mirror of https://github.com/ethereum/go-ethereum
les: historical data garbage collection (#19570)
This change introduces garbage collection for the light client. Historical chain data is deleted periodically. If you want to disable the GC, use the --light.nopruning flag.pull/21325/head
gary rong
4 years ago
committed by
GitHub
parent
b8dd0890b3
commit
6eef141aef
@ -0,0 +1,98 @@ |
|||||||
|
// Copyright 2019 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package les |
||||||
|
|
||||||
|
import ( |
||||||
|
"sync" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/common/math" |
||||||
|
"github.com/ethereum/go-ethereum/core" |
||||||
|
"github.com/ethereum/go-ethereum/ethdb" |
||||||
|
"github.com/ethereum/go-ethereum/log" |
||||||
|
) |
||||||
|
|
||||||
|
// pruner is responsible for pruning historical light chain data.
|
||||||
|
type pruner struct { |
||||||
|
db ethdb.Database |
||||||
|
indexers []*core.ChainIndexer |
||||||
|
closeCh chan struct{} |
||||||
|
wg sync.WaitGroup |
||||||
|
} |
||||||
|
|
||||||
|
// newPruner returns a light chain pruner instance.
|
||||||
|
func newPruner(db ethdb.Database, indexers ...*core.ChainIndexer) *pruner { |
||||||
|
pruner := &pruner{ |
||||||
|
db: db, |
||||||
|
indexers: indexers, |
||||||
|
closeCh: make(chan struct{}), |
||||||
|
} |
||||||
|
pruner.wg.Add(1) |
||||||
|
go pruner.loop() |
||||||
|
return pruner |
||||||
|
} |
||||||
|
|
||||||
|
// close notifies all background goroutines belonging to pruner to exit.
|
||||||
|
func (p *pruner) close() { |
||||||
|
close(p.closeCh) |
||||||
|
p.wg.Wait() |
||||||
|
} |
||||||
|
|
||||||
|
// loop periodically queries the status of chain indexers and prunes useless
|
||||||
|
// historical chain data. Notably, whenever Geth restarts, it will iterate
|
||||||
|
// all historical sections even they don't exist at all(below checkpoint) so
|
||||||
|
// that light client can prune cached chain data that was ODRed after pruning
|
||||||
|
// that section.
|
||||||
|
func (p *pruner) loop() { |
||||||
|
defer p.wg.Done() |
||||||
|
|
||||||
|
// cleanTicker is the ticker used to trigger a history clean 2 times a day.
|
||||||
|
var cleanTicker = time.NewTicker(12 * time.Hour) |
||||||
|
|
||||||
|
// pruning finds the sections that have been processed by all indexers
|
||||||
|
// and deletes all historical chain data.
|
||||||
|
// Note, if some indexers don't support pruning(e.g. eth.BloomIndexer),
|
||||||
|
// pruning operations can be silently ignored.
|
||||||
|
pruning := func() { |
||||||
|
min := uint64(math.MaxUint64) |
||||||
|
for _, indexer := range p.indexers { |
||||||
|
sections, _, _ := indexer.Sections() |
||||||
|
if sections < min { |
||||||
|
min = sections |
||||||
|
} |
||||||
|
} |
||||||
|
// Always keep the latest section data in database.
|
||||||
|
if min < 2 || len(p.indexers) == 0 { |
||||||
|
return |
||||||
|
} |
||||||
|
for _, indexer := range p.indexers { |
||||||
|
if err := indexer.Prune(min - 2); err != nil { |
||||||
|
log.Debug("Failed to prune historical data", "err", err) |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
p.db.Compact(nil, nil) // Compact entire database, ensure all removed data are deleted.
|
||||||
|
} |
||||||
|
for { |
||||||
|
pruning() |
||||||
|
select { |
||||||
|
case <-cleanTicker.C: |
||||||
|
case <-p.closeCh: |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,197 @@ |
|||||||
|
// Copyright 2019 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package les |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"context" |
||||||
|
"encoding/binary" |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/core" |
||||||
|
"github.com/ethereum/go-ethereum/light" |
||||||
|
) |
||||||
|
|
||||||
|
func TestLightPruner(t *testing.T) { |
||||||
|
config := light.TestClientIndexerConfig |
||||||
|
|
||||||
|
waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { |
||||||
|
for { |
||||||
|
cs, _, _ := cIndexer.Sections() |
||||||
|
bts, _, _ := btIndexer.Sections() |
||||||
|
if cs >= 3 && bts >= 3 { |
||||||
|
break |
||||||
|
} |
||||||
|
time.Sleep(10 * time.Millisecond) |
||||||
|
} |
||||||
|
} |
||||||
|
server, client, tearDown := newClientServerEnv(t, int(3*config.ChtSize+config.ChtConfirms), 2, waitIndexers, nil, 0, false, true, false) |
||||||
|
defer tearDown() |
||||||
|
|
||||||
|
// checkDB iterates the chain with given prefix, resolves the block number
|
||||||
|
// with given callback and ensures this entry should exist or not.
|
||||||
|
checkDB := func(from, to uint64, prefix []byte, resolve func(key, value []byte) *uint64, exist bool) bool { |
||||||
|
it := client.db.NewIterator(prefix, nil) |
||||||
|
defer it.Release() |
||||||
|
|
||||||
|
var next = from |
||||||
|
for it.Next() { |
||||||
|
number := resolve(it.Key(), it.Value()) |
||||||
|
if number == nil || *number < from { |
||||||
|
continue |
||||||
|
} else if *number > to { |
||||||
|
return true |
||||||
|
} |
||||||
|
if exist { |
||||||
|
if *number != next { |
||||||
|
return false |
||||||
|
} |
||||||
|
next++ |
||||||
|
} else { |
||||||
|
return false |
||||||
|
} |
||||||
|
} |
||||||
|
return true |
||||||
|
} |
||||||
|
// checkPruned checks and ensures the stale chain data has been pruned.
|
||||||
|
checkPruned := func(from, to uint64) { |
||||||
|
// Iterate canonical hash
|
||||||
|
if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { |
||||||
|
if len(key) == 1+8+1 && bytes.Equal(key[9:10], []byte("n")) { |
||||||
|
n := binary.BigEndian.Uint64(key[1:9]) |
||||||
|
return &n |
||||||
|
} |
||||||
|
return nil |
||||||
|
}, false) { |
||||||
|
t.Fatalf("canonical hash mappings are not properly pruned") |
||||||
|
} |
||||||
|
// Iterate header
|
||||||
|
if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { |
||||||
|
if len(key) == 1+8+32 { |
||||||
|
n := binary.BigEndian.Uint64(key[1:9]) |
||||||
|
return &n |
||||||
|
} |
||||||
|
return nil |
||||||
|
}, false) { |
||||||
|
t.Fatalf("headers are not properly pruned") |
||||||
|
} |
||||||
|
// Iterate body
|
||||||
|
if !checkDB(from, to, []byte("b"), func(key, value []byte) *uint64 { |
||||||
|
if len(key) == 1+8+32 { |
||||||
|
n := binary.BigEndian.Uint64(key[1:9]) |
||||||
|
return &n |
||||||
|
} |
||||||
|
return nil |
||||||
|
}, false) { |
||||||
|
t.Fatalf("block bodies are not properly pruned") |
||||||
|
} |
||||||
|
// Iterate receipts
|
||||||
|
if !checkDB(from, to, []byte("r"), func(key, value []byte) *uint64 { |
||||||
|
if len(key) == 1+8+32 { |
||||||
|
n := binary.BigEndian.Uint64(key[1:9]) |
||||||
|
return &n |
||||||
|
} |
||||||
|
return nil |
||||||
|
}, false) { |
||||||
|
t.Fatalf("receipts are not properly pruned") |
||||||
|
} |
||||||
|
// Iterate td
|
||||||
|
if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { |
||||||
|
if len(key) == 1+8+32+1 && bytes.Equal(key[41:42], []byte("t")) { |
||||||
|
n := binary.BigEndian.Uint64(key[1:9]) |
||||||
|
return &n |
||||||
|
} |
||||||
|
return nil |
||||||
|
}, false) { |
||||||
|
t.Fatalf("tds are not properly pruned") |
||||||
|
} |
||||||
|
} |
||||||
|
// Start light pruner.
|
||||||
|
time.Sleep(1500 * time.Millisecond) // Ensure light client has finished the syncing and indexing
|
||||||
|
newPruner(client.db, client.chtIndexer, client.bloomTrieIndexer) |
||||||
|
|
||||||
|
time.Sleep(1500 * time.Millisecond) // Ensure pruner have enough time to prune data.
|
||||||
|
checkPruned(1, config.ChtSize-1) |
||||||
|
|
||||||
|
// Ensure all APIs still work after pruning.
|
||||||
|
var cases = []struct { |
||||||
|
from, to uint64 |
||||||
|
methodName string |
||||||
|
method func(uint64) bool |
||||||
|
}{ |
||||||
|
{ |
||||||
|
1, 10, "GetHeaderByNumber", |
||||||
|
func(n uint64) bool { |
||||||
|
_, err := light.GetHeaderByNumber(context.Background(), client.handler.backend.odr, n) |
||||||
|
return err == nil |
||||||
|
}, |
||||||
|
}, |
||||||
|
{ |
||||||
|
11, 20, "GetCanonicalHash", |
||||||
|
func(n uint64) bool { |
||||||
|
_, err := light.GetCanonicalHash(context.Background(), client.handler.backend.odr, n) |
||||||
|
return err == nil |
||||||
|
}, |
||||||
|
}, |
||||||
|
{ |
||||||
|
21, 30, "GetTd", |
||||||
|
func(n uint64) bool { |
||||||
|
_, err := light.GetTd(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||||
|
return err == nil |
||||||
|
}, |
||||||
|
}, |
||||||
|
{ |
||||||
|
31, 40, "GetBodyRLP", |
||||||
|
func(n uint64) bool { |
||||||
|
_, err := light.GetBodyRLP(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||||
|
return err == nil |
||||||
|
}, |
||||||
|
}, |
||||||
|
{ |
||||||
|
41, 50, "GetBlock", |
||||||
|
func(n uint64) bool { |
||||||
|
_, err := light.GetBlock(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||||
|
return err == nil |
||||||
|
}, |
||||||
|
}, |
||||||
|
{ |
||||||
|
51, 60, "GetBlockReceipts", |
||||||
|
func(n uint64) bool { |
||||||
|
_, err := light.GetBlockReceipts(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||||
|
return err == nil |
||||||
|
}, |
||||||
|
}, |
||||||
|
} |
||||||
|
for _, c := range cases { |
||||||
|
for i := c.from; i <= c.to; i++ { |
||||||
|
if !c.method(i) { |
||||||
|
t.Fatalf("rpc method %s failed, number %d", c.methodName, i) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
// Check GetBloombits
|
||||||
|
_, err := light.GetBloomBits(context.Background(), client.handler.backend.odr, 0, []uint64{0}) |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Failed to retrieve bloombits of pruned section: %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
// Ensure the ODR cached data can be cleaned by pruner.
|
||||||
|
newPruner(client.db, client.chtIndexer, client.bloomTrieIndexer) |
||||||
|
time.Sleep(50 * time.Millisecond) // Ensure pruner have enough time to prune data.
|
||||||
|
checkPruned(1, config.ChtSize-1) // Ensure all cached data(by odr) is cleaned.
|
||||||
|
} |
||||||
Loading…
Reference in new issue