mirror of https://github.com/ethereum/go-ethereum
les: historical data garbage collection (#19570)
This change introduces garbage collection for the light client. Historical chain data is deleted periodically. If you want to disable the GC, use the --light.nopruning flag.pull/21325/head
gary rong
4 years ago
committed by
GitHub
parent
b8dd0890b3
commit
6eef141aef
@ -0,0 +1,98 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package les |
||||
|
||||
import ( |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/common/math" |
||||
"github.com/ethereum/go-ethereum/core" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/ethereum/go-ethereum/log" |
||||
) |
||||
|
||||
// pruner is responsible for pruning historical light chain data.
|
||||
type pruner struct { |
||||
db ethdb.Database |
||||
indexers []*core.ChainIndexer |
||||
closeCh chan struct{} |
||||
wg sync.WaitGroup |
||||
} |
||||
|
||||
// newPruner returns a light chain pruner instance.
|
||||
func newPruner(db ethdb.Database, indexers ...*core.ChainIndexer) *pruner { |
||||
pruner := &pruner{ |
||||
db: db, |
||||
indexers: indexers, |
||||
closeCh: make(chan struct{}), |
||||
} |
||||
pruner.wg.Add(1) |
||||
go pruner.loop() |
||||
return pruner |
||||
} |
||||
|
||||
// close notifies all background goroutines belonging to pruner to exit.
|
||||
func (p *pruner) close() { |
||||
close(p.closeCh) |
||||
p.wg.Wait() |
||||
} |
||||
|
||||
// loop periodically queries the status of chain indexers and prunes useless
|
||||
// historical chain data. Notably, whenever Geth restarts, it will iterate
|
||||
// all historical sections even they don't exist at all(below checkpoint) so
|
||||
// that light client can prune cached chain data that was ODRed after pruning
|
||||
// that section.
|
||||
func (p *pruner) loop() { |
||||
defer p.wg.Done() |
||||
|
||||
// cleanTicker is the ticker used to trigger a history clean 2 times a day.
|
||||
var cleanTicker = time.NewTicker(12 * time.Hour) |
||||
|
||||
// pruning finds the sections that have been processed by all indexers
|
||||
// and deletes all historical chain data.
|
||||
// Note, if some indexers don't support pruning(e.g. eth.BloomIndexer),
|
||||
// pruning operations can be silently ignored.
|
||||
pruning := func() { |
||||
min := uint64(math.MaxUint64) |
||||
for _, indexer := range p.indexers { |
||||
sections, _, _ := indexer.Sections() |
||||
if sections < min { |
||||
min = sections |
||||
} |
||||
} |
||||
// Always keep the latest section data in database.
|
||||
if min < 2 || len(p.indexers) == 0 { |
||||
return |
||||
} |
||||
for _, indexer := range p.indexers { |
||||
if err := indexer.Prune(min - 2); err != nil { |
||||
log.Debug("Failed to prune historical data", "err", err) |
||||
return |
||||
} |
||||
} |
||||
p.db.Compact(nil, nil) // Compact entire database, ensure all removed data are deleted.
|
||||
} |
||||
for { |
||||
pruning() |
||||
select { |
||||
case <-cleanTicker.C: |
||||
case <-p.closeCh: |
||||
return |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,197 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package les |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/binary" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/core" |
||||
"github.com/ethereum/go-ethereum/light" |
||||
) |
||||
|
||||
func TestLightPruner(t *testing.T) { |
||||
config := light.TestClientIndexerConfig |
||||
|
||||
waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { |
||||
for { |
||||
cs, _, _ := cIndexer.Sections() |
||||
bts, _, _ := btIndexer.Sections() |
||||
if cs >= 3 && bts >= 3 { |
||||
break |
||||
} |
||||
time.Sleep(10 * time.Millisecond) |
||||
} |
||||
} |
||||
server, client, tearDown := newClientServerEnv(t, int(3*config.ChtSize+config.ChtConfirms), 2, waitIndexers, nil, 0, false, true, false) |
||||
defer tearDown() |
||||
|
||||
// checkDB iterates the chain with given prefix, resolves the block number
|
||||
// with given callback and ensures this entry should exist or not.
|
||||
checkDB := func(from, to uint64, prefix []byte, resolve func(key, value []byte) *uint64, exist bool) bool { |
||||
it := client.db.NewIterator(prefix, nil) |
||||
defer it.Release() |
||||
|
||||
var next = from |
||||
for it.Next() { |
||||
number := resolve(it.Key(), it.Value()) |
||||
if number == nil || *number < from { |
||||
continue |
||||
} else if *number > to { |
||||
return true |
||||
} |
||||
if exist { |
||||
if *number != next { |
||||
return false |
||||
} |
||||
next++ |
||||
} else { |
||||
return false |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
// checkPruned checks and ensures the stale chain data has been pruned.
|
||||
checkPruned := func(from, to uint64) { |
||||
// Iterate canonical hash
|
||||
if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { |
||||
if len(key) == 1+8+1 && bytes.Equal(key[9:10], []byte("n")) { |
||||
n := binary.BigEndian.Uint64(key[1:9]) |
||||
return &n |
||||
} |
||||
return nil |
||||
}, false) { |
||||
t.Fatalf("canonical hash mappings are not properly pruned") |
||||
} |
||||
// Iterate header
|
||||
if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { |
||||
if len(key) == 1+8+32 { |
||||
n := binary.BigEndian.Uint64(key[1:9]) |
||||
return &n |
||||
} |
||||
return nil |
||||
}, false) { |
||||
t.Fatalf("headers are not properly pruned") |
||||
} |
||||
// Iterate body
|
||||
if !checkDB(from, to, []byte("b"), func(key, value []byte) *uint64 { |
||||
if len(key) == 1+8+32 { |
||||
n := binary.BigEndian.Uint64(key[1:9]) |
||||
return &n |
||||
} |
||||
return nil |
||||
}, false) { |
||||
t.Fatalf("block bodies are not properly pruned") |
||||
} |
||||
// Iterate receipts
|
||||
if !checkDB(from, to, []byte("r"), func(key, value []byte) *uint64 { |
||||
if len(key) == 1+8+32 { |
||||
n := binary.BigEndian.Uint64(key[1:9]) |
||||
return &n |
||||
} |
||||
return nil |
||||
}, false) { |
||||
t.Fatalf("receipts are not properly pruned") |
||||
} |
||||
// Iterate td
|
||||
if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { |
||||
if len(key) == 1+8+32+1 && bytes.Equal(key[41:42], []byte("t")) { |
||||
n := binary.BigEndian.Uint64(key[1:9]) |
||||
return &n |
||||
} |
||||
return nil |
||||
}, false) { |
||||
t.Fatalf("tds are not properly pruned") |
||||
} |
||||
} |
||||
// Start light pruner.
|
||||
time.Sleep(1500 * time.Millisecond) // Ensure light client has finished the syncing and indexing
|
||||
newPruner(client.db, client.chtIndexer, client.bloomTrieIndexer) |
||||
|
||||
time.Sleep(1500 * time.Millisecond) // Ensure pruner have enough time to prune data.
|
||||
checkPruned(1, config.ChtSize-1) |
||||
|
||||
// Ensure all APIs still work after pruning.
|
||||
var cases = []struct { |
||||
from, to uint64 |
||||
methodName string |
||||
method func(uint64) bool |
||||
}{ |
||||
{ |
||||
1, 10, "GetHeaderByNumber", |
||||
func(n uint64) bool { |
||||
_, err := light.GetHeaderByNumber(context.Background(), client.handler.backend.odr, n) |
||||
return err == nil |
||||
}, |
||||
}, |
||||
{ |
||||
11, 20, "GetCanonicalHash", |
||||
func(n uint64) bool { |
||||
_, err := light.GetCanonicalHash(context.Background(), client.handler.backend.odr, n) |
||||
return err == nil |
||||
}, |
||||
}, |
||||
{ |
||||
21, 30, "GetTd", |
||||
func(n uint64) bool { |
||||
_, err := light.GetTd(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||
return err == nil |
||||
}, |
||||
}, |
||||
{ |
||||
31, 40, "GetBodyRLP", |
||||
func(n uint64) bool { |
||||
_, err := light.GetBodyRLP(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||
return err == nil |
||||
}, |
||||
}, |
||||
{ |
||||
41, 50, "GetBlock", |
||||
func(n uint64) bool { |
||||
_, err := light.GetBlock(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||
return err == nil |
||||
}, |
||||
}, |
||||
{ |
||||
51, 60, "GetBlockReceipts", |
||||
func(n uint64) bool { |
||||
_, err := light.GetBlockReceipts(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) |
||||
return err == nil |
||||
}, |
||||
}, |
||||
} |
||||
for _, c := range cases { |
||||
for i := c.from; i <= c.to; i++ { |
||||
if !c.method(i) { |
||||
t.Fatalf("rpc method %s failed, number %d", c.methodName, i) |
||||
} |
||||
} |
||||
} |
||||
// Check GetBloombits
|
||||
_, err := light.GetBloomBits(context.Background(), client.handler.backend.odr, 0, []uint64{0}) |
||||
if err != nil { |
||||
t.Fatalf("Failed to retrieve bloombits of pruned section: %v", err) |
||||
} |
||||
|
||||
// Ensure the ODR cached data can be cleaned by pruner.
|
||||
newPruner(client.db, client.chtIndexer, client.bloomTrieIndexer) |
||||
time.Sleep(50 * time.Millisecond) // Ensure pruner have enough time to prune data.
|
||||
checkPruned(1, config.ChtSize-1) // Ensure all cached data(by odr) is cleaned.
|
||||
} |
||||
Loading…
Reference in new issue