mirror of https://github.com/ethereum/go-ethereum
eth/downloader: refactor downloader + queue (#21263)
* eth/downloader: refactor downloader + queue downloader, fetcher: throttle-metrics, fetcher filter improvements, standalone resultcache downloader: more accurate deliverytime calculation, less mem overhead in state requests downloader/queue: increase underlying buffer of results, new throttle mechanism eth/downloader: updates to tests eth/downloader: fix up some review concerns eth/downloader/queue: minor fixes eth/downloader: minor fixes after review call eth/downloader: testcases for queue.go eth/downloader: minor change, don't set progress unless progress... eth/downloader: fix flaw which prevented useless peers from being dropped eth/downloader: try to fix tests eth/downloader: verify non-deliveries against advertised remote head eth/downloader: fix flaw with checking closed-status causing hang eth/downloader: hashing avoidance eth/downloader: review concerns + simplify resultcache and queue eth/downloader: add back some locks, address review concerns downloader/queue: fix remaining lock flaw * eth/downloader: nitpick fixes * eth/downloader: remove the *2*3/4 throttling threshold dance * eth/downloader: print correct throttle threshold in stats Co-authored-by: Péter Szilágyi <peterke@gmail.com>pull/21368/head
Martin Holst Swende
4 years ago
committed by
GitHub
parent
3a57eecc69
commit
105922180f
@ -0,0 +1,53 @@ |
|||||||
|
// Copyright 2020 The go-ethereum Authors
|
||||||
|
// This file is part of go-ethereum.
|
||||||
|
//
|
||||||
|
// go-ethereum is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// go-ethereum is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package downloader |
||||||
|
|
||||||
|
import ( |
||||||
|
"sort" |
||||||
|
"testing" |
||||||
|
) |
||||||
|
|
||||||
|
func TestPeerThroughputSorting(t *testing.T) { |
||||||
|
a := &peerConnection{ |
||||||
|
id: "a", |
||||||
|
headerThroughput: 1.25, |
||||||
|
} |
||||||
|
b := &peerConnection{ |
||||||
|
id: "b", |
||||||
|
headerThroughput: 1.21, |
||||||
|
} |
||||||
|
c := &peerConnection{ |
||||||
|
id: "c", |
||||||
|
headerThroughput: 1.23, |
||||||
|
} |
||||||
|
|
||||||
|
peers := []*peerConnection{a, b, c} |
||||||
|
tps := []float64{a.headerThroughput, |
||||||
|
b.headerThroughput, c.headerThroughput} |
||||||
|
sortPeers := &peerThroughputSort{peers, tps} |
||||||
|
sort.Sort(sortPeers) |
||||||
|
if got, exp := sortPeers.p[0].id, "a"; got != exp { |
||||||
|
t.Errorf("sort fail, got %v exp %v", got, exp) |
||||||
|
} |
||||||
|
if got, exp := sortPeers.p[1].id, "c"; got != exp { |
||||||
|
t.Errorf("sort fail, got %v exp %v", got, exp) |
||||||
|
} |
||||||
|
if got, exp := sortPeers.p[2].id, "b"; got != exp { |
||||||
|
t.Errorf("sort fail, got %v exp %v", got, exp) |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,426 @@ |
|||||||
|
// Copyright 2019 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package downloader |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"math/big" |
||||||
|
"math/rand" |
||||||
|
"sync" |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/common" |
||||||
|
"github.com/ethereum/go-ethereum/consensus/ethash" |
||||||
|
"github.com/ethereum/go-ethereum/core" |
||||||
|
"github.com/ethereum/go-ethereum/core/rawdb" |
||||||
|
"github.com/ethereum/go-ethereum/core/types" |
||||||
|
"github.com/ethereum/go-ethereum/log" |
||||||
|
"github.com/ethereum/go-ethereum/params" |
||||||
|
) |
||||||
|
|
||||||
|
var ( |
||||||
|
testdb = rawdb.NewMemoryDatabase() |
||||||
|
genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) |
||||||
|
) |
||||||
|
|
||||||
|
// makeChain creates a chain of n blocks starting at and including parent.
|
||||||
|
// the returned hash chain is ordered head->parent. In addition, every 3rd block
|
||||||
|
// contains a transaction and every 5th an uncle to allow testing correct block
|
||||||
|
// reassembly.
|
||||||
|
func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { |
||||||
|
blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { |
||||||
|
block.SetCoinbase(common.Address{seed}) |
||||||
|
// Add one tx to every secondblock
|
||||||
|
if !empty && i%2 == 0 { |
||||||
|
signer := types.MakeSigner(params.TestChainConfig, block.Number()) |
||||||
|
tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) |
||||||
|
if err != nil { |
||||||
|
panic(err) |
||||||
|
} |
||||||
|
block.AddTx(tx) |
||||||
|
} |
||||||
|
}) |
||||||
|
return blocks, receipts |
||||||
|
} |
||||||
|
|
||||||
|
type chainData struct { |
||||||
|
blocks []*types.Block |
||||||
|
offset int |
||||||
|
} |
||||||
|
|
||||||
|
var chain *chainData |
||||||
|
var emptyChain *chainData |
||||||
|
|
||||||
|
func init() { |
||||||
|
// Create a chain of blocks to import
|
||||||
|
targetBlocks := 128 |
||||||
|
blocks, _ := makeChain(targetBlocks, 0, genesis, false) |
||||||
|
chain = &chainData{blocks, 0} |
||||||
|
|
||||||
|
blocks, _ = makeChain(targetBlocks, 0, genesis, true) |
||||||
|
emptyChain = &chainData{blocks, 0} |
||||||
|
} |
||||||
|
|
||||||
|
func (chain *chainData) headers() []*types.Header { |
||||||
|
hdrs := make([]*types.Header, len(chain.blocks)) |
||||||
|
for i, b := range chain.blocks { |
||||||
|
hdrs[i] = b.Header() |
||||||
|
} |
||||||
|
return hdrs |
||||||
|
} |
||||||
|
|
||||||
|
func (chain *chainData) Len() int { |
||||||
|
return len(chain.blocks) |
||||||
|
} |
||||||
|
|
||||||
|
func dummyPeer(id string) *peerConnection { |
||||||
|
p := &peerConnection{ |
||||||
|
id: id, |
||||||
|
lacking: make(map[common.Hash]struct{}), |
||||||
|
} |
||||||
|
return p |
||||||
|
} |
||||||
|
|
||||||
|
func TestBasics(t *testing.T) { |
||||||
|
q := newQueue(10) |
||||||
|
if !q.Idle() { |
||||||
|
t.Errorf("new queue should be idle") |
||||||
|
} |
||||||
|
q.Prepare(1, FastSync) |
||||||
|
if res := q.Results(false); len(res) != 0 { |
||||||
|
t.Fatal("new queue should have 0 results") |
||||||
|
} |
||||||
|
|
||||||
|
// Schedule a batch of headers
|
||||||
|
q.Schedule(chain.headers(), 1) |
||||||
|
if q.Idle() { |
||||||
|
t.Errorf("queue should not be idle") |
||||||
|
} |
||||||
|
if got, exp := q.PendingBlocks(), chain.Len(); got != exp { |
||||||
|
t.Errorf("wrong pending block count, got %d, exp %d", got, exp) |
||||||
|
} |
||||||
|
// Only non-empty receipts get added to task-queue
|
||||||
|
if got, exp := q.PendingReceipts(), 64; got != exp { |
||||||
|
t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) |
||||||
|
} |
||||||
|
// Items are now queued for downloading, next step is that we tell the
|
||||||
|
// queue that a certain peer will deliver them for us
|
||||||
|
{ |
||||||
|
peer := dummyPeer("peer-1") |
||||||
|
fetchReq, _, throttle := q.ReserveBodies(peer, 50) |
||||||
|
if !throttle { |
||||||
|
// queue size is only 10, so throttling should occur
|
||||||
|
t.Fatal("should throttle") |
||||||
|
} |
||||||
|
// But we should still get the first things to fetch
|
||||||
|
if got, exp := len(fetchReq.Headers), 5; got != exp { |
||||||
|
t.Fatalf("expected %d requests, got %d", exp, got) |
||||||
|
} |
||||||
|
if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { |
||||||
|
t.Fatalf("expected header %d, got %d", exp, got) |
||||||
|
} |
||||||
|
} |
||||||
|
{ |
||||||
|
peer := dummyPeer("peer-2") |
||||||
|
fetchReq, _, throttle := q.ReserveBodies(peer, 50) |
||||||
|
|
||||||
|
// The second peer should hit throttling
|
||||||
|
if !throttle { |
||||||
|
t.Fatalf("should not throttle") |
||||||
|
} |
||||||
|
// And not get any fetches at all, since it was throttled to begin with
|
||||||
|
if fetchReq != nil { |
||||||
|
t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) |
||||||
|
} |
||||||
|
} |
||||||
|
//fmt.Printf("blockTaskQueue len: %d\n", q.blockTaskQueue.Size())
|
||||||
|
//fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size())
|
||||||
|
{ |
||||||
|
// The receipt delivering peer should not be affected
|
||||||
|
// by the throttling of body deliveries
|
||||||
|
peer := dummyPeer("peer-3") |
||||||
|
fetchReq, _, throttle := q.ReserveReceipts(peer, 50) |
||||||
|
if !throttle { |
||||||
|
// queue size is only 10, so throttling should occur
|
||||||
|
t.Fatal("should throttle") |
||||||
|
} |
||||||
|
// But we should still get the first things to fetch
|
||||||
|
if got, exp := len(fetchReq.Headers), 5; got != exp { |
||||||
|
t.Fatalf("expected %d requests, got %d", exp, got) |
||||||
|
} |
||||||
|
if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { |
||||||
|
t.Fatalf("expected header %d, got %d", exp, got) |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
//fmt.Printf("blockTaskQueue len: %d\n", q.blockTaskQueue.Size())
|
||||||
|
//fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size())
|
||||||
|
//fmt.Printf("processable: %d\n", q.resultCache.countCompleted())
|
||||||
|
} |
||||||
|
|
||||||
|
func TestEmptyBlocks(t *testing.T) { |
||||||
|
q := newQueue(10) |
||||||
|
|
||||||
|
q.Prepare(1, FastSync) |
||||||
|
// Schedule a batch of headers
|
||||||
|
q.Schedule(emptyChain.headers(), 1) |
||||||
|
if q.Idle() { |
||||||
|
t.Errorf("queue should not be idle") |
||||||
|
} |
||||||
|
if got, exp := q.PendingBlocks(), len(emptyChain.blocks); got != exp { |
||||||
|
t.Errorf("wrong pending block count, got %d, exp %d", got, exp) |
||||||
|
} |
||||||
|
if got, exp := q.PendingReceipts(), 0; got != exp { |
||||||
|
t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) |
||||||
|
} |
||||||
|
// They won't be processable, because the fetchresults haven't been
|
||||||
|
// created yet
|
||||||
|
if got, exp := q.resultCache.countCompleted(), 0; got != exp { |
||||||
|
t.Errorf("wrong processable count, got %d, exp %d", got, exp) |
||||||
|
} |
||||||
|
|
||||||
|
// Items are now queued for downloading, next step is that we tell the
|
||||||
|
// queue that a certain peer will deliver them for us
|
||||||
|
// That should trigger all of them to suddenly become 'done'
|
||||||
|
{ |
||||||
|
// Reserve blocks
|
||||||
|
peer := dummyPeer("peer-1") |
||||||
|
fetchReq, _, _ := q.ReserveBodies(peer, 50) |
||||||
|
|
||||||
|
// there should be nothing to fetch, blocks are empty
|
||||||
|
if fetchReq != nil { |
||||||
|
t.Fatal("there should be no body fetch tasks remaining") |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
if q.blockTaskQueue.Size() != len(emptyChain.blocks)-10 { |
||||||
|
t.Errorf("expected block task queue to be 0, got %d", q.blockTaskQueue.Size()) |
||||||
|
} |
||||||
|
if q.receiptTaskQueue.Size() != 0 { |
||||||
|
t.Errorf("expected receipt task queue to be 0, got %d", q.receiptTaskQueue.Size()) |
||||||
|
} |
||||||
|
//fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size())
|
||||||
|
{ |
||||||
|
peer := dummyPeer("peer-3") |
||||||
|
fetchReq, _, _ := q.ReserveReceipts(peer, 50) |
||||||
|
|
||||||
|
// there should be nothing to fetch, blocks are empty
|
||||||
|
if fetchReq != nil { |
||||||
|
t.Fatal("there should be no body fetch tasks remaining") |
||||||
|
} |
||||||
|
} |
||||||
|
if got, exp := q.resultCache.countCompleted(), 10; got != exp { |
||||||
|
t.Errorf("wrong processable count, got %d, exp %d", got, exp) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// XTestDelivery does some more extensive testing of events that happen,
|
||||||
|
// blocks that become known and peers that make reservations and deliveries.
|
||||||
|
// disabled since it's not really a unit-test, but can be executed to test
|
||||||
|
// some more advanced scenarios
|
||||||
|
func XTestDelivery(t *testing.T) { |
||||||
|
// the outside network, holding blocks
|
||||||
|
blo, rec := makeChain(128, 0, genesis, false) |
||||||
|
world := newNetwork() |
||||||
|
world.receipts = rec |
||||||
|
world.chain = blo |
||||||
|
world.progress(10) |
||||||
|
if false { |
||||||
|
log.Root().SetHandler(log.StdoutHandler) |
||||||
|
|
||||||
|
} |
||||||
|
q := newQueue(10) |
||||||
|
var wg sync.WaitGroup |
||||||
|
q.Prepare(1, FastSync) |
||||||
|
wg.Add(1) |
||||||
|
go func() { |
||||||
|
// deliver headers
|
||||||
|
defer wg.Done() |
||||||
|
c := 1 |
||||||
|
for { |
||||||
|
//fmt.Printf("getting headers from %d\n", c)
|
||||||
|
hdrs := world.headers(c) |
||||||
|
l := len(hdrs) |
||||||
|
//fmt.Printf("scheduling %d headers, first %d last %d\n",
|
||||||
|
// l, hdrs[0].Number.Uint64(), hdrs[len(hdrs)-1].Number.Uint64())
|
||||||
|
q.Schedule(hdrs, uint64(c)) |
||||||
|
c += l |
||||||
|
} |
||||||
|
}() |
||||||
|
wg.Add(1) |
||||||
|
go func() { |
||||||
|
// collect results
|
||||||
|
defer wg.Done() |
||||||
|
tot := 0 |
||||||
|
for { |
||||||
|
res := q.Results(true) |
||||||
|
tot += len(res) |
||||||
|
fmt.Printf("got %d results, %d tot\n", len(res), tot) |
||||||
|
// Now we can forget about these
|
||||||
|
world.forget(res[len(res)-1].Header.Number.Uint64()) |
||||||
|
|
||||||
|
} |
||||||
|
}() |
||||||
|
wg.Add(1) |
||||||
|
go func() { |
||||||
|
defer wg.Done() |
||||||
|
// reserve body fetch
|
||||||
|
i := 4 |
||||||
|
for { |
||||||
|
peer := dummyPeer(fmt.Sprintf("peer-%d", i)) |
||||||
|
f, _, _ := q.ReserveBodies(peer, rand.Intn(30)) |
||||||
|
if f != nil { |
||||||
|
var emptyList []*types.Header |
||||||
|
var txs [][]*types.Transaction |
||||||
|
var uncles [][]*types.Header |
||||||
|
numToSkip := rand.Intn(len(f.Headers)) |
||||||
|
for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { |
||||||
|
txs = append(txs, world.getTransactions(hdr.Number.Uint64())) |
||||||
|
uncles = append(uncles, emptyList) |
||||||
|
} |
||||||
|
time.Sleep(100 * time.Millisecond) |
||||||
|
_, err := q.DeliverBodies(peer.id, txs, uncles) |
||||||
|
if err != nil { |
||||||
|
fmt.Printf("delivered %d bodies %v\n", len(txs), err) |
||||||
|
} |
||||||
|
} else { |
||||||
|
i++ |
||||||
|
time.Sleep(200 * time.Millisecond) |
||||||
|
} |
||||||
|
} |
||||||
|
}() |
||||||
|
go func() { |
||||||
|
defer wg.Done() |
||||||
|
// reserve receiptfetch
|
||||||
|
peer := dummyPeer("peer-3") |
||||||
|
for { |
||||||
|
f, _, _ := q.ReserveReceipts(peer, rand.Intn(50)) |
||||||
|
if f != nil { |
||||||
|
var rcs [][]*types.Receipt |
||||||
|
for _, hdr := range f.Headers { |
||||||
|
rcs = append(rcs, world.getReceipts(hdr.Number.Uint64())) |
||||||
|
} |
||||||
|
_, err := q.DeliverReceipts(peer.id, rcs) |
||||||
|
if err != nil { |
||||||
|
fmt.Printf("delivered %d receipts %v\n", len(rcs), err) |
||||||
|
} |
||||||
|
time.Sleep(100 * time.Millisecond) |
||||||
|
} else { |
||||||
|
time.Sleep(200 * time.Millisecond) |
||||||
|
} |
||||||
|
} |
||||||
|
}() |
||||||
|
wg.Add(1) |
||||||
|
go func() { |
||||||
|
defer wg.Done() |
||||||
|
for i := 0; i < 50; i++ { |
||||||
|
time.Sleep(300 * time.Millisecond) |
||||||
|
//world.tick()
|
||||||
|
//fmt.Printf("trying to progress\n")
|
||||||
|
world.progress(rand.Intn(100)) |
||||||
|
} |
||||||
|
for i := 0; i < 50; i++ { |
||||||
|
time.Sleep(2990 * time.Millisecond) |
||||||
|
|
||||||
|
} |
||||||
|
}() |
||||||
|
wg.Add(1) |
||||||
|
go func() { |
||||||
|
defer wg.Done() |
||||||
|
for { |
||||||
|
time.Sleep(990 * time.Millisecond) |
||||||
|
fmt.Printf("world block tip is %d\n", |
||||||
|
world.chain[len(world.chain)-1].Header().Number.Uint64()) |
||||||
|
fmt.Println(q.Stats()) |
||||||
|
} |
||||||
|
}() |
||||||
|
wg.Wait() |
||||||
|
} |
||||||
|
|
||||||
|
func newNetwork() *network { |
||||||
|
var l sync.RWMutex |
||||||
|
return &network{ |
||||||
|
cond: sync.NewCond(&l), |
||||||
|
offset: 1, // block 1 is at blocks[0]
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// represents the network
|
||||||
|
type network struct { |
||||||
|
offset int |
||||||
|
chain []*types.Block |
||||||
|
receipts []types.Receipts |
||||||
|
lock sync.RWMutex |
||||||
|
cond *sync.Cond |
||||||
|
} |
||||||
|
|
||||||
|
func (n *network) getTransactions(blocknum uint64) types.Transactions { |
||||||
|
index := blocknum - uint64(n.offset) |
||||||
|
return n.chain[index].Transactions() |
||||||
|
} |
||||||
|
func (n *network) getReceipts(blocknum uint64) types.Receipts { |
||||||
|
index := blocknum - uint64(n.offset) |
||||||
|
if got := n.chain[index].Header().Number.Uint64(); got != blocknum { |
||||||
|
fmt.Printf("Err, got %d exp %d\n", got, blocknum) |
||||||
|
panic("sd") |
||||||
|
} |
||||||
|
return n.receipts[index] |
||||||
|
} |
||||||
|
|
||||||
|
func (n *network) forget(blocknum uint64) { |
||||||
|
index := blocknum - uint64(n.offset) |
||||||
|
n.chain = n.chain[index:] |
||||||
|
n.receipts = n.receipts[index:] |
||||||
|
n.offset = int(blocknum) |
||||||
|
|
||||||
|
} |
||||||
|
func (n *network) progress(numBlocks int) { |
||||||
|
|
||||||
|
n.lock.Lock() |
||||||
|
defer n.lock.Unlock() |
||||||
|
//fmt.Printf("progressing...\n")
|
||||||
|
newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false) |
||||||
|
n.chain = append(n.chain, newBlocks...) |
||||||
|
n.receipts = append(n.receipts, newR...) |
||||||
|
n.cond.Broadcast() |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
func (n *network) headers(from int) []*types.Header { |
||||||
|
numHeaders := 128 |
||||||
|
var hdrs []*types.Header |
||||||
|
index := from - n.offset |
||||||
|
|
||||||
|
for index >= len(n.chain) { |
||||||
|
// wait for progress
|
||||||
|
n.cond.L.Lock() |
||||||
|
//fmt.Printf("header going into wait\n")
|
||||||
|
n.cond.Wait() |
||||||
|
index = from - n.offset |
||||||
|
n.cond.L.Unlock() |
||||||
|
} |
||||||
|
n.lock.RLock() |
||||||
|
defer n.lock.RUnlock() |
||||||
|
for i, b := range n.chain[index:] { |
||||||
|
hdrs = append(hdrs, b.Header()) |
||||||
|
if i >= numHeaders { |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
return hdrs |
||||||
|
} |
||||||
@ -0,0 +1,194 @@ |
|||||||
|
// Copyright 2019 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package downloader |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"sync" |
||||||
|
"sync/atomic" |
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/core/types" |
||||||
|
) |
||||||
|
|
||||||
|
// resultStore implements a structure for maintaining fetchResults, tracking their
|
||||||
|
// download-progress and delivering (finished) results.
|
||||||
|
type resultStore struct { |
||||||
|
items []*fetchResult // Downloaded but not yet delivered fetch results
|
||||||
|
resultOffset uint64 // Offset of the first cached fetch result in the block chain
|
||||||
|
|
||||||
|
// Internal index of first non-completed entry, updated atomically when needed.
|
||||||
|
// If all items are complete, this will equal length(items), so
|
||||||
|
// *important* : is not safe to use for indexing without checking against length
|
||||||
|
indexIncomplete int32 // atomic access
|
||||||
|
|
||||||
|
// throttleThreshold is the limit up to which we _want_ to fill the
|
||||||
|
// results. If blocks are large, we want to limit the results to less
|
||||||
|
// than the number of available slots, and maybe only fill 1024 out of
|
||||||
|
// 8192 possible places. The queue will, at certain times, recalibrate
|
||||||
|
// this index.
|
||||||
|
throttleThreshold uint64 |
||||||
|
|
||||||
|
lock sync.RWMutex |
||||||
|
} |
||||||
|
|
||||||
|
func newResultStore(size int) *resultStore { |
||||||
|
return &resultStore{ |
||||||
|
resultOffset: 0, |
||||||
|
items: make([]*fetchResult, size), |
||||||
|
throttleThreshold: uint64(size), |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// SetThrottleThreshold updates the throttling threshold based on the requested
|
||||||
|
// limit and the total queue capacity. It returns the (possibly capped) threshold
|
||||||
|
func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { |
||||||
|
r.lock.Lock() |
||||||
|
defer r.lock.Unlock() |
||||||
|
|
||||||
|
limit := uint64(len(r.items)) |
||||||
|
if threshold >= limit { |
||||||
|
threshold = limit |
||||||
|
} |
||||||
|
r.throttleThreshold = threshold |
||||||
|
return r.throttleThreshold |
||||||
|
} |
||||||
|
|
||||||
|
// AddFetch adds a header for body/receipt fetching. This is used when the queue
|
||||||
|
// wants to reserve headers for fetching.
|
||||||
|
//
|
||||||
|
// It returns the following:
|
||||||
|
// stale - if true, this item is already passed, and should not be requested again
|
||||||
|
// throttled - if true, the store is at capacity, this particular header is not prio now
|
||||||
|
// item - the result to store data into
|
||||||
|
// err - any error that occurred
|
||||||
|
func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { |
||||||
|
r.lock.Lock() |
||||||
|
defer r.lock.Unlock() |
||||||
|
|
||||||
|
var index int |
||||||
|
item, index, stale, throttled, err = r.getFetchResult(header.Number.Uint64()) |
||||||
|
if err != nil || stale || throttled { |
||||||
|
return stale, throttled, item, err |
||||||
|
} |
||||||
|
if item == nil { |
||||||
|
item = newFetchResult(header, fastSync) |
||||||
|
r.items[index] = item |
||||||
|
} |
||||||
|
return stale, throttled, item, err |
||||||
|
} |
||||||
|
|
||||||
|
// GetDeliverySlot returns the fetchResult for the given header. If the 'stale' flag
|
||||||
|
// is true, that means the header has already been delivered 'upstream'. This method
|
||||||
|
// does not bubble up the 'throttle' flag, since it's moot at the point in time when
|
||||||
|
// the item is downloaded and ready for delivery
|
||||||
|
func (r *resultStore) GetDeliverySlot(headerNumber uint64) (*fetchResult, bool, error) { |
||||||
|
r.lock.RLock() |
||||||
|
defer r.lock.RUnlock() |
||||||
|
|
||||||
|
res, _, stale, _, err := r.getFetchResult(headerNumber) |
||||||
|
return res, stale, err |
||||||
|
} |
||||||
|
|
||||||
|
// getFetchResult returns the fetchResult corresponding to the given item, and
|
||||||
|
// the index where the result is stored.
|
||||||
|
func (r *resultStore) getFetchResult(headerNumber uint64) (item *fetchResult, index int, stale, throttle bool, err error) { |
||||||
|
index = int(int64(headerNumber) - int64(r.resultOffset)) |
||||||
|
throttle = index >= int(r.throttleThreshold) |
||||||
|
stale = index < 0 |
||||||
|
|
||||||
|
if index >= len(r.items) { |
||||||
|
err = fmt.Errorf("%w: index allocation went beyond available resultStore space "+ |
||||||
|
"(index [%d] = header [%d] - resultOffset [%d], len(resultStore) = %d", errInvalidChain, |
||||||
|
index, headerNumber, r.resultOffset, len(r.items)) |
||||||
|
return nil, index, stale, throttle, err |
||||||
|
} |
||||||
|
if stale { |
||||||
|
return nil, index, stale, throttle, nil |
||||||
|
} |
||||||
|
item = r.items[index] |
||||||
|
return item, index, stale, throttle, nil |
||||||
|
} |
||||||
|
|
||||||
|
// hasCompletedItems returns true if there are processable items available
|
||||||
|
// this method is cheaper than countCompleted
|
||||||
|
func (r *resultStore) HasCompletedItems() bool { |
||||||
|
r.lock.RLock() |
||||||
|
defer r.lock.RUnlock() |
||||||
|
|
||||||
|
if len(r.items) == 0 { |
||||||
|
return false |
||||||
|
} |
||||||
|
if item := r.items[0]; item != nil && item.AllDone() { |
||||||
|
return true |
||||||
|
} |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// countCompleted returns the number of items ready for delivery, stopping at
|
||||||
|
// the first non-complete item.
|
||||||
|
//
|
||||||
|
// The mthod assumes (at least) rlock is held.
|
||||||
|
func (r *resultStore) countCompleted() int { |
||||||
|
// We iterate from the already known complete point, and see
|
||||||
|
// if any more has completed since last count
|
||||||
|
index := atomic.LoadInt32(&r.indexIncomplete) |
||||||
|
for ; ; index++ { |
||||||
|
if index >= int32(len(r.items)) { |
||||||
|
break |
||||||
|
} |
||||||
|
result := r.items[index] |
||||||
|
if result == nil || !result.AllDone() { |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
atomic.StoreInt32(&r.indexIncomplete, index) |
||||||
|
return int(index) |
||||||
|
} |
||||||
|
|
||||||
|
// GetCompleted returns the next batch of completed fetchResults
|
||||||
|
func (r *resultStore) GetCompleted(limit int) []*fetchResult { |
||||||
|
r.lock.Lock() |
||||||
|
defer r.lock.Unlock() |
||||||
|
|
||||||
|
completed := r.countCompleted() |
||||||
|
if limit > completed { |
||||||
|
limit = completed |
||||||
|
} |
||||||
|
results := make([]*fetchResult, limit) |
||||||
|
copy(results, r.items[:limit]) |
||||||
|
|
||||||
|
// Delete the results from the cache and clear the tail.
|
||||||
|
copy(r.items, r.items[limit:]) |
||||||
|
for i := len(r.items) - limit; i < len(r.items); i++ { |
||||||
|
r.items[i] = nil |
||||||
|
} |
||||||
|
// Advance the expected block number of the first cache entry
|
||||||
|
r.resultOffset += uint64(limit) |
||||||
|
atomic.AddInt32(&r.indexIncomplete, int32(-limit)) |
||||||
|
|
||||||
|
return results |
||||||
|
} |
||||||
|
|
||||||
|
// Prepare initialises the offset with the given block number
|
||||||
|
func (r *resultStore) Prepare(offset uint64) { |
||||||
|
r.lock.Lock() |
||||||
|
defer r.lock.Unlock() |
||||||
|
|
||||||
|
if r.resultOffset < offset { |
||||||
|
r.resultOffset = offset |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue