forked from mirror/go-ethereum
eth/downloader: refactor downloader + queue (#21263)
* eth/downloader: refactor downloader + queue downloader, fetcher: throttle-metrics, fetcher filter improvements, standalone resultcache downloader: more accurate deliverytime calculation, less mem overhead in state requests downloader/queue: increase underlying buffer of results, new throttle mechanism eth/downloader: updates to tests eth/downloader: fix up some review concerns eth/downloader/queue: minor fixes eth/downloader: minor fixes after review call eth/downloader: testcases for queue.go eth/downloader: minor change, don't set progress unless progress... eth/downloader: fix flaw which prevented useless peers from being dropped eth/downloader: try to fix tests eth/downloader: verify non-deliveries against advertised remote head eth/downloader: fix flaw with checking closed-status causing hang eth/downloader: hashing avoidance eth/downloader: review concerns + simplify resultcache and queue eth/downloader: add back some locks, address review concerns downloader/queue: fix remaining lock flaw * eth/downloader: nitpick fixes * eth/downloader: remove the *2*3/4 throttling threshold dance * eth/downloader: print correct throttle threshold in stats Co-authored-by: Péter Szilágyi <peterke@gmail.com>release/1.9
Martin Holst Swende
4 years ago
committed by
GitHub
parent
3a57eecc69
commit
105922180f
@ -0,0 +1,53 @@ |
||||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package downloader |
||||
|
||||
import ( |
||||
"sort" |
||||
"testing" |
||||
) |
||||
|
||||
func TestPeerThroughputSorting(t *testing.T) { |
||||
a := &peerConnection{ |
||||
id: "a", |
||||
headerThroughput: 1.25, |
||||
} |
||||
b := &peerConnection{ |
||||
id: "b", |
||||
headerThroughput: 1.21, |
||||
} |
||||
c := &peerConnection{ |
||||
id: "c", |
||||
headerThroughput: 1.23, |
||||
} |
||||
|
||||
peers := []*peerConnection{a, b, c} |
||||
tps := []float64{a.headerThroughput, |
||||
b.headerThroughput, c.headerThroughput} |
||||
sortPeers := &peerThroughputSort{peers, tps} |
||||
sort.Sort(sortPeers) |
||||
if got, exp := sortPeers.p[0].id, "a"; got != exp { |
||||
t.Errorf("sort fail, got %v exp %v", got, exp) |
||||
} |
||||
if got, exp := sortPeers.p[1].id, "c"; got != exp { |
||||
t.Errorf("sort fail, got %v exp %v", got, exp) |
||||
} |
||||
if got, exp := sortPeers.p[2].id, "b"; got != exp { |
||||
t.Errorf("sort fail, got %v exp %v", got, exp) |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,426 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package downloader |
||||
|
||||
import ( |
||||
"fmt" |
||||
"math/big" |
||||
"math/rand" |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/consensus/ethash" |
||||
"github.com/ethereum/go-ethereum/core" |
||||
"github.com/ethereum/go-ethereum/core/rawdb" |
||||
"github.com/ethereum/go-ethereum/core/types" |
||||
"github.com/ethereum/go-ethereum/log" |
||||
"github.com/ethereum/go-ethereum/params" |
||||
) |
||||
|
||||
var ( |
||||
testdb = rawdb.NewMemoryDatabase() |
||||
genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) |
||||
) |
||||
|
||||
// makeChain creates a chain of n blocks starting at and including parent.
|
||||
// the returned hash chain is ordered head->parent. In addition, every 3rd block
|
||||
// contains a transaction and every 5th an uncle to allow testing correct block
|
||||
// reassembly.
|
||||
func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { |
||||
blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { |
||||
block.SetCoinbase(common.Address{seed}) |
||||
// Add one tx to every secondblock
|
||||
if !empty && i%2 == 0 { |
||||
signer := types.MakeSigner(params.TestChainConfig, block.Number()) |
||||
tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
block.AddTx(tx) |
||||
} |
||||
}) |
||||
return blocks, receipts |
||||
} |
||||
|
||||
type chainData struct { |
||||
blocks []*types.Block |
||||
offset int |
||||
} |
||||
|
||||
var chain *chainData |
||||
var emptyChain *chainData |
||||
|
||||
func init() { |
||||
// Create a chain of blocks to import
|
||||
targetBlocks := 128 |
||||
blocks, _ := makeChain(targetBlocks, 0, genesis, false) |
||||
chain = &chainData{blocks, 0} |
||||
|
||||
blocks, _ = makeChain(targetBlocks, 0, genesis, true) |
||||
emptyChain = &chainData{blocks, 0} |
||||
} |
||||
|
||||
func (chain *chainData) headers() []*types.Header { |
||||
hdrs := make([]*types.Header, len(chain.blocks)) |
||||
for i, b := range chain.blocks { |
||||
hdrs[i] = b.Header() |
||||
} |
||||
return hdrs |
||||
} |
||||
|
||||
func (chain *chainData) Len() int { |
||||
return len(chain.blocks) |
||||
} |
||||
|
||||
func dummyPeer(id string) *peerConnection { |
||||
p := &peerConnection{ |
||||
id: id, |
||||
lacking: make(map[common.Hash]struct{}), |
||||
} |
||||
return p |
||||
} |
||||
|
||||
func TestBasics(t *testing.T) { |
||||
q := newQueue(10) |
||||
if !q.Idle() { |
||||
t.Errorf("new queue should be idle") |
||||
} |
||||
q.Prepare(1, FastSync) |
||||
if res := q.Results(false); len(res) != 0 { |
||||
t.Fatal("new queue should have 0 results") |
||||
} |
||||
|
||||
// Schedule a batch of headers
|
||||
q.Schedule(chain.headers(), 1) |
||||
if q.Idle() { |
||||
t.Errorf("queue should not be idle") |
||||
} |
||||
if got, exp := q.PendingBlocks(), chain.Len(); got != exp { |
||||
t.Errorf("wrong pending block count, got %d, exp %d", got, exp) |
||||
} |
||||
// Only non-empty receipts get added to task-queue
|
||||
if got, exp := q.PendingReceipts(), 64; got != exp { |
||||
t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) |
||||
} |
||||
// Items are now queued for downloading, next step is that we tell the
|
||||
// queue that a certain peer will deliver them for us
|
||||
{ |
||||
peer := dummyPeer("peer-1") |
||||
fetchReq, _, throttle := q.ReserveBodies(peer, 50) |
||||
if !throttle { |
||||
// queue size is only 10, so throttling should occur
|
||||
t.Fatal("should throttle") |
||||
} |
||||
// But we should still get the first things to fetch
|
||||
if got, exp := len(fetchReq.Headers), 5; got != exp { |
||||
t.Fatalf("expected %d requests, got %d", exp, got) |
||||
} |
||||
if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { |
||||
t.Fatalf("expected header %d, got %d", exp, got) |
||||
} |
||||
} |
||||
{ |
||||
peer := dummyPeer("peer-2") |
||||
fetchReq, _, throttle := q.ReserveBodies(peer, 50) |
||||
|
||||
// The second peer should hit throttling
|
||||
if !throttle { |
||||
t.Fatalf("should not throttle") |
||||
} |
||||
// And not get any fetches at all, since it was throttled to begin with
|
||||
if fetchReq != nil { |
||||
t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) |
||||
} |
||||
} |
||||
//fmt.Printf("blockTaskQueue len: %d\n", q.blockTaskQueue.Size())
|
||||
//fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size())
|
||||
{ |
||||
// The receipt delivering peer should not be affected
|
||||
// by the throttling of body deliveries
|
||||
peer := dummyPeer("peer-3") |
||||
fetchReq, _, throttle := q.ReserveReceipts(peer, 50) |
||||
if !throttle { |
||||
// queue size is only 10, so throttling should occur
|
||||
t.Fatal("should throttle") |
||||
} |
||||
// But we should still get the first things to fetch
|
||||
if got, exp := len(fetchReq.Headers), 5; got != exp { |
||||
t.Fatalf("expected %d requests, got %d", exp, got) |
||||
} |
||||
if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { |
||||
t.Fatalf("expected header %d, got %d", exp, got) |
||||
} |
||||
|
||||
} |
||||
//fmt.Printf("blockTaskQueue len: %d\n", q.blockTaskQueue.Size())
|
||||
//fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size())
|
||||
//fmt.Printf("processable: %d\n", q.resultCache.countCompleted())
|
||||
} |
||||
|
||||
func TestEmptyBlocks(t *testing.T) { |
||||
q := newQueue(10) |
||||
|
||||
q.Prepare(1, FastSync) |
||||
// Schedule a batch of headers
|
||||
q.Schedule(emptyChain.headers(), 1) |
||||
if q.Idle() { |
||||
t.Errorf("queue should not be idle") |
||||
} |
||||
if got, exp := q.PendingBlocks(), len(emptyChain.blocks); got != exp { |
||||
t.Errorf("wrong pending block count, got %d, exp %d", got, exp) |
||||
} |
||||
if got, exp := q.PendingReceipts(), 0; got != exp { |
||||
t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) |
||||
} |
||||
// They won't be processable, because the fetchresults haven't been
|
||||
// created yet
|
||||
if got, exp := q.resultCache.countCompleted(), 0; got != exp { |
||||
t.Errorf("wrong processable count, got %d, exp %d", got, exp) |
||||
} |
||||
|
||||
// Items are now queued for downloading, next step is that we tell the
|
||||
// queue that a certain peer will deliver them for us
|
||||
// That should trigger all of them to suddenly become 'done'
|
||||
{ |
||||
// Reserve blocks
|
||||
peer := dummyPeer("peer-1") |
||||
fetchReq, _, _ := q.ReserveBodies(peer, 50) |
||||
|
||||
// there should be nothing to fetch, blocks are empty
|
||||
if fetchReq != nil { |
||||
t.Fatal("there should be no body fetch tasks remaining") |
||||
} |
||||
|
||||
} |
||||
if q.blockTaskQueue.Size() != len(emptyChain.blocks)-10 { |
||||
t.Errorf("expected block task queue to be 0, got %d", q.blockTaskQueue.Size()) |
||||
} |
||||
if q.receiptTaskQueue.Size() != 0 { |
||||
t.Errorf("expected receipt task queue to be 0, got %d", q.receiptTaskQueue.Size()) |
||||
} |
||||
//fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size())
|
||||
{ |
||||
peer := dummyPeer("peer-3") |
||||
fetchReq, _, _ := q.ReserveReceipts(peer, 50) |
||||
|
||||
// there should be nothing to fetch, blocks are empty
|
||||
if fetchReq != nil { |
||||
t.Fatal("there should be no body fetch tasks remaining") |
||||
} |
||||
} |
||||
if got, exp := q.resultCache.countCompleted(), 10; got != exp { |
||||
t.Errorf("wrong processable count, got %d, exp %d", got, exp) |
||||
} |
||||
} |
||||
|
||||
// XTestDelivery does some more extensive testing of events that happen,
|
||||
// blocks that become known and peers that make reservations and deliveries.
|
||||
// disabled since it's not really a unit-test, but can be executed to test
|
||||
// some more advanced scenarios
|
||||
func XTestDelivery(t *testing.T) { |
||||
// the outside network, holding blocks
|
||||
blo, rec := makeChain(128, 0, genesis, false) |
||||
world := newNetwork() |
||||
world.receipts = rec |
||||
world.chain = blo |
||||
world.progress(10) |
||||
if false { |
||||
log.Root().SetHandler(log.StdoutHandler) |
||||
|
||||
} |
||||
q := newQueue(10) |
||||
var wg sync.WaitGroup |
||||
q.Prepare(1, FastSync) |
||||
wg.Add(1) |
||||
go func() { |
||||
// deliver headers
|
||||
defer wg.Done() |
||||
c := 1 |
||||
for { |
||||
//fmt.Printf("getting headers from %d\n", c)
|
||||
hdrs := world.headers(c) |
||||
l := len(hdrs) |
||||
//fmt.Printf("scheduling %d headers, first %d last %d\n",
|
||||
// l, hdrs[0].Number.Uint64(), hdrs[len(hdrs)-1].Number.Uint64())
|
||||
q.Schedule(hdrs, uint64(c)) |
||||
c += l |
||||
} |
||||
}() |
||||
wg.Add(1) |
||||
go func() { |
||||
// collect results
|
||||
defer wg.Done() |
||||
tot := 0 |
||||
for { |
||||
res := q.Results(true) |
||||
tot += len(res) |
||||
fmt.Printf("got %d results, %d tot\n", len(res), tot) |
||||
// Now we can forget about these
|
||||
world.forget(res[len(res)-1].Header.Number.Uint64()) |
||||
|
||||
} |
||||
}() |
||||
wg.Add(1) |
||||
go func() { |
||||
defer wg.Done() |
||||
// reserve body fetch
|
||||
i := 4 |
||||
for { |
||||
peer := dummyPeer(fmt.Sprintf("peer-%d", i)) |
||||
f, _, _ := q.ReserveBodies(peer, rand.Intn(30)) |
||||
if f != nil { |
||||
var emptyList []*types.Header |
||||
var txs [][]*types.Transaction |
||||
var uncles [][]*types.Header |
||||
numToSkip := rand.Intn(len(f.Headers)) |
||||
for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { |
||||
txs = append(txs, world.getTransactions(hdr.Number.Uint64())) |
||||
uncles = append(uncles, emptyList) |
||||
} |
||||
time.Sleep(100 * time.Millisecond) |
||||
_, err := q.DeliverBodies(peer.id, txs, uncles) |
||||
if err != nil { |
||||
fmt.Printf("delivered %d bodies %v\n", len(txs), err) |
||||
} |
||||
} else { |
||||
i++ |
||||
time.Sleep(200 * time.Millisecond) |
||||
} |
||||
} |
||||
}() |
||||
go func() { |
||||
defer wg.Done() |
||||
// reserve receiptfetch
|
||||
peer := dummyPeer("peer-3") |
||||
for { |
||||
f, _, _ := q.ReserveReceipts(peer, rand.Intn(50)) |
||||
if f != nil { |
||||
var rcs [][]*types.Receipt |
||||
for _, hdr := range f.Headers { |
||||
rcs = append(rcs, world.getReceipts(hdr.Number.Uint64())) |
||||
} |
||||
_, err := q.DeliverReceipts(peer.id, rcs) |
||||
if err != nil { |
||||
fmt.Printf("delivered %d receipts %v\n", len(rcs), err) |
||||
} |
||||
time.Sleep(100 * time.Millisecond) |
||||
} else { |
||||
time.Sleep(200 * time.Millisecond) |
||||
} |
||||
} |
||||
}() |
||||
wg.Add(1) |
||||
go func() { |
||||
defer wg.Done() |
||||
for i := 0; i < 50; i++ { |
||||
time.Sleep(300 * time.Millisecond) |
||||
//world.tick()
|
||||
//fmt.Printf("trying to progress\n")
|
||||
world.progress(rand.Intn(100)) |
||||
} |
||||
for i := 0; i < 50; i++ { |
||||
time.Sleep(2990 * time.Millisecond) |
||||
|
||||
} |
||||
}() |
||||
wg.Add(1) |
||||
go func() { |
||||
defer wg.Done() |
||||
for { |
||||
time.Sleep(990 * time.Millisecond) |
||||
fmt.Printf("world block tip is %d\n", |
||||
world.chain[len(world.chain)-1].Header().Number.Uint64()) |
||||
fmt.Println(q.Stats()) |
||||
} |
||||
}() |
||||
wg.Wait() |
||||
} |
||||
|
||||
func newNetwork() *network { |
||||
var l sync.RWMutex |
||||
return &network{ |
||||
cond: sync.NewCond(&l), |
||||
offset: 1, // block 1 is at blocks[0]
|
||||
} |
||||
} |
||||
|
||||
// represents the network
|
||||
type network struct { |
||||
offset int |
||||
chain []*types.Block |
||||
receipts []types.Receipts |
||||
lock sync.RWMutex |
||||
cond *sync.Cond |
||||
} |
||||
|
||||
func (n *network) getTransactions(blocknum uint64) types.Transactions { |
||||
index := blocknum - uint64(n.offset) |
||||
return n.chain[index].Transactions() |
||||
} |
||||
func (n *network) getReceipts(blocknum uint64) types.Receipts { |
||||
index := blocknum - uint64(n.offset) |
||||
if got := n.chain[index].Header().Number.Uint64(); got != blocknum { |
||||
fmt.Printf("Err, got %d exp %d\n", got, blocknum) |
||||
panic("sd") |
||||
} |
||||
return n.receipts[index] |
||||
} |
||||
|
||||
func (n *network) forget(blocknum uint64) { |
||||
index := blocknum - uint64(n.offset) |
||||
n.chain = n.chain[index:] |
||||
n.receipts = n.receipts[index:] |
||||
n.offset = int(blocknum) |
||||
|
||||
} |
||||
func (n *network) progress(numBlocks int) { |
||||
|
||||
n.lock.Lock() |
||||
defer n.lock.Unlock() |
||||
//fmt.Printf("progressing...\n")
|
||||
newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false) |
||||
n.chain = append(n.chain, newBlocks...) |
||||
n.receipts = append(n.receipts, newR...) |
||||
n.cond.Broadcast() |
||||
|
||||
} |
||||
|
||||
func (n *network) headers(from int) []*types.Header { |
||||
numHeaders := 128 |
||||
var hdrs []*types.Header |
||||
index := from - n.offset |
||||
|
||||
for index >= len(n.chain) { |
||||
// wait for progress
|
||||
n.cond.L.Lock() |
||||
//fmt.Printf("header going into wait\n")
|
||||
n.cond.Wait() |
||||
index = from - n.offset |
||||
n.cond.L.Unlock() |
||||
} |
||||
n.lock.RLock() |
||||
defer n.lock.RUnlock() |
||||
for i, b := range n.chain[index:] { |
||||
hdrs = append(hdrs, b.Header()) |
||||
if i >= numHeaders { |
||||
break |
||||
} |
||||
} |
||||
return hdrs |
||||
} |
||||
@ -0,0 +1,194 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package downloader |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
"sync/atomic" |
||||
|
||||
"github.com/ethereum/go-ethereum/core/types" |
||||
) |
||||
|
||||
// resultStore implements a structure for maintaining fetchResults, tracking their
|
||||
// download-progress and delivering (finished) results.
|
||||
type resultStore struct { |
||||
items []*fetchResult // Downloaded but not yet delivered fetch results
|
||||
resultOffset uint64 // Offset of the first cached fetch result in the block chain
|
||||
|
||||
// Internal index of first non-completed entry, updated atomically when needed.
|
||||
// If all items are complete, this will equal length(items), so
|
||||
// *important* : is not safe to use for indexing without checking against length
|
||||
indexIncomplete int32 // atomic access
|
||||
|
||||
// throttleThreshold is the limit up to which we _want_ to fill the
|
||||
// results. If blocks are large, we want to limit the results to less
|
||||
// than the number of available slots, and maybe only fill 1024 out of
|
||||
// 8192 possible places. The queue will, at certain times, recalibrate
|
||||
// this index.
|
||||
throttleThreshold uint64 |
||||
|
||||
lock sync.RWMutex |
||||
} |
||||
|
||||
func newResultStore(size int) *resultStore { |
||||
return &resultStore{ |
||||
resultOffset: 0, |
||||
items: make([]*fetchResult, size), |
||||
throttleThreshold: uint64(size), |
||||
} |
||||
} |
||||
|
||||
// SetThrottleThreshold updates the throttling threshold based on the requested
|
||||
// limit and the total queue capacity. It returns the (possibly capped) threshold
|
||||
func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { |
||||
r.lock.Lock() |
||||
defer r.lock.Unlock() |
||||
|
||||
limit := uint64(len(r.items)) |
||||
if threshold >= limit { |
||||
threshold = limit |
||||
} |
||||
r.throttleThreshold = threshold |
||||
return r.throttleThreshold |
||||
} |
||||
|
||||
// AddFetch adds a header for body/receipt fetching. This is used when the queue
|
||||
// wants to reserve headers for fetching.
|
||||
//
|
||||
// It returns the following:
|
||||
// stale - if true, this item is already passed, and should not be requested again
|
||||
// throttled - if true, the store is at capacity, this particular header is not prio now
|
||||
// item - the result to store data into
|
||||
// err - any error that occurred
|
||||
func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { |
||||
r.lock.Lock() |
||||
defer r.lock.Unlock() |
||||
|
||||
var index int |
||||
item, index, stale, throttled, err = r.getFetchResult(header.Number.Uint64()) |
||||
if err != nil || stale || throttled { |
||||
return stale, throttled, item, err |
||||
} |
||||
if item == nil { |
||||
item = newFetchResult(header, fastSync) |
||||
r.items[index] = item |
||||
} |
||||
return stale, throttled, item, err |
||||
} |
||||
|
||||
// GetDeliverySlot returns the fetchResult for the given header. If the 'stale' flag
|
||||
// is true, that means the header has already been delivered 'upstream'. This method
|
||||
// does not bubble up the 'throttle' flag, since it's moot at the point in time when
|
||||
// the item is downloaded and ready for delivery
|
||||
func (r *resultStore) GetDeliverySlot(headerNumber uint64) (*fetchResult, bool, error) { |
||||
r.lock.RLock() |
||||
defer r.lock.RUnlock() |
||||
|
||||
res, _, stale, _, err := r.getFetchResult(headerNumber) |
||||
return res, stale, err |
||||
} |
||||
|
||||
// getFetchResult returns the fetchResult corresponding to the given item, and
|
||||
// the index where the result is stored.
|
||||
func (r *resultStore) getFetchResult(headerNumber uint64) (item *fetchResult, index int, stale, throttle bool, err error) { |
||||
index = int(int64(headerNumber) - int64(r.resultOffset)) |
||||
throttle = index >= int(r.throttleThreshold) |
||||
stale = index < 0 |
||||
|
||||
if index >= len(r.items) { |
||||
err = fmt.Errorf("%w: index allocation went beyond available resultStore space "+ |
||||
"(index [%d] = header [%d] - resultOffset [%d], len(resultStore) = %d", errInvalidChain, |
||||
index, headerNumber, r.resultOffset, len(r.items)) |
||||
return nil, index, stale, throttle, err |
||||
} |
||||
if stale { |
||||
return nil, index, stale, throttle, nil |
||||
} |
||||
item = r.items[index] |
||||
return item, index, stale, throttle, nil |
||||
} |
||||
|
||||
// hasCompletedItems returns true if there are processable items available
|
||||
// this method is cheaper than countCompleted
|
||||
func (r *resultStore) HasCompletedItems() bool { |
||||
r.lock.RLock() |
||||
defer r.lock.RUnlock() |
||||
|
||||
if len(r.items) == 0 { |
||||
return false |
||||
} |
||||
if item := r.items[0]; item != nil && item.AllDone() { |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// countCompleted returns the number of items ready for delivery, stopping at
|
||||
// the first non-complete item.
|
||||
//
|
||||
// The mthod assumes (at least) rlock is held.
|
||||
func (r *resultStore) countCompleted() int { |
||||
// We iterate from the already known complete point, and see
|
||||
// if any more has completed since last count
|
||||
index := atomic.LoadInt32(&r.indexIncomplete) |
||||
for ; ; index++ { |
||||
if index >= int32(len(r.items)) { |
||||
break |
||||
} |
||||
result := r.items[index] |
||||
if result == nil || !result.AllDone() { |
||||
break |
||||
} |
||||
} |
||||
atomic.StoreInt32(&r.indexIncomplete, index) |
||||
return int(index) |
||||
} |
||||
|
||||
// GetCompleted returns the next batch of completed fetchResults
|
||||
func (r *resultStore) GetCompleted(limit int) []*fetchResult { |
||||
r.lock.Lock() |
||||
defer r.lock.Unlock() |
||||
|
||||
completed := r.countCompleted() |
||||
if limit > completed { |
||||
limit = completed |
||||
} |
||||
results := make([]*fetchResult, limit) |
||||
copy(results, r.items[:limit]) |
||||
|
||||
// Delete the results from the cache and clear the tail.
|
||||
copy(r.items, r.items[limit:]) |
||||
for i := len(r.items) - limit; i < len(r.items); i++ { |
||||
r.items[i] = nil |
||||
} |
||||
// Advance the expected block number of the first cache entry
|
||||
r.resultOffset += uint64(limit) |
||||
atomic.AddInt32(&r.indexIncomplete, int32(-limit)) |
||||
|
||||
return results |
||||
} |
||||
|
||||
// Prepare initialises the offset with the given block number
|
||||
func (r *resultStore) Prepare(offset uint64) { |
||||
r.lock.Lock() |
||||
defer r.lock.Unlock() |
||||
|
||||
if r.resultOffset < offset { |
||||
r.resultOffset = offset |
||||
} |
||||
} |
||||
Loading…
Reference in new issue