mirror
/
go-ethereum
mirror of https://github.com/ethereum/go-ethereum
2
0
Fork 1
Code Issues Releases Wiki Activity

Merge pull request #1784 from karalabe/standard-sync-stats

Browse Source 
eth, rpc: standardize the chain sync progress counters
pull/1813/head
Jeffrey Wilcke 9 years ago
parent e9a80518c7 d4d3fc6a70
commit 1cc2f08041
8 changed files with 1174 additions and 251 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 188
      eth/downloader/downloader.go
  2. 398
      eth/downloader/downloader_test.go
  3. 17
      eth/downloader/queue.go
  4. 789
      jsre/ethereum_js.go
  5. 12
      rpc/api/admin.go
  6. 4
      rpc/api/admin_js.go
  7. 15
      rpc/api/eth.go
  8. 2
      rpc/api/utils.go

188
eth/downloader/downloader.go
Unescape View File

@ -130,10 +130,9 @@ type Downloader struct {
interrupt int32 // Atomic boolean to signal termination
// Statistics
importStart time.Time // Instance when the last blocks were taken from the cache
importQueue []*Block // Previously taken blocks to check import progress
importDone int // Number of taken blocks already imported from the last batch
importLock sync.Mutex
syncStatsOrigin uint64 // Origin block number where syncing started at
syncStatsHeight uint64 // Highest block number known when syncing started
syncStatsLock sync.RWMutex // Lock protecting the sync stats fields
// Callbacks
hasBlock hashCheckFn // Checks if a block is present in the chain
@ -161,6 +160,7 @@ type Downloader struct {
cancelLock sync.RWMutex // Lock to protect the cancel channel in delivers
// Testing hooks
syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run
bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch
chainInsertHook func([]*Block) // Method to call upon inserting a chain of blocks (possibly in multiple invocations)
}
@ -192,27 +192,14 @@ func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, he
}
}
// Stats retrieves the current status of the downloader.
func (d *Downloader) Stats() (pending int, cached int, importing int, estimate time.Duration) {
// Fetch the download status
pending, cached = d.queue.Size()
// Boundaries retrieves the synchronisation boundaries, specifically the origin
// block where synchronisation started at (may have failed/suspended) and the
// latest known block which the synchonisation targets.
func (d *Downloader) Boundaries() (uint64, uint64) {
d.syncStatsLock.RLock()
defer d.syncStatsLock.RUnlock()
// Figure out the import progress
d.importLock.Lock()
defer d.importLock.Unlock()
for len(d.importQueue) > 0 && d.hasBlock(d.importQueue[0].RawBlock.Hash()) {
d.importQueue = d.importQueue[1:]
d.importDone++
}
importing = len(d.importQueue)
// Make an estimate on the total sync
estimate = 0
if d.importDone > 0 {
estimate = time.Since(d.importStart) / time.Duration(d.importDone) * time.Duration(pending+cached+importing)
}
return
return d.syncStatsOrigin, d.syncStatsHeight
}
// Synchronising returns whether the downloader is currently retrieving blocks.
@ -333,14 +320,29 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
switch {
case p.version == eth61:
// Old eth/61, use forward, concurrent hash and block retrieval algorithm
number, err := d.findAncestor61(p)
// Look up the sync boundaries: the common ancestor and the target block
latest, err := d.fetchHeight61(p)
if err != nil {
return err
}
origin, err := d.findAncestor61(p)
if err != nil {
return err
}
d.syncStatsLock.Lock()
if d.syncStatsHeight <= origin || d.syncStatsOrigin > origin {
d.syncStatsOrigin = origin
}
d.syncStatsHeight = latest
d.syncStatsLock.Unlock()
// Initiate the sync using a concurrent hash and block retrieval algorithm
if d.syncInitHook != nil {
d.syncInitHook(origin, latest)
}
errc := make(chan error, 2)
go func() { errc <- d.fetchHashes61(p, td, number+1) }()
go func() { errc <- d.fetchBlocks61(number + 1) }()
go func() { errc <- d.fetchHashes61(p, td, origin+1) }()
go func() { errc <- d.fetchBlocks61(origin + 1) }()
// If any fetcher fails, cancel the other
if err := <-errc; err != nil {
@ -351,14 +353,29 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
return <-errc
case p.version >= eth62:
// New eth/62, use forward, concurrent header and block body retrieval algorithm
number, err := d.findAncestor(p)
// Look up the sync boundaries: the common ancestor and the target block
latest, err := d.fetchHeight(p)
if err != nil {
return err
}
origin, err := d.findAncestor(p)
if err != nil {
return err
}
d.syncStatsLock.Lock()
if d.syncStatsHeight <= origin || d.syncStatsOrigin > origin {
d.syncStatsOrigin = origin
}
d.syncStatsHeight = latest
d.syncStatsLock.Unlock()
// Initiate the sync using a concurrent hash and block retrieval algorithm
if d.syncInitHook != nil {
d.syncInitHook(origin, latest)
}
errc := make(chan error, 2)
go func() { errc <- d.fetchHeaders(p, td, number+1) }()
go func() { errc <- d.fetchBodies(number + 1) }()
go func() { errc <- d.fetchHeaders(p, td, origin+1) }()
go func() { errc <- d.fetchBodies(origin + 1) }()
// If any fetcher fails, cancel the other
if err := <-errc; err != nil {
@ -401,6 +418,50 @@ func (d *Downloader) Terminate() {
d.cancel()
}
// fetchHeight61 retrieves the head block of the remote peer to aid in estimating
// the total time a pending synchronisation would take.
func (d *Downloader) fetchHeight61(p *peer) (uint64, error) {
glog.V(logger.Debug).Infof("%v: retrieving remote chain height", p)
// Request the advertised remote head block and wait for the response
go p.getBlocks([]common.Hash{p.head})
timeout := time.After(blockSoftTTL)
for {
select {
case <-d.cancelCh:
return 0, errCancelBlockFetch
case <-d.headerCh:
// Out of bounds eth/62 block headers received, ignore them
case <-d.bodyCh:
// Out of bounds eth/62 block bodies received, ignore them
case <-d.hashCh:
// Out of bounds hashes received, ignore them
case blockPack := <-d.blockCh:
// Discard anything not from the origin peer
if blockPack.peerId != p.id {
glog.V(logger.Debug).Infof("Received blocks from incorrect peer(%s)", blockPack.peerId)
break
}
// Make sure the peer actually gave something valid
blocks := blockPack.blocks
if len(blocks) != 1 {
glog.V(logger.Debug).Infof("%v: invalid number of head blocks: %d != 1", p, len(blocks))
return 0, errBadPeer
}
return blocks[0].NumberU64(), nil
case <-timeout:
glog.V(logger.Debug).Infof("%v: head block timeout", p)
return 0, errTimeout
}
}
}
// findAncestor61 tries to locate the common ancestor block of the local chain and
// a remote peers blockchain. In the general case when our node was in sync and
// on the correct chain, checking the top N blocks should already get us a match.
@ -776,6 +837,50 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
}
}
// fetchHeight retrieves the head header of the remote peer to aid in estimating
// the total time a pending synchronisation would take.
func (d *Downloader) fetchHeight(p *peer) (uint64, error) {
glog.V(logger.Debug).Infof("%v: retrieving remote chain height", p)
// Request the advertised remote head block and wait for the response
go p.getRelHeaders(p.head, 1, 0, false)
timeout := time.After(headerTTL)
for {
select {
case <-d.cancelCh:
return 0, errCancelBlockFetch
case headerPack := <-d.headerCh:
// Discard anything not from the origin peer
if headerPack.peerId != p.id {
glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId)
break
}
// Make sure the peer actually gave something valid
headers := headerPack.headers
if len(headers) != 1 {
glog.V(logger.Debug).Infof("%v: invalid number of head headers: %d != 1", p, len(headers))
return 0, errBadPeer
}
return headers[0].Number.Uint64(), nil
case <-d.bodyCh:
// Out of bounds block bodies received, ignore them
case <-d.hashCh:
// Out of bounds eth/61 hashes received, ignore them
case <-d.blockCh:
// Out of bounds eth/61 blocks received, ignore them
case <-timeout:
glog.V(logger.Debug).Infof("%v: head header timeout", p)
return 0, errTimeout
}
}
}
// findAncestor tries to locate the common ancestor block of the local chain and
// a remote peers blockchain. In the general case when our node was in sync and
// on the correct chain, checking the top N blocks should already get us a match.
@ -973,7 +1078,7 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
// Otherwise insert all the new headers, aborting in case of junk
glog.V(logger.Detail).Infof("%v: inserting %d headers from #%d", p, len(headerPack.headers), from)
inserts := d.queue.Insert(headerPack.headers)
inserts := d.queue.Insert(headerPack.headers, from)
if len(inserts) != len(headerPack.headers) {
glog.V(logger.Debug).Infof("%v: stale headers", p)
return errBadPeer
@ -1203,16 +1308,10 @@ func (d *Downloader) process() {
d.process()
}
}()
// Release the lock upon exit (note, before checking for reentry!), and set
// Release the lock upon exit (note, before checking for reentry!)
// the import statistics to zero.
defer func() {
d.importLock.Lock()
d.importQueue = nil
d.importDone = 0
d.importLock.Unlock()
defer atomic.StoreInt32(&d.processing, 0)
atomic.StoreInt32(&d.processing, 0)
}()
// Repeat the processing as long as there are blocks to import
for {
// Fetch the next batch of blocks
@ -1223,13 +1322,6 @@ func (d *Downloader) process() {
if d.chainInsertHook != nil {
d.chainInsertHook(blocks)
}
// Reset the import statistics
d.importLock.Lock()
d.importStart = time.Now()
d.importQueue = blocks
d.importDone = 0
d.importLock.Unlock()
// Actually import the blocks
glog.V(logger.Debug).Infof("Inserting chain with %d blocks (#%v - #%v)\n", len(blocks), blocks[0].RawBlock.Number(), blocks[len(blocks)-1].RawBlock.Number())
for len(blocks) != 0 {

398
eth/downloader/downloader_test.go
Unescape View File

@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"math/big"
"sync"
"sync/atomic"
"testing"
"time"
@ -99,6 +100,8 @@ type downloadTester struct {
peerHashes map[string][]common.Hash // Hash chain belonging to different test peers
peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers
peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains
lock sync.RWMutex
}
// newTester creates a new downloader test mocker.
@ -118,8 +121,8 @@ func newTester() *downloadTester {
// sync starts synchronizing with a remote peer, blocking until it completes.
func (dl *downloadTester) sync(id string, td *big.Int) error {
dl.lock.RLock()
hash := dl.peerHashes[id][0]
// If no particular TD was requested, load from the peer's blockchain
if td == nil {
td = big.NewInt(1)
@ -127,8 +130,9 @@ func (dl *downloadTester) sync(id string, td *big.Int) error {
td = diff
}
}
err := dl.downloader.synchronise(id, hash, td)
dl.lock.RUnlock()
err := dl.downloader.synchronise(id, hash, td)
for {
// If the queue is empty and processing stopped, break
hashes, blocks := dl.downloader.queue.Size()
@ -143,26 +147,41 @@ func (dl *downloadTester) sync(id string, td *big.Int) error {
// hasBlock checks if a block is pres ent in the testers canonical chain.
func (dl *downloadTester) hasBlock(hash common.Hash) bool {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.getBlock(hash) != nil
}
// getBlock retrieves a block from the testers canonical chain.
func (dl *downloadTester) getBlock(hash common.Hash) *types.Block {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.ownBlocks[hash]
}
// headBlock retrieves the current head block from the canonical chain.
func (dl *downloadTester) headBlock() *types.Block {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.getBlock(dl.ownHashes[len(dl.ownHashes)-1])
}
// getTd retrieves the block's total difficulty from the canonical chain.
func (dl *downloadTester) getTd(hash common.Hash) *big.Int {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.ownChainTd[hash]
}
// insertChain injects a new batch of blocks into the simulated chain.
func (dl *downloadTester) insertChain(blocks types.Blocks) (int, error) {
dl.lock.Lock()
defer dl.lock.Unlock()
for i, block := range blocks {
if _, ok := dl.ownBlocks[block.ParentHash()]; !ok {
return i, errors.New("unknown parent")
@ -183,9 +202,12 @@ func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash,
// specific delay time on processing the network packets sent to it, simulating
// potentially slow network IO.
func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, blocks map[common.Hash]*types.Block, delay time.Duration) error {
dl.lock.Lock()
defer dl.lock.Unlock()
err := dl.downloader.RegisterPeer(id, version, hashes[0],
dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay),
nil, dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay))
dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay))
if err == nil {
// Assign the owned hashes and blocks to the peer (deep copy)
dl.peerHashes[id] = make([]common.Hash, len(hashes))
@ -207,6 +229,9 @@ func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Ha
// dropPeer simulates a hard peer removal from the connection pool.
func (dl *downloadTester) dropPeer(id string) {
dl.lock.Lock()
defer dl.lock.Unlock()
delete(dl.peerHashes, id)
delete(dl.peerBlocks, id)
delete(dl.peerChainTds, id)
@ -221,6 +246,9 @@ func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) fun
return func(head common.Hash) error {
time.Sleep(delay)
dl.lock.RLock()
defer dl.lock.RUnlock()
// Gather the next batch of hashes
hashes := dl.peerHashes[id]
result := make([]common.Hash, 0, MaxHashFetch)
@ -250,6 +278,9 @@ func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) fun
return func(head uint64, count int) error {
time.Sleep(delay)
dl.lock.RLock()
defer dl.lock.RUnlock()
// Gather the next batch of hashes
hashes := dl.peerHashes[id]
result := make([]common.Hash, 0, count)
@ -271,6 +302,10 @@ func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) fun
func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([]common.Hash) error {
return func(hashes []common.Hash) error {
time.Sleep(delay)
dl.lock.RLock()
defer dl.lock.RUnlock()
blocks := dl.peerBlocks[id]
result := make([]*types.Block, 0, len(hashes))
for _, hash := range hashes {
@ -284,6 +319,27 @@ func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([
}
}
// peerGetRelHeadersFn constructs a GetBlockHeaders function based on a hashed
// origin; associated with a particular peer in the download tester. The returned
// function can be used to retrieve batches of headers from the particular peer.
func (dl *downloadTester) peerGetRelHeadersFn(id string, delay time.Duration) func(common.Hash, int, int, bool) error {
return func(origin common.Hash, amount int, skip int, reverse bool) error {
// Find the canonical number of the hash
dl.lock.RLock()
number := uint64(0)
for num, hash := range dl.peerHashes[id] {
if hash == origin {
number = uint64(len(dl.peerHashes[id]) - num - 1)
break
}
}
dl.lock.RUnlock()
// Use the absolute header fetcher to satisfy the query
return dl.peerGetAbsHeadersFn(id, delay)(number, amount, skip, reverse)
}
}
// peerGetAbsHeadersFn constructs a GetBlockHeaders function based on a numbered
// origin; associated with a particular peer in the download tester. The returned
// function can be used to retrieve batches of headers from the particular peer.
@ -291,6 +347,9 @@ func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) fu
return func(origin uint64, amount int, skip int, reverse bool) error {
time.Sleep(delay)
dl.lock.RLock()
defer dl.lock.RUnlock()
// Gather the next batch of hashes
hashes := dl.peerHashes[id]
blocks := dl.peerBlocks[id]
@ -315,6 +374,10 @@ func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) fu
func (dl *downloadTester) peerGetBodiesFn(id string, delay time.Duration) func([]common.Hash) error {
return func(hashes []common.Hash) error {
time.Sleep(delay)
dl.lock.RLock()
defer dl.lock.RUnlock()
blocks := dl.peerBlocks[id]
transactions := make([][]*types.Transaction, 0, len(hashes))
@ -384,13 +447,23 @@ func testThrottling(t *testing.T, protocol int) {
errc <- tester.sync("peer", nil)
}()
// Iteratively take some blocks, always checking the retrieval count
for len(tester.ownBlocks) < targetBlocks+1 {
for {
// Check the retrieval count synchronously (! reason for this ugly block)
tester.lock.RLock()
retrieved := len(tester.ownBlocks)
tester.lock.RUnlock()
if retrieved >= targetBlocks+1 {
break
}
// Wait a bit for sync to throttle itself
var cached int
for start := time.Now(); time.Since(start) < time.Second; {
time.Sleep(25 * time.Millisecond)
tester.downloader.queue.lock.RLock()
cached = len(tester.downloader.queue.blockPool)
tester.downloader.queue.lock.RUnlock()
if cached == blockCacheLimit || len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) == targetBlocks+1 {
break
}
@ -583,6 +656,67 @@ func testEmptyBlockShortCircuit(t *testing.T, protocol int) {
}
}
// Tests that headers are enqueued continuously, preventing malicious nodes from
// stalling the downloader by feeding gapped header chains.
func TestMissingHeaderAttack62(t *testing.T) { testMissingHeaderAttack(t, 62) }
func TestMissingHeaderAttack63(t *testing.T) { testMissingHeaderAttack(t, 63) }
func TestMissingHeaderAttack64(t *testing.T) { testMissingHeaderAttack(t, 64) }
func testMissingHeaderAttack(t *testing.T, protocol int) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
hashes, blocks := makeChain(targetBlocks, 0, genesis)
tester := newTester()
// Attempt a full sync with an attacker feeding gapped headers
tester.newPeer("attack", protocol, hashes, blocks)
missing := targetBlocks / 2
delete(tester.peerBlocks["attack"], hashes[missing])
if err := tester.sync("attack", nil); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
tester.newPeer("valid", protocol, hashes, blocks)
if err := tester.sync("valid", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
if imported := len(tester.ownBlocks); imported != len(hashes) {
t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes))
}
}
// Tests that if requested headers are shifted (i.e. first is missing), the queue
// detects the invalid numbering.
func TestShiftedHeaderAttack62(t *testing.T) { testShiftedHeaderAttack(t, 62) }
func TestShiftedHeaderAttack63(t *testing.T) { testShiftedHeaderAttack(t, 63) }
func TestShiftedHeaderAttack64(t *testing.T) { testShiftedHeaderAttack(t, 64) }
func testShiftedHeaderAttack(t *testing.T, protocol int) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
hashes, blocks := makeChain(targetBlocks, 0, genesis)
tester := newTester()
// Attempt a full sync with an attacker feeding shifted headers
tester.newPeer("attack", protocol, hashes, blocks)
delete(tester.peerBlocks["attack"], hashes[len(hashes)-2])
if err := tester.sync("attack", nil); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
tester.newPeer("valid", protocol, hashes, blocks)
if err := tester.sync("valid", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
if imported := len(tester.ownBlocks); imported != len(hashes) {
t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes))
}
}
// Tests that if a peer sends an invalid body for a requested block, it gets
// dropped immediately by the downloader.
func TestInvalidBlockBodyAttack62(t *testing.T) { testInvalidBlockBodyAttack(t, 62) }
@ -727,3 +861,259 @@ func testBlockBodyAttackerDropping(t *testing.T, protocol int) {
}
}
}
// Tests that synchronisation boundaries (origin block number and highest block
// number) is tracked and updated correctly.
func TestSyncBoundaries61(t *testing.T) { testSyncBoundaries(t, 61) }
func TestSyncBoundaries62(t *testing.T) { testSyncBoundaries(t, 62) }
func TestSyncBoundaries63(t *testing.T) { testSyncBoundaries(t, 63) }
func TestSyncBoundaries64(t *testing.T) { testSyncBoundaries(t, 64) }
func testSyncBoundaries(t *testing.T, protocol int) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
hashes, blocks := makeChain(targetBlocks, 0, genesis)
// Set a sync init hook to catch boundary changes
starting := make(chan struct{})
progress := make(chan struct{})
tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
// Retrieve the sync boundaries and ensure they are zero (pristine sync)
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
}
// Synchronise half the blocks and check initial boundaries
tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], blocks)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("peer-half", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks/2+1) {
t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks/2+1)
}
progress <- struct{}{}
pending.Wait()
// Synchronise all the blocks and check continuation boundaries
tester.newPeer("peer-full", protocol, hashes, blocks)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("peer-full", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, targetBlocks/2+1, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
}
// Tests that synchronisation boundaries (origin block number and highest block
// number) is tracked and updated correctly in case of a fork (or manual head
// revertal).
func TestForkedSyncBoundaries61(t *testing.T) { testForkedSyncBoundaries(t, 61) }
func TestForkedSyncBoundaries62(t *testing.T) { testForkedSyncBoundaries(t, 62) }
func TestForkedSyncBoundaries63(t *testing.T) { testForkedSyncBoundaries(t, 63) }
func TestForkedSyncBoundaries64(t *testing.T) { testForkedSyncBoundaries(t, 64) }
func testForkedSyncBoundaries(t *testing.T, protocol int) {
// Create a forked chain to simulate origin revertal
common, fork := MaxHashFetch, 2*MaxHashFetch
hashesA, hashesB, blocksA, blocksB := makeChainFork(common+fork, fork, genesis)
// Set a sync init hook to catch boundary changes
starting := make(chan struct{})
progress := make(chan struct{})
tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
// Retrieve the sync boundaries and ensure they are zero (pristine sync)
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
}
// Synchronise with one of the forks and check boundaries
tester.newPeer("fork A", protocol, hashesA, blocksA)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("fork A", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(len(hashesA)-1) {
t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, len(hashesA)-1)
}
progress <- struct{}{}
pending.Wait()
// Simulate a successful sync above the fork
tester.downloader.syncStatsOrigin = tester.downloader.syncStatsHeight
// Synchronise with the second fork and check boundary resets
tester.newPeer("fork B", protocol, hashesB, blocksB)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("fork B", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != uint64(common) || latest != uint64(len(hashesB)-1) {
t.Fatalf("Forking boundary mismatch: have %v/%v, want %v/%v", origin, latest, common, len(hashesB)-1)
}
progress <- struct{}{}
pending.Wait()
}
// Tests that if synchronisation is aborted due to some failure, then the boundary
// origin is not updated in the next sync cycle, as it should be considered the
// continuation of the previous sync and not a new instance.
func TestFailedSyncBoundaries61(t *testing.T) { testFailedSyncBoundaries(t, 61) }
func TestFailedSyncBoundaries62(t *testing.T) { testFailedSyncBoundaries(t, 62) }
func TestFailedSyncBoundaries63(t *testing.T) { testFailedSyncBoundaries(t, 63) }
func TestFailedSyncBoundaries64(t *testing.T) { testFailedSyncBoundaries(t, 64) }
func testFailedSyncBoundaries(t *testing.T, protocol int) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
hashes, blocks := makeChain(targetBlocks, 0, genesis)
// Set a sync init hook to catch boundary changes
starting := make(chan struct{})
progress := make(chan struct{})
tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
// Retrieve the sync boundaries and ensure they are zero (pristine sync)
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
}
// Attempt a full sync with a faulty peer
tester.newPeer("faulty", protocol, hashes, blocks)
missing := targetBlocks / 2
delete(tester.peerBlocks["faulty"], hashes[missing])
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("faulty", nil); err == nil {
t.Fatalf("succeeded faulty synchronisation")
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
// Synchronise with a good peer and check that the boundary origin remind the same after a failure
tester.newPeer("valid", protocol, hashes, blocks)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("valid", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
}
// Tests that if an attacker fakes a chain height, after the attack is detected,
// the boundary height is successfully reduced at the next sync invocation.
func TestFakedSyncBoundaries61(t *testing.T) { testFakedSyncBoundaries(t, 61) }
func TestFakedSyncBoundaries62(t *testing.T) { testFakedSyncBoundaries(t, 62) }
func TestFakedSyncBoundaries63(t *testing.T) { testFakedSyncBoundaries(t, 63) }
func TestFakedSyncBoundaries64(t *testing.T) { testFakedSyncBoundaries(t, 64) }
func testFakedSyncBoundaries(t *testing.T, protocol int) {
// Create a small block chain
targetBlocks := blockCacheLimit - 15
hashes, blocks := makeChain(targetBlocks+3, 0, genesis)
// Set a sync init hook to catch boundary changes
starting := make(chan struct{})
progress := make(chan struct{})
tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
// Retrieve the sync boundaries and ensure they are zero (pristine sync)
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
}
// Create and sync with an attacker that promises a higher chain than available
tester.newPeer("attack", protocol, hashes, blocks)
for i := 1; i < 3; i++ {
delete(tester.peerBlocks["attack"], hashes[i])
}
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("attack", nil); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks+3) {
t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks+3)
}
progress <- struct{}{}
pending.Wait()
// Synchronise with a good peer and check that the boundary height has been reduced to the true value
tester.newPeer("valid", protocol, hashes[3:], blocks)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("valid", nil); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
}

17
eth/downloader/queue.go
Unescape View File

@ -57,6 +57,7 @@ type queue struct {
headerPool map[common.Hash]*types.Header // [eth/62] Pending headers, mapping from their hashes
headerQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the bodies for
headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
pendPool map[string]*fetchRequest // Currently pending block retrieval operations
@ -91,6 +92,7 @@ func (q *queue) Reset() {
q.headerPool = make(map[common.Hash]*types.Header)
q.headerQueue.Reset()
q.headerHead = common.Hash{}
q.pendPool = make(map[string]*fetchRequest)
@ -186,7 +188,7 @@ func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash {
// Insert adds a set of headers for the download queue for scheduling, returning
// the new headers encountered.
func (q *queue) Insert(headers []*types.Header) []*types.Header {
func (q *queue) Insert(headers []*types.Header, from uint64) []*types.Header {
q.lock.Lock()
defer q.lock.Unlock()
@ -196,13 +198,24 @@ func (q *queue) Insert(headers []*types.Header) []*types.Header {
// Make sure no duplicate requests are executed
hash := header.Hash()
if _, ok := q.headerPool[hash]; ok {
glog.V(logger.Warn).Infof("Header %x already scheduled", hash)
glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled", header.Number.Uint64(), hash[:4])
continue
}
// Make sure chain order is honored and preserved throughout
if header.Number == nil || header.Number.Uint64() != from {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ordering, expected %d", header.Number, hash[:4], from)
break
}
if q.headerHead != (common.Hash{}) && q.headerHead != header.ParentHash {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ancestry", header.Number, hash[:4])
break
}
// Queue the header for body retrieval
inserts = append(inserts, header)
q.headerPool[hash] = header
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
q.headerHead = hash
from++
}
return inserts
}

789
jsre/ethereum_js.go
View File

File diff suppressed because it is too large Load Diff

12
rpc/api/admin.go
Unescape View File

@ -55,7 +55,6 @@ var (
"admin_exportChain": (*adminApi).ExportChain,
"admin_importChain": (*adminApi).ImportChain,
"admin_verbosity": (*adminApi).Verbosity,
"admin_chainSyncStatus": (*adminApi).ChainSyncStatus,
"admin_setSolc": (*adminApi).SetSolc,
"admin_datadir": (*adminApi).DataDir,
"admin_startRPC": (*adminApi).StartRPC,
@ -232,17 +231,6 @@ func (self *adminApi) Verbosity(req *shared.Request) (interface{}, error) {
return true, nil
}
func (self *adminApi) ChainSyncStatus(req *shared.Request) (interface{}, error) {
pending, cached, importing, estimate := self.ethereum.Downloader().Stats()
return map[string]interface{}{
"blocksAvailable": pending,
"blocksWaitingForImport": cached,
"importing": importing,
"estimate": estimate.String(),
}, nil
}
func (self *adminApi) SetSolc(req *shared.Request) (interface{}, error) {
args := new(SetSolcArgs)
if err := self.coder.Decode(req.Params, &args); err != nil {

4
rpc/api/admin_js.go
Unescape View File

@ -143,10 +143,6 @@ web3._extend({
new web3._extend.Property({
name: 'datadir',
getter: 'admin_datadir'
}),
new web3._extend.Property({
name: 'chainSyncStatus',
getter: 'admin_chainSyncStatus'
})
]
});

15
rpc/api/eth.go
Unescape View File

@ -55,6 +55,7 @@ var (
"eth_protocolVersion": (*ethApi).ProtocolVersion,
"eth_coinbase": (*ethApi).Coinbase,
"eth_mining": (*ethApi).IsMining,
"eth_syncing": (*ethApi).IsSyncing,
"eth_gasPrice": (*ethApi).GasPrice,
"eth_getStorage": (*ethApi).GetStorage,
"eth_storageAt": (*ethApi).GetStorage,
@ -166,6 +167,20 @@ func (self *ethApi) IsMining(req *shared.Request) (interface{}, error) {
return self.xeth.IsMining(), nil
}
func (self *ethApi) IsSyncing(req *shared.Request) (interface{}, error) {
current := self.ethereum.ChainManager().CurrentBlock().NumberU64()
origin, height := self.ethereum.Downloader().Boundaries()
if current < height {
return map[string]interface{}{
"startingBlock": newHexNum(big.NewInt(int64(origin)).Bytes()),
"currentBlock": newHexNum(big.NewInt(int64(current)).Bytes()),
"highestBlock": newHexNum(big.NewInt(int64(height)).Bytes()),
}, nil
}
return false, nil
}
func (self *ethApi) GasPrice(req *shared.Request) (interface{}, error) {
return newHexNum(self.xeth.DefaultGasPrice().Bytes()), nil
}

2
rpc/api/utils.go
Unescape View File

@ -32,7 +32,6 @@ var (
AutoCompletion = map[string][]string{
"admin": []string{
"addPeer",
"chainSyncStatus",
"datadir",
"exportChain",
"getContractInfo",
@ -99,6 +98,7 @@ var (
"sendRawTransaction",
"sendTransaction",
"sign",
"syncing",
},
"miner": []string{
"hashrate",

Write Preview
Loading…
Cancel
Save