Merge branch 'release/0.6.6'

block_pool.go

@@ -1,17 +1,27 @@
 package eth
 
 import (
+	"bytes"
+	"container/list"
 	"math"
 	"math/big"
 	"sync"
+	"time"
 
 	"github.com/ethereum/eth-go/ethchain"
+	"github.com/ethereum/eth-go/ethlog"
 	"github.com/ethereum/eth-go/ethutil"
+	"github.com/ethereum/eth-go/ethwire"
 )
 
+var poollogger = ethlog.NewLogger("[BPOOL]")
+
 type block struct {
-	peer  *Peer
-	block *ethchain.Block
+	from      *Peer
+	peer      *Peer
+	block     *ethchain.Block
+	reqAt     time.Time
+	requested int
 }
 
 type BlockPool struct {
@@ -22,7 +32,10 @@ type BlockPool struct {
 	hashPool [][]byte
 	pool     map[string]*block
 
-	td *big.Int
+	td   *big.Int
+	quit chan bool
+
+	ChainLength, BlocksProcessed int
 }
 
 func NewBlockPool(eth *Ethereum) *BlockPool {
@@ -30,6 +43,7 @@ func NewBlockPool(eth *Ethereum) *BlockPool {
 		eth:  eth,
 		pool: make(map[string]*block),
 		td:   ethutil.Big0,
+		quit: make(chan bool),
 	}
 }
 
@@ -45,9 +59,9 @@ func (self *BlockPool) HasCommonHash(hash []byte) bool {
 	return self.eth.BlockChain().GetBlock(hash) != nil
 }
 
-func (self *BlockPool) AddHash(hash []byte) {
+func (self *BlockPool) AddHash(hash []byte, peer *Peer) {
 	if self.pool[string(hash)] == nil {
-		self.pool[string(hash)] = &block{nil, nil}
+		self.pool[string(hash)] = &block{peer, nil, nil, time.Now(), 0}
 
 		self.hashPool = append([][]byte{hash}, self.hashPool...)
 	}
@@ -58,24 +72,54 @@ func (self *BlockPool) SetBlock(b *ethchain.Block, peer *Peer) {
 
 	if self.pool[hash] == nil && !self.eth.BlockChain().HasBlock(b.Hash()) {
 		self.hashPool = append(self.hashPool, b.Hash())
-		self.pool[hash] = &block{peer, b}
+		self.pool[hash] = &block{peer, peer, b, time.Now(), 0}
 	} else if self.pool[hash] != nil {
 		self.pool[hash].block = b
 	}
+
+	self.BlocksProcessed++
 }
 
-func (self *BlockPool) CheckLinkAndProcess(f func(block *ethchain.Block)) {
+func (self *BlockPool) getParent(block *ethchain.Block) *ethchain.Block {
+	for _, item := range self.pool {
+		if item.block != nil {
+			if bytes.Compare(item.block.Hash(), block.PrevHash) == 0 {
+				return item.block
+			}
+		}
+	}
+
+	return nil
+}
 
+func (self *BlockPool) GetChainFromBlock(block *ethchain.Block) ethchain.Blocks {
 	var blocks ethchain.Blocks
+
+	for b := block; b != nil; b = self.getParent(b) {
+		blocks = append(ethchain.Blocks{b}, blocks...)
+	}
+
+	return blocks
+}
+
+func (self *BlockPool) Blocks() (blocks ethchain.Blocks) {
 	for _, item := range self.pool {
 		if item.block != nil {
 			blocks = append(blocks, item.block)
 		}
 	}
 
+	return
+}
+
+func (self *BlockPool) ProcessCanonical(f func(block *ethchain.Block)) (procAmount int) {
+	blocks := self.Blocks()
+
 	ethchain.BlockBy(ethchain.Number).Sort(blocks)
 	for _, block := range blocks {
 		if self.eth.BlockChain().HasBlock(block.PrevHash) {
+			procAmount++
+
 			f(block)
 
 			hash := block.Hash()
@@ -84,23 +128,108 @@ func (self *BlockPool) CheckLinkAndProcess(f func(block *ethchain.Block)) {
 		}
 
 	}
+
+	return
 }
 
-func (self *BlockPool) Take(amount int, peer *Peer) (hashes [][]byte) {
-	self.mut.Lock()
-	defer self.mut.Unlock()
+func (self *BlockPool) DistributeHashes() {
+	var (
+		peerLen = self.eth.peers.Len()
+		amount  = 200 * peerLen
+		dist    = make(map[*Peer][][]byte)
+	)
 
 	num := int(math.Min(float64(amount), float64(len(self.pool))))
-	j := 0
-	for i := 0; i < len(self.hashPool) && j < num; i++ {
-		hash := string(self.hashPool[i])
-		if self.pool[hash] != nil && (self.pool[hash].peer == nil || self.pool[hash].peer == peer) && self.pool[hash].block == nil {
-			self.pool[hash].peer = peer
-
-			hashes = append(hashes, self.hashPool[i])
-			j++
+	for i, j := 0, 0; i < len(self.hashPool) && j < num; i++ {
+		hash := self.hashPool[i]
+		item := self.pool[string(hash)]
+
+		if item != nil && item.block == nil {
+			var peer *Peer
+			lastFetchFailed := time.Since(item.reqAt) > 5*time.Second
+
+			// Handle failed requests
+			if lastFetchFailed && item.requested > 0 && item.peer != nil {
+				if item.requested < 100 {
+					// Select peer the hash was retrieved off
+					peer = item.from
+				} else {
+					// Remove it
+					self.hashPool = ethutil.DeleteFromByteSlice(self.hashPool, hash)
+					delete(self.pool, string(hash))
+				}
+			} else if lastFetchFailed || item.peer == nil {
+				// Find a suitable, available peer
+				eachPeer(self.eth.peers, func(p *Peer, v *list.Element) {
+					if peer == nil && len(dist[p]) < amount/peerLen {
+						peer = p
+					}
+				})
+			}
+
+			if peer != nil {
+				item.reqAt = time.Now()
+				item.peer = peer
+				item.requested++
+
+				dist[peer] = append(dist[peer], hash)
+			}
 		}
 	}
 
-	return
+	for peer, hashes := range dist {
+		peer.FetchBlocks(hashes)
+	}
+}
+
+func (self *BlockPool) Start() {
+	go self.update()
+}
+
+func (self *BlockPool) Stop() {
+	close(self.quit)
+}
+
+func (self *BlockPool) update() {
+	serviceTimer := time.NewTicker(100 * time.Millisecond)
+	procTimer := time.NewTicker(500 * time.Millisecond)
+out:
+	for {
+		select {
+		case <-self.quit:
+			break out
+		case <-serviceTimer.C:
+			// Check if we're catching up. If not distribute the hashes to
+			// the peers and download the blockchain
+			done := true
+			eachPeer(self.eth.peers, func(p *Peer, v *list.Element) {
+				if p.statusKnown && p.FetchingHashes() {
+					done = false
+				}
+			})
+
+			if done && len(self.hashPool) > 0 {
+				self.DistributeHashes()
+			}
+
+			if self.ChainLength < len(self.hashPool) {
+				self.ChainLength = len(self.hashPool)
+			}
+		case <-procTimer.C:
+			// XXX We can optimize this lifting this on to a new goroutine.
+			// We'd need to make sure that the pools are properly protected by a mutex
+			amount := self.ProcessCanonical(func(block *ethchain.Block) {
+				err := self.eth.StateManager().Process(block, false)
+				if err != nil {
+					poollogger.Infoln(err)
+				}
+			})
+
+			// Do not propagate to the network on catchups
+			if amount == 1 {
+				block := self.eth.BlockChain().CurrentBlock
+				self.eth.Broadcast(ethwire.MsgBlockTy, []interface{}{block.Value().Val})
+			}
+		}
+	}
 }

ethchain/state_manager.go

@@ -250,15 +250,13 @@ func (sm *StateManager) Process(block *Block, dontReact bool) (err error) {
 		fk := append([]byte("bloom"), block.Hash()...)
 		sm.Ethereum.Db().Put(fk, filter.Bin())
 
-		statelogger.Infof("Added block #%d (%x)\n", block.Number, block.Hash())
+		statelogger.Debugf("Added block #%d (%x)\n", block.Number, block.Hash())
 		if dontReact == false {
 			sm.Ethereum.Reactor().Post("newBlock", block)
 
 			state.Manifest().Reset()
 		}
 
-		sm.Ethereum.Broadcast(ethwire.MsgBlockTy, []interface{}{block.Value().Val})
-
 		sm.Ethereum.TxPool().RemoveInvalid(state)
 	} else {
 		statelogger.Errorln("total diff failed")

ethereum.go

@@ -158,6 +158,9 @@ func (s *Ethereum) StateManager() *ethchain.StateManager {
 func (s *Ethereum) TxPool() *ethchain.TxPool {
 	return s.txPool
 }
+func (s *Ethereum) BlockPool() *BlockPool {
+	return s.blockPool
+}
 func (self *Ethereum) Db() ethutil.Database {
 	return self.db
 }
@@ -383,7 +386,6 @@ func (s *Ethereum) ReapDeadPeerHandler() {
 // Start the ethereum
 func (s *Ethereum) Start(seed bool) {
 	s.reactor.Start()
-	s.blockPool.Start()
 	// Bind to addr and port
 	ln, err := net.Listen("tcp", ":"+s.Port)
 	if err != nil {
@@ -504,6 +506,7 @@ func (s *Ethereum) Stop() {
 	s.stateManager.Stop()
 	s.reactor.Flush()
 	s.reactor.Stop()
+	s.blockPool.Stop()
 
 	ethlogger.Infoln("Server stopped")
 	close(s.shutdownChan)

ethminer/miner.go

@@ -3,7 +3,6 @@ package ethminer
 import (
 	"bytes"
 	"sort"
-	"time"
 
 	"github.com/ethereum/eth-go/ethchain"
 	"github.com/ethereum/eth-go/ethlog"
@@ -136,12 +135,6 @@ func (miner *Miner) listener() {
 				}
 			}
 		default:
-			// This hack is only temporarily
-			if len(miner.txs) == 0 {
-				time.Sleep(2 * time.Second)
-				continue
-			}
-
 			miner.mineNewBlock()
 		}
 	}

ethpipe/js_types.go

@@ -36,13 +36,6 @@ func NewJSBlock(block *ethchain.Block) *JSBlock {
 		ptxs = append(ptxs, *NewJSTx(tx))
 	}
 
-	/*
-		txJson, err := json.Marshal(ptxs)
-		if err != nil {
-			return nil
-		}
-		return &JSBlock{ref: block, Size: block.Size().String(), Number: int(block.Number.Uint64()), GasUsed: block.GasUsed.String(), GasLimit: block.GasLimit.String(), Hash: ethutil.Bytes2Hex(block.Hash()), Transactions: string(txJson), Time: block.Time, Coinbase: ethutil.Bytes2Hex(block.Coinbase)}
-	*/
 	list := ethutil.NewList(ptxs)
 
 	return &JSBlock{ref: block, Size: block.Size().String(), Number: int(block.Number.Uint64()), GasUsed: block.GasUsed.String(), GasLimit: block.GasLimit.String(), Hash: ethutil.Bytes2Hex(block.Hash()), Transactions: list, Time: block.Time, Coinbase: ethutil.Bytes2Hex(block.Coinbase)}

ethtrie/trie.go

@@ -196,8 +196,8 @@ func (t *Trie) Update(key, value string) {
 }
 
 func (t *Trie) Get(key string) string {
-	t.mut.RLock()
-	defer t.mut.RUnlock()
+	t.mut.Lock()
+	defer t.mut.Unlock()
 
 	k := CompactHexDecode(key)
 	c := ethutil.NewValue(t.getState(t.Root, k))

ethutil/list.go

@@ -2,7 +2,6 @@ package ethutil
 
 import (
 	"encoding/json"
-	"fmt"
 	"reflect"
 )
 
@@ -10,6 +9,7 @@ import (
 // for containing any slice type to use in an environment which
 // does not support slice types (e.g., JavaScript, QML)
 type List struct {
+	val    interface{}
 	list   reflect.Value
 	Length int
 }
@@ -21,7 +21,7 @@ func NewList(t interface{}) *List {
 		panic("list container initialized with a non-slice type")
 	}
 
-	return &List{list, list.Len()}
+	return &List{t, list, list.Len()}
 }
 
 func EmptyList() *List {
@@ -30,17 +30,24 @@ func EmptyList() *List {
 
 // Get N element from the embedded slice. Returns nil if OOB.
 func (self *List) Get(i int) interface{} {
-	if self.list.Len() == 3 {
-		fmt.Println("get", i, self.list.Index(i).Interface())
-	}
 
 	if self.list.Len() > i {
-		return self.list.Index(i).Interface()
+		i := self.list.Index(i).Interface()
+
+		return i
 	}
 
 	return nil
 }
 
+func (self *List) GetAsJson(i int) interface{} {
+	e := self.Get(i)
+
+	r, _ := json.Marshal(e)
+
+	return string(r)
+}
+
 // Appends value at the end of the slice. Panics when incompatible value
 // is given.
 func (self *List) Append(v interface{}) {

ethvm/vm.go

@@ -961,7 +961,6 @@ func (self *Message) Addr() []byte {
 }
 
 func (self *Message) Exec(codeAddr []byte, caller ClosureRef) (ret []byte, err error) {
-	fmt.Printf("%x %x\n", codeAddr[0:4], self.address[0:4])
 	queue := self.vm.queue
 	self.vm.queue = list.New()
 

peer.go

@@ -131,6 +131,7 @@ type Peer struct {
 	// Last received pong message
 	lastPong          int64
 	lastBlockReceived time.Time
+	LastHashReceived  time.Time
 
 	host             []byte
 	port             uint16
@@ -176,6 +177,7 @@ func NewPeer(conn net.Conn, ethereum *Ethereum, inbound bool) *Peer {
 		caps:            ethereum.ServerCaps(),
 		version:         ethereum.ClientIdentity().String(),
 		protocolCaps:    ethutil.NewValue(nil),
+		td:              big.NewInt(0),
 	}
 }
 
@@ -191,6 +193,7 @@ func NewOutboundPeer(addr string, ethereum *Ethereum, caps Caps) *Peer {
 		caps:         caps,
 		version:      ethereum.ClientIdentity().String(),
 		protocolCaps: ethutil.NewValue(nil),
+		td:           big.NewInt(0),
 	}
 
 	// Set up the connection in another goroutine so we don't block the main thread
@@ -505,22 +508,19 @@ func (p *Peer) HandleInbound() {
 					for it.Next() {
 						hash := it.Value().Bytes()
 
+						p.lastReceivedHash = hash
+						p.LastHashReceived = time.Now()
+
 						if blockPool.HasCommonHash(hash) {
 							foundCommonHash = true
 
 							break
 						}
 
-						blockPool.AddHash(hash)
-
-						p.lastReceivedHash = hash
-
-						p.lastBlockReceived = time.Now()
+						blockPool.AddHash(hash, p)
 					}
 
-					if foundCommonHash || msg.Data.Len() == 0 {
-						p.FetchBlocks()
-					} else {
+					if !foundCommonHash && msg.Data.Len() != 0 {
 						p.FetchHashes()
 					}
 
@@ -538,20 +538,6 @@ func (p *Peer) HandleInbound() {
 
 						p.lastBlockReceived = time.Now()
 					}
-
-					var err error
-					blockPool.CheckLinkAndProcess(func(block *ethchain.Block) {
-						err = p.ethereum.StateManager().Process(block, false)
-					})
-
-					if err != nil {
-						peerlogger.Infoln(err)
-					} else {
-						// Don't trigger if there's just one block.
-						if blockPool.Len() != 0 && msg.Data.Len() > 1 {
-							p.FetchBlocks()
-						}
-					}
 				}
 			}
 		}
@@ -560,10 +546,7 @@ func (p *Peer) HandleInbound() {
 	p.Stop()
 }
 
-func (self *Peer) FetchBlocks() {
-	blockPool := self.ethereum.blockPool
-
-	hashes := blockPool.Take(100, self)
+func (self *Peer) FetchBlocks(hashes [][]byte) {
 	if len(hashes) > 0 {
 		self.QueueMessage(ethwire.NewMessage(ethwire.MsgGetBlocksTy, ethutil.ByteSliceToInterface(hashes)))
 	}
@@ -572,7 +555,7 @@ func (self *Peer) FetchBlocks() {
 func (self *Peer) FetchHashes() {
 	blockPool := self.ethereum.blockPool
 
-	if self.td.Cmp(blockPool.td) >= 0 {
+	if self.td.Cmp(self.ethereum.HighestTDPeer()) >= 0 {
 		blockPool.td = self.td
 
 		if !blockPool.HasLatestHash() {
@@ -581,19 +564,27 @@ func (self *Peer) FetchHashes() {
 	}
 }
 
+func (self *Peer) FetchingHashes() bool {
+	return time.Since(self.LastHashReceived) < 5*time.Second
+}
+
 // General update method
 func (self *Peer) update() {
-	serviceTimer := time.NewTicker(5 * time.Second)
+	serviceTimer := time.NewTicker(100 * time.Millisecond)
 
 out:
 	for {
 		select {
 		case <-serviceTimer.C:
-			since := time.Since(self.lastBlockReceived)
-			if since > 10*time.Second && self.ethereum.blockPool.Len() != 0 && self.IsCap("eth") {
-				self.FetchHashes()
-			} else if since > 5*time.Second {
-				self.catchingUp = false
+			if self.IsCap("eth") {
+				var (
+					sinceBlock = time.Since(self.lastBlockReceived)
+					sinceHash  = time.Since(self.LastHashReceived)
+				)
+
+				if sinceBlock > 5*time.Second && sinceHash > 5*time.Second {
+					self.catchingUp = false
+				}
 			}
 		case <-self.quit:
 			break out
@@ -717,7 +708,7 @@ func (self *Peer) handleStatus(msg *ethwire.Msg) {
 	// Compare the total TD with the blockchain TD. If remote is higher
 	// fetch hashes from highest TD node.
 	if self.td.Cmp(self.ethereum.BlockChain().TD) > 0 {
-		self.ethereum.blockPool.AddHash(self.lastReceivedHash)
+		self.ethereum.blockPool.AddHash(self.lastReceivedHash, self)
 		self.FetchHashes()
 	}
 
@@ -761,6 +752,14 @@ func (p *Peer) handleHandshake(msg *ethwire.Msg) {
 		return
 	}
 
+	// Self connect detection
+	pubkey := p.ethereum.KeyManager().PublicKey()
+	if bytes.Compare(pubkey[1:], pub) == 0 {
+		p.Stop()
+
+		return
+	}
+
 	usedPub := 0
 	// This peer is already added to the peerlist so we expect to find a double pubkey at least once
 	eachPeer(p.ethereum.Peers(), func(peer *Peer, e *list.Element) {
@@ -779,16 +778,8 @@ func (p *Peer) handleHandshake(msg *ethwire.Msg) {
 	// If this is an inbound connection send an ack back
 	if p.inbound {
 		p.port = uint16(port)
-
-		// Self connect detection
-		pubkey := p.ethereum.KeyManager().PublicKey()
-		if bytes.Compare(pubkey, p.pubkey) == 0 {
-			p.Stop()
-
-			return
-		}
-
 	}
+
 	p.SetVersion(clientId)
 
 	p.versionKnown = true