eth/downloader: resolve local header by hash for beacon sync (#24691)

* eth/downlaoder: resolve local header by hash for beacon sync

* eth/downloader: fix error message

* eth/downloader: cap the reverse header resolving

* eth/downloader: re-enable tests

* eth/downloader: add warning logs

eth/downloader/beaconsync.go

@@ -263,10 +263,23 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) {
 // fetchBeaconHeaders feeds skeleton headers to the downloader queue for scheduling
 // until sync errors or is finished.
 func (d *Downloader) fetchBeaconHeaders(from uint64) error {
-	head, _, err := d.skeleton.Bounds()
+	head, tail, err := d.skeleton.Bounds()
 	if err != nil {
 		return err
 	}
+	// A part of headers are not in the skeleton space, try to resolve
+	// them from the local chain. Note the range should be very short
+	// and it should only happen when there are less than 64 post-merge
+	// blocks in the network.
+	var localHeaders []*types.Header
+	if from < tail.Number.Uint64() {
+		count := tail.Number.Uint64() - from
+		if count > uint64(fsMinFullBlocks) {
+			return fmt.Errorf("invalid origin (%d) of beacon sync (%d)", from, tail.Number)
+		}
+		localHeaders = d.readHeaderRange(tail, int(count))
+		log.Warn("Retrieved beacon headers from local", "from", from, "count", count)
+	}
 	for {
 		// Retrieve a batch of headers and feed it to the header processor
 		var (
@@ -275,8 +288,18 @@ func (d *Downloader) fetchBeaconHeaders(from uint64) error {
 		)
 		for i := 0; i < maxHeadersProcess && from <= head.Number.Uint64(); i++ {
 			header := d.skeleton.Header(from)
+
+			// The header is not found in skeleton space, try to find it in local chain.
+			if header == nil && from < tail.Number.Uint64() {
+				dist := tail.Number.Uint64() - from
+				if len(localHeaders) >= int(dist) {
+					header = localHeaders[dist-1]
+				}
+			}
+			// The header is still missing, the beacon sync is corrupted and bail out
+			// the error here.
 			if header == nil {
-				header = d.lightchain.GetHeaderByNumber(from)
+				return fmt.Errorf("missing beacon header %d", from)
 			}
 			headers = append(headers, header)
 			hashes = append(hashes, headers[i].Hash())

eth/downloader/downloader.go

@@ -159,9 +159,6 @@ type LightChain interface {
 	// GetHeaderByHash retrieves a header from the local chain.
 	GetHeaderByHash(common.Hash) *types.Header
 
-	// GetHeaderByNumber retrieves a block header from the local chain by number.
-	GetHeaderByNumber(number uint64) *types.Header
-
 	// CurrentHeader retrieves the head header from the local chain.
 	CurrentHeader() *types.Header
 
@@ -486,7 +483,15 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd *
 			// Retrieve the pivot header from the skeleton chain segment but
 			// fallback to local chain if it's not found in skeleton space.
 			if pivot = d.skeleton.Header(number); pivot == nil {
-				pivot = d.lightchain.GetHeaderByNumber(number)
+				_, oldest, _ := d.skeleton.Bounds() // error is already checked
+				if number < oldest.Number.Uint64() {
+					count := int(oldest.Number.Uint64() - number) // it's capped by fsMinFullBlocks
+					headers := d.readHeaderRange(oldest, count)
+					if len(headers) == count {
+						pivot = headers[len(headers)-1]
+						log.Warn("Retrieved pivot header from local", "number", pivot.Number, "hash", pivot.Hash(), "latest", latest.Number, "oldest", oldest.Number)
+					}
+				}
 			}
 			// Print an error log and return directly in case the pivot header
 			// is still not found. It means the skeleton chain is not linked
@@ -1772,3 +1777,25 @@ func (d *Downloader) DeliverSnapPacket(peer *snap.Peer, packet snap.Packet) erro
 		return fmt.Errorf("unexpected snap packet type: %T", packet)
 	}
 }
+
+// readHeaderRange returns a list of headers, using the given last header as the base,
+// and going backwards towards genesis. This method assumes that the caller already has
+// placed a reasonable cap on count.
+func (d *Downloader) readHeaderRange(last *types.Header, count int) []*types.Header {
+	var (
+		current = last
+		headers []*types.Header
+	)
+	for {
+		parent := d.lightchain.GetHeaderByHash(current.ParentHash)
+		if parent == nil {
+			break // The chain is not continuous, or the chain is exhausted
+		}
+		headers = append(headers, parent)
+		if len(headers) >= count {
+			break
+		}
+		current = parent
+	}
+	return headers
+}