beacon/engine, eth/catalyst: EIP-4844 updates for the engine API (#27736)

This is a spin-out from the EIP-4844 devnet branch, containing just the Engine API modifications
and nothing else. The newPayloadV3 endpoint won't really work in this version, but we need the
data structures for testing so I'd like to get this in early.

Co-authored-by: Marius van der Wijden <m.vanderwijden@live.de>

beacon/engine/gen_ed.go

@@ -32,6 +32,8 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) {
 		BlockHash     common.Hash         `json:"blockHash"     gencodec:"required"`
 		Transactions  []hexutil.Bytes     `json:"transactions"  gencodec:"required"`
 		Withdrawals   []*types.Withdrawal `json:"withdrawals"`
+		DataGasUsed   *hexutil.Uint64     `json:"dataGasUsed"`
+		ExcessDataGas *hexutil.Uint64     `json:"excessDataGas"`
 	}
 	var enc ExecutableData
 	enc.ParentHash = e.ParentHash
@@ -54,6 +56,8 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) {
 		}
 	}
 	enc.Withdrawals = e.Withdrawals
+	enc.DataGasUsed = (*hexutil.Uint64)(e.DataGasUsed)
+	enc.ExcessDataGas = (*hexutil.Uint64)(e.ExcessDataGas)
 	return json.Marshal(&enc)
 }
 
@@ -75,6 +79,8 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error {
 		BlockHash     *common.Hash        `json:"blockHash"     gencodec:"required"`
 		Transactions  []hexutil.Bytes     `json:"transactions"  gencodec:"required"`
 		Withdrawals   []*types.Withdrawal `json:"withdrawals"`
+		DataGasUsed   *hexutil.Uint64     `json:"dataGasUsed"`
+		ExcessDataGas *hexutil.Uint64     `json:"excessDataGas"`
 	}
 	var dec ExecutableData
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -142,5 +148,11 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error {
 	if dec.Withdrawals != nil {
 		e.Withdrawals = dec.Withdrawals
 	}
+	if dec.DataGasUsed != nil {
+		e.DataGasUsed = (*uint64)(dec.DataGasUsed)
+	}
+	if dec.ExcessDataGas != nil {
+		e.ExcessDataGas = (*uint64)(dec.ExcessDataGas)
+	}
 	return nil
 }

beacon/engine/gen_epe.go

@@ -17,10 +17,12 @@ func (e ExecutionPayloadEnvelope) MarshalJSON() ([]byte, error) {
 	type ExecutionPayloadEnvelope struct {
 		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
+		BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
 	}
 	var enc ExecutionPayloadEnvelope
 	enc.ExecutionPayload = e.ExecutionPayload
 	enc.BlockValue = (*hexutil.Big)(e.BlockValue)
+	enc.BlobsBundle = e.BlobsBundle
 	return json.Marshal(&enc)
 }
 
@@ -29,6 +31,7 @@ func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error {
 	type ExecutionPayloadEnvelope struct {
 		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
+		BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
 	}
 	var dec ExecutionPayloadEnvelope
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -42,5 +45,8 @@ func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error {
 		return errors.New("missing required field 'blockValue' for ExecutionPayloadEnvelope")
 	}
 	e.BlockValue = (*big.Int)(dec.BlockValue)
+	if dec.BlobsBundle != nil {
+		e.BlobsBundle = dec.BlobsBundle
+	}
 	return nil
 }

beacon/engine/types.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
@@ -61,6 +62,8 @@ type ExecutableData struct {
 	BlockHash     common.Hash         `json:"blockHash"     gencodec:"required"`
 	Transactions  [][]byte            `json:"transactions"  gencodec:"required"`
 	Withdrawals   []*types.Withdrawal `json:"withdrawals"`
+	DataGasUsed   *uint64             `json:"dataGasUsed"`
+	ExcessDataGas *uint64             `json:"excessDataGas"`
 }
 
 // JSON type overrides for executableData.
@@ -73,6 +76,8 @@ type executableDataMarshaling struct {
 	ExtraData     hexutil.Bytes
 	LogsBloom     hexutil.Bytes
 	Transactions  []hexutil.Bytes
+	DataGasUsed   *hexutil.Uint64
+	ExcessDataGas *hexutil.Uint64
 }
 
 //go:generate go run github.com/fjl/gencodec -type ExecutionPayloadEnvelope -field-override executionPayloadEnvelopeMarshaling -out gen_epe.go
@@ -80,6 +85,13 @@ type executableDataMarshaling struct {
 type ExecutionPayloadEnvelope struct {
 	ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 	BlockValue       *big.Int        `json:"blockValue"  gencodec:"required"`
+	BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
+}
+
+type BlobsBundleV1 struct {
+	Commitments []hexutil.Bytes `json:"commitments"`
+	Proofs      []hexutil.Bytes `json:"proofs"`
+	Blobs       []hexutil.Bytes `json:"blobs"`
 }
 
 // JSON type overrides for ExecutionPayloadEnvelope.
@@ -152,14 +164,15 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) {
 // ExecutableDataToBlock constructs a block from executable data.
 // It verifies that the following fields:
 //
-//	len(extraData) <= 32
+//		len(extraData) <= 32
-//	uncleHash = emptyUncleHash
+//		uncleHash = emptyUncleHash
-//	difficulty = 0
+//		difficulty = 0
+//	 	if versionedHashes != nil, versionedHashes match to blob transactions
 //
 // and that the blockhash of the constructed block matches the parameters. Nil
 // Withdrawals value will propagate through the returned block. Empty
 // Withdrawals value must be passed via non-nil, length 0 value in params.
-func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) {
+func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash) (*types.Block, error) {
 	txs, err := decodeTransactions(params.Transactions)
 	if err != nil {
 		return nil, err
@@ -174,6 +187,18 @@ func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) {
 	if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) {
 		return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas)
 	}
+	var blobHashes []common.Hash
+	for _, tx := range txs {
+		blobHashes = append(blobHashes, tx.BlobHashes()...)
+	}
+	if len(blobHashes) != len(versionedHashes) {
+		return nil, fmt.Errorf("invalid number of versionedHashes: %v blobHashes: %v", versionedHashes, blobHashes)
+	}
+	for i := 0; i < len(blobHashes); i++ {
+		if blobHashes[i] != versionedHashes[i] {
+			return nil, fmt.Errorf("invalid versionedHash at %v: %v blobHashes: %v", i, versionedHashes, blobHashes)
+		}
+	}
 	// Only set withdrawalsRoot if it is non-nil. This allows CLs to use
 	// ExecutableData before withdrawals are enabled by marshaling
 	// Withdrawals as the json null value.
@@ -199,6 +224,8 @@ func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) {
 		Extra:           params.ExtraData,
 		MixDigest:       params.Random,
 		WithdrawalsHash: withdrawalsRoot,
+		ExcessDataGas:   params.ExcessDataGas,
+		DataGasUsed:     params.DataGasUsed,
 	}
 	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */).WithWithdrawals(params.Withdrawals)
 	if block.Hash() != params.BlockHash {
@@ -209,7 +236,7 @@ func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) {
 
 // BlockToExecutableData constructs the ExecutableData structure by filling the
 // fields from the given block. It assumes the given block is post-merge block.
-func BlockToExecutableData(block *types.Block, fees *big.Int) *ExecutionPayloadEnvelope {
+func BlockToExecutableData(block *types.Block, fees *big.Int, blobs []kzg4844.Blob, commitments []kzg4844.Commitment, proofs []kzg4844.Proof) *ExecutionPayloadEnvelope {
 	data := &ExecutableData{
 		BlockHash:     block.Hash(),
 		ParentHash:    block.ParentHash(),
@@ -226,8 +253,20 @@ func BlockToExecutableData(block *types.Block, fees *big.Int) *ExecutionPayloadE
 		Random:        block.MixDigest(),
 		ExtraData:     block.Extra(),
 		Withdrawals:   block.Withdrawals(),
+		DataGasUsed:   block.DataGasUsed(),
+		ExcessDataGas: block.ExcessDataGas(),
+	}
+	blobsBundle := BlobsBundleV1{
+		Commitments: make([]hexutil.Bytes, 0),
+		Blobs:       make([]hexutil.Bytes, 0),
+		Proofs:      make([]hexutil.Bytes, 0),
+	}
+	for i := range blobs {
+		blobsBundle.Blobs = append(blobsBundle.Blobs, hexutil.Bytes(blobs[i][:]))
+		blobsBundle.Commitments = append(blobsBundle.Commitments, hexutil.Bytes(commitments[i][:]))
+		blobsBundle.Proofs = append(blobsBundle.Proofs, hexutil.Bytes(proofs[i][:]))
 	}
-	return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees}
+	return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees, BlobsBundle: &blobsBundle}
 }
 
 // ExecutionPayloadBodyV1 is used in the response to GetPayloadBodiesByHashV1 and GetPayloadBodiesByRangeV1

eth/catalyst/api.go

@@ -81,8 +81,10 @@ var caps = []string{
 	"engine_exchangeTransitionConfigurationV1",
 	"engine_getPayloadV1",
 	"engine_getPayloadV2",
+	"engine_getPayloadV3",
 	"engine_newPayloadV1",
 	"engine_newPayloadV2",
+	"engine_newPayloadV3",
 	"engine_getPayloadBodiesByHashV1",
 	"engine_getPayloadBodiesByRangeV1",
 }
@@ -405,23 +407,13 @@ func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.Execu
 	return api.getPayload(payloadID)
 }
 
-func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) {
+// GetPayloadV3 returns a cached payload by id.
-	log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID)
+func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) {
-	data := api.localBlocks.get(payloadID, false)
+	return api.getPayload(payloadID)
-	if data == nil {
-		return nil, engine.UnknownPayload
-	}
-	return data, nil
 }
 
-// getFullPayload returns a cached payload by it. The difference is that this
+func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) {
-// function always expects a non-empty payload, but can also return empty one
+	log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID)
-// if no transaction is executable.
-//
-// Note, this function is not a part of standard engine API, meant to be used
-// by consensus client mock in dev mode.
-func (api *ConsensusAPI) getFullPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) {
-	log.Trace("Engine API request received", "method", "GetFullPayload", "id", payloadID)
 	data := api.localBlocks.get(payloadID, true)
 	if data == nil {
 		return nil, engine.UnknownPayload
@@ -434,7 +426,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl
 	if params.Withdrawals != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1"))
 	}
-	return api.newPayload(params)
+	return api.newPayload(params, nil)
 }
 
 // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
@@ -446,10 +438,29 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl
 	} else if params.Withdrawals != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai"))
 	}
-	return api.newPayload(params)
+	if api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("newPayloadV2 called post-cancun"))
+	}
+	return api.newPayload(params, nil)
 }
 
-func (api *ConsensusAPI) newPayload(params engine.ExecutableData) (engine.PayloadStatusV1, error) {
+// NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
+func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes *[]common.Hash) (engine.PayloadStatusV1, error) {
+	if !api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("newPayloadV3 called pre-cancun"))
+	}
+
+	if params.ExcessDataGas == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(fmt.Errorf("nil excessDataGas post-cancun"))
+	}
+	var hashes []common.Hash
+	if versionedHashes != nil {
+		hashes = *versionedHashes
+	}
+	return api.newPayload(params, hashes)
+}
+
+func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash) (engine.PayloadStatusV1, error) {
 	// The locking here is, strictly, not required. Without these locks, this can happen:
 	//
 	// 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to
@@ -467,9 +478,9 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData) (engine.Payloa
 	defer api.newPayloadLock.Unlock()
 
 	log.Trace("Engine API request received", "method", "NewPayload", "number", params.Number, "hash", params.BlockHash)
-	block, err := engine.ExecutableDataToBlock(params)
+	block, err := engine.ExecutableDataToBlock(params, versionedHashes)
 	if err != nil {
-		log.Debug("Invalid NewPayload params", "params", params, "error", err)
+		log.Warn("Invalid NewPayload params", "params", params, "error", err)
 		return engine.PayloadStatusV1{Status: engine.INVALID}, nil
 	}
 	// Stash away the last update to warn the user if the beacon client goes offline
@@ -730,8 +741,8 @@ func (api *ConsensusAPI) ExchangeCapabilities([]string) []string {
 	return caps
 }
 
-// GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which
+// GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which allows for retrieval of a list
-// allows for retrieval of a list of block bodies by the engine api.
+// of block bodies by the engine api.
 func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engine.ExecutionPayloadBodyV1 {
 	var bodies = make([]*engine.ExecutionPayloadBodyV1, len(hashes))
 	for i, hash := range hashes {
@@ -741,8 +752,8 @@ func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engin
 	return bodies
 }
 
-// GetPayloadBodiesByRangeV1 implements engine_getPayloadBodiesByRangeV1 which
+// GetPayloadBodiesByRangeV1 implements engine_getPayloadBodiesByRangeV1 which allows for retrieval of a range
-// allows for retrieval of a range of block bodies by the engine api.
+// of block bodies by the engine api.
 func (api *ConsensusAPI) GetPayloadBodiesByRangeV1(start, count hexutil.Uint64) ([]*engine.ExecutionPayloadBodyV1, error) {
 	if start == 0 || count == 0 {
 		return nil, engine.InvalidParams.With(fmt.Errorf("invalid start or count, start: %v count: %v", start, count))
@@ -768,19 +779,23 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 {
 	if block == nil {
 		return nil
 	}
+
 	var (
 		body        = block.Body()
 		txs         = make([]hexutil.Bytes, len(body.Transactions))
 		withdrawals = body.Withdrawals
 	)
+
 	for j, tx := range body.Transactions {
 		data, _ := tx.MarshalBinary()
 		txs[j] = hexutil.Bytes(data)
 	}
+
 	// Post-shanghai withdrawals MUST be set to empty slice instead of nil
 	if withdrawals == nil && block.Header().WithdrawalsHash != nil {
 		withdrawals = make([]*types.Withdrawal, 0)
 	}
+
 	return &engine.ExecutionPayloadBodyV1{
 		TransactionData: txs,
 		Withdrawals:     withdrawals,

eth/catalyst/api_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/txpool"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
@@ -322,7 +323,7 @@ func TestEth2NewBlock(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to create the executable data %v", err)
 		}
-		block, err := engine.ExecutableDataToBlock(*execData)
+		block, err := engine.ExecutableDataToBlock(*execData, nil)
 		if err != nil {
 			t.Fatalf("Failed to convert executable data to block %v", err)
 		}
@@ -364,7 +365,7 @@ func TestEth2NewBlock(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to create the executable data %v", err)
 		}
-		block, err := engine.ExecutableDataToBlock(*execData)
+		block, err := engine.ExecutableDataToBlock(*execData, nil)
 		if err != nil {
 			t.Fatalf("Failed to convert executable data to block %v", err)
 		}
@@ -996,7 +997,7 @@ func TestSimultaneousNewBlock(t *testing.T) {
 				t.Fatal(testErr)
 			}
 		}
-		block, err := engine.ExecutableDataToBlock(*execData)
+		block, err := engine.ExecutableDataToBlock(*execData, nil)
 		if err != nil {
 			t.Fatalf("Failed to convert executable data to block %v", err)
 		}
@@ -1519,3 +1520,38 @@ func equalBody(a *types.Body, b *engine.ExecutionPayloadBodyV1) bool {
 	}
 	return reflect.DeepEqual(a.Withdrawals, b.Withdrawals)
 }
+
+func TestBlockToPayloadWithBlobs(t *testing.T) {
+	header := types.Header{}
+	var txs []*types.Transaction
+
+	inner := types.BlobTx{
+		BlobHashes: make([]common.Hash, 1),
+	}
+
+	txs = append(txs, types.NewTx(&inner))
+
+	blobs := make([]kzg4844.Blob, 1)
+	commitments := make([]kzg4844.Commitment, 1)
+	proofs := make([]kzg4844.Proof, 1)
+
+	block := types.NewBlock(&header, txs, nil, nil, trie.NewStackTrie(nil))
+	envelope := engine.BlockToExecutableData(block, nil, blobs, commitments, proofs)
+	var want int
+	for _, tx := range txs {
+		want += len(tx.BlobHashes())
+	}
+	if got := len(envelope.BlobsBundle.Commitments); got != want {
+		t.Fatalf("invalid number of commitments: got %v, want %v", got, want)
+	}
+	if got := len(envelope.BlobsBundle.Proofs); got != want {
+		t.Fatalf("invalid number of proofs: got %v, want %v", got, want)
+	}
+	if got := len(envelope.BlobsBundle.Blobs); got != want {
+		t.Fatalf("invalid number of blobs: got %v, want %v", got, want)
+	}
+	_, err := engine.ExecutableDataToBlock(*envelope.ExecutionPayload, make([]common.Hash, 1))
+	if err != nil {
+		t.Error(err)
+	}
+}

eth/catalyst/simulated_beacon.go

@@ -151,7 +151,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error {
 		return fmt.Errorf("error calling forkchoice update: %v", err)
 	}
 
-	envelope, err := c.engineAPI.getFullPayload(*fcResponse.PayloadID)
+	envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID)
 	if err != nil {
 		return fmt.Errorf("error retrieving payload: %v", err)
 	}

miner/payload_building.go

@@ -120,9 +120,9 @@ func (payload *Payload) Resolve() *engine.ExecutionPayloadEnvelope {
 		close(payload.stop)
 	}
 	if payload.full != nil {
-		return engine.BlockToExecutableData(payload.full, payload.fullFees)
+		return engine.BlockToExecutableData(payload.full, payload.fullFees, nil, nil, nil)
 	}
-	return engine.BlockToExecutableData(payload.empty, big.NewInt(0))
+	return engine.BlockToExecutableData(payload.empty, big.NewInt(0), nil, nil, nil)
 }
 
 // ResolveEmpty is basically identical to Resolve, but it expects empty block only.
@@ -131,7 +131,7 @@ func (payload *Payload) ResolveEmpty() *engine.ExecutionPayloadEnvelope {
 	payload.lock.Lock()
 	defer payload.lock.Unlock()
 
-	return engine.BlockToExecutableData(payload.empty, big.NewInt(0))
+	return engine.BlockToExecutableData(payload.empty, big.NewInt(0), nil, nil, nil)
 }
 
 // ResolveFull is basically identical to Resolve, but it expects full block only.
@@ -157,7 +157,7 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope {
 	default:
 		close(payload.stop)
 	}
-	return engine.BlockToExecutableData(payload.full, payload.fullFees)
+	return engine.BlockToExecutableData(payload.full, payload.fullFees, nil, nil, nil)
 }
 
 // buildPayload builds the payload according to the provided parameters.