build, tests: add execution-spec-tests (#26985)

This makes it possible to run the execution-spec-tests (a.k.a. pyspec) in CI.

---------

Co-authored-by: Felix Lange <fjl@twurst.com>

build/checksums.txt

@@ -1,5 +1,9 @@
 # This file contains sha256 checksums of optional build dependencies.
 
+# https://github.com/ethereum/execution-spec-tests/releases
+24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81  fixtures.tar.gz
+
+# https://go.dev/dl/
 818d46ede85682dd551ad378ef37a4d247006f12ec59b5b755601d2ce114369a  go1.21.0.src.tar.gz
 b314de9f704ab122c077d2ec8e67e3670affe8865479d1f01991e7ac55d65e70  go1.21.0.darwin-amd64.tar.gz
 3aca44de55c5e098de2f406e98aba328898b05d509a2e2a356416faacf2c4566  go1.21.0.darwin-arm64.tar.gz
@@ -15,7 +19,7 @@ af920fbb74fc3d173118dc3cc35f02a709c1de642700e92a91a7d16981df3fec  go1.21.0.windo
 732121e64e0ecb07c77fdf6cc1bc5ce7b242c2d40d4ac29021ad4c64a08731f6  go1.21.0.windows-amd64.zip
 41342f5a0f8c083b14c68bde738ddcd313a4f53a5854bfdfab47f0e88247de12  go1.21.0.windows-arm64.zip
 
-
+# https://github.com/golangci/golangci-lint/releases
 fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc  golangci-lint-1.51.1-darwin-amd64.tar.gz
 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2  golangci-lint-1.51.1-darwin-arm64.tar.gz
 e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414  golangci-lint-1.51.1-freebsd-386.tar.gz

build/ci.go

@@ -123,12 +123,12 @@ var (
 	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish,
 	//   kinetic
 	debDistroGoBoots = map[string]string{
-		"trusty":  "golang-1.11", // EOL: 04/2024
+		"trusty": "golang-1.11", // EOL: 04/2024
-		"xenial":  "golang-go",   // EOL: 04/2026
+		"xenial": "golang-go",   // EOL: 04/2026
-		"bionic":  "golang-go",   // EOL: 04/2028
+		"bionic": "golang-go",   // EOL: 04/2028
-		"focal":   "golang-go",   // EOL: 04/2030
+		"focal":  "golang-go",   // EOL: 04/2030
-		"jammy":   "golang-go",   // EOL: 04/2032
+		"jammy":  "golang-go",   // EOL: 04/2032
-		"lunar":   "golang-go",   // EOL: 01/2024
+		"lunar":  "golang-go",   // EOL: 01/2024
 	}
 
 	debGoBootPaths = map[string]string{
@@ -148,6 +148,13 @@ var (
 	// we need to switch over to a recursive builder to jumpt across supported
 	// versions.
 	gobootVersion = "1.19.6"
+
+	// This is the version of execution-spec-tests that we are using.
+	// When updating, you must also update build/checksums.txt.
+	executionSpecTestsVersion = "1.0.2"
+
+	// This is where the tests should be unpacked.
+	executionSpecTestsDir = "tests/spec-tests"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -294,13 +301,17 @@ func doTest(cmdline []string) {
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 		verbose  = flag.Bool("v", false, "Whether to log verbosely")
 		race     = flag.Bool("race", false, "Execute the race detector")
+		cachedir = flag.String("cachedir", "./build/cache", "directory for caching downloads")
 	)
 	flag.CommandLine.Parse(cmdline)
 
+	// Get test fixtures.
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	downloadSpecTestFixtures(csdb, *cachedir)
+
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		csdb := build.MustLoadChecksums("build/checksums.txt")
 		tc.Root = build.DownloadGo(csdb, dlgoVersion)
 	}
 	gotest := tc.Go("test")
@@ -332,6 +343,21 @@ func doTest(cmdline []string) {
 	build.MustRun(gotest)
 }
 
+// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
+func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+	ext := ".tar.gz"
+	base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
+	url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext)
+	archivePath := filepath.Join(cachedir, base+ext)
+	if err := csdb.DownloadFile(url, archivePath); err != nil {
+		log.Fatal(err)
+	}
+	if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil {
+		log.Fatal(err)
+	}
+	return filepath.Join(cachedir, base)
+}
+
 // doLint runs golangci-lint on requested packages.
 func doLint(cmdline []string) {
 	var (

tests/block_test.go

@@ -19,12 +19,11 @@ package tests
 import (
 	"testing"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 )
 
 func TestBlockchain(t *testing.T) {
-	t.Parallel()
-
 	bt := new(testMatcher)
 	// General state tests are 'exported' as blockchain tests, but we can run them natively.
 	// For speedier CI-runs, the line below can be uncommented, so those are skipped.
@@ -50,20 +49,40 @@ func TestBlockchain(t *testing.T) {
 	bt.skipLoad(`.*randomStatetest94.json.*`)
 
 	bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
-		if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil {
+		execBlockTest(t, bt, test)
-			t.Errorf("test in hash mode without snapshotter failed: %v", err)
-		}
-		if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil {
-			t.Errorf("test in hash mode with snapshotter failed: %v", err)
-		}
-		if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil {
-			t.Errorf("test in path mode without snapshotter failed: %v", err)
-		}
-		if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil {
-			t.Errorf("test in path mode with snapshotter failed: %v", err)
-		}
 	})
 	// There is also a LegacyTests folder, containing blockchain tests generated
 	// prior to Istanbul. However, they are all derived from GeneralStateTests,
 	// which run natively, so there's no reason to run them here.
 }
+
+// TestExecutionSpec runs the test fixtures from execution-spec-tests.
+func TestExecutionSpec(t *testing.T) {
+	if !common.FileExist(executionSpecDir) {
+		t.Skipf("directory %s does not exist", executionSpecDir)
+	}
+	bt := new(testMatcher)
+
+	// cancun tests are not complete yet
+	bt.skipLoad(`^cancun/`)
+	bt.skipLoad(`-fork=Cancun`)
+
+	bt.walk(t, executionSpecDir, func(t *testing.T, name string, test *BlockTest) {
+		execBlockTest(t, bt, test)
+	})
+}
+
+func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) {
+	if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil {
+		t.Errorf("test in hash mode without snapshotter failed: %v", err)
+	}
+	if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil {
+		t.Errorf("test in hash mode with snapshotter failed: %v", err)
+	}
+	if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil {
+		t.Errorf("test in path mode without snapshotter failed: %v", err)
+	}
+	if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil {
+		t.Errorf("test in path mode with snapshotter failed: %v", err)
+	}
+}

tests/block_test_util.go

@@ -73,24 +73,27 @@ type btBlock struct {
 //go:generate go run github.com/fjl/gencodec -type btHeader -field-override btHeaderMarshaling -out gen_btheader.go
 
 type btHeader struct {
-	Bloom            types.Bloom
+	Bloom                 types.Bloom
-	Coinbase         common.Address
+	Coinbase              common.Address
-	MixHash          common.Hash
+	MixHash               common.Hash
-	Nonce            types.BlockNonce
+	Nonce                 types.BlockNonce
-	Number           *big.Int
+	Number                *big.Int
-	Hash             common.Hash
+	Hash                  common.Hash
-	ParentHash       common.Hash
+	ParentHash            common.Hash
-	ReceiptTrie      common.Hash
+	ReceiptTrie           common.Hash
-	StateRoot        common.Hash
+	StateRoot             common.Hash
-	TransactionsTrie common.Hash
+	TransactionsTrie      common.Hash
-	UncleHash        common.Hash
+	UncleHash             common.Hash
-	ExtraData        []byte
+	ExtraData             []byte
-	Difficulty       *big.Int
+	Difficulty            *big.Int
-	GasLimit         uint64
+	GasLimit              uint64
-	GasUsed          uint64
+	GasUsed               uint64
-	Timestamp        uint64
+	Timestamp             uint64
-	BaseFeePerGas    *big.Int
+	BaseFeePerGas         *big.Int
-	WithdrawalsRoot  *common.Hash
+	WithdrawalsRoot       *common.Hash
+	BlobGasUsed           *uint64
+	ExcessBlobGas         *uint64
+	ParentBeaconBlockRoot *common.Hash
 }
 
 type btHeaderMarshaling struct {
@@ -101,6 +104,8 @@ type btHeaderMarshaling struct {
 	GasUsed       math.HexOrDecimal64
 	Timestamp     math.HexOrDecimal64
 	BaseFeePerGas *math.HexOrDecimal256
+	BlobGasUsed   *math.HexOrDecimal64
+	ExcessBlobGas *math.HexOrDecimal64
 }
 
 func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error {
@@ -175,18 +180,20 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) er
 
 func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis {
 	return &core.Genesis{
-		Config:     config,
+		Config:        config,
-		Nonce:      t.json.Genesis.Nonce.Uint64(),
+		Nonce:         t.json.Genesis.Nonce.Uint64(),
-		Timestamp:  t.json.Genesis.Timestamp,
+		Timestamp:     t.json.Genesis.Timestamp,
-		ParentHash: t.json.Genesis.ParentHash,
+		ParentHash:    t.json.Genesis.ParentHash,
-		ExtraData:  t.json.Genesis.ExtraData,
+		ExtraData:     t.json.Genesis.ExtraData,
-		GasLimit:   t.json.Genesis.GasLimit,
+		GasLimit:      t.json.Genesis.GasLimit,
-		GasUsed:    t.json.Genesis.GasUsed,
+		GasUsed:       t.json.Genesis.GasUsed,
-		Difficulty: t.json.Genesis.Difficulty,
+		Difficulty:    t.json.Genesis.Difficulty,
-		Mixhash:    t.json.Genesis.MixHash,
+		Mixhash:       t.json.Genesis.MixHash,
-		Coinbase:   t.json.Genesis.Coinbase,
+		Coinbase:      t.json.Genesis.Coinbase,
-		Alloc:      t.json.Pre,
+		Alloc:         t.json.Pre,
-		BaseFee:    t.json.Genesis.BaseFeePerGas,
+		BaseFee:       t.json.Genesis.BaseFeePerGas,
+		BlobGasUsed:   t.json.Genesis.BlobGasUsed,
+		ExcessBlobGas: t.json.Genesis.ExcessBlobGas,
 	}
 }
 
@@ -295,6 +302,15 @@ func validateHeader(h *btHeader, h2 *types.Header) error {
 	if !reflect.DeepEqual(h.WithdrawalsRoot, h2.WithdrawalsHash) {
 		return fmt.Errorf("withdrawalsRoot: want: %v have: %v", h.WithdrawalsRoot, h2.WithdrawalsHash)
 	}
+	if !reflect.DeepEqual(h.BlobGasUsed, h2.BlobGasUsed) {
+		return fmt.Errorf("blobGasUsed: want: %v have: %v", h.BlobGasUsed, h2.BlobGasUsed)
+	}
+	if !reflect.DeepEqual(h.ExcessBlobGas, h2.ExcessBlobGas) {
+		return fmt.Errorf("excessBlobGas: want: %v have: %v", h.ExcessBlobGas, h2.ExcessBlobGas)
+	}
+	if !reflect.DeepEqual(h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) {
+		return fmt.Errorf("parentBeaconBlockRoot: want: %v have: %v", h.ParentBeaconBlockRoot, h2.ParentBeaconRoot)
+	}
 	return nil
 }
 

tests/gen_btheader.go

@@ -17,24 +17,27 @@ var _ = (*btHeaderMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (b btHeader) MarshalJSON() ([]byte, error) {
 	type btHeader struct {
-		Bloom            types.Bloom
+		Bloom                 types.Bloom
-		Coinbase         common.Address
+		Coinbase              common.Address
-		MixHash          common.Hash
+		MixHash               common.Hash
-		Nonce            types.BlockNonce
+		Nonce                 types.BlockNonce
-		Number           *math.HexOrDecimal256
+		Number                *math.HexOrDecimal256
-		Hash             common.Hash
+		Hash                  common.Hash
-		ParentHash       common.Hash
+		ParentHash            common.Hash
-		ReceiptTrie      common.Hash
+		ReceiptTrie           common.Hash
-		StateRoot        common.Hash
+		StateRoot             common.Hash
-		TransactionsTrie common.Hash
+		TransactionsTrie      common.Hash
-		UncleHash        common.Hash
+		UncleHash             common.Hash
-		ExtraData        hexutil.Bytes
+		ExtraData             hexutil.Bytes
-		Difficulty       *math.HexOrDecimal256
+		Difficulty            *math.HexOrDecimal256
-		GasLimit         math.HexOrDecimal64
+		GasLimit              math.HexOrDecimal64
-		GasUsed          math.HexOrDecimal64
+		GasUsed               math.HexOrDecimal64
-		Timestamp        math.HexOrDecimal64
+		Timestamp             math.HexOrDecimal64
-		BaseFeePerGas    *math.HexOrDecimal256
+		BaseFeePerGas         *math.HexOrDecimal256
-		WithdrawalsRoot  *common.Hash
+		WithdrawalsRoot       *common.Hash
+		BlobGasUsed           *math.HexOrDecimal64
+		ExcessBlobGas         *math.HexOrDecimal64
+		ParentBeaconBlockRoot *common.Hash
 	}
 	var enc btHeader
 	enc.Bloom = b.Bloom
@@ -55,30 +58,36 @@ func (b btHeader) MarshalJSON() ([]byte, error) {
 	enc.Timestamp = math.HexOrDecimal64(b.Timestamp)
 	enc.BaseFeePerGas = (*math.HexOrDecimal256)(b.BaseFeePerGas)
 	enc.WithdrawalsRoot = b.WithdrawalsRoot
+	enc.BlobGasUsed = (*math.HexOrDecimal64)(b.BlobGasUsed)
+	enc.ExcessBlobGas = (*math.HexOrDecimal64)(b.ExcessBlobGas)
+	enc.ParentBeaconBlockRoot = b.ParentBeaconBlockRoot
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
 func (b *btHeader) UnmarshalJSON(input []byte) error {
 	type btHeader struct {
-		Bloom            *types.Bloom
+		Bloom                 *types.Bloom
-		Coinbase         *common.Address
+		Coinbase              *common.Address
-		MixHash          *common.Hash
+		MixHash               *common.Hash
-		Nonce            *types.BlockNonce
+		Nonce                 *types.BlockNonce
-		Number           *math.HexOrDecimal256
+		Number                *math.HexOrDecimal256
-		Hash             *common.Hash
+		Hash                  *common.Hash
-		ParentHash       *common.Hash
+		ParentHash            *common.Hash
-		ReceiptTrie      *common.Hash
+		ReceiptTrie           *common.Hash
-		StateRoot        *common.Hash
+		StateRoot             *common.Hash
-		TransactionsTrie *common.Hash
+		TransactionsTrie      *common.Hash
-		UncleHash        *common.Hash
+		UncleHash             *common.Hash
-		ExtraData        *hexutil.Bytes
+		ExtraData             *hexutil.Bytes
-		Difficulty       *math.HexOrDecimal256
+		Difficulty            *math.HexOrDecimal256
-		GasLimit         *math.HexOrDecimal64
+		GasLimit              *math.HexOrDecimal64
-		GasUsed          *math.HexOrDecimal64
+		GasUsed               *math.HexOrDecimal64
-		Timestamp        *math.HexOrDecimal64
+		Timestamp             *math.HexOrDecimal64
-		BaseFeePerGas    *math.HexOrDecimal256
+		BaseFeePerGas         *math.HexOrDecimal256
-		WithdrawalsRoot  *common.Hash
+		WithdrawalsRoot       *common.Hash
+		BlobGasUsed           *math.HexOrDecimal64
+		ExcessBlobGas         *math.HexOrDecimal64
+		ParentBeaconBlockRoot *common.Hash
 	}
 	var dec btHeader
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -138,5 +147,14 @@ func (b *btHeader) UnmarshalJSON(input []byte) error {
 	if dec.WithdrawalsRoot != nil {
 		b.WithdrawalsRoot = dec.WithdrawalsRoot
 	}
+	if dec.BlobGasUsed != nil {
+		b.BlobGasUsed = (*uint64)(dec.BlobGasUsed)
+	}
+	if dec.ExcessBlobGas != nil {
+		b.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas)
+	}
+	if dec.ParentBeaconBlockRoot != nil {
+		b.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot
+	}
 	return nil
 }

tests/init.go

@@ -318,6 +318,25 @@ var Forks = map[string]*params.ChainConfig{
 		ShanghaiTime:            u64(0),
 		CancunTime:              u64(0),
 	},
+	"ShanghaiToCancunAtTime15k": {
+		ChainID:                 big.NewInt(1),
+		HomesteadBlock:          big.NewInt(0),
+		EIP150Block:             big.NewInt(0),
+		EIP155Block:             big.NewInt(0),
+		EIP158Block:             big.NewInt(0),
+		ByzantiumBlock:          big.NewInt(0),
+		ConstantinopleBlock:     big.NewInt(0),
+		PetersburgBlock:         big.NewInt(0),
+		IstanbulBlock:           big.NewInt(0),
+		MuirGlacierBlock:        big.NewInt(0),
+		BerlinBlock:             big.NewInt(0),
+		LondonBlock:             big.NewInt(0),
+		ArrowGlacierBlock:       big.NewInt(0),
+		MergeNetsplitBlock:      big.NewInt(0),
+		TerminalTotalDifficulty: big.NewInt(0),
+		ShanghaiTime:            u64(0),
+		CancunTime:              u64(15_000),
+	},
 }
 
 // AvailableForks returns the set of defined fork names

tests/init_test.go

@@ -41,6 +41,7 @@ var (
 	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
 	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
 	difficultyTestDir  = filepath.Join(baseDir, "BasicTests")
+	executionSpecDir   = filepath.Join(".", "spec-tests", "fixtures")
 	benchmarksDir      = filepath.Join(".", "evm-benchmarks", "benchmarks")
 )