build, tests: add execution-spec-tests (#26985)

This makes it possible to run the execution-spec-tests (a.k.a. pyspec) in CI.

---------

Co-authored-by: Felix Lange <fjl@twurst.com>

build/checksums.txt

@@ -1,5 +1,9 @@
 # This file contains sha256 checksums of optional build dependencies.
 
+# https://github.com/ethereum/execution-spec-tests/releases
+24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81  fixtures.tar.gz
+
+# https://go.dev/dl/
 818d46ede85682dd551ad378ef37a4d247006f12ec59b5b755601d2ce114369a  go1.21.0.src.tar.gz
 b314de9f704ab122c077d2ec8e67e3670affe8865479d1f01991e7ac55d65e70  go1.21.0.darwin-amd64.tar.gz
 3aca44de55c5e098de2f406e98aba328898b05d509a2e2a356416faacf2c4566  go1.21.0.darwin-arm64.tar.gz
@@ -15,7 +19,7 @@ af920fbb74fc3d173118dc3cc35f02a709c1de642700e92a91a7d16981df3fec  go1.21.0.windo
 732121e64e0ecb07c77fdf6cc1bc5ce7b242c2d40d4ac29021ad4c64a08731f6  go1.21.0.windows-amd64.zip
 41342f5a0f8c083b14c68bde738ddcd313a4f53a5854bfdfab47f0e88247de12  go1.21.0.windows-arm64.zip
 
-
+# https://github.com/golangci/golangci-lint/releases
 fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc  golangci-lint-1.51.1-darwin-amd64.tar.gz
 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2  golangci-lint-1.51.1-darwin-arm64.tar.gz
 e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414  golangci-lint-1.51.1-freebsd-386.tar.gz

build/ci.go

@@ -148,6 +148,13 @@ var (
 	// we need to switch over to a recursive builder to jumpt across supported
 	// versions.
 	gobootVersion = "1.19.6"
+
+	// This is the version of execution-spec-tests that we are using.
+	// When updating, you must also update build/checksums.txt.
+	executionSpecTestsVersion = "1.0.2"
+
+	// This is where the tests should be unpacked.
+	executionSpecTestsDir = "tests/spec-tests"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -294,13 +301,17 @@ func doTest(cmdline []string) {
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 		verbose  = flag.Bool("v", false, "Whether to log verbosely")
 		race     = flag.Bool("race", false, "Execute the race detector")
+		cachedir = flag.String("cachedir", "./build/cache", "directory for caching downloads")
 	)
 	flag.CommandLine.Parse(cmdline)
 
+	// Get test fixtures.
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	downloadSpecTestFixtures(csdb, *cachedir)
+
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		csdb := build.MustLoadChecksums("build/checksums.txt")
 		tc.Root = build.DownloadGo(csdb, dlgoVersion)
 	}
 	gotest := tc.Go("test")
@@ -332,6 +343,21 @@ func doTest(cmdline []string) {
 	build.MustRun(gotest)
 }
 
+// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
+func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+	ext := ".tar.gz"
+	base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
+	url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext)
+	archivePath := filepath.Join(cachedir, base+ext)
+	if err := csdb.DownloadFile(url, archivePath); err != nil {
+		log.Fatal(err)
+	}
+	if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil {
+		log.Fatal(err)
+	}
+	return filepath.Join(cachedir, base)
+}
+
 // doLint runs golangci-lint on requested packages.
 func doLint(cmdline []string) {
 	var (

tests/block_test.go

@@ -19,12 +19,11 @@ package tests
 import (
 	"testing"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 )
 
 func TestBlockchain(t *testing.T) {
-	t.Parallel()
-
 	bt := new(testMatcher)
 	// General state tests are 'exported' as blockchain tests, but we can run them natively.
 	// For speedier CI-runs, the line below can be uncommented, so those are skipped.
@@ -50,6 +49,30 @@ func TestBlockchain(t *testing.T) {
 	bt.skipLoad(`.*randomStatetest94.json.*`)
 
 	bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
+		execBlockTest(t, bt, test)
+	})
+	// There is also a LegacyTests folder, containing blockchain tests generated
+	// prior to Istanbul. However, they are all derived from GeneralStateTests,
+	// which run natively, so there's no reason to run them here.
+}
+
+// TestExecutionSpec runs the test fixtures from execution-spec-tests.
+func TestExecutionSpec(t *testing.T) {
+	if !common.FileExist(executionSpecDir) {
+		t.Skipf("directory %s does not exist", executionSpecDir)
+	}
+	bt := new(testMatcher)
+
+	// cancun tests are not complete yet
+	bt.skipLoad(`^cancun/`)
+	bt.skipLoad(`-fork=Cancun`)
+
+	bt.walk(t, executionSpecDir, func(t *testing.T, name string, test *BlockTest) {
+		execBlockTest(t, bt, test)
+	})
+}
+
+func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) {
 	if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil {
 		t.Errorf("test in hash mode without snapshotter failed: %v", err)
 	}
@@ -62,8 +85,4 @@ func TestBlockchain(t *testing.T) {
 	if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil {
 		t.Errorf("test in path mode with snapshotter failed: %v", err)
 	}
-	})
-	// There is also a LegacyTests folder, containing blockchain tests generated
-	// prior to Istanbul. However, they are all derived from GeneralStateTests,
-	// which run natively, so there's no reason to run them here.
 }

tests/block_test_util.go

@@ -91,6 +91,9 @@ type btHeader struct {
 	Timestamp             uint64
 	BaseFeePerGas         *big.Int
 	WithdrawalsRoot       *common.Hash
+	BlobGasUsed           *uint64
+	ExcessBlobGas         *uint64
+	ParentBeaconBlockRoot *common.Hash
 }
 
 type btHeaderMarshaling struct {
@@ -101,6 +104,8 @@ type btHeaderMarshaling struct {
 	GasUsed       math.HexOrDecimal64
 	Timestamp     math.HexOrDecimal64
 	BaseFeePerGas *math.HexOrDecimal256
+	BlobGasUsed   *math.HexOrDecimal64
+	ExcessBlobGas *math.HexOrDecimal64
 }
 
 func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error {
@@ -187,6 +192,8 @@ func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis {
 		Coinbase:      t.json.Genesis.Coinbase,
 		Alloc:         t.json.Pre,
 		BaseFee:       t.json.Genesis.BaseFeePerGas,
+		BlobGasUsed:   t.json.Genesis.BlobGasUsed,
+		ExcessBlobGas: t.json.Genesis.ExcessBlobGas,
 	}
 }
 
@@ -295,6 +302,15 @@ func validateHeader(h *btHeader, h2 *types.Header) error {
 	if !reflect.DeepEqual(h.WithdrawalsRoot, h2.WithdrawalsHash) {
 		return fmt.Errorf("withdrawalsRoot: want: %v have: %v", h.WithdrawalsRoot, h2.WithdrawalsHash)
 	}
+	if !reflect.DeepEqual(h.BlobGasUsed, h2.BlobGasUsed) {
+		return fmt.Errorf("blobGasUsed: want: %v have: %v", h.BlobGasUsed, h2.BlobGasUsed)
+	}
+	if !reflect.DeepEqual(h.ExcessBlobGas, h2.ExcessBlobGas) {
+		return fmt.Errorf("excessBlobGas: want: %v have: %v", h.ExcessBlobGas, h2.ExcessBlobGas)
+	}
+	if !reflect.DeepEqual(h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) {
+		return fmt.Errorf("parentBeaconBlockRoot: want: %v have: %v", h.ParentBeaconBlockRoot, h2.ParentBeaconRoot)
+	}
 	return nil
 }
 

tests/gen_btheader.go

@@ -35,6 +35,9 @@ func (b btHeader) MarshalJSON() ([]byte, error) {
 		Timestamp             math.HexOrDecimal64
 		BaseFeePerGas         *math.HexOrDecimal256
 		WithdrawalsRoot       *common.Hash
+		BlobGasUsed           *math.HexOrDecimal64
+		ExcessBlobGas         *math.HexOrDecimal64
+		ParentBeaconBlockRoot *common.Hash
 	}
 	var enc btHeader
 	enc.Bloom = b.Bloom
@@ -55,6 +58,9 @@ func (b btHeader) MarshalJSON() ([]byte, error) {
 	enc.Timestamp = math.HexOrDecimal64(b.Timestamp)
 	enc.BaseFeePerGas = (*math.HexOrDecimal256)(b.BaseFeePerGas)
 	enc.WithdrawalsRoot = b.WithdrawalsRoot
+	enc.BlobGasUsed = (*math.HexOrDecimal64)(b.BlobGasUsed)
+	enc.ExcessBlobGas = (*math.HexOrDecimal64)(b.ExcessBlobGas)
+	enc.ParentBeaconBlockRoot = b.ParentBeaconBlockRoot
 	return json.Marshal(&enc)
 }
 
@@ -79,6 +85,9 @@ func (b *btHeader) UnmarshalJSON(input []byte) error {
 		Timestamp             *math.HexOrDecimal64
 		BaseFeePerGas         *math.HexOrDecimal256
 		WithdrawalsRoot       *common.Hash
+		BlobGasUsed           *math.HexOrDecimal64
+		ExcessBlobGas         *math.HexOrDecimal64
+		ParentBeaconBlockRoot *common.Hash
 	}
 	var dec btHeader
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -138,5 +147,14 @@ func (b *btHeader) UnmarshalJSON(input []byte) error {
 	if dec.WithdrawalsRoot != nil {
 		b.WithdrawalsRoot = dec.WithdrawalsRoot
 	}
+	if dec.BlobGasUsed != nil {
+		b.BlobGasUsed = (*uint64)(dec.BlobGasUsed)
+	}
+	if dec.ExcessBlobGas != nil {
+		b.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas)
+	}
+	if dec.ParentBeaconBlockRoot != nil {
+		b.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot
+	}
 	return nil
 }

tests/init.go

@@ -318,6 +318,25 @@ var Forks = map[string]*params.ChainConfig{
 		ShanghaiTime:            u64(0),
 		CancunTime:              u64(0),
 	},
+	"ShanghaiToCancunAtTime15k": {
+		ChainID:                 big.NewInt(1),
+		HomesteadBlock:          big.NewInt(0),
+		EIP150Block:             big.NewInt(0),
+		EIP155Block:             big.NewInt(0),
+		EIP158Block:             big.NewInt(0),
+		ByzantiumBlock:          big.NewInt(0),
+		ConstantinopleBlock:     big.NewInt(0),
+		PetersburgBlock:         big.NewInt(0),
+		IstanbulBlock:           big.NewInt(0),
+		MuirGlacierBlock:        big.NewInt(0),
+		BerlinBlock:             big.NewInt(0),
+		LondonBlock:             big.NewInt(0),
+		ArrowGlacierBlock:       big.NewInt(0),
+		MergeNetsplitBlock:      big.NewInt(0),
+		TerminalTotalDifficulty: big.NewInt(0),
+		ShanghaiTime:            u64(0),
+		CancunTime:              u64(15_000),
+	},
 }
 
 // AvailableForks returns the set of defined fork names

tests/init_test.go

@@ -41,6 +41,7 @@ var (
 	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
 	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
 	difficultyTestDir  = filepath.Join(baseDir, "BasicTests")
+	executionSpecDir   = filepath.Join(".", "spec-tests", "fixtures")
 	benchmarksDir      = filepath.Join(".", "evm-benchmarks", "benchmarks")
 )