From f174ddba7af5c150ecad37430c167194d66d75f1 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 26 Aug 2023 15:42:27 +0200 Subject: [PATCH] build, tests: add execution-spec-tests (#26985) This makes it possible to run the execution-spec-tests (a.k.a. pyspec) in CI. --------- Co-authored-by: Felix Lange --- build/checksums.txt | 6 ++- build/ci.go | 40 ++++++++++++++---- tests/block_test.go | 47 ++++++++++++++------- tests/block_test_util.go | 76 +++++++++++++++++++-------------- tests/gen_btheader.go | 90 ++++++++++++++++++++++++---------------- tests/init.go | 19 +++++++++ tests/init_test.go | 1 + 7 files changed, 191 insertions(+), 88 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index ea92437e87..adfa0bc34d 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,5 +1,9 @@ # This file contains sha256 checksums of optional build dependencies. +# https://github.com/ethereum/execution-spec-tests/releases +24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81 fixtures.tar.gz + +# https://go.dev/dl/ 818d46ede85682dd551ad378ef37a4d247006f12ec59b5b755601d2ce114369a go1.21.0.src.tar.gz b314de9f704ab122c077d2ec8e67e3670affe8865479d1f01991e7ac55d65e70 go1.21.0.darwin-amd64.tar.gz 3aca44de55c5e098de2f406e98aba328898b05d509a2e2a356416faacf2c4566 go1.21.0.darwin-arm64.tar.gz @@ -15,7 +19,7 @@ af920fbb74fc3d173118dc3cc35f02a709c1de642700e92a91a7d16981df3fec go1.21.0.windo 732121e64e0ecb07c77fdf6cc1bc5ce7b242c2d40d4ac29021ad4c64a08731f6 go1.21.0.windows-amd64.zip 41342f5a0f8c083b14c68bde738ddcd313a4f53a5854bfdfab47f0e88247de12 go1.21.0.windows-arm64.zip - +# https://github.com/golangci/golangci-lint/releases fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc golangci-lint-1.51.1-darwin-amd64.tar.gz 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2 golangci-lint-1.51.1-darwin-arm64.tar.gz e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414 golangci-lint-1.51.1-freebsd-386.tar.gz diff --git a/build/ci.go b/build/ci.go index 8b2b8fa9c1..6f77e5df60 100644 --- a/build/ci.go +++ b/build/ci.go @@ -123,12 +123,12 @@ var ( // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish, // kinetic debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", // EOL: 04/2024 - "xenial": "golang-go", // EOL: 04/2026 - "bionic": "golang-go", // EOL: 04/2028 - "focal": "golang-go", // EOL: 04/2030 - "jammy": "golang-go", // EOL: 04/2032 - "lunar": "golang-go", // EOL: 01/2024 + "trusty": "golang-1.11", // EOL: 04/2024 + "xenial": "golang-go", // EOL: 04/2026 + "bionic": "golang-go", // EOL: 04/2028 + "focal": "golang-go", // EOL: 04/2030 + "jammy": "golang-go", // EOL: 04/2032 + "lunar": "golang-go", // EOL: 01/2024 } debGoBootPaths = map[string]string{ @@ -148,6 +148,13 @@ var ( // we need to switch over to a recursive builder to jumpt across supported // versions. gobootVersion = "1.19.6" + + // This is the version of execution-spec-tests that we are using. + // When updating, you must also update build/checksums.txt. + executionSpecTestsVersion = "1.0.2" + + // This is where the tests should be unpacked. + executionSpecTestsDir = "tests/spec-tests" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -294,13 +301,17 @@ func doTest(cmdline []string) { coverage = flag.Bool("coverage", false, "Whether to record code coverage") verbose = flag.Bool("v", false, "Whether to log verbosely") race = flag.Bool("race", false, "Execute the race detector") + cachedir = flag.String("cachedir", "./build/cache", "directory for caching downloads") ) flag.CommandLine.Parse(cmdline) + // Get test fixtures. + csdb := build.MustLoadChecksums("build/checksums.txt") + downloadSpecTestFixtures(csdb, *cachedir) + // Configure the toolchain. tc := build.GoToolchain{GOARCH: *arch, CC: *cc} if *dlgo { - csdb := build.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb, dlgoVersion) } gotest := tc.Go("test") @@ -332,6 +343,21 @@ func doTest(cmdline []string) { build.MustRun(gotest) } +// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures. +func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string { + ext := ".tar.gz" + base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename + url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext) + archivePath := filepath.Join(cachedir, base+ext) + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) + } + if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil { + log.Fatal(err) + } + return filepath.Join(cachedir, base) +} + // doLint runs golangci-lint on requested packages. func doLint(cmdline []string) { var ( diff --git a/tests/block_test.go b/tests/block_test.go index ad522ad653..645b4b702d 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -19,12 +19,11 @@ package tests import ( "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" ) func TestBlockchain(t *testing.T) { - t.Parallel() - bt := new(testMatcher) // General state tests are 'exported' as blockchain tests, but we can run them natively. // For speedier CI-runs, the line below can be uncommented, so those are skipped. @@ -50,20 +49,40 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil { - t.Errorf("test in hash mode without snapshotter failed: %v", err) - } - if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil { - t.Errorf("test in hash mode with snapshotter failed: %v", err) - } - if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil { - t.Errorf("test in path mode without snapshotter failed: %v", err) - } - if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil { - t.Errorf("test in path mode with snapshotter failed: %v", err) - } + execBlockTest(t, bt, test) }) // There is also a LegacyTests folder, containing blockchain tests generated // prior to Istanbul. However, they are all derived from GeneralStateTests, // which run natively, so there's no reason to run them here. } + +// TestExecutionSpec runs the test fixtures from execution-spec-tests. +func TestExecutionSpec(t *testing.T) { + if !common.FileExist(executionSpecDir) { + t.Skipf("directory %s does not exist", executionSpecDir) + } + bt := new(testMatcher) + + // cancun tests are not complete yet + bt.skipLoad(`^cancun/`) + bt.skipLoad(`-fork=Cancun`) + + bt.walk(t, executionSpecDir, func(t *testing.T, name string, test *BlockTest) { + execBlockTest(t, bt, test) + }) +} + +func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) { + if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil { + t.Errorf("test in hash mode without snapshotter failed: %v", err) + } + if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil { + t.Errorf("test in hash mode with snapshotter failed: %v", err) + } + if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil { + t.Errorf("test in path mode without snapshotter failed: %v", err) + } + if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil { + t.Errorf("test in path mode with snapshotter failed: %v", err) + } +} diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 712174884b..ad1d34fb2b 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -73,24 +73,27 @@ type btBlock struct { //go:generate go run github.com/fjl/gencodec -type btHeader -field-override btHeaderMarshaling -out gen_btheader.go type btHeader struct { - Bloom types.Bloom - Coinbase common.Address - MixHash common.Hash - Nonce types.BlockNonce - Number *big.Int - Hash common.Hash - ParentHash common.Hash - ReceiptTrie common.Hash - StateRoot common.Hash - TransactionsTrie common.Hash - UncleHash common.Hash - ExtraData []byte - Difficulty *big.Int - GasLimit uint64 - GasUsed uint64 - Timestamp uint64 - BaseFeePerGas *big.Int - WithdrawalsRoot *common.Hash + Bloom types.Bloom + Coinbase common.Address + MixHash common.Hash + Nonce types.BlockNonce + Number *big.Int + Hash common.Hash + ParentHash common.Hash + ReceiptTrie common.Hash + StateRoot common.Hash + TransactionsTrie common.Hash + UncleHash common.Hash + ExtraData []byte + Difficulty *big.Int + GasLimit uint64 + GasUsed uint64 + Timestamp uint64 + BaseFeePerGas *big.Int + WithdrawalsRoot *common.Hash + BlobGasUsed *uint64 + ExcessBlobGas *uint64 + ParentBeaconBlockRoot *common.Hash } type btHeaderMarshaling struct { @@ -101,6 +104,8 @@ type btHeaderMarshaling struct { GasUsed math.HexOrDecimal64 Timestamp math.HexOrDecimal64 BaseFeePerGas *math.HexOrDecimal256 + BlobGasUsed *math.HexOrDecimal64 + ExcessBlobGas *math.HexOrDecimal64 } func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error { @@ -175,18 +180,20 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) er func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis { return &core.Genesis{ - Config: config, - Nonce: t.json.Genesis.Nonce.Uint64(), - Timestamp: t.json.Genesis.Timestamp, - ParentHash: t.json.Genesis.ParentHash, - ExtraData: t.json.Genesis.ExtraData, - GasLimit: t.json.Genesis.GasLimit, - GasUsed: t.json.Genesis.GasUsed, - Difficulty: t.json.Genesis.Difficulty, - Mixhash: t.json.Genesis.MixHash, - Coinbase: t.json.Genesis.Coinbase, - Alloc: t.json.Pre, - BaseFee: t.json.Genesis.BaseFeePerGas, + Config: config, + Nonce: t.json.Genesis.Nonce.Uint64(), + Timestamp: t.json.Genesis.Timestamp, + ParentHash: t.json.Genesis.ParentHash, + ExtraData: t.json.Genesis.ExtraData, + GasLimit: t.json.Genesis.GasLimit, + GasUsed: t.json.Genesis.GasUsed, + Difficulty: t.json.Genesis.Difficulty, + Mixhash: t.json.Genesis.MixHash, + Coinbase: t.json.Genesis.Coinbase, + Alloc: t.json.Pre, + BaseFee: t.json.Genesis.BaseFeePerGas, + BlobGasUsed: t.json.Genesis.BlobGasUsed, + ExcessBlobGas: t.json.Genesis.ExcessBlobGas, } } @@ -295,6 +302,15 @@ func validateHeader(h *btHeader, h2 *types.Header) error { if !reflect.DeepEqual(h.WithdrawalsRoot, h2.WithdrawalsHash) { return fmt.Errorf("withdrawalsRoot: want: %v have: %v", h.WithdrawalsRoot, h2.WithdrawalsHash) } + if !reflect.DeepEqual(h.BlobGasUsed, h2.BlobGasUsed) { + return fmt.Errorf("blobGasUsed: want: %v have: %v", h.BlobGasUsed, h2.BlobGasUsed) + } + if !reflect.DeepEqual(h.ExcessBlobGas, h2.ExcessBlobGas) { + return fmt.Errorf("excessBlobGas: want: %v have: %v", h.ExcessBlobGas, h2.ExcessBlobGas) + } + if !reflect.DeepEqual(h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) { + return fmt.Errorf("parentBeaconBlockRoot: want: %v have: %v", h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) + } return nil } diff --git a/tests/gen_btheader.go b/tests/gen_btheader.go index 985ea692d7..80ad89e03b 100644 --- a/tests/gen_btheader.go +++ b/tests/gen_btheader.go @@ -17,24 +17,27 @@ var _ = (*btHeaderMarshaling)(nil) // MarshalJSON marshals as JSON. func (b btHeader) MarshalJSON() ([]byte, error) { type btHeader struct { - Bloom types.Bloom - Coinbase common.Address - MixHash common.Hash - Nonce types.BlockNonce - Number *math.HexOrDecimal256 - Hash common.Hash - ParentHash common.Hash - ReceiptTrie common.Hash - StateRoot common.Hash - TransactionsTrie common.Hash - UncleHash common.Hash - ExtraData hexutil.Bytes - Difficulty *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - GasUsed math.HexOrDecimal64 - Timestamp math.HexOrDecimal64 - BaseFeePerGas *math.HexOrDecimal256 - WithdrawalsRoot *common.Hash + Bloom types.Bloom + Coinbase common.Address + MixHash common.Hash + Nonce types.BlockNonce + Number *math.HexOrDecimal256 + Hash common.Hash + ParentHash common.Hash + ReceiptTrie common.Hash + StateRoot common.Hash + TransactionsTrie common.Hash + UncleHash common.Hash + ExtraData hexutil.Bytes + Difficulty *math.HexOrDecimal256 + GasLimit math.HexOrDecimal64 + GasUsed math.HexOrDecimal64 + Timestamp math.HexOrDecimal64 + BaseFeePerGas *math.HexOrDecimal256 + WithdrawalsRoot *common.Hash + BlobGasUsed *math.HexOrDecimal64 + ExcessBlobGas *math.HexOrDecimal64 + ParentBeaconBlockRoot *common.Hash } var enc btHeader enc.Bloom = b.Bloom @@ -55,30 +58,36 @@ func (b btHeader) MarshalJSON() ([]byte, error) { enc.Timestamp = math.HexOrDecimal64(b.Timestamp) enc.BaseFeePerGas = (*math.HexOrDecimal256)(b.BaseFeePerGas) enc.WithdrawalsRoot = b.WithdrawalsRoot + enc.BlobGasUsed = (*math.HexOrDecimal64)(b.BlobGasUsed) + enc.ExcessBlobGas = (*math.HexOrDecimal64)(b.ExcessBlobGas) + enc.ParentBeaconBlockRoot = b.ParentBeaconBlockRoot return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (b *btHeader) UnmarshalJSON(input []byte) error { type btHeader struct { - Bloom *types.Bloom - Coinbase *common.Address - MixHash *common.Hash - Nonce *types.BlockNonce - Number *math.HexOrDecimal256 - Hash *common.Hash - ParentHash *common.Hash - ReceiptTrie *common.Hash - StateRoot *common.Hash - TransactionsTrie *common.Hash - UncleHash *common.Hash - ExtraData *hexutil.Bytes - Difficulty *math.HexOrDecimal256 - GasLimit *math.HexOrDecimal64 - GasUsed *math.HexOrDecimal64 - Timestamp *math.HexOrDecimal64 - BaseFeePerGas *math.HexOrDecimal256 - WithdrawalsRoot *common.Hash + Bloom *types.Bloom + Coinbase *common.Address + MixHash *common.Hash + Nonce *types.BlockNonce + Number *math.HexOrDecimal256 + Hash *common.Hash + ParentHash *common.Hash + ReceiptTrie *common.Hash + StateRoot *common.Hash + TransactionsTrie *common.Hash + UncleHash *common.Hash + ExtraData *hexutil.Bytes + Difficulty *math.HexOrDecimal256 + GasLimit *math.HexOrDecimal64 + GasUsed *math.HexOrDecimal64 + Timestamp *math.HexOrDecimal64 + BaseFeePerGas *math.HexOrDecimal256 + WithdrawalsRoot *common.Hash + BlobGasUsed *math.HexOrDecimal64 + ExcessBlobGas *math.HexOrDecimal64 + ParentBeaconBlockRoot *common.Hash } var dec btHeader if err := json.Unmarshal(input, &dec); err != nil { @@ -138,5 +147,14 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { if dec.WithdrawalsRoot != nil { b.WithdrawalsRoot = dec.WithdrawalsRoot } + if dec.BlobGasUsed != nil { + b.BlobGasUsed = (*uint64)(dec.BlobGasUsed) + } + if dec.ExcessBlobGas != nil { + b.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) + } + if dec.ParentBeaconBlockRoot != nil { + b.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot + } return nil } diff --git a/tests/init.go b/tests/init.go index a04e227dc7..99b7e4d333 100644 --- a/tests/init.go +++ b/tests/init.go @@ -318,6 +318,25 @@ var Forks = map[string]*params.ChainConfig{ ShanghaiTime: u64(0), CancunTime: u64(0), }, + "ShanghaiToCancunAtTime15k": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(0), + ShanghaiTime: u64(0), + CancunTime: u64(15_000), + }, } // AvailableForks returns the set of defined fork names diff --git a/tests/init_test.go b/tests/init_test.go index 7d8743efcc..3ab15e7658 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -41,6 +41,7 @@ var ( transactionTestDir = filepath.Join(baseDir, "TransactionTests") rlpTestDir = filepath.Join(baseDir, "RLPTests") difficultyTestDir = filepath.Join(baseDir, "BasicTests") + executionSpecDir = filepath.Join(".", "spec-tests", "fixtures") benchmarksDir = filepath.Join(".", "evm-benchmarks", "benchmarks") )