From ac0637c41332de1f49fb0955f4fbe0fb908a77d5 Mon Sep 17 00:00:00 2001 From: Taylor Gerring Date: Wed, 10 Jun 2015 17:04:06 -0400 Subject: [PATCH] More consistent test interfaces + test skipping --- tests/block_test.go | 50 +++++++++++++++++++++++++++------- tests/block_test_util.go | 27 ++++++++++-------- tests/init.go | 5 ++++ tests/state_test_util.go | 36 ++++++++++++------------ tests/transaction_test.go | 25 ++--------------- tests/transaction_test_util.go | 27 ++++++++++++------ tests/vm_test_util.go | 16 +++++------ 7 files changed, 108 insertions(+), 78 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index f1a92be49..9d21ba28d 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -6,38 +6,68 @@ import ( ) func TestBcValidBlockTests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcValidBlockTest.json"), []string{"SimpleTx3"}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json")) + if err != nil { + t.Fatal(err) + } } func TestBcUncleTests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleTest.json"), []string{}) - runBlockTestsInFile(filepath.Join(blockTestDir, "bcBruncleTest.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json")) + if err != nil { + t.Fatal(err) + } + err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json")) + if err != nil { + t.Fatal(err) + } } func TestBcUncleHeaderValidityTests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json")) + if err != nil { + t.Fatal(err) + } } func TestBcInvalidHeaderTests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json")) + if err != nil { + t.Fatal(err) + } } func TestBcInvalidRLPTests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json")) + if err != nil { + t.Fatal(err) + } } func TestBcRPCAPITests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json")) + if err != nil { + t.Fatal(err) + } } func TestBcForkBlockTests(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcForkBlockTest.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json")) + if err != nil { + t.Fatal(err) + } } func TestBcTotalDifficulty(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json")) + if err != nil { + t.Fatal(err) + } } func TestBcWallet(t *testing.T) { - runBlockTestsInFile(filepath.Join(blockTestDir, "bcWalletTest.json"), []string{}) + err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json")) + if err != nil { + t.Fatal(err) + } } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index ec532d178..a04019111 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -86,28 +86,35 @@ type btTransaction struct { Value string } -func runBlockTestsInFile(filepath string, snafus []string) error { +func RunBlockTest(filepath string) error { bt, err := LoadBlockTests(filepath) if err != nil { return nil } - notWorking := make(map[string]bool, 100) - for _, name := range snafus { - notWorking[name] = true + // map skipped tests to boolean set + skipTest := make(map[string]bool, len(blockSkipTests)) + for _, name := range blockSkipTests { + skipTest[name] = true } for name, test := range bt { - if !notWorking[name] { - if err := runBlockTest(name, test); err != nil { - return err - } + // if the test should be skipped, return + if skipTest[name] { + fmt.Println("Skipping state test", name) + return nil + } + // test the block + if err := testBlock(test); err != nil { + return err } + fmt.Println("Block test passed: ", name) + } return nil } -func runBlockTest(name string, test *BlockTest) error { +func testBlock(test *BlockTest) error { cfg := testEthConfig() ethereum, err := eth.New(cfg) if err != nil { @@ -136,8 +143,6 @@ func runBlockTest(name string, test *BlockTest) error { if err = test.ValidatePostState(statedb); err != nil { return fmt.Errorf("post state validation failed: %v", err) } - fmt.Println("Block test passed: ", name) - // t.Log("Block test passed: ", name) return nil } diff --git a/tests/init.go b/tests/init.go index e6644ae60..74d9499f1 100644 --- a/tests/init.go +++ b/tests/init.go @@ -15,6 +15,11 @@ var ( stateTestDir = filepath.Join(baseDir, "StateTests") transactionTestDir = filepath.Join(baseDir, "TransactionTests") vmTestDir = filepath.Join(baseDir, "VMTests") + + blockSkipTests = []string{} + transSkipTests = []string{"TransactionWithHihghNonce256"} + stateSkipTests = []string{"mload32bitBound_return", "mload32bitBound_return2"} + vmSkipTests = []string{} ) func readJSON(reader io.Reader, value interface{}) error { diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 29d7cebe8..b507de47f 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -15,19 +15,19 @@ import ( ) func RunStateTest(p string) error { + skipTest := make(map[string]bool, len(stateSkipTests)) + for _, name := range stateSkipTests { + skipTest[name] = true + } tests := make(map[string]VmTest) CreateFileTests(p, &tests) for name, test := range tests { - /* - vm.Debug = true - glog.SetV(4) - glog.SetToStderr(true) - if name != "Call50000_sha256" { - continue - } - */ + if skipTest[name] { + fmt.Println("Skipping state test", name) + return nil + } db, _ := ethdb.NewMemDatabase() statedb := state.New(common.Hash{}, db) for addr, account := range test.Pre { @@ -60,17 +60,17 @@ func RunStateTest(p string) error { ret, logs, _, _ = RunState(statedb, env, test.Transaction) - // Compare expected and actual return - switch name { - // the memory required for these tests (4294967297 bytes) would take too much time. - // on 19 May 2015 decided to skip these tests their output. - case "mload32bitBound_return", "mload32bitBound_return2": - default: - rexp := common.FromHex(test.Out) - if bytes.Compare(rexp, ret) != 0 { - return fmt.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret) - } + // // Compare expected and actual return + // switch name { + // // the memory required for these tests (4294967297 bytes) would take too much time. + // // on 19 May 2015 decided to skip these tests their output. + // case "mload32bitBound_return", "mload32bitBound_return2": + // default: + rexp := common.FromHex(test.Out) + if bytes.Compare(rexp, ret) != 0 { + return fmt.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret) } + // } // check post state for addr, account := range test.Post { diff --git a/tests/transaction_test.go b/tests/transaction_test.go index e1237dee2..41a20a1bb 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -6,40 +6,21 @@ import ( ) func TestTransactions(t *testing.T) { - notWorking := make(map[string]bool, 100) - - // TODO: all these tests should work! remove them from the array when they work - snafus := []string{ - "TransactionWithHihghNonce256", // fails due to testing upper bound of 256 bit nonce - } - - for _, name := range snafus { - notWorking[name] = true - } - - var err error - err = RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"), - notWorking) + err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json")) if err != nil { t.Fatal(err) } } func TestWrongRLPTransactions(t *testing.T) { - notWorking := make(map[string]bool, 100) - var err error - err = RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"), - notWorking) + err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json")) if err != nil { t.Fatal(err) } } func Test10MBtx(t *testing.T) { - notWorking := make(map[string]bool, 100) - var err error - err = RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"), - notWorking) + err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json")) if err != nil { t.Fatal(err) } diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index a6cea972a..65e2c7591 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -30,20 +30,29 @@ type TransactionTest struct { Transaction TtTransaction } -func RunTransactionTests(file string, notWorking map[string]bool) error { +func RunTransactionTests(file string) error { + skipTest := make(map[string]bool, len(transSkipTests)) + for _, name := range transSkipTests { + skipTest[name] = true + } + bt := make(map[string]TransactionTest) if err := LoadJSON(file, &bt); err != nil { return err } - for name, in := range bt { - var err error - // TODO: remove this, we currently ignore some tests which are broken - if !notWorking[name] { - if err = runTest(in); err != nil { - return fmt.Errorf("bad test %s: %v", name, err) - } - fmt.Println("Transaction test passed:", name) + + for name, test := range bt { + // if the test should be skipped, return + if skipTest[name] { + fmt.Println("Skipping state test", name) + return nil + } + // test the block + if err := runTest(test); err != nil { + return err } + fmt.Println("Transaction test passed: ", name) + } return nil } diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index 066217620..55036ed82 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -13,6 +13,10 @@ import ( ) func RunVmTest(p string) error { + skipTest := make(map[string]bool, len(vmSkipTests)) + for _, name := range vmSkipTests { + skipTest[name] = true + } tests := make(map[string]VmTest) err := CreateFileTests(p, &tests) @@ -21,14 +25,10 @@ func RunVmTest(p string) error { } for name, test := range tests { - /* - vm.Debug = true - glog.SetV(4) - glog.SetToStderr(true) - if name != "Call50000_sha256" { - continue - } - */ + if skipTest[name] { + fmt.Println("Skipping state test", name) + return nil + } db, _ := ethdb.NewMemDatabase() statedb := state.New(common.Hash{}, db) for addr, account := range test.Pre {