More consistent test interfaces + test skipping

tests/block_test.go

@@ -6,38 +6,68 @@ import (
 )
 
 func TestBcValidBlockTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcValidBlockTest.json"), []string{"SimpleTx3"})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcUncleTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleTest.json"), []string{})
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcBruncleTest.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcUncleHeaderValidityTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcInvalidHeaderTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcInvalidRLPTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcRPCAPITests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcForkBlockTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcForkBlockTest.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcTotalDifficulty(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestBcWallet(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcWalletTest.json"), []string{})
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
 }

tests/block_test_util.go

@@ -86,28 +86,35 @@ type btTransaction struct {
 	Value    string
 }
 
-func runBlockTestsInFile(filepath string, snafus []string) error {
+func RunBlockTest(filepath string) error {
 	bt, err := LoadBlockTests(filepath)
 	if err != nil {
 		return nil
 	}
 
-	notWorking := make(map[string]bool, 100)
-	for _, name := range snafus {
-		notWorking[name] = true
+	// map skipped tests to boolean set
+	skipTest := make(map[string]bool, len(blockSkipTests))
+	for _, name := range blockSkipTests {
+		skipTest[name] = true
 	}
 
 	for name, test := range bt {
-		if !notWorking[name] {
-			if err := runBlockTest(name, test); err != nil {
-				return err
-			}
+		// if the test should be skipped, return
+		if skipTest[name] {
+			fmt.Println("Skipping state test", name)
+			return nil
+		}
+		// test the block
+		if err := testBlock(test); err != nil {
+			return err
 		}
+		fmt.Println("Block test passed: ", name)
+
 	}
 	return nil
 }
 
-func runBlockTest(name string, test *BlockTest) error {
+func testBlock(test *BlockTest) error {
 	cfg := testEthConfig()
 	ethereum, err := eth.New(cfg)
 	if err != nil {
@@ -136,8 +143,6 @@ func runBlockTest(name string, test *BlockTest) error {
 	if err = test.ValidatePostState(statedb); err != nil {
 		return fmt.Errorf("post state validation failed: %v", err)
 	}
-	fmt.Println("Block test passed: ", name)
-	// t.Log("Block test passed: ", name)
 	return nil
 }
 

tests/init.go

@@ -15,6 +15,11 @@ var (
 	stateTestDir       = filepath.Join(baseDir, "StateTests")
 	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
 	vmTestDir          = filepath.Join(baseDir, "VMTests")
+
+	blockSkipTests = []string{}
+	transSkipTests = []string{"TransactionWithHihghNonce256"}
+	stateSkipTests = []string{"mload32bitBound_return", "mload32bitBound_return2"}
+	vmSkipTests    = []string{}
 )
 
 func readJSON(reader io.Reader, value interface{}) error {

tests/state_test_util.go

@@ -15,19 +15,19 @@ import (
 )
 
 func RunStateTest(p string) error {
+	skipTest := make(map[string]bool, len(stateSkipTests))
+	for _, name := range stateSkipTests {
+		skipTest[name] = true
+	}
 
 	tests := make(map[string]VmTest)
 	CreateFileTests(p, &tests)
 
 	for name, test := range tests {
-		/*
-		   vm.Debug = true
-		   glog.SetV(4)
-		   glog.SetToStderr(true)
-		   if name != "Call50000_sha256" {
-		     continue
-		   }
-		*/
+		if skipTest[name] {
+			fmt.Println("Skipping state test", name)
+			return nil
+		}
 		db, _ := ethdb.NewMemDatabase()
 		statedb := state.New(common.Hash{}, db)
 		for addr, account := range test.Pre {
@@ -60,17 +60,17 @@ func RunStateTest(p string) error {
 
 		ret, logs, _, _ = RunState(statedb, env, test.Transaction)
 
-		// Compare expected  and actual return
-		switch name {
-		// the memory required for these tests (4294967297 bytes) would take too much time.
-		// on 19 May 2015 decided to skip these tests their output.
-		case "mload32bitBound_return", "mload32bitBound_return2":
-		default:
-			rexp := common.FromHex(test.Out)
-			if bytes.Compare(rexp, ret) != 0 {
-				return fmt.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret)
-			}
+		// // Compare expected  and actual return
+		// switch name {
+		// // the memory required for these tests (4294967297 bytes) would take too much time.
+		// // on 19 May 2015 decided to skip these tests their output.
+		// case "mload32bitBound_return", "mload32bitBound_return2":
+		// default:
+		rexp := common.FromHex(test.Out)
+		if bytes.Compare(rexp, ret) != 0 {
+			return fmt.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret)
 		}
+		// }
 
 		// check post state
 		for addr, account := range test.Post {

tests/transaction_test.go

@@ -6,40 +6,21 @@ import (
 )
 
 func TestTransactions(t *testing.T) {
-	notWorking := make(map[string]bool, 100)
-
-	// TODO: all these tests should work! remove them from the array when they work
-	snafus := []string{
-		"TransactionWithHihghNonce256", // fails due to testing upper bound of 256 bit nonce
-	}
-
-	for _, name := range snafus {
-		notWorking[name] = true
-	}
-
-	var err error
-	err = RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"),
-		notWorking)
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"))
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestWrongRLPTransactions(t *testing.T) {
-	notWorking := make(map[string]bool, 100)
-	var err error
-	err = RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"),
-		notWorking)
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"))
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func Test10MBtx(t *testing.T) {
-	notWorking := make(map[string]bool, 100)
-	var err error
-	err = RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"),
-		notWorking)
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"))
 	if err != nil {
 		t.Fatal(err)
 	}

tests/transaction_test_util.go

@@ -30,20 +30,29 @@ type TransactionTest struct {
 	Transaction TtTransaction
 }
 
-func RunTransactionTests(file string, notWorking map[string]bool) error {
+func RunTransactionTests(file string) error {
+	skipTest := make(map[string]bool, len(transSkipTests))
+	for _, name := range transSkipTests {
+		skipTest[name] = true
+	}
+
 	bt := make(map[string]TransactionTest)
 	if err := LoadJSON(file, &bt); err != nil {
 		return err
 	}
-	for name, in := range bt {
-		var err error
-		// TODO: remove this, we currently ignore some tests which are broken
-		if !notWorking[name] {
-			if err = runTest(in); err != nil {
-				return fmt.Errorf("bad test %s: %v", name, err)
-			}
-			fmt.Println("Transaction test passed:", name)
+
+	for name, test := range bt {
+		// if the test should be skipped, return
+		if skipTest[name] {
+			fmt.Println("Skipping state test", name)
+			return nil
+		}
+		// test the block
+		if err := runTest(test); err != nil {
+			return err
 		}
+		fmt.Println("Transaction test passed: ", name)
+
 	}
 	return nil
 }

tests/vm_test_util.go

@@ -13,6 +13,10 @@ import (
 )
 
 func RunVmTest(p string) error {
+	skipTest := make(map[string]bool, len(vmSkipTests))
+	for _, name := range vmSkipTests {
+		skipTest[name] = true
+	}
 
 	tests := make(map[string]VmTest)
 	err := CreateFileTests(p, &tests)
@@ -21,14 +25,10 @@ func RunVmTest(p string) error {
 	}
 
 	for name, test := range tests {
-		/*
-		   vm.Debug = true
-		   glog.SetV(4)
-		   glog.SetToStderr(true)
-		   if name != "Call50000_sha256" {
-		     continue
-		   }
-		*/
+		if skipTest[name] {
+			fmt.Println("Skipping state test", name)
+			return nil
+		}
 		db, _ := ethdb.NewMemDatabase()
 		statedb := state.New(common.Hash{}, db)
 		for addr, account := range test.Pre {