tests: update test suite for istanbul (ethereum#20082)

* update tests for istanbul

* tests: updated blockchaintests, see ethereum/tests#637

* tests: update again, hopefully fixed this time

* tests: skip time consuming, run legacy tests

* tests: update again

* build: disable long-running tests on travis

* tests: fix formatting nits

* tests: I hate github's editor

build/ci.go

@@ -315,7 +315,7 @@ func doTest(cmdline []string) {
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
 	gotest := goTool("test", buildFlags(env)...)
-	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m", "--short")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}

tests/block_test.go

@@ -25,17 +25,9 @@ func TestBlockchain(t *testing.T) {
 
 	bt := new(testMatcher)
 	// General state tests are 'exported' as blockchain tests, but we can run them natively.
-	bt.skipLoad(`^ValidBlocks/bcStateTests/`)
-	// Skip random failures due to selfish mining test.
+	bt.skipLoad(`^GeneralStateTests/`)
+	// Skip random failures due to selfish mining test
 	bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`)
-	bt.skipLoad(`.*bcMultiChainTest/(ChainAtoChainB_blockorder|CallContractFromNotBestBlock)`)
-	bt.skipLoad(`.*bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`)
-
-	// These are not formatted like the rest -- due to the large postState, the postState
-	// was replaced by a hash, instead of a genesisAlloc map
-	// See https://github.com/ethereum/tests/pull/616
-	bt.skipLoad(`.*bcExploitTest/ShanghaiLove.json`)
-	bt.skipLoad(`.*bcExploitTest/SuicideIssue.json`)
 
 	// Slow tests
 	bt.slow(`.*bcExploitTest/DelegateCallSpam.json`)
@@ -45,9 +37,20 @@ func TestBlockchain(t *testing.T) {
 	bt.slow(`.*/bcGasPricerTest/RPC_API_Test.json`)
 	bt.slow(`.*/bcWalletTest/`)
 
+	// Very slow test
+	bt.skipLoad(`.*/stTimeConsuming/.*`)
+
+	// test takes a lot for time and goes easily OOM because of sha3 calculation on a huge range,
+	// using 4.6 TGas
+	bt.skipLoad(`.*randomStatetest94.json.*`)
+
 	bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
 		if err := bt.checkFailure(t, name, test.Run()); err != nil {
 			t.Error(err)
 		}
 	})
+
+	// There is also a LegacyTests folder, containing blockchain tests generated
+	// prior to Istanbul. However, they are all derived from GeneralStateTests,
+	// which run natively, so there's no reason to run them here.
 }

tests/init_test.go

@@ -49,6 +49,7 @@ var (
 	baseDir            = filepath.Join(".", "testdata")
 	blockTestDir       = filepath.Join(baseDir, "BlockchainTests")
 	stateTestDir       = filepath.Join(baseDir, "GeneralStateTests")
+	legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests")
 	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
 	vmTestDir          = filepath.Join(baseDir, "VMTests")
 	rlpTestDir         = filepath.Join(baseDir, "RLPTests")

tests/state_test.go

@@ -40,9 +40,11 @@ func TestState(t *testing.T) {
 	st.slow(`^stStaticCall/static_Call1MB`)
 	st.slow(`^stSystemOperationsTest/CallRecursiveBomb`)
 	st.slow(`^stTransactionTest/Opcodes_TransactionInit`)
+
+	// Very time consuming
+	st.skipLoad(`^stTimeConsuming/`)
+
 	// Broken tests:
-	st.skipLoad(`^stTransactionTest/OverflowGasRequire\.json`) // gasLimit > 256 bits
-	st.skipLoad(`^stTransactionTest/zeroSigTransa[^/]*\.json`) // EIP-86 is not supported yet
 	// Expected failures:
 	//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test")
 	//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test")
@@ -64,6 +66,20 @@ func TestState(t *testing.T) {
 			})
 		}
 	})
+	// For Istanbul, older tests were moved into LegacyTests
+	st.walk(t, legacyStateTestDir, func(t *testing.T, name string, test *StateTest) {
+		for _, subtest := range test.Subtests() {
+			subtest := subtest
+			key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
+			name := name + "/" + key
+			t.Run(key, func(t *testing.T) {
+				withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
+					_, err := test.Run(subtest, vmconfig)
+					return st.checkFailure(t, name, err)
+				})
+			})
+		}
+	})
 }
 
 // Transactions with gasLimit above this value will not get a VM trace on failure.