From bb0c08fe01e7e4089301c6a72e534e5cadd2cb17 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 Sep 2018 12:24:40 +0200 Subject: [PATCH] Add constantinople conf to EvmTestClient. (#9570) * Add constantinople conf to EvmTestClient. * Skip some test to update submodule etheureum/tests submodule to latest. * Put skipping 'under issue' test behind a feature. * Change blockReward for const-test to pass ethereum/tests * Update tests to new constantinple definition (change of reward at block 5). Switch 'reference' to string, that way we can include issues from others repo (more flexible)Update tests to new constantinple definition (change of reward at block 5). Switch 'reference' to string, that way we can include issues from others repo (more flexible). * Fix modexp and bn128_mul gas prices in chain config * Changes `run_test_path` method to append its directory results (without that it stop testing at the first file failure). Add some missing tests. Add skip for those (block create2 is one hundred percent false but on hive we can see that geth and aleth got similar issue for this item). * retab current.json * Update reference to parity issue for failing tests. --- Cargo.toml | 1 + ethcore/Cargo.toml | 2 + ethcore/res/ethereum/constantinople_test.json | 10 +- ethcore/res/ethereum/tests | 2 +- .../res/ethereum/tests-issues/currents.json | 370 ++++++++++++++++++ ethcore/src/client/evm_test_client.rs | 2 +- ethcore/src/json_tests/chain.rs | 18 + ethcore/src/json_tests/mod.rs | 3 + ethcore/src/json_tests/skip.rs | 37 ++ ethcore/src/json_tests/state.rs | 20 +- ethcore/src/json_tests/test_common.rs | 31 +- json/src/test/mod.rs | 57 +++ test.sh | 2 +- 13 files changed, 544 insertions(+), 11 deletions(-) create mode 100644 ethcore/res/ethereum/tests-issues/currents.json create mode 100644 ethcore/src/json_tests/skip.rs diff --git a/Cargo.toml b/Cargo.toml index 646f63599dc..9b09550d3b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ daemonize = { git = "https://github.com/paritytech/daemonize" } [features] miner-debug = ["ethcore/miner-debug"] json-tests = ["ethcore/json-tests"] +ci-skip-issue = ["ethcore/ci-skip-issue"] test-heavy = ["ethcore/test-heavy"] evm-debug = ["ethcore/evm-debug"] evm-debug-tests = ["ethcore/evm-debug-tests"] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 82dd2230dd6..80f7e432600 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -102,6 +102,8 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"] slow-blocks = [] # Run JSON consensus tests. json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"] +# Skip JSON consensus tests with pending issues. +ci-skip-issue = [] # Run memory/cpu heavy tests. test-heavy = [] # Compile benches diff --git a/ethcore/res/ethereum/constantinople_test.json b/ethcore/res/ethereum/constantinople_test.json index 56c556f4cec..fd7474dc2d7 100644 --- a/ethcore/res/ethereum/constantinople_test.json +++ b/ethcore/res/ethereum/constantinople_test.json @@ -6,12 +6,12 @@ "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", "durationLimit": "0x0d", - "blockReward": "0x1BC16D674EC80000", + "blockReward": "0x29A2241AF62C0000", "homesteadTransition": "0x0", "eip100bTransition": "0x0", - "eip649Transition": "0x0", + "eip649Transition": "0x5", "eip649Reward": "0x1BC16D674EC80000", - "eip1234Transition": "0x0", + "eip1234Transition": "0x5", "eip1234Reward": "0x1BC16D674EC80000" } } @@ -59,9 +59,9 @@ "0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, "0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, "0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, - "0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 100 } } } }, + "0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 20 } } } }, "0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } }, - "0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 2000, "word": 0 } } } }, + "0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 40000, "word": 0 } } } }, "0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } } } } diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index b8a21c19369..3f5febc9019 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit b8a21c193696976ca3b33b6d82107601063a5d26 +Subproject commit 3f5febc901913ef698f1b09dda8705babd729e4a diff --git a/ethcore/res/ethereum/tests-issues/currents.json b/ethcore/res/ethereum/tests-issues/currents.json new file mode 100644 index 00000000000..371211cab82 --- /dev/null +++ b/ethcore/res/ethereum/tests-issues/currents.json @@ -0,0 +1,370 @@ +{ "block": + [ + { + "reference": "9590", + "failing": "stCreateTest", + "subtests": [ + "CreateOOGafterInitCodeReturndata2_d0g1v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stCreate2", + "subtests": [ + "RevertDepthCreateAddressCollision_d0g1v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g1v1_Constantinople", + "CREATE2_Suicide_d5g0v0_Constantinople", + "CREATE2_Suicide_d7g0v0_Constantinople", + "create2collisionSelfdestructedOOG_d2g0v0_Byzantium", + "create2collisionSelfdestructedOOG_d2g0v0_Constantinople", + "create2collisionNonce_d1g0v0_Byzantium", + "create2collisionNonce_d1g0v0_Constantinople", + "CreateMessageRevertedOOGInInit_d0g1v0_Constantinople", + "create2callPrecompiles_d3g0v0_Constantinople", + "create2collisionCode_d1g0v0_Byzantium", + "create2collisionCode_d1g0v0_Constantinople", + "create2collisionStorage_d0g0v0_Byzantium", + "create2collisionStorage_d0g0v0_Constantinople", + "create2callPrecompiles_d4g0v0_Constantinople", + "create2collisionSelfdestructedRevert_d0g0v0_Byzantium", + "create2collisionSelfdestructedRevert_d0g0v0_Constantinople", + "CreateMessageReverted_d0g1v0_Constantinople", + "RevertOpcodeCreate_d0g1v0_Constantinople", + "CREATE2_Suicide_d11g0v0_Constantinople", + "create2checkFieldsInInitcode_d5g0v0_Constantinople", + "create2collisionSelfdestructedOOG_d1g0v0_Byzantium", + "create2collisionSelfdestructedOOG_d1g0v0_Constantinople", + "returndatacopy_following_create_d1g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g1v1_Constantinople", + "create2collisionSelfdestructed_d2g0v0_Byzantium", + "create2collisionSelfdestructed_d2g0v0_Constantinople", + "create2callPrecompiles_d2g0v0_Constantinople", + "create2InitCodes_d2g0v0_Constantinople", + "create2collisionNonce_d2g0v0_Byzantium", + "create2collisionNonce_d2g0v0_Constantinople", + "create2collisionCode_d0g0v0_Byzantium", + "create2collisionCode_d0g0v0_Constantinople", + "CREATE2_Bounds_d0g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g0v0_Constantinople", + "CREATE2_Suicide_d1g0v0_Constantinople", + "CREATE2_Bounds3_d0g1v0_Constantinople", + "create2collisionStorage_d2g0v0_Byzantium", + "create2collisionStorage_d2g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d0g0v1_Constantinople", + "create2callPrecompiles_d5g0v0_Constantinople", + "create2collisionCode2_d0g0v0_Byzantium", + "create2collisionCode2_d0g0v0_Constantinople", + "create2noCash_d0g0v0_Byzantium", + "create2noCash_d0g0v0_Constantinople", + "create2checkFieldsInInitcode_d7g0v0_Constantinople", + "create2SmartInitCode_d1g0v0_Constantinople", + "create2InitCodes_d6g0v0_Constantinople", + "create2noCash_d1g0v0_Byzantium", + "create2noCash_d1g0v0_Constantinople", + "CREATE2_ContractSuicideDuringInit_ThenStoreThenReturn_d0g0v0_Constantinople", + "RevertOpcodeInCreateReturns_d0g0v0_Constantinople", + "create2collisionStorage_d1g0v0_Byzantium", + "create2collisionStorage_d1g0v0_Constantinople", + "create2checkFieldsInInitcode_d3g0v0_Constantinople", + "create2collisionBalance_d0g0v0_Byzantium", + "create2collisionBalance_d0g0v0_Constantinople", + "create2collisionSelfdestructed2_d0g0v0_Constantinople", + "create2InitCodes_d3g0v0_Constantinople", + "create2collisionCode2_d1g0v0_Byzantium", + "create2collisionCode2_d1g0v0_Constantinople", + "create2checkFieldsInInitcode_d1g0v0_Constantinople", + "create2collisionBalance_d1g0v0_Byzantium", + "create2collisionBalance_d1g0v0_Constantinople", + "CREATE2_Bounds3_d0g2v0_Constantinople", + "create2callPrecompiles_d6g0v0_Constantinople", + "Create2Recursive_d0g0v0_Constantinople", + "create2collisionSelfdestructedOOG_d0g0v0_Byzantium", + "create2collisionSelfdestructedOOG_d0g0v0_Constantinople", + "CREATE2_Suicide_d3g0v0_Constantinople", + "returndatacopy_following_create_d0g0v0_Constantinople", + "create2InitCodes_d8g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g0v1_Constantinople", + "create2checkFieldsInInitcode_d2g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g0v1_Constantinople", + "Create2OnDepth1024_d0g0v0_Constantinople", + "create2collisionSelfdestructed2_d1g0v0_Constantinople", + "create2collisionSelfdestructedRevert_d2g0v0_Byzantium", + "create2collisionSelfdestructedRevert_d2g0v0_Constantinople", + "create2callPrecompiles_d0g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d0g1v1_Constantinople", + "create2collisionSelfdestructed_d1g0v0_Byzantium", + "create2collisionSelfdestructed_d1g0v0_Constantinople", + "call_outsize_then_create2_successful_then_returndatasize_d0g0v0_Byzantium", + "call_outsize_then_create2_successful_then_returndatasize_d0g0v0_Constantinople", + "Create2OOGafterInitCodeRevert_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata3_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndataSize_d0g0v0_Constantinople", + "create2InitCodes_d7g0v0_Constantinople", + "CREATE2_Suicide_d10g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g1v0_Constantinople", + "create2InitCodes_d5g0v0_Constantinople", + "create2collisionSelfdestructedRevert_d1g0v0_Byzantium", + "create2collisionSelfdestructedRevert_d1g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g1v0_Constantinople", + "create2collisionSelfdestructed_d0g0v0_Byzantium", + "create2collisionSelfdestructed_d0g0v0_Constantinople", + "create2noCash_d2g0v0_Byzantium", + "create2noCash_d2g0v0_Constantinople", + "CREATE2_Bounds3_d0g0v0_Constantinople", + "create2collisionNonce_d0g0v0_Byzantium", + "create2collisionNonce_d0g0v0_Constantinople", + "CREATE2_Suicide_d2g0v0_Constantinople", + "Create2OOGafterInitCode_d0g0v0_Constantinople", + "call_then_create2_successful_then_returndatasize_d0g0v0_Byzantium", + "call_then_create2_successful_then_returndatasize_d0g0v0_Constantinople", + "create2collisionBalance_d2g0v0_Byzantium", + "create2collisionBalance_d2g0v0_Constantinople", + "create2checkFieldsInInitcode_d6g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g1v1_Constantinople", + "returndatacopy_afterFailing_create_d0g0v0_Constantinople", + "returndatacopy_following_revert_in_create_d0g0v0_Constantinople", + "CREATE2_Suicide_d9g0v0_Constantinople", + "create2callPrecompiles_d7g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g0v1_Constantinople", + "create2InitCodes_d1g0v0_Constantinople", + "CREATE2_Bounds_d0g1v0_Constantinople", + "Create2OOGafterInitCodeReturndata_d0g0v0_Constantinople", + "create2checkFieldsInInitcode_d4g0v0_Constantinople", + "CreateMessageRevertedOOGInInit_d0g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g1v0_Constantinople", + "returndatacopy_following_successful_create_d0g0v0_Constantinople", + "create2checkFieldsInInitcode_d0g0v0_Constantinople", + "CreateMessageReverted_d0g0v0_Constantinople", + "create2SmartInitCode_d0g0v0_Constantinople", + "CREATE2_Bounds2_d0g0v0_Constantinople", + "returndatasize_following_successful_create_d0g0v0_Constantinople", + "CREATE2_Bounds2_d0g1v0_Constantinople", + "returndatacopy_0_0_following_successful_create_d0g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d0g0v0_Constantinople", + "CREATE2_Suicide_d0g0v0_Constantinople", + "create2InitCodes_d0g0v0_Constantinople", + "Create2OnDepth1023_d0g0v0_Constantinople", + "create2InitCodes_d4g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata2_d0g0v0_Constantinople", + "create2collisionBalance_d3g0v0_Byzantium", + "create2collisionBalance_d3g0v0_Constantinople", + "CREATE2_Suicide_d4g0v0_Constantinople", + "Create2OOGafterInitCode_d0g1v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g0v0_Constantinople", + "Create2OOGafterInitCodeRevert2_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata_d0g1v0_Constantinople", + "Create2Recursive_d0g1v0_Constantinople", + "create2collisionCode_d2g0v0_Byzantium", + "create2collisionCode_d2g0v0_Constantinople", + "CREATE2_Suicide_d6g0v0_Constantinople", + "CREATE2_Suicide_d8g0v0_Constantinople", + "RevertOpcodeCreate_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata2_d0g1v0_Constantinople", + "create2callPrecompiles_d1g0v0_Constantinople", + "RevertInCreateInInit_d0g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g0v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "bcStateTest", + "subtests": [ + "suicideStorageCheck_Byzantium", + "suicideStorageCheck_Constantinople", + "suicideStorageCheckVCreate2_Byzantium", + "suicideStorageCheckVCreate2_Constantinople", + "create2collisionwithSelfdestructSameBlock_Constantinople", + "blockhashNonConstArg_Constantinople", + "suicideThenCheckBalance_Constantinople", + "suicideThenCheckBalance_Homestead", + "suicideStorageCheckVCreate_Byzantium", + "suicideStorageCheckVCreate_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stEIP158Specific", + "subtests": [ + "callToEmptyThenCallError_d0g0v0_Byzantium", + "callToEmptyThenCallError_d0g0v0_Constantinople", + "callToEmptyThenCallError_d0g0v0_EIP158" + ] + }, + { + "reference": "9590", + "failing": "stPreCompiledContracts", + "subtests": [ + "identity_to_smaller_d0g0v0_Constantinople", + "identity_to_bigger_d0g0v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stReturnDataTest", + "subtests": [ + "modexp_modsize0_returndatasize_d0g1v0_Constantinople", + "modexp_modsize0_returndatasize_d0g2v0_Constantinople", + "modexp_modsize0_returndatasize_d0g3v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stSpecialTest", + "subtests": [ + "push32withoutByte_d0g0v0_Constantinople" + ] + } + + ], + "state": + [ + { + "reference": "9590", + "failing": "stCreateTest", + "subtests": { + "CreateOOGafterInitCodeReturndata2": { + "subnumbers": ["2"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stCreate2Test", + "subtests": { + "RevertInCreateInInit": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stEIP150Specific", + "subtests": { + "NewGasPriceForCodes": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stInitCodeTest", + "subtests": { + "OutOfGasContractCreation": { + "subnumbers": ["4"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stPreCompiledContracts", + "subtests": { + "modexp": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stRevertTest", + "subtests": { + "LoopCallsDepthThenRevert3": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertOpcodeCreate": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertSubCallStorageOOG2": { + "subnumbers": ["1","3"], + "chain": "Constantinople (test)" + }, + "RevertDepthCreateOOG": { + "subnumbers": ["3","4"], + "chain": "Constantinople (test)" + }, + "RevertOpcodeMultipleSubCalls": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "RevertOpcodeDirectCall": { + "subnumbers": ["1","2"], + "chain": "Constantinople (test)" + }, + "LoopCallsDepthThenRevert2": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertDepth2": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertRemoteSubCallStorageOOG2": { + "subnumbers": ["1","2"], + "chain": "Constantinople (test)" + }, + "RevertDepthCreateAddressCollision": { + "subnumbers": ["3","4"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stStaticCall", + "subtests": { + "static_RevertDepth2": { + "subnumbers": ["1","3"], + "chain": "Constantinople (test)" + }, + "static_CheckOpcodes4": { + "subnumbers": ["3"], + "chain": "Constantinople (test)" + }, + "static_CheckOpcodes3": { + "subnumbers": ["5","6","7","8"], + "chain": "Constantinople (test)" + }, + "static_callBasic": { + "subnumbers": ["1","2"], + "chain": "Constantinople (test)" + }, + "static_CheckOpcodes2": { + "subnumbers": ["5","6","7","8"], + "chain": "Constantinople (test)" + }, + "static_callCreate": { + "subnumbers": ["2"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "https://github.com/ethereum/tests/issues/512", + "failing": "stZeroKnowledge", + "subtests": { + "pointAddTrunc": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "pointAdd": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "pointMulAdd": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "pointMulAdd2": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + } + } + } + ] +} diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index 838add4f6a8..f9699af46cd 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -86,9 +86,9 @@ impl<'a> EvmTestClient<'a> { ForkSpec::EIP150 => Some(ethereum::new_eip150_test()), ForkSpec::EIP158 => Some(ethereum::new_eip161_test()), ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()), + ForkSpec::Constantinople => Some(ethereum::new_constantinople_test()), ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()), ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None, - _ => None, } } diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 2d643e75fe4..ae78d4d58c4 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -23,6 +23,7 @@ use ethjson; use miner::Miner; use io::IoChannel; use test_helpers; +use super::SKIP_TEST_STATE; use super::HookType; @@ -36,12 +37,20 @@ pub fn run_test_file(p: &Path, h: &mut H) { ::json_tests::test_common::run_test_file(p, json_chain_test, h) } +fn skip_test(name: &String) -> bool { + SKIP_TEST_STATE.block.iter().any(|block_test|block_test.subtests.contains(name)) +} + pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::blockchain::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, blockchain) in tests.into_iter() { + if skip_test(&name) { + println!(" - {} | {:?} Ignoring tests because in skip list", name, blockchain.network); + continue; + } start_stop_hook(&name, HookType::OnStart); let mut fail = false; @@ -122,19 +131,24 @@ mod block_tests { declare_test!{BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"} declare_test!{BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"} declare_test!{BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"} + declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"} declare_test!{BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"} declare_test!{BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"} declare_test!{BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"} declare_test!{BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"} declare_test!{BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"} + declare_test!{BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"} declare_test!{BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"} declare_test!{BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"} + declare_test!{BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"} declare_test!{BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"} + declare_test!{BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"} declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"} declare_test!{BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"} declare_test!{BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"} + declare_test!{BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"} declare_test!{BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"} declare_test!{BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"} declare_test!{BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"} @@ -149,12 +163,15 @@ mod block_tests { declare_test!{BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"} declare_test!{BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"} declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"} + declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"} declare_test!{heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"} declare_test!{BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"} + declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"} declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"} declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"} declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"} declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"} + declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"} declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"} declare_test!{BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"} declare_test!{BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"} @@ -166,6 +183,7 @@ mod block_tests { declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"} declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"} declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"} + declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"} declare_test!{BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"} declare_test!{BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"} diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index fa1c822ce2a..35d5034806a 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -24,10 +24,12 @@ mod executive; mod state; mod chain; mod trie; +mod skip; #[cfg(test)] mod difficulty; + pub use self::test_common::HookType; pub use self::transaction::run_test_path as run_transaction_test_path; @@ -42,3 +44,4 @@ pub use self::trie::run_generic_test_path as run_generic_trie_test_path; pub use self::trie::run_generic_test_file as run_generic_trie_test_file; pub use self::trie::run_secure_test_path as run_secure_trie_test_path; pub use self::trie::run_secure_test_file as run_secure_trie_test_file; +use self::skip::SKIP_TEST_STATE; diff --git a/ethcore/src/json_tests/skip.rs b/ethcore/src/json_tests/skip.rs new file mode 100644 index 00000000000..784e2975d73 --- /dev/null +++ b/ethcore/src/json_tests/skip.rs @@ -0,0 +1,37 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! State tests to skip. + +use ethjson; + +#[cfg(all(not(test), feature = "ci-skip-tests"))] +compile_error!("ci-skip-tests can only be enabled for testing builds."); + +#[cfg(feature="ci-skip-issue")] +lazy_static!{ + pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = { + let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json"); + ethjson::test::SkipStates::load(&skip_data[..]).expect("No invalid json allowed") + }; +} + +#[cfg(not(feature="ci-skip-issue"))] +lazy_static!{ + pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = { + ethjson::test::SkipStates::empty() + }; +} diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 5837f191c7d..04ca947bf19 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -22,7 +22,7 @@ use client::{EvmTestClient, EvmTestError, TransactResult}; use ethjson; use transaction::SignedTransaction; use vm::EnvInfo; - +use super::SKIP_TEST_STATE; use super::HookType; /// Run state jsontests on a given folder. @@ -35,6 +35,18 @@ pub fn run_test_file(p: &Path, h: &mut H) { ::json_tests::test_common::run_test_file(p, json_chain_test, h) } +fn skip_test(subname: &str, chain: &String, number: usize) -> bool { + SKIP_TEST_STATE.state.iter().any(|state_test|{ + if let Some(subtest) = state_test.subtests.get(subname) { + chain == &subtest.chain && + (subtest.subnumbers[0] == "*" + || subtest.subnumbers.contains(&number.to_string())) + } else { + false + } + }) +} + pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::state::test::Test::load(json_data).unwrap(); @@ -60,6 +72,10 @@ pub fn json_chain_test(json_data: &[u8], start_stop_ho for (i, state) in states.into_iter().enumerate() { let info = format!(" - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total); + if skip_test(&name, &spec.name, i + 1) { + println!("{} in skip list : SKIPPED", info); + continue; + } let post_root: H256 = state.hash.into(); let transaction: SignedTransaction = multitransaction.select(&state.indexes).into(); @@ -125,11 +141,13 @@ mod state_tests { declare_test!{GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"} declare_test!{GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"} declare_test!{GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"} + declare_test!{GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"} declare_test!{GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"} declare_test!{GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"} declare_test!{GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"} declare_test!{GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"} declare_test!{GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"} + declare_test!{GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"} declare_test!{GeneralStateTest_stExample, "GeneralStateTests/stExample/"} declare_test!{GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"} declare_test!{GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"} diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index ce5c68a73a0..ce16395d228 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -34,6 +34,18 @@ pub fn run_test_path( p: &Path, skip: &[&'static str], runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, start_stop_hook: &mut H +) { + let mut errors = Vec::new(); + run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors); + let empty: [String; 0] = []; + assert_eq!(errors, empty); +} + +fn run_test_path_inner( + p: &Path, skip: &[&'static str], + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H, + errors: &mut Vec ) { let path = Path::new(p); let s: HashSet = skip.iter().map(|s| { @@ -50,17 +62,32 @@ pub fn run_test_path( } else { Some(e.path()) }}) { - run_test_path(&p, skip, runner, start_stop_hook) + run_test_path_inner(&p, skip, runner, start_stop_hook, errors); } } else if extension == Some("swp") || extension == None { // Ignore junk } else { let mut path = p.to_path_buf(); path.set_extension("json"); - run_test_file(&path, runner, start_stop_hook) + run_test_file_append(&path, runner, start_stop_hook, errors) } } +fn run_test_file_append( + path: &Path, + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H, + errors: &mut Vec +) { + let mut data = Vec::new(); + let mut file = match File::open(&path) { + Ok(file) => file, + Err(_) => panic!("Error opening test file at: {:?}", path), + }; + file.read_to_end(&mut data).expect("Error reading test file"); + errors.append(&mut runner(&data, start_stop_hook)); +} + pub fn run_test_file( path: &Path, runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, diff --git a/json/src/test/mod.rs b/json/src/test/mod.rs index 8f95a9aec49..e6148e0d08c 100644 --- a/json/src/test/mod.rs +++ b/json/src/test/mod.rs @@ -45,6 +45,7 @@ pub struct DifficultyTestCase { #[serde(rename="currentBlockNumber")] pub current_block_number: Uint, } + /// Blockchain test deserializer. #[derive(Debug, PartialEq, Deserialize)] pub struct DifficultyTest(BTreeMap); @@ -64,3 +65,59 @@ impl DifficultyTest { serde_json::from_reader(reader) } } + +/// Test to skip (only if issue ongoing) +#[derive(Debug, PartialEq, Deserialize)] +pub struct SkipStates { + /// Block tests + pub block: Vec, + /// State tests + pub state: Vec, + +} + +/// Block test to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct BlockSkipStates { + /// Issue reference. + pub reference: String, + /// Test failing name. + pub failing: String, + /// Items failing for the test. + pub subtests: Vec, +} + +/// State test to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateSkipStates { + /// Issue reference. + pub reference: String, + /// Test failing name. + pub failing: String, + /// Items failing for the test. + pub subtests: BTreeMap +} + +/// State subtest to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateSkipSubStates { + /// State test number of this item. Or '*' for all state. + pub subnumbers: Vec, + /// Chain for this items. + pub chain: String, +} + +impl SkipStates { + /// Loads skip states from json. + pub fn load(reader: R) -> Result where R: Read { + serde_json::from_reader(reader) + } + + /// Empty skip states. + pub fn empty() -> Self { + SkipStates { + block: Vec::new(), + state: Vec::new(), + } + } +} diff --git a/test.sh b/test.sh index 82e05d954c5..d1bb9e8a26e 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,7 @@ #!/bin/sh # Running Parity Full Test Suite -FEATURES="json-tests" +FEATURES="json-tests,ci-skip-issue" OPTIONS="--release" VALIDATE=1