From 3b9b1a8f147f870878e88d5edf7e5fca74988c3e Mon Sep 17 00:00:00 2001 From: Afri Schoedon <5chdn@users.noreply.github.com> Date: Wed, 13 Feb 2019 11:00:41 +0100 Subject: [PATCH] Backports for Beta 2.3.3 (#10333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * version: bump beta to 2.3.3 * import rpc transactions sequentially (#10051) * import rpc transactions sequentially * use impl trait in argument position, renamed ProspectiveDispatcher to WithPostSign * grouped imports * integrates PostSign with ProspectiveSigner * fix spaces, removed unnecessary type cast and duplicate polling * clean up code style * Apply suggestions from code review * Fix Windows build (#10284) * Don't run the CPP example on CI (#10285) * Don't run the CPP example on CI * Add comment * CI optimizations (#10297) * CI optimizations * fix stripping * new dockerfile * no need n submodule upd * review * moved dockerfile * it becomes large * onchain update depends on s3 * fix dependency * fix cache status * fix cache status * new cache status * fix publish job (#10317) * fix publish job * dashes and colonels * Add Statetest support for Constantinople Fix (#10323) * Update Ethereum tests repo to v6.0.0-beta.3 tag * Add spec for St.Peter's / ConstantinopleFix statetests * Properly handle check_epoch_end_signal errors (#10015) * Make check_epoch_end_signal to only use immutable data * Move check_epoch_end_signals out of commit_block * Make check_epoch_end_signals possible to fail * Actually return the error from check_epoch_end_signals * Remove a clone * Fix import error * cargo: fix compilation * fix(add helper for timestamp overflows) (#10330) * fix(add helper timestamp overflows) * fix(simplify code) * fix(make helper private) * Remove CallContract and RegistryInfo re-exports from `ethcore/client` (#10205) * Remove re-export of `CallContract` and `RegistryInfo` from `ethcore/client` * Remove CallContract and RegistryInfo re-exports again This was missed while fixing merge conflicts * fix(docker): fix not receives SIGINT (#10059) * fix(docker): fix not receives SIGINT * fix: update with reviews * update with review * update * update * snap: official image / test (#10168) * official image / test * fix / test * bit more necromancy * fix paths * add source bin/df /test * add source bin/df /test2 * something w paths /test * something w paths /test * add source-type /test * show paths /test * copy plugin /test * plugin -> nil * install rhash * no questions while installing rhash * publish snap only for release * Don't add discovery initiators to the node table (#10305) * Don't add discovery initiators to the node table * Use enums for tracking state of the nodes in discovery * Dont try to ping ourselves * Fix minor nits * Update timeouts when observing an outdated node * Extracted update_bucket_record from update_node * Fixed typo * Fix two final nits from @todr * Extract CallContract and RegistryInfo traits into their own crate (#10178) * Create call-contract crate * Add license * First attempt at using extracted CallContract trait * Remove unneeded `extern crate` calls * Move RegistryInfo trait into call-contract crate * Move service-transaction-checker from ethcore to ethcore-miner * Update Cargo.lock file * Re-export call_contract * Merge CallContract and RegistryInfo imports * Remove commented code * Add documentation to call_contract crate * Add TODO for removal of re-exports * Update call-contract crate description Co-Authored-By: HCastano * Rename call-contract crate to ethcore-call-contract * Remove CallContract and RegistryInfo re-exports from `ethcore/client` (#10205) * Remove re-export of `CallContract` and `RegistryInfo` from `ethcore/client` * Remove CallContract and RegistryInfo re-exports again This was missed while fixing merge conflicts * fixed: types::transaction::SignedTransaction; (#10229) * fix daemonize dependency * fix build * change docker image based on debian instead of ubuntu due to the chan… (#10336) * change docker image based on debian instead of ubuntu due to the changes of the build container * role back docker build image and docker deploy image to ubuntu:xenial based (#10338) * perform stripping during build (#10208) * perform stripping during build (#10208) * perform stripping during build * var RUSTFLAGS --- .gitlab-ci.yml | 97 +++++++--- Cargo.lock | 32 +++- Cargo.toml | 13 +- ethcore/Cargo.toml | 1 + ethcore/call-contract/Cargo.toml | 11 ++ ethcore/call-contract/src/call_contract.rs | 33 ++++ ethcore/call-contract/src/lib.rs | 27 +++ ethcore/private-tx/Cargo.toml | 1 + ethcore/private-tx/src/lib.rs | 4 +- ethcore/res/ethereum/st_peters_test.json | 65 +++++++ ethcore/res/ethereum/tests | 2 +- ethcore/src/client/client.rs | 73 +++++--- ethcore/src/client/evm_test_client.rs | 1 + ethcore/src/client/mod.rs | 2 +- ethcore/src/client/test_client.rs | 3 +- ethcore/src/client/traits.rs | 13 +- ethcore/src/engines/mod.rs | 3 + ethcore/src/engines/validator_set/contract.rs | 3 +- ethcore/src/error.rs | 3 + ethcore/src/ethereum/mod.rs | 6 + ethcore/src/json_tests/state.rs | 2 +- ethcore/src/lib.rs | 1 + ethcore/src/machine.rs | 3 +- ethcore/src/miner/miner.rs | 3 +- ethcore/src/miner/mod.rs | 4 +- ethcore/src/miner/pool_client.rs | 5 +- ethcore/src/state/mod.rs | 7 + ethcore/src/tx_filter.rs | 3 +- ethcore/src/verification/canon_verifier.rs | 3 +- ethcore/src/verification/mod.rs | 3 +- ethcore/src/verification/noop_verifier.rs | 3 +- ethcore/src/verification/verification.rs | 38 +++- ethcore/src/verification/verifier.rs | 3 +- json/src/spec/spec.rs | 1 + miner/Cargo.toml | 4 + .../res/contracts/service_transaction.json | 0 miner/src/lib.rs | 7 + .../src}/service_transaction_checker.rs | 3 +- parity/lib.rs | 3 +- parity/run.rs | 3 +- rpc/src/v1/helpers/dispatch.rs | 176 +++++++++++++----- rpc/src/v1/impls/personal.rs | 48 +++-- scripts/docker/hub/Dockerfile | 27 ++- scripts/gitlab/build-unix.sh | 23 +-- scripts/gitlab/cargo-audit.sh | 7 - ...blish-awss3.sh => publish-onnet-update.sh} | 16 -- scripts/gitlab/publish-snap.sh | 14 +- scripts/gitlab/test-all.sh | 5 - scripts/snap/snapcraft.template.yaml | 6 +- secret-store/Cargo.toml | 1 + secret-store/src/acl_storage.rs | 3 +- secret-store/src/key_server_set.rs | 3 +- secret-store/src/lib.rs | 1 + secret-store/src/listener/service_contract.rs | 3 +- secret-store/src/trusted_client.rs | 3 +- test.sh | 23 +-- util/network-devp2p/Cargo.toml | 1 + util/network-devp2p/src/discovery.rs | 138 ++++++++++---- util/network-devp2p/src/lib.rs | 1 + util/version/Cargo.toml | 2 +- 60 files changed, 690 insertions(+), 303 deletions(-) create mode 100644 ethcore/call-contract/Cargo.toml create mode 100644 ethcore/call-contract/src/call_contract.rs create mode 100644 ethcore/call-contract/src/lib.rs create mode 100644 ethcore/res/ethereum/st_peters_test.json rename {ethcore => miner}/res/contracts/service_transaction.json (100%) rename {ethcore/src/miner => miner/src}/service_transaction_checker.rs (96%) delete mode 100755 scripts/gitlab/cargo-audit.sh rename scripts/gitlab/{publish-awss3.sh => publish-onnet-update.sh} (73%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1a1c979987..a4109ad8597 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,11 +2,14 @@ stages: - test - build - publish + - publish-onchain - optional image: parity/rust:gitlab-ci variables: + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: recursive CI_SERVER_NAME: "GitLab CI" CARGO_HOME: "${CI_PROJECT_DIR}/.cargo" CARGO_TARGET: x86_64-unknown-linux-gnu @@ -40,25 +43,29 @@ test-linux: variables: RUN_TESTS: all script: - - scripts/gitlab/test-all.sh stable + - scripts/gitlab/test-all.sh + - sccache -s tags: - - rust-stable + - linux-docker test-audit: stage: test script: - - scripts/gitlab/cargo-audit.sh + - set -e + - set -u + - cargo audit tags: - - rust-stable + - linux-docker build-linux: stage: build only: *releaseable_branches script: - scripts/gitlab/build-unix.sh + - sccache -s <<: *collect_artifacts tags: - - rust-stable + - linux-docker build-darwin: stage: build @@ -96,9 +103,9 @@ publish-docker: - scripts/gitlab/publish-docker.sh parity publish-snap: - stage: publish + stage: optional #publish only: *releaseable_branches - image: parity/snapcraft:gitlab-ci + image: snapcore/snapcraft variables: BUILD_ARCH: amd64 cache: {} @@ -112,19 +119,66 @@ publish-snap: allow_failure: true <<: *collect_artifacts -publish-awss3: - stage: publish +publish-onnet-update: + stage: publish-onchain only: *releaseable_branches - cache: {} + cache: {} dependencies: - build-linux - build-darwin - build-windows + - publish-awss3-release before_script: *determine_version script: - - scripts/gitlab/publish-awss3.sh + - scripts/gitlab/publish-onnet-update.sh tags: - - shell + - linux-docker + +# configures aws for fast uploads/syncs +.s3-before-script: &s3-before-script + before_script: + - mkdir -p ${HOME}/.aws + - | + cat > ${HOME}/.aws/config <"] @@ -34,6 +34,7 @@ ethcore = { path = "ethcore", features = ["parity"] } parity-bytes = "0.1" common-types = { path = "ethcore/types" } ethcore-blockchain = { path = "ethcore/blockchain" } +ethcore-call-contract = { path = "ethcore/call-contract"} ethcore-db = { path = "ethcore/db" } ethcore-io = { path = "util/io" } ethcore-light = { path = "ethcore/light" } @@ -79,12 +80,12 @@ ipnetwork = "0.12.6" tempdir = "0.3" fake-fetch = { path = "util/fake-fetch" } -[target.'cfg(windows)'.dependencies] -winapi = { version = "0.3.4", features = ["winsock2", "winuser", "shellapi"] } - [target.'cfg(not(windows))'.dependencies] daemonize = "0.3" +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3.4", features = ["winsock2", "winuser", "shellapi"] } + [features] miner-debug = ["ethcore/miner-debug"] json-tests = ["ethcore/json-tests"] @@ -132,6 +133,10 @@ members = [ "evmbin", "parity-clib", "whisper/cli", + "util/triehash-ethereum", + "util/keccak-hasher", + "util/patricia-trie-ethereum", + "util/fastmap" ] [patch.crates-io] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 5179584c112..6adb69f67a2 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -21,6 +21,7 @@ ethabi-derive = "6.0" ethash = { path = "../ethash" } ethcore-blockchain = { path = "./blockchain" } ethcore-bloom-journal = { path = "../util/bloom" } +ethcore-call-contract = { path = "./call-contract" } ethcore-db = { path = "./db" } ethcore-io = { path = "../util/io" } ethcore-miner = { path = "../miner" } diff --git a/ethcore/call-contract/Cargo.toml b/ethcore/call-contract/Cargo.toml new file mode 100644 index 00000000000..068434a1dec --- /dev/null +++ b/ethcore/call-contract/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ethcore-call-contract" +version = "0.1.0" +license = "GPL-3.0" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +types = { path = "../types", package = "common-types" } +ethereum-types = "0.4" +bytes = { version = "0.1", package = "parity-bytes" } diff --git a/ethcore/call-contract/src/call_contract.rs b/ethcore/call-contract/src/call_contract.rs new file mode 100644 index 00000000000..8b042f0833c --- /dev/null +++ b/ethcore/call-contract/src/call_contract.rs @@ -0,0 +1,33 @@ +// Copyright 2015-2019 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Provides CallContract and RegistryInfo traits + +use bytes::Bytes; +use ethereum_types::Address; +use types::ids::BlockId; + +/// Provides `call_contract` method +pub trait CallContract { + /// Like `call`, but with various defaults. Designed to be used for calling contracts. + fn call_contract(&self, id: BlockId, address: Address, data: Bytes) -> Result; +} + +/// Provides information on a blockchain service and it's registry +pub trait RegistryInfo { + /// Get the address of a particular blockchain service, if available. + fn registry_address(&self, name: String, block: BlockId) -> Option
; +} diff --git a/ethcore/call-contract/src/lib.rs b/ethcore/call-contract/src/lib.rs new file mode 100644 index 00000000000..1cbfb11378e --- /dev/null +++ b/ethcore/call-contract/src/lib.rs @@ -0,0 +1,27 @@ +// Copyright 2015-2019 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +#![warn(missing_docs)] + +//! Call Contract module +//! +//! This crate exposes traits required to call contracts at particular block. +//! All utilities that depend on on-chain data should use those traits to access it. + +pub mod call_contract; + +// Re-export +pub use self::call_contract::*; diff --git a/ethcore/private-tx/Cargo.toml b/ethcore/private-tx/Cargo.toml index 1dd328e11cb..5151deee448 100644 --- a/ethcore/private-tx/Cargo.toml +++ b/ethcore/private-tx/Cargo.toml @@ -12,6 +12,7 @@ ethabi = "6.0" ethabi-contract = "6.0" ethabi-derive = "6.0" ethcore = { path = ".." } +ethcore-call-contract = { path = "../call-contract" } ethcore-io = { path = "../../util/io" } ethcore-miner = { path = "../../miner" } ethereum-types = "0.4" diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index f55a30334a1..3ab39d9e4ea 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -28,6 +28,7 @@ mod error; extern crate common_types as types; extern crate ethabi; extern crate ethcore; +extern crate ethcore_call_contract as call_contract; extern crate ethcore_io as io; extern crate ethcore_miner; extern crate ethereum_types; @@ -82,11 +83,12 @@ use types::transaction::{SignedTransaction, Transaction, Action, UnverifiedTrans use ethcore::{contract_address as ethcore_contract_address}; use ethcore::client::{ Client, ChainNotify, NewBlocks, ChainMessageType, ClientIoMessage, BlockId, - CallContract, Call, BlockInfo + Call, BlockInfo }; use ethcore::account_provider::AccountProvider; use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; use ethcore::trace::{Tracer, VMTracer}; +use call_contract::CallContract; use rustc_hex::FromHex; use ethkey::Password; use ethabi::FunctionOutputDecoder; diff --git a/ethcore/res/ethereum/st_peters_test.json b/ethcore/res/ethereum/st_peters_test.json new file mode 100644 index 00000000000..ee88008f668 --- /dev/null +++ b/ethcore/res/ethereum/st_peters_test.json @@ -0,0 +1,65 @@ +{ + "name": "St. Peter's (test)", + "engine": { + "Ethash": { + "params": { + "minimumDifficulty": "0x020000", + "difficultyBoundDivisor": "0x0800", + "durationLimit": "0x0d", + "blockReward": "0x1BC16D674EC80000", + "homesteadTransition": "0x0", + "eip100bTransition": "0x0", + "difficultyBombDelays": { + "0": 5000000 + } + } + } + }, + "params": { + "gasLimitBoundDivisor": "0x0400", + "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", + "accountStartNonce": "0x00", + "maximumExtraDataSize": "0x20", + "minGasLimit": "0x1388", + "networkID" : "0x1", + "maxCodeSize": 24576, + "maxCodeSizeTransition": "0x0", + "eip150Transition": "0x0", + "eip160Transition": "0x0", + "eip161abcTransition": "0x0", + "eip161dTransition": "0x0", + "eip140Transition": "0x0", + "eip211Transition": "0x0", + "eip214Transition": "0x0", + "eip155Transition": "0x0", + "eip658Transition": "0x0", + "eip145Transition": "0x0", + "eip1014Transition": "0x0", + "eip1052Transition": "0x0", + "eip1283DisableTransition": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "nonce": "0x0000000000000042", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty": "0x400000000", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa", + "gasLimit": "0x1388" + }, + "accounts": { + "0000000000000000000000000000000000000001": { "balance": "1", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, + "0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, + "0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, + "0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, + "0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 20 } } } }, + "0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } }, + "0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 40000, "word": 0 } } } }, + "0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } } + } +} diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index 420f443477c..725dbc73a54 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit 420f443477caa8516f1f9ee8122fafc3415c0f34 +Subproject commit 725dbc73a54649e22a00330bd0f4d6699a5060e5 diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 995b11520f6..bcac9bbad30 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -23,6 +23,7 @@ use std::time::{Instant, Duration}; use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert}; use bytes::Bytes; +use call_contract::{CallContract, RegistryInfo}; use ethcore_miner::pool::VerifiedTransaction; use ethereum_types::{H256, Address, U256}; use evm::Schedule; @@ -46,8 +47,8 @@ use vm::{EnvInfo, LastHashes}; use block::{IsBlock, LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock}; use client::ancient_import::AncientVerifier; use client::{ - Nonce, Balance, ChainInfo, BlockInfo, CallContract, TransactionInfo, - RegistryInfo, ReopenBlock, PrepareOpenBlock, ScheduleInfo, ImportSealedBlock, + Nonce, Balance, ChainInfo, BlockInfo, TransactionInfo, + ReopenBlock, PrepareOpenBlock, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, StateOrBlock, StateInfo, StateClient, Call, AccountData, BlockChain as BlockChainTrait, BlockProducer, SealedBlockImporter, ClientIoMessage, @@ -59,7 +60,8 @@ use client::{ IoClient, BadBlocks, }; use client::bad_blocks; -use engines::{EthEngine, EpochTransition, ForkChoice}; +use engines::{EthEngine, EpochTransition, ForkChoice, EngineError}; +use engines::epoch::PendingTransition; use error::{ ImportErrorKind, ExecutionError, CallError, BlockError, QueueError, QueueErrorKind, Error as EthcoreError, EthcoreResult, ErrorKind as EthcoreErrorKind @@ -291,8 +293,8 @@ impl Importer { continue; } - match self.check_and_lock_block(block, client) { - Ok(closed_block) => { + match self.check_and_lock_block(&bytes, block, client) { + Ok((closed_block, pending)) => { if self.engine.is_proposal(&header) { self.block_queue.mark_as_good(&[hash]); proposed_blocks.push(bytes); @@ -301,7 +303,7 @@ impl Importer { let transactions_len = closed_block.transactions().len(); - let route = self.commit_block(closed_block, &header, encoded::Block::new(bytes), client); + let route = self.commit_block(closed_block, &header, encoded::Block::new(bytes), pending, client); import_results.push(route); client.report.write().accrue_block(&header, transactions_len); @@ -353,7 +355,7 @@ impl Importer { imported } - fn check_and_lock_block(&self, block: PreverifiedBlock, client: &Client) -> EthcoreResult { + fn check_and_lock_block(&self, bytes: &[u8], block: PreverifiedBlock, client: &Client) -> EthcoreResult<(LockedBlock, Option)> { let engine = &*self.engine; let header = block.header.clone(); @@ -437,7 +439,15 @@ impl Importer { bail!(e); } - Ok(locked_block) + let pending = self.check_epoch_end_signal( + &header, + bytes, + locked_block.receipts(), + locked_block.state().db(), + client + )?; + + Ok((locked_block, pending)) } /// Import a block with transaction receipts. @@ -469,7 +479,8 @@ impl Importer { // it is for reconstructing the state transition. // // The header passed is from the original block data and is sealed. - fn commit_block(&self, block: B, header: &Header, block_data: encoded::Block, client: &Client) -> ImportRoute where B: Drain { + // TODO: should return an error if ImportRoute is none, issue #9910 + fn commit_block(&self, block: B, header: &Header, block_data: encoded::Block, pending: Option, client: &Client) -> ImportRoute where B: Drain { let hash = &header.hash(); let number = header.number(); let parent = header.parent_hash(); @@ -524,15 +535,9 @@ impl Importer { // check epoch end signal, potentially generating a proof on the current // state. - self.check_epoch_end_signal( - &header, - block_data.raw(), - &receipts, - &state, - &chain, - &mut batch, - client - ); + if let Some(pending) = pending { + chain.insert_pending_transition(&mut batch, header.hash(), pending); + } state.journal_under(&mut batch, number, hash).expect("DB commit failed"); @@ -587,10 +592,8 @@ impl Importer { block_bytes: &[u8], receipts: &[Receipt], state_db: &StateDB, - chain: &BlockChain, - batch: &mut DBTransaction, client: &Client, - ) { + ) -> EthcoreResult> { use engines::EpochChange; let hash = header.hash(); @@ -601,7 +604,6 @@ impl Importer { match self.engine.signals_epoch_end(header, auxiliary) { EpochChange::Yes(proof) => { - use engines::epoch::PendingTransition; use engines::Proof; let proof = match proof { @@ -638,11 +640,9 @@ impl Importer { .transact(&transaction, options); let res = match res { - Err(ExecutionError::Internal(e)) => - Err(format!("Internal error: {}", e)), Err(e) => { trace!(target: "client", "Proved call failed: {}", e); - Ok((Vec::new(), state.drop().1.extract_proof())) + Err(e.to_string()) } Ok(res) => Ok((res.output, state.drop().1.extract_proof())), }; @@ -655,7 +655,7 @@ impl Importer { Err(e) => { warn!(target: "client", "Failed to generate transition proof for block {}: {}", hash, e); warn!(target: "client", "Snapshots produced by this client may be incomplete"); - Vec::new() + return Err(EngineError::FailedSystemCall(e).into()) } } } @@ -663,13 +663,13 @@ impl Importer { debug!(target: "client", "Block {} signals epoch end.", hash); - let pending = PendingTransition { proof: proof }; - chain.insert_pending_transition(batch, hash, pending); + Ok(Some(PendingTransition { proof: proof })) }, - EpochChange::No => {}, + EpochChange::No => Ok(None), EpochChange::Unsure(_) => { warn!(target: "client", "Detected invalid engine implementation."); warn!(target: "client", "Engine claims to require more block data, but everything provided."); + Err(EngineError::InvalidEngine.into()) } } } @@ -2328,7 +2328,20 @@ impl ImportSealedBlock for Client { let block_data = block.rlp_bytes(); - let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self); + let pending = self.importer.check_epoch_end_signal( + &header, + &block_data, + block.receipts(), + block.state().db(), + self + )?; + let route = self.importer.commit_block( + block, + &header, + encoded::Block::new(block_data), + pending, + self + ); trace!(target: "client", "Imported sealed block #{} ({})", header.number(), hash); self.state_db.write().sync_cache(&route.enacted, &route.retracted, false); route diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index 7a2158209b5..a65e2b313ec 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -93,6 +93,7 @@ impl<'a> EvmTestClient<'a> { ForkSpec::EIP158 => Some(ethereum::new_eip161_test()), ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()), ForkSpec::Constantinople => Some(ethereum::new_constantinople_test()), + ForkSpec::ConstantinopleFix => Some(ethereum::new_constantinople_fix_test()), ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()), ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None, } diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 9c534d470c6..7d9053ca60b 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -36,7 +36,7 @@ pub use self::io_message::ClientIoMessage; pub use self::test_client::{TestBlockChainClient, EachBlockWith}; pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType}; pub use self::traits::{ - Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, PrepareOpenBlock, CallContract, TransactionInfo, RegistryInfo, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, + Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, PrepareOpenBlock, TransactionInfo, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, StateOrBlock, StateClient, Call, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter, BadBlocks, }; pub use state::StateInfo; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 1ca826097f0..5236f7cd409 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -49,8 +49,9 @@ use types::views::BlockView; use vm::Schedule; use block::{OpenBlock, SealedBlock, ClosedBlock}; +use call_contract::{CallContract, RegistryInfo}; use client::{ - Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, CallContract, TransactionInfo, RegistryInfo, + Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, TransactionInfo, PrepareOpenBlock, BlockChainClient, BlockChainInfo, BlockStatus, BlockId, Mode, TransactionId, UncleId, TraceId, TraceFilter, LastHashes, CallAnalytics, ProvingBlockChainClient, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, StateOrBlock, diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index ff0148beefc..b273bd3dcb2 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use blockchain::{BlockReceipts, TreeRoute}; use bytes::Bytes; +use call_contract::{CallContract, RegistryInfo}; use ethcore_miner::pool::VerifiedTransaction; use ethereum_types::{H256, U256, Address}; use evm::Schedule; @@ -157,12 +158,6 @@ pub trait StateClient { /// Provides various blockchain information, like block header, chain state etc. pub trait BlockChain: ChainInfo + BlockInfo + TransactionInfo {} -/// Provides information on a blockchain service and it's registry -pub trait RegistryInfo { - /// Get the address of a particular blockchain service, if available. - fn registry_address(&self, name: String, block: BlockId) -> Option
; -} - // FIXME Why these methods belong to BlockChainClient and not MiningBlockChainClient? /// Provides methods to import block into blockchain pub trait ImportBlock { @@ -170,12 +165,6 @@ pub trait ImportBlock { fn import_block(&self, block: Unverified) -> EthcoreResult; } -/// Provides `call_contract` method -pub trait CallContract { - /// Like `call`, but with various defaults. Designed to be used for calling contracts. - fn call_contract(&self, id: BlockId, address: Address, data: Bytes) -> Result; -} - /// Provides `call` and `call_many` methods pub trait Call { /// Type representing chain state diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index 64d05e5c466..deb85cd35f6 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -81,6 +81,8 @@ pub enum EngineError { MalformedMessage(String), /// Requires client ref, but none registered. RequiresClient, + /// Invalid engine specification or implementation. + InvalidEngine, } impl fmt::Display for EngineError { @@ -96,6 +98,7 @@ impl fmt::Display for EngineError { FailedSystemCall(ref msg) => format!("Failed to make system call: {}", msg), MalformedMessage(ref msg) => format!("Received malformed consensus message: {}", msg), RequiresClient => format!("Call requires client but none registered"), + InvalidEngine => format!("Invalid engine specification or implementation"), }; f.write_fmt(format_args!("Engine error ({})", msg)) diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index 714a092d7e5..b13ebdd105d 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -145,7 +145,8 @@ mod tests { use miner::MinerService; use types::ids::BlockId; use test_helpers::generate_dummy_client_with_spec_and_accounts; - use client::{BlockChainClient, ChainInfo, BlockInfo, CallContract}; + use call_contract::CallContract; + use client::{BlockChainClient, ChainInfo, BlockInfo}; use super::super::ValidatorSet; use super::ValidatorContract; diff --git a/ethcore/src/error.rs b/ethcore/src/error.rs index 5c434d48706..712a35f4bec 100644 --- a/ethcore/src/error.rs +++ b/ethcore/src/error.rs @@ -90,6 +90,8 @@ pub enum BlockError { InvalidNumber(Mismatch), /// Block number isn't sensible. RidiculousNumber(OutOfBounds), + /// Timestamp header overflowed + TimestampOverflow, /// Too many transactions from a particular address. TooManyTransactions(Address), /// Parent given is unknown. @@ -139,6 +141,7 @@ impl fmt::Display for BlockError { UnknownParent(ref hash) => format!("Unknown parent: {}", hash), UnknownUncleParent(ref hash) => format!("Unknown uncle parent: {}", hash), UnknownEpochTransition(ref num) => format!("Unknown transition to epoch number: {}", num), + TimestampOverflow => format!("Timestamp overflow"), TooManyTransactions(ref address) => format!("Too many transactions from: {}", address), }; diff --git a/ethcore/src/ethereum/mod.rs b/ethcore/src/ethereum/mod.rs index 104155532cb..b7c60789a3d 100644 --- a/ethcore/src/ethereum/mod.rs +++ b/ethcore/src/ethereum/mod.rs @@ -148,6 +148,9 @@ pub fn new_byzantium_test() -> Spec { load(None, include_bytes!("../../res/ether /// Create a new Foundation Constantinople era spec. pub fn new_constantinople_test() -> Spec { load(None, include_bytes!("../../res/ethereum/constantinople_test.json")) } +/// Create a new Foundation St. Peter's (Contantinople Fix) era spec. +pub fn new_constantinople_fix_test() -> Spec { load(None, include_bytes!("../../res/ethereum/st_peters_test.json")) } + /// Create a new Musicoin-MCIP3-era spec. pub fn new_mcip3_test() -> Spec { load(None, include_bytes!("../../res/ethereum/mcip3_test.json")) } @@ -168,6 +171,9 @@ pub fn new_byzantium_test_machine() -> EthereumMachine { load_machine(include_by /// Create a new Foundation Constantinople era spec. pub fn new_constantinople_test_machine() -> EthereumMachine { load_machine(include_bytes!("../../res/ethereum/constantinople_test.json")) } +/// Create a new Foundation St. Peter's (Contantinople Fix) era spec. +pub fn new_constantinople_fix_test_machine() -> EthereumMachine { load_machine(include_bytes!("../../res/ethereum/st_peters_test.json")) } + /// Create a new Musicoin-MCIP3-era spec. pub fn new_mcip3_test_machine() -> EthereumMachine { load_machine(include_bytes!("../../res/ethereum/mcip3_test.json")) } diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 45d9c035f69..33fc6d522d5 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -165,6 +165,7 @@ mod state_tests { declare_test!{GeneralStateTest_stRefundTest, "GeneralStateTests/stRefundTest/"} declare_test!{GeneralStateTest_stReturnDataTest, "GeneralStateTests/stReturnDataTest/"} declare_test!{GeneralStateTest_stRevertTest, "GeneralStateTests/stRevertTest/"} + declare_test!{GeneralStateTest_stSStoreTest, "GeneralStateTests/stSStoreTest/"} declare_test!{GeneralStateTest_stShift, "GeneralStateTests/stShift/"} declare_test!{GeneralStateTest_stSolidityTest, "GeneralStateTests/stSolidityTest/"} declare_test!{GeneralStateTest_stSpecialTest, "GeneralStateTests/stSpecialTest/"} @@ -177,7 +178,6 @@ mod state_tests { declare_test!{GeneralStateTest_stZeroCallsRevert, "GeneralStateTests/stZeroCallsRevert/"} declare_test!{GeneralStateTest_stZeroCallsTest, "GeneralStateTests/stZeroCallsTest/"} declare_test!{GeneralStateTest_stZeroKnowledge, "GeneralStateTests/stZeroKnowledge/"} - declare_test!{GeneralStateTest_stSStoreTest, "GeneralStateTests/stSStoreTest/"} // Attempts to send a transaction that requires more than current balance: // Tx: diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 89f6f8e09e5..2e291aecf40 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -66,6 +66,7 @@ extern crate ethabi; extern crate ethash; extern crate ethcore_blockchain as blockchain; extern crate ethcore_bloom_journal as bloom_journal; +extern crate ethcore_call_contract as call_contract; extern crate ethcore_db as db; extern crate ethcore_io as io; extern crate ethcore_miner; diff --git a/ethcore/src/machine.rs b/ethcore/src/machine.rs index 4ceab9bf028..a12702e846f 100644 --- a/ethcore/src/machine.rs +++ b/ethcore/src/machine.rs @@ -30,7 +30,8 @@ use vm::{EnvInfo, Schedule, CreateContractAddress}; use block::{ExecutedBlock, IsBlock}; use builtin::Builtin; -use client::{BlockInfo, CallContract}; +use call_contract::CallContract; +use client::BlockInfo; use error::Error; use executive::Executive; use spec::CommonParams; diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index d30c5a67d04..998ce628041 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use ansi_term::Colour; use bytes::Bytes; +use call_contract::CallContract; use ethcore_miner::gas_pricer::GasPricer; use ethcore_miner::pool::{self, TransactionQueue, VerifiedTransaction, QueueStatus, PrioritizationStrategy}; #[cfg(feature = "work-notify")] @@ -48,7 +49,7 @@ use using_queue::{UsingQueue, GetAction}; use account_provider::{AccountProvider, SignError as AccountError}; use block::{ClosedBlock, IsBlock, SealedBlock}; use client::{ - BlockChain, ChainInfo, CallContract, BlockProducer, SealedBlockImporter, Nonce, TransactionInfo, TransactionId + BlockChain, ChainInfo, BlockProducer, SealedBlockImporter, Nonce, TransactionInfo, TransactionId }; use client::{BlockId, ClientIoMessage}; use engines::{EthEngine, Seal}; diff --git a/ethcore/src/miner/mod.rs b/ethcore/src/miner/mod.rs index 2696306e71d..4f5ba4c1f84 100644 --- a/ethcore/src/miner/mod.rs +++ b/ethcore/src/miner/mod.rs @@ -20,7 +20,6 @@ //! Keeps track of transactions and currently sealed pending block. mod miner; -mod service_transaction_checker; pub mod pool_client; #[cfg(feature = "stratum")] @@ -43,8 +42,9 @@ use types::header::Header; use types::receipt::RichReceipt; use block::SealedBlock; +use call_contract::{CallContract, RegistryInfo}; use client::{ - CallContract, RegistryInfo, ScheduleInfo, + ScheduleInfo, BlockChain, BlockProducer, SealedBlockImporter, ChainInfo, AccountData, Nonce, }; diff --git a/ethcore/src/miner/pool_client.rs b/ethcore/src/miner/pool_client.rs index e748d9bc201..a75454a3fd5 100644 --- a/ethcore/src/miner/pool_client.rs +++ b/ethcore/src/miner/pool_client.rs @@ -25,6 +25,7 @@ use std::{ use ethereum_types::{H256, U256, Address}; use ethcore_miner::pool; use ethcore_miner::pool::client::NonceClient; +use ethcore_miner::service_transaction_checker::ServiceTransactionChecker; use types::transaction::{ self, UnverifiedTransaction, @@ -34,10 +35,10 @@ use types::header::Header; use parking_lot::RwLock; use account_provider::AccountProvider; -use client::{TransactionId, BlockInfo, CallContract, Nonce}; +use call_contract::CallContract; +use client::{TransactionId, BlockInfo, Nonce}; use engines::EthEngine; use miner; -use miner::service_transaction_checker::ServiceTransactionChecker; use transaction_ext::Transaction; /// Cache for state nonces. diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 27274c0e619..8cc06fa83cc 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -1289,6 +1289,13 @@ impl fmt::Debug for State { } } +impl State { + /// Get a reference to the underlying state DB. + pub fn db(&self) -> &StateDB { + &self.db + } +} + // TODO: cloning for `State` shouldn't be possible in general; Remove this and use // checkpoints where possible. impl Clone for State { diff --git a/ethcore/src/tx_filter.rs b/ethcore/src/tx_filter.rs index 403e4f7bcbe..3f32ab365a8 100644 --- a/ethcore/src/tx_filter.rs +++ b/ethcore/src/tx_filter.rs @@ -20,7 +20,8 @@ use ethereum_types::{H256, U256, Address}; use lru_cache::LruCache; use ethabi::FunctionOutputDecoder; -use client::{BlockInfo, CallContract, BlockId}; +use call_contract::CallContract; +use client::{BlockInfo, BlockId}; use parking_lot::Mutex; use spec::CommonParams; use types::transaction::{Action, SignedTransaction}; diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/src/verification/canon_verifier.rs index 4a8ed111c96..03a1c7155f8 100644 --- a/ethcore/src/verification/canon_verifier.rs +++ b/ethcore/src/verification/canon_verifier.rs @@ -16,7 +16,8 @@ //! Canonical verifier. -use client::{BlockInfo, CallContract}; +use call_contract::CallContract; +use client::BlockInfo; use engines::EthEngine; use error::Error; use types::header::Header; diff --git a/ethcore/src/verification/mod.rs b/ethcore/src/verification/mod.rs index acffb8c0a54..5546bd60c91 100644 --- a/ethcore/src/verification/mod.rs +++ b/ethcore/src/verification/mod.rs @@ -28,7 +28,8 @@ pub use self::canon_verifier::CanonVerifier; pub use self::noop_verifier::NoopVerifier; pub use self::queue::{BlockQueue, Config as QueueConfig, VerificationQueue, QueueInfo}; -use client::{BlockInfo, CallContract}; +use call_contract::CallContract; +use client::BlockInfo; /// Verifier type. #[derive(Debug, PartialEq, Clone)] diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs index 727e99a66c0..d68f1eb8856 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/src/verification/noop_verifier.rs @@ -16,7 +16,8 @@ //! No-op verifier. -use client::{BlockInfo, CallContract}; +use call_contract::CallContract; +use client::BlockInfo; use engines::EthEngine; use error::Error; use types::header::Header; diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 31eb9dfa685..3f5008a2b86 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -32,13 +32,33 @@ use triehash::ordered_trie_root; use unexpected::{Mismatch, OutOfBounds}; use blockchain::*; -use client::{BlockInfo, CallContract}; +use call_contract::CallContract; +use client::BlockInfo; use engines::EthEngine; use error::{BlockError, Error}; use types::{BlockNumber, header::Header}; use types::transaction::SignedTransaction; use verification::queue::kind::blocks::Unverified; + +/// Returns `Ok` when the result less or equal to `i32::max_value` to prevent `SystemTime` to panic because +/// it is platform specific, may be i32 or i64. +/// +/// `Err Result { + let d1 = sys.duration_since(UNIX_EPOCH).map_err(|_| BlockError::TimestampOverflow)?; + let total_time = d1.checked_add(d2).ok_or(BlockError::TimestampOverflow)?; + + if total_time.as_secs() <= i32::max_value() as u64 { + Ok(sys + d2) + } else { + Err(BlockError::TimestampOverflow) + } +} + /// Preprocessed block data gathered in `verify_block_unordered` call pub struct PreverifiedBlock { /// Populated block header @@ -305,7 +325,7 @@ pub fn verify_header_params(header: &Header, engine: &EthEngine, is_full: bool, const ACCEPTABLE_DRIFT: Duration = Duration::from_secs(15); let max_time = SystemTime::now() + ACCEPTABLE_DRIFT; let invalid_threshold = max_time + ACCEPTABLE_DRIFT * 9; - let timestamp = UNIX_EPOCH + Duration::from_secs(header.timestamp()); + let timestamp = timestamp_checked_add(UNIX_EPOCH, Duration::from_secs(header.timestamp()))?; if timestamp > invalid_threshold { return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: Some(max_time), min: None, found: timestamp }))) @@ -327,8 +347,8 @@ fn verify_parent(header: &Header, parent: &Header, engine: &EthEngine) -> Result let gas_limit_divisor = engine.params().gas_limit_bound_divisor; if !engine.is_timestamp_valid(header.timestamp(), parent.timestamp()) { - let min = SystemTime::now() + Duration::from_secs(parent.timestamp() + 1); - let found = SystemTime::now() + Duration::from_secs(header.timestamp()); + let min = timestamp_checked_add(SystemTime::now(), Duration::from_secs(parent.timestamp().saturating_add(1)))?; + let found = timestamp_checked_add(SystemTime::now(), Duration::from_secs(header.timestamp()))?; return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: None, min: Some(min), found }))) } if header.number() != parent.number() + 1 { @@ -742,7 +762,8 @@ mod tests { check_fail_timestamp(family_test(&create_test_block_with_data(&header, &good_transactions, &good_uncles), engine, &bc), false); header = good.clone(); - header.set_timestamp(2450000000); + // will return `BlockError::TimestampOverflow` when timestamp > `i32::max_value()` + header.set_timestamp(i32::max_value() as u64); check_fail_timestamp(basic_test(&create_test_block_with_data(&header, &good_transactions, &good_uncles), engine), false); header = good.clone(); @@ -814,4 +835,11 @@ mod tests { check_fail(unordered_test(&create_test_block_with_data(&header, &bad_transactions, &[]), &engine), TooManyTransactions(keypair.address())); unordered_test(&create_test_block_with_data(&header, &good_transactions, &[]), &engine).unwrap(); } + + #[test] + fn checked_add_systime_dur() { + assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 + 1, 0)).is_err()); + assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64, 0)).is_ok()); + assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 - 1, 1_000_000_000)).is_ok()); + } } diff --git a/ethcore/src/verification/verifier.rs b/ethcore/src/verification/verifier.rs index a1a5d32bc95..76eb60b9a18 100644 --- a/ethcore/src/verification/verifier.rs +++ b/ethcore/src/verification/verifier.rs @@ -16,7 +16,8 @@ //! A generic verifier trait. -use client::{BlockInfo, CallContract}; +use call_contract::CallContract; +use client::BlockInfo; use engines::EthEngine; use error::Error; use types::header::Header; diff --git a/json/src/spec/spec.rs b/json/src/spec/spec.rs index f8f1217b479..68824cad99b 100644 --- a/json/src/spec/spec.rs +++ b/json/src/spec/spec.rs @@ -30,6 +30,7 @@ pub enum ForkSpec { Homestead, Byzantium, Constantinople, + ConstantinopleFix, EIP158ToByzantiumAt5, FrontierToHomesteadAt5, HomesteadToDaoAt5, diff --git a/miner/Cargo.toml b/miner/Cargo.toml index d37ed692c73..02e64ad4f8d 100644 --- a/miner/Cargo.toml +++ b/miner/Cargo.toml @@ -17,6 +17,10 @@ url = { version = "1", optional = true } ansi_term = "0.10" common-types = { path = "../ethcore/types" } error-chain = "0.12" +ethabi = "6.0" +ethabi-derive = "6.0" +ethabi-contract = "6.0" +ethcore-call-contract = { path = "../ethcore/call-contract" } ethereum-types = "0.4" futures = "0.1" heapsize = "0.4" diff --git a/ethcore/res/contracts/service_transaction.json b/miner/res/contracts/service_transaction.json similarity index 100% rename from ethcore/res/contracts/service_transaction.json rename to miner/res/contracts/service_transaction.json diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 3b2513f678e..921e6dbaadc 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -21,6 +21,8 @@ extern crate ansi_term; extern crate common_types as types; +extern crate ethabi; +extern crate ethcore_call_contract as call_contract; extern crate ethereum_types; extern crate futures; extern crate heapsize; @@ -33,6 +35,10 @@ extern crate price_info; extern crate rlp; extern crate transaction_pool as txpool; +#[macro_use] +extern crate ethabi_contract; +#[macro_use] +extern crate ethabi_derive; #[macro_use] extern crate error_chain; #[macro_use] @@ -52,5 +58,6 @@ pub mod external; pub mod gas_price_calibrator; pub mod gas_pricer; pub mod pool; +pub mod service_transaction_checker; #[cfg(feature = "work-notify")] pub mod work_notify; diff --git a/ethcore/src/miner/service_transaction_checker.rs b/miner/src/service_transaction_checker.rs similarity index 96% rename from ethcore/src/miner/service_transaction_checker.rs rename to miner/src/service_transaction_checker.rs index 8d4c461cea2..17776f1569e 100644 --- a/ethcore/src/miner/service_transaction_checker.rs +++ b/miner/src/service_transaction_checker.rs @@ -16,7 +16,8 @@ //! A service transactions contract checker. -use client::{RegistryInfo, CallContract, BlockId}; +use call_contract::{CallContract, RegistryInfo}; +use types::ids::BlockId; use types::transaction::SignedTransaction; use ethabi::FunctionOutputDecoder; diff --git a/parity/lib.rs b/parity/lib.rs index 14ec8f98ed2..b96995910c3 100644 --- a/parity/lib.rs +++ b/parity/lib.rs @@ -43,8 +43,9 @@ extern crate toml; extern crate blooms_db; extern crate cli_signer; extern crate common_types as types; -extern crate ethcore; extern crate parity_bytes as bytes; +extern crate ethcore; +extern crate ethcore_call_contract as call_contract; extern crate ethcore_db; extern crate ethcore_io as io; extern crate ethcore_light as light; diff --git a/parity/run.rs b/parity/run.rs index 8b4bfff5b95..d0034ed7e4b 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -21,8 +21,9 @@ use std::thread; use ansi_term::Colour; use bytes::Bytes; +use call_contract::CallContract; use ethcore::account_provider::{AccountProvider, AccountProviderSettings}; -use ethcore::client::{BlockId, CallContract, Client, Mode, DatabaseCompactionProfile, VMType, BlockChainClient, BlockInfo}; +use ethcore::client::{BlockId, Client, Mode, DatabaseCompactionProfile, VMType, BlockChainClient, BlockInfo}; use ethstore::ethkey; use ethcore::miner::{stratum, Miner, MinerService, MinerOptions}; use ethcore::snapshot::{self, SnapshotConfiguration}; diff --git a/rpc/src/v1/helpers/dispatch.rs b/rpc/src/v1/helpers/dispatch.rs index e6d0d3fef67..c26fde45f14 100644 --- a/rpc/src/v1/helpers/dispatch.rs +++ b/rpc/src/v1/helpers/dispatch.rs @@ -41,11 +41,11 @@ use types::basic_account::BasicAccount; use types::ids::BlockId; use jsonrpc_core::{BoxFuture, Result, Error}; -use jsonrpc_core::futures::{future, Future, Poll, Async}; +use jsonrpc_core::futures::{future, Future, Poll, Async, IntoFuture}; use jsonrpc_core::futures::future::Either; use v1::helpers::{errors, nonce, TransactionRequest, FilledTransactionRequest, ConfirmationPayload}; use v1::types::{ - H256 as RpcH256, H520 as RpcH520, Bytes as RpcBytes, + H520 as RpcH520, Bytes as RpcBytes, RichRawTransaction as RpcRichRawTransaction, ConfirmationPayload as RpcConfirmationPayload, ConfirmationResponse, @@ -69,12 +69,20 @@ pub trait Dispatcher: Send + Sync + Clone { fn fill_optional_fields(&self, request: TransactionRequest, default_sender: Address, force_nonce: bool) -> BoxFuture; - /// Sign the given transaction request without dispatching, fetching appropriate nonce. - fn sign(&self, accounts: Arc, filled: FilledTransactionRequest, password: SignWith) - -> BoxFuture>; + /// Sign the given transaction request, fetching appropriate nonce and executing the PostSign action + fn sign

( + &self, + accounts: Arc, + filled: FilledTransactionRequest, + password: SignWith, + post_sign: P + ) -> BoxFuture + where + P: PostSign + 'static, + ::Future: Send; /// Converts a `SignedTransaction` into `RichRawTransaction` - fn enrich(&self, SignedTransaction) -> RpcRichRawTransaction; + fn enrich(&self, signed: SignedTransaction) -> RpcRichRawTransaction; /// "Dispatch" a local transaction. fn dispatch_transaction(&self, signed_transaction: PendingTransaction) @@ -164,19 +172,30 @@ impl Dispatcher })) } - fn sign(&self, accounts: Arc, filled: FilledTransactionRequest, password: SignWith) - -> BoxFuture> + fn sign

( + &self, + accounts: Arc, + filled: FilledTransactionRequest, + password: SignWith, + post_sign: P + ) -> BoxFuture + where + P: PostSign + 'static, + ::Future: Send { let chain_id = self.client.signing_chain_id(); if let Some(nonce) = filled.nonce { - return Box::new(future::done(sign_transaction(&*accounts, filled, chain_id, nonce, password))); - } - - let state = self.state_nonce(&filled.from); - let reserved = self.nonces.lock().reserve(filled.from, state); + let future = sign_transaction(&*accounts, filled, chain_id, nonce, password) + .into_future() + .and_then(move |signed| post_sign.execute(signed)); + Box::new(future) + } else { + let state = self.state_nonce(&filled.from); + let reserved = self.nonces.lock().reserve(filled.from, state); - Box::new(ProspectiveSigner::new(accounts, filled, chain_id, reserved, password)) + Box::new(ProspectiveSigner::new(accounts, filled, chain_id, reserved, password, post_sign)) + } } fn enrich(&self, signed_transaction: SignedTransaction) -> RpcRichRawTransaction { @@ -396,12 +415,24 @@ impl Dispatcher for LightDispatcher { })) } - fn sign(&self, accounts: Arc, filled: FilledTransactionRequest, password: SignWith) - -> BoxFuture> + fn sign

( + &self, + accounts: Arc, + filled: FilledTransactionRequest, + password: SignWith, + post_sign: P + ) -> BoxFuture + where + P: PostSign + 'static, + ::Future: Send { let chain_id = self.client.signing_chain_id(); let nonce = filled.nonce.expect("nonce is always provided; qed"); - Box::new(future::done(sign_transaction(&*accounts, filled, chain_id, nonce, password))) + + let future = sign_transaction(&*accounts, filled, chain_id, nonce, password) + .into_future() + .and_then(move |signed| post_sign.execute(signed)); + Box::new(future) } fn enrich(&self, signed_transaction: SignedTransaction) -> RpcRichRawTransaction { @@ -449,28 +480,60 @@ fn sign_transaction( #[derive(Debug, Clone, Copy)] enum ProspectiveSignerState { TryProspectiveSign, + WaitForPostSign, WaitForNonce, - Finish, } -struct ProspectiveSigner { +struct ProspectiveSigner { accounts: Arc, filled: FilledTransactionRequest, chain_id: Option, reserved: nonce::Reserved, password: SignWith, state: ProspectiveSignerState, - prospective: Option>>, + prospective: Option>, ready: Option, + post_sign: Option

, + post_sign_future: Option<::Future> } -impl ProspectiveSigner { +/// action to execute after signing +/// e.g importing a transaction into the chain +pub trait PostSign: Send { + /// item that this PostSign returns + type Item: Send; + /// incase you need to perform async PostSign actions + type Out: IntoFuture + Send; + /// perform an action with the signed transaction + fn execute(self, signer: WithToken) -> Self::Out; +} + +impl PostSign for () { + type Item = WithToken; + type Out = Result; + fn execute(self, signed: WithToken) -> Self::Out { + Ok(signed) + } +} + +impl PostSign for F + where F: FnOnce(WithToken) -> Result +{ + type Item = T; + type Out = Result; + fn execute(self, signed: WithToken) -> Self::Out { + (self)(signed) + } +} + +impl ProspectiveSigner

{ pub fn new( accounts: Arc, filled: FilledTransactionRequest, chain_id: Option, reserved: nonce::Reserved, password: SignWith, + post_sign: P ) -> Self { // If the account is permanently unlocked we can try to sign // using prospective nonce. This should speed up sending @@ -491,6 +554,8 @@ impl ProspectiveSigner { }, prospective: None, ready: None, + post_sign: Some(post_sign), + post_sign_future: None } } @@ -509,8 +574,8 @@ impl ProspectiveSigner { } } -impl Future for ProspectiveSigner { - type Item = WithToken; +impl Future for ProspectiveSigner

{ + type Item = P::Item; type Error = Error; fn poll(&mut self) -> Poll { @@ -523,32 +588,45 @@ impl Future for ProspectiveSigner { match self.poll_reserved()? { Async::NotReady => { self.state = WaitForNonce; - self.prospective = Some(self.sign(self.reserved.prospective_value())); + self.prospective = Some(self.sign(self.reserved.prospective_value())?); }, Async::Ready(nonce) => { - self.state = Finish; - self.prospective = Some(self.sign(nonce.value())); + self.state = WaitForPostSign; + self.post_sign_future = Some(self.post_sign.take() + .expect("post_sign is set on creation; qed") + .execute(self.sign(nonce.value())?) + .into_future()); self.ready = Some(nonce); }, } }, WaitForNonce => { let nonce = try_ready!(self.poll_reserved()); - let result = match (self.prospective.take(), nonce.matches_prospective()) { + let prospective = match (self.prospective.take(), nonce.matches_prospective()) { (Some(prospective), true) => prospective, - _ => self.sign(nonce.value()), + _ => self.sign(nonce.value())?, }; - self.state = Finish; - self.prospective = Some(result); self.ready = Some(nonce); + self.state = WaitForPostSign; + self.post_sign_future = Some(self.post_sign.take() + .expect("post_sign is set on creation; qed") + .execute(prospective) + .into_future()); }, - Finish => { - if let (Some(result), Some(nonce)) = (self.prospective.take(), self.ready.take()) { - // Mark nonce as used on successful signing - return result.map(move |tx| { - nonce.mark_used(); - Async::Ready(tx) - }) + WaitForPostSign => { + if let Some(mut fut) = self.post_sign_future.as_mut() { + match fut.poll()? { + Async::Ready(item) => { + let nonce = self.ready + .take() + .expect("nonce is set before state transitions to WaitForPostSign; qed"); + nonce.mark_used(); + return Ok(Async::Ready(item)) + }, + Async::NotReady => { + return Ok(Async::NotReady) + } + } } else { panic!("Poll after ready."); } @@ -655,19 +733,21 @@ pub fn execute( match payload { ConfirmationPayload::SendTransaction(request) => { let condition = request.condition.clone().map(Into::into); - Box::new(dispatcher.sign(accounts, request, pass) - .map(move |v| v.map(move |tx| PendingTransaction::new(tx, condition))) - .map(WithToken::into_tuple) - .map(|(tx, token)| (tx, token, dispatcher)) - .and_then(|(tx, tok, dispatcher)| { - dispatcher.dispatch_transaction(tx) - .map(RpcH256::from) - .map(ConfirmationResponse::SendTransaction) - .map(move |h| WithToken::from((h, tok))) - })) + let cloned_dispatcher = dispatcher.clone(); + let post_sign = move |with_token_signed: WithToken| { + let (signed, token) = with_token_signed.into_tuple(); + let signed_transaction = PendingTransaction::new(signed, condition); + cloned_dispatcher.dispatch_transaction(signed_transaction) + .map(|hash| (hash, token)) + }; + let future = dispatcher.sign(accounts, request, pass, post_sign) + .map(|(hash, token)| { + WithToken::from((ConfirmationResponse::SendTransaction(hash.into()), token)) + }); + Box::new(future) }, ConfirmationPayload::SignTransaction(request) => { - Box::new(dispatcher.sign(accounts, request, pass) + Box::new(dispatcher.sign(accounts, request, pass, ()) .map(move |result| result .map(move |tx| dispatcher.enrich(tx)) .map(ConfirmationResponse::SignTransaction) diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index 56135544a68..e413903c598 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -18,16 +18,17 @@ use std::sync::Arc; use std::time::Duration; -use bytes::{Bytes, ToPretty}; +use bytes::Bytes; use ethcore::account_provider::AccountProvider; use types::transaction::PendingTransaction; +use types::transaction::SignedTransaction; use ethereum_types::{H520, U128, Address}; use ethkey::{public_to_address, recover, Signature}; use jsonrpc_core::{BoxFuture, Result}; use jsonrpc_core::futures::{future, Future}; use v1::helpers::{errors, eip191}; -use v1::helpers::dispatch::{self, eth_data_hash, Dispatcher, SignWith}; +use v1::helpers::dispatch::{self, eth_data_hash, Dispatcher, SignWith, PostSign, WithToken}; use v1::traits::Personal; use v1::types::{ H160 as RpcH160, H256 as RpcH256, H520 as RpcH520, U128 as RpcU128, @@ -68,7 +69,16 @@ impl PersonalClient { } impl PersonalClient { - fn do_sign_transaction(&self, _meta: Metadata, request: TransactionRequest, password: String) -> BoxFuture<(PendingTransaction, D)> { + fn do_sign_transaction

( + &self, + _meta: Metadata, + request: TransactionRequest, + password: String, + post_sign: P + ) -> BoxFuture + where P: PostSign + 'static, + ::Future: Send + { let dispatcher = self.dispatcher.clone(); let accounts = self.accounts.clone(); @@ -86,11 +96,7 @@ impl PersonalClient { Box::new(dispatcher.fill_optional_fields(request.into(), default, false) .and_then(move |filled| { - let condition = filled.condition.clone().map(Into::into); - dispatcher.sign(accounts, filled, SignWith::Password(password.into())) - .map(|tx| tx.into_value()) - .map(move |tx| PendingTransaction::new(tx, condition)) - .map(move |tx| (tx, dispatcher)) + dispatcher.sign(accounts, filled, SignWith::Password(password.into()), post_sign) }) ) } @@ -223,18 +229,26 @@ impl Personal for PersonalClient { } fn sign_transaction(&self, meta: Metadata, request: TransactionRequest, password: String) -> BoxFuture { - Box::new(self.do_sign_transaction(meta, request, password) - .map(|(pending_tx, dispatcher)| dispatcher.enrich(pending_tx.transaction))) + let condition = request.condition.clone().map(Into::into); + let dispatcher = self.dispatcher.clone(); + Box::new(self.do_sign_transaction(meta, request, password, ()) + .map(move |tx| PendingTransaction::new(tx.into_value(), condition)) + .map(move |pending_tx| dispatcher.enrich(pending_tx.transaction))) } fn send_transaction(&self, meta: Metadata, request: TransactionRequest, password: String) -> BoxFuture { - Box::new(self.do_sign_transaction(meta, request, password) - .and_then(|(pending_tx, dispatcher)| { - let chain_id = pending_tx.chain_id(); - trace!(target: "miner", "send_transaction: dispatching tx: {} for chain ID {:?}", - ::rlp::encode(&*pending_tx).pretty(), chain_id); - - dispatcher.dispatch_transaction(pending_tx).map(Into::into) + let condition = request.condition.clone().map(Into::into); + let dispatcher = self.dispatcher.clone(); + Box::new(self.do_sign_transaction(meta, request, password, move |signed: WithToken| { + dispatcher.dispatch_transaction( + PendingTransaction::new( + signed.into_value(), + condition + ) + ) + }) + .and_then(|hash| { + Ok(RpcH256::from(hash)) }) ) } diff --git a/scripts/docker/hub/Dockerfile b/scripts/docker/hub/Dockerfile index d40a662111f..c2249a56d45 100644 --- a/scripts/docker/hub/Dockerfile +++ b/scripts/docker/hub/Dockerfile @@ -1,8 +1,5 @@ FROM ubuntu:xenial -MAINTAINER Parity Technologies -#set ENVIROMENT -ARG TARGET -ENV TARGET ${TARGET} +LABEL MAINTAINER="Parity Technologies " # install tools and dependencies RUN apt update && apt install -y --no-install-recommends openssl libudev-dev file curl jq @@ -10,27 +7,25 @@ RUN apt update && apt install -y --no-install-recommends openssl libudev-dev fil # show backtraces ENV RUST_BACKTRACE 1 -#cleanup Docker image -RUN apt autoremove -y -RUN apt clean -y -RUN rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* +# cleanup Docker image +RUN apt autoremove -y \ + && apt clean -y \ + && rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* RUN groupadd -g 1000 parity \ && useradd -m -u 1000 -g parity -s /bin/sh parity +WORKDIR /home/parity -#add TARGET to docker image -COPY artifacts/x86_64-unknown-linux-gnu/$TARGET /bin/$TARGET - -# Build a shell script because the ENTRYPOINT command doesn't like using ENV -RUN echo "#!/bin/bash \n ${TARGET} \$@" > ./entrypoint.sh -RUN chmod +x ./entrypoint.sh +# add parity-ethereum to docker image +COPY artifacts/x86_64-unknown-linux-gnu/parity /bin/parity COPY scripts/docker/hub/check_sync.sh /check_sync.sh # switch to user parity here USER parity -# setup ENTRYPOINT +VOLUME [ "/home/parity/.local/share/io.parity.ethereum" ] EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp -ENTRYPOINT ["./entrypoint.sh"] + +ENTRYPOINT ["/bin/parity"] diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index 9bb6cd0f3f2..6244dc8460c 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -9,11 +9,12 @@ echo "CARGO_HOME: " $CARGO_HOME echo "CARGO_TARGET: " $CARGO_TARGET echo "CC: " $CC echo "CXX: " $CXX +#strip ON +export RUSTFLAGS=" -C link-arg=-s" echo "_____ Building target: "$CARGO_TARGET" _____" if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then -# only thing we need for android time cargo build --target $CARGO_TARGET --release -p parity-clib --features final else time cargo build --target $CARGO_TARGET --release --features final @@ -24,14 +25,11 @@ else fi echo "_____ Post-processing binaries _____" -rm -rf artifacts -mkdir -p artifacts -cd artifacts -mkdir -p $CARGO_TARGET -cd $CARGO_TARGET +mkdir -p artifacts/$CARGO_TARGET +cd artifacts/$CARGO_TARGET + if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then -# only thing we need for android cp -v ../../target/$CARGO_TARGET/release/libparity.so ./libparity.so else cp -v ../../target/$CARGO_TARGET/release/parity ./parity @@ -41,16 +39,6 @@ else cp -v ../../target/$CARGO_TARGET/release/whisper ./whisper fi - -# stripping can also be done on release build time -# export RUSTFLAGS="${RUSTFLAGS} -C link-arg=-s" -if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] -then - arm-linux-androideabi-strip -v ./* -else - strip -v ./* -fi - echo "_____ Calculating checksums _____" for binary in $(ls) do @@ -62,4 +50,3 @@ do ./parity tools hash $binary > $binary.sha3 fi done - diff --git a/scripts/gitlab/cargo-audit.sh b/scripts/gitlab/cargo-audit.sh deleted file mode 100755 index 16f0dc934ad..00000000000 --- a/scripts/gitlab/cargo-audit.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e # fail on any error -set -u # treat unset variables as error - -cargo install cargo-audit -cargo audit diff --git a/scripts/gitlab/publish-awss3.sh b/scripts/gitlab/publish-onnet-update.sh similarity index 73% rename from scripts/gitlab/publish-awss3.sh rename to scripts/gitlab/publish-onnet-update.sh index af768a63283..588cbdfb579 100755 --- a/scripts/gitlab/publish-awss3.sh +++ b/scripts/gitlab/publish-onnet-update.sh @@ -36,19 +36,3 @@ do esac cd .. done - -echo "__________Push binaries to AWS S3____________" -aws configure set aws_access_key_id $s3_key -aws configure set aws_secret_access_key $s3_secret - -case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in - (beta|stable|nightly) - export S3_BUCKET=releases.parity.io/ethereum; - ;; - (*) - export S3_BUCKET=builds-parity; - ;; -esac - -aws s3 sync ./ s3://$S3_BUCKET/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/ - diff --git a/scripts/gitlab/publish-snap.sh b/scripts/gitlab/publish-snap.sh index f001bbff0d8..386abdf3706 100755 --- a/scripts/gitlab/publish-snap.sh +++ b/scripts/gitlab/publish-snap.sh @@ -21,7 +21,19 @@ SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap" echo "__________Create snap package__________" echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME echo $VERSION:$GRADE:$BUILD_ARCH -cat scripts/snap/snapcraft.template.yaml | envsubst '$VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET' > snapcraft.yaml +# cat scripts/snap/snapcraft.template.yaml | envsubst '$VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET' > snapcraft.yaml +# a bit more necromancy (substitutions): +pwd +cd /builds/$CI_PROJECT_PATH/scripts/snap/ +sed -e 's/$VERSION/'"$VERSION"'/g' \ + -e 's/$GRADE/'"$GRADE"'/g' \ + -e 's/$BUILD_ARCH/'"$BUILD_ARCH"'/g' \ + -e 's/$CARGO_TARGET/'"$CARGO_TARGET"'/g' \ + snapcraft.template.yaml > /builds/$CI_PROJECT_PATH/snapcraft.yaml +cd /builds/$CI_PROJECT_PATH +pwd +apt update +apt install -y --no-install-recommends rhash cat snapcraft.yaml snapcraft --target-arch=$BUILD_ARCH ls *.snap diff --git a/scripts/gitlab/test-all.sh b/scripts/gitlab/test-all.sh index 15c22870626..925124b7ac2 100755 --- a/scripts/gitlab/test-all.sh +++ b/scripts/gitlab/test-all.sh @@ -1,6 +1,4 @@ #!/bin/bash -# ARGUMENT $1 Rust flavor to test with (stable/beta/nightly) - set -e # fail on any error set -u # treat unset variables as error @@ -27,9 +25,6 @@ then exit 0 fi -rustup default $1 - -git submodule update --init --recursive rustup show exec ./test.sh diff --git a/scripts/snap/snapcraft.template.yaml b/scripts/snap/snapcraft.template.yaml index eb67ba12820..d170241dbec 100644 --- a/scripts/snap/snapcraft.template.yaml +++ b/scripts/snap/snapcraft.template.yaml @@ -50,8 +50,4 @@ parts: cp -v ethkey $SNAPCRAFT_PART_INSTALL/usr/bin/ethkey cp -v ethstore $SNAPCRAFT_PART_INSTALL/usr/bin/ethstore cp -v whisper $SNAPCRAFT_PART_INSTALL/usr/bin/whisper - stage-packages: [libc6, libudev1, libstdc++6, cmake, libdb] - df: - plugin: nil - stage-packages: [coreutils] - stage: [bin/df] + stage-packages: [libc6, libudev1, libstdc++6, cmake, libdb5.3] diff --git a/secret-store/Cargo.toml b/secret-store/Cargo.toml index 8d41d48a87a..016028a0bf0 100644 --- a/secret-store/Cargo.toml +++ b/secret-store/Cargo.toml @@ -8,6 +8,7 @@ authors = ["Parity Technologies "] [dependencies] byteorder = "1.0" common-types = { path = "../ethcore/types" } +ethcore-call-contract = { path = "../ethcore/call-contract" } log = "0.4" parking_lot = "0.7" hyper = { version = "0.12", default-features = false } diff --git a/secret-store/src/acl_storage.rs b/secret-store/src/acl_storage.rs index 98eab5d932c..8c6656b1084 100644 --- a/secret-store/src/acl_storage.rs +++ b/secret-store/src/acl_storage.rs @@ -17,7 +17,8 @@ use std::sync::Arc; use std::collections::{HashMap, HashSet}; use parking_lot::{Mutex, RwLock}; -use ethcore::client::{BlockId, ChainNotify, NewBlocks, CallContract}; +use call_contract::CallContract; +use ethcore::client::{BlockId, ChainNotify, NewBlocks}; use ethereum_types::Address; use ethabi::FunctionOutputDecoder; use trusted_client::TrustedClient; diff --git a/secret-store/src/key_server_set.rs b/secret-store/src/key_server_set.rs index 50036d04d5d..5b25641aeec 100644 --- a/secret-store/src/key_server_set.rs +++ b/secret-store/src/key_server_set.rs @@ -18,8 +18,9 @@ use std::sync::Arc; use std::net::SocketAddr; use std::collections::{BTreeMap, HashSet}; use parking_lot::Mutex; +use call_contract::CallContract; use ethabi::FunctionOutputDecoder; -use ethcore::client::{Client, BlockChainClient, BlockId, ChainNotify, NewBlocks, CallContract}; +use ethcore::client::{Client, BlockChainClient, BlockId, ChainNotify, NewBlocks}; use ethereum_types::{H256, Address}; use ethkey::public_to_address; use bytes::Bytes; diff --git a/secret-store/src/lib.rs b/secret-store/src/lib.rs index 76ae09ffb34..915cec52889 100644 --- a/secret-store/src/lib.rs +++ b/secret-store/src/lib.rs @@ -18,6 +18,7 @@ extern crate byteorder; extern crate common_types; extern crate ethabi; extern crate ethcore; +extern crate ethcore_call_contract as call_contract; extern crate ethcore_sync as sync; extern crate ethereum_types; extern crate ethkey; diff --git a/secret-store/src/listener/service_contract.rs b/secret-store/src/listener/service_contract.rs index 47bb996e358..795e75d2c6d 100644 --- a/secret-store/src/listener/service_contract.rs +++ b/secret-store/src/listener/service_contract.rs @@ -19,7 +19,8 @@ use parking_lot::RwLock; use common_types::filter::Filter; use ethabi::RawLog; use ethabi::FunctionOutputDecoder; -use ethcore::client::{Client, BlockChainClient, BlockId, CallContract}; +use call_contract::CallContract; +use ethcore::client::{Client, BlockChainClient, BlockId}; use ethkey::{Public, public_to_address}; use hash::keccak; use bytes::Bytes; diff --git a/secret-store/src/trusted_client.rs b/secret-store/src/trusted_client.rs index d66e7f43dab..a20373ad0d0 100644 --- a/secret-store/src/trusted_client.rs +++ b/secret-store/src/trusted_client.rs @@ -16,9 +16,10 @@ use std::sync::{Arc, Weak}; use bytes::Bytes; +use call_contract::RegistryInfo; use common_types::transaction::{Transaction, SignedTransaction, Action}; use ethereum_types::Address; -use ethcore::client::{Client, BlockChainClient, ChainInfo, Nonce, BlockId, RegistryInfo}; +use ethcore::client::{Client, BlockChainClient, ChainInfo, Nonce, BlockId}; use ethcore::miner::{Miner, MinerService}; use sync::SyncProvider; use helpers::{get_confirmed_block_hash, REQUEST_CONFIRMATIONS_REQUIRED}; diff --git a/test.sh b/test.sh index 1b05194e471..a25f41d9a94 100755 --- a/test.sh +++ b/test.sh @@ -1,33 +1,12 @@ #!/bin/sh # Running Parity Full Test Suite +echo "________Running test.sh________" FEATURES="json-tests,ci-skip-issue" OPTIONS="--release" VALIDATE=1 THREADS=8 -case $1 in - --no-json) - FEATURES="ipc" - shift # past argument=value - ;; - --no-release) - OPTIONS="" - shift - ;; - --no-validate) - VALIDATE=0 - shift - ;; - --no-run) - OPTIONS="--no-run" - shift - ;; - *) - # unknown option - ;; -esac - set -e diff --git a/util/network-devp2p/Cargo.toml b/util/network-devp2p/Cargo.toml index 5254e8a65d4..bea3f6e31cb 100644 --- a/util/network-devp2p/Cargo.toml +++ b/util/network-devp2p/Cargo.toml @@ -34,6 +34,7 @@ serde = "1.0" serde_json = "1.0" serde_derive = "1.0" error-chain = { version = "0.12", default-features = false } +lru-cache = "0.1" [dev-dependencies] env_logger = "0.5" diff --git a/util/network-devp2p/src/discovery.rs b/util/network-devp2p/src/discovery.rs index f6eaf494b99..7bf8dc62e5e 100644 --- a/util/network-devp2p/src/discovery.rs +++ b/util/network-devp2p/src/discovery.rs @@ -20,6 +20,7 @@ use std::collections::{HashSet, HashMap, VecDeque}; use std::collections::hash_map::Entry; use std::default::Default; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use lru_cache::LruCache; use hash::keccak; use ethereum_types::{H256, H520}; use rlp::{Rlp, RlpStream}; @@ -55,6 +56,8 @@ const REQUEST_BACKOFF: [Duration; 4] = [ const NODE_LAST_SEEN_TIMEOUT: Duration = Duration::from_secs(24*60*60); +const OBSERVED_NODES_MAX_SIZE: usize = 10_000; + #[derive(Clone, Debug)] pub struct NodeEntry { pub id: NodeId, @@ -95,7 +98,27 @@ struct FindNodeRequest { #[derive(Clone, Copy)] enum PingReason { Default, - FromDiscoveryRequest(NodeId) + FromDiscoveryRequest(NodeId, NodeValidity), +} + +#[derive(Clone, Copy, PartialEq)] +enum NodeCategory { + Bucket, + Observed +} + +#[derive(Clone, Copy, PartialEq)] +enum NodeValidity { + Ourselves, + ValidNode(NodeCategory), + ExpiredNode(NodeCategory), + UnknownNode +} + +#[derive(Debug)] +enum BucketError { + Ourselves, + NotInTheBucket{node_entry: NodeEntry, bucket_distance: usize}, } struct PingRequest { @@ -145,6 +168,12 @@ pub struct Discovery<'a> { discovery_id: NodeId, discovery_nodes: HashSet, node_buckets: Vec, + + // Sometimes we don't want to add nodes to the NodeTable, but still want to + // keep track of them to avoid excessive pinging (happens when an unknown node sends + // a discovery request to us -- the node might be on a different net). + other_observed_nodes: LruCache, + in_flight_pings: HashMap, in_flight_find_nodes: HashMap, send_queue: VecDeque, @@ -171,6 +200,7 @@ impl<'a> Discovery<'a> { discovery_id: NodeId::new(), discovery_nodes: HashSet::new(), node_buckets: (0..ADDRESS_BITS).map(|_| NodeBucket::new()).collect(), + other_observed_nodes: LruCache::new(OBSERVED_NODES_MAX_SIZE), in_flight_pings: HashMap::new(), in_flight_find_nodes: HashMap::new(), send_queue: VecDeque::new(), @@ -200,41 +230,53 @@ impl<'a> Discovery<'a> { } } - fn update_node(&mut self, e: NodeEntry) -> Option { - trace!(target: "discovery", "Inserting {:?}", &e); + fn update_bucket_record(&mut self, e: NodeEntry) -> Result<(), BucketError> { let id_hash = keccak(e.id); let dist = match Discovery::distance(&self.id_hash, &id_hash) { Some(dist) => dist, None => { debug!(target: "discovery", "Attempted to update own entry: {:?}", e); - return None; + return Err(BucketError::Ourselves); } }; + let bucket = &mut self.node_buckets[dist]; + bucket.nodes.iter_mut().find(|n| n.address.id == e.id) + .map_or(Err(BucketError::NotInTheBucket{node_entry: e.clone(), bucket_distance: dist}.into()), |entry| { + entry.address = e; + entry.last_seen = Instant::now(); + entry.backoff_until = Instant::now(); + entry.fail_count = 0; + Ok(()) + }) + } - let mut added_map = HashMap::new(); - let ping = { - let bucket = &mut self.node_buckets[dist]; - let updated = if let Some(node) = bucket.nodes.iter_mut().find(|n| n.address.id == e.id) { - node.address = e.clone(); - node.last_seen = Instant::now(); - node.backoff_until = Instant::now(); - node.fail_count = 0; - true - } else { false }; + fn update_node(&mut self, e: NodeEntry) -> Option { + trace!(target: "discovery", "Inserting {:?}", &e); + + match self.update_bucket_record(e) { + Ok(()) => None, + Err(BucketError::Ourselves) => None, + Err(BucketError::NotInTheBucket{node_entry, bucket_distance}) => Some((node_entry, bucket_distance)) + }.map(|(node_entry, bucket_distance)| { + trace!(target: "discovery", "Adding a new node {:?} into our bucket {}", &node_entry, bucket_distance); - if !updated { - added_map.insert(e.id, e.clone()); - bucket.nodes.push_front(BucketEntry::new(e)); + let mut added = HashMap::with_capacity(1); + added.insert(node_entry.id, node_entry.clone()); + let node_to_ping = { + let bucket = &mut self.node_buckets[bucket_distance]; + bucket.nodes.push_front(BucketEntry::new(node_entry)); if bucket.nodes.len() > BUCKET_SIZE { select_bucket_ping(bucket.nodes.iter()) - } else { None } - } else { None } - }; - if let Some(node) = ping { - self.try_ping(node, PingReason::Default); - } - Some(TableUpdates { added: added_map, removed: HashSet::new() }) + } else { + None + } + }; + if let Some(node) = node_to_ping { + self.try_ping(node, PingReason::Default); + }; + TableUpdates{added, removed: HashSet::new()} + }) } /// Starts the discovery process at round 0 @@ -541,10 +583,28 @@ impl<'a> Discovery<'a> { }; if let Some((node, ping_reason)) = expected_node { - if let PingReason::FromDiscoveryRequest(target) = ping_reason { + if let PingReason::FromDiscoveryRequest(target, validity) = ping_reason { self.respond_with_discovery(target, &node)?; + // kirushik: I would prefer to probe the network id of the remote node here, and add it to the nodes list if it's on "our" net -- + // but `on_packet` happens synchronously, so doing the full TCP handshake ceremony here is a bad idea. + // So instead we just LRU-caching most recently seen nodes to avoid unnecessary pinging + match validity { + NodeValidity::ValidNode(NodeCategory::Bucket) | NodeValidity::ExpiredNode(NodeCategory::Bucket) => { + trace!(target: "discovery", "Updating node {:?} in our Kad buckets", &node); + self.update_bucket_record(node).unwrap_or_else(|error| { + debug!(target: "discovery", "Error occured when processing ping from a bucket node: {:?}", &error); + }); + }, + NodeValidity::UnknownNode | NodeValidity::ExpiredNode(NodeCategory::Observed) | NodeValidity::ValidNode(NodeCategory::Observed)=> { + trace!(target: "discovery", "Updating node {:?} in the list of other_observed_nodes", &node); + self.other_observed_nodes.insert(node.id, (node.endpoint, Instant::now())); + }, + NodeValidity::Ourselves => (), + } + Ok(None) + } else { + Ok(self.update_node(node)) } - Ok(self.update_node(node)) } else { debug!(target: "discovery", "Got unexpected Pong from {:?} ; request not found", &from); Ok(None) @@ -565,31 +625,41 @@ impl<'a> Discovery<'a> { } }; - if self.is_a_valid_known_node(&node) { - self.respond_with_discovery(target, &node)?; - } else { + match self.check_validity(&node) { + NodeValidity::Ourselves => (), // It makes no sense to respond to the discovery request from ourselves + NodeValidity::ValidNode(_) => self.respond_with_discovery(target, &node)?, // Make sure the request source is actually there and responds to pings before actually responding - self.try_ping(node, PingReason::FromDiscoveryRequest(target)); + invalidity_reason => self.try_ping(node, PingReason::FromDiscoveryRequest(target, invalidity_reason)) } Ok(None) } - fn is_a_valid_known_node(&self, node: &NodeEntry) -> bool { + fn check_validity(&mut self, node: &NodeEntry) -> NodeValidity { let id_hash = keccak(node.id); let dist = match Discovery::distance(&self.id_hash, &id_hash) { Some(dist) => dist, None => { debug!(target: "discovery", "Got an incoming discovery request from self: {:?}", node); - return false; + return NodeValidity::Ourselves; } }; let bucket = &self.node_buckets[dist]; if let Some(known_node) = bucket.nodes.iter().find(|n| n.address.id == node.id) { debug!(target: "discovery", "Found a known node in a bucket when processing discovery: {:?}/{:?}", known_node, node); - (known_node.address.endpoint == node.endpoint) && (known_node.last_seen.elapsed() < NODE_LAST_SEEN_TIMEOUT) + match ((known_node.address.endpoint == node.endpoint), (known_node.last_seen.elapsed() < NODE_LAST_SEEN_TIMEOUT)) { + (true, true) => NodeValidity::ValidNode(NodeCategory::Bucket), + (true, false) => NodeValidity::ExpiredNode(NodeCategory::Bucket), + _ => NodeValidity::UnknownNode + } } else { - false + self.other_observed_nodes.get_mut(&node.id).map_or(NodeValidity::UnknownNode, |(endpoint, observed_at)| { + match ((node.endpoint==*endpoint), (observed_at.elapsed() < NODE_LAST_SEEN_TIMEOUT)) { + (true, true) => NodeValidity::ValidNode(NodeCategory::Observed), + (true, false) => NodeValidity::ExpiredNode(NodeCategory::Observed), + _ => NodeValidity::UnknownNode + } + }) } } diff --git a/util/network-devp2p/src/lib.rs b/util/network-devp2p/src/lib.rs index 531c6ee506f..6082049e890 100644 --- a/util/network-devp2p/src/lib.rs +++ b/util/network-devp2p/src/lib.rs @@ -84,6 +84,7 @@ extern crate keccak_hash as hash; extern crate serde; extern crate serde_json; extern crate parity_snappy as snappy; +extern crate lru_cache; #[macro_use] extern crate error_chain; diff --git a/util/version/Cargo.toml b/util/version/Cargo.toml index d479cb9482c..5135f3b2a71 100644 --- a/util/version/Cargo.toml +++ b/util/version/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "parity-version" # NOTE: this value is used for Parity Ethereum version string (via env CARGO_PKG_VERSION) -version = "2.3.2" +version = "2.3.3" authors = ["Parity Technologies "] build = "build.rs"