Change to new, experimental cargo feature resolver

The [new cargo feature resolver](rust-lang/cargo#7820) won't unify features across build deps, dev deps, and targets. This solves our problem of needing to work around feature unification to avoid Clone implementations on private key material. This commit puts back our true dependency information into the Cargo.tomls. Specifically, it adds dev-dependencies that enable features that aren't enabled on normal dependencies. This will cause the feature unification to happen when using the old resolver, but the build will be correct under the new resolver. In order to maintain correct builds in CI, this commit also changes CI to use the nightly cargo with `-Z features=all` but still preserving the rustc toolchain specified in `rustc-toolchain`. Local developer builds will likely still use the `rustc-toolchain` version of cargo with the old resolver, but this shouldn't cause any problems for development.
metajack · Mar 26, 2020 · 412fcd0 · 412fcd0
1 parent 5ad239b
commit 412fcd0
Show file tree

Hide file tree

Showing 31 changed files with 107 additions and 69 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -19,20 +19,12 @@ executors:
     resource_class: small
 
 commands:
-  rust_setup:
-    description: Set rustc version
-    steps:
-      - run:
-          name: Set rustc version
-          command: |
-            rustup default stable
-            rustup update stable
   print_versions:
     description: Version Info
     steps:
       - run:
           name: Version Info
-          command: rustc --version; cargo --version; rustup --version
+          command: rustup --version
   env_setup:
     description: Environment Setup
     steps:
@@ -44,6 +36,7 @@ commands:
             echo 'export LIBRA_DUMP_LOGS=1' >> $BASH_ENV
             echo 'export CARGO_INCREMENTAL=0' >> $BASH_ENV
             echo 'export CI_TIMEOUT="timeout 40m"' >> $BASH_ENV
+            echo 'export RUST_NIGHTLY=nightly-2020-03-18' >> $BASH_ENV
   install_deps:
     steps:
       - run:
@@ -52,14 +45,23 @@ commands:
             sudo apt-get update
             sudo apt-get install -y cmake curl clang llvm
             rustup component add clippy rustfmt
+            rustup toolchain install $RUST_NIGHTLY
+      - run:
+          name: Set cargo Environment
+          command: |
+            # Use nightly version of cargo to access the new feature resolver
+            echo 'export CARGO=$(rustup which --toolchain $RUST_NIGHTLY cargo)' >> $BASH_ENV
+            # Turn on the experimental feature resolver in cargo. See:
+            # https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#features
+            echo 'export CARGOFLAGS=-Zfeatures=all' >> $BASH_ENV
   install_code_coverage_deps:
     steps:
       - run:
           name: Install grcov and lcov
           command: |
             sudo apt-get update
             sudo apt-get install lcov
-            cargo install --force grcov
+            $CARGO $CARGOFLAGS install --force grcov
   install_docker_linter:
     steps:
       - run:
@@ -68,11 +70,6 @@ commands:
              export HADOLINT=${HOME}/hadolint
              export HADOLINT_VER=v1.17.4
              curl -sL -o ${HADOLINT} "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VER}/hadolint-$(uname -s)-$(uname -m)" && chmod 700 ${HADOLINT}
-  install_rust_nightly_toolchain:
-    steps:
-      - run:
-          name: Install nightly toolchain for features not in beta/stable
-          command: rustup install nightly
   find_dockerfile_changes:
     steps:
       - run:
@@ -106,7 +103,6 @@ commands:
   build_setup:
     steps:
       - checkout
-      - rust_setup
       - print_versions
       - env_setup
       - install_deps
@@ -127,7 +123,7 @@ jobs:
           command: ./scripts/git-checks.sh
       - run:
           name: Fetch workspace dependencies over network
-          command: cargo fetch
+          command: $CARGO $CARGOFLAGS fetch
       - save_cargo_package_cache
       - persist_to_workspace:
           root: /home/circleci/project
@@ -148,20 +144,20 @@ jobs:
       - restore_cargo_package_cache
       - run:
           name: cargo lint
-          command: cargo x lint
+          command: $CARGO $CARGOFLAGS x lint
       - run:
           name: cargo clippy
-          command: cargo xclippy --workspace --all-targets
+          command: $CARGO $CARGOFLAGS xclippy --workspace --all-targets
       - run:
           name: cargo fmt
-          command: cargo xfmt --check
+          command: $CARGO $CARGOFLAGS xfmt --check
       - run:
           name: cargo guppy
           command: |
-            cargo install cargo-guppy \
+            $CARGO $CARGOFLAGS install cargo-guppy \
               --git http://github.com/calibra/cargo-guppy \
               --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
-            [[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu \
+            [[ -z $($CARGO $CARGOFLAGS guppy dups --target x86_64-unknown-linux-gnu \
               --kind directthirdparty) ]]
   build-dev:
     executor: build-executor
@@ -172,19 +168,19 @@ jobs:
           at: /home/circleci/project
       - restore_cargo_package_cache
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p libra-swarm
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p cluster-test
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p libra-fuzzer
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p language_benchmarks
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p cost-synthesis
       - run:
-          command: RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p test-generation
       - build_teardown
   build-release:
     executor: build-executor
@@ -196,7 +192,7 @@ jobs:
       - restore_cargo_package_cache
       - run:
           name: Build release
-          command: RUST_BACKTRACE=1 cargo build -j 16 --release
+          command: RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 --release
       - build_teardown
   build-e2e-test:
     executor: build-executor
@@ -209,7 +205,7 @@ jobs:
       - run:
           name: Find all e2e tests
           command: |
-            cargo x test --package testsuite -- --list | grep "::" | \
+            $CARGO $CARGOFLAGS x test --package testsuite -- --list | grep "::" | \
               sed 's/: .*$//' > e2e_tests
             cat e2e_tests
       - persist_to_workspace:
@@ -246,7 +242,7 @@ jobs:
                 RUST_BACKTRACE=1 $CI_TIMEOUT $libratest $target \
                   --test-threads 1 --exact --nocapture
               else
-                RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test $target \
+                RUST_BACKTRACE=1 $CI_TIMEOUT $CARGO $CARGOFLAGS x test $target \
                   --package testsuite -- --test-threads 1 --exact --nocapture
               fi
             done
@@ -262,7 +258,7 @@ jobs:
       - run:
           name: Run all unit tests
           command: |
-            RUST_BACKTRACE=1 $CI_TIMEOUT cargo test \
+            RUST_BACKTRACE=1 $CI_TIMEOUT $CARGO $CARGOFLAGS test \
               --all-features \
               --workspace \
               --exclude libra-node \
@@ -281,7 +277,7 @@ jobs:
           name: Run crypto unit tests
           command: |
             cd crypto/crypto && \
-            RUST_BACKTRACE=1 cargo test \
+            RUST_BACKTRACE=1 $CARGO $CARGOFLAGS test \
               --features='std fiat_u64_backend fuzzing' \
               --no-default-features
   run-flaky-unit-test:
@@ -312,16 +308,16 @@ jobs:
     steps:
       - build_setup
       - run:
-          name: Install Cargo Audit
+          name: Install cargo-audit
           command: |
-            cargo install --force cargo-audit
+            $CARGO $CARGOFLAGS install --force cargo-audit
       - run:
           # NOTE ignored advisory rules
           # RUSTSEC-2018-0015 - term
           # RUSTSEC-2019-0031 - spin
           name: Audit crates
           command: |
-            cargo audit --deny-warnings \
+            $CARGO $CARGOFLAGS audit --deny-warnings \
               --ignore RUSTSEC-2018-0015 \
               --ignore RUSTSEC-2019-0031
       - build_teardown
@@ -331,7 +327,6 @@ jobs:
     steps:
       - build_setup
       - install_code_coverage_deps
-      - install_rust_nightly_toolchain
       - run:
           name: Setup code coverage output
           command: echo "export CODECOV_OUTPUT=codecov" >> $BASH_ENV

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -108,7 +108,8 @@ members = [
 
 # NOTE: These members should never include crates that require fuzzing
 # features or test features. These are the crates we want built with no extra
-# test-only code included.
+# test-only code included. These are essentially the main libra release
+# binaries.
 default-members = [
     "client/cli",
     "language/compiler",

diff --git a/admission_control/admission-control-service/Cargo.toml b/admission_control/admission-control-service/Cargo.toml
@@ -32,6 +32,12 @@ serde_json = "1.0"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
+proptest = "0.9.4"
+libra-mempool = { path = "../../mempool", version = "0.1.0", features = ["fuzzing"] }
+libra-proptest-helpers = { path = "../../common/proptest-helpers" }
+libra-prost-ext = { path = "../../common/prost-ext", version = "0.1.0" }
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
+storage-service = { path = "../../storage/storage-service" }
 vm-validator = { path = "../../vm-validator", version = "0.1.0" }
 
 [features]

diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml
@@ -42,6 +42,7 @@ transaction-builder = { path = "../../language/transaction-builder", version = "
 
 [dev-dependencies]
 proptest = "0.9.2"
+libra-crypto = { path = "../../crypto/crypto", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

diff --git a/config/Cargo.toml b/config/Cargo.toml
@@ -29,6 +29,9 @@ libra-logger = { path = "../common/logger", version = "0.1.0" }
 libra-temppath = { path = "../common/temppath", version = "0.1.0" }
 libra-types = { path = "../types", version = "0.1.0" }
 
+[dev-dependencies]
+libra-crypto = { path = "../crypto/crypto", version = "0.1.0", features = ["fuzzing"] }
+
 [features]
 default = []
 fuzzing = ["libra-crypto/fuzzing", "libra-types/fuzzing"]
diff --git a/consensus/Cargo.toml b/consensus/Cargo.toml
@@ -31,7 +31,7 @@ prometheus = { version = "0.8.0", default-features = false }
 proptest = { version = "0.9.4", optional = true }
 
 channel = { path = "../common/channel", version = "0.1.0" }
-consensus-types = { path = "consensus-types", version = "0.1.0", default-features = false }
+consensus-types = { path = "consensus-types", version = "0.1.0" }
 crash-handler = { path = "../common/crash-handler", version = "0.1.0" }
 debug-interface = { path = "../common/debug-interface", version = "0.1.0" }
 executor = { path = "../execution/executor", version = "0.1.0" }
@@ -57,6 +57,9 @@ cached = "0.12.0"
 proptest = "0.9.4"
 tempfile = "3.1.0"
 
+consensus-types = { path = "consensus-types", version = "0.1.0", features = ["fuzzing"] }
+libra-config = { path = "../config", version = "0.1.0", features = ["fuzzing"] }
+libra-mempool = { path = "../mempool", version = "0.1.0", features = ["fuzzing"] }
 vm-genesis = { path = "../language/tools/vm-genesis", version = "0.1.0" }
 vm-validator = { path = "../vm-validator", version = "0.1.0" }
 

diff --git a/consensus/consensus-types/Cargo.toml b/consensus/consensus-types/Cargo.toml
@@ -21,6 +21,7 @@ libra-types = { path = "../../types", version = "0.1.0" }
 
 [dev-dependencies]
 proptest = "0.9.4"
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

diff --git a/consensus/safety-rules/Cargo.toml b/consensus/safety-rules/Cargo.toml
@@ -28,6 +28,8 @@ workspace-builder = { path = "../../common/workspace-builder", version = "0.1.0"
 criterion = "0.3"
 rand = { version = "0.6.5", default-features = false }
 tempfile = "3.1.0"
+consensus-types = { path = "../consensus-types", version = "0.1.0", features = ["fuzzing"] }
+libra-config = { path = "../../config", version = "0.1.0", features = ["fuzzing"] }
 
 [[bench]]
 name = "safety_rules"

diff --git a/consensus/src/chained_bft/chained_bft_smr_test.rs b/consensus/src/chained_bft/chained_bft_smr_test.rs
@@ -44,7 +44,6 @@ struct SMRNode {
     commit_cb_receiver: mpsc::UnboundedReceiver<LedgerInfoWithSignatures>,
     storage: Arc<MockStorage<TestPayload>>,
     state_sync: mpsc::UnboundedReceiver<Vec<usize>>,
-    shared_mempool: MockSharedMempool,
 }
 
 impl SMRNode {
@@ -75,7 +74,7 @@ impl SMRNode {
         let (state_sync_client, state_sync) = mpsc::unbounded();
         let (commit_cb_sender, commit_cb_receiver) = mpsc::unbounded::<LedgerInfoWithSignatures>();
         let shared_mempool = MockSharedMempool::new(None);
-        let consensus_to_mempool_sender = shared_mempool.consensus_sender.clone();
+        let consensus_to_mempool_sender = shared_mempool.consensus_sender;
 
         let mut smr = ChainedBftSMR::new(
             network_sender,
@@ -101,7 +100,6 @@ impl SMRNode {
             commit_cb_receiver,
             storage,
             state_sync,
-            shared_mempool,
         }
     }
 

diff --git a/consensus/src/chained_bft/event_processor_test.rs b/consensus/src/chained_bft/event_processor_test.rs
@@ -49,7 +49,7 @@ use libra_types::{
     block_info::BlockInfo,
     ledger_info::LedgerInfoWithSignatures,
     validator_signer::ValidatorSigner,
-    validator_verifier::{random_validator_verifier, ValidatorVerifier},
+    validator_verifier::{random_validator_verifier},
 };
 use network::peer_manager::{
     conn_status_channel, ConnectionRequestSender, PeerManagerRequestSender,
@@ -65,7 +65,6 @@ pub struct NodeSetup {
     storage: Arc<MockStorage<TestPayload>>,
     signer: ValidatorSigner,
     proposer_author: Author,
-    validators: Arc<ValidatorVerifier>,
     safety_rules_manager: SafetyRulesManager<TestPayload>,
 }
 
@@ -143,7 +142,7 @@ impl NodeSetup {
         playground.add_node(author, consensus_tx, network_reqs_rx, conn_mgr_reqs_rx);
 
         let (self_sender, self_receiver) = channel::new_test(8);
-        let network = NetworkSender::new(author, network_sender, self_sender, validators.clone());
+        let network = NetworkSender::new(author, network_sender, self_sender, validators);
 
         let (task, _receiver) = NetworkTask::<TestPayload>::new(network_events, self_receiver);
 
@@ -199,7 +198,6 @@ impl NodeSetup {
             storage,
             signer,
             proposer_author,
-            validators,
             safety_rules_manager,
         }
     }

diff --git a/consensus/src/chained_bft/test_utils/mod.rs b/consensus/src/chained_bft/test_utils/mod.rs
@@ -61,20 +61,6 @@ pub fn build_simple_tree() -> (
     (vec![genesis_block, a1, a2, a3, b1, b2, c1], block_store)
 }
 
-pub fn build_chain() -> Vec<Arc<ExecutedBlock<TestPayload>>> {
-    let mut inserter = TreeInserter::default();
-    let block_store = inserter.block_store();
-    let genesis = block_store.root();
-    let a1 = inserter.insert_block_with_qc(certificate_for_genesis(), &genesis, 1);
-    let a2 = inserter.insert_block(&a1, 2, None);
-    let a3 = inserter.insert_block(&a2, 3, Some(genesis.block_info()));
-    let a4 = inserter.insert_block(&a3, 4, Some(a1.block_info()));
-    let a5 = inserter.insert_block(&a4, 5, Some(a2.block_info()));
-    let a6 = inserter.insert_block(&a5, 6, Some(a3.block_info()));
-    let a7 = inserter.insert_block(&a6, 7, Some(a4.block_info()));
-    vec![genesis, a1, a2, a3, a4, a5, a6, a7]
-}
-
 pub fn build_empty_tree() -> Arc<BlockStore<TestPayload>> {
     let (initial_data, storage) = EmptyStorage::start_for_testing();
     Arc::new(BlockStore::new(

diff --git a/execution/executor/Cargo.toml b/execution/executor/Cargo.toml
@@ -42,7 +42,8 @@ executor-utils = { path = "../executor-utils", version = "0.1.0" }
 storage-service = { path = "../../storage/storage-service", version = "0.1.0" }
 stdlib = { path = "../../language/stdlib", version = "0.1.0" }
 transaction-builder = { path = "../../language/transaction-builder", version = "0.1.0" }
-
+libra-config = { path = "../../config", version = "0.1.0", features = ["fuzzing"] }
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 vm-genesis = { path = "../../language/tools/vm-genesis", version = "0.1.0" }
 
 [features]