[CI] parallel CI build and test

Currently the end-to-end CI workflow takes 23 minutes to complete. This
lengthens the PR land time as Homu lands them serially.  When the CI
workflow fails due to flaky tests and has to re-run, dev speed and
experience suffers.  The E2E (smoke) test job is the current critical
path.

This commit attempts to reduce the CI workflow time to 14 mintues, by
parallelizing the build and test.
  - prefetch cargo crates
  - pre-build e2e test binaries
  - distribute e2e tests between containers

Future improvements include parallelzing the unit test (ie the next
critical path) and generating/storing/analyzing test artifacts.

Closes: diem#3025
Approved by: davidiw

.circleci/config.yml

@@ -5,6 +5,10 @@ executors:
     docker:
       - image: circleci/rust:buster
     resource_class: 2xlarge
+  test-executor:
+    docker:
+      - image: circleci/rust:buster
+    resource_class: xlarge
   audit-executor:
     docker:
       - image: circleci/rust:buster
@@ -79,6 +83,26 @@ commands:
                 git diff-tree --no-commit-id --name-only -r "$commit" -- "*.Dockerfile";
               done
             )' >> $BASH_ENV
+  save_cargo_package_cache:
+    description: Save cargo package cache for subsequent jobs
+    steps:
+      - run:
+          name: Save cargo package cache
+          command: |
+            mv /usr/local/cargo/git .
+            mv /usr/local/cargo/registry .
+            mv /usr/local/cargo/.package-cache  .
+  restore_cargo_package_cache:
+    description: Restore Cargo package cache from prev job
+    steps:
+      - run:
+          name: Check cargo package cache
+          command: |
+            mv git /usr/local/cargo/
+            mv registry /usr/local/cargo/
+            mv .package-cache /usr/local/cargo/
+            ls -all /usr/local/cargo
+            du -ssh /usr/local/cargo
   build_setup:
     steps:
       - checkout
@@ -93,53 +117,186 @@ commands:
           command: ./scripts/changed-files.sh
 
 jobs:
-  build:
-    executor: build-executor
-    parallelism: 5
+  prefetch-crates:
+    executor: test-executor
+    description: Prefetch cargo crates for subsequent jobs.
+    steps:
+      - build_setup
+      - run:
+          name: Git Hooks and Checks
+          command: ./scripts/git-checks.sh
+      - run:
+          name: Fetch workspace dependencies over network
+          command: cargo fetch
+      - save_cargo_package_cache
+      - persist_to_workspace:
+          root: /home/circleci/project
+          paths:
+            - git
+            - registry
+            - .package-cache
+  lint:
+    executor: test-executor
+    description: Run Rust linting tools.
     steps:
       - build_setup
       - run:
           name: Git Hooks and Checks
           command: ./scripts/git-checks.sh
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
+      - run:
+          name: cargo lint
+          command: cargo x lint
       - run:
-          name: Linting
+          name: cargo clippy
+          command: cargo xclippy --workspace --all-targets
+      - run:
+          name: cargo fmt
+          command: cargo xfmt --check
+      - run:
+          name: cargo guppy
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo x lint
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xclippy --workspace --all-targets
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xfmt --check
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo install cargo-guppy --git http://github.com/calibra/cargo-guppy --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || [[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu --kind directthirdparty) ]]
+            cargo install cargo-guppy \
+              --git http://github.com/calibra/cargo-guppy \
+              --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
+            [[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu \
+              --kind directthirdparty) ]]
+  build-dev:
+    executor: build-executor
+    description: Development Build
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
+      - run:
+          command: RUST_BACKTRACE=1 cargo build -j 16
+      - run:
+          command: RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
+      - run:
+          command: RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
+      - run:
+          command: RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
       - run:
-          name: Build Release
+          command: RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
+      - run:
+          command: RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
+      - run:
+          command: RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
+      - build_teardown
+  build-release:
+    executor: build-executor
+    description: Release Build
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
+      - run:
+          name: Build release
+          command: RUST_BACKTRACE=1 cargo build -j 16 --release
+      - build_teardown
+  build-e2e-test:
+    executor: build-executor
+    description: Generate a list of E2E test targets to be distributed.
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
+      - run:
+          name: Find all e2e tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0234] ]] || RUST_BACKTRACE=1 cargo build -j 16 --release
+            cargo x test --package testsuite -- --list | grep "::" | \
+              sed 's/: .*$//' > e2e_tests
+            cat e2e_tests
+      - persist_to_workspace:
+          root: /home/circleci/project
+          paths:
+            - e2e_tests
+            - target/debug/deps/libratest*
+  run-e2e-test:
+    executor: build-executor
+    parallelism: 8
+    description: Run E2E tests in parallel. Each container runs a subset of
+      test targets.
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
       - run:
-          name: Build Dev
+          name: Determine test targets for this container.
+          # NOTE Currently the tests are distributed by name order. Once test
+          # metadata is enabled, the tests can be distributed by run time to
+          # speed up this job.
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
+            ls -all target/debug/*
+            echo -e "Found $(wc -l e2e_tests) tests."
+            cat e2e_tests | circleci tests split > /tmp/tests_to_run
+            echo -e "This runner will run these tests\n$(cat /tmp/tests_to_run)"
       - run:
-          name: Run All Non Flaky Unit Tests
+          name: Run E2E tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo test --all-features --workspace --exclude libra-node --exclude libra-crypto --exclude testsuite --exclude consensus
+            set -x
+            libratest=$(find target/debug/deps/ -name "libratest*" -executable)
+            for target in $(cat /tmp/tests_to_run) ; do
+              if [ ! -z "$libratest" ]; then
+                RUST_BACKTRACE=1 $CI_TIMEOUT $libratest $target \
+                  --test-threads 1 --exact --nocapture
+              else
+                RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test $target \
+                  --package testsuite -- --test-threads 1 --exact --nocapture
+              fi
+            done
+  run-unit-test:
+    executor: build-executor
+    description: Run all unit tests, excluding E2E and flaky tests that are
+      explicitly ignored.
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
       - run:
-          name: Run Cryptography Unit Tests with the formally verified backend
+          name: Run all unit tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || ( RUST_BACKTRACE=1 cd crypto/crypto && $CI_TIMEOUT cargo test --features='std fiat_u64_backend fuzzing' --no-default-features )
+            RUST_BACKTRACE=1 $CI_TIMEOUT cargo test \
+              --all-features \
+              --workspace \
+              --exclude libra-node \
+              --exclude libra-crypto \
+              --exclude testsuite \
+              --exclude consensus
+  run-crypto-unit-test:
+    executor: test-executor
+    description: Run crypto unit tests
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
       - run:
-          name: Run All End to End Tests
+          name: Run crypto unit tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test --package testsuite -- --test-threads 1
+            cd crypto/crypto && \
+            RUST_BACKTRACE=1 cargo test \
+              --features='std fiat_u64_backend fuzzing' \
+              --no-default-features
+  run-flaky-unit-test:
+    executor: test-executor
+    description: Run a list of known flaky tests.
+    steps:
+      - build_setup
+      - attach_workspace:
+          at: /home/circleci/project
+      - restore_cargo_package_cache
       - run:
-          name: Run Quarantined Unit Tests 3 (consensus) times
+          name: Run flaky tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0123] ]] || timeout 20m ./scripts/run_quarantined.sh -c consensus -r 3 -f
-      - build_teardown
+            RUST_BACKTRACE=1 $CI_TIMEOUT \
+            ./scripts/run_quarantined.sh -c consensus -r 3 -f
   validate-cluster-test-dockerfile:
     description: Validate that committed docker files for cluster test are up to date
     executor: build-executor
@@ -233,10 +390,34 @@ jobs:
 workflows:
   commit-workflow:
     jobs:
-      - build
       - build-docker
       - validate-cluster-test-dockerfile
       - terraform
+      - prefetch-crates
+      - lint:
+          requires:
+            - prefetch-crates
+      - build-dev:
+          requires:
+            - prefetch-crates
+      - build-release:
+          requires:
+            - prefetch-crates
+      - build-e2e-test:
+          requires:
+            - prefetch-crates
+      - run-e2e-test:
+          requires:
+            - build-e2e-test
+      - run-unit-test:
+          requires:
+            - prefetch-crates
+      - run-crypto-unit-test:
+          requires:
+            - prefetch-crates
+      - run-flaky-unit-test:
+          requires:
+            - prefetch-crates
 
   scheduled-workflow:
     triggers: