From 783294352cea3e19adadeb101f3e3457a9f344cc Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Tue, 12 Sep 2023 13:16:36 -0700 Subject: [PATCH] chore(nix): Reduce nix to building final rust & wasm packages --- .github/Cross.toml | 1 - .github/workflows/cross.yml | 28 +++ flake.lock | 24 --- flake.nix | 343 ++++++++++-------------------------- 4 files changed, 121 insertions(+), 275 deletions(-) create mode 100644 .github/workflows/cross.yml diff --git a/.github/Cross.toml b/.github/Cross.toml index 09e6316a59c..d8516b9ae09 100644 --- a/.github/Cross.toml +++ b/.github/Cross.toml @@ -2,7 +2,6 @@ passthrough = [ "HOME", "RUST_BACKTRACE", - "BARRETENBERG_BIN_DIR" ] volumes = [ "HOME", diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml new file mode 100644 index 00000000000..6099452f0dd --- /dev/null +++ b/.github/workflows/cross.yml @@ -0,0 +1,28 @@ +name: Cross + +on: + pull_request: + merge_group: + push: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-cross: + runs-on: macos-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Setup Nix + uses: cachix/install-nix-action@v22 + with: + nix_path: nixpkgs=channel:nixos-22.11 + github_access_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build cross package + run: | + nix build -L . diff --git a/flake.lock b/flake.lock index 659daec5d3a..465d56c0838 100644 --- a/flake.lock +++ b/flake.lock @@ -1,28 +1,5 @@ { "nodes": { - "barretenberg": { - "inputs": { - "flake-utils": [ - "flake-utils" - ], - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1688820427, - "narHash": "sha256-w7yMeYp50KrlTn23TTKfYmLOQL4uIgw0wSX67v2tvvc=", - "owner": "AztecProtocol", - "repo": "barretenberg", - "rev": "fdd46f77531a6fcc9d9b24a698c56590d54d487e", - "type": "github" - }, - "original": { - "owner": "AztecProtocol", - "repo": "barretenberg", - "type": "github" - } - }, "crane": { "inputs": { "flake-compat": [ @@ -104,7 +81,6 @@ }, "root": { "inputs": { - "barretenberg": "barretenberg", "crane": "crane", "flake-compat": "flake-compat", "flake-utils": "flake-utils", diff --git a/flake.nix b/flake.nix index b6d5a3e750f..71b6911352d 100644 --- a/flake.nix +++ b/flake.nix @@ -36,31 +36,22 @@ rust-overlay.follows = "rust-overlay"; }; }; - - barretenberg = { - url = "github:AztecProtocol/barretenberg"; - # All of these inputs (a.k.a. dependencies) need to align with inputs we - # use so they use the `inputs.*.follows` syntax to reference our inputs - inputs = { - nixpkgs.follows = "nixpkgs"; - flake-utils.follows = "flake-utils"; - }; - }; }; outputs = - { self, nixpkgs, crane, flake-utils, rust-overlay, barretenberg, ... }: + { self, nixpkgs, crane, flake-utils, rust-overlay, ... }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlays.default - barretenberg.overlays.default ]; }; - rustToolchain = pkgs.rust-bin.stable."1.66.0".default.override { + rustVersion = "1.66.1"; + + rustToolchain = pkgs.rust-bin.stable.${rustVersion}.default.override { # We include rust-src to ensure rust-analyzer works. # See https://discourse.nixos.org/t/rust-src-not-found-and-other-misadventures-of-developing-rust-on-nixos/11570/4 extensions = [ "rust-src" ]; @@ -73,66 +64,28 @@ craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; - sharedEnvironment = { - # We enable backtraces on any failure for help with debugging - RUST_BACKTRACE = "1"; - - BARRETENBERG_ARCHIVE = builtins.fetchurl { - url = "https://github.com/AztecProtocol/barretenberg/releases/download/barretenberg-v0.4.5/acvm_backend.wasm.tar.gz"; - sha256 = "sha256:0z24yhvxc0dr13xj7y4xs9p42lzxwpazrmsrdpcgynfajkk6vqy4"; - }; - }; - - nativeEnvironment = sharedEnvironment // { - # rust-bindgen needs to know the location of libclang - LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; - }; - - wasmEnvironment = sharedEnvironment // { - # We set the environment variable because barretenberg must be compiled in a special way for wasm - BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin"; - }; - - testEnvironment = sharedEnvironment // { }; - # The `self.rev` property is only available when the working tree is not dirty GIT_COMMIT = if (self ? rev) then self.rev else "unknown"; GIT_DIRTY = if (self ? rev) then "false" else "true"; - # We use `include_str!` macro to embed the solidity verifier template so we need to create a special - # source filter to include .sol files in addition to usual rust/cargo source files. - solidityFilter = path: _type: builtins.match ".*sol$" path != null; - # We use `.bytecode` and `.tr` files to test interactions with `bb` so we add a source filter to include these. - bytecodeFilter = path: _type: builtins.match ".*bytecode$" path != null; - witnessFilter = path: _type: builtins.match ".*tr$" path != null; - # We use `.nr` and `.toml` files in tests so we need to create a special source - # filter to include those files in addition to usual rust/cargo source files - noirFilter = path: _type: builtins.match ".*nr$" path != null; - tomlFilter = path: _type: builtins.match ".*toml$" path != null; + # Custom filter with various file extensions that we rely upon to build the compiler or wasm packages + # Currently: `.nr`, `.sol`, `.sh`, `.json`, `.md` sourceFilter = path: type: - (solidityFilter path type) || (bytecodeFilter path type)|| (witnessFilter path type) || (noirFilter path type) || (tomlFilter path type) || (craneLib.filterCargoSources path type); - - # As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses - # gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++ - stdenv = - if (pkgs.stdenv.targetPlatform.isGnu && pkgs.stdenv.targetPlatform.isAarch64) then - pkgs.overrideCC pkgs.llvmPackages.stdenv (pkgs.llvmPackages.clang.override { gccForLibs = pkgs.gcc11.cc; }) - else - pkgs.llvmPackages.stdenv; + (builtins.match ".*\.(nr|sol|sh|json|md)$" path != null) || (craneLib.filterCargoSources path type); - extraBuildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [ + extraBuildInputs = [ ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ # Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156 pkgs.libiconv pkgs.darwin.apple_sdk.frameworks.Security - ] ++ [ - # Need to install various packages used by the `bb` binary. - pkgs.curl - stdenv.cc.cc.lib - pkgs.gcc.cc.lib - pkgs.gzip ]; - sharedArgs = { + environment = { + # We enable backtraces on any failure for help with debugging + RUST_BACKTRACE = "1"; + }; + + # Configuration shared between builds + config = { # x-release-please-start-version version = "0.11.1"; # x-release-please-end @@ -150,128 +103,88 @@ }; # Combine the environment and other configuration needed for crane to build our Rust packages - nativeArgs = nativeEnvironment // sharedArgs // { - pname = "noir-native"; - - # Use our custom stdenv to build and test our Rust project - inherit stdenv; - - nativeBuildInputs = [ - # This provides the pkg-config tool to find barretenberg & other native libraries - pkgs.pkg-config - # This provides the `lld` linker to cargo - pkgs.llvmPackages.bintools - ] ++ pkgs.lib.optionals stdenv.isLinux [ - # This is linux specific and used to patch the rpath and interpreter of the bb binary - pkgs.patchelf - ]; + nativeConfig = environment // config // { + nativeBuildInputs = [ ]; - buildInputs = [ - ] ++ extraBuildInputs; + buildInputs = [ ] ++ extraBuildInputs; }; # Combine the environmnet with cargo args needed to build wasm package - noirWasmArgs = sharedEnvironment // sharedArgs // { - pname = "noir_wasm"; - - src = ./.; + wasmConfig = environment // config // rec { + CARGO_TARGET_DIR = "./target"; - cargoExtraArgs = "--package noir_wasm --target wasm32-unknown-unknown"; + nativeBuildInputs = with pkgs; [ + which + git + jq + rustToolchain + wasm-bindgen-cli + binaryen + toml2json + ]; buildInputs = [ ] ++ extraBuildInputs; - - doCheck = false; }; - # Combine the environment with cargo args needed to build wasm package - noirc_abi_WasmArgs = sharedEnvironment // sharedArgs // { + # Build *just* the cargo dependencies, so we can reuse all of that work between runs + native-cargo-artifacts = craneLib.buildDepsOnly (nativeConfig // { + pname = "nargo"; + }); + noir-wasm-cargo-artifacts = craneLib.buildDepsOnly (wasmConfig // { + pname = "noir_wasm"; + }); + noirc-abi-wasm-cargo-artifacts = craneLib.buildDepsOnly (wasmConfig // { pname = "noirc_abi_wasm"; + }); - src = ./.; + nargo = craneLib.buildPackage (nativeConfig // { + pname = "nargo"; - cargoExtraArgs = "--package noirc_abi_wasm --target wasm32-unknown-unknown"; + inherit GIT_COMMIT GIT_DIRTY; - buildInputs = [ ] ++ extraBuildInputs; + cargoArtifacts = native-cargo-artifacts; + # We don't want to run checks or tests when just building the project doCheck = false; - }; - - # Conditionally download the binary based on whether it is linux or mac - bb_binary = let - platformSpecificUrl = if stdenv.hostPlatform.isLinux then - "https://github.com/AztecProtocol/barretenberg/releases/download/barretenberg-v0.4.3/bb-ubuntu.tar.gz" - else if stdenv.hostPlatform.isDarwin then - "https://github.com/AztecProtocol/barretenberg/releases/download/barretenberg-v0.4.3/barretenberg-x86_64-apple-darwin.tar.gz" - else - throw "Unsupported platform"; - - platformSpecificHash = if stdenv.hostPlatform.isLinux then - "sha256:0rcsjws87f4v28cw9734c10pg7c49apigf4lg3m0ji5vbhhmfnhr" - else if stdenv.hostPlatform.isDarwin then - "sha256:0pnsd56z0vkai7m0advawfgcvq9jbnpqm7lk98n5flqj583x3w35" - else - throw "Unsupported platform"; - in builtins.fetchurl { - url = platformSpecificUrl; - sha256 = platformSpecificHash; - }; + }); + + noir_wasm = craneLib.buildPackage (wasmConfig // rec { + pname = "noir_wasm"; - # The `port` is parameterized to support parallel test runs without colliding static servers - testArgs = port: testEnvironment // { - BB_BINARY_PATH = "/tmp/backend_binary"; - - BB_BINARY_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf bb_binary}"; - - # We provide `barretenberg-transcript00` from the overlay to the tests as a URL hosted via a static server - # This is necessary because the Nix sandbox has no network access and downloading during tests would fail - BARRETENBERG_TRANSCRIPT_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf pkgs.barretenberg-transcript00}"; - - # This copies the `barretenberg-transcript00` from the Nix store into this sandbox - # which avoids exposing the entire Nix store to the static server it starts - # The static server is moved to the background and killed after checks are completed - # - # We also set the NARGO_BACKEND_CACHE_DIR environment variable to the $TMP directory so we can successfully cache - # the transcript; which isn't possible with the default path because the Nix sandbox disabled $HOME - preCheck = '' - echo "Extracting bb binary" - mkdir extracted - tar -xf ${bb_binary} -C extracted - - # Conditionally patch the binary for Linux - ${if stdenv.hostPlatform.isLinux then '' - - cp extracted/cpp/build/bin/bb /tmp/backend_binary - - echo "Patching bb binary for Linux" - patchelf --set-rpath "${stdenv.cc.cc.lib}/lib:${pkgs.gcc.cc.lib}/lib" /tmp/backend_binary - patchelf --set-interpreter ${stdenv.cc.libc}/lib/ld-linux-x86-64.so.2 /tmp/backend_binary - '' else if stdenv.hostPlatform.isDarwin then '' - cp extracted/bb /tmp/backend_binary - '' else - throw "Unsupported platform" - } - - export NARGO_BACKEND_CACHE_DIR=$TMP - cp ${pkgs.barretenberg-transcript00} . - echo "Starting simple static server" - ${pkgs.simple-http-server}/bin/simple-http-server --port ${toString port} --silent & - HTTP_SERVER_PID=$! + inherit GIT_COMMIT GIT_DIRTY; + + cargoArtifacts = noir-wasm-cargo-artifacts; + + cargoExtraArgs = "--package ${pname} --target wasm32-unknown-unknown"; + + buildPhaseCargoCommand = '' + bash compiler/wasm/buildPhaseCargoCommand.sh release ''; - postCheck = '' - kill $HTTP_SERVER_PID + installPhase = '' + bash compiler/wasm/installPhase.sh ''; - }; - # Build *just* the cargo dependencies, so we can reuse all of that work between runs - native-cargo-artifacts = craneLib.buildDepsOnly nativeArgs; - noir-wasm-cargo-artifacts = craneLib.buildDepsOnly noirWasmArgs; - noirc-abi-wasm-cargo-artifacts = craneLib.buildDepsOnly noirc_abi_WasmArgs; + # We don't want to run checks or tests when just building the project + doCheck = false; + }); + + noirc_abi_wasm = craneLib.buildPackage (wasmConfig // rec { + pname = "noirc_abi_wasm"; - noir-native = craneLib.buildPackage (nativeArgs // { inherit GIT_COMMIT GIT_DIRTY; - cargoArtifacts = native-cargo-artifacts; + cargoArtifacts = noirc-abi-wasm-cargo-artifacts; + + cargoExtraArgs = "--package ${pname} --target wasm32-unknown-unknown"; + + buildPhaseCargoCommand = '' + bash tooling/noirc_abi_wasm/buildPhaseCargoCommand.sh release + ''; + + installPhase = '' + bash tooling/noirc_abi_wasm/installPhase.sh + ''; # We don't want to run checks or tests when just building the project doCheck = false; @@ -286,30 +199,30 @@ in rec { checks = { - cargo-clippy = craneLib.cargoClippy (nativeArgs // { - inherit GIT_COMMIT GIT_DIRTY; + cargo-clippy = craneLib.cargoClippy (nativeConfig // { + pname = "noir"; - cargoArtifacts = native-cargo-artifacts; - }); - - cargo-fmt = craneLib.cargoFmt (nativeArgs // { inherit GIT_COMMIT GIT_DIRTY; cargoArtifacts = native-cargo-artifacts; - doCheck = true; }); - cargo-test = craneLib.cargoTest (nativeArgs // (testArgs 8000) // { + cargo-fmt = craneLib.cargoFmt (nativeConfig // { + pname = "noir"; + inherit GIT_COMMIT GIT_DIRTY; cargoArtifacts = native-cargo-artifacts; + doCheck = true; }); }; packages = { - default = noir-native; + default = nargo; - inherit noir-native; + inherit nargo; + inherit noir_wasm; + inherit noirc_abi_wasm; # We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI inherit native-cargo-artifacts; @@ -317,16 +230,17 @@ inherit noirc-abi-wasm-cargo-artifacts; }; - # TODO(#1197): Look into installable apps with Nix flakes - # apps.default = flake-utils.lib.mkApp { drv = nargo; }; - - # Setup the environment to match the stdenv from `nix build` & `nix flake check`, and - # combine it with the environment settings, the inputs from our checks derivations, + # Setup the environment to match the environment settings, the inputs from our checks derivations, # and extra tooling via `nativeBuildInputs` - devShells.default = pkgs.mkShell.override { inherit stdenv; } (nativeEnvironment // wasmEnvironment // testEnvironment // { + devShells.default = pkgs.mkShell (environment // { inputsFrom = builtins.attrValues checks; nativeBuildInputs = with pkgs; [ + # Need to install various packages used by the `bb` binary. + # pkgs.curl + # stdenv.cc.cc.lib + # pkgs.gcc.cc.lib + # pkgs.gzip curl gzip which @@ -340,87 +254,16 @@ jq binaryen yarn - rust-bin.stable."1.66.1".default + rust-bin.stable.${rustVersion}.default rust-analyzer rustup - nodejs-18_x + nodejs-18_x ]; shellHook = '' eval "$(starship init bash)" ''; }); - - # TODO: This fails with a "section too large" error on MacOS so we should limit to linux targets - # or fix the failure - packages.wasm = craneLib.buildPackage (noirWasmArgs // { - - inherit GIT_COMMIT; - inherit GIT_DIRTY; - doCheck = false; - - cargoArtifacts = noir-wasm-cargo-artifacts; - - COMMIT_SHORT = builtins.substring 0 7 GIT_COMMIT; - VERSION_APPENDIX = if GIT_DIRTY == "true" then "-dirty" else ""; - PKG_PATH = "./pkg"; - CARGO_TARGET_DIR = "./target"; - - nativeBuildInputs = with pkgs; [ - which - git - jq - rustToolchain - wasm-bindgen-cli - binaryen - toml2json - ]; - - buildPhaseCargoCommand = '' - bash compiler/wasm/buildPhaseCargoCommand.sh release - ''; - - installPhase = '' - bash compiler/wasm/installPhase.sh - ''; - - }); - - # TODO: This fails with a "section too large" error on MacOS so we should limit to linux targets - # or fix the failure - packages.noirc_abi_wasm = craneLib.buildPackage (noirc_abi_WasmArgs // { - - inherit GIT_COMMIT; - inherit GIT_DIRTY; - doCheck = false; - - cargoArtifacts = noirc-abi-wasm-cargo-artifacts; - - COMMIT_SHORT = builtins.substring 0 7 GIT_COMMIT; - VERSION_APPENDIX = if GIT_DIRTY == "true" then "-dirty" else ""; - PKG_PATH = "./pkg"; - CARGO_TARGET_DIR = "./target"; - - nativeBuildInputs = with pkgs; [ - which - git - jq - rustToolchain - wasm-bindgen-cli - binaryen - toml2json - ]; - - buildPhaseCargoCommand = '' - bash tooling/noirc_abi_wasm/buildPhaseCargoCommand.sh release - ''; - - installPhase = '' - bash tooling/noirc_abi_wasm/installPhase.sh - ''; - - }); - }); }