From ac20bcf44925e7d290bf2f3135bd234ad6bc1bf7 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 12 Jan 2024 13:02:17 -0800 Subject: [PATCH 1/6] writeShellApplication: Add `bashOptions` argument --- pkgs/build-support/trivial-builders/default.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index 9643c9ba048eb93..a8a07f6d071e38e 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -272,6 +272,7 @@ rec { , meta ? { } , checkPhase ? null , excludeShellChecks ? [ ] + , bashOptions ? [ "errexit" "nounset" "pipefail" ] }: writeTextFile { inherit name meta; @@ -281,9 +282,7 @@ rec { preferLocalBuild = false; text = '' #!${runtimeShell} - set -o errexit - set -o nounset - set -o pipefail + ${lib.concatMapStringsSep "\n" (option: "set -o ${option}") bashOptions} '' + lib.optionalString (runtimeInputs != [ ]) '' export PATH="${lib.makeBinPath runtimeInputs}:$PATH" From a64766913f229e5e63e5a0377df9b4837c87d36f Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 12 Jan 2024 13:30:36 -0800 Subject: [PATCH 2/6] writeShellApplication: Add `runtimeEnv` argument --- pkgs/build-support/trivial-builders/default.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index a8a07f6d071e38e..8a5fc288f1ecd8c 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -269,6 +269,7 @@ rec { { name , text , runtimeInputs ? [ ] + , runtimeEnv ? null , meta ? { } , checkPhase ? null , excludeShellChecks ? [ ] @@ -283,7 +284,15 @@ rec { text = '' #!${runtimeShell} ${lib.concatMapStringsSep "\n" (option: "set -o ${option}") bashOptions} - '' + lib.optionalString (runtimeInputs != [ ]) '' + '' + lib.optionalString (runtimeEnv != null) + (lib.concatStrings + (lib.mapAttrsToList + (name: value: '' + ${lib.toShellVar name value} + export ${name} + '') + runtimeEnv)) + + lib.optionalString (runtimeInputs != [ ]) '' export PATH="${lib.makeBinPath runtimeInputs}:$PATH" '' + '' From 863786b98b36025ddca4c0678b39bbdf383468a0 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 29 Jan 2024 10:03:51 -0800 Subject: [PATCH 3/6] writeTextFile,writeShellApplication: Allow setting extra arguments --- pkgs/build-support/trivial-builders/default.nix | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index 8a5fc288f1ecd8c..3e63e12febd2293 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -152,19 +152,21 @@ rec { , meta ? { } , allowSubstitutes ? false , preferLocalBuild ? true + , derivationArgs ? { } # Extra arguments to pass to `stdenv.mkDerivation` }: let matches = builtins.match "/bin/([^/]+)" destination; in runCommand name - { + ({ inherit text executable checkPhase allowSubstitutes preferLocalBuild; - passAsFile = [ "text" ]; + passAsFile = [ "text" ] + ++ derivationArgs.passAsFile or [ ]; meta = lib.optionalAttrs (executable && matches != null) { mainProgram = lib.head matches; - } // meta; - } + } // meta // derivationArgs.meta or {}; + } // removeAttrs derivationArgs [ "passAsFile" "meta" ]) '' target=$out${lib.escapeShellArg destination} mkdir -p "$(dirname "$target")" @@ -274,9 +276,10 @@ rec { , checkPhase ? null , excludeShellChecks ? [ ] , bashOptions ? [ "errexit" "nounset" "pipefail" ] + , derivationArgs ? { } # Extra arguments to pass to `stdenv.mkDerivation` }: writeTextFile { - inherit name meta; + inherit name meta derivationArgs; executable = true; destination = "/bin/${name}"; allowSubstitutes = true; From 63f7c9b415fa09a4d7ae5b01f3ecdd420c30cbed Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 12 Jan 2024 13:42:53 -0800 Subject: [PATCH 4/6] writeShellApplication: Document arguments --- .../trivial-builders/default.nix | 104 ++++++++++++------ 1 file changed, 68 insertions(+), 36 deletions(-) diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index 3e63e12febd2293..a38231bdcaa32f5 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -240,43 +240,75 @@ rec { meta.mainProgram = name; }; - /* - Similar to writeShellScriptBin and writeScriptBin. - Writes an executable Shell script to /nix/store//bin/ and - checks its syntax with shellcheck and the shell's -n option. - Individual checks can be foregone by putting them in the excludeShellChecks - list, e.g. [ "SC2016" ]. - Automatically includes sane set of shellopts (errexit, nounset, pipefail) - and handles creation of PATH based on runtimeInputs - - Note that the checkPhase uses stdenv.shell for the test run of the script, - while the generated shebang uses runtimeShell. If, for whatever reason, - those were to mismatch you might lose fidelity in the default checks. - - Example: - - Writes my-file to /nix/store//bin/my-file and makes executable. - - - writeShellApplication { - name = "my-file"; - runtimeInputs = [ curl w3m ]; - text = '' - curl -s 'https://nixos.org' | w3m -dump -T text/html - ''; - } - - */ + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing writeShellApplication = - { name - , text - , runtimeInputs ? [ ] - , runtimeEnv ? null - , meta ? { } - , checkPhase ? null - , excludeShellChecks ? [ ] - , bashOptions ? [ "errexit" "nounset" "pipefail" ] - , derivationArgs ? { } # Extra arguments to pass to `stdenv.mkDerivation` + { + /* + The name of the script to write. + + Type: String + */ + name, + /* + The shell script's text, not including a shebang. + + Type: String + */ + text, + /* + Inputs to add to the shell script's `$PATH` at runtime. + + Type: [String|Derivation] + */ + runtimeInputs ? [ ], + /* + Extra environment variables to set at runtime. + + Type: AttrSet + */ + runtimeEnv ? null, + /* + `stdenv.mkDerivation`'s `meta` argument. + + Type: AttrSet + */ + meta ? { }, + /* + The `checkPhase` to run. Defaults to `shellcheck` on supported + platforms and `bash -n`. + + The script path will be given as `$target` in the `checkPhase`. + + Type: String + */ + checkPhase ? null, + /* + Checks to exclude when running `shellcheck`, e.g. `[ "SC2016" ]`. + + See for a list of checks. + + Type: [String] + */ + excludeShellChecks ? [ ], + /* + Bash options to activate with `set -o` at the start of the script. + + Defaults to `[ "errexit" "nounset" "pipefail" ]`. + + Type: [String] + */ + bashOptions ? [ "errexit" "nounset" "pipefail" ], + /* Extra arguments to pass to `stdenv.mkDerivation`. + + :::{.caution} + Certain derivation attributes are used internally, + overriding those could cause problems. + ::: + + Type: AttrSet + */ + derivationArgs ? { }, }: writeTextFile { inherit name meta derivationArgs; From 88ce0b00191ff1d7093709696538e403c990285a Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 12 Jan 2024 16:39:00 -0800 Subject: [PATCH 5/6] writeShellApplication: Expand test suite --- .../test/writeShellApplication.nix | 152 +++++++++++++++--- 1 file changed, 132 insertions(+), 20 deletions(-) diff --git a/pkgs/build-support/trivial-builders/test/writeShellApplication.nix b/pkgs/build-support/trivial-builders/test/writeShellApplication.nix index 6ce6f0720fcf6a5..c50f5a4d283f9ec 100644 --- a/pkgs/build-support/trivial-builders/test/writeShellApplication.nix +++ b/pkgs/build-support/trivial-builders/test/writeShellApplication.nix @@ -1,29 +1,141 @@ -/* - Run with: +# Run with: +# nix-build -A tests.trivial-builders.writeShellApplication +{ writeShellApplication +, writeTextFile +, runCommand +, lib +, linkFarm +, diffutils +, hello +}: +let + checkShellApplication = args@{name, expected, ...}: + let + writeShellApplicationArgs = builtins.removeAttrs args ["expected"]; + script = writeShellApplication writeShellApplicationArgs; + executable = lib.getExe script; + expected' = writeTextFile { + name = "${name}-expected"; + text = expected; + }; + actual = "${name}-actual"; + in + runCommand name { } '' + echo "Running test executable ${name}" + ${executable} > ${actual} + echo "Got output from test executable:" + cat ${actual} + echo "Checking test output against expected output:" + ${diffutils}/bin/diff --color --unified ${expected'} ${actual} + touch $out + ''; +in +linkFarm "writeShellApplication-tests" { + test-meta = + let + script = writeShellApplication { + name = "test-meta"; + text = ""; + meta.description = "A test for the `writeShellApplication` `meta` argument."; + }; + in + assert script.meta.mainProgram == "test-meta"; + assert script.meta.description == "A test for the `writeShellApplication` `meta` argument."; + script; - cd nixpkgs - nix-build -A tests.trivial-builders.writeShellApplication -*/ + test-runtime-inputs = + checkShellApplication { + name = "test-runtime-inputs"; + text = '' + hello + ''; + runtimeInputs = [ hello ]; + expected = "Hello, world!\n"; + }; -{ lib, writeShellApplication, runCommand }: -let - pkg = writeShellApplication { - name = "test-script"; + test-runtime-env = + checkShellApplication { + name = "test-runtime-env"; + runtimeEnv = { + MY_COOL_ENV_VAR = "my-cool-env-value"; + MY_OTHER_COOL_ENV_VAR = "my-other-cool-env-value"; + # Check that we can serialize a bunch of different types: + BOOL = true; + INT = 1; + LIST = [1 2 3]; + MAP = { + a = "a"; + b = "b"; + }; + }; + text = '' + echo "$MY_COOL_ENV_VAR" + echo "$MY_OTHER_COOL_ENV_VAR" + ''; + expected = '' + my-cool-env-value + my-other-cool-env-value + ''; + }; + + test-check-phase = + checkShellApplication { + name = "test-check-phase"; + text = ""; + checkPhase = '' + echo "echo -n hello" > $target + ''; + expected = "hello"; + }; + + test-argument-forwarding = + checkShellApplication { + name = "test-argument-forwarding"; + text = ""; + derivationArgs.MY_BUILD_TIME_VARIABLE = "puppy"; + derivationArgs.postCheck = '' + if [[ "$MY_BUILD_TIME_VARIABLE" != puppy ]]; then + echo "\$MY_BUILD_TIME_VARIABLE is not set to 'puppy'!" + exit 1 + fi + ''; + meta.description = "A test checking that `writeShellApplication` forwards extra arguments to `stdenv.mkDerivation`."; + expected = ""; + }; + + test-exclude-shell-checks = writeShellApplication { + name = "test-exclude-shell-checks"; excludeShellChecks = [ "SC2016" ]; text = '' - echo -e '#!/usr/bin/env bash\n' \ - 'echo "$SHELL"' > /tmp/something.sh # this line would normally - # ...cause shellcheck error + # Triggers SC2016: Expressions don't expand in single quotes, use double + # quotes for that. + echo '$SHELL' ''; }; -in - assert pkg.meta.mainProgram == "test-script"; - runCommand "test-writeShellApplication" { } '' - echo Testing if writeShellApplication builds without shellcheck error... - - target=${lib.getExe pkg} + test-bash-options-pipefail = checkShellApplication { + name = "test-bash-options-pipefail"; + text = '' + touch my-test-file + echo puppy | grep doggy | sed 's/doggy/puppy/g' + # ^^^^^^^^^^ This will fail. + true + ''; + # Don't use `pipefail`: + bashOptions = ["errexit" "nounset"]; + expected = ""; + }; - touch $out - '' + test-bash-options-nounset = checkShellApplication { + name = "test-bash-options-nounset"; + text = '' + echo -n "$someUndefinedVariable" + ''; + # Don't use `nounset`: + bashOptions = []; + # Don't warn about the undefined variable at build time: + excludeShellChecks = [ "SC2154" ]; + expected = ""; + }; +} From 41376dd064e8be5c1d936decca6da2cdd8a7a731 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 22 Jan 2024 09:33:49 -0800 Subject: [PATCH 6/6] writeShellApplication: Update manual --- doc/build-helpers/trivial-build-helpers.chapter.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/build-helpers/trivial-build-helpers.chapter.md b/doc/build-helpers/trivial-build-helpers.chapter.md index 4648c7985542376..384e25035060e35 100644 --- a/doc/build-helpers/trivial-build-helpers.chapter.md +++ b/doc/build-helpers/trivial-build-helpers.chapter.md @@ -502,9 +502,14 @@ concatScript "my-file" [ file1 file2 ] ## `writeShellApplication` {#trivial-builder-writeShellApplication} -This can be used to easily produce a shell script that has some dependencies (`runtimeInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck). +`writeShellApplication` is similar to `writeShellScriptBin` and `writeScriptBin` but supports runtime dependencies with `runtimeInputs`. +Writes an executable shell script to `/nix/store//bin/` and checks its syntax with [`shellcheck`](https://github.com/koalaman/shellcheck) and the `bash`'s `-n` option. +Some basic Bash options are set by default (`errexit`, `nounset`, and `pipefail`), but can be overridden with `bashOptions`. -For example, look at the following code: +Extra arguments may be passed to `stdenv.mkDerivation` by setting `derivationArgs`; note that variables set in this manner will be set when the shell script is _built,_ not when it's run. +Runtime environment variables can be set with the `runtimeEnv` argument. + +For example, the following shell application can refer to `curl` directly, rather than needing to write `${curl}/bin/curl`: ```nix writeShellApplication { @@ -518,10 +523,6 @@ writeShellApplication { } ``` -Unlike with normal `writeShellScriptBin`, there is no need to manually write out `${curl}/bin/curl`, setting the PATH -was handled by `writeShellApplication`. Moreover, the script is being checked with `shellcheck` for more strict -validation. - ## `symlinkJoin` {#trivial-builder-symlinkJoin} This can be used to put many derivations into the same directory structure. It works by creating a new derivation and adding symlinks to each of the paths listed. It expects two arguments, `name`, and `paths`. `name` is the name used in the Nix store path for the created derivation. `paths` is a list of paths that will be symlinked. These paths can be to Nix store derivations or any other subdirectory contained within.