Skip to content

Commit

Permalink
Merge pull request #280592 from 9999years/write-shell-application-opt…
Browse files Browse the repository at this point in the history
…ions

writeShellApplication: Document and extend arguments
  • Loading branch information
infinisil committed Feb 2, 2024
2 parents 55ae7c5 + 41376dd commit 3c29f55
Show file tree
Hide file tree
Showing 3 changed files with 224 additions and 68 deletions.
13 changes: 7 additions & 6 deletions doc/build-helpers/trivial-build-helpers.chapter.md
Original file line number Diff line number Diff line change
Expand Up @@ -502,9 +502,14 @@ concatScript "my-file" [ file1 file2 ]

## `writeShellApplication` {#trivial-builder-writeShellApplication}

This can be used to easily produce a shell script that has some dependencies (`runtimeInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck).
`writeShellApplication` is similar to `writeShellScriptBin` and `writeScriptBin` but supports runtime dependencies with `runtimeInputs`.
Writes an executable shell script to `/nix/store/<store path>/bin/<name>` and checks its syntax with [`shellcheck`](https://github.com/koalaman/shellcheck) and the `bash`'s `-n` option.
Some basic Bash options are set by default (`errexit`, `nounset`, and `pipefail`), but can be overridden with `bashOptions`.

For example, look at the following code:
Extra arguments may be passed to `stdenv.mkDerivation` by setting `derivationArgs`; note that variables set in this manner will be set when the shell script is _built,_ not when it's run.
Runtime environment variables can be set with the `runtimeEnv` argument.

For example, the following shell application can refer to `curl` directly, rather than needing to write `${curl}/bin/curl`:

```nix
writeShellApplication {
Expand All @@ -518,10 +523,6 @@ writeShellApplication {
}
```

Unlike with normal `writeShellScriptBin`, there is no need to manually write out `${curl}/bin/curl`, setting the PATH
was handled by `writeShellApplication`. Moreover, the script is being checked with `shellcheck` for more strict
validation.

## `symlinkJoin` {#trivial-builder-symlinkJoin}

This can be used to put many derivations into the same directory structure. It works by creating a new derivation and adding symlinks to each of the paths listed. It expects two arguments, `name`, and `paths`. `name` is the name used in the Nix store path for the created derivation. `paths` is a list of paths that will be symlinked. These paths can be to Nix store derivations or any other subdirectory contained within.
Expand Down
127 changes: 85 additions & 42 deletions pkgs/build-support/trivial-builders/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -152,19 +152,21 @@ rec {
, meta ? { }
, allowSubstitutes ? false
, preferLocalBuild ? true
, derivationArgs ? { } # Extra arguments to pass to `stdenv.mkDerivation`
}:
let
matches = builtins.match "/bin/([^/]+)" destination;
in
runCommand name
{
({
inherit text executable checkPhase allowSubstitutes preferLocalBuild;
passAsFile = [ "text" ];
passAsFile = [ "text" ]
++ derivationArgs.passAsFile or [ ];
meta = lib.optionalAttrs (executable && matches != null)
{
mainProgram = lib.head matches;
} // meta;
}
} // meta // derivationArgs.meta or {};
} // removeAttrs derivationArgs [ "passAsFile" "meta" ])
''
target=$out${lib.escapeShellArg destination}
mkdir -p "$(dirname "$target")"
Expand Down Expand Up @@ -238,53 +240,94 @@ rec {
meta.mainProgram = name;
};

/*
Similar to writeShellScriptBin and writeScriptBin.
Writes an executable Shell script to /nix/store/<store path>/bin/<name> and
checks its syntax with shellcheck and the shell's -n option.
Individual checks can be foregone by putting them in the excludeShellChecks
list, e.g. [ "SC2016" ].
Automatically includes sane set of shellopts (errexit, nounset, pipefail)
and handles creation of PATH based on runtimeInputs
Note that the checkPhase uses stdenv.shell for the test run of the script,
while the generated shebang uses runtimeShell. If, for whatever reason,
those were to mismatch you might lose fidelity in the default checks.
Example:
Writes my-file to /nix/store/<store path>/bin/my-file and makes executable.
writeShellApplication {
name = "my-file";
runtimeInputs = [ curl w3m ];
text = ''
curl -s 'https://nixos.org' | w3m -dump -T text/html
'';
}
*/
# See doc/build-helpers/trivial-build-helpers.chapter.md
# or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing
writeShellApplication =
{ name
, text
, runtimeInputs ? [ ]
, meta ? { }
, checkPhase ? null
, excludeShellChecks ? [ ]
{
/*
The name of the script to write.
Type: String
*/
name,
/*
The shell script's text, not including a shebang.
Type: String
*/
text,
/*
Inputs to add to the shell script's `$PATH` at runtime.
Type: [String|Derivation]
*/
runtimeInputs ? [ ],
/*
Extra environment variables to set at runtime.
Type: AttrSet
*/
runtimeEnv ? null,
/*
`stdenv.mkDerivation`'s `meta` argument.
Type: AttrSet
*/
meta ? { },
/*
The `checkPhase` to run. Defaults to `shellcheck` on supported
platforms and `bash -n`.
The script path will be given as `$target` in the `checkPhase`.
Type: String
*/
checkPhase ? null,
/*
Checks to exclude when running `shellcheck`, e.g. `[ "SC2016" ]`.
See <https://www.shellcheck.net/wiki/> for a list of checks.
Type: [String]
*/
excludeShellChecks ? [ ],
/*
Bash options to activate with `set -o` at the start of the script.
Defaults to `[ "errexit" "nounset" "pipefail" ]`.
Type: [String]
*/
bashOptions ? [ "errexit" "nounset" "pipefail" ],
/* Extra arguments to pass to `stdenv.mkDerivation`.
:::{.caution}
Certain derivation attributes are used internally,
overriding those could cause problems.
:::
Type: AttrSet
*/
derivationArgs ? { },
}:
writeTextFile {
inherit name meta;
inherit name meta derivationArgs;
executable = true;
destination = "/bin/${name}";
allowSubstitutes = true;
preferLocalBuild = false;
text = ''
#!${runtimeShell}
set -o errexit
set -o nounset
set -o pipefail
'' + lib.optionalString (runtimeInputs != [ ]) ''
${lib.concatMapStringsSep "\n" (option: "set -o ${option}") bashOptions}
'' + lib.optionalString (runtimeEnv != null)
(lib.concatStrings
(lib.mapAttrsToList
(name: value: ''
${lib.toShellVar name value}
export ${name}
'')
runtimeEnv))
+ lib.optionalString (runtimeInputs != [ ]) ''
export PATH="${lib.makeBinPath runtimeInputs}:$PATH"
'' + ''
Expand Down
152 changes: 132 additions & 20 deletions pkgs/build-support/trivial-builders/test/writeShellApplication.nix
Original file line number Diff line number Diff line change
@@ -1,29 +1,141 @@
/*
Run with:
# Run with:
# nix-build -A tests.trivial-builders.writeShellApplication
{ writeShellApplication
, writeTextFile
, runCommand
, lib
, linkFarm
, diffutils
, hello
}:
let
checkShellApplication = args@{name, expected, ...}:
let
writeShellApplicationArgs = builtins.removeAttrs args ["expected"];
script = writeShellApplication writeShellApplicationArgs;
executable = lib.getExe script;
expected' = writeTextFile {
name = "${name}-expected";
text = expected;
};
actual = "${name}-actual";
in
runCommand name { } ''
echo "Running test executable ${name}"
${executable} > ${actual}
echo "Got output from test executable:"
cat ${actual}
echo "Checking test output against expected output:"
${diffutils}/bin/diff --color --unified ${expected'} ${actual}
touch $out
'';
in
linkFarm "writeShellApplication-tests" {
test-meta =
let
script = writeShellApplication {
name = "test-meta";
text = "";
meta.description = "A test for the `writeShellApplication` `meta` argument.";
};
in
assert script.meta.mainProgram == "test-meta";
assert script.meta.description == "A test for the `writeShellApplication` `meta` argument.";
script;

cd nixpkgs
nix-build -A tests.trivial-builders.writeShellApplication
*/
test-runtime-inputs =
checkShellApplication {
name = "test-runtime-inputs";
text = ''
hello
'';
runtimeInputs = [ hello ];
expected = "Hello, world!\n";
};

{ lib, writeShellApplication, runCommand }:
let
pkg = writeShellApplication {
name = "test-script";
test-runtime-env =
checkShellApplication {
name = "test-runtime-env";
runtimeEnv = {
MY_COOL_ENV_VAR = "my-cool-env-value";
MY_OTHER_COOL_ENV_VAR = "my-other-cool-env-value";
# Check that we can serialize a bunch of different types:
BOOL = true;
INT = 1;
LIST = [1 2 3];
MAP = {
a = "a";
b = "b";
};
};
text = ''
echo "$MY_COOL_ENV_VAR"
echo "$MY_OTHER_COOL_ENV_VAR"
'';
expected = ''
my-cool-env-value
my-other-cool-env-value
'';
};

test-check-phase =
checkShellApplication {
name = "test-check-phase";
text = "";
checkPhase = ''
echo "echo -n hello" > $target
'';
expected = "hello";
};

test-argument-forwarding =
checkShellApplication {
name = "test-argument-forwarding";
text = "";
derivationArgs.MY_BUILD_TIME_VARIABLE = "puppy";
derivationArgs.postCheck = ''
if [[ "$MY_BUILD_TIME_VARIABLE" != puppy ]]; then
echo "\$MY_BUILD_TIME_VARIABLE is not set to 'puppy'!"
exit 1
fi
'';
meta.description = "A test checking that `writeShellApplication` forwards extra arguments to `stdenv.mkDerivation`.";
expected = "";
};

test-exclude-shell-checks = writeShellApplication {
name = "test-exclude-shell-checks";
excludeShellChecks = [ "SC2016" ];
text = ''
echo -e '#!/usr/bin/env bash\n' \
'echo "$SHELL"' > /tmp/something.sh # this line would normally
# ...cause shellcheck error
# Triggers SC2016: Expressions don't expand in single quotes, use double
# quotes for that.
echo '$SHELL'
'';
};
in
assert pkg.meta.mainProgram == "test-script";
runCommand "test-writeShellApplication" { } ''

echo Testing if writeShellApplication builds without shellcheck error...
target=${lib.getExe pkg}
test-bash-options-pipefail = checkShellApplication {
name = "test-bash-options-pipefail";
text = ''
touch my-test-file
echo puppy | grep doggy | sed 's/doggy/puppy/g'
# ^^^^^^^^^^ This will fail.
true
'';
# Don't use `pipefail`:
bashOptions = ["errexit" "nounset"];
expected = "";
};

touch $out
''
test-bash-options-nounset = checkShellApplication {
name = "test-bash-options-nounset";
text = ''
echo -n "$someUndefinedVariable"
'';
# Don't use `nounset`:
bashOptions = [];
# Don't warn about the undefined variable at build time:
excludeShellChecks = [ "SC2154" ];
expected = "";
};

}

0 comments on commit 3c29f55

Please sign in to comment.