From 0cb623c3d996638c548c8622c7df96d2cb4d1b05 Mon Sep 17 00:00:00 2001 From: Dan Peebles Date: Mon, 15 Aug 2016 10:27:39 -0400 Subject: [PATCH 1/4] fetchurl: add user agent It would be nice to be able to track Nix requests. It's not trustworthy, but can be helpful for stats and routing in HTTP logs. Since `fetchurl` is used so widely, we should "magically" get a UA on `fetchzip`, `fetchFromGitHub`, and other related fetchers. Since `fetchurl` is only used for fixed-output derivations, this should cause no mass rebuild. User-Agent example: curl/7.57.0 Nixpkgs/18.03 --- pkgs/build-support/fetchurl/builder.sh | 25 ++++++++++++++----------- pkgs/build-support/fetchurl/default.nix | 4 +++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index 7c2bdf260b4e2..530864742f603 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -2,20 +2,23 @@ source $stdenv/setup source $mirrorsFile +curlVersion=$(curl -V | head -1 | cut -d' ' -f2) # Curl flags to handle redirects, not use EPSV, handle cookies for # servers to need them during redirects, and work on SSL without a # certificate (this isn't a security problem because we check the # cryptographic hash of the output anyway). -curl="curl \ - --location --max-redirs 20 \ - --retry 3 \ - --disable-epsv \ - --cookie-jar cookies \ - --insecure \ - $curlOpts \ - $NIX_CURL_FLAGS" - +curl=( + curl + --location + --max-redirs 20 + --disable-epsv + --cookie-jar cookies + --insecure + --user-agent "curl/$curlVersion Nixpkgs/$nixpkgsVersion" + $curlOpts + $NIX_CURL_FLAGS +) downloadedFile="$out" if [ -n "$downloadToTemp" ]; then downloadedFile="$TMPDIR/file"; fi @@ -32,7 +35,7 @@ tryDownload() { # if we get error code 18, resume partial download while [ $curlexit -eq 18 ]; do # keep this inside an if statement, since on failure it doesn't abort the script - if $curl -C - --fail "$url" --output "$downloadedFile"; then + if "${curl[@]}" -C - --fail "$url" --output "$downloadedFile"; then success=1 break else @@ -61,7 +64,7 @@ tryHashedMirrors() { for mirror in $hashedMirrors; do url="$mirror/$outputHashAlgo/$outputHash" - if $curl --retry 0 --connect-timeout "${NIX_CONNECT_TIMEOUT:-15}" \ + if "${curl[@]}" --retry 0 --connect-timeout "${NIX_CONNECT_TIMEOUT:-15}" \ --fail --silent --show-error --head "$url" \ --write-out "%{http_code}" --output /dev/null > code 2> log; then tryDownload "$url" diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 8dac273eb1ca9..9ab3494b2b0a9 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -95,7 +95,7 @@ assert sha512 != "" -> builtins.compareVersions "1.11" builtins.nixVersion <= 0; let - + inherit (stdenv.lib) fileContents; hasHash = showURLs || (outputHash != "" && outputHashAlgo != "") || sha1 != "" || sha256 != "" || sha512 != ""; urls_ = if urls != [] then urls else [url]; @@ -132,6 +132,8 @@ else stdenv.mkDerivation { impureEnvVars = impureEnvVars ++ netrcImpureEnvVars; + nixpkgsVersion = fileContents ../../../.version; + # Doing the download on a remote machine just duplicates network # traffic, so don't do that. preferLocalBuild = true; From a26d9e3ba8680865fea4b2cb906e7de954f3aa27 Mon Sep 17 00:00:00 2001 From: Yegor Timoshenko Date: Mon, 11 Jun 2018 01:48:00 +0000 Subject: [PATCH 2/4] Readd --retry 3 --- pkgs/build-support/fetchurl/builder.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index 530864742f603..f9bc8b602f4cc 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -12,6 +12,7 @@ curl=( curl --location --max-redirs 20 + --retry 3 --disable-epsv --cookie-jar cookies --insecure From 5a232b0f5b71814bdf60c757c4edd2b535bff487 Mon Sep 17 00:00:00 2001 From: Yegor Timoshenko Date: Mon, 11 Jun 2018 01:50:06 +0000 Subject: [PATCH 3/4] Leverage lib.release --- pkgs/build-support/fetchurl/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index f74465be9b439..88d89feb8232a 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -133,7 +133,7 @@ stdenvNoCC.mkDerivation { impureEnvVars = impureEnvVars ++ netrcImpureEnvVars; - nixpkgsVersion = lib.fileContents ../../../.version; + nixpkgsVersion = lib.release; # Doing the download on a remote machine just duplicates network # traffic, so don't do that. From f44012ba10537e1a2a94b2be443a86e4dd0ad38f Mon Sep 17 00:00:00 2001 From: Yegor Timoshenko Date: Mon, 11 Jun 2018 02:14:18 +0000 Subject: [PATCH 4/4] lib.release -> lib.trivial.release --- pkgs/build-support/fetchurl/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 88d89feb8232a..5f0c1384c79ed 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -133,7 +133,7 @@ stdenvNoCC.mkDerivation { impureEnvVars = impureEnvVars ++ netrcImpureEnvVars; - nixpkgsVersion = lib.release; + nixpkgsVersion = lib.trivial.release; # Doing the download on a remote machine just duplicates network # traffic, so don't do that.