Add explicit allRefs = true; argument to fetchGit

Sometimes it's necessary to fetch a git repository at a revision and
it's unknown which ref contains the revision in question. An example
would be a Cargo.lock which only provides the URL and the revision when
using a git repository as build input.

However it's considered a bad practice to perform a full checkout of a
repository since this may take a lot of time and can eat up a lot of
disk space. This patch makes a full checkout explicit by adding an
`allRefs` argument to `builtins.fetchGit` which fetches all refs if
explicitly set to true.

Closes NixOS#2409

doc/manual/expressions/builtins.xml

@@ -432,6 +432,14 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+         <term>allRefs</term>
+         <listitem>
+          <para>
+           If set to <literal>true</literal>, it fetches all <literal>refs</literal> of a git repository.
+          </para>
+         </listitem>
+        </varlistentry>
       </variablelist>
 
       <example>

src/libexpr/primops/fetchGit.cc

@@ -14,6 +14,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     std::optional<Hash> rev;
     std::string name = "source";
     bool fetchSubmodules = false;
+    bool allRefs = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -34,6 +35,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "submodules")
                 fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
+            else if (n == "allRefs")
+                allRefs = state.forceBool(*attr.value, *attr.pos);
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
@@ -63,6 +66,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
     if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
+    if (allRefs) attrs.insert_or_assign("allRefs", true);
     auto input = fetchers::inputFromAttrs(attrs);
 
     // FIXME: use name?

src/libfetchers/git.cc

@@ -29,6 +29,7 @@ struct GitInput : Input
     std::optional<Hash> rev;
     bool shallow = false;
     bool submodules = false;
+    bool allRefs = false;
 
     GitInput(const ParsedURL & url) : url(url)
     { }
@@ -100,6 +101,7 @@ struct GitInput : Input
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
         if (submodules) cacheType += "-submodules";
+        if (allRefs) cacheType += "-all-refs";
 
         auto getImmutableAttrs = [&]()
         {
@@ -269,11 +271,15 @@ struct GitInput : Input
                     }
                 }
             } else {
-                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-                   git fetch to update the local ref to the remote ref. */
-                struct stat st;
-                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+                if (allRefs) {
+                    doFetch = true;
+                } else {
+                    /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                       git fetch to update the local ref to the remote ref. */
+                    struct stat st;
+                    doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+                }
             }
 
             if (doFetch) {
@@ -282,9 +288,11 @@ struct GitInput : Input
                 // FIXME: git stderr messes up our progress indicator, so
                 // we're using --quiet for now. Should process its stderr.
                 try {
-                    auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
-                        ? *input->ref
-                        : "refs/heads/" + *input->ref;
+                    auto fetchRef = allRefs
+                        ? "refs/*"
+                        : input->ref->compare(0, 5, "refs/") == 0
+                            ? *input->ref
+                            : "refs/heads/" + *input->ref;
                     runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
@@ -432,7 +440,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "allRefs")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
@@ -448,6 +456,11 @@ struct GitInputScheme : InputScheme
 
         input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
 
+        input->allRefs = maybeGetBoolAttr(attrs, "allRefs").value_or(false);
+
+        if (input->allRefs && input->shallow)
+            throw Error("Cannot perform a shallow git-fetch if all refs should be fetched!");
+
         return input;
     }
 };