Git object hashing

Part of RFC 133 Extracted from our old IPFS branches. Co-Authored-By: Matthew Bauer <mjbauer95@gmail.com> Co-Authored-By: Carlo Nucera <carlo.nucera@protonmail.com>
NixOS · Sep 8, 2023 · 3c52443 · 3c52443
1 parent 00d8f9a
commit 3c52443
Show file tree

Hide file tree

Showing 35 changed files with 918 additions and 162 deletions.
diff --git a/Makefile b/Makefile
@@ -29,6 +29,7 @@ makefiles += \
   tests/local.mk \
   tests/ca/local.mk \
   tests/dyn-drv/local.mk \
+  tests/git-hashing/local.mk \
   tests/test-libstoreconsumer/local.mk \
   tests/plugins/local.mk
 else

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
@@ -1111,7 +1111,10 @@ drvName, Bindings * attrs, Value & v)
         auto handleHashMode = [&](const std::string_view s) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else if (s == "text") {
+            else if (s == "flat") {
+                experimentalFeatureSettings.require(Xp::GitHashing);
+                ingestionMethod = FileIngestionMethod::Flat;
+            } else if (s == "text") {
                 experimentalFeatureSettings.require(Xp::DynamicDerivations);
                 ingestionMethod = TextIngestionMethod {};
             } else

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
@@ -30,9 +30,13 @@ void emitTreeAttrs(
 
     // FIXME: support arbitrary input attributes.
 
-    auto narHash = input.getNarHash();
-    assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
+    if (auto narHash = input.getNarHash()) {
+        attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
+    } else if (auto treeHash = input.getTreeHash()) {
+        attrs.alloc("treeHash").mkString(treeHash->to_string(SRI, true));
+    } else
+        /* Must have either tree hash or NAR hash */
+        assert(false);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(
@@ -50,6 +54,11 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
+        if (auto treeHash = input.getTreeHash()) {
+            attrs.alloc("treeHash").mkString(treeHash->gitRev());
+            attrs.alloc("shortTreeHash").mkString(treeHash->gitRev());
+        }
+
         if (auto revCount = input.getRevCount())
             attrs.alloc("revCount").mkInt(*revCount);
         else if (emptyRevFallback)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -25,6 +25,8 @@ static void fixupInput(Input & input)
     input.getRef();
     if (input.getRev())
         input.locked = true;
+    if (input.getTreeHash())
+        input.locked = true;
     input.getRevCount();
     input.getLastModified();
     if (input.getNarHash())
@@ -89,7 +91,7 @@ Attrs Input::toAttrs() const
 
 bool Input::hasAllInfo() const
 {
-    return getNarHash() && scheme && scheme->hasAllInfo(*this);
+    return scheme && scheme->hasAllInfo(*this);
 }
 
 bool Input::operator ==(const Input & other) const
@@ -213,14 +215,19 @@ std::string Input::getName() const
 
 StorePath Input::computeStorePath(Store & store) const
 {
-    auto narHash = getNarHash();
-    if (!narHash)
-        throw Error("cannot compute store path for unlocked input '%s'", to_string());
-    return store.makeFixedOutputPath(getName(), FixedOutputInfo {
-        .method = FileIngestionMethod::Recursive,
-        .hash = *narHash,
-        .references = {},
-    });
+    if (auto treeHash = getTreeHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Git,
+            .hash = *treeHash,
+            .references = {},
+        });
+    if (auto narHash = getNarHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Recursive,
+            .hash = *narHash,
+            .references = {},
+        });
+    throw Error("cannot compute store path for unlocked input '%s'", to_string());
 }
 
 std::string Input::getType() const
@@ -262,6 +269,15 @@ std::optional<Hash> Input::getRev() const
     return hash;
 }
 
+std::optional<Hash> Input::getTreeHash() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "treeHash")) {
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return Hash::parseAny(*s, htSHA1);
+    }
+    return {};
+}
+
 std::optional<uint64_t> Input::getRevCount() const
 {
     if (auto n = maybeGetIntAttr(attrs, "revCount"))

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
@@ -110,6 +110,7 @@ public:
     std::optional<Hash> getNarHash() const;
     std::optional<std::string> getRef() const;
     std::optional<Hash> getRev() const;
+    std::optional<Hash> getTreeHash() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
 };