From 845fc3f605be6dd1140ab81eefb969da1cc5346b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2022 22:09:32 +0100 Subject: [PATCH 1/8] Merge toDerivations() into toDerivedPaths() toDerivedPaths() now returns DerivedPathWithInfo, which is DerivedPath with some attributes needed by 'nix profile' etc. Preparation for #7417. --- src/libcmd/installables.cc | 175 +++++++++++++++++-------------------- src/libcmd/installables.hh | 47 +++++----- src/nix/app.cc | 5 +- src/nix/build.cc | 10 ++- src/nix/log.cc | 2 +- src/nix/profile.cc | 67 ++++++++------ src/nix/store-copy-log.cc | 8 +- src/nix/why-depends.cc | 2 +- 8 files changed, 154 insertions(+), 162 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5cdd3e12cee..97bad5c4528 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -358,7 +358,7 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -DerivedPath Installable::toDerivedPath() +DerivedPathWithInfo Installable::toDerivedPath() { auto buildables = toDerivedPaths(); if (buildables.size() != 1) @@ -422,21 +422,9 @@ struct InstallableStorePath : Installable return req.to_string(*store); } - DerivedPaths toDerivedPaths() override + DerivedPathsWithInfo toDerivedPaths() override { - return { req }; - } - - StorePathSet toDrvPaths(ref store) override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> StorePathSet { - return { bfd.drvPath }; - }, - [&](const DerivedPath::Opaque & bo) -> StorePathSet { - return { getDeriver(store, *this, bo.path) }; - }, - }, req.raw()); + return {{req}}; } std::optional getStorePath() override @@ -452,34 +440,6 @@ struct InstallableStorePath : Installable } }; -DerivedPaths InstallableValue::toDerivedPaths() -{ - DerivedPaths res; - - std::map> drvsToOutputs; - RealisedPath::Set drvsToCopy; - - // Group by derivation, helps with .all in particular - for (auto & drv : toDerivations()) { - for (auto & outputName : drv.outputsToInstall) - drvsToOutputs[drv.drvPath].insert(outputName); - drvsToCopy.insert(drv.drvPath); - } - - for (auto & i : drvsToOutputs) - res.push_back(DerivedPath::Built { i.first, i.second }); - - return res; -} - -StorePathSet InstallableValue::toDrvPaths(ref store) -{ - StorePathSet res; - for (auto & drv : toDerivations()) - res.insert(drv.drvPath); - return res; -} - struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; @@ -509,40 +469,52 @@ struct InstallableAttrPath : InstallableValue return {vRes, pos}; } - virtual std::vector toDerivations() override; -}; + DerivedPathsWithInfo toDerivedPaths() override + { + auto v = toValue(*state).first; -std::vector InstallableAttrPath::toDerivations() -{ - auto v = toValue(*state).first; + Bindings & autoArgs = *cmd.getAutoArgs(*state); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); + DerivedPathsWithInfo res; - std::vector res; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map byDrvPath; - std::set outputsToInstall; + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); - if (auto outputNames = std::get_if(&outputsSpec)) - outputsToInstall = *outputNames; - else - for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) - outputsToInstall.insert(output.first); + std::set outputsToInstall; - res.push_back(DerivationInfo { - .drvPath = *drvPath, - .outputsToInstall = std::move(outputsToInstall) - }); - } + if (auto outputNames = std::get_if(&outputsSpec)) + outputsToInstall = *outputNames; + else + for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) + outputsToInstall.insert(output.first); - return res; -} + auto i = byDrvPath.find(*drvPath); + if (i == byDrvPath.end()) { + byDrvPath[*drvPath] = res.size(); + res.push_back({ + .path = DerivedPath::Built { + .drvPath = std::move(*drvPath), + .outputs = std::move(outputsToInstall), + } + }); + } else { + for (auto & output : outputsToInstall) + std::get(res[i->second].path).outputs.insert(output); + } + } + + return res; + } +}; std::vector InstallableFlake::getActualAttrPaths() { @@ -630,7 +602,7 @@ InstallableFlake::InstallableFlake( throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); } -std::tuple InstallableFlake::toDerivation() +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() { Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); @@ -674,20 +646,19 @@ std::tuple InstallableF if (auto outputNames = std::get_if(&outputsSpec)) outputsToInstall = *outputNames; - auto drvInfo = DerivationInfo { - .drvPath = std::move(drvPath), - .outputsToInstall = std::move(outputsToInstall), - .priority = priority, - }; - - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; -} - -std::vector InstallableFlake::toDerivations() -{ - std::vector res; - res.push_back(std::get<2>(toDerivation())); - return res; + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::move(outputsToInstall), + }, + .info = { + .priority = priority, + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + .attrPath = attrPath, + .outputsSpec = outputsSpec, + } + }}; } std::pair InstallableFlake::toValue(EvalState & state) @@ -895,13 +866,19 @@ std::vector, BuiltPathWithResult>> Instal if (mode == Realise::Nothing) settings.readOnlyMode = true; + struct Aux + { + ExtraInfo info; + std::shared_ptr installable; + }; + std::vector pathsToBuild; - std::map>> backmap; + std::map> backmap; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b); - backmap[b].push_back(i); + pathsToBuild.push_back(b.path); + backmap[b.path].push_back({.info = b.info, .installable = i}); } } @@ -914,7 +891,7 @@ std::vector, BuiltPathWithResult>> Instal printMissing(store, pathsToBuild, lvlError); for (auto & path : pathsToBuild) { - for (auto & installable : backmap[path]) { + for (auto & aux : backmap[path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { OutputPathMap outputs; @@ -946,10 +923,14 @@ std::vector, BuiltPathWithResult>> Instal output, *drvOutput->second); } } - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info}}); }, }, path.raw()); } @@ -965,16 +946,22 @@ std::vector, BuiltPathWithResult>> Instal if (!buildResult.success()) buildResult.rethrow(); - for (auto & installable : backmap[buildResult.path]) { + for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; for (auto & path : buildResult.builtOutputs) outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info, + .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info, + .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -1059,7 +1046,7 @@ StorePathSet Installable::toDerivations( [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); return drvPaths; } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 02ea351d3f9..95ab4a40ea5 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -52,26 +52,42 @@ enum class OperateOn { Derivation }; +struct ExtraInfo +{ + std::optional priority; + std::optional originalRef; + std::optional resolvedRef; + std::optional attrPath; + // FIXME: merge with DerivedPath's 'outputs' field? + std::optional outputsSpec; +}; + +/* A derived path with any additional info that commands might + need from the derivation. */ +struct DerivedPathWithInfo +{ + DerivedPath path; + ExtraInfo info; +}; + struct BuiltPathWithResult { BuiltPath path; + ExtraInfo info; std::optional result; }; +typedef std::vector DerivedPathsWithInfo; + struct Installable { virtual ~Installable() { } virtual std::string what() const = 0; - virtual DerivedPaths toDerivedPaths() = 0; - - virtual StorePathSet toDrvPaths(ref store) - { - throw Error("'%s' cannot be converted to a derivation path", what()); - } + virtual DerivedPathsWithInfo toDerivedPaths() = 0; - DerivedPath toDerivedPath(); + DerivedPathWithInfo toDerivedPath(); UnresolvedApp toApp(EvalState & state); @@ -146,19 +162,6 @@ struct InstallableValue : Installable ref state; InstallableValue(ref state) : state(state) {} - - struct DerivationInfo - { - StorePath drvPath; - std::set outputsToInstall; - std::optional priority; - }; - - virtual std::vector toDerivations() = 0; - - DerivedPaths toDerivedPaths() override; - - StorePathSet toDrvPaths(ref store) override; }; struct InstallableFlake : InstallableValue @@ -186,9 +189,7 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - std::tuple toDerivation(); - - std::vector toDerivations() override; + DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; diff --git a/src/nix/app.cc b/src/nix/app.cc index 5658f2a52f8..a8d7e115bcc 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -19,12 +19,11 @@ struct InstallableDerivedPath : Installable { } - std::string what() const override { return derivedPath.to_string(*store); } - DerivedPaths toDerivedPaths() override + DerivedPathsWithInfo toDerivedPaths() override { - return {derivedPath}; + return {{derivedPath}}; } std::optional getStorePath() override diff --git a/src/nix/build.cc b/src/nix/build.cc index 94b169167e6..12b22d999a3 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (dryRun) { std::vector pathsToBuild; - for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); - } + for (auto & i : installables) + for (auto & b : i->toDerivedPaths()) + pathsToBuild.push_back(b.path); + printMissing(store, pathsToBuild, lvlError); + if (json) logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; } diff --git a/src/nix/log.cc b/src/nix/log.cc index 72d02ef11bb..a0598ca1333 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand [&](const DerivedPath::Built & bfd) { return logSub.getBuildLog(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 11910523d29..db702db1bdc 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -32,12 +32,14 @@ struct ProfileElementSource } }; +const int defaultPriority = 5; + struct ProfileElement { StorePathSet storePaths; std::optional source; bool active = true; - int priority = 5; + int priority = defaultPriority; std::string describe() const { @@ -251,13 +253,19 @@ struct ProfileManifest } }; -static std::map +static std::map> builtPathsPerInstallable( const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map res; - for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath.path); + std::map> res; + for (auto & [installable, builtPath] : builtPaths) { + auto & r = res[installable.get()]; + /* Note that there could be conflicting info + (e.g. meta.priority fields) if the installable returned + multiple derivations. So pick one arbitrarily. */ + r.first.push_back(builtPath.path); + r.second = builtPath.info; + } return res; } @@ -297,28 +305,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + auto & [res, info] = builtPaths[installable.get()]; - - if (auto installable2 = std::dynamic_pointer_cast(installable)) { - // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + if (info.originalRef && info.resolvedRef && info.attrPath && info.outputsSpec) { element.source = ProfileElementSource { - installable2->flakeRef, - resolvedRef, - attrPath, - installable2->outputsSpec + .originalRef = *info.originalRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = *info.outputsSpec, }; - - if(drv.priority) { - element.priority = *drv.priority; - } } - if(priority) { // if --priority was specified we want to override the priority of the installable - element.priority = *priority; - }; + // If --priority was specified we want to override the + // priority of the installable. + element.priority = + priority + ? *priority + : info.priority.value_or(defaultPriority); - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, res); manifest.elements.push_back(std::move(element)); } @@ -476,18 +481,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto & info = derivedPaths[0].info; + + assert(info.resolvedRef && info.attrPath); - if (element.source->resolvedRef == resolvedRef) continue; + if (element.source->resolvedRef == info.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, resolvedRef); + element.source->attrPath, element.source->resolvedRef, *info.resolvedRef); element.source = ProfileElementSource { - installable->flakeRef, - resolvedRef, - attrPath, - installable->outputsSpec + .originalRef = installable->flakeRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = installable->outputsSpec, }; installables.push_back(installable); @@ -515,7 +524,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first); } updateProfile(manifest.build(store)); diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 2e288f74391..d5fab5f2f38 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand auto dstStore = getDstStore(); auto & dstLogStore = require(*dstStore); - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & drvPath : i->toDrvPaths(getEvalStore())) - drvPaths.insert(drvPath); - - for (auto & drvPath : drvPaths) { + for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) { if (auto log = srcLogStore.getBuildLog(drvPath)) dstLogStore.addBuildLog(drvPath, *log); else diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 72301749728..661df965e98 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -111,7 +111,7 @@ struct CmdWhyDepends : SourceExprCommand } return maybePath->second; }, - }, derivedDependency.raw()); + }, derivedDependency.path.raw()); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); From bda879170fbf8ff8c5397948ccc0e1695d23871a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 20 Dec 2022 14:58:39 +0100 Subject: [PATCH 2/8] EvalState::copyPathToStore(): Return a StorePath --- src/libexpr/eval.cc | 28 ++++++++++++++-------------- src/libexpr/eval.hh | 2 +- src/libexpr/value-to-json.cc | 3 ++- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 084ccbee2f3..29d109ac430 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2250,7 +2250,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet if (canonicalizePath) path = canonPath(*path); if (copyToStore) - path = copyPathToStore(context, std::move(path).toOwned()); + path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned())); return path; } @@ -2293,26 +2293,26 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet } -std::string EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) throwEvalError("file names are not allowed to end in '%1%'", drvExtension); - Path dstPath; - auto i = srcToStore.find(path); - if (i != srcToStore.end()) - dstPath = store->printStorePath(i->second); - else { - auto p = settings.readOnlyMode + auto dstPath = [&]() -> StorePath + { + auto i = srcToStore.find(path); + if (i != srcToStore.end()) return i->second; + + auto dstPath = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - dstPath = store->printStorePath(p); - allowPath(p); - srcToStore.insert_or_assign(path, std::move(p)); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); - } + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); - context.insert(dstPath); + context.insert(store->printStorePath(dstPath)); return dstPath; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 21666339b7e..52b1736fe22 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -400,7 +400,7 @@ public: bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - std::string copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(PathSet & context, const Path & path); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5dc453b2e60..c35c876e383 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" #include "util.hh" +#include "store-api.hh" #include #include @@ -35,7 +36,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) - out = state.copyPathToStore(context, v.path); + out = state.store->printStorePath(state.copyPathToStore(context, v.path)); else out = v.path; break; From 5c97b5a3988c7dd28e617734c2eba669ee0c1288 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2022 23:01:15 +0100 Subject: [PATCH 3/8] InstallableFlake::toDerivedPaths(): Support paths and store paths This makes 'nix build' work on paths (which will be copied to the store) and store paths (returned as is). E.g. the following flake output attributes can be built using 'nix build .#foo': foo = ./src; foo = self.outPath; foo = builtins.fetchTarball { ... }; foo = (builtins.fetchTree { .. }).outPath; foo = builtins.fetchTree { .. } + "/README.md"; foo = builtins.storePath /nix/store/...; Note that this is potentially risky, e.g. foo = /.; will cause Nix to try to copy the entire file system to the store. What doesn't work yet: foo = self; foo = builtins.fetchTree { .. }; because we don't handle attrsets with an outPath attribute in it yet, and foo = builtins.storePath /nix/store/.../README.md; since result symlinks have to point to a store path currently (rather than a file inside a store path). Fixes #7417. --- src/libcmd/installables.cc | 34 +++++++++++++++++-- tests/flakes/build-paths.sh | 66 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 tests/flakes/build-paths.sh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 97bad5c4528..f4486bc2f03 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -610,8 +610,38 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto attrPath = attr->getAttrPathStr(); - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (v.type() == nPath) { + PathSet context; + auto storePath = state->copyPathToStore(context, Path(v.path)); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + } + }}; + } + + else if (v.type() == nString) { + PathSet context; + auto s = state->forceString(v, context); + auto storePath = state->store->maybeParseStorePath(s); + if (storePath && context.count(std::string(s))) { + return {{ + .path = DerivedPath::Opaque { + .path = std::move(*storePath), + } + }}; + } else + throw Error("flake output attribute '%s' evaluates to a string that does not denote a store path", attrPath); + } + + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); + } auto drvPath = attr->forceDerivation(); diff --git a/tests/flakes/build-paths.sh b/tests/flakes/build-paths.sh new file mode 100644 index 00000000000..08b4d176394 --- /dev/null +++ b/tests/flakes/build-paths.sh @@ -0,0 +1,66 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +mkdir -p $flake1Dir $flake2Dir + +writeSimpleFlake $flake2Dir +tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake2 +hash=$(nix hash path $flake2Dir) + +dep=$(nix store add-path ./common.sh) + +cat > $flake1Dir/flake.nix < $flake1Dir/foo + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 +[[ $(cat $TEST_ROOT/result) = bar ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 +diff common.sh $TEST_ROOT/result + +(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9) diff --git a/tests/local.mk b/tests/local.mk index 2f7f762617f..55913e9777e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -9,6 +9,7 @@ nix_tests = \ flakes/check.sh \ flakes/unlocked-override.sh \ flakes/absolute-paths.sh \ + flakes/build-paths.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ From b80e4b57dae78490ed644a2e2a840dae39d1da4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 14:52:49 +0100 Subject: [PATCH 4/8] ExtraInfo -> ExtraPathInfo --- src/libcmd/installables.cc | 2 +- src/libcmd/installables.hh | 6 +++--- src/nix/profile.cc | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f5a436fbec4..28021237974 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -898,7 +898,7 @@ std::vector, BuiltPathWithResult>> Instal struct Aux { - ExtraInfo info; + ExtraPathInfo info; std::shared_ptr installable; }; diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 95ab4a40ea5..250cf7e831a 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -52,7 +52,7 @@ enum class OperateOn { Derivation }; -struct ExtraInfo +struct ExtraPathInfo { std::optional priority; std::optional originalRef; @@ -67,13 +67,13 @@ struct ExtraInfo struct DerivedPathWithInfo { DerivedPath path; - ExtraInfo info; + ExtraPathInfo info; }; struct BuiltPathWithResult { BuiltPath path; - ExtraInfo info; + ExtraPathInfo info; std::optional result; }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index db702db1bdc..3da47568c00 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -253,11 +253,11 @@ struct ProfileManifest } }; -static std::map> +static std::map> builtPathsPerInstallable( const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map> res; + std::map> res; for (auto & [installable, builtPath] : builtPaths) { auto & r = res[installable.get()]; /* Note that there could be conflicting info From b4dc68f0be07edc3511a4ef276de1cb4e13a676d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 14:56:03 +0100 Subject: [PATCH 5/8] Show string in error message --- src/libcmd/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 28021237974..d7fdbb13dba 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -636,7 +636,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } }}; } else - throw Error("flake output attribute '%s' evaluates to a string that does not denote a store path", attrPath); + throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); } else From 1123c42f9016c5df7ade9d917d5e1900af6688e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 14:57:35 +0100 Subject: [PATCH 6/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libcmd/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d7fdbb13dba..409afd76287 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -424,7 +424,7 @@ struct InstallableStorePath : Installable DerivedPathsWithInfo toDerivedPaths() override { - return {{req}}; + return {{.path = req, .info = {} }}; } std::optional getStorePath() override From 7f1af270ddffa1cd6d18c167bcebe56b93bfd127 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 15:08:46 +0100 Subject: [PATCH 7/8] Clean up toDerivedPaths() logic --- src/libcmd/installables.cc | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 409afd76287..c0db2a715c5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -478,11 +478,9 @@ struct InstallableAttrPath : InstallableValue DrvInfos drvInfos; getDerivations(*state, *v, "", autoArgs, drvInfos, false); - DerivedPathsWithInfo res; - // Backward compatibility hack: group results by drvPath. This // helps keep .all output together. - std::map byDrvPath; + std::map byDrvPath; for (auto & drvInfo : drvInfos) { auto drvPath = drvInfo.queryDrvPath(); @@ -497,21 +495,16 @@ struct InstallableAttrPath : InstallableValue for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) outputsToInstall.insert(output.first); - auto i = byDrvPath.find(*drvPath); - if (i == byDrvPath.end()) { - byDrvPath[*drvPath] = res.size(); - res.push_back({ - .path = DerivedPath::Built { - .drvPath = std::move(*drvPath), - .outputs = std::move(outputsToInstall), - } - }); - } else { - for (auto & output : outputsToInstall) - std::get(res[i->second].path).outputs.insert(output); - } + auto derivedPath = byDrvPath.emplace(*drvPath, DerivedPath::Built { .drvPath = *drvPath }).first; + + for (auto & output : outputsToInstall) + derivedPath->second.outputs.insert(output); } + DerivedPathsWithInfo res; + for (auto & [_, info] : byDrvPath) + res.push_back({ .path = { info } }); + return res; } }; From 59cc920cc0751f93d4a3b717da72505a7bab2ee7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 15:20:30 +0100 Subject: [PATCH 8/8] Add a FIXME --- src/nix/profile.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 3da47568c00..22ee51ab936 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -262,7 +262,8 @@ builtPathsPerInstallable( auto & r = res[installable.get()]; /* Note that there could be conflicting info (e.g. meta.priority fields) if the installable returned - multiple derivations. So pick one arbitrarily. */ + multiple derivations. So pick one arbitrarily. FIXME: + print a warning? */ r.first.push_back(builtPath.path); r.second = builtPath.info; }