From 76f75e76915329c4f3502abcbea1df8e3ee658c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 15:28:49 +0200 Subject: [PATCH 01/26] nix copy: Add --profile flag This allows `nix copy` to atomically copy a store path and point a profile to it, without the risk that the store path might be GC'ed in between. This is useful for instance when deploying a new NixOS system profile from a remote store. --- src/libcmd/command.cc | 35 ++++++++++++++++++++++------------- src/libcmd/command.hh | 12 ++++++++---- src/nix/build.cc | 2 +- src/nix/copy.cc | 8 +++++--- src/nix/copy.md | 9 +++++++++ src/nix/develop.cc | 2 +- src/nix/profile.cc | 6 +++--- src/nix/realisation.cc | 2 +- tests/functional/zstd.sh | 4 +++- 9 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 6d8bfc19b1e..e38f982d896 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -179,30 +179,34 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive) void BuiltPathsCommand::run(ref store, Installables && installables) { - BuiltPaths paths; + BuiltPaths rootPaths, allPaths; + if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.emplace_back(BuiltPath::Opaque{p}); + rootPaths.emplace_back(BuiltPath::Opaque{p}); + allPaths = rootPaths; } else { - paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); + rootPaths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); + allPaths = rootPaths; + if (recursive) { // XXX: This only computes the store path closure, ignoring // intermediate realisations StorePathSet pathsRoots, pathsClosure; - for (auto & root : paths) { + for (auto & root : rootPaths) { auto rootFromThis = root.outPaths(); pathsRoots.insert(rootFromThis.begin(), rootFromThis.end()); } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.emplace_back(BuiltPath::Opaque{path}); + allPaths.emplace_back(BuiltPath::Opaque{path}); } } - run(store, std::move(paths)); + run(store, std::move(allPaths), std::move(rootPaths)); } StorePathsCommand::StorePathsCommand(bool recursive) @@ -210,10 +214,10 @@ StorePathsCommand::StorePathsCommand(bool recursive) { } -void StorePathsCommand::run(ref store, BuiltPaths && paths) +void StorePathsCommand::run(ref store, BuiltPaths && allPaths, BuiltPaths && rootPaths) { StorePathSet storePaths; - for (auto & builtPath : paths) + for (auto & builtPath : allPaths) for (auto & p : builtPath.outPaths()) storePaths.insert(p); @@ -242,17 +246,21 @@ MixProfile::MixProfile() }); } -void MixProfile::updateProfile(const StorePath & storePath) +void MixProfile::updateProfile( + const StorePath & storePath, + ref store_) { if (!profile) return; - auto store = getStore().dynamic_pointer_cast(); + auto store = store_.dynamic_pointer_cast(); if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, createGeneration(*store, profile2, storePath)); } -void MixProfile::updateProfile(const BuiltPaths & buildables) +void MixProfile::updateProfile( + const BuiltPaths & buildables, + ref store) { if (!profile) return; @@ -274,7 +282,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) if (result.size() != 1) throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(result[0]); + updateProfile(result[0], store); } MixDefaultProfile::MixDefaultProfile() @@ -308,7 +316,8 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) }); } -void MixEnvironment::setEnviron() { +void MixEnvironment::setEnviron() +{ if (ignoreEnvironment) { if (!unset.empty()) throw UsageError("--unset does not make sense with --ignore-environment"); diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 4a72627ed4d..8ada7878226 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -238,7 +238,7 @@ public: BuiltPathsCommand(bool recursive = false); - virtual void run(ref store, BuiltPaths && paths) = 0; + virtual void run(ref store, BuiltPaths && allPaths, BuiltPaths && rootPaths) = 0; void run(ref store, Installables && installables) override; @@ -251,7 +251,7 @@ struct StorePathsCommand : public BuiltPathsCommand virtual void run(ref store, StorePaths && storePaths) = 0; - void run(ref store, BuiltPaths && paths) override; + void run(ref store, BuiltPaths && allPaths, BuiltPaths && rootPaths) override; }; /** @@ -301,11 +301,15 @@ struct MixProfile : virtual StoreCommand MixProfile(); /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile(const StorePath & storePath); + void updateProfile( + const StorePath & storePath, + ref store); /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const BuiltPaths & buildables); + void updateProfile( + const BuiltPaths & buildables, + ref store); }; struct MixDefaultProfile : MixProfile diff --git a/src/nix/build.cc b/src/nix/build.cc index da9132d02f0..cffbeccb6bf 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -161,7 +161,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile BuiltPaths buildables2; for (auto & b : buildables) buildables2.push_back(b.path); - updateProfile(buildables2); + updateProfile(buildables2, store); } }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 151d282772f..c058a744678 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -4,7 +4,7 @@ using namespace nix; -struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand +struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile { CheckSigsFlag checkSigs = CheckSigs; @@ -43,19 +43,21 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand Category category() override { return catSecondary; } - void run(ref srcStore, BuiltPaths && paths) override + void run(ref srcStore, BuiltPaths && allPaths, BuiltPaths && rootPaths) override { auto dstStore = getDstStore(); RealisedPath::Set stuffToCopy; - for (auto & builtPath : paths) { + for (auto & builtPath : allPaths) { auto theseRealisations = builtPath.toRealisedPaths(*srcStore); stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end()); } copyPaths( *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); + + updateProfile(rootPaths, dstStore); } }; diff --git a/src/nix/copy.md b/src/nix/copy.md index 6ab7cdee3e1..813050fcbab 100644 --- a/src/nix/copy.md +++ b/src/nix/copy.md @@ -55,6 +55,15 @@ R""( # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs ``` +* Update the NixOS system profile to point to a closure copied from a + remote machine: + + ```console + # nix copy --from ssh://server \ + --profile /nix/var/nix/profiles/system \ + /nix/store/r14v3km89zm3prwsa521fab5kgzvfbw4-nixos-system-foobar-24.05.20240925.759537f + ``` + # Description `nix copy` copies store path closures between two Nix stores. The diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c7a7330255d..ca9f6a4af03 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -502,7 +502,7 @@ struct Common : InstallableCommand, MixProfile auto strPath = store->printStorePath(shellOutPath); - updateProfile(shellOutPath); + updateProfile(shellOutPath, store); debug("reading environment file '%s'", strPath); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 324fd633003..ae1f3e6e9f8 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -424,7 +424,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } try { - updateProfile(manifest.build(store)); + updateProfile(manifest.build(store), store); } catch (BuildEnvFileConflictError & conflictError) { // FIXME use C++20 std::ranges once macOS has it // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 @@ -669,7 +669,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem removedCount, newManifest.elements.size()); - updateProfile(newManifest.build(store)); + updateProfile(newManifest.build(store), store); } }; @@ -779,7 +779,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf builtPaths.find(&*installable)->second.first); } - updateProfile(manifest.build(store)); + updateProfile(manifest.build(store), store); } }; diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index e1f23122255..a386d98eac9 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -36,7 +36,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON Category category() override { return catSecondary; } - void run(ref store, BuiltPaths && paths) override + void run(ref store, BuiltPaths && paths, BuiltPaths && rootPaths) override { experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index 90fe58539b1..cdc30c66e7a 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -18,7 +18,9 @@ HASH=$(nix hash path $outPath) clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs +nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile + +[[ -e $TEST_ROOT/profile ]] if ls $cacheDir/nar/*.zst &> /dev/null; then echo "files do exist" From 43ad8c5eb2bdf4995467595f83efd6d9a747b440 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 15:36:21 +0200 Subject: [PATCH 02/26] Make getDstStore() a virtual method in StoreCommand --- src/libcmd/command.cc | 12 ++++-------- src/libcmd/command.hh | 23 ++++++++++++++++------- src/nix/build.cc | 2 +- src/nix/copy.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/profile.cc | 6 +++--- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index e38f982d896..0fb4f5680dc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -246,21 +246,17 @@ MixProfile::MixProfile() }); } -void MixProfile::updateProfile( - const StorePath & storePath, - ref store_) +void MixProfile::updateProfile(const StorePath & storePath) { if (!profile) return; - auto store = store_.dynamic_pointer_cast(); + auto store = getDstStore().dynamic_pointer_cast(); if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, createGeneration(*store, profile2, storePath)); } -void MixProfile::updateProfile( - const BuiltPaths & buildables, - ref store) +void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; @@ -282,7 +278,7 @@ void MixProfile::updateProfile( if (result.size() != 1) throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(result[0], store); + updateProfile(result[0]); } MixDefaultProfile::MixDefaultProfile() diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8ada7878226..73aaab741fe 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -45,7 +45,20 @@ struct StoreCommand : virtual Command { StoreCommand(); void run() override; + + /** + * Return the default Nix store. + */ ref getStore(); + + /** + * Return the destination Nix store. + */ + virtual ref getDstStore() + { + return getStore(); + } + virtual ref createStore(); /** * Main entry point, with a `Store` provided @@ -68,7 +81,7 @@ struct CopyCommand : virtual StoreCommand ref createStore() override; - ref getDstStore(); + ref getDstStore() override; }; /** @@ -301,15 +314,11 @@ struct MixProfile : virtual StoreCommand MixProfile(); /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile( - const StorePath & storePath, - ref store); + void updateProfile(const StorePath & storePath); /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile( - const BuiltPaths & buildables, - ref store); + void updateProfile(const BuiltPaths & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/nix/build.cc b/src/nix/build.cc index cffbeccb6bf..da9132d02f0 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -161,7 +161,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile BuiltPaths buildables2; for (auto & b : buildables) buildables2.push_back(b.path); - updateProfile(buildables2, store); + updateProfile(buildables2); } }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index c058a744678..60a64723cd2 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -57,7 +57,7 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile copyPaths( *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); - updateProfile(rootPaths, dstStore); + updateProfile(rootPaths); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ca9f6a4af03..c7a7330255d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -502,7 +502,7 @@ struct Common : InstallableCommand, MixProfile auto strPath = store->printStorePath(shellOutPath); - updateProfile(shellOutPath, store); + updateProfile(shellOutPath); debug("reading environment file '%s'", strPath); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ae1f3e6e9f8..324fd633003 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -424,7 +424,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } try { - updateProfile(manifest.build(store), store); + updateProfile(manifest.build(store)); } catch (BuildEnvFileConflictError & conflictError) { // FIXME use C++20 std::ranges once macOS has it // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 @@ -669,7 +669,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem removedCount, newManifest.elements.size()); - updateProfile(newManifest.build(store), store); + updateProfile(newManifest.build(store)); } }; @@ -779,7 +779,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf builtPaths.find(&*installable)->second.first); } - updateProfile(manifest.build(store), store); + updateProfile(manifest.build(store)); } }; From 7f6d006bebfb56b2967165c7f2e8c940abafac24 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 16:35:53 +0200 Subject: [PATCH 03/26] nix copy: Add --out-link --- src/libcmd/command.cc | 25 +++++++++++++++++++++++++ src/libcmd/command.hh | 10 ++++++++++ src/libcmd/installables.cc | 8 ++++++++ src/libcmd/installables.hh | 2 ++ src/nix/build.cc | 25 +------------------------ src/nix/copy.cc | 18 ++++++++++++++++++ tests/functional/zstd.sh | 3 ++- 7 files changed, 66 insertions(+), 25 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0fb4f5680dc..8053e1da674 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -334,4 +334,29 @@ void MixEnvironment::setEnviron() } } +void createOutLinks( + const std::filesystem::path & outLink, + const BuiltPaths & buildables, + LocalFSStore & store) +{ + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + auto symlink = outLink; + if (i) symlink += fmt("-%d", i); + store.addPermRoot(bo.path, absPath(symlink.string())); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + auto symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store.addPermRoot(output.second, absPath(symlink.string())); + } + }, + }, buildable.raw()); + } +} + } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 73aaab741fe..e9fcf3df999 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -18,6 +18,7 @@ extern char * * savedArgv; class EvalState; struct Pos; class Store; +class LocalFSStore; static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; @@ -367,4 +368,13 @@ void printClosureDiff( const StorePath & afterPath, std::string_view indent); +/** + * Create symlinks prefixed by `outLink` to the store paths in + * `buildables`. + */ +void createOutLinks( + const std::filesystem::path & outLink, + const BuiltPaths & buildables, + LocalFSStore & store); + } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f9d6d8ce8e1..86d26f6c4e7 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -917,4 +917,12 @@ void BuiltPathsCommand::applyDefaultInstallables(std::vector & rawI rawInstallables.push_back("."); } +BuiltPaths toBuiltPaths(const std::vector & builtPathsWithResult) +{ + BuiltPaths res; + for (auto & i : builtPathsWithResult) + res.push_back(i.path); + return res; +} + } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index bf5759230f3..c995c3019f4 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -86,6 +86,8 @@ struct BuiltPathWithResult std::optional result; }; +BuiltPaths toBuiltPaths(const std::vector & builtPathsWithResult); + /** * Shorthand, for less typing and helping us keep the choice of * collection in sync. diff --git a/src/nix/build.cc b/src/nix/build.cc index da9132d02f0..3569b0cde2a 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -42,29 +42,6 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vector& buildables, LocalFSStore& store2) -{ - for (const auto & [_i, buildable] : enumerate(buildables)) { - auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - store2.addPermRoot(bo.path, absPath(symlink.string())); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { - auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2.addPermRoot(output.second, absPath(symlink.string())); - } - }, - }, buildable.path.raw()); - } -} - struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; @@ -140,7 +117,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, buildables, *store2); + createOutLinks(outLink, toBuiltPaths(buildables), *store2); if (printOutputPaths) { stopProgressBar(); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 60a64723cd2..399a6c0fd34 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -1,11 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" using namespace nix; struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile { + std::optional outLink; CheckSigsFlag checkSigs = CheckSigs; SubstituteFlag substitute = NoSubstitute; @@ -13,6 +15,15 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile CmdCopy() : BuiltPathsCommand(true) { + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "Create symlinks prefixed with *path* to the top-level store paths fetched from the source store.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + addFlag({ .longName = "no-check-sigs", .description = "Do not require that paths are signed by trusted keys.", @@ -58,6 +69,13 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); updateProfile(rootPaths); + + if (outLink) { + if (auto store2 = dstStore.dynamic_pointer_cast()) + createOutLinks(*outLink, rootPaths, *store2); + else + throw Error("'--out-link' is not supported for this Nix store"); + } } }; diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index cdc30c66e7a..1eb1b534327 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -18,9 +18,10 @@ HASH=$(nix hash path $outPath) clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile +nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile --out-link $TEST_ROOT/result [[ -e $TEST_ROOT/profile ]] +[[ -e $TEST_ROOT/result ]] if ls $cacheDir/nar/*.zst &> /dev/null; then echo "files do exist" From e9b5704d1c3bdead2d9d259b3c443a391306b6d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 16:49:35 +0200 Subject: [PATCH 04/26] Add release note --- doc/manual/rl-next/nix-copy-flags.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/manual/rl-next/nix-copy-flags.md diff --git a/doc/manual/rl-next/nix-copy-flags.md b/doc/manual/rl-next/nix-copy-flags.md new file mode 100644 index 00000000000..f5b2b9716a4 --- /dev/null +++ b/doc/manual/rl-next/nix-copy-flags.md @@ -0,0 +1,18 @@ +--- +synopsis: "`nix copy` supports `--profile` and `--out-link`" +prs: [11657] +--- + +The `nix copy` command now has flags `--profile` and `--out-link`, similar to `nix build`. `--profile` makes a profile point to the +top-level store path, while `--out-link` create symlinks to the top-level store paths. + +For example, when updating the local NixOS system profile from a NixOS system closure on a remote machine, instead of +``` +# nix copy --from ssh://server $path +# nix build --profile /nix/var/nix/profiles/system $path +``` +you can now do +``` +# nix copy --from ssh://server --profile /nix/var/nix/profiles/system $path +``` +The advantage is that this avoids a time window where *path* is not a garbage collector root, and so could be deleted by a concurrent `nix store gc` process. From 27ea43781371cad717077ae723b11a79c0d0fc78 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2024 16:54:11 +0200 Subject: [PATCH 05/26] Support fine-grained database schema migrations Backward-compatible schema changes (e.g. those that add tables or nullable columns) now no longer need a change to the global schema file (/nix/var/nix/db/schema). Thus, old Nix versions can continue to access the database. This is especially useful for schema changes required by experimental features. In particular, it replaces the ad-hoc handling of the schema changes for CA derivations (i.e. the file /nix/var/nix/db/ca-schema). Schema versions 8 and 10 could have been handled by this mechanism in a backward-compatible way as well. --- src/libstore/local-store.cc | 105 ++++++++++++++++++------------------ src/libstore/local-store.hh | 2 + 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eafdac0cd33..f708bd1b008 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -95,51 +95,6 @@ struct LocalStore::State::Stmts { SQLiteStmt AddRealisationReference; }; -static int getSchema(Path schemaPath) -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - auto s = readFile(schemaPath); - auto n = string2Int(s); - if (!n) - throw Error("'%1%' is corrupt", schemaPath); - curSchema = *n; - } - return curSchema; -} - -void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) -{ - const int nixCASchemaVersion = 4; - int curCASchema = getSchema(schemaPath); - if (curCASchema != nixCASchemaVersion) { - if (curCASchema > nixCASchemaVersion) { - throw Error("current Nix store ca-schema is version %1%, but I only support %2%", - curCASchema, nixCASchemaVersion); - } - - if (!lockFile(lockFd.get(), ltWrite, false)) { - printInfo("waiting for exclusive access to the Nix store for ca drvs..."); - lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks - lockFile(lockFd.get(), ltWrite, true); - } - - if (curCASchema == 0) { - static const char schema[] = - #include "ca-specific-schema.sql.gen.hh" - ; - db.exec(schema); - curCASchema = nixCASchemaVersion; - } - - if (curCASchema < 4) - throw Error("experimental CA schema version %d is no longer supported", curCASchema); - - writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true); - lockFile(lockFd.get(), ltRead, true); - } -} - LocalStore::LocalStore( std::string_view scheme, PathView path, @@ -316,6 +271,10 @@ LocalStore::LocalStore( openDB(*state, false); + /* Legacy database schema migrations. Don't bump 'schema' for + new migrations; instead, add a migration to + upgradeDBSchema(). */ + if (curSchema < 8) { SQLiteTxn txn(state->db); state->db.exec("alter table ValidPaths add column ultimate integer"); @@ -342,13 +301,7 @@ LocalStore::LocalStore( else openDB(*state, false); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - if (!readOnly) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); - } else { - throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); - } - } + upgradeDBSchema(*state); /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, @@ -483,7 +436,17 @@ std::string LocalStore::getUri() int LocalStore::getSchema() -{ return nix::getSchema(schemaPath); } +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + auto s = readFile(schemaPath); + auto n = string2Int(s); + if (!n) + throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; + } + return curSchema; +} void LocalStore::openDB(State & state, bool create) { @@ -566,6 +529,42 @@ void LocalStore::openDB(State & state, bool create) } +void LocalStore::upgradeDBSchema(State & state) +{ + state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); + + std::set schemaMigrations; + + { + SQLiteStmt querySchemaMigrations; + querySchemaMigrations.create(state.db, "select migration from SchemaMigrations;"); + auto useQuerySchemaMigrations(querySchemaMigrations.use()); + while (useQuerySchemaMigrations.next()) + schemaMigrations.insert(useQuerySchemaMigrations.getStr(0)); + } + + auto doUpgrade = [&](const std::string & migrationName, const std::string & stmt) + { + if (schemaMigrations.contains(migrationName)) + return; + + debug("executing Nix database schema migration '%s'...", migrationName); + + SQLiteTxn txn(state.db); + state.db.exec(stmt + fmt(";\ninsert into SchemaMigrations values('%s')", migrationName)); + txn.commit(); + + schemaMigrations.insert(migrationName); + }; + + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + doUpgrade( + "20220326-ca-derivations", + #include "ca-specific-schema.sql.gen.hh" + ); +} + + /* To improve purity, users may want to make the Nix store a read-only bind mount. So make the Nix store writable for this process. */ void LocalStore::makeStoreWritable() diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 21848cc4d10..83154d65193 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -356,6 +356,8 @@ private: void openDB(State & state, bool create); + void upgradeDBSchema(State & state); + void makeStoreWritable(); uint64_t queryValidPathId(State & state, const StorePath & path); From 850281908cd65b7ccfdfe17b1e4a43f8ec59ef9a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Nov 2024 16:50:13 +0100 Subject: [PATCH 06/26] Clean up flakeref parsing This factors out some commonality in calling fromURL() and handling the "dir" parameter into a fromParsedURL() helper function. --- src/libflake/flake/flakeref.cc | 43 ++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 01fe747f9c0..ed47ad7375a 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -67,6 +67,20 @@ std::optional maybeParseFlakeRef( } } +static std::pair fromParsedURL( + const fetchers::Settings & fetchSettings, + ParsedURL && parsedURL, + bool isFlake) +{ + auto dir = getOr(parsedURL.query, "dir", ""); + parsedURL.query.erase("dir"); + + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment); +} + std::pair parsePathFlakeRefWithFragment( const fetchers::Settings & fetchSettings, const std::string & url, @@ -89,7 +103,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); } if (baseDir) { @@ -153,6 +167,7 @@ std::pair parsePathFlakeRefWithFragment( .authority = "", .path = flakeRoot, .query = query, + .fragment = fragment, }; if (subdir != "") { @@ -164,9 +179,7 @@ std::pair parsePathFlakeRefWithFragment( if (pathExists(flakeRoot + "/.git/shallow")) parsedURL.query.insert_or_assign("shallow", "1"); - return std::make_pair( - FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); + return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake); } subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); @@ -185,11 +198,12 @@ std::pair parsePathFlakeRefWithFragment( attrs.insert_or_assign("path", path); return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); -}; - +} -/* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ +/** + * Check if `url` is a flake ID. This is an abbreviated syntax for + * `flake:?ref=&rev=`. + */ static std::optional> parseFlakeIdRef( const fetchers::Settings & fetchSettings, const std::string & url, @@ -227,22 +241,11 @@ std::optional> parseURLFlakeRef( bool isFlake ) { - ParsedURL parsedURL; try { - parsedURL = parseURL(url); + return fromParsedURL(fetchSettings, parseURL(url), isFlake); } catch (BadURL &) { return std::nullopt; } - - std::string fragment; - std::swap(fragment, parsedURL.fragment); - - auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); - input.parent = baseDir; - - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); } std::pair parseFlakeRefWithFragment( From e948c8e03304a899893fffc1238d0d6025ed2564 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 19 Nov 2024 18:59:08 +0100 Subject: [PATCH 07/26] Bump fetcher cache version We're getting more reports in https://github.com/NixOS/nix/issues/10985 It appears that something hasn't gone right process-wise. I find this mistake not to be worth investigating, but rather something to pay attention to going forward. Let's nip this in the bud. Closes https://github.com/NixOS/nix/issues/10985 --- src/libfetchers/cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index b0b6cb8873d..6c2241f3af7 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -36,7 +36,7 @@ struct CacheImpl : Cache { auto state(_state.lock()); - auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite"; + auto dbPath = getCacheDir() + "/fetcher-cache-v3.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); From 4fca22b0dc6fe8054cb18e4e622b6dd2e3b8635c Mon Sep 17 00:00:00 2001 From: Gavin John Date: Tue, 19 Nov 2024 11:52:45 -0800 Subject: [PATCH 08/26] Update issue and pull request templates --- .github/ISSUE_TEMPLATE/bug_report.md | 39 +++++++++++++------ .github/ISSUE_TEMPLATE/feature_request.md | 35 ++++++++++++----- .github/ISSUE_TEMPLATE/installer.md | 17 ++++++-- .../ISSUE_TEMPLATE/missing_documentation.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 8 ++-- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 984f9a9ea2d..7ea82dcbcc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,36 +1,51 @@ --- name: Bug report -about: Create a report to help us improve +about: Report unexpected or incorrect behaviour title: '' labels: bug assignees: '' --- -**Describe the bug** +## Describe the bug -A clear and concise description of what the bug is. + -**Steps To Reproduce** +## Steps To Reproduce 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behavior** +## Expected behavior -A clear and concise description of what you expected to happen. + -**`nix-env --version` output** +## Metadata -**Additional context** + -Add any other context about the problem here. +## Additional context -**Priorities** + + +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source +[open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug + +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 42c658b522f..c75a4695170 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,24 +1,39 @@ --- name: Feature request -about: Suggest an idea for this project +about: Suggest a new feature title: '' labels: feature assignees: '' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +## Is your feature request related to a problem? -**Describe the solution you'd like** -A clear and concise description of what you want to happen. + -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +## Proposed solution -**Additional context** -Add any other context or screenshots about the feature request here. + -**Priorities** +## Alternative solutions + + + +## Additional context + + + +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open feature issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source +[open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature + +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index 3768a49c9a3..ed5e1ce87b9 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -23,14 +23,25 @@ assignees: ''
Output -```log + - +```log ```
-## Priorities +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open installer issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source +[open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer + +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index cf663e28d33..6c334b72206 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -26,6 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Priorities +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 69da87db727..c6843d86fa7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,10 +17,12 @@ so you understand the process and the expectations. --> -# Motivation +## Motivation + -# Context +## Context + @@ -29,7 +31,7 @@ so you understand the process and the expectations. -# Priorities and Process +--- Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). From ced8d311a593fcf9c3823e4e118474ac132d8e60 Mon Sep 17 00:00:00 2001 From: Picnoir Date: Wed, 20 Nov 2024 13:33:00 +0100 Subject: [PATCH 09/26] gc: resume GC after a pathinuse error First the motivation: I recently faced a bug that I assume is coming from the topoSortPaths function where the GC was trying to delete a path having some alive referrers. I resolved this by manually deleting the faulty path referrers using nix-store --query --referrers. I sadly did not manage to reproduce this bug. This bug alone is not a big deal. However, this bug is triggering a cascading failure: invalidatePathChecked is throwing a PathInUse exception. This exception is not catched and fails the whole GC run. From there, the machine (a builder machine) was unable to GC its Nix store, which led to an almost full disk with no way to automatically delete the dead Nix paths. Instead, I think we should log the error for the specific store path we're trying to delete, specifying we can't delete this path because it still has referrers. Once we're done with logging that, the GC run should continue to delete the dead store paths it can delete. --- src/libstore/gc.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 73195794a5c..45dfe4ad871 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -4,6 +4,7 @@ #include "finally.hh" #include "unix-domain-socket.hh" #include "signals.hh" +#include "posix-fs-canonicalise.hh" #if !defined(__linux__) // For shelling out to lsof @@ -763,13 +764,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } } } - for (auto & path : topoSortPaths(visited)) { if (!dead.insert(path).second) continue; if (shouldDelete) { - invalidatePathChecked(path); - deleteFromStore(path.to_string()); - referrersCache.erase(path); + try { + invalidatePathChecked(path); + deleteFromStore(path.to_string()); + referrersCache.erase(path); + } catch (PathInUse &e) { + // If we end up here, it's likely a new occurence + // of https://github.com/NixOS/nix/issues/11923 + printError("BUG: %s", e.what()); + } } } }; From 1800853b2a450b8f80514d2f4acb8ab394a22705 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:03:58 +0300 Subject: [PATCH 10/26] fix(libexpr/eval-inline): get rid of references to nullptr env When diagnosing infinite recursion references to nullptr `Env` can be formed. This happens only with `ExprBlackHole` is evaluated, which always leads to `InfiniteRecursionError`. UBSAN log for one such case: ``` ../src/libexpr/eval-inline.hh:94:31: runtime error: reference binding to null pointer of type 'Env' SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior ../src/libexpr/eval-inline.hh:94:31 in ``` --- src/libexpr/eval-inline.hh | 6 +++++- src/libexpr/eval.cc | 7 +++++-- src/libexpr/nixexpr.hh | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index d5ce238b2ed..631c0f39610 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -87,11 +87,15 @@ void EvalState::forceValue(Value & v, const PosIdx pos) { if (v.isThunk()) { Env * env = v.payload.thunk.env; + assert(env || v.isBlackhole()); Expr * expr = v.payload.thunk.expr; try { v.mkBlackhole(); //checkInterrupt(); - expr->eval(*this, *env, v); + if (env) [[likely]] + expr->eval(*this, *env, v); + else + ExprBlackHole::throwInfiniteRecursionError(*this, v); } catch (...) { v.mkThunk(env, expr); tryFixupBlackHolePos(v, pos); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2fe2d5249d1..05f58957ee3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2052,9 +2052,12 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) state.mkPos(v, pos); } - -void ExprBlackHole::eval(EvalState & state, Env & env, Value & v) +void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value & v) { + throwInfiniteRecursionError(state, v); +} + +[[gnu::noinline]] [[noreturn]] void ExprBlackHole::throwInfiniteRecursionError(EvalState & state, Value &v) { state.error("infinite recursion encountered") .atPos(v.determinePos(noPos)) .debugThrow(); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 948839bd9f8..2950ff1fd7c 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -468,6 +468,7 @@ struct ExprBlackHole : Expr void show(const SymbolTable & symbols, std::ostream & str) const override {} void eval(EvalState & state, Env & env, Value & v) override; void bindVars(EvalState & es, const std::shared_ptr & env) override {} + [[noreturn]] static void throwInfiniteRecursionError(EvalState & state, Value & v); }; extern ExprBlackHole eBlackHole; From ad7ad017ea5a7f7835148a03e81ef63a2689e8c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 16:35:47 +0100 Subject: [PATCH 11/26] EvalState::callPathFilter(): Remove unnecessary pathArg argument --- src/libexpr/eval.hh | 1 - src/libexpr/primops.cc | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 107334ffeb2..3ac3c8a8a6f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -819,7 +819,6 @@ public: bool callPathFilter( Value * filterFun, const SourcePath & path, - std::string_view pathArg, PosIdx pos); DocComment getDocCommentForPos(PosIdx pos); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 503632328de..53bfce6c5d0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2438,7 +2438,6 @@ static RegisterPrimOp primop_toFile({ bool EvalState::callPathFilter( Value * filterFun, const SourcePath & path, - std::string_view pathArg, PosIdx pos) { auto st = path.lstat(); @@ -2446,7 +2445,7 @@ bool EvalState::callPathFilter( /* Call the filter function. The first argument is the path, the second is a string indicating the type of the file. */ Value arg1; - arg1.mkString(pathArg); + arg1.mkString(path.path.abs()); // assert that type is not "unknown" Value * args []{&arg1, fileTypeToString(*this, st.type)}; @@ -2489,7 +2488,7 @@ static void addPath( if (filterFun) filter = std::make_unique([&](const Path & p) { auto p2 = CanonPath(p); - return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + return state.callPathFilter(filterFun, {path.accessor, p2}, pos); }); std::optional expectedStorePath; From 5533b0c735bea0e7277b75ba1024bfbc2521c635 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 18:08:31 +0100 Subject: [PATCH 12/26] Move shebang flake tests into a separate test --- tests/functional/flakes/flakes.sh | 102 +----------------------- tests/functional/flakes/meson.build | 1 + tests/functional/flakes/shebang.sh | 117 ++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 101 deletions(-) create mode 100644 tests/functional/flakes/shebang.sh diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 5e901c5d179..2b503df0223 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -74,100 +74,9 @@ cat > "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" -#! nix --offline shell -#! nix --impure --expr `` -#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; -#! nix fooScript = flake.${builtins.currentSystem}.fooScript; -#! nix /* just a comment !@#$%^&*()__+ # */ -#! nix in fooScript -#! nix `` -#! nix --no-write-lock-file --command bash -set -ex -foo -echo "$@" -EOF -chmod +x $nonFlakeDir/shebang-inline-expr.sh - -cat > $nonFlakeDir/fooScript.nix <<"EOF" -let flake = (builtins.getFlake (toString ../flake1)).packages; - fooScript = flake.${builtins.currentSystem}.fooScript; - in fooScript -EOF - -cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" -#! nix --offline shell -#! nix --impure --file ./fooScript.nix -#! nix --no-write-lock-file --command bash -set -ex -foo -echo "$@" -EOF -chmod +x $nonFlakeDir/shebang-file.sh - # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -644,15 +553,6 @@ nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridd # reference-lock-file can only be used if allow-dirty is set. expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock -# Test shebang -[[ $($nonFlakeDir/shebang.sh) = "foo" ]] -[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] -[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] -[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] -[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] -[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] -expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' - # Test that the --commit-lock-file-summary flag and its alias work cat > "$lockfileSummaryFlake/flake.nix" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + +[[ $($nonFlakeDir/shebang.sh) = "foo" ]] +[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From fd2df5f02f59f5f739c1e1931f8ff8c11a7720e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 18:23:20 +0100 Subject: [PATCH 13/26] Rename nonFlakeDir -> scriptDir --- tests/functional/flakes/shebang.sh | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh index ac00ca76457..23db19baaa5 100644 --- a/tests/functional/flakes/shebang.sh +++ b/tests/functional/flakes/shebang.sh @@ -7,15 +7,15 @@ TODO_NixOS requireGit flake1Dir=$TEST_ROOT/flake1 -nonFlakeDir=$TEST_ROOT/nonFlake +scriptDir=$TEST_ROOT/nonFlake createGitRepo "$flake1Dir" "" createSimpleGitFlake "$flake1Dir" nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" -mkdir -p "$nonFlakeDir" +mkdir -p "$scriptDir" -cat > "$nonFlakeDir/shebang.sh" < "$scriptDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $scriptDir/shebang-comments.sh < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $scriptDir/shebang-different-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $scriptDir/shebang-reject.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh < $scriptDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +cat >> $scriptDir/shebang-inline-expr.sh <<"EOF" #! nix --offline shell #! nix --impure --expr `` #! nix let flake = (builtins.getFlake (toString ../flake1)).packages; @@ -87,18 +87,18 @@ set -ex foo echo "$@" EOF -chmod +x $nonFlakeDir/shebang-inline-expr.sh +chmod +x $scriptDir/shebang-inline-expr.sh -cat > $nonFlakeDir/fooScript.nix <<"EOF" +cat > $scriptDir/fooScript.nix <<"EOF" let flake = (builtins.getFlake (toString ../flake1)).packages; fooScript = flake.${builtins.currentSystem}.fooScript; in fooScript EOF -cat > $nonFlakeDir/shebang-file.sh < $scriptDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +cat >> $scriptDir/shebang-file.sh <<"EOF" #! nix --offline shell #! nix --impure --file ./fooScript.nix #! nix --no-write-lock-file --command bash @@ -106,12 +106,12 @@ set -ex foo echo "$@" EOF -chmod +x $nonFlakeDir/shebang-file.sh +chmod +x $scriptDir/shebang-file.sh -[[ $($nonFlakeDir/shebang.sh) = "foo" ]] -[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] -[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] -[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] -[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] -[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] -expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' +[[ $($scriptDir/shebang.sh) = "foo" ]] +[[ $($scriptDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($scriptDir/shebang-comments.sh ) = "foo" ]] +[[ "$($scriptDir/shebang-different-comments.sh)" = "$(cat $scriptDir/shebang-different-comments.sh)" ]] +[[ $($scriptDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($scriptDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $scriptDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From e1cb905acae40b6bb55d30312ba9997c418eebf8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 18:42:33 +0100 Subject: [PATCH 14/26] Move --commit-lock-file-summary tests into a separate file --- .../flakes/commit-lock-file-summary.sh | 50 +++++++++++++++++++ tests/functional/flakes/flakes.sh | 40 +-------------- tests/functional/flakes/meson.build | 1 + 3 files changed, 52 insertions(+), 39 deletions(-) create mode 100644 tests/functional/flakes/commit-lock-file-summary.sh diff --git a/tests/functional/flakes/commit-lock-file-summary.sh b/tests/functional/flakes/commit-lock-file-summary.sh new file mode 100644 index 00000000000..21504eb2132 --- /dev/null +++ b/tests/functional/flakes/commit-lock-file-summary.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +source ./common.sh + +TODO_NixOS + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake + +createGitRepo "$flake1Dir" "" +createSimpleGitFlake "$flake1Dir" +nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" + +createGitRepo "$lockfileSummaryFlake" "--initial-branch=main" + +# Test that the --commit-lock-file-summary flag and its alias work +cat > "$lockfileSummaryFlake/flake.nix" < "$lockfileSummaryFlake/flake.nix" < Date: Wed, 20 Nov 2024 18:51:23 +0100 Subject: [PATCH 15/26] Add a utility function for creating/registering a simple flake --- tests/functional/flakes/commit-lock-file-summary.sh | 8 +------- tests/functional/flakes/common.sh | 9 +++++++++ tests/functional/flakes/dubious-query.sh | 7 ++----- tests/functional/flakes/edit.sh | 7 +------ tests/functional/flakes/shebang.sh | 8 +------- 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/tests/functional/flakes/commit-lock-file-summary.sh b/tests/functional/flakes/commit-lock-file-summary.sh index 21504eb2132..314d43ec3b9 100644 --- a/tests/functional/flakes/commit-lock-file-summary.sh +++ b/tests/functional/flakes/commit-lock-file-summary.sh @@ -4,15 +4,9 @@ source ./common.sh TODO_NixOS -requireGit +createFlake1 -flake1Dir=$TEST_ROOT/flake1 lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake - -createGitRepo "$flake1Dir" "" -createSimpleGitFlake "$flake1Dir" -nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" - createGitRepo "$lockfileSummaryFlake" "--initial-branch=main" # Test that the --commit-lock-file-summary flag and its alias work diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index cc9b2e466a3..0a020a1d733 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -38,12 +38,21 @@ EOF } createSimpleGitFlake() { + requireGit local flakeDir="$1" writeSimpleFlake "$flakeDir" git -C "$flakeDir" add flake.nix simple.nix shell.nix simple.builder.sh config.nix git -C "$flakeDir" commit -m 'Initial' } +# Create a simple Git flake and add it to the registry as "flake1". +createFlake1() { + flake1Dir="$TEST_ROOT/flake1" + createGitRepo "$flake1Dir" "" + createSimpleGitFlake "$flake1Dir" + nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" +} + writeDependentFlake() { local flakeDir="$1" cat > "$flakeDir/flake.nix" < "$scriptDir/shebang.sh" < Date: Wed, 20 Nov 2024 19:51:04 +0100 Subject: [PATCH 16/26] Move non-flake input tests into a separate file --- tests/functional/flakes/common.sh | 23 +++ tests/functional/flakes/flakes.sh | 162 ++------------------ tests/functional/flakes/meson.build | 1 + tests/functional/flakes/non-flake-inputs.sh | 126 +++++++++++++++ 4 files changed, 162 insertions(+), 150 deletions(-) create mode 100644 tests/functional/flakes/non-flake-inputs.sh diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 0a020a1d733..b1c3988e342 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -53,6 +53,29 @@ createFlake1() { nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" } +createFlake2() { + flake2Dir="$TEST_ROOT/flake 2" + percentEncodedFlake2Dir="$TEST_ROOT/flake%202" + + # Give one repo a non-main initial branch. + createGitRepo "$flake2Dir" "--initial-branch=main" + + cat > "$flake2Dir/flake.nix" < "$flakeDir/flake.nix" < "$flake2Dir/flake.nix" < "$flake3Dir/flake.nix" < "$nonFlakeDir/README.md" < "$flake3Dir/flake.nix" < \$out - [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] - [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] - ''; - }; - }; -} -EOF - -cp "${config_nix}" "$flake3Dir" - -git -C "$flake3Dir" add flake.nix config.nix -git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' - -# Check whether `nix build` works with a lockfile which is missing a -# nonFlakeInputs. -nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file - -nix build -o "$TEST_ROOT/result" flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] - -# Check whether flake input fetching is lazy: flake3#sth does not -# depend on flake2, so this shouldn't fail. -rm -rf "$TEST_HOME/.cache" -clearStore -mv "$flake2Dir" "$flake2Dir.tmp" -mv "$nonFlakeDir" "$nonFlakeDir.tmp" -nix build -o "$TEST_ROOT/result" flake3#sth -(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) -(! nix build -o "$TEST_ROOT/result" flake3#fnord) -mv "$flake2Dir.tmp" "$flake2Dir" -mv "$nonFlakeDir.tmp" "$nonFlakeDir" -nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord - # Test doing multiple `lookupFlake`s -nix build -o "$TEST_ROOT/result" flake4#xyzzy +nix build -o "$TEST_ROOT/result" flake3#xyzzy # Test 'nix flake update' and --override-flake. nix flake lock "$flake3Dir" @@ -330,53 +230,15 @@ nix flake lock "$flake3Dir" nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] -# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore -git -C "$flake3Dir" checkout -b removeXyzzy -rm "$flake3Dir/flake.nix" - -cat > "$flake3Dir/flake.nix" < \$out - ''; - }; - }; -} -EOF -nix flake lock "$flake3Dir" -git -C "$flake3Dir" add flake.nix flake.lock -git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' -git -C "$flake3Dir" checkout master - -# Test whether fuzzy-matching works for registry entries. -(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) -nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth - # Testing the nix CLI nix registry add flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] +[[ $(nix registry list | wc -l) == 5 ]] nix registry pin flake1 -[[ $(nix registry list | wc -l) == 6 ]] +[[ $(nix registry list | wc -l) == 5 ]] nix registry pin flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry remove flake1 [[ $(nix registry list | wc -l) == 5 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 4 ]] # Test 'nix registry list' with a disabled global registry. nix registry add user-flake1 git+file://$flake1Dir @@ -386,7 +248,7 @@ nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in nix --flake-registry "" registry list | grepQuiet '^user' nix registry remove user-flake1 nix registry remove user-flake2 -[[ $(nix registry list | wc -l) == 5 ]] +[[ $(nix registry list | wc -l) == 4 ]] # Test 'nix flake clone'. rm -rf $TEST_ROOT/flake1-v2 diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index a646abc0084..00060e3c927 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -26,6 +26,7 @@ suites += { 'dubious-query.sh', 'shebang.sh', 'commit-lock-file-summary.sh', + 'non-flake-inputs.sh', ], 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh new file mode 100644 index 00000000000..66dc74e0c60 --- /dev/null +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +source ./common.sh + +createFlake1 +createFlake2 + +nonFlakeDir=$TEST_ROOT/nonFlake +createGitRepo "$nonFlakeDir" "" + +cat > "$nonFlakeDir/README.md" < "$flake3Dir/flake.nix" < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] + ''; + }; + }; +} +EOF + +cp "${config_nix}" "$flake3Dir" + +git -C "$flake3Dir" add flake.nix config.nix +git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file + +nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" + +nix build -o "$TEST_ROOT/result" flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf "$TEST_HOME/.cache" +clearStore +mv "$flake2Dir" "$flake2Dir.tmp" +mv "$nonFlakeDir" "$nonFlakeDir.tmp" +nix build -o "$TEST_ROOT/result" flake3#sth +(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! nix build -o "$TEST_ROOT/result" flake3#fnord) +mv "$flake2Dir.tmp" "$flake2Dir" +mv "$nonFlakeDir.tmp" "$nonFlakeDir" +nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C "$flake3Dir" checkout -b removeXyzzy +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < \$out + ''; + }; + }; +} +EOF +nix flake lock "$flake3Dir" +git -C "$flake3Dir" add flake.nix flake.lock +git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' +git -C "$flake3Dir" checkout master + +# Test whether fuzzy-matching works for registry entries. +nix registry add --registry "$registry" flake4 flake3 +(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) +nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth From db0525692dcb94e4f274e033091ad1ce54a30cf3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 21:07:22 +0100 Subject: [PATCH 17/26] Formatting --- src/libcmd/command.cc | 36 +++++++++++++++++++----------------- src/libcmd/command.hh | 5 +---- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0a3a516c235..86d13fab796 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -369,28 +369,30 @@ void MixEnvironment::setEnviron() return; } -void createOutLinks( - const std::filesystem::path & outLink, - const BuiltPaths & buildables, - LocalFSStore & store) +void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store) { for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - store.addPermRoot(bo.path, absPath(symlink.string())); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store.addPermRoot(output.second, absPath(symlink.string())); - } + if (i) + symlink += fmt("-%d", i); + store.addPermRoot(bo.path, absPath(symlink.string())); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + auto symlink = outLink; + if (i) + symlink += fmt("-%d", i); + if (output.first != "out") + symlink += fmt("-%s", output.first); + store.addPermRoot(output.second, absPath(symlink.string())); + } + }, }, - }, buildable.raw()); + buildable.raw()); } } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 67c208d28bc..23529848f6b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -372,9 +372,6 @@ void printClosureDiff( * Create symlinks prefixed by `outLink` to the store paths in * `buildables`. */ -void createOutLinks( - const std::filesystem::path & outLink, - const BuiltPaths & buildables, - LocalFSStore & store); +void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store); } From 2f24030bffa8f8f6769ac8ce0002d6c201312933 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Wed, 20 Nov 2024 13:23:02 -0800 Subject: [PATCH 18/26] Move bug report list to comment and make it more nix-specific --- .github/ISSUE_TEMPLATE/bug_report.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7ea82dcbcc4..a5005f8a002 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -18,10 +18,13 @@ assignees: '' ## Steps To Reproduce -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error + ## Expected behavior From 671df02bf7d0cbf339d2626266260e0b7ccf5988 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 20:42:13 +0100 Subject: [PATCH 19/26] shellcheck --- tests/functional/flakes/non-flake-inputs.sh | 2 +- tests/functional/flakes/shebang.sh | 44 ++++++++++----------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index 66dc74e0c60..c9c9803d37a 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -71,7 +71,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" nix build -o "$TEST_ROOT/result" flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] +[[ $(cat "$TEST_ROOT/result") = FNORD ]] # Check whether flake input fetching is lazy: flake3#sth does not # depend on flake2, so this shouldn't fail. diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh index 645b1fa3895..576a62e2cb4 100644 --- a/tests/functional/flakes/shebang.sh +++ b/tests/functional/flakes/shebang.sh @@ -6,7 +6,7 @@ TODO_NixOS createFlake1 -scriptDir=$TEST_ROOT/nonFlake +scriptDir="$TEST_ROOT/nonFlake" mkdir -p "$scriptDir" cat > "$scriptDir/shebang.sh" < $scriptDir/shebang-comments.sh < "$scriptDir/shebang-comments.sh" < $scriptDir/shebang-comments.sh < $scriptDir/shebang-different-comments.sh < "$scriptDir/shebang-different-comments.sh" < $scriptDir/shebang-different-comments.sh < $scriptDir/shebang-reject.sh < "$scriptDir/shebang-reject.sh" < $scriptDir/shebang-reject.sh < $scriptDir/shebang-inline-expr.sh < "$scriptDir/shebang-inline-expr.sh" <> $scriptDir/shebang-inline-expr.sh <<"EOF" +cat >> "$scriptDir/shebang-inline-expr.sh" <<"EOF" #! nix --offline shell #! nix --impure --expr `` #! nix let flake = (builtins.getFlake (toString ../flake1)).packages; @@ -81,18 +79,18 @@ set -ex foo echo "$@" EOF -chmod +x $scriptDir/shebang-inline-expr.sh +chmod +x "$scriptDir/shebang-inline-expr.sh" -cat > $scriptDir/fooScript.nix <<"EOF" +cat > "$scriptDir/fooScript.nix" <<"EOF" let flake = (builtins.getFlake (toString ../flake1)).packages; fooScript = flake.${builtins.currentSystem}.fooScript; in fooScript EOF -cat > $scriptDir/shebang-file.sh < "$scriptDir/shebang-file.sh" <> $scriptDir/shebang-file.sh <<"EOF" +cat >> "$scriptDir/shebang-file.sh" <<"EOF" #! nix --offline shell #! nix --impure --file ./fooScript.nix #! nix --no-write-lock-file --command bash @@ -100,12 +98,12 @@ set -ex foo echo "$@" EOF -chmod +x $scriptDir/shebang-file.sh +chmod +x "$scriptDir/shebang-file.sh" -[[ $($scriptDir/shebang.sh) = "foo" ]] -[[ $($scriptDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] -[[ $($scriptDir/shebang-comments.sh ) = "foo" ]] -[[ "$($scriptDir/shebang-different-comments.sh)" = "$(cat $scriptDir/shebang-different-comments.sh)" ]] -[[ $($scriptDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] -[[ $($scriptDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] -expect 1 $scriptDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' +[[ $("$scriptDir/shebang.sh") = "foo" ]] +[[ $("$scriptDir/shebang.sh" "bar") = "foo"$'\n'"bar" ]] +[[ $("$scriptDir/shebang-comments.sh" ) = "foo" ]] +[[ "$("$scriptDir/shebang-different-comments.sh")" = "$(cat "$scriptDir/shebang-different-comments.sh")" ]] +[[ $("$scriptDir/shebang-inline-expr.sh" baz) = "foo"$'\n'"baz" ]] +[[ $("$scriptDir/shebang-file.sh" baz) = "foo"$'\n'"baz" ]] +expect 1 "$scriptDir/shebang-reject.sh" 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From e122acef97edd4a1190dba63e8f91b2b363a7df2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 20:55:45 +0100 Subject: [PATCH 20/26] Fix VM test --- tests/functional/flakes/non-flake-inputs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index c9c9803d37a..f5e12cd0141 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -2,6 +2,8 @@ source ./common.sh +TODO_NixOS + createFlake1 createFlake2 From 4a18c783853858b7bf897ac0a69baa3be7248237 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 21:26:20 +0100 Subject: [PATCH 21/26] flake_regressions: Pass -L to nix build --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb97fd211dd..9918875d9b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,4 +220,4 @@ jobs: path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From f4f4b698f6bbd19cec767686eb3f017567d9fef2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Nov 2024 16:53:34 +0100 Subject: [PATCH 22/26] fetchTree: Don't crash if narHash is missing Fixes nix: ../src/libexpr/primops/fetchTree.cc:37: void nix::emitTreeAttrs(EvalState&, const StorePath&, const fetchers::Input&, Value&, bool, bool): Assertion `narHash' failed. on a lock file with an input that doesn't have a narHash. This can happen when using a lock file created by the lazy-trees branch. Cherry-picked from lazy-trees. --- src/libexpr/primops/fetchTree.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c207da8ade8..47d1be1ccdd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -33,9 +33,8 @@ void emitTreeAttrs( // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); + if (auto narHash = input.getNarHash()) + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( From 965ca18db85638d6a04f4ecaff16b48fd661fe33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Nov 2024 18:44:57 +0100 Subject: [PATCH 23/26] Merge build-utils-meson/{diagnostics,threads} into build-utils-meson/common This reduces the amount of boilerplate. More importantly, it provides a place to add compiler flags (such as -O3) without having to add it to every subproject (and the risk of forgetting to include it). --- build-utils-meson/{diagnostics => common}/meson.build | 7 +++++++ build-utils-meson/threads/meson.build | 6 ------ src/libcmd/meson.build | 4 +--- src/libexpr-c/meson.build | 4 +--- src/libexpr-test-support/meson.build | 4 +--- src/libexpr-tests/meson.build | 4 +--- src/libexpr/meson.build | 4 +--- src/libfetchers-tests/meson.build | 4 +--- src/libfetchers/meson.build | 4 +--- src/libflake-tests/meson.build | 4 +--- src/libflake/meson.build | 4 +--- src/libmain-c/meson.build | 4 +--- src/libmain/meson.build | 4 +--- src/libstore-c/meson.build | 4 +--- src/libstore-test-support/meson.build | 4 +--- src/libstore-tests/meson.build | 4 +--- src/libstore/meson.build | 3 +-- src/libutil-c/meson.build | 4 +--- src/libutil-test-support/meson.build | 4 +--- src/libutil-tests/meson.build | 4 +--- src/libutil/meson.build | 3 +-- src/nix/meson.build | 4 +--- 22 files changed, 27 insertions(+), 64 deletions(-) rename build-utils-meson/{diagnostics => common}/meson.build (51%) delete mode 100644 build-utils-meson/threads/meson.build diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/common/meson.build similarity index 51% rename from build-utils-meson/diagnostics/meson.build rename to build-utils-meson/common/meson.build index 30eedfc13f8..67b6658f594 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/common/meson.build @@ -1,3 +1,10 @@ +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') + deps_private += dependency('threads') +endif + add_project_arguments( '-Wdeprecated-copy', '-Werror=suggest-override', diff --git a/build-utils-meson/threads/meson.build b/build-utils-meson/threads/meson.build deleted file mode 100644 index 294160de130..00000000000 --- a/build-utils-meson/threads/meson.build +++ /dev/null @@ -1,6 +0,0 @@ -# This is only conditional to work around -# https://github.com/mesonbuild/meson/issues/13293. It should be -# unconditional. -if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') - deps_private += dependency('threads') -endif diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index c484cf998aa..1f27c161402 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -30,8 +30,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -72,7 +70,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'built-path.cc', diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 4160f0d5a7b..5bcca29e0b8 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -29,8 +29,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -55,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_expr.cc', diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index b9e7f390d74..bdfd435a877 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -27,8 +27,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -41,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'tests/value/context.cc', diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 5a5c9f1d48a..b50c18c9c8f 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -51,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'derived-path.cc', diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 4d8a38b435c..28318579e92 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -27,8 +27,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - boost = dependency( 'boost', modules : ['container', 'context'], @@ -79,7 +77,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') parser_tab = custom_target( input : 'parser.y', diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index d948dbad69a..fdab6ba6c41 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -24,8 +24,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -44,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'public-key.cc', diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index ff638578d18..07a1178cced 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -26,8 +26,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -43,7 +41,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'attrs.cc', diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 592a7493b9e..c0a9b88478e 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -24,8 +24,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -44,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'flakeref.cc', diff --git a/src/libflake/meson.build b/src/libflake/meson.build index d2bb179dfd6..2c1a70a1840 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -26,8 +26,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -41,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'flake/config.cc', diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 0ec0e3f6db1..3cb1e4baae7 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -29,8 +29,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -55,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_main.cc', diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 7fcadf06ddc..6c6298e2be7 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -26,8 +26,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - pubsetbuf_test = ''' #include @@ -60,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'common-args.cc', diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index d4f86eeffdb..44b5fe11d81 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -27,8 +27,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -51,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_store.cc', diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 98ec9882edf..f8308e7bbf6 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -38,7 +36,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'tests/derived-path.cc', diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index f4f67d73a39..fc9152f2fb6 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -52,7 +50,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'common-protocol.cc', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 101879e9090..f836b8d4f62 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -82,7 +82,6 @@ if host_machine.system() == 'windows' endif subdir('build-utils-meson/libatomic') -subdir('build-utils-meson/threads') boost = dependency( 'boost', @@ -180,7 +179,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'binary-cache-store.cc', diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 3d5a0b9c23c..d44453676e9 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -47,7 +45,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_util.cc', diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index c5e1ba80b2b..fa1df732022 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -23,8 +23,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -35,7 +33,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'tests/hash.cc', diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 5c3b5e5a3aa..f593507742b 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -44,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'args.cc', diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a6dc8639478..11b4ea59243 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -54,7 +54,6 @@ endforeach configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) subdir('build-utils-meson/libatomic') -subdir('build-utils-meson/threads') if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') @@ -121,7 +120,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'archive.cc', diff --git a/src/nix/meson.build b/src/nix/meson.build index 60ee480358c..5c70c8216f7 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -32,8 +32,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -65,7 +63,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') subdir('build-utils-meson/generate-header') nix_sources = [config_h] + files( From ed120a61abc35b2a0a782dfce3935013cc108d50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Nov 2024 19:16:27 +0100 Subject: [PATCH 24/26] Use -O3 again This was lost in the switch to the new build system. -O3 provides around a 10% performance gain compared to -O2, see e.g. nix-env.qaAggressive.time in https://hydra.nixos.org/job/nix/master/metrics.nixpkgs#tabs-charts. --- build-utils-meson/common/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-utils-meson/common/meson.build b/build-utils-meson/common/meson.build index 67b6658f594..f0322183e8a 100644 --- a/build-utils-meson/common/meson.build +++ b/build-utils-meson/common/meson.build @@ -16,3 +16,7 @@ add_project_arguments( '-Wno-deprecated-declarations', language : 'cpp', ) + +if get_option('buildtype') not in ['debug'] + add_project_arguments('-O3', language : 'cpp') +endif From ba074465ba249da4df0752d0dc53529d0ab1a60d Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Thu, 21 Nov 2024 20:08:13 +0000 Subject: [PATCH 25/26] doc: Clarify that nix-shell still uses shell from host environment (#8809) * doc: Clarify that nix-shell still uses shell from host environment * doc: Fix NIX_BUILD_SHELL description * doc: Add anchor and link to NIX_BUILD_SHELL * doc: Add example of default shell trickiness Co-authored-by: Valentin Gagarin --- doc/manual/source/command-ref/nix-shell.md | 33 ++++++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md index 69a711bd5e6..e95db9bea5d 100644 --- a/doc/manual/source/command-ref/nix-shell.md +++ b/doc/manual/source/command-ref/nix-shell.md @@ -88,7 +88,9 @@ All options not listed here are passed to `nix-store cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + retained. Note that the shell used to run commands is obtained from + [`NIX_BUILD_SHELL`](#env-NIX_BUILD_SHELL) / `` from + `NIX_PATH`, and therefore not affected by `--pure`. - `--packages` / `-p` *packages*… @@ -112,11 +114,30 @@ All options not listed here are passed to `nix-store # Environment variables -- `NIX_BUILD_SHELL` - - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- [`NIX_BUILD_SHELL`](#env-NIX_BUILD_SHELL) + + Shell used to start the interactive environment. + Defaults to the `bash` from `bashInteractive` found in ``, falling back to the `bash` found in `PATH` if not found. + + > **Note** + > + > The shell obtained using this method may not necessarily be the same as any shells requested in *path*. + + + + > **Example + > + > Despite `--pure`, this invocation will not result in a fully reproducible shell environment: + > + > ```nix + > #!/usr/bin/env -S nix-shell --pure + > let + > pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/854fdc68881791812eddd33b2fed94b954979a8e.tar.gz") {}; + > in + > pkgs.mkShell { + > buildInputs = pkgs.bashInteractive; + > } + > ``` {{#include ./env-common.md}} From ebb19cc1cdc2f04c291e2e8e34f434e5defa5bbd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2024 09:14:01 +0100 Subject: [PATCH 26/26] Drop std::make_pair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- src/libflake/flake/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index ed47ad7375a..cdcdcf87f55 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -78,7 +78,7 @@ static std::pair fromParsedURL( std::string fragment; std::swap(fragment, parsedURL.fragment); - return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment); + return {FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment}; } std::pair parsePathFlakeRefWithFragment(