From dd4838974eb8afbde4cf3dc60ffd327084affb06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Nov 2024 13:24:11 +0100 Subject: [PATCH 01/93] document shallow clone options in git fetchers --- src/nix/flake.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index 8f0f9936c13..5412afcddb0 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -245,6 +245,9 @@ Currently the `type` attribute can be one of the following: * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` + * `git+https://example.org/my/repo?shallow=1` A shallow clone of the repository. + For large repositories, the shallow clone option can significantly speed up fresh clones compared + to non-shallow clones, while still providing faster updates than other fetch methods such as `tarball:` or `github:`. * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` * `git+file:///home/my-user/some-repo/some-repo` From 756758d968375c3f346412f8f01c82a9178689fb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:05:52 +0300 Subject: [PATCH 02/93] chore: get rid of dead code and unused variables where appropriate Looks like some cruft has been left over from previous refactorings. This removes dead variables, which should not have side effects in their constructors. In cases where the variable initialization has a purpose [[maybe_unused]] is inserted to silence compiler warnings. --- src/libexpr/attr-path.cc | 1 - src/libexpr/primops.cc | 2 -- src/libfetchers/git.cc | 2 -- src/libflake/flake/flakeref.cc | 2 -- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/export-import.cc | 5 ----- src/libstore/unix/build/local-derivation-goal.cc | 2 +- src/libutil-tests/nix_api_util.cc | 1 - src/libutil/serialise.cc | 3 +-- src/libutil/terminal.cc | 2 +- src/nix/search.cc | 1 - src/nix/sigs.cc | 1 - 12 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2f67260c532..822ec7620c2 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -129,7 +129,6 @@ std::pair findPackageFilename(EvalState & state, Value & v try { auto colon = fn.rfind(':'); if (colon == std::string::npos) fail(); - std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 53bfce6c5d0..5d2f75373a5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -66,14 +66,12 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - auto ctxS = store->printStorePath(o.path); ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, [&](const NixStringContextElem::DrvDeep & d) { /* Treat same as Opaque */ - auto ctxS = store->printStorePath(d.drvPath); ensureValid(d.drvPath); if (maybePathsOut) maybePathsOut->emplace(d.drvPath); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 99d91919eb4..a6883a2d355 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -514,8 +514,6 @@ struct GitInputScheme : InputScheme auto origRev = input.getRev(); - std::string name = input.getName(); - auto originalRef = input.getRef(); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); input.attrs.insert_or_assign("ref", ref); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index cdcdcf87f55..9616fe0eaff 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -257,8 +257,6 @@ std::pair parseFlakeRefWithFragment( { using namespace fetchers; - std::smatch match; - if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 794be15682a..bf1a25db1f0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1229,7 +1229,7 @@ HookReply DerivationGoal::tryBuildHook() hook->toHook.writeSide.close(); /* Create the log file and pipe. */ - Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = openLogFile(); std::set fds; fds.insert(hook->fromHook.readSide.get()); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb36c0c1b47..1c62cdfad64 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -13,14 +13,9 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) auto sorted = topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - std::string doneLabel("paths exported"); - //logger->incExpected(doneLabel, sorted.size()); - for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); - //logger->incProgress(doneLabel); } sink << 0; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index dcfaadeef4f..06a2f85be84 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -883,7 +883,7 @@ void LocalDerivationGoal::startBuilder() printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ - Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ builderOut = posix_openpt(O_RDWR | O_NOCTTY); diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index b36f71042de..7b77bd87fac 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -136,7 +136,6 @@ TEST_F(nix_api_util_context, nix_err_name) // no error EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); - std::string err_msg_ref; try { throw nix::Error("testing error"); } catch (...) { diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 168d2ed32fb..381e7ae3825 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -90,7 +90,6 @@ void Source::operator () (std::string_view data) void Source::drainInto(Sink & sink) { - std::string s; std::array buf; while (true) { size_t n; @@ -427,7 +426,7 @@ Error readError(Source & source) auto type = readString(source); assert(type == "Error"); auto level = (Verbosity) readInt(source); - auto name = readString(source); // removed + [[maybe_unused]] auto name = readString(source); // removed auto msg = readString(source); ErrorInfo info { .level = level, diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index db7a6fcd1ed..4c127ddb078 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -26,7 +26,7 @@ bool isTTY() std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) { - std::string t, e; + std::string t; size_t w = 0; auto i = s.begin(); diff --git a/src/nix/search.cc b/src/nix/search.cc index c8d0b9e9641..30b96c5008d 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -161,7 +161,6 @@ struct CmdSearch : InstallableValueCommand, MixJSON {"description", description}, }; } else { - auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 2afe4b2678d..134d4f34a17 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -41,7 +41,6 @@ struct CmdCopySigs : StorePathsCommand ThreadPool pool; - std::string doneLabel = "done"; std::atomic added{0}; //logger->setExpected(doneLabel, storePaths.size()); From 09ddc34b62bf762cbe7e0a9adce4bdea9ff7fc6a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sun, 24 Nov 2024 00:01:11 +0300 Subject: [PATCH 03/93] refactor(libfetchers/registry): use standard remove_if + erase Get rid of this fixme. This does not appear to be used anywhere in the nix codebase itself. Not sure why the comment mentioned C++20 erase member function with predicate, but iterator-based algorithms are also fine. --- src/libfetchers/registry.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 7f7a0905384..c761028ab55 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -94,12 +94,9 @@ void Registry::add( void Registry::remove(const Input & input) { - // FIXME: use C++20 std::erase. - for (auto i = entries.begin(); i != entries.end(); ) - if (i->from == input) - i = entries.erase(i); - else - ++i; + entries.erase( + std::remove_if(entries.begin(), entries.end(), [&](const Entry & entry) { return entry.from == input; }), + entries.end()); } static Path getSystemRegistryPath() From fbffd47fb715396f47be21ae0ed1a7f484852b4b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sun, 24 Nov 2024 00:35:47 +0300 Subject: [PATCH 04/93] build(lib{expr,store,util}-test-support): depend on -c libraries Since lib{expr,store,util}-test-support subprojects define nix_api_* helpers for testing nix c bindings, they need to publicly depend on -c counterparts. This makes their headers self-sufficient and does not rely on the -tests to add necessary dependencies. --- src/libexpr-test-support/meson.build | 1 + src/libexpr-test-support/package.nix | 2 ++ src/libstore-test-support/meson.build | 1 + src/libstore-test-support/package.nix | 2 ++ src/libutil-test-support/meson.build | 1 + src/libutil-test-support/package.nix | 2 ++ 6 files changed, 9 insertions(+) diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index bdfd435a877..33d9e17a6a8 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -24,6 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-store-test-support'), dependency('nix-expr'), + dependency('nix-expr-c'), ] subdir('build-utils-meson/subprojects') diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index bcf6118e043..7e92d145f78 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -4,6 +4,7 @@ , nix-store-test-support , nix-expr +, nix-expr-c , rapidcheck @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-store-test-support nix-expr + nix-expr-c rapidcheck ]; diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index f8308e7bbf6..1f230914f6c 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -22,6 +22,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-util-test-support'), dependency('nix-store'), + dependency('nix-store-c'), ] subdir('build-utils-meson/subprojects') diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 48f8b5e6bad..2543049fe73 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -4,6 +4,7 @@ , nix-util-test-support , nix-store +, nix-store-c , rapidcheck @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-util-test-support nix-store + nix-store-c rapidcheck ]; diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index fa1df732022..4afed01ca0d 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -20,6 +20,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ dependency('nix-util'), + dependency('nix-util-c'), ] subdir('build-utils-meson/subprojects') diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 2525e16028f..c403e762c1a 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -3,6 +3,7 @@ , mkMesonLibrary , nix-util +, nix-util-c , rapidcheck @@ -33,6 +34,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-util + nix-util-c rapidcheck ]; From 4145d18435f5e7073f0c99f813d45dd9058430dd Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sat, 23 Nov 2024 08:36:51 +0300 Subject: [PATCH 05/93] Rename install-nix-from-closure.sh into install-nix-from-tarball.sh Because it is only used as /install script from tarball. --- maintainers/flake-module.nix | 2 +- scripts/binary-tarball.nix | 2 +- ...{install-nix-from-closure.sh => install-nix-from-tarball.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename scripts/{install-nix-from-closure.sh => install-nix-from-tarball.sh} (100%) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index fdb031302eb..ba6cd2816ca 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -496,7 +496,7 @@ ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' - ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-nix-from-tarball\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 104189b0c81..671c8e96e38 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -23,7 +23,7 @@ in runCommand "nix-binary-tarball-${version}" env '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ + substitute ${./install-nix-from-tarball.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-tarball.sh similarity index 100% rename from scripts/install-nix-from-closure.sh rename to scripts/install-nix-from-tarball.sh From 82a23d9b6b96bf08e7c28008fcc346c0bdb671be Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 16:24:20 +0100 Subject: [PATCH 06/93] libexpr-c: Add nix_eval_state_builder --- src/libexpr-c/nix_api_expr.cc | 79 ++++++++++++++++++++++++--- src/libexpr-c/nix_api_expr.h | 65 +++++++++++++++++++++- src/libexpr-c/nix_api_expr_internal.h | 11 ++++ src/libexpr-tests/nix_api_expr.cc | 37 +++++++++++++ 4 files changed, 183 insertions(+), 9 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 6144a7986c4..27a70afb6d6 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -6,6 +6,7 @@ #include "eval-gc.hh" #include "globals.hh" #include "eval-settings.hh" +#include "ref.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -93,7 +94,46 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val NIXC_CATCH_ERRS } -EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // Allocate ahead of time, because .settings needs self-reference + void * p = ::operator new( + sizeof(nix_eval_state_builder), + static_cast(alignof(nix_eval_state_builder))); + auto * p2 = static_cast(p); + new (p) nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ p2->readOnlyMode}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = true, + }; + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_eval_state_builder_free(nix_eval_state_builder * builder) +{ + delete builder; +} + +nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // TODO: load in one go? + builder->settings.readOnlyMode = nix::settings.readOnlyMode; + loadConfFile(builder->settings); + loadConfFile(builder->fetchSettings); + } + NIXC_CATCH_ERRS +} + +nix_err nix_eval_state_builder_set_lookup_path(nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath_c) { if (context) context->last_err_code = NIX_OK; @@ -102,28 +142,51 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c if (lookupPath_c != nullptr) for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); + builder->lookupPath = nix::LookupPath::parse(lookupPath); + } + NIXC_CATCH_ERRS +} +EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // Allocate ahead of time, because .state init needs self-reference void * p = ::operator new( sizeof(EvalState), static_cast(alignof(EvalState))); auto * p2 = static_cast(p); new (p) EvalState { - .fetchSettings = nix::fetchers::Settings{}, - .settings = nix::EvalSettings{ - nix::settings.readOnlyMode, - }, + .fetchSettings = std::move(builder->fetchSettings), + .settings = std::move(builder->settings), .state = nix::EvalState( - nix::LookupPath::parse(lookupPath), - store->ptr, + builder->lookupPath, + builder->store, p2->fetchSettings, p2->settings), }; - loadConfFile(p2->settings); return p2; } NIXC_CATCH_ERRS_NULL } +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + auto builder = nix_eval_state_builder_new(context, store); + if (builder == nullptr) + return nullptr; + + if (nix_eval_state_builder_load(context, builder) != NIX_OK) + return nullptr; + + if (nix_eval_state_builder_set_lookup_path(context, builder, lookupPath_c) + != NIX_OK) + return nullptr; + + return nix_eval_state_build(context, builder); +} + void nix_state_free(EvalState * state) { delete state; diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index e680f5ff190..f8d1814521b 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -30,6 +30,11 @@ extern "C" { // cffi start // Type definitions +/** + * @brief Builder for EvalState + */ +typedef struct nix_eval_state_builder nix_eval_state_builder; + /** * @brief Represents a state of the Nix language evaluator. * @@ -174,12 +179,70 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); /** - * @brief Create a new Nix language evaluator state. + * @brief Create a new nix_eval_state_builder + * + * The settings are initialized to their default value. + * Values can be sourced elsewhere with nix_eval_state_builder_load. + * + * @param[out] context Optional, stores error information + * @param[in] store The Nix store to use. + * @return A new nix_eval_state_builder or NULL on failure. + */ +nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store); + +/** + * @brief Read settings from the ambient environment + * + * Settings are sourced from environment variables and configuration files, + * as documented in the Nix manual. + * + * @param[out] context Optional, stores error information + * @param[out] builder The builder to modify. + * @return NIX_OK if successful, an error code otherwise. + */ +nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder); + +/** + * @brief Set the lookup path for `<...>` expressions + * + * @param[in] context Optional, stores error information + * @param[in] builder The builder to modify. + * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. + */ +nix_err nix_eval_state_builder_set_lookup_path( + nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath); + +/** + * @brief Create a new Nix language evaluator state + * + * Remember to nix_eval_state_builder_free after building the state. + * + * @param[out] context Optional, stores error information + * @param[in] builder The builder to use and free + * @return A new Nix state or NULL on failure. + * @see nix_eval_state_builder_new, nix_eval_state_builder_free + */ +EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder); + +/** + * @brief Free a nix_eval_state_builder + * + * Does not fail. + * + * @param[in] builder The builder to free. + */ +void nix_eval_state_builder_free(nix_eval_state_builder * builder); + +/** + * @brief Create a new Nix language evaluator state + * + * For more control, use nix_eval_state_builder * * @param[out] context Optional, stores error information * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. * @param[in] store The Nix store to use. * @return A new Nix state or NULL on failure. + * @see nix_state_builder_new */ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 12f24b6eb07..f596640115f 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -6,6 +6,17 @@ #include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" +#include "search-path.hh" + +struct nix_eval_state_builder +{ + nix::ref store; + nix::EvalSettings settings; + nix::fetchers::Settings fetchSettings; + nix::LookupPath lookupPath; + // TODO: make an EvalSettings setting own this instead? + bool readOnlyMode; +}; struct EvalState { diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index b37ac44b317..5ed78d2fcd9 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -7,12 +7,49 @@ #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" +#include "file-system.hh" #include #include namespace nixC { +TEST_F(nix_api_store_test, nix_eval_state_lookup_path) +{ + auto tmpDir = nix::createTempDir(); + auto delTmpDir = std::make_unique(tmpDir, true); + auto nixpkgs = tmpDir + "/pkgs"; + auto nixos = tmpDir + "/cfg"; + std::filesystem::create_directories(nixpkgs); + std::filesystem::create_directories(nixos); + + std::string nixpkgsEntry = "nixpkgs=" + nixpkgs; + std::string nixosEntry = "nixos-config=" + nixos; + const char * lookupPath[] = {nixpkgsEntry.c_str(), nixosEntry.c_str(), nullptr}; + + auto builder = nix_eval_state_builder_new(ctx, store); + assert_ctx_ok(); + + ASSERT_EQ(NIX_OK, nix_eval_state_builder_set_lookup_path(ctx, builder, lookupPath)); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + + nix_eval_state_builder_free(builder); + + Value * value = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "builtins.seq ", ".", value); + assert_ctx_ok(); + + ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH); + assert_ctx_ok(); + + auto pathStr = nix_get_path_string(ctx, value); + assert_ctx_ok(); + ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str())); +} + TEST_F(nix_api_expr_test, nix_expr_eval_from_string) { nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value); From 1bd75178017102e098e7b9a6aa2ddf858b486b53 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 16:27:17 +0100 Subject: [PATCH 07/93] Doc nix_get_path_string --- src/libexpr-c/nix_api_value.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 8a0813ebe1f..711b0adbc82 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -213,7 +213,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_ /** @brief Get path as string * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return string + * @return string, if the type is NIX_TYPE_PATH * @return NULL in case of error. */ const char * nix_get_path_string(nix_c_context * context, const nix_value * value); From f06f611ff3f1e7beee51171b3ab13d4883e187f3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 17:26:26 +0100 Subject: [PATCH 08/93] refactor: Extract unsafe_new_with_self --- src/libexpr-c/nix_api_expr.cc | 69 +++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 27a70afb6d6..a024248cdd0 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,6 +19,29 @@ # include #endif +/** + * @brief Allocate and initialize using self-reference + * + * This allows a brace initializer to reference the object being constructed. + * + * @warning Use with care, as the pointer points to an object that is not fully constructed yet. + * + * @tparam T Type to allocate + * @tparam F A function type for `init`, taking a T* and returning the initializer for T + * @param init Function that takes a T* and returns the initializer for T + * @return Pointer to allocated and initialized object + */ +template +static T * unsafe_new_with_self(F && init) +{ + // Allocate + void * p = ::operator new( + sizeof(T), + static_cast(alignof(T))); + // Initialize with placement new + return new (p) T(init(static_cast(p))); +} + nix_err nix_libexpr_init(nix_c_context * context) { if (context) @@ -99,18 +122,14 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - // Allocate ahead of time, because .settings needs self-reference - void * p = ::operator new( - sizeof(nix_eval_state_builder), - static_cast(alignof(nix_eval_state_builder))); - auto * p2 = static_cast(p); - new (p) nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ p2->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - return p2; + return unsafe_new_with_self([&](auto * self) { + return nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = true, + }; + }); } NIXC_CATCH_ERRS_NULL } @@ -152,21 +171,17 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder if (context) context->last_err_code = NIX_OK; try { - // Allocate ahead of time, because .state init needs self-reference - void * p = ::operator new( - sizeof(EvalState), - static_cast(alignof(EvalState))); - auto * p2 = static_cast(p); - new (p) EvalState { - .fetchSettings = std::move(builder->fetchSettings), - .settings = std::move(builder->settings), - .state = nix::EvalState( - builder->lookupPath, - builder->store, - p2->fetchSettings, - p2->settings), - }; - return p2; + return unsafe_new_with_self([&](auto * self) { + return EvalState{ + .fetchSettings = std::move(builder->fetchSettings), + .settings = std::move(builder->settings), + .state = nix::EvalState( + builder->lookupPath, + builder->store, + self->fetchSettings, + self->settings), + }; + }); } NIXC_CATCH_ERRS_NULL } From 4eecf3c20ab454b4427363a276d757298f9220dc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 16:18:55 +0100 Subject: [PATCH 09/93] Add nix-flake-c, nix_flake_init_global, nix_flake_settings_new --- meson.build | 1 + packaging/components.nix | 1 + packaging/everything.nix | 4 + src/external-api-docs/doxygen.cfg.in | 1 + src/external-api-docs/package.nix | 1 + src/libflake-c/.version | 1 + src/libflake-c/build-utils-meson | 1 + src/libflake-c/meson.build | 93 ++++++++++++++++++++++++ src/libflake-c/nix_api_flake.cc | 32 ++++++++ src/libflake-c/nix_api_flake.h | 46 ++++++++++++ src/libflake-c/nix_api_flake_internal.hh | 9 +++ src/libflake-c/package.nix | 60 +++++++++++++++ src/libflake-tests/meson.build | 3 + src/libflake-tests/nix_api_flake.cc | 51 +++++++++++++ src/libflake-tests/package.nix | 3 + 15 files changed, 307 insertions(+) create mode 120000 src/libflake-c/.version create mode 120000 src/libflake-c/build-utils-meson create mode 100644 src/libflake-c/meson.build create mode 100644 src/libflake-c/nix_api_flake.cc create mode 100644 src/libflake-c/nix_api_flake.h create mode 100644 src/libflake-c/nix_api_flake_internal.hh create mode 100644 src/libflake-c/package.nix create mode 100644 src/libflake-tests/nix_api_flake.cc diff --git a/meson.build b/meson.build index 8985b631ead..49adf9832ed 100644 --- a/meson.build +++ b/meson.build @@ -34,6 +34,7 @@ endif subproject('libutil-c') subproject('libstore-c') subproject('libexpr-c') +subproject('libflake-c') subproject('libmain-c') # Language Bindings diff --git a/packaging/components.nix b/packaging/components.nix index c29e04ae9fe..e1f661be8fb 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -44,6 +44,7 @@ in nix-expr-tests = callPackage ../src/libexpr-tests/package.nix { }; nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-c = callPackage ../src/libflake-c/package.nix { }; nix-flake-tests = callPackage ../src/libflake-tests/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; diff --git a/packaging/everything.nix b/packaging/everything.nix index b09b9d2a91c..0b04d2c6d93 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -19,6 +19,7 @@ nix-expr-tests, nix-flake, + nix-flake-c, nix-flake-tests, nix-main, @@ -53,6 +54,7 @@ let nix-expr-c nix-fetchers nix-flake + nix-flake-c nix-main nix-main-c nix-store @@ -86,6 +88,7 @@ let "nix-expr-c" "nix-fetchers" "nix-flake" + "nix-flake-c" "nix-main" "nix-main-c" "nix-store" @@ -169,6 +172,7 @@ in nix-expr nix-expr-c nix-flake + nix-flake-c nix-main nix-main-c ; diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 8e235dae5ce..3af2f5b813f 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -40,6 +40,7 @@ GENERATE_LATEX = NO INPUT = \ @src@/src/libutil-c \ @src@/src/libexpr-c \ + @src@/src/libflake-c \ @src@/src/libstore-c \ @src@/src/external-api-docs/README.md diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index 0c592955a7b..57c5138cfdb 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libflake-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; diff --git a/src/libflake-c/.version b/src/libflake-c/.version new file mode 120000 index 00000000000..b7badcd0cc8 --- /dev/null +++ b/src/libflake-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake-c/build-utils-meson b/src/libflake-c/build-utils-meson new file mode 120000 index 00000000000..91937f18372 --- /dev/null +++ b/src/libflake-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson/ \ No newline at end of file diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build new file mode 100644 index 00000000000..00d9650e770 --- /dev/null +++ b/src/libflake-c/meson.build @@ -0,0 +1,93 @@ +project('nix-flake-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), + dependency('nix-flake'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), + dependency('nix-expr-c'), +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-flake.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + # not generated (yet?) + # '-include', 'config-flake.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + '-include', 'config-flake.h', + language : 'cpp', +) + +subdir('build-utils-meson/common') + +sources = files( + 'nix_api_flake.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_flake.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_flake.h') + +subdir('build-utils-meson/export-all-symbols') +subdir('build-utils-meson/windows-version') + +this_library = library( + 'nixflakec', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc new file mode 100644 index 00000000000..17cf6572da2 --- /dev/null +++ b/src/libflake-c/nix_api_flake.cc @@ -0,0 +1,32 @@ +#include "nix_api_flake.h" +#include "nix_api_flake_internal.hh" +#include "nix_api_util_internal.h" + +#include "flake/flake.hh" + +nix_flake_settings * nix_flake_settings_new(nix_c_context * context) +{ + try { + auto settings = nix::make_ref(); + return new nix_flake_settings{settings}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_flake_settings_free(nix_flake_settings * settings) +{ + delete settings; +} + +nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings) +{ + static std::shared_ptr registeredSettings; + try { + if (registeredSettings) + throw nix::Error("nix_flake_init_global already initialized"); + + registeredSettings = settings->settings; + nix::flake::initLib(*registeredSettings); + } + NIXC_CATCH_ERRS +} diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h new file mode 100644 index 00000000000..80051298d28 --- /dev/null +++ b/src/libflake-c/nix_api_flake.h @@ -0,0 +1,46 @@ +#ifndef NIX_API_FLAKE_H +#define NIX_API_FLAKE_H +/** @defgroup libflake libflake + * @brief Bindings to the Nix Flakes library + * + * @{ + */ +/** @file + * @brief Main entry for the libflake C bindings + */ + +#include "nix_api_store.h" +#include "nix_api_util.h" +#include "nix_api_expr.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +typedef struct nix_flake_settings nix_flake_settings; + +// Function prototypes +/** + * Create a nix_flake_settings initialized with default values. + * @param[out] context Optional, stores error information + * @return A new nix_flake_settings or NULL on failure. + * @see nix_flake_settings_free + */ +nix_flake_settings * nix_flake_settings_new(nix_c_context * context); + +/** + * @brief Release the resources associated with a nix_flake_settings. + */ +void nix_flake_settings_free(nix_flake_settings * settings); + +/** + * @brief Register Flakes support process-wide. + */ +nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/libflake-c/nix_api_flake_internal.hh b/src/libflake-c/nix_api_flake_internal.hh new file mode 100644 index 00000000000..4c154a34229 --- /dev/null +++ b/src/libflake-c/nix_api_flake_internal.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "ref.hh" +#include "flake/settings.hh" + +struct nix_flake_settings +{ + nix::ref settings; +}; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix new file mode 100644 index 00000000000..a70cbf94ef5 --- /dev/null +++ b/src/libflake-c/package.nix @@ -0,0 +1,60 @@ +{ lib +, stdenv +, mkMesonLibrary + +, nix-store-c +, nix-expr-c +, nix-flake + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonLibrary (finalAttrs: { + pname = "nix-flake-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + propagatedBuildInputs = [ + nix-expr-c + nix-store-c + nix-flake + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index c0a9b88478e..c494c414e99 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -19,6 +19,7 @@ subdir('build-utils-meson/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr-test-support'), dependency('nix-flake'), + dependency('nix-flake-c'), ] deps_public_maybe_subproject = [ ] @@ -46,6 +47,7 @@ subdir('build-utils-meson/common') sources = files( 'flakeref.cc', + 'nix_api_flake.cc', 'url-name.cc', ) @@ -68,6 +70,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'NIX_CONFIG': 'extra-experimental-features = flakes', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/nix_api_flake.cc b/src/libflake-tests/nix_api_flake.cc new file mode 100644 index 00000000000..21109d181a4 --- /dev/null +++ b/src/libflake-tests/nix_api_flake.cc @@ -0,0 +1,51 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "nix_api_flake.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include +#include + +namespace nixC { + +TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists) +{ + nix_libstore_init(ctx); + assert_ctx_ok(); + nix_libexpr_init(ctx); + assert_ctx_ok(); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + nix_flake_init_global(ctx, settings); + assert_ctx_ok(); + + nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store); + ASSERT_NE(nullptr, builder); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + ASSERT_NE(nullptr, state); + + nix_eval_state_builder_free(builder); + + auto value = nix_alloc_value(ctx, state); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, err); + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value)); +} + +} // namespace nixC diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 67e7169795b..b3a8ac46644 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -4,6 +4,7 @@ , mkMesonExecutable , nix-flake +, nix-flake-c , nix-expr-test-support , rapidcheck @@ -38,6 +39,7 @@ mkMesonExecutable (finalAttrs: { buildInputs = [ nix-flake + nix-flake-c nix-expr-test-support rapidcheck gtest @@ -67,6 +69,7 @@ mkMesonExecutable (finalAttrs: { mkdir -p "$HOME" '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + export NIX_CONFIG="extra-experimental-features = flakes" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); From d004c524b84651c2eebb5bbb55f6a3a8324437e9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 24 Nov 2024 13:52:40 +0100 Subject: [PATCH 10/93] test: Change FAIL to throw [FAIL()] is a macro with `return`, making it unsuitable for helpers. This uses std::runtime_error, because gtest does not seem to provide an exception type of its own for this purpose. [AssertionException] is for a different use case. [FAIL()]: https://google.github.io/googletest/reference/assertions.html#FAIL [AssertionException]: https://github.com/google/googletest/blob/35d0c365609296fa4730d62057c487e3cfa030ff/docs/reference/testing.md#assertionexception-assertionexception --- src/libutil-test-support/tests/gtest-with-params.hh | 2 +- src/libutil-test-support/tests/nix_api_util.hh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libutil-test-support/tests/gtest-with-params.hh b/src/libutil-test-support/tests/gtest-with-params.hh index d72aec4fd78..a6e23ad8965 100644 --- a/src/libutil-test-support/tests/gtest-with-params.hh +++ b/src/libutil-test-support/tests/gtest-with-params.hh @@ -40,7 +40,7 @@ void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) } else { std::ostringstream ss; printResultMessage(result, ss); - FAIL() << ss.str() << std::endl; + throw std::runtime_error(ss.str()); } } } diff --git a/src/libutil-test-support/tests/nix_api_util.hh b/src/libutil-test-support/tests/nix_api_util.hh index efd2001167d..006dc497ccf 100644 --- a/src/libutil-test-support/tests/nix_api_util.hh +++ b/src/libutil-test-support/tests/nix_api_util.hh @@ -26,14 +26,13 @@ protected: inline void assert_ctx_ok() { - if (nix_err_code(ctx) == NIX_OK) { return; } unsigned int n; const char * p = nix_err_msg(nullptr, ctx, &n); std::string msg(p, n); - FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg; + throw std::runtime_error(std::string("nix_err_code(ctx) != NIX_OK, message: ") + msg); } inline void assert_ctx_err() @@ -41,7 +40,7 @@ protected: if (nix_err_code(ctx) != NIX_OK) { return; } - FAIL() << "Got NIX_OK, but expected an error!"; + throw std::runtime_error("Got NIX_OK, but expected an error!"); } }; From 6db6b269ed70788314209d35499812c90949057f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Nov 2024 01:16:02 +0100 Subject: [PATCH 11/93] .github/ci: Set max-jobs to 1, to reduce peak memory usage --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9918875d9b0..be96bb48469 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,9 @@ jobs: - uses: cachix/install-nix-action@v30 with: # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: "sandbox = true" + extra_nix_config: | + sandbox = true + max-jobs = 1 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' From 6502dc4d6af5baad369578ee0b4d2e1295d199a7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Nov 2024 12:06:54 +0100 Subject: [PATCH 12/93] ci(Mergify): configuration update Signed-off-by: Robert Hensing --- .mergify.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index c297d3d5ef4..86623a13838 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -90,3 +90,13 @@ pull_request_rules: - "2.24-maintenance" labels: - merge-queue + + - name: backport patches to 2.25 + conditions: + - label=backport 2.25-maintenance + actions: + backport: + branches: + - "2.25-maintenance" + labels: + - merge-queue From 3fb7481e64fc22313211c6c4ea79ac314457f81b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 24 Nov 2024 11:17:17 +0100 Subject: [PATCH 13/93] source-accessor: fix case where normalization goes beyond root fixes https://github.com/NixOS/nix/issues/11936 --- src/libutil/source-accessor.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e797951c76e..d3e304f740d 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -84,9 +84,10 @@ CanonPath SourceAccessor::resolveSymlinks( todo.pop_front(); if (c == "" || c == ".") ; - else if (c == "..") - res.pop(); - else { + else if (c == "..") { + if (!res.isRoot()) + res.pop(); + } else { res.push(c); if (mode == SymlinkResolution::Full || !todo.empty()) { if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) { From 57fea81f8a6ab3b91b1003d27bd48effad30b25a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Nov 2024 15:59:43 +0100 Subject: [PATCH 14/93] Work around gcc warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This shuts up a 300-line warning that includes /nix/store/fg7ass3a5m5pgl26qzfdniicbwbgzccy-gcc-13.2.0/include/c++/13.2.0/bits/stl_tree.h:182:25: warning: ‘*(std::_Rb_tree_header*)((char*)& + offsetof(nix::value_type, nix::DerivedPath::.std::variant::.std::__detail::__variant::_Variant_base::.std::__detail::__variant::_Move_assign_base::.std::__detail::__variant::_Copy_assign_base::.std::__detail::__variant::_Move_ctor_base::.std::__detail::__variant::_Copy_ctor_base::.std::__detail::__variant::_Variant_storage::_M_u) + 24).std::_Rb_tree_header::_M_header.std::_Rb_tree_node_base::_M_parent’ may be used uninitialized [-Wmaybe-uninitialized] 182 | if (__x._M_header._M_parent != nullptr) | ~~~~~~~~~~~~~~^~~~~~~~~ --- src/nix/flake.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ce2faacb034..925b66d3e26 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -643,10 +643,11 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { - drvPaths.push_back(DerivedPath::Built { + auto path = DerivedPath::Built { .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, - }); + }; + drvPaths.push_back(std::move(path)); } } } From fafaec5ac35d517a6e6217416336346b7353ea05 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:05:08 +0300 Subject: [PATCH 15/93] fix(treewide): remove unnecessary copying in range for loops This gets rid of unnecessary copies in range-based-for loops and local variables, when they are used solely as `const &`. Also added a fixme comment about a suspicious move out of const, which might not be intended. --- src/libcmd/installables.cc | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/context.cc | 2 ++ src/libexpr/primops/fetchClosure.cc | 2 +- src/libstore/build/drv-output-substitution-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/keys.cc | 4 ++-- src/libstore/local-overlay-store.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/store-api.cc | 4 ++-- src/libutil-tests/hilite.cc | 3 +-- src/libutil/args.cc | 2 +- src/libutil/url.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/flake.cc | 2 +- src/nix/hash.cc | 4 ++-- src/nix/run.cc | 4 ++-- src/nix/verify.cc | 2 +- 20 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f2b27af7c87..250cd141399 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -858,7 +858,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() applyDefaultInstallables(rawInstallables); std::vector res; res.reserve(rawInstallables.size()); - for (auto i : rawInstallables) + for (const auto & i : rawInstallables) res.push_back(parseFlakeRefWithFragment( fetchSettings, expandTilde(i), diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d2f75373a5..7e13e945cf2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4383,7 +4383,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) for (auto i = begin; i != end; ++i) { assert(idx <= 2 * len + 1 - 3); - auto match = *i; + const auto & match = *i; // Add a string for non-matched characters. list[idx++] = mkString(state, match.prefix()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 02683b173aa..ede7d97ba34 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -132,6 +132,8 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V }, [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { /* Reuse original item because we want this to be idempotent. */ + /* FIXME: Suspicious move out of const. This is actually a copy, so the comment + above does not make much sense. */ return std::move(c); }, }, context.begin()->raw) }), diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index fc5bb31454f..04b8d059599 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -40,7 +40,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor }); } - auto toPath = *toPathMaybe; + const auto & toPath = *toPathMaybe; // check and return diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index dedcad2b100..f069c0d9404 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -32,7 +32,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() bool substituterFailed = false; - for (auto sub : subs) { + for (const auto & sub : subs) { trace("trying next substituter"); /* The callback of the curl download below can outlive `this` (if diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 315500719a0..983c86601d8 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -57,7 +57,7 @@ Goal::Co PathSubstitutionGoal::init() bool substituterFailed = false; - for (auto sub : subs) { + for (const auto & sub : subs) { trace("trying next substituter"); cleanup(); diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc index 70478e7ad01..668725fc7e8 100644 --- a/src/libstore/keys.cc +++ b/src/libstore/keys.cc @@ -10,12 +10,12 @@ PublicKeys getDefaultPublicKeys() // FIXME: filter duplicates - for (auto s : settings.trustedPublicKeys.get()) { + for (const auto & s : settings.trustedPublicKeys.get()) { PublicKey key(s); publicKeys.emplace(key.name, key); } - for (auto secretKeyFile : settings.secretKeyFiles.get()) { + for (const auto & secretKeyFile : settings.secretKeyFiles.get()) { try { SecretKey secretKey(readFile(secretKeyFile)); publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index b86beba2caf..56ff6bef3e5 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -156,7 +156,7 @@ void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) { auto res = LocalStore::queryValidDerivers(path); - for (auto p : lowerStore->queryValidDerivers(path)) + for (const auto & p : lowerStore->queryValidDerivers(path)) res.insert(p); return res; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 8b255706072..27fcc286411 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -118,7 +118,7 @@ std::string NarInfo::to_string(const Store & store) const if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; - for (auto sig : sigs) + for (const auto & sig : sigs) res += "Sig: " + sig + "\n"; if (ca) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index bcbf0b55ebc..bf351a56d31 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -454,7 +454,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual debug("got %d keys, next marker '%s'", contents.size(), res.GetNextMarker()); - for (auto object : contents) { + for (const auto & object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 10577fa2a32..78cc3b917d2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1332,7 +1332,7 @@ ref openStore(StoreReference && storeURI) return std::make_shared(params); }, [&](const StoreReference::Specified & g) { - for (auto implem : *Implementations::registered) + for (const auto & implem : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) return implem.create(g.scheme, g.authority, params); @@ -1363,7 +1363,7 @@ std::list> getDefaultSubstituters() } }; - for (auto uri : settings.substituters.get()) + for (const auto & uri : settings.substituters.get()) addStore(uri); stores.sort([](ref & a, ref & b) { diff --git a/src/libutil-tests/hilite.cc b/src/libutil-tests/hilite.cc index 1ff5980d583..5ef58188884 100644 --- a/src/libutil-tests/hilite.cc +++ b/src/libutil-tests/hilite.cc @@ -52,8 +52,7 @@ namespace nix { std::regex("pt"), }; std::vector matches; - for(auto regex : regexes) - { + for (const auto & regex : regexes) { for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { matches.push_back(*it); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 385b6cd346b..05ecf724ef6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -348,7 +348,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) /* Now that all the other args are processed, run the deferred completions. */ - for (auto d : deferredCompletions) + for (const auto & d : deferredCompletions) d.completer(*completions, d.n, d.prefix); } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9ed49dcbe2d..63b9734ee51 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -77,7 +77,7 @@ std::map decodeQuery(const std::string & query) { std::map result; - for (auto s : tokenizeString(query, "&")) { + for (const auto & s : tokenizeString(query, "&")) { auto e = s.find('='); if (e == std::string::npos) { warn("dubious URI query '%s' is missing equal sign '%s', ignoring", s, "="); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c823c930e18..b731b25afed 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -222,7 +222,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { auto method = FileIngestionMethod::Flat; - for (auto i : opFlags) + for (const auto & i : opFlags) if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9a95bc69598..1736add9abd 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -611,7 +611,7 @@ struct CmdDevelop : Common, MixEnvironment else if (!command.empty()) { std::vector args; args.reserve(command.size()); - for (auto s : command) + for (const auto & s : command) args.push_back(shellEscape(s)); script += fmt("exec %s\n", concatStringsSep(" ", args)); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 925b66d3e26..cbc3cdb6579 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -94,7 +94,7 @@ struct CmdFlakeUpdate : FlakeCommand .label="inputs", .optional=true, .handler={[&](std::vector inputsToUpdate){ - for (auto inputToUpdate : inputsToUpdate) { + for (const auto & inputToUpdate : inputsToUpdate) { InputPath inputPath; try { inputPath = flake::parseInputPath(inputToUpdate); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 62266fda15a..2f9b3fe7c6f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -79,7 +79,7 @@ struct CmdHashBase : Command void run() override { - for (auto path : paths) { + for (const auto & path : paths) { auto makeSink = [&]() -> std::unique_ptr { if (modulus) return std::make_unique(hashAlgo, *modulus); @@ -182,7 +182,7 @@ struct CmdToBase : Command void run() override { warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); - for (auto s : args) + for (const auto & s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; diff --git a/src/nix/run.cc b/src/nix/run.cc index c9857e13e3d..a9f9ef60ff2 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -180,9 +180,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (auto entry : fs::directory_iterator{"/"}) { + for (const auto & entry : fs::directory_iterator{"/"}) { checkInterrupt(); - auto src = entry.path(); + const auto & src = entry.path(); fs::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; auto st = entry.symlink_status(); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 124a05bed2c..52585fe08d5 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -129,7 +129,7 @@ struct CmdVerify : StorePathsCommand size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { - for (auto sig : sigs) { + for (const auto & sig : sigs) { if (!sigsSeen.insert(sig).second) continue; if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig)) validSigs++; From f9980b5715fc403383f8e99d5da8bb91538c996d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Tue, 26 Nov 2024 00:13:54 +0300 Subject: [PATCH 16/93] fix(libutil/config): declare virtual dtor for AbstractConfig This prevents any potential cases of deletion through base pointer and its non-virtual dtor, which might leak memory. Also gets rid of the warning: /nix/store/fg7ass3a5m5pgl26qzfdniicbwbgzccy-gcc-13.2.0/include/c++/13.2.0/bits/stl_construct.h:88:2: warning: destructor called on non-final 'nix::flake::Settings' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] 88 | __location->~_Tp(); .... ../src/libflake-c/nix_api_flake.cc:10:30: note: in instantiation of function template specialization 'nix::make_ref' requested here 10 | auto settings = nix::make_ref(); --- src/libutil/config.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index c0c59ac68f5..e98e09bf76c 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -115,6 +115,8 @@ public: * Re-applies all previously attempted changes to unknown settings */ void reapplyUnknownSettings(); + + virtual ~AbstractConfig() = default; }; /** From 5b8728d393dd1c9bbbf6737500669853da7de1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 26 Nov 2024 07:07:31 +0100 Subject: [PATCH 17/93] more readable errors if symlinks cannot be created Before: filesystem error: cannot create symlink: Permission denied [/nix/store/1s2p3a4rs172336hj2l8n20nz74hf71j-nix-eval-jobs-2.24.1.drv] [/1s2p3a4rs172336hj2l8n20nz74hf71j-nix-eval-jobs-2.24.1.drv.tmp-2772352-1316231068] Now: creating symlink '/wfxz2q489c811n08cdqj7ywxm3n4z6m5-nix-eval-jobs-2.24.1.drv.tmp-2971297-324653080' -> '/nix/store/wfxz2q489c811n08cdqj7ywxm3n4z6m5-nix-eval-jobs-2.24.1.drv': Permission denied --- src/libutil/file-system.cc | 16 +++++++++++++--- src/libutil/file-system.hh | 2 -- src/nix/flake.cc | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 92996ea471c..829700336c1 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -602,7 +602,11 @@ std::pair createTempFile(const Path & prefix) void createSymlink(const Path & target, const Path & link) { - fs::create_symlink(target, link); + try { + fs::create_symlink(target, link); + } catch (fs::filesystem_error & e) { + throw SysError("creating symlink '%1%' -> '%2%'", link, target); + } } void replaceSymlink(const fs::path & target, const fs::path & link) @@ -615,10 +619,16 @@ void replaceSymlink(const fs::path & target, const fs::path & link) fs::create_symlink(target, tmp); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw; + throw SysError("creating symlink '%1%' -> '%2%'", tmp, target); + } + + try { + fs::rename(tmp, link); + } catch (fs::filesystem_error & e) { + if (e.code() == std::errc::file_exists) continue; + throw SysError("renaming '%1%' to '%2%'", tmp, link); } - fs::rename(tmp, link); break; } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4c08cdf58cc..3c49181a0ad 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -250,8 +250,6 @@ void setWriteTime(const std::filesystem::path & path, const struct stat & st); /** * Create a symlink. * - * In the process of being deprecated for - * `std::filesystem::create_symlink`. */ void createSymlink(const Path & target, const Path & link); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 925b66d3e26..4bb5c329e24 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -937,7 +937,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - fs::create_symlink(target, to2); + createSymlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); From d67aa03414ad6e75b2ac2145406fcc936c0f1798 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 26 Nov 2024 18:35:18 +0000 Subject: [PATCH 18/93] src/perl/meson.build: fall back to 'bz2' library lookup Upstream `bzip2` does not provide `pkg-config` files. As a result an attempt to build `nix` on some distributions like Gentoo failos the configure as: $ meson setup .. ... Executing subproject perl ... perl| Run-time dependency bzip2 found: NO (tried pkgconfig and cmake) ../src/perl/meson.build:68:12: ERROR: Dependency "bzip2" not found, tried pkgconfig and cmake The change falls back to `bz2` library for such cases. --- src/perl/meson.build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/perl/meson.build b/src/perl/meson.build index dcb6a68a46d..52d85fd6042 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -65,7 +65,13 @@ yath = find_program('yath', required : false) # Required Libraries #------------------------------------------------- -bzip2_dep = dependency('bzip2') +bzip2_dep = dependency('bzip2', required: false) +if not bzip2_dep.found() + bzip2_dep = cpp.find_library('bz2') + if not bzip2_dep.found() + error('No "bzip2" pkg-config or "bz2" library found') + endif +endif curl_dep = dependency('libcurl') libsodium_dep = dependency('libsodium') From 2679e55232af74b0325877b6a49ed83502711fc0 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 26 Nov 2024 23:08:10 +0000 Subject: [PATCH 19/93] tests/functional/meson.build: always look up `ls` as a `coreutils` proxy Without the change `meson setup` fails on `Gentoo or Debian as those don't use multicall binary: $ meson setup .. ... Executing subproject nix-functional-tests ... ../src/nix-functional-tests/meson.build:24:14: ERROR: Program 'coreutils' not found or not executable The change always uses `ls` to look `coreutils` up. Closes: https://github.com/NixOS/nix/issues/11975 --- tests/functional/meson.build | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 0d46f9ce219..933595cd5ef 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -17,12 +17,10 @@ fs = import('fs') nix = find_program('nix') bash = find_program('bash', native : true) busybox = find_program('busybox', native : true, required : false) -if host_machine.system() == 'windows' - # Because of the state of symlinks on Windows, coreutils.exe doesn't usually exist, but things like ls.exe will - coreutils = find_program('ls', native : true) -else - coreutils = find_program('coreutils', native : true) -endif +# Look up `coreutils` package by searching for `ls` binary. +# Previously we looked up `coreutils` on `linux`, but that is not +# guaranteed to exist either. +coreutils = find_program('ls', native : true) dot = find_program('dot', native : true, required : false) nix_bin_dir = fs.parent(nix.full_path()) From 21ddd2022e40d4727684cadf7aca44b0b4ec622c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 27 Nov 2024 07:39:30 +0100 Subject: [PATCH 20/93] mergify: drop installer test --- .mergify.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index c297d3d5ef4..ac1bee11126 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,9 +2,6 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=installer - - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-latest) - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) - check-success=vm_tests From a5c7709f97ceb567ffd1903aa1cd921ca70d7c7b Mon Sep 17 00:00:00 2001 From: h0nIg Date: Wed, 27 Nov 2024 13:24:46 +0100 Subject: [PATCH 21/93] docker: Fix command "nix profile install", Don't require --impure --- docker.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index 19479e57c18..e2e9da72831 100644 --- a/docker.nix +++ b/docker.nix @@ -258,14 +258,14 @@ let mkdir -p $out/nix/var/nix/profiles/per-user/${uname} ln -s ${profile} $out/nix/var/nix/profiles/default-1-link - ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link - ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels + ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels mkdir -p $out${userHome}/.nix-defexpr - ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels + ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels mkdir -p $out/bin $out/usr/bin From 37fd80588fff0ee5e9e27c9d6a1dbc7d2f740b6d Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 27 Nov 2024 17:11:36 +0300 Subject: [PATCH 22/93] shellcheck: simplify install-nix-from-tarball.sh --- maintainers/flake-module.nix | 1 - scripts/install-nix-from-tarball.sh | 15 +++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index ba6cd2816ca..1d4e85c8c5b 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -496,7 +496,6 @@ ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' - ''^scripts/install-nix-from-tarball\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index 794622530e4..007fe85ee94 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -48,15 +48,14 @@ case "$(uname -s)" in INSTALL_MODE=no-daemon;; esac -# space-separated string -ACTIONS= +ACTION= # handle the command line flags while [ $# -gt 0 ]; do case $1 in --daemon) INSTALL_MODE=daemon - ACTIONS="${ACTIONS}install " + ACTION=install ;; --no-daemon) if [ "$(uname -s)" = "Darwin" ]; then @@ -65,18 +64,14 @@ while [ $# -gt 0 ]; do fi INSTALL_MODE=no-daemon # intentional tail space - ACTIONS="${ACTIONS}install " + ACTION=install ;; - # --uninstall) - # # intentional tail space - # ACTIONS="${ACTIONS}uninstall " - # ;; --yes) export NIX_INSTALLER_YES=1;; --no-channel-add) export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) - export NIX_USER_COUNT=$2 + export NIX_USER_COUNT="$2" shift;; --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; @@ -128,7 +123,7 @@ done if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' - exec "$self/install-multi-user" $ACTIONS # let ACTIONS split + exec "$self/install-multi-user" $ACTION exit 0 fi From 8034589d7eb299c126f169ff0780b5242936acdd Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 14 Aug 2024 00:05:06 -0400 Subject: [PATCH 23/93] parser-state: fix attribute merging --- src/libexpr-tests/trivial.cc | 51 +++++++++ src/libexpr/parser-state.hh | 108 +++++++++++------- ...fail-attrset-merge-drops-later-rec.err.exp | 5 + ...val-fail-attrset-merge-drops-later-rec.nix | 1 + ...val-okay-regrettable-rec-attrset-merge.exp | 1 + ...val-okay-regrettable-rec-attrset-merge.nix | 3 + .../parse-fail-mixed-nested-attrs1.err.exp | 8 +- .../parse-fail-mixed-nested-attrs2.err.exp | 8 +- 8 files changed, 133 insertions(+), 52 deletions(-) create mode 100644 tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp create mode 100644 tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix create mode 100644 tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp create mode 100644 tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix diff --git a/src/libexpr-tests/trivial.cc b/src/libexpr-tests/trivial.cc index e455a571b18..d77b4d53b47 100644 --- a/src/libexpr-tests/trivial.cc +++ b/src/libexpr-tests/trivial.cc @@ -177,6 +177,57 @@ namespace nix { ) ); +// The following macros ultimately define 48 tests (16 variations on three +// templates). Each template tests an expression that can be written in 2^4 +// different ways, by making four choices about whether to write a particular +// attribute path segment as `x.y = ...;` (collapsed) or `x = { y = ...; };` +// (expanded). +// +// The nestedAttrsetMergeXXXX tests check that the expression +// `{ a.b.c = 1; a.b.d = 2; }` has the same value regardless of how it is +// expanded. (That exact expression is exercised in test +// nestedAttrsetMerge0000, because it is fully collapsed. The test +// nestedAttrsetMerge1001 would instead examine +// `{ a = { b.c = 1; }; a.b = { d = 2; }; }`.) +// +// The nestedAttrsetMergeDupXXXX tests check that the expression +// `{ a.b.c = 1; a.b.c = 2; }` throws a duplicate attribute error, again +// regardless of how it is expanded. +// +// The nestedAttrsetMergeLetXXXX tests check that the expression +// `let a.b.c = 1; a.b.d = 2; in a` has the same value regardless of how it is +// expanded. +#define X_EXPAND_IF0(k, v) k "." v +#define X_EXPAND_IF1(k, v) k " = { " v " };" +#define X4(w, x, y, z) \ + TEST_F(TrivialExpressionTest, nestedAttrsetMerge##w##x##y##z) { \ + auto v = eval("{ a.b = { c = 1; d = 2; }; } == { " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " }"); \ + ASSERT_THAT(v, IsTrue()); \ + }; \ + TEST_F(TrivialExpressionTest, nestedAttrsetMergeDup##w##x##y##z) { \ + ASSERT_THROW(eval("{ " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "c = 2;")) " }"), Error); \ + }; \ + TEST_F(TrivialExpressionTest, nestedAttrsetMergeLet##w##x##y##z) { \ + auto v = eval("{ b = { c = 1; d = 2; }; } == (let " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " in a)"); \ + ASSERT_THAT(v, IsTrue()); \ + }; +#define X3(...) X4(__VA_ARGS__, 0) X4(__VA_ARGS__, 1) +#define X2(...) X3(__VA_ARGS__, 0) X3(__VA_ARGS__, 1) +#define X1(...) X2(__VA_ARGS__, 0) X2(__VA_ARGS__, 1) + X1(0) X1(1) +#undef X_EXPAND_IF0 +#undef X_EXPAND_IF1 +#undef X1 +#undef X2 +#undef X3 +#undef X4 + TEST_F(TrivialExpressionTest, functor) { auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5"); ASSERT_THAT(v, IsIntEq(15)); diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 8ad0d9ad714..21a880e8eb7 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -88,6 +88,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); + void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -120,64 +121,29 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + ExprAttrs * nested; if (i->symbol) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(attrPath, pos, j->second.pos); - attrs = attrs2; - } else + nested = dynamic_cast(j->second.e); + if (!nested) { + attrPath.erase(i + 1, attrPath.end()); dupAttr(attrPath, pos, j->second.pos); + } } else { - ExprAttrs * nested = new ExprAttrs; + nested = new ExprAttrs; attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; } } else { - ExprAttrs *nested = new ExprAttrs; + nested = new ExprAttrs; attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; } + attrs = nested; } // Expr insertion. // ========================== if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) - jAttrs->inheritFromExprs = std::make_unique>(); - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { - auto & sel = dynamic_cast(*ad.second.e); - auto & from = dynamic_cast(*sel.e); - from.displ += jAttrs->inheritFromExprs->size(); - } - } - jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); - if (ae->inheritFromExprs) { - jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), - ae->inheritFromExprs->begin(), ae->inheritFromExprs->end()); - } - } else { - dupAttr(attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } + addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos)); } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } @@ -189,6 +155,60 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const } } +/** + * Precondition: attrPath is used for error messages and should already contain + * symbol as its last element. + */ +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def) +{ + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(def.e); + auto jAttrs = dynamic_cast(j->second.e); + + // N.B. In a world in which we are less bound by our past mistakes, we + // would also test that jAttrs and ae are not recursive. The effect of + // not doing so is that any `rec` marker on ae is discarded, and any + // `rec` marker on jAttrs will apply to the attributes in ae. + // See https://github.com/NixOS/nix/issues/9020. + if (jAttrs && ae) { + if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) + jAttrs->inheritFromExprs = std::make_unique>(); + for (auto & ad : ae->attrs) { + if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { + auto & sel = dynamic_cast(*ad.second.e); + auto & from = dynamic_cast(*sel.e); + from.displ += jAttrs->inheritFromExprs->size(); + } + attrPath.emplace_back(AttrName(ad.first)); + addAttr(jAttrs, attrPath, ad.first, std::move(ad.second)); + attrPath.pop_back(); + } + ae->attrs.clear(); + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), + std::make_move_iterator(ae->dynamicAttrs.begin()), + std::make_move_iterator(ae->dynamicAttrs.end())); + ae->dynamicAttrs.clear(); + if (ae->inheritFromExprs) { + jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), + std::make_move_iterator(ae->inheritFromExprs->begin()), + std::make_move_iterator(ae->inheritFromExprs->end())); + ae->inheritFromExprs = nullptr; + } + } else { + dupAttr(attrPath, def.pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(symbol, def); + def.e->setName(symbol); + } +} + inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) { std::sort(formals->formals.begin(), formals->formals.end(), diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp new file mode 100644 index 00000000000..d1cdc7b769f --- /dev/null +++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp @@ -0,0 +1,5 @@ +error: undefined variable 'd' + at /pwd/lang/eval-fail-attrset-merge-drops-later-rec.nix:1:26: + 1| { a.b = 1; a = rec { c = d + 2; d = 3; }; }.c + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix new file mode 100644 index 00000000000..fdb314b9193 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix @@ -0,0 +1 @@ +{ a.b = 1; a = rec { c = d + 2; d = 3; }; }.c diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp new file mode 100644 index 00000000000..1e8b3149621 --- /dev/null +++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp @@ -0,0 +1 @@ +6 diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix new file mode 100644 index 00000000000..8df6a2ad81d --- /dev/null +++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix @@ -0,0 +1,3 @@ +# This is for backwards compatibility, not because we like it. +# See https://github.com/NixOS/nix/issues/9020. +{ a = rec { b = c + 1; d = 2; }; a.c = d + 3; }.a.b diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp index a4472156b29..49a07323f34 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -1,6 +1,6 @@ -error: attribute 'z' already defined at «stdin»:3:16 - at «stdin»:2:3: - 1| { +error: attribute 'x.z' already defined at «stdin»:2:3 + at «stdin»:3:16: 2| x.z = 3; - | ^ 3| x = { y = 3; z = 3; }; + | ^ + 4| } diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp index ead1f0dbd82..36fab2fe68d 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -1,6 +1,6 @@ -error: attribute 'y' already defined at «stdin»:3:9 - at «stdin»:2:3: - 1| { +error: attribute 'x.y.y' already defined at «stdin»:2:3 + at «stdin»:3:9: 2| x.y.y = 3; - | ^ 3| x = { y.y= 3; z = 3; }; + | ^ + 4| } From e5e09006f97700f35c68411df6fe4f8a9d7dc807 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Nov 2024 15:25:51 +0100 Subject: [PATCH 24/93] Work around gcc warning Same as 57fea81f8a6ab3b91b1003d27bd48effad30b25a. --- src/nix-env/nix-env.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ba2baccee85..e9eb5270895 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,12 +481,13 @@ static void printMissing(EvalState & state, PackageInfos & elems) { std::vector targets; for (auto & i : elems) - if (auto drvPath = i.queryDrvPath()) - targets.emplace_back(DerivedPath::Built{ + if (auto drvPath = i.queryDrvPath()) { + auto path = DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, - }); - else + }; + targets.emplace_back(std::move(path)); + } else targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); From 04975f7c32e91cda35adc0c3257c98542be97d0e Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Mon, 28 Oct 2024 09:53:44 +0300 Subject: [PATCH 25/93] install: Allow to specify alternative `sudo` command --- scripts/install-multi-user.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a487d459f40..518c955b4a2 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -56,6 +56,9 @@ readonly NIX_INSTALLED_CACERT="@cacert@" #readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" readonly EXTRACTED_NIX_PATH="$(dirname "$0")" +# allow to override identity change command +readonly NIX_BECOME=${NIX_BECOME:-sudo} + readonly ROOT_HOME=~root if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then @@ -123,7 +126,7 @@ uninstall_directions() { cat < Date: Sat, 23 Nov 2024 09:51:56 +0300 Subject: [PATCH 26/93] Allow `sudo` alternatives when installing from tarball --- scripts/install-nix-from-tarball.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index 007fe85ee94..8d127a9c52c 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -9,6 +9,8 @@ self="$(dirname "$0")" nix="@nix@" cacert="@cacert@" +# allow to override identity change command +readonly NIX_BECOME="${NIX_BECOME:-sudo}" if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 @@ -63,7 +65,6 @@ while [ $# -gt 0 ]; do exit 1 fi INSTALL_MODE=no-daemon - # intentional tail space ACTION=install ;; --yes) @@ -135,8 +136,8 @@ echo "performing a single-user installation of Nix..." >&2 if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" - echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 - if ! sudo sh -c "$cmd"; then + echo "directory $dest does not exist; creating it by running '$cmd' using $NIX_BECOME" >&2 + if ! $NIX_BECOME sh -c "$cmd"; then echo "$0: please manually run '$cmd' as root to create $dest" >&2 exit 1 fi From 747cf4e50f43b510ff054ec14bdef87634231237 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Nov 2024 22:49:04 +0100 Subject: [PATCH 27/93] fix: Add splicing to fix the manual in cross We *could* use a "native" manual instead - ie reusing a native `nixpkgsFor.${buildPlatform}`, but this works, and also works for possible cases where we have a custom or patched build tool. --- flake.nix | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 06025e3b79c..794736af4ec 100644 --- a/flake.nix +++ b/flake.nix @@ -124,18 +124,36 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { - inherit (final) lib; - inherit officialRelease; - src = self; - }); + nixComponents = + lib.makeScopeWithSplicing' + { + inherit (final) splicePackages; + inherit (final.nixDependencies) newScope; + } + { + otherSplices = final.generateSplicesForMkScope "nixComponents"; + f = import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; + }; + }; # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. - nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { - inherit inputs stdenv; - pkgs = final; - }); + nixDependencies = + lib.makeScopeWithSplicing' + { + inherit (final) splicePackages; + inherit (final) newScope; # layered directly on pkgs, unlike nixComponents above + } + { + otherSplices = final.generateSplicesForMkScope "nixDependencies"; + f = import ./packaging/dependencies.nix { + inherit inputs stdenv; + pkgs = final; + }; + }; nix = final.nixComponents.nix-cli; From d67e24afec185eee740180b5f023fc4df462013c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 2 Dec 2024 17:42:11 +0100 Subject: [PATCH 28/93] fix: Add missing manpages to meson.build --- doc/manual/meson.build | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 3630e2dc828..c4cc9b893c0 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -199,6 +199,7 @@ nix3_manpages = [ 'nix3-build', 'nix3-bundle', 'nix3-config', + 'nix3-config-check', 'nix3-config-show', 'nix3-copy', 'nix3-daemon', @@ -206,8 +207,8 @@ nix3_manpages = [ 'nix3-derivation', 'nix3-derivation-show', 'nix3-develop', - #'nix3-doctor', 'nix3-edit', + 'nix3-env-shell', 'nix3-eval', 'nix3-flake-archive', 'nix3-flake-check', @@ -224,6 +225,7 @@ nix3_manpages = [ 'nix3-fmt', 'nix3-hash-file', 'nix3-hash', + 'nix3-hash-convert', 'nix3-hash-path', 'nix3-hash-to-base16', 'nix3-hash-to-base32', @@ -238,6 +240,7 @@ nix3_manpages = [ 'nix3-nar-cat', 'nix3-nar-dump-path', 'nix3-nar-ls', + 'nix3-nar-pack', 'nix3-nar', 'nix3-path-info', 'nix3-print-dev-env', @@ -260,7 +263,7 @@ nix3_manpages = [ 'nix3-repl', 'nix3-run', 'nix3-search', - #'nix3-shell', + 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', 'nix3-store-cat', @@ -270,6 +273,7 @@ nix3_manpages = [ 'nix3-store-diff-closures', 'nix3-store-dump-path', 'nix3-store-gc', + 'nix3-store-info', 'nix3-store-ls', 'nix3-store-make-content-addressed', 'nix3-store', From 038ab46d7a08b34e9706a5f7a3c75995cd145899 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Dec 2024 16:37:07 +0100 Subject: [PATCH 29/93] Restore org.nixos.nix-daemon.plist installation --- misc/launchd/meson.build | 13 +++++++++++++ misc/meson.build | 1 + 2 files changed, 14 insertions(+) create mode 100644 misc/launchd/meson.build diff --git a/misc/launchd/meson.build b/misc/launchd/meson.build new file mode 100644 index 00000000000..5168131d198 --- /dev/null +++ b/misc/launchd/meson.build @@ -0,0 +1,13 @@ +configure_file( + input : 'org.nixos.nix-daemon.plist.in', + output : 'org.nixos.nix-daemon.plist', + install : true, + install_dir : get_option('prefix') / 'Library/LaunchDaemons', + install_mode : 'rw-r--r--', + configuration : { + # TODO: unhardcode paths with something like: + # 'storedir' : store_dir, + # 'localstatedir' : localstatedir, + # 'bindir' : bindir, + }, +) diff --git a/misc/meson.build b/misc/meson.build index a6d1f944bbc..78ae3c06cd4 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -2,4 +2,5 @@ subdir('bash') subdir('fish') subdir('zsh') +subdir('launchd') subdir('systemd') From 63c0f0dcd258b6763970a80c43bed431e691e5ba Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Dec 2024 16:51:01 +0100 Subject: [PATCH 30/93] Install init system configs only when relevant --- misc/meson.build | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/misc/meson.build b/misc/meson.build index 78ae3c06cd4..82f2b0c6571 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -2,5 +2,10 @@ subdir('bash') subdir('fish') subdir('zsh') -subdir('launchd') -subdir('systemd') +if host_machine.system() == 'linux' + subdir('systemd') +endif + +if host_machine.system() == 'darwin' + subdir('launchd') +endif From abcfdb4bdab966016d3c9d725fafdad8eadbbf5d Mon Sep 17 00:00:00 2001 From: mupdt <25388474+mupdt@users.noreply.github.com> Date: Wed, 4 Dec 2024 05:49:21 -0500 Subject: [PATCH 31/93] s3-binary-cache: show the error's request ID The request ID is essential for traceability and debugging purposes. It allows us to connect client-side to server-side events. --- src/libstore/s3-binary-cache-store.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index bf351a56d31..cfa713b00c2 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -48,7 +48,11 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - s + ": " + outcome.GetError().GetMessage()); + fmt( + "%s: %s (request id: %s)", + s, + outcome.GetError().GetMessage(), + outcome.GetError().GetRequestId())); return outcome.GetResultWithOwnership(); } @@ -121,9 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy checkInterrupt(); auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) - printError("AWS error '%s' (%s), will retry in %d ms", + printError("AWS error '%s' (%s; request id: %s), will retry in %d ms", error.GetExceptionName(), error.GetMessage(), + error.GetRequestId(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } From 3b21ea40cc6b6b9d512c15facb2e24c32a368993 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 16:52:30 +0100 Subject: [PATCH 32/93] HttpBinaryCacheStore: Improve error message for unauthorized caches Instead of the unhelpful warning: 'https://cache.flakehub.com' does not appear to be a binary cache you now get warning: unable to download 'https://cache.flakehub.com/nix-cache-info': HTTP error 401 response body: {"code":401,"error":"Unauthorized","message":"Unauthorized."} --- src/libstore/binary-cache-store.cc | 13 ++++++++----- src/libstore/binary-cache-store.hh | 8 ++++++++ src/libstore/http-binary-cache-store.cc | 13 +++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b337..896779f85fc 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -39,15 +39,13 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) void BinaryCacheStore::init() { - std::string cacheInfoFile = "nix-cache-info"; - - auto cacheInfo = getFile(cacheInfoFile); + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { - size_t colon= line.find(':'); - if (colon ==std::string::npos) continue; + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { @@ -63,6 +61,11 @@ void BinaryCacheStore::init() } } +std::optional BinaryCacheStore::getNixCacheInfo() +{ + return getFile(cacheInfoFile); +} + void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc925277..6bd7fd14ac9 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -64,6 +64,8 @@ protected: // The prefix under which realisation infos will be stored const std::string realisationsPrefix = "realisations"; + const std::string cacheInfoFile = "nix-cache-info"; + BinaryCacheStore(const Params & params); public: @@ -84,6 +86,12 @@ public: */ virtual void getFile(const std::string & path, Sink & sink); + /** + * Get the contents of /nix-cache-info. Return std::nullopt if it + * doesn't exist. + */ + virtual std::optional getNixCacheInfo(); + /** * Fetch the specified file and call the specified callback with * the result. A subclass may implement this asynchronously. diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index fc7ac2deac8..f32616f94a6 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,19 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v } } + std::optional getNixCacheInfo() override + { + try { + auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); + return result.data; + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound) + return std::nullopt; + maybeDisable(); + throw; + } + } + /** * This isn't actually necessary read only. We support "upsert" now, so we * have a notion of authentication via HTTP POST/PUT. From d1894f34566cb1a7c56b4225e0faf700dfd5b22d Mon Sep 17 00:00:00 2001 From: wh0 Date: Wed, 4 Dec 2024 23:21:07 -0800 Subject: [PATCH 33/93] tests: derivation-advanced-attributes unset NIX_STORE when built by nix, NIX_STORE is set, which breaks $got when it is not the default /nix/store --- tests/functional/derivation-advanced-attributes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 271f17dc69c..6707b345cc3 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -13,7 +13,7 @@ badExitCode=0 store="$TEST_ROOT/store" for nixFile in derivation/*.nix; do - drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") testName=$(basename "$nixFile" .nix) got="${store}${drvPath}" expected="derivation/$testName.drv" From 52f1cd05956b7745985c5565a463633f0b805639 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2024 16:02:35 +0100 Subject: [PATCH 34/93] nix hash convert: Support SRI hashes that lack trailing '=' characters Fixes #11996. --- src/libutil/hash.cc | 3 ++- src/nix/hash.cc | 9 ++++++--- tests/functional/hash-convert.sh | 14 ++++++++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 748176d3370..b69dec685f5 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -134,7 +134,8 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const Hash Hash::dummy(HashAlgorithm::SHA256); -Hash Hash::parseSRI(std::string_view original) { +Hash Hash::parseSRI(std::string_view original) +{ auto rest = original; // Parse the has type before the separater, if there was one. diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2f9b3fe7c6f..654e67437c0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -222,9 +222,12 @@ struct CmdHashConvert : Command Category category() override { return catUtility; } void run() override { - for (const auto& s: hashStrings) { - Hash h = Hash::parseAny(s, algo); - if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + for (const auto & s: hashStrings) { + Hash h = + from == HashFormat::SRI + ? Hash::parseSRI(s) + : Hash::parseAny(s, algo); + if (from && from != HashFormat::SRI && h.to_string(*from, false) != s) { auto from_as_string = printHashFormat(*from); throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); } diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index 3a099950ff9..1cbe864f326 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -93,15 +93,17 @@ try3() { # Asserting input format fails. # - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] + expectStderr 1 nix hash convert --hash-algo "$1" --from sri "$2" | grepQuiet "is not SRI" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$2" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" + +# Test SRI hashes that lack trailing '=' characters. These are incorrect but we need to support them for backward compatibility. +[[ $(nix hash convert --from sri "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] +[[ $(nix hash convert --from sri "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]] From 33b645cedfabd4f275644ac4df0eac220f444278 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2024 16:19:21 +0100 Subject: [PATCH 35/93] nix hash convert: Don't fail on uppercase base-16 hashes --- src/nix/hash.cc | 10 +++++++--- tests/functional/hash-convert.sh | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 654e67437c0..fd07fa7d1d5 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -222,14 +222,18 @@ struct CmdHashConvert : Command Category category() override { return catUtility; } void run() override { - for (const auto & s: hashStrings) { + for (const auto & s : hashStrings) { Hash h = from == HashFormat::SRI ? Hash::parseSRI(s) : Hash::parseAny(s, algo); - if (from && from != HashFormat::SRI && h.to_string(*from, false) != s) { + if (from + && from != HashFormat::SRI + && h.to_string(*from, false) != + (from == HashFormat::Base16 ? toLower(s) : s)) + { auto from_as_string = printHashFormat(*from); - throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + throw BadHash("input hash '%s' does not have the expected format for '--from %s'", s, from_as_string); } logger->cout(h.to_string(to, to == HashFormat::SRI)); } diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index 1cbe864f326..c40cb469c76 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -98,6 +98,8 @@ try3() { expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash" expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" + # Base-16 hashes can be in uppercase. + nix hash convert --hash-algo "$1" --from base16 "$(echo $2 | tr [a-z] [A-Z])" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" From 408c2faf9365ab1b6f9e28551429d9352f9e7f1a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2024 16:25:05 +0100 Subject: [PATCH 36/93] nix hash: Don't print 'nix hash' deprecation message Fixes #11997. --- src/nix/hash.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index fd07fa7d1d5..416cd19b317 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -163,8 +163,11 @@ struct CmdToBase : Command HashFormat hashFormat; std::optional hashAlgo; std::vector args; + bool legacyCli; - CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) + CmdToBase(HashFormat hashFormat, bool legacyCli = false) + : hashFormat(hashFormat) + , legacyCli(legacyCli) { addFlag(flag::hashAlgoOpt("type", &hashAlgo)); expectArgs("strings", &args); @@ -181,7 +184,8 @@ struct CmdToBase : Command void run() override { - warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); + if (!legacyCli) + warn("The old format conversion subcommands of `nix hash` were deprecated in favor of `nix hash convert`."); for (const auto & s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } @@ -328,7 +332,7 @@ static int compatNixHash(int argc, char * * argv) } else { - CmdToBase cmd(hashFormat); + CmdToBase cmd(hashFormat, true); cmd.args = ss; if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo; cmd.run(); From 4137ead7a1011c3f410899be089822e8cf33862e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 5 Dec 2024 14:01:00 -0500 Subject: [PATCH 37/93] Disable suid and atime on the /nix mount point on Darwin The Determinate Nix Installer has set nosuid and noatime in https://github.com/DeterminateSystems/nix-installer/pull/1338, and figured this perf and security improvement is worthy of upstreaming. The /nix volume shouldn't have setuid binaries anyway, and filesystems seem to generally be noatime on macOS. Further, the garbage collector doesn't use atime. --- scripts/create-darwin-volume.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 103e1e3916c..7a61764d4f3 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -463,7 +463,7 @@ EOF EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" < Date: Thu, 5 Dec 2024 20:10:59 +0100 Subject: [PATCH 38/93] Fix typo (#12015) --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7e13e945cf2..5202ef7d72b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1100,7 +1100,7 @@ static RegisterPrimOp primop_warn({ .name = "__warn", .args = {"e1", "e2"}, .doc = R"( - Evaluate *e1*, which must be a string and print iton standard error as a warning. + Evaluate *e1*, which must be a string, and print it on standard error as a warning. Then return *e2*. This function is useful for non-critical situations where attention is advisable. From ad296eae2a1f1b6ad6399f364f8dc91c80923922 Mon Sep 17 00:00:00 2001 From: "Shahar \"Dawn\" Or" Date: Sat, 7 Dec 2024 06:37:30 +0700 Subject: [PATCH 39/93] Test: more specific error message for `head` Sorry, I'm not sure how to implement this. So just a test change. And hopefully will be picked up by someone who is paying attention. A hero. --- src/libexpr-tests/error_traces.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index be379a90993..011a0848d61 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -712,7 +712,7 @@ namespace nix { ASSERT_TRACE1("head []", Error, - HintFmt("list index %d is out of bounds", 0)); + HintFmt("'head' called on an empty list")); } From ad3a67a2a0dc815219593a7aae306db8ed8ec3a3 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 7 Dec 2024 04:42:04 +0100 Subject: [PATCH 40/93] optimize string concat --- src/libutil/util.hh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0fb6ff837ec..4d5683e2bda 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -338,7 +338,9 @@ std::string showBytes(uint64_t bytes); */ inline std::string operator + (const std::string & s1, std::string_view s2) { - auto s = s1; + std::string s; + s.reserve(s1.size() + s2.size()); + s.append(s1); s.append(s2); return s; } @@ -351,10 +353,11 @@ inline std::string operator + (std::string && s, std::string_view s2) inline std::string operator + (std::string_view s1, const char * s2) { + auto s2Size = strlen(s2); std::string s; - s.reserve(s1.size() + strlen(s2)); + s.reserve(s1.size() + s2Size); s.append(s1); - s.append(s2); + s.append(s2, s2Size); return s; } From 8b9e0f86e406b53911b6c2d1ec4066a767c44b3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Dec 2024 13:56:03 +0100 Subject: [PATCH 41/93] nix flake lock: Fail if there is an unlocked input Since the only purpose of `nix flake lock` is to write a new lock file, it should be a fatal error if we can't write the lock file. --- src/libflake/flake/flake.cc | 2 ++ src/libflake/flake/flake.hh | 5 +++++ src/nix/flake.cc | 1 + tests/functional/flakes/unlocked-override.sh | 3 +++ 4 files changed, 11 insertions(+) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 19b622a34af..2165ffd8dc9 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -672,6 +672,8 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { + if (lockFlags.failOnUnlocked) + throw Error("cannot write lock file of flake '%s' because it has an unlocked input ('%s').\n", topRef, *unlockedInput); if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index cc2bea76e59..0dfd9440d6e 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -156,6 +156,11 @@ struct LockFlags */ bool writeLockFile = true; + /** + * Throw an exception when the flake has an unlocked input. + */ + bool failOnUnlocked = false; + /** * Whether to use the registries to lookup indirect flake * references like 'nixpkgs'. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9f3584a11ac..55aa8971e12 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -162,6 +162,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; lockFlake(); diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index a17a0c2afe7..ebad332d011 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -30,3 +30,6 @@ git -C "$flake2Dir" add flake.nix echo 456 > "$flake1Dir"/x.nix [[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] + +expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" | + grepQuiet "cannot write lock file.*because it has an unlocked input" From e82ff51726b33b95152c4c64dc870eab831936f9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 20 Nov 2024 11:05:55 +0100 Subject: [PATCH 42/93] tests/functional/dependencies.nix: Refactor, replace arcane let --- tests/functional/dependencies.nix | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index be1a7ae9a6e..4ff29227fd3 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,7 +1,7 @@ { hashInvalidator ? "" }: with import ./config.nix; -let { +let input0 = mkDerivation { name = "dependencies-input-0"; @@ -33,16 +33,15 @@ let { outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - body = mkDerivation { - name = "dependencies-top"; - builder = ./dependencies.builder0.sh + "/FOOBAR/../."; - input1 = input1 + "/."; - input2 = "${input2}/."; - input1_drv = input1; - input2_drv = input2; - input0_drv = input0; - fod_input_drv = fod_input; - meta.description = "Random test package"; - }; - +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + fod_input_drv = fod_input; + meta.description = "Random test package"; } From ee03fd478e99d90eb02baefda6dee7eb9a3c347f Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Fri, 11 Oct 2024 17:16:32 +0200 Subject: [PATCH 43/93] libutil: handle json builder log messages with unexpected format Before this change, expressions like: with import {}; runCommand "foo" {} '' echo '@nix {}' >&$NIX_LOG_FD '' would result in Lix crashing, because accessing nonexistent fields of a JSON object throws an exception. Rather than handling each field individually, we just catch JSON exceptions wholesale. Since these log messages are an unusual circumstance, log a warning when this happens. Fixes #544. Change-Id: Idc2d8acf6e37046b3ec212f42e29269163dca893 (cherry picked from commit e55cd3beea710db727fd966f265a1b715b7285f3) --- src/libutil/logging.cc | 60 +++++++++++++++++-------------- tests/functional/dependencies.nix | 13 +++++++ 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 80c107ef538..0eceee6d4ce 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -295,37 +295,45 @@ bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, bool trusted) { - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actFileTransfer) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); - } + try { + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } - else if (action == "stop") - activities.erase((ActivityId) json["id"]); + else if (action == "stop") + activities.erase((ActivityId) json["id"]); - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } - return true; + return true; + } catch (const nlohmann::json::exception &e) { + warn( + "warning: Unable to handle a JSON message from the builder: %s", + e.what() + ); + return false; + } } bool handleJSONLogMessage(const std::string & msg, diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index 4ff29227fd3..64d9d236009 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -33,6 +33,18 @@ let outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; + unusual-logging = mkDerivation { + name = "unusual-logging"; + buildCommand = '' + { + echo "@nix 1" + echo "@nix {}" + echo '@nix {"action": null}' + } >&$NIX_LOG_FD + touch $out + ''; + }; + in mkDerivation { name = "dependencies-top"; @@ -42,6 +54,7 @@ mkDerivation { input1_drv = input1; input2_drv = input2; input0_drv = input0; + unusual_logging_drv = unusual-logging; fod_input_drv = fod_input; meta.description = "Random test package"; } From 1485937b897bc88cda0848b54b8ad9310230c47b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 20 Nov 2024 11:16:39 +0100 Subject: [PATCH 44/93] tests/functional/dependencies.nix: Check that we tolerate syntax and type errors --- tests/functional/dependencies.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index 64d9d236009..d177f5b6358 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -40,6 +40,8 @@ let echo "@nix 1" echo "@nix {}" echo '@nix {"action": null}' + echo '@nix {"action": 123}' + echo '@nix ][' } >&$NIX_LOG_FD touch $out ''; From 03d4bfd852dce9a050f984e887c887a43581796c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 21 Nov 2024 16:03:21 +0100 Subject: [PATCH 45/93] Push log source description out of libutil and report build hook @nix warning correctly --- src/libstore/build/derivation-goal.cc | 8 ++++---- src/libutil/logging.cc | 17 ++++++++++------- src/libutil/logging.hh | 13 ++++++++++++- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bf1a25db1f0..2ff0ef92f2c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1161,7 +1161,7 @@ HookReply DerivationGoal::tryBuildHook() throw; } }(); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, "the build hook", true)) ; else if (s.substr(0, 2) == "# ") { reply = s.substr(2); @@ -1346,9 +1346,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { - auto json = parseJSONMessage(currentHookLine); + auto json = parseJSONMessage(currentHookLine, "the derivation builder"); if (json) { - auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); + auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. if (s && !isWrittenToLog && logSink) { @@ -1390,7 +1390,7 @@ void DerivationGoal::handleEOF(Descriptor fd) void DerivationGoal::flushLine() { - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) ; else { diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 0eceee6d4ce..4c706101651 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -280,20 +280,22 @@ static Logger::Fields getFields(nlohmann::json & json) return fields; } -std::optional parseJSONMessage(const std::string & msg) +std::optional parseJSONMessage(const std::string & msg, std::string_view source) { if (!hasPrefix(msg, "@nix ")) return std::nullopt; try { return nlohmann::json::parse(std::string(msg, 5)); } catch (std::exception & e) { - printError("bad JSON log message from builder: %s", e.what()); + printError("bad JSON log message from %s: %s", + Uncolored(source), + e.what()); } return std::nullopt; } bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, - bool trusted) + std::string_view source, bool trusted) { try { std::string action = json["action"]; @@ -329,7 +331,8 @@ bool handleJSONLogMessage(nlohmann::json & json, return true; } catch (const nlohmann::json::exception &e) { warn( - "warning: Unable to handle a JSON message from the builder: %s", + "warning: Unable to handle a JSON message from %s: %s", + Uncolored(source), e.what() ); return false; @@ -337,12 +340,12 @@ bool handleJSONLogMessage(nlohmann::json & json, } bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map & activities, bool trusted) + const Activity & act, std::map & activities, std::string_view source, bool trusted) { - auto json = parseJSONMessage(msg); + auto json = parseJSONMessage(msg, source); if (!json) return false; - return handleJSONLogMessage(*json, act, activities, trusted); + return handleJSONLogMessage(*json, act, activities, source, trusted); } Activity::~Activity() diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 250f9209972..11e4033a59d 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -185,14 +185,25 @@ Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); -std::optional parseJSONMessage(const std::string & msg); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ +std::optional parseJSONMessage(const std::string & msg, std::string_view source); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, + std::string_view source, bool trusted); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, + std::string_view source, bool trusted); /** From 8490fba42d49cec068eea0a442bdc125c0030cd2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Dec 2024 16:15:23 +0100 Subject: [PATCH 46/93] Improve error messages for head/elemAt --- src/libexpr-tests/error_traces.cc | 14 ++++++------ src/libexpr/primops.cc | 36 ++++++++++++++++--------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index 011a0848d61..2aa13cf62de 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -691,15 +691,15 @@ namespace nix { ASSERT_TRACE2("elemAt \"foo\" (-1)", TypeError, HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); ASSERT_TRACE1("elemAt [] (-1)", Error, - HintFmt("list index %d is out of bounds", -1)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); ASSERT_TRACE1("elemAt [\"foo\"] 3", Error, - HintFmt("list index %d is out of bounds", 3)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); } @@ -708,11 +708,11 @@ namespace nix { ASSERT_TRACE2("head 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.head'")); ASSERT_TRACE1("head []", Error, - HintFmt("'head' called on an empty list")); + HintFmt("'builtins.head' called on an empty list")); } @@ -721,11 +721,11 @@ namespace nix { ASSERT_TRACE2("tail 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.tail")); + HintFmt("while evaluating the first argument passed to 'builtins.tail'")); ASSERT_TRACE1("tail []", Error, - HintFmt("'tail' called on an empty list")); + HintFmt("'builtins.tail' called on an empty list")); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5202ef7d72b..7c5c6ea9b85 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3259,23 +3259,19 @@ static RegisterPrimOp primop_isList({ .fun = prim_isList, }); -static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) -{ - state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); - if (n < 0 || (unsigned int) n >= list.listSize()) - state.error( - "list index %1% is out of bounds", - n - ).atPos(pos).debugThrow(); - state.forceValue(*list.listElems()[n], pos); - v = *list.listElems()[n]; -} - /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; - elemAt(state, pos, *args[0], elem, v); + NixInt::Inner n = state.forceInt(*args[1], pos, "while evaluating the second argument passed to 'builtins.elemAt'").value; + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.elemAt'"); + if (n < 0 || (unsigned int) n >= args[0]->listSize()) + state.error( + "'builtins.elemAt' called with index %d on a list of size %d", + n, + args[0]->listSize() + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[n], pos); + v = *args[0]->listElems()[n]; } static RegisterPrimOp primop_elemAt({ @@ -3291,7 +3287,13 @@ static RegisterPrimOp primop_elemAt({ /* Return the first element of a list. */ static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], 0, v); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.head'"); + if (args[0]->listSize() == 0) + state.error( + "'builtins.head' called on an empty list" + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[0], pos); + v = *args[0]->listElems()[0]; } static RegisterPrimOp primop_head({ @@ -3310,9 +3312,9 @@ static RegisterPrimOp primop_head({ don't want to use it! */ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.tail'"); if (args[0]->listSize() == 0) - state.error("'tail' called on an empty list").atPos(pos).debugThrow(); + state.error("'builtins.tail' called on an empty list").atPos(pos).debugThrow(); auto list = state.buildList(args[0]->listSize() - 1); for (const auto & [n, v] : enumerate(list)) From 1edf868213a5462b67c3c269f34bf8555c77758a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 16:49:02 +0100 Subject: [PATCH 47/93] rename: nix-tests -> nix-daemon-compat-tests I think I have failed to read the very long version-garbage-like string for the second time now, leaving me oblivious to the crucial info that a test failure happens in the context of an older daemon. --- packaging/hydra.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 81406a2493c..6109f479eae 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -19,11 +19,11 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { pname = - "nix-tests" + "nix-daemon-compat-tests" + lib.optionalString (lib.versionAtLeast daemon.version "2.4pre20211005" && lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "-${pkgs.nix.version}-against-${daemon.version}"; + "-${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; From d0b4db924a80d74540b81a6b717aa01498df8ceb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 16:39:48 +0100 Subject: [PATCH 48/93] rename: build-utils-meson -> nix-meson-build-support Fix a footgun. In my case, I had a couple of build ("output") directories sitting around. rm -rf build-* Was confused for a bit why a meson.build file was missing. Probably also helps with autocompletion. I tried meson-build-support first, but I had to add something like a nix- prefix, in order to make meson happy. They've reserved the meson- prefix. --- .../common/meson.build | 0 .../deps-lists/meson.build | 0 .../export-all-symbols/meson.build | 0 .../export/meson.build | 0 .../generate-header/meson.build | 0 .../libatomic/meson.build | 0 .../subprojects/meson.build | 0 .../windows-version/meson.build | 0 src/libcmd/build-utils-meson | 1 - src/libcmd/meson.build | 8 ++++---- src/libcmd/nix-meson-build-support | 1 + src/libcmd/package.nix | 4 ++-- src/libexpr-c/build-utils-meson | 1 - src/libexpr-c/meson.build | 12 ++++++------ src/libexpr-c/nix-meson-build-support | 1 + src/libexpr-c/package.nix | 4 ++-- src/libexpr-test-support/build-utils-meson | 1 - src/libexpr-test-support/meson.build | 12 ++++++------ src/libexpr-test-support/nix-meson-build-support | 1 + src/libexpr-test-support/package.nix | 4 ++-- src/libexpr-tests/build-utils-meson | 1 - src/libexpr-tests/meson.build | 10 +++++----- src/libexpr-tests/nix-meson-build-support | 1 + src/libexpr-tests/package.nix | 4 ++-- src/libexpr/build-utils-meson | 1 - src/libexpr/meson.build | 10 +++++----- src/libexpr/nix-meson-build-support | 1 + src/libexpr/package.nix | 4 ++-- src/libfetchers-tests/build-utils-meson | 1 - src/libfetchers-tests/meson.build | 10 +++++----- src/libfetchers-tests/nix-meson-build-support | 1 + src/libfetchers-tests/package.nix | 4 ++-- src/libfetchers/build-utils-meson | 1 - src/libfetchers/meson.build | 8 ++++---- src/libfetchers/nix-meson-build-support | 1 + src/libfetchers/package.nix | 4 ++-- src/libflake-c/build-utils-meson | 1 - src/libflake-c/meson.build | 12 ++++++------ src/libflake-c/nix-meson-build-support | 1 + src/libflake-c/package.nix | 4 ++-- src/libflake-tests/build-utils-meson | 1 - src/libflake-tests/meson.build | 10 +++++----- src/libflake-tests/nix-meson-build-support | 1 + src/libflake-tests/package.nix | 4 ++-- src/libflake/build-utils-meson | 1 - src/libflake/meson.build | 8 ++++---- src/libflake/nix-meson-build-support | 1 + src/libflake/package.nix | 4 ++-- src/libmain-c/build-utils-meson | 1 - src/libmain-c/meson.build | 12 ++++++------ src/libmain-c/nix-meson-build-support | 1 + src/libmain-c/package.nix | 4 ++-- src/libmain/build-utils-meson | 1 - src/libmain/meson.build | 8 ++++---- src/libmain/nix-meson-build-support | 1 + src/libmain/package.nix | 4 ++-- src/libstore-c/build-utils-meson | 1 - src/libstore-c/meson.build | 12 ++++++------ src/libstore-c/nix-meson-build-support | 1 + src/libstore-c/package.nix | 4 ++-- src/libstore-test-support/build-utils-meson | 1 - src/libstore-test-support/meson.build | 12 ++++++------ .../nix-meson-build-support | 1 + src/libstore-test-support/package.nix | 4 ++-- src/libstore-tests/build-utils-meson | 1 - src/libstore-tests/meson.build | 10 +++++----- src/libstore-tests/nix-meson-build-support | 1 + src/libstore-tests/package.nix | 4 ++-- src/libstore/build-utils-meson | 1 - src/libstore/meson.build | 16 ++++++++-------- src/libstore/nix-meson-build-support | 1 + src/libstore/package.nix | 4 ++-- src/libutil-c/build-utils-meson | 1 - src/libutil-c/meson.build | 12 ++++++------ src/libutil-c/nix-meson-build-support | 1 + src/libutil-c/package.nix | 4 ++-- src/libutil-test-support/build-utils-meson | 1 - src/libutil-test-support/meson.build | 12 ++++++------ src/libutil-test-support/nix-meson-build-support | 1 + src/libutil-test-support/package.nix | 4 ++-- src/libutil-tests/build-utils-meson | 1 - src/libutil-tests/meson.build | 10 +++++----- src/libutil-tests/nix-meson-build-support | 1 + src/libutil-tests/package.nix | 4 ++-- src/libutil/build-utils-meson | 1 - src/libutil/meson.build | 14 +++++++------- src/libutil/nix-meson-build-support | 1 + src/libutil/package.nix | 4 ++-- src/nix/build-utils-meson | 1 - src/nix/meson.build | 12 ++++++------ src/nix/nix-meson-build-support | 1 + src/nix/package.nix | 4 ++-- 92 files changed, 178 insertions(+), 178 deletions(-) rename {build-utils-meson => nix-meson-build-support}/common/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/deps-lists/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/export-all-symbols/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/export/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/generate-header/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/libatomic/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/subprojects/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/windows-version/meson.build (100%) delete mode 120000 src/libcmd/build-utils-meson create mode 120000 src/libcmd/nix-meson-build-support delete mode 120000 src/libexpr-c/build-utils-meson create mode 120000 src/libexpr-c/nix-meson-build-support delete mode 120000 src/libexpr-test-support/build-utils-meson create mode 120000 src/libexpr-test-support/nix-meson-build-support delete mode 120000 src/libexpr-tests/build-utils-meson create mode 120000 src/libexpr-tests/nix-meson-build-support delete mode 120000 src/libexpr/build-utils-meson create mode 120000 src/libexpr/nix-meson-build-support delete mode 120000 src/libfetchers-tests/build-utils-meson create mode 120000 src/libfetchers-tests/nix-meson-build-support delete mode 120000 src/libfetchers/build-utils-meson create mode 120000 src/libfetchers/nix-meson-build-support delete mode 120000 src/libflake-c/build-utils-meson create mode 120000 src/libflake-c/nix-meson-build-support delete mode 120000 src/libflake-tests/build-utils-meson create mode 120000 src/libflake-tests/nix-meson-build-support delete mode 120000 src/libflake/build-utils-meson create mode 120000 src/libflake/nix-meson-build-support delete mode 120000 src/libmain-c/build-utils-meson create mode 120000 src/libmain-c/nix-meson-build-support delete mode 120000 src/libmain/build-utils-meson create mode 120000 src/libmain/nix-meson-build-support delete mode 120000 src/libstore-c/build-utils-meson create mode 120000 src/libstore-c/nix-meson-build-support delete mode 120000 src/libstore-test-support/build-utils-meson create mode 120000 src/libstore-test-support/nix-meson-build-support delete mode 120000 src/libstore-tests/build-utils-meson create mode 120000 src/libstore-tests/nix-meson-build-support delete mode 120000 src/libstore/build-utils-meson create mode 120000 src/libstore/nix-meson-build-support delete mode 120000 src/libutil-c/build-utils-meson create mode 120000 src/libutil-c/nix-meson-build-support delete mode 120000 src/libutil-test-support/build-utils-meson create mode 120000 src/libutil-test-support/nix-meson-build-support delete mode 120000 src/libutil-tests/build-utils-meson create mode 120000 src/libutil-tests/nix-meson-build-support delete mode 120000 src/libutil/build-utils-meson create mode 120000 src/libutil/nix-meson-build-support delete mode 120000 src/nix/build-utils-meson create mode 120000 src/nix/nix-meson-build-support diff --git a/build-utils-meson/common/meson.build b/nix-meson-build-support/common/meson.build similarity index 100% rename from build-utils-meson/common/meson.build rename to nix-meson-build-support/common/meson.build diff --git a/build-utils-meson/deps-lists/meson.build b/nix-meson-build-support/deps-lists/meson.build similarity index 100% rename from build-utils-meson/deps-lists/meson.build rename to nix-meson-build-support/deps-lists/meson.build diff --git a/build-utils-meson/export-all-symbols/meson.build b/nix-meson-build-support/export-all-symbols/meson.build similarity index 100% rename from build-utils-meson/export-all-symbols/meson.build rename to nix-meson-build-support/export-all-symbols/meson.build diff --git a/build-utils-meson/export/meson.build b/nix-meson-build-support/export/meson.build similarity index 100% rename from build-utils-meson/export/meson.build rename to nix-meson-build-support/export/meson.build diff --git a/build-utils-meson/generate-header/meson.build b/nix-meson-build-support/generate-header/meson.build similarity index 100% rename from build-utils-meson/generate-header/meson.build rename to nix-meson-build-support/generate-header/meson.build diff --git a/build-utils-meson/libatomic/meson.build b/nix-meson-build-support/libatomic/meson.build similarity index 100% rename from build-utils-meson/libatomic/meson.build rename to nix-meson-build-support/libatomic/meson.build diff --git a/build-utils-meson/subprojects/meson.build b/nix-meson-build-support/subprojects/meson.build similarity index 100% rename from build-utils-meson/subprojects/meson.build rename to nix-meson-build-support/subprojects/meson.build diff --git a/build-utils-meson/windows-version/meson.build b/nix-meson-build-support/windows-version/meson.build similarity index 100% rename from build-utils-meson/windows-version/meson.build rename to nix-meson-build-support/windows-version/meson.build diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libcmd/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 1f27c161402..222817c81e0 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -14,7 +14,7 @@ project('nix-cmd', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -28,7 +28,7 @@ deps_public_maybe_subproject = [ dependency('nix-flake'), dependency('nix-main'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -70,7 +70,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'built-path.cc', @@ -125,4 +125,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libcmd/nix-meson-build-support b/src/libcmd/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libcmd/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 244179ee4f7..53e54d2f8d2 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -39,8 +39,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libexpr-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 5bcca29e0b8..1556dae519c 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -14,7 +14,7 @@ project('nix-expr-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_expr.cc', @@ -72,8 +72,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_expr_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixexprc', @@ -89,4 +89,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-c/nix-meson-build-support b/src/libexpr-c/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libexpr-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index df49a8bdc2f..727b3a81138 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-test-support/build-utils-meson b/src/libexpr-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libexpr-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index 33d9e17a6a8..64d4fe21862 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-expr-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -26,7 +26,7 @@ deps_public_maybe_subproject = [ dependency('nix-expr'), dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -40,7 +40,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/value/context.cc', @@ -54,8 +54,8 @@ headers = files( 'tests/value/context.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-expr-test-support', @@ -73,4 +73,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-test-support/nix-meson-build-support b/src/libexpr-test-support/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libexpr-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 7e92d145f78..4842f5f17f0 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-tests/build-utils-meson b/src/libexpr-tests/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libexpr-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index b50c18c9c8f..f37e85e5722 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -14,7 +14,7 @@ project('nix-expr-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'derived-path.cc', diff --git a/src/libexpr-tests/nix-meson-build-support b/src/libexpr-tests/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libexpr-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 959d6b84efd..70e497b7ec5 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libexpr/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 28318579e92..b3c559ba71c 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -14,7 +14,7 @@ project('nix-expr', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-fetchers'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') boost = dependency( 'boost', @@ -77,7 +77,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') parser_tab = custom_target( input : 'parser.y', @@ -121,7 +121,7 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -205,4 +205,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr/nix-meson-build-support b/src/libexpr/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libexpr/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d97e7f3a82f..5171d70fd64 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -40,8 +40,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers-tests/build-utils-meson b/src/libfetchers-tests/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libfetchers-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index fdab6ba6c41..3e82c61110d 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store-test-support'), @@ -22,10 +22,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'public-key.cc', diff --git a/src/libfetchers-tests/nix-meson-build-support b/src/libfetchers-tests/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libfetchers-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index 7b2ba8f2cd8..e18d644ed4c 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libfetchers/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 07a1178cced..b4408e94318 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -41,7 +41,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', @@ -90,4 +90,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libfetchers/nix-meson-build-support b/src/libfetchers/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libfetchers/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 70973bdb26c..7dad0002577 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake-c/build-utils-meson b/src/libflake-c/build-utils-meson deleted file mode 120000 index 91937f18372..00000000000 --- a/src/libflake-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index 00d9650e770..b7669fe9778 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -14,7 +14,7 @@ project('nix-flake-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_public_maybe_subproject = [ dependency('nix-store-c'), dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_flake.cc', @@ -73,8 +73,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_flake.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixflakec', @@ -90,4 +90,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake-c/nix-meson-build-support b/src/libflake-c/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libflake-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index a70cbf94ef5..7425d614050 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -21,8 +21,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake-tests/build-utils-meson b/src/libflake-tests/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libflake-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index c494c414e99..5c3c58e53a8 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -14,7 +14,7 @@ project('nix-flake-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr-test-support'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -43,7 +43,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flakeref.cc', diff --git a/src/libflake-tests/nix-meson-build-support b/src/libflake-tests/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libflake-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index b3a8ac46644..f1abbb32d01 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libflake/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 2c1a70a1840..f9e2177294e 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -14,7 +14,7 @@ project('nix-flake', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-fetchers'), dependency('nix-expr'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -39,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flake/config.cc', @@ -72,4 +72,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake/nix-meson-build-support b/src/libflake/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libflake/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake/package.nix b/src/libflake/package.nix index fff481720d9..92445739f12 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libmain-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 3cb1e4baae7..5a5684b8d63 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -14,7 +14,7 @@ project('nix-main-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_main.cc', @@ -65,8 +65,8 @@ headers = [config_h] + files( 'nix_api_main.h', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixmainc', @@ -82,4 +82,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain-c/nix-meson-build-support b/src/libmain-c/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libmain-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 5522037f3ec..d657929210c 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libmain/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 6c6298e2be7..87fc8b8d29f 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -14,7 +14,7 @@ project('nix-main', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') pubsetbuf_test = ''' #include @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-args.cc', @@ -96,4 +96,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain/nix-meson-build-support b/src/libmain/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libmain/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7e7b8047284..7d9d99b6183 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libstore-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 44b5fe11d81..1ac331ad0fb 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -14,7 +14,7 @@ project('nix-store-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_store.cc', @@ -64,8 +64,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` headers += files('nix_api_store_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstorec', @@ -81,4 +81,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-c/nix-meson-build-support b/src/libstore-c/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libstore-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 896a1a39f05..351d4510af8 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-test-support/build-utils-meson b/src/libstore-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libstore-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 1f230914f6c..2a07e56ac7f 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-store-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -37,7 +37,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/derived-path.cc', @@ -56,8 +56,8 @@ headers = files( 'tests/protocol.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-store-test-support', @@ -75,4 +75,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-test-support/nix-meson-build-support b/src/libstore-test-support/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libstore-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 2543049fe73..62f783c4fc9 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-tests/build-utils-meson b/src/libstore-tests/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libstore-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fc9152f2fb6..c4e42634d8d 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -14,7 +14,7 @@ project('nix-store-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite @@ -50,7 +50,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-protocol.cc', diff --git a/src/libstore-tests/nix-meson-build-support b/src/libstore-tests/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libstore-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 3704d8c5ce8..4937b532958 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -28,8 +28,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libstore/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/meson.build b/src/libstore/meson.build index f836b8d4f62..12a0e637644 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -15,7 +15,7 @@ project('nix-store', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') run_command('ln', '-s', meson.project_build_root() / '__nothing_link_target', @@ -81,7 +81,7 @@ if host_machine.system() == 'windows' deps_other += [wsock32] endif -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') boost = dependency( 'boost', @@ -131,7 +131,7 @@ if aws_s3.found() endif deps_other += aws_s3 -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -179,7 +179,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'binary-cache-store.cc', @@ -416,8 +416,8 @@ foreach name, value : cpp_str_defines ] endforeach -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstore', @@ -446,4 +446,4 @@ if host_machine.system() != 'darwin' } endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore/nix-meson-build-support b/src/libstore/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libstore/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore/package.nix b/src/libstore/package.nix index f04e3b95fc3..47a203f83c3 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -32,8 +32,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libutil-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index d44453676e9..44cec1afc8e 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -14,7 +14,7 @@ project('nix-util-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -23,7 +23,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -45,7 +45,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_util.cc', @@ -60,8 +60,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it. headers += files('nix_api_util_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutilc', @@ -77,4 +77,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/nix-meson-build-support b/src/libutil-c/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libutil-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 35533f98136..4caf968047b 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -19,8 +19,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-test-support/build-utils-meson b/src/libutil-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libutil-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 4afed01ca0d..03ae63f1a31 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-util-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -22,7 +22,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -34,7 +34,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/hash.cc', @@ -51,8 +51,8 @@ headers = files( 'tests/string_callback.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-util-test-support', @@ -70,4 +70,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-test-support/nix-meson-build-support b/src/libutil-test-support/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libutil-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index c403e762c1a..19b5d6b772e 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-tests/build-utils-meson b/src/libutil-tests/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libutil-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f593507742b..83ac79e92b8 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -14,7 +14,7 @@ project('nix-util-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-util'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'args.cc', diff --git a/src/libutil-tests/nix-meson-build-support b/src/libutil-tests/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libutil-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index b099037eee3..f06704e26f1 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson deleted file mode 120000 index 5fff21bab55..00000000000 --- a/src/libutil/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 11b4ea59243..bbe7872cf10 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -14,7 +14,7 @@ project('nix-util', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -22,7 +22,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # Check for each of these functions, and create a define like `#define # HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 @@ -53,7 +53,7 @@ endforeach configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') @@ -120,7 +120,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'archive.cc', @@ -256,8 +256,8 @@ else subdir('unix') endif -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutil', @@ -278,4 +278,4 @@ if host_machine.system() == 'windows' libraries_private += ['-lws2_32'] endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil/nix-meson-build-support b/src/libutil/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/libutil/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 17a15674008..69ebbf726e9 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -25,8 +25,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson deleted file mode 120000 index 91937f18372..00000000000 --- a/src/nix/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/meson.build b/src/nix/meson.build index 5c70c8216f7..1d4840b1215 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -15,7 +15,7 @@ project('nix', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') nix_store = dependency('nix-store') @@ -30,10 +30,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') configdata = configuration_data() @@ -63,8 +63,8 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/generate-header') nix_sources = [config_h] + files( 'add-to-store.cc', diff --git a/src/nix/nix-meson-build-support b/src/nix/nix-meson-build-support new file mode 120000 index 00000000000..0b140f56bde --- /dev/null +++ b/src/nix/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix index c7b24efce72..9bc139c3b35 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -22,8 +22,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions ([ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build From 1421420e862434321c46511a3152016e443dd479 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:18:10 +0100 Subject: [PATCH 49/93] test: Move unusual-logging to run only in logging test case --- tests/functional/dependencies.nix | 15 --------------- tests/functional/logging.sh | 3 +++ tests/functional/logging/unusual-logging.nix | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 tests/functional/logging/unusual-logging.nix diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index d177f5b6358..4ff29227fd3 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -33,20 +33,6 @@ let outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - unusual-logging = mkDerivation { - name = "unusual-logging"; - buildCommand = '' - { - echo "@nix 1" - echo "@nix {}" - echo '@nix {"action": null}' - echo '@nix {"action": 123}' - echo '@nix ][' - } >&$NIX_LOG_FD - touch $out - ''; - }; - in mkDerivation { name = "dependencies-top"; @@ -56,7 +42,6 @@ mkDerivation { input1_drv = input1; input2_drv = input2; input0_drv = input0; - unusual_logging_drv = unusual-logging; fod_input_drv = fod_input; meta.description = "Random test package"; } diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index c026ac9c211..bbc1bac96e3 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -28,3 +28,6 @@ outp="$(nix-build -E \ test -d "$outp" nix log "$outp" + +# Build works despite ill-formed structured build log entries. +expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:' diff --git a/tests/functional/logging/unusual-logging.nix b/tests/functional/logging/unusual-logging.nix new file mode 100644 index 00000000000..ddb8aa53010 --- /dev/null +++ b/tests/functional/logging/unusual-logging.nix @@ -0,0 +1,16 @@ +let + inherit (import ../config.nix) mkDerivation; +in +mkDerivation { + name = "unusual-logging"; + buildCommand = '' + { + echo "@nix 1" + echo "@nix {}" + echo '@nix {"action": null}' + echo '@nix {"action": 123}' + echo '@nix ][' + } >&$NIX_LOG_FD + touch $out + ''; +} From f3c722cab24f7a0de8c3573d25e91749f4f16234 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:20:27 +0100 Subject: [PATCH 50/93] Remove redundant warning: prefix from structured build log warning --- src/libutil/logging.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 4c706101651..a5add5565df 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -331,7 +331,7 @@ bool handleJSONLogMessage(nlohmann::json & json, return true; } catch (const nlohmann::json::exception &e) { warn( - "warning: Unable to handle a JSON message from %s: %s", + "Unable to handle a JSON message from %s: %s", Uncolored(source), e.what() ); From c783cd22ac1c693871979c4248d7f7b2b7f6bacd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:29:45 +0100 Subject: [PATCH 51/93] tests: Make unusual logging test conditional on fixed daemon version --- tests/functional/logging.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index bbc1bac96e3..ddb1913adf2 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -29,5 +29,7 @@ test -d "$outp" nix log "$outp" -# Build works despite ill-formed structured build log entries. -expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:' +if isDaemonNewer "2.26"; then + # Build works despite ill-formed structured build log entries. + expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:' +fi From 6d86839a0265cf06a384499d9d9aac0daf31a8ad Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:34:56 +0100 Subject: [PATCH 52/93] Use version correctly in nix-daemon-compat-tests derivation Previously, a version would be appended to the pname which already contained a version. --- packaging/hydra.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 6109f479eae..e4562cd212e 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -18,12 +18,12 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { - pname = - "nix-daemon-compat-tests" - + lib.optionalString + pname = "nix-daemon-compat-tests"; + version = + lib.optionalString (lib.versionAtLeast daemon.version "2.4pre20211005" && lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "-${pkgs.nix.version}-with-daemon-${daemon.version}"; + "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; From 62d3957268399ef20c517801fd8672b863ceb16d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:35:57 +0100 Subject: [PATCH 53/93] Remove dead code from nix-daemon-compat-tests packaging We're not testing against these versions anymore. If we bring that back (I would support that), we could do so in a clean way, by making sure that the packaging we test against has a proper version attribute. --- packaging/hydra.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index e4562cd212e..17ba720321c 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -19,11 +19,7 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { pname = "nix-daemon-compat-tests"; - version = - lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "${pkgs.nix.version}-with-daemon-${daemon.version}"; + version = "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; From 8768239517dd75955979fc43cee0d411b4e04139 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 9 Dec 2024 11:15:53 -0800 Subject: [PATCH 54/93] Install .nix-channels and nix.conf with 644 instead of 664 As far as I can tell, there's no real reason either of these need to be 664. I'm willing to bet they were just a typo that has lasted for 7 years. While this shouldn't change anything, this is, IMHO, more correct, so let's stop perpetuating the wrong mode! --- scripts/install-multi-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a487d459f40..19f787ab578 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -690,7 +690,7 @@ place_channel_configuration() { if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + install -m 0644 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" fi } @@ -964,7 +964,7 @@ $NIX_EXTRA_CONF build-users-group = $NIX_BUILD_GROUP_NAME EOF _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf + install -m 0644 "$SCRATCH/nix.conf" /etc/nix/nix.conf } From 35dd19d7851fea20eebcd3cd7cbb6e64cdc47a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Dec 2024 15:37:17 +0100 Subject: [PATCH 55/93] ensure clang-format is using the same version on all platforms --- maintainers/flake-module.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 1d4e85c8c5b..1a134b91a5e 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -12,6 +12,8 @@ hooks = { clang-format = { enable = true; + # https://github.com/cachix/git-hooks.nix/pull/532 + package = pkgs.llvmPackages_latest.clang-tools; excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' From 9b40618d2e455951b699e2e181e4b93ca97e7b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Dec 2024 16:05:37 +0100 Subject: [PATCH 56/93] tests/nixos: disable documentation to improve eval speed we are not testing any nixos modules, so we don't need to generate documentation. This will give us a bit of speed up. --- tests/nixos/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c5f4a23aa08..ff1220f35d0 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -23,6 +23,9 @@ let nix.checkAllErrors = false; # TODO: decide which packaging stage to use. `nix-cli` is efficient, but not the same as the user-facing `everything.nix` package (`default`). Perhaps a good compromise is `everything.nix` + `noTests` defined above? nix.package = nixpkgsFor.${system}.native.nixComponents.nix-cli; + + # Evaluate VMs faster + documentation.enable = false; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; From 3a9d64b8e354176b41936b2b9e1d3b20fe0290f2 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 7 Dec 2024 20:46:11 +0100 Subject: [PATCH 57/93] fromJSON/fromTOML: throw if string contains null byte --- src/libexpr/eval.cc | 9 +++++++++ src/libexpr/json-to-value.cc | 2 ++ src/libexpr/primops/fromTOML.cc | 14 +++++++++++--- src/libexpr/value.hh | 2 ++ .../eval-fail-fromJSON-keyWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromJSON-keyWithNullByte.nix | 1 + .../eval-fail-fromJSON-valueWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromJSON-valueWithNullByte.nix | 1 + .../eval-fail-fromTOML-keyWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromTOML-keyWithNullByte.nix | 1 + .../eval-fail-fromTOML-valueWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromTOML-valueWithNullByte.nix | 1 + 12 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix create mode 100644 tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix create mode 100644 tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix create mode 100644 tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 05f58957ee3..7b6e89534e3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3178,5 +3178,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } +void forceNoNullByte(std::string_view s) +{ + if (s.find('\0') != s.npos) { + using namespace std::string_view_literals; + auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); + throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + } +} + } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ac56541ab9..17cab7ad5da 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -50,6 +50,7 @@ class JSONSax : nlohmann::json_sax { public: void key(string_t & name, EvalState & state) { + forceNoNullByte(name); attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; @@ -122,6 +123,7 @@ class JSONSax : nlohmann::json_sax { bool string(string_t & val) override { + forceNoNullByte(val); rs->value(state).mkString(val); rs->add(); return true; diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 264046711a6..40442505407 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -28,8 +28,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V auto attrs = state.buildBindings(size); - for(auto & elem : table) + for(auto & elem : table) { + forceNoNullByte(elem.first); visit(attrs.alloc(elem.first), elem.second); + } v.mkAttrs(attrs); } @@ -54,7 +56,11 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V v.mkFloat(toml::get(t)); break;; case toml::value_t::string: - v.mkString(toml::get(t)); + { + auto s = toml::get(t); + forceNoNullByte(s); + v.mkString(s); + } break;; case toml::value_t::local_datetime: case toml::value_t::offset_datetime: @@ -66,7 +72,9 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V attrs.alloc("_type").mkString("timestamp"); std::ostringstream s; s << t; - attrs.alloc("value").mkString(toView(s)); + auto str = toView(s); + forceNoNullByte(str); + attrs.alloc("value").mkString(str); v.mkAttrs(attrs); } else { throw std::runtime_error("Dates and times are not supported"); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d9816148852..88fcae98606 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,4 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); +void forceNoNullByte(std::string_view s); + } diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp new file mode 100644 index 00000000000..a16192c5983 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-keyWithNullByte.nix:1:1: + 1| builtins.fromJSON ''{"a\u0000b": 1}'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix new file mode 100644 index 00000000000..ffaa6a97d22 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"a\u0000b": 1}'' diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp new file mode 100644 index 00000000000..c5c08e6ff5d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-valueWithNullByte.nix:1:1: + 1| builtins.fromJSON ''"a\u0000b"'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix new file mode 100644 index 00000000000..c71ab990ded --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''"a\u0000b"'' diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp new file mode 100644 index 00000000000..dc2180f0bc0 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-keyWithNullByte.nix:1:1: + 1| builtins.fromTOML ''"a\u0000b" = 1'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix new file mode 100644 index 00000000000..b622dc4dcdc --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''"a\u0000b" = 1'' diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp new file mode 100644 index 00000000000..0235692a8ed --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-valueWithNullByte.nix:1:1: + 1| builtins.fromTOML ''k = "a\u0000b"'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix new file mode 100644 index 00000000000..183cab6b3b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''k = "a\u0000b"'' From 41a464c68de54d637f086c9a5f97111feacfa328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 24 Sep 2024 11:18:19 +0200 Subject: [PATCH 58/93] tests/libstore.hh: reformat with clang-format after update --- src/libstore-test-support/tests/libstore.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore-test-support/tests/libstore.hh b/src/libstore-test-support/tests/libstore.hh index 84be52c230b..699ba957ec8 100644 --- a/src/libstore-test-support/tests/libstore.hh +++ b/src/libstore-test-support/tests/libstore.hh @@ -19,12 +19,12 @@ public: protected: LibStoreTest() : store(openStore({ - .variant = - StoreReference::Specified{ - .scheme = "dummy", - }, - .params = {}, - })) + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) { } From b5cdf2e268c7dce5933dbbcc8cae64d170a04b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 10 Dec 2024 05:33:32 +0100 Subject: [PATCH 59/93] mergify: don't use rebase merge method The repository configuration doesn't allow rebase merge method. --- .mergify.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index c545bbe6a24..5b41c5c8ded 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -5,7 +5,6 @@ queue_rules: - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) - check-success=vm_tests - merge_method: rebase batch_size: 5 pull_request_rules: From 6d97d57dc78c156c2dfd1fb830cfbc0152683827 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Dec 2024 16:20:29 +0100 Subject: [PATCH 60/93] Prune unneeded .gitignore entries --- .gitignore | 122 +-------------------------------------- tests/functional/repl.sh | 2 +- 2 files changed, 4 insertions(+), 120 deletions(-) diff --git a/.gitignore b/.gitignore index de1183977b3..337a7c15450 100644 --- a/.gitignore +++ b/.gitignore @@ -1,110 +1,12 @@ -Makefile.config -perl/Makefile.config - -# / -/aclocal.m4 -/autom4te.cache -/precompiled-headers.h.gch -/config.* -/configure -/stamp-h1 -/svn-revision -/libtool -/config/config.* # Default meson build dir /build -# /doc/manual/ -/doc/manual/*.1 -/doc/manual/*.5 -/doc/manual/*.8 -/doc/manual/generated/* -/doc/manual/nix.json -/doc/manual/conf-file.json -/doc/manual/language.json -/doc/manual/xp-features.json -/doc/manual/source/SUMMARY.md -/doc/manual/source/SUMMARY-rl-next.md -/doc/manual/source/store/types/* -!/doc/manual/source/store/types/index.md.in -/doc/manual/source/command-ref/new-cli -/doc/manual/source/command-ref/conf-file.md -/doc/manual/source/command-ref/experimental-features-shortlist.md -/doc/manual/source/contributing/experimental-feature-descriptions.md -/doc/manual/source/language/builtins.md -/doc/manual/source/language/builtin-constants.md -/doc/manual/source/release-notes/rl-next.md - -# /scripts/ -/scripts/nix-profile.sh -/scripts/nix-profile-daemon.sh -/scripts/nix-profile.fish -/scripts/nix-profile-daemon.fish - -# /src/libexpr/ -/src/libexpr/lexer-tab.cc -/src/libexpr/lexer-tab.hh -/src/libexpr/parser-tab.cc -/src/libexpr/parser-tab.hh -/src/libexpr/parser-tab.output -/src/libexpr/nix.tbl -/src/libexpr/tests -/src/libexpr-tests/libnixexpr-tests - -# /src/libfetchers -/src/libfetchers-tests/libnixfetchers-tests - -# /src/libflake -/src/libflake-tests/libnixflake-tests - -# /src/libstore/ -*.gen.* -/src/libstore/tests -/src/libstore-tests/libnixstore-tests - -# /src/libutil/ -/src/libutil/tests -/src/libutil-tests/libnixutil-tests - -/src/nix/nix - -/src/nix/generated-doc - -# /src/nix-env/ -/src/nix-env/nix-env - -# /src/nix-instantiate/ -/src/nix-instantiate/nix-instantiate - -# /src/nix-store/ -/src/nix-store/nix-store - -/src/nix-prefetch-url/nix-prefetch-url - -/src/nix-collect-garbage/nix-collect-garbage - -# /src/nix-channel/ -/src/nix-channel/nix-channel - -# /src/nix-build/ -/src/nix-build/nix-build - -/src/nix-copy-closure/nix-copy-closure - -/src/error-demo/error-demo - -/src/build-remote/build-remote - # /tests/functional/ -/tests/functional/test-tmp /tests/functional/common/subst-vars.sh -/tests/functional/result* /tests/functional/restricted-innocent -/tests/functional/shell -/tests/functional/shell.drv -/tests/functional/repl-result-out /tests/functional/debugger-test-out /tests/functional/test-libstoreconsumer/test-libstoreconsumer +/tests/functional/nix-shell # /tests/functional/lang/ /tests/functional/lang/*.out @@ -112,27 +14,9 @@ perl/Makefile.config /tests/functional/lang/*.err /tests/functional/lang/*.ast -/perl/lib/Nix/Config.pm -/perl/lib/Nix/Store.cc - -/misc/systemd/nix-daemon.service -/misc/systemd/nix-daemon.socket -/misc/systemd/nix-daemon.conf -/misc/upstart/nix-daemon.conf - outputs/ -*.a -*.o -*.o.tmp -*.so -*.dylib -*.dll -*.exe -*.dep *~ -*.pc -*.plist # GNU Global GPATH @@ -147,8 +31,6 @@ GTAGS compile_commands.json *.compile_commands.json -nix-rust/target - result result-* @@ -163,3 +45,5 @@ result-* # Mac OS .DS_Store + +flake-regressions diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 706e0f5dba9..59d1f1be02a 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -309,7 +309,7 @@ runRepl () { for test in $(cd "$testDir/repl"; echo *.in); do test="$(basename "$test" .in)" in="$testDir/repl/$test.in" - actual="$testDir/repl/$test.actual" + actual="$TEST_ROOT/$test.actual" expected="$testDir/repl/$test.expected" (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 From 691b1ea237427845f073fbc790f960023bacdfe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 09:44:36 +0100 Subject: [PATCH 61/93] speed up ci by splitting off eval and build - This speeds up macOS builds from 30 minutes to 11 minutes (3x faster). - Also improve error reporting e.g. printing out what actually failed to build. - As a result we also no longer need swap. --- .github/workflows/ci.yml | 28 +++++++++------------------- scripts/build-checks | 6 ++++++ 2 files changed, 15 insertions(+), 19 deletions(-) create mode 100755 scripts/build-checks diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be96bb48469..5fb672769f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,14 @@ on: permissions: read-all jobs: + eval: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v30 + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: needs: [check_secrets] @@ -33,25 +41,7 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - if: matrix.os == 'ubuntu-latest' - run: | - free -h - swapon --show - swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') - echo "Found swap: $swap" - sudo swapoff $swap - # resize it (fallocate) - sudo fallocate -l 10G $swap - sudo mkswap $swap - sudo swapon $swap - free -h - ( - while sleep 60; do - free -h - done - ) & - - run: nix --experimental-features 'nix-command flakes' flake check -L - - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json + - run: scripts/build-checks # Steps to test CI automation in your own fork. # Cachix: diff --git a/scripts/build-checks b/scripts/build-checks new file mode 100755 index 00000000000..e0ee70631b5 --- /dev/null +++ b/scripts/build-checks @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +system=$(nix eval --raw --impure --expr builtins.currentSystem) +nix eval --json ".#checks.$system" --apply builtins.attrNames | \ + jq -r '.[]' | \ + xargs -P0 -I '{}' sh -c "nix build -L .#checks.$system.{} || { echo 'FAILED: \033[0;31mnix build -L .#checks.$system.{}\\033[0m'; kill 0; }" From 31b3a34e5bb761a9eb84454bbb4d45261f0247ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 10:47:41 +0100 Subject: [PATCH 62/93] binary-tarball: create tarball without listing files This is producing quite a bit of log output in CI. --- scripts/binary-tarball.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 671c8e96e38..9de90b7fb56 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -65,7 +65,7 @@ runCommand "nix-binary-tarball-${version}" env '' fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ + tar cfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --mtime='1970-01-01' \ --absolute-names \ From 9f2b9256b6107a2db1f77c6eb7576c7d155e9644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 12:05:49 +0100 Subject: [PATCH 63/93] ci: disable apparmor restrictions For our tests we need to map the root user for some tests. However ubuntu no longer allows this by default: https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fb672769f2..3af1e85249f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,10 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: + # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces + - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + if: matrix.os == 'ubuntu-latest' - run: scripts/build-checks # Steps to test CI automation in your own fork. From 4f831e2be5ec5ec82b42c838522ff3d285b88cb0 Mon Sep 17 00:00:00 2001 From: Greg Curtis Date: Thu, 12 Dec 2024 16:29:17 -0500 Subject: [PATCH 64/93] Fix `nix upgrade-nix` profile search Commit cfe66dbec updated `nix upgrade-nix` to use `ExecutablePath::load().find`, which broke the logic for finding the profile associated with the nix executable. The error looks something like: ``` $ sudo -i nix upgrade-nix --debug found Nix in '"/nix/store/46p1z0w9ad605kky62dr53z4h24k2a5r-nix-2.25.2/bin/nix"' found profile '/nix/store/46p1z0w9ad605kky62dr53z4h24k2a5r-nix-2.25.2/bin' error: directory '"/nix/store/46p1z0w9ad605kky62dr53z4h24k2a5r-nix-2.25.2/bin/nix"' does not appear to be part of a Nix profile ``` This seems to happen for two reasons: 1. The original PATH search resulted in a directory, but `find` returns the path to the executable. Fixed by getting the path's parent. 2. The profile symlink cannot be found because `ExecutablePath::load().find` canonicalizes the executable path. I updated find to normalize the path instead, which seems more in line with how other programs resolve paths. I'm not sure if this affects other callers though. I manually tested this on macOS and Linux, and it seemed to fix upgrading from 2.25.2 to 2.25.3. --- src/libutil/executable-path.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ebd522a415c..8d665c7df95 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -73,7 +73,7 @@ ExecutablePath::findName(const OsString & exe, std::functionparent_path(); printInfo("found Nix in '%s'", where); From 366611391ec749ff02545ab147a9d4f2792490aa Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Fri, 6 Dec 2024 15:54:47 +0100 Subject: [PATCH 65/93] Implement shellSplitString for proper handling of NIX_SSHOPTS with spaces and quotes --- doc/manual/rl-next/nix-sshopts-parsing.md | 21 +++++ src/libstore/ssh.cc | 13 ++- src/libutil-tests/strings.cc | 107 +++++++++++++++++++++- src/libutil/strings.cc | 104 +++++++++++++++++++++ src/libutil/strings.hh | 7 ++ 5 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 doc/manual/rl-next/nix-sshopts-parsing.md diff --git a/doc/manual/rl-next/nix-sshopts-parsing.md b/doc/manual/rl-next/nix-sshopts-parsing.md new file mode 100644 index 00000000000..65fe6f5625a --- /dev/null +++ b/doc/manual/rl-next/nix-sshopts-parsing.md @@ -0,0 +1,21 @@ +--- +synopsis: "Improved `NIX_SSHOPTS` parsing for better SSH option handling" +issues: [5181] +prs: [12020] +--- + +The parsing of the `NIX_SSHOPTS` environment variable has been improved to handle spaces and quotes correctly. +Previously, incorrectly split SSH options could cause failures in CLIs like `nix-copy-closure`, +especially when using complex ssh invocations such as `-o ProxyCommand="ssh -W %h:%p ..."`. + +This change introduces a `shellSplitString` function to ensure +that `NIX_SSHOPTS` is parsed in a manner consistent with shell +behavior, addressing common parsing errors. + +For example, the following now works as expected: + +```bash +export NIX_SSHOPTS='-o ProxyCommand="ssh -W %h:%p ..."' +``` + +This update improves the reliability of SSH-related operations using `NIX_SSHOPTS` across Nix CLIs. diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index dec733fd516..116a480bacc 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -41,8 +41,17 @@ void SSHMaster::addCommonSSHOpts(Strings & args) { auto state(state_.lock()); - for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) - args.push_back(i); + std::string sshOpts = getEnv("NIX_SSHOPTS").value_or(""); + + try { + std::list opts = shellSplitString(sshOpts); + for (auto & i : opts) + args.push_back(i); + } catch (Error & e) { + e.addTrace({}, "while splitting NIX_SSHOPTS '%s'", sshOpts); + throw; + } + if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); if (!sshPublicHostKey.empty()) { diff --git a/src/libutil-tests/strings.cc b/src/libutil-tests/strings.cc index 8ceb1676760..206890bcf19 100644 --- a/src/libutil-tests/strings.cc +++ b/src/libutil-tests/strings.cc @@ -2,11 +2,10 @@ #include #include "strings.hh" +#include "error.hh" namespace nix { -using Strings = std::vector; - /* ---------------------------------------------------------------------------- * concatStringsSep * --------------------------------------------------------------------------*/ @@ -345,4 +344,108 @@ RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); } +/* ---------------------------------------------------------------------------- + * shellSplitString + * --------------------------------------------------------------------------*/ + +TEST(shellSplitString, empty) +{ + std::list expected = {}; + + ASSERT_EQ(shellSplitString(""), expected); +} + +TEST(shellSplitString, oneWord) +{ + std::list expected = {"foo"}; + + ASSERT_EQ(shellSplitString("foo"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpaces) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("'foo bar'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpacesAndDoubleQuoteInSingleQuote) +{ + std::list expected = {"foo bar\""}; + + ASSERT_EQ(shellSplitString("'foo bar\"'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithDoubleQuotes) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("\"foo bar\""), expected); +} + +TEST(shellSplitString, twoWords) +{ + std::list expected = {"foo", "bar"}; + + ASSERT_EQ(shellSplitString("foo bar"), expected); +} + +TEST(shellSplitString, twoWordsWithSpacesAndQuotesQuoted) +{ + std::list expected = {"foo bar'", "baz\""}; + + ASSERT_EQ(shellSplitString("\"foo bar'\" 'baz\"'"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedSingleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo '' bar baz ''"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedDoubleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo \"\" bar baz \"\""), expected); +} + +TEST(shellSplitString, singleQuoteDoesNotUseEscapes) +{ + std::list expected = {"foo\\\"bar"}; + + ASSERT_EQ(shellSplitString("'foo\\\"bar'"), expected); +} + +TEST(shellSplitString, doubleQuoteDoesUseEscapes) +{ + std::list expected = {"foo\"bar"}; + + ASSERT_EQ(shellSplitString("\"foo\\\"bar\""), expected); +} + +TEST(shellSplitString, backslashEscapesSpaces) +{ + std::list expected = {"foo bar", "baz", "qux quux"}; + + ASSERT_EQ(shellSplitString("foo\\ bar baz qux\\ quux"), expected); +} + +TEST(shellSplitString, backslashEscapesQuotes) +{ + std::list expected = {"foo\"bar", "baz", "qux'quux"}; + + ASSERT_EQ(shellSplitString("foo\\\"bar baz qux\\'quux"), expected); +} + +TEST(shellSplitString, testUnbalancedQuotes) +{ + ASSERT_THROW(shellSplitString("foo'"), Error); + ASSERT_THROW(shellSplitString("foo\""), Error); + ASSERT_THROW(shellSplitString("foo'bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar\\\""), Error); +} + } // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index c221a43c6f1..402b7ae98a3 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -4,6 +4,7 @@ #include "strings-inline.hh" #include "os-string.hh" +#include "error.hh" namespace nix { @@ -48,4 +49,107 @@ template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const s template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + * + * Read https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html for the + * POSIX shell specification, which is technically what we are implementing here. + */ +std::list shellSplitString(std::string_view s) +{ + std::list result; + std::string current; + bool startedCurrent = false; + bool escaping = false; + + auto pushCurrent = [&]() { + if (startedCurrent) { + result.push_back(current); + current.clear(); + startedCurrent = false; + } + }; + + auto pushChar = [&](char c) { + current.push_back(c); + startedCurrent = true; + }; + + auto pop = [&]() { + auto c = s[0]; + s.remove_prefix(1); + return c; + }; + + auto inDoubleQuotes = [&]() { + startedCurrent = true; + // in double quotes, escaping with backslash is only effective for $, `, ", and backslash + while (!s.empty()) { + auto c = pop(); + if (escaping) { + switch (c) { + case '$': + case '`': + case '"': + case '\\': + pushChar(c); + break; + default: + pushChar('\\'); + pushChar(c); + break; + } + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == '"') { + return; + } else { + pushChar(c); + } + } + if (s.empty()) { + throw Error("unterminated double quote"); + } + }; + + auto inSingleQuotes = [&]() { + startedCurrent = true; + while (!s.empty()) { + auto c = pop(); + if (c == '\'') { + return; + } + pushChar(c); + } + if (s.empty()) { + throw Error("unterminated single quote"); + } + }; + + while (!s.empty()) { + auto c = pop(); + if (escaping) { + pushChar(c); + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == ' ' || c == '\t') { + pushCurrent(); + } else if (c == '"') { + inDoubleQuotes(); + } else if (c == '\'') { + inSingleQuotes(); + } else { + pushChar(c); + } + } + + pushCurrent(); + + return result; +} } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 533126be1e0..c4fd3daa194 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -71,4 +71,11 @@ extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + */ +std::list shellSplitString(std::string_view s); } From 7d1c41f74cc02c6a941c690b655d3d5338074bc5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 13:09:03 +0100 Subject: [PATCH 66/93] C API: typo --- src/libutil-c/nix_api_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 43f9fa9dc63..5f42641d426 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -47,7 +47,7 @@ extern "C" { */ // Error codes /** - * @brief Type for error codes in the NIX system + * @brief Type for error codes in the Nix system * * This type can have one of several predefined constants: * - NIX_OK: No error occurred (0) From 80ee736b02bb3f400e656f65e3beb1b80a932cf7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 13:05:46 +0100 Subject: [PATCH 67/93] C API: document nix_store_open NULL URI tl;dr NULL is better than "auto" --- src/libstore-c/nix_api_store.h | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 282ccc28514..60a28755983 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -48,12 +48,27 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context); * Store instances may share state and resources behind the scenes. * * @param[out] context Optional, stores error information - * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference + * + * @param[in] uri @parblock + * URI of the Nix store, copied. + * + * If `NULL`, the store from the settings will be used. + * Note that `"auto"` holds a strange middle ground, reading part of the general environment, but not all of it. It + * ignores `NIX_REMOTE` and the `store` option. For this reason, `NULL` is most likely the better choice. + * + * For supported store URLs, see [*Store URL format* in the Nix Reference * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format). - * @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint", - * "https://s3.local"}}. See [*Store Types* in the Nix Reference - * Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * + * @param[in] params @parblock + * optional, null-terminated array of key-value pairs, e.g. {{"endpoint", + * "https://s3.local"}}. + * + * See [*Store Types* in the Nix Reference Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * * @return a Store pointer, NULL in case of errors + * * @see nix_store_free */ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params); From 472912f7ca9dbf90fdee18bc0a9b0994974c6660 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 12 Dec 2024 17:43:01 +0100 Subject: [PATCH 68/93] C API: Add nix_store_get_storedir --- src/libstore-c/nix_api_store.cc | 11 ++++++++++ src/libstore-c/nix_api_store.h | 13 +++++++++++- src/libstore-tests/meson.build | 5 ++++- src/libstore-tests/nix_api_store.cc | 33 +++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fb7391276c4..6e5dd559f7c 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -67,6 +67,17 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string NIXC_CATCH_ERRS } +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return call_nix_get_string_callback(store->ptr->storeDir, callback, user_data); + } + NIXC_CATCH_ERRS +} + nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) { diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 60a28755983..d421ee4d317 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -93,7 +93,18 @@ void nix_store_free(Store * store); */ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); -// returns: owned StorePath* +/** + * @brief get the storeDir of a Nix store, typically `"/nix/store"` + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the URI. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + /** * @brief Parse a Nix store path into a StorePath * diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index c4e42634d8d..b706fa12c2f 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -16,8 +16,10 @@ cxx = meson.get_compiler('cpp') subdir('nix-meson-build-support/deps-lists') +nix_store = dependency('nix-store') + deps_private_maybe_subproject = [ - dependency('nix-store'), + nix_store, dependency('nix-store-c'), dependency('nix-store-test-support'), ] @@ -90,6 +92,7 @@ this_exe = executable( include_directories : include_dirs, # TODO: -lrapidcheck, see ../libutil-support/build.meson link_args: linker_export_flags + ['-lrapidcheck'], + cpp_args : [ '-DNIX_STORE_DIR="' + nix_store.get_variable('storedir') + '"' ], # get main from gtest install : true, ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 7c6ec078057..d47cd1dfd6b 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -24,6 +24,39 @@ TEST_F(nix_api_store_test, nix_store_get_uri) ASSERT_STREQ("local", str.c_str()); } +TEST_F(nix_api_util_context, nix_store_get_storedir_default) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles + GTEST_SKIP(); + } + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, nullptr, nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(NIX_STORE_DIR, str.c_str()); + + nix_store_free(store); +} + +TEST_F(nix_api_store_test, nix_store_get_storedir) +{ + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(nixStoreDir.c_str(), str.c_str()); +} + TEST_F(nix_api_store_test, InvalidPathFails) { nix_store_parse_path(ctx, store, "invalid-path"); From 2a981689423a531f5462f51aea17774ed4f17d0d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 12 Dec 2024 18:51:11 +0100 Subject: [PATCH 69/93] C API: Add nix_store_real_path --- src/libstore-c/nix_api_store.cc | 12 +++++ src/libstore-c/nix_api_store.h | 20 ++++++++ src/libstore-tests/nix_api_store.cc | 80 +++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 6e5dd559f7c..bc306e0d0ad 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -100,6 +100,18 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * NIXC_CATCH_ERRS_RES(false); } +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->toRealPath(path->path); + return call_nix_get_string_callback(res, callback, user_data); + } + NIXC_CATCH_ERRS +} + StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index d421ee4d317..e55bc3f59ff 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -149,6 +149,26 @@ void nix_store_path_free(StorePath * p); * @return true or false, error info in context */ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); + +/** + * @brief Get the physical location of a store path + * + * A store may reside at a different location than its `storeDir` suggests. + * This situation is called a relocated store. + * Relocated stores are used during NixOS installation, as well as in restricted computing environments that don't offer + * a writable `/nix/store`. + * + * Not all types of stores support this operation. + * + * @param[in] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] path the path to get the real path from + * @param[in] callback called with the real path + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data); + // nix_err nix_store_ensure(Store*, const char*); // nix_err nix_store_build_paths(Store*); /** diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index d47cd1dfd6b..a8b7b8e5fc8 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -119,4 +119,84 @@ TEST_F(nix_api_store_test, nix_store_is_valid_path_not_in_store) ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, path)); } +TEST_F(nix_api_store_test, nix_store_real_path) +{ + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((nixStoreDir + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); +} + +TEST_F(nix_api_util_context, nix_store_real_path_relocated) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // Can't open default store from within sandbox + GTEST_SKIP(); + } + auto tmp = nix::createTempDir(); + std::string storeRoot = tmp + "/store"; + std::string stateDir = tmp + "/state"; + std::string logDir = tmp + "/log"; + const char * rootkv[] = {"root", storeRoot.c_str()}; + const char * statekv[] = {"state", stateDir.c_str()}; + const char * logkv[] = {"log", logDir.c_str()}; + // const char * rokv[] = {"read-only", "true"}; + const char ** kvs[] = {rootkv, statekv, logkv, NULL}; + + nix_libstore_init(ctx); + assert_ctx_ok(); + + Store * store = nix_store_open(ctx, "local", kvs); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string nixStoreDir; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(nixStoreDir)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(NIX_STORE_DIR, nixStoreDir.c_str()); + + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((storeRoot + NIX_STORE_DIR + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); } + +TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // TODO: override NIX_CACHE_HOME? + // skipping test in sandbox because narinfo cache can't be written + GTEST_SKIP(); + } + + Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string path_raw = std::string(NIX_STORE_DIR) + PATH_SUFFIX; + StorePath * path = nix_store_parse_path(ctx, store, path_raw.c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(path_raw.c_str(), rp.c_str()); +} + +} // namespace nixC From da7f7ba810e8fbcdc28bca490b5a5c618b1cef2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 12:55:45 +0100 Subject: [PATCH 70/93] functional-tests: skip tests if the kernel restricts unprivileged user namespaces Update tests/functional/common/functions.sh Co-authored-by: Robert Hensing --- .../build-remote-trustless-should-fail-0.sh | 1 + tests/functional/build-remote-trustless.sh | 1 + tests/functional/build-remote.sh | 2 ++ tests/functional/chroot-store.sh | 1 + tests/functional/common/functions.sh | 11 +++++++++++ tests/functional/linux-sandbox.sh | 1 + tests/functional/local-overlay-store/bad-uris.sh | 2 +- tests/functional/local-overlay-store/common.sh | 4 ---- tests/functional/nested-sandboxing.sh | 1 + tests/functional/nested-sandboxing/command.sh | 1 + tests/functional/shell.sh | 1 + tests/functional/supplementary-groups.sh | 2 +- 12 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 4eccb73e0ee..3401de1b073 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -8,6 +8,7 @@ TODO_NixOS restartDaemon requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh index c498d46c301..9f91a91a969 100644 --- a/tests/functional/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -5,6 +5,7 @@ # shellcheck disable=SC2154 requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "$busybox" =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index 1a53345778d..3231341cbf6 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -3,6 +3,7 @@ : "${file?must be defined by caller (remote building test case using this)}" requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "${busybox-}" =~ busybox ]] || skipTest "no busybox" # Avoid store dir being inside sandbox build-dir @@ -27,6 +28,7 @@ builders=( chmod -R +w "$TEST_ROOT/machine"* || true rm -rf "$TEST_ROOT/machine"* || true + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 46e91f0aaea..ccde3e90bb5 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -40,6 +40,7 @@ EOF cp simple.nix shell.nix simple.builder.sh "${config_nix}" "$flakeDir/" TODO_NixOS + requiresUnprivilegedUserNamespaces outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir") diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 7195149cbfc..bf3dd2ca861 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -345,4 +345,15 @@ count() { trap onError ERR +requiresUnprivilegedUserNamespaces() { + if [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; then + skipTest "Unprivileged user namespaces are disabled. Run 'sudo sysctl -w /proc/sys/kernel/apparmor_restrict_unprivileged_userns=0' to allow, and run these tests." + fi +} + +execUnshare () { + requiresUnprivilegedUserNamespaces + exec unshare --mount --map-root-user "$SHELL" "$@" +} + fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 81ef3623796..abb635f1195 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -9,6 +9,7 @@ TODO_NixOS clearStore requireSandboxSupport +requiresUnprivilegedUserNamespaces # Note: we need to bind-mount $SHELL into the chroot. Currently we # only support the case where $SHELL is in the Nix store, because diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index b7930e32e24..f0c6a151c35 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -19,7 +19,7 @@ TODO_NixOS for i in "${storesBad[@]}"; do echo $i - unshare --mount --map-root-user bash < Date: Sat, 14 Dec 2024 17:17:29 +0100 Subject: [PATCH 71/93] installer: make sure we can always test the installer in ci and locally Just now there is a dependency on cachix, which means we cannot test the installer in CI if forks do not have the necessary secrets set up. We replace this with a simple http server that serves the installer and can be both used in CI and locally. --- .github/workflows/ci.yml | 119 +++++++------------ flake.nix | 4 +- packaging/hydra.nix | 13 +- scripts/prepare-installer-for-github-actions | 13 +- scripts/serve-installer-for-github-actions | 22 ++++ 5 files changed, 82 insertions(+), 89 deletions(-) create mode 100755 scripts/serve-installer-for-github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3af1e85249f..addafb9f82d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,6 @@ jobs: - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: - needs: [check_secrets] strategy: fail-fast: false matrix: @@ -34,72 +33,21 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 if: matrix.os == 'ubuntu-latest' - run: scripts/build-checks - - # Steps to test CI automation in your own fork. - # Cachix: - # 1. Sign-up for https://www.cachix.org/ - # 2. Create a cache for $githubuser-nix-install-tests - # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN - # Dockerhub: - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check Cachix and Docker secrets present for installer tests - runs-on: ubuntu-latest - outputs: - cachix: ${{ steps.secret.outputs.cachix }} - docker: ${{ steps.secret.outputs.docker }} - steps: - - name: Check for secrets - id: secret - env: - _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - installer: - needs: [tests, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - runs-on: ubuntu-latest - outputs: - installerURL: ${{ steps.prepare-installer.outputs.installerURL }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: cachix/cachix-action@v15 + - run: scripts/prepare-installer-for-github-actions + - name: Upload installer tarball + uses: actions/upload-artifact@v4 with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - cachixArgs: '-v' - - id: prepare-installer - run: scripts/prepare-installer-for-github-actions + name: installer-${{matrix.os}} + path: out/* installer_test: - needs: [installer, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' + needs: [tests] strategy: fail-fast: false matrix: @@ -107,11 +55,18 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - name: Download installer tarball + uses: actions/download-artifact@v4 + with: + name: installer-${{matrix.os}} + path: out + - name: Serving installer + id: serving_installer + run: ./scripts/serve-installer-for-github-actions - uses: cachix/install-nix-action@v30 with: - install_url: '${{needs.installer.outputs.installerURL}}' - install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" + install_url: 'http://localhost:8126/install' + install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-latest' - run: brew install fish @@ -123,32 +78,50 @@ jobs: - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + # Steps to test CI automation in your own fork. + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork + check_secrets: + permissions: + contents: none + name: Check Docker secrets present for installer tests + runs-on: ubuntu-latest + outputs: + docker: ${{ steps.secret.outputs.docker }} + steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" + docker_push_image: - needs: [check_secrets, tests, vm_tests] + needs: [tests, vm_tests, check_secrets] permissions: contents: read packages: write if: >- + needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && - github.ref_name == 'master' && - needs.check_secrets.outputs.cachix == 'true' && - needs.check_secrets.outputs.docker == 'true' + github.ref_name == 'master' runs-on: ubuntu-latest steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION diff --git a/flake.nix b/flake.nix index 794736af4ec..a5e68609eda 100644 --- a/flake.nix +++ b/flake.nix @@ -186,7 +186,7 @@ }; checks = forAllSystems (system: { - binaryTarball = self.hydraJobs.binaryTarball.${system}; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = @@ -237,6 +237,8 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; + binaryTarball = self.hydraJobs.binaryTarball.${system}; # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 17ba720321c..5b1e4755948 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -123,15 +123,10 @@ in self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; + + installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix { + tarballs = [ self.hydraJobs.binaryTarball.${system} ]; + }); # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 4b994a7532c..0fbecf25c2a 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -1,10 +1,11 @@ #!/usr/bin/env bash -set -e +set -euo pipefail -script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo "$script" | cut -b12-43 -) +nix build -L ".#installerScriptForGHA" ".#binaryTarball" -installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install - -echo "::set-output name=installerURL::$installerURL" +mkdir -p out +cp ./result/install "out/install" +name="$(basename "$(realpath ./result-1)")" +# everything before the first dash +cp -r ./result-1 "out/${name%%-*}" diff --git a/scripts/serve-installer-for-github-actions b/scripts/serve-installer-for-github-actions new file mode 100755 index 00000000000..2efd2aa329e --- /dev/null +++ b/scripts/serve-installer-for-github-actions @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [[ ! -d out ]]; then + echo "run prepare-installer-for-github-actions first" + exit 1 +fi +cd out +PORT=${PORT:-8126} +nohup python -m http.server "$PORT" >/dev/null 2>&1 & +pid=$! + +while ! curl -s "http://localhost:$PORT"; do + sleep 1 + if ! kill -0 $pid; then + echo "Failed to start http server" + exit 1 + fi +done + +echo 'To install nix, run the following command:' +echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT" From df3eb4f3f6e80fa3bdcd17f36ec6b0a9156825fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 18:45:03 +0100 Subject: [PATCH 72/93] mergify: updated required tests --- .mergify.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 5b41c5c8ded..70fccae4911 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,6 +4,8 @@ queue_rules: merge_conditions: - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) + - check-success=installer_test (macos-latest) + - check-success=installer_test (ubuntu-latest) - check-success=vm_tests batch_size: 5 From ccaa4c259ad33d5378e0641f9bb891430bf9bbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 21:04:26 +0100 Subject: [PATCH 73/93] upgrade-nix: don't double quote path on error the format error already adds quotes. --- src/nix/upgrade-nix.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a4722abf5f4..77fb798a7c4 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -126,7 +126,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) - throw Error("directory '%s' does not appear to be part of a Nix profile", where); + throw Error("directory %s does not appear to be part of a Nix profile", where); if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); From 7ddf7300b52c8500718ca1fbaca76bab017f6edf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 6 Dec 2024 11:45:26 +0100 Subject: [PATCH 74/93] Ignore local registries for lock file generation When resolving indirect flake references like `nixpkgs` in `flake.nix` files, Nix will no longer use the system and user flake registries. It will only use the global flake registry and overrides given on the command line via `--override-flake`. --- doc/manual/rl-next/ignore-local-registries.md | 22 +++++++++++++++++++ src/libfetchers/registry.cc | 4 +++- src/libfetchers/registry.hh | 9 +++++++- src/libflake/flake/flake.cc | 8 ++++++- src/libflake/flake/flakeref.cc | 5 +++-- src/libflake/flake/flakeref.hh | 5 ++++- src/nix/flake.md | 3 ++- src/nix/registry.md | 2 ++ 8 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 doc/manual/rl-next/ignore-local-registries.md diff --git a/doc/manual/rl-next/ignore-local-registries.md b/doc/manual/rl-next/ignore-local-registries.md new file mode 100644 index 00000000000..8d5e333dd0a --- /dev/null +++ b/doc/manual/rl-next/ignore-local-registries.md @@ -0,0 +1,22 @@ +--- +synopsis: "Flake lock file generation now ignores local registries" +prs: [12019] +--- + +When resolving indirect flake references like `nixpkgs` in `flake.nix` files, Nix will no longer use the system and user flake registries. It will only use the global flake registry and overrides given on the command line via `--override-flake`. + +This avoids accidents where users have local registry overrides that map `nixpkgs` to a `path:` flake in the local file system, which then end up in committed lock files pushed to other users. + +In the future, we may remove the use of the registry during lock file generation altogether. It's better to explicitly specify the URL of a flake input. For example, instead of +```nix +{ + outputs = { self, nixpkgs }: { ... }; +} +``` +write +```nix +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + outputs = { self, nixpkgs }: { ... }; +} +``` diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index c761028ab55..171afcea777 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -178,7 +178,8 @@ Registries getRegistries(const Settings & settings, ref store) std::pair lookupInRegistries( ref store, - const Input & _input) + const Input & _input, + const RegistryFilter & filter) { Attrs extraAttrs; int n = 0; @@ -190,6 +191,7 @@ std::pair lookupInRegistries( if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(*input.settings, store)) { + if (filter && !filter(registry->type)) continue; // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 0d68ac395e9..8f47e15905e 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -65,8 +65,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs); +using RegistryFilter = std::function; + +/** + * Rewrite a flakeref using the registries. If `filter` is set, only + * use the registries for which the filter function returns true. + */ std::pair lookupInRegistries( ref store, - const Input & input); + const Input & input, + const RegistryFilter & filter = {}); } diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 19b622a34af..01cd8db6550 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -54,7 +54,13 @@ static std::tuple fetchOrSubstituteTree( fetched.emplace(originalRef.fetchTree(state.store)); } else { if (allowLookup) { - resolvedRef = originalRef.resolve(state.store); + resolvedRef = originalRef.resolve( + state.store, + [](fetchers::Registry::RegistryType type) { + /* Only use the global registry and CLI flags + to resolve indirect flakerefs. */ + return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; + }); auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); flakeCache.push_back({resolvedRef, *fetchedResolved}); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 9616fe0eaff..ab882fdaba6 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -3,7 +3,6 @@ #include "url.hh" #include "url-parts.hh" #include "fetchers.hh" -#include "registry.hh" namespace nix { @@ -36,7 +35,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -FlakeRef FlakeRef::resolve(ref store) const +FlakeRef FlakeRef::resolve( + ref store, + const fetchers::RegistryFilter & filter) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 1064538a72a..80013e87ea4 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "types.hh" #include "fetchers.hh" #include "outputs-spec.hh" +#include "registry.hh" namespace nix { @@ -57,7 +58,9 @@ struct FlakeRef fetchers::Attrs toAttrs() const; - FlakeRef resolve(ref store) const; + FlakeRef resolve( + ref store, + const fetchers::RegistryFilter & filter = {}) const; static FlakeRef fromAttrs( const fetchers::Settings & fetchSettings, diff --git a/src/nix/flake.md b/src/nix/flake.md index 1e9895f6e44..1028dc8079a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -165,7 +165,8 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `indirect`: *The default*. Indirection through the flake registry. +* `indirect`: *The default*. These are symbolic references to flakes + that are looked up in [the flake registries](./nix3-registry.md). These have the form ``` diff --git a/src/nix/registry.md b/src/nix/registry.md index bd3575d1b5a..d6f8af5e958 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -34,6 +34,8 @@ highest precedence: * Overrides specified on the command line using the option `--override-flake`. +Note that the system and user registries are not used to resolve flake references in `flake.nix`. They are only used to resolve flake references on the command line. + # Registry format A registry is a JSON file with the following format: From d8117c8c0de0a533a42b563ed01e67976368da09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 11:46:02 +0100 Subject: [PATCH 75/93] libexpr: always initialize displacement Otherwise the value is undefined in some cases. --- src/libexpr/nixexpr.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 2950ff1fd7c..a7ad580d2df 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -168,7 +168,7 @@ struct ExprVar : Expr the set stored in the environment that is `level` levels up from the current one.*/ Level level; - Displacement displ; + Displacement displ = 0; ExprVar(Symbol name) : name(name) { }; ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { }; @@ -242,7 +242,7 @@ struct ExprAttrs : Expr Kind kind; Expr * e; PosIdx pos; - Displacement displ; // displacement + Displacement displ = 0; // displacement AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain) : kind(kind), e(e), pos(pos) { }; AttrDef() { }; From d1f20e3510ef39bc36287773cff35383e824ebc0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Dec 2024 14:58:25 +0100 Subject: [PATCH 76/93] Make FetchedFlake a struct --- src/libflake/flake/flake.cc | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index bc62e0bc508..0bdaeb789e1 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -21,7 +21,12 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +struct FetchedFlake +{ + FlakeRef lockedRef; + StorePath storePath; +}; + typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -32,7 +37,7 @@ static std::optional lookupInFlakeCache( for (auto & i : flakeCache) { if (flakeRef == i.first) { debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i.first, i.second.second); + flakeRef, i.first, i.second.lockedRef); return i.second; } } @@ -51,7 +56,8 @@ static std::tuple fetchOrSubstituteTree( if (!fetched) { if (originalRef.input.isDirect()) { - fetched.emplace(originalRef.fetchTree(state.store)); + auto [storePath, lockedRef] = originalRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } else { if (allowLookup) { resolvedRef = originalRef.resolve( @@ -61,10 +67,12 @@ static std::tuple fetchOrSubstituteTree( to resolve indirect flakerefs. */ return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; }); - auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); - if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); - flakeCache.push_back({resolvedRef, *fetchedResolved}); - fetched.emplace(*fetchedResolved); + fetched = lookupInFlakeCache(flakeCache, originalRef); + if (!fetched) { + auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); + } + flakeCache.push_back({resolvedRef, *fetched}); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); @@ -73,16 +81,14 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [storePath, lockedRef] = *fetched; - debug("got tree '%s' from '%s'", - state.store->printStorePath(storePath), lockedRef); + state.store->printStorePath(fetched->storePath), fetched->lockedRef); - state.allowPath(storePath); + state.allowPath(fetched->storePath); - assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(storePath), resolvedRef, lockedRef}; + return {fetched->storePath, resolvedRef, fetched->lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) From d2e1d4916a7a74ba75dbbdff0f696acb0859dbfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Dec 2024 15:30:55 +0100 Subject: [PATCH 77/93] lookupInFlakeCache(): Fix O(n) time lookup --- src/libfetchers/fetchers.hh | 5 +++++ src/libflake/flake/flake.cc | 21 ++++++++------------- src/libflake/flake/flakeref.hh | 2 ++ src/libutil/types.hh | 7 +++---- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b28ec456864..94b2320f919 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -104,6 +104,11 @@ public: bool operator ==(const Input & other) const noexcept; + auto operator <=>(const Input & other) const + { + return attrs <=> other.attrs; + } + bool contains(const Input & other) const; /** diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 0bdaeb789e1..29090b90057 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -27,22 +27,17 @@ struct FetchedFlake StorePath storePath; }; -typedef std::vector> FlakeCache; +typedef std::map FlakeCache; static std::optional lookupInFlakeCache( const FlakeCache & flakeCache, const FlakeRef & flakeRef) { - // FIXME: inefficient. - for (auto & i : flakeCache) { - if (flakeRef == i.first) { - debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i.first, i.second.lockedRef); - return i.second; - } - } - - return std::nullopt; + auto i = flakeCache.find(flakeRef); + if (i == flakeCache.end()) return std::nullopt; + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i->first, i->second.lockedRef); + return i->second; } static std::tuple fetchOrSubstituteTree( @@ -72,13 +67,13 @@ static std::tuple fetchOrSubstituteTree( auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } - flakeCache.push_back({resolvedRef, *fetched}); + flakeCache.insert_or_assign(resolvedRef, *fetched); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); } } - flakeCache.push_back({originalRef, *fetched}); + flakeCache.insert_or_assign(originalRef, *fetched); } debug("got tree '%s' from '%s'", diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 80013e87ea4..2ba01c72b05 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -49,6 +49,8 @@ struct FlakeRef bool operator ==(const FlakeRef & other) const = default; + auto operator <=>(const FlakeRef &) const = default; + FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) { } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 325e3ea7351..f30dd910e64 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -43,10 +43,9 @@ template struct Explicit { T t; - bool operator ==(const Explicit & other) const - { - return t == other.t; - } + bool operator ==(const Explicit & other) const = default; + + auto operator <=>(const Explicit & other) const = default; }; From b167e2c415684518626265abfbdb3b756f04ee1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Dec 2024 20:59:59 +0100 Subject: [PATCH 78/93] Work around clang/libc++ issue --- src/libfetchers/fetchers.hh | 4 ++-- src/libflake/flake/flakeref.hh | 5 ++++- src/libutil/types.hh | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 94b2320f919..ff04d5551ad 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -104,9 +104,9 @@ public: bool operator ==(const Input & other) const noexcept; - auto operator <=>(const Input & other) const + bool operator <(const Input & other) const { - return attrs <=> other.attrs; + return attrs < other.attrs; } bool contains(const Input & other) const; diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 2ba01c72b05..ec755399d78 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -49,7 +49,10 @@ struct FlakeRef bool operator ==(const FlakeRef & other) const = default; - auto operator <=>(const FlakeRef &) const = default; + bool operator <(const FlakeRef & other) const + { + return std::tie(input, subdir) < std::tie(other.input, other.subdir); + } FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index f30dd910e64..9f5c7582784 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -45,7 +45,10 @@ struct Explicit { bool operator ==(const Explicit & other) const = default; - auto operator <=>(const Explicit & other) const = default; + bool operator <(const Explicit & other) const + { + return t < other.t; + } }; From 08361f031d65703aa35cb3b7e7715ff63256e311 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Dec 2024 16:49:48 +0100 Subject: [PATCH 79/93] EvalState::realiseContext(): Allow access to the entire closure Fixes #11030. --- src/libexpr/eval.cc | 15 +++++++++++---- src/libexpr/eval.hh | 5 +++++ src/libexpr/primops.cc | 8 +++----- tests/functional/import-from-derivation.nix | 21 ++++++++++++++++++++- tests/functional/import-from-derivation.sh | 5 +++++ 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7b6e89534e3..fe5f05ab887 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -347,6 +347,16 @@ void EvalState::allowPath(const StorePath & storePath) rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath))); } +void EvalState::allowClosure(const StorePath & storePath) +{ + if (!rootFS.dynamic_pointer_cast()) return; + + StorePathSet closure; + store->computeFSClosure(storePath, closure); + for (auto & p : closure) + allowPath(p); +} + void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) { allowPath(storePath); @@ -3099,10 +3109,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat allowPath(path.path.abs()); if (store->isInStore(path.path.abs())) { try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); - for (auto & p : closure) - allowPath(p); + allowClosure(store->toStorePath(path.path.abs()).first); } catch (InvalidPath &) { } } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3ac3c8a8a6f..84b7d823c36 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -400,6 +400,11 @@ public: */ void allowPath(const StorePath & storePath); + /** + * Allow access to the closure of a store path. + */ + void allowClosure(const StorePath & storePath); + /** * Allow access to a store path and return it as a string. */ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7c5c6ea9b85..c1f1cf4c18a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -119,11 +119,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); if (isIFD) { - for (auto & outputPath : outputsToCopyAndAllow) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(outputPath); - } + /* Allow access to the output closures of this derivation. */ + for (auto & outputPath : outputsToCopyAndAllow) + allowClosure(outputPath); } return res; diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix index cc53451cff3..770dd86cf73 100644 --- a/tests/functional/import-from-derivation.nix +++ b/tests/functional/import-from-derivation.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import ; rec { bar = mkDerivation { @@ -30,4 +30,23 @@ rec { echo -n BLA$(cat $src) > $out ''; }; + + step1 = mkDerivation { + name = "step1"; + buildCommand = '' + mkdir -p $out + echo 'foo' > $out/bla + ''; + }; + + addPathExpr = mkDerivation { + name = "add-path"; + inherit step1; + buildCommand = '' + mkdir -p $out + echo "builtins.path { path = \"$step1\"; sha256 = \"7ptL+pnrZXnSa5hwwB+2SXTLkcSb5264WGGokN8OXto=\"; }" > $out/default.nix + ''; + }; + + importAddPathExpr = import addPathExpr; } diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index 83ef92a6f58..a007612350c 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -6,6 +6,8 @@ TODO_NixOS clearStoreIfPossible +export NIX_PATH=config="${config_nix}" + if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then echo "read-only evaluation of an imported derivation unexpectedly failed" exit 1 @@ -15,6 +17,9 @@ outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) [ "$(cat "$outPath")" = FOO579 ] +# Check that we can have access to the entire closure of a derivation output. +nix build --no-link --restrict-eval -I src=. -f ./import-from-derivation.nix importAddPathExpr -v + # FIXME: the next tests are broken on CA. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then exit 0 From 51463d228010360e91366e2fc06f8d54ccfc9288 Mon Sep 17 00:00:00 2001 From: Jason Yundt Date: Mon, 16 Dec 2024 13:55:41 -0500 Subject: [PATCH 80/93] Fix flake.lock/flake.nix mix-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/nix/flake.md describes the format of flake.lock files. Before this change, it said that the original field was “The original input specification from `flake.lock`[…]” The original input specification is in flake.nix, not flake.lock. --- src/nix/flake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 1028dc8079a..fa17375df6e 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -654,7 +654,7 @@ following fields: * `inputs`: The dependencies of this node, as a mapping from input names (e.g. `nixpkgs`) to node labels (e.g. `n2`). -* `original`: The original input specification from `flake.lock`, as a +* `original`: The original input specification from `flake.nix`, as a set of `builtins.fetchTree` arguments. * `locked`: The locked input specification, as a set of From 331bf3e2613d2b1f730f3867dd32bd0dc1ca43a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Nov 2024 16:55:27 +0100 Subject: [PATCH 81/93] Git fetcher: Calculate a fingerprint for dirty workdirs This restores evaluation caching for dirty Git workdirs. --- src/libfetchers/git-utils.cc | 18 ++++++++++++++++-- src/libfetchers/git-utils.hh | 7 ++++++- src/libfetchers/git.cc | 30 +++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 74e68fe1281..bd578685763 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -437,7 +437,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && !(statusFlags & GIT_STATUS_WT_DELETED)) - info.files.insert(CanonPath(path)); + info.files.emplace(CanonPath(path), + statusFlags == GIT_STATUS_CURRENT + ? WorkdirInfo::State::Clean + : WorkdirInfo::State::Dirty); + else + info.deletedFiles.insert(CanonPath(path)); if (statusFlags != GIT_STATUS_CURRENT) info.isDirty = true; return 0; @@ -1202,6 +1207,15 @@ ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore } } +template +std::set getKeys(const std::map & c) +{ + std::set res; + for (auto & i : c) + res.insert(i.first); + return res; +} + ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); @@ -1214,7 +1228,7 @@ ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool export ? makeEmptySourceAccessor() : AllowListSourceAccessor::create( makeFSSourceAccessor(path), - std::set { wd.files }, + std::set { getKeys(wd.files) }, std::move(makeNotAllowedError)).cast(); if (exportIgnore) return make_ref(self, fileAccessor, std::nullopt); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index f45b5a50425..12cee5db107 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -55,9 +55,14 @@ struct GitRepo in the repo yet. */ std::optional headRev; + enum State { Clean, Dirty }; + /* All files in the working directory that are unchanged, modified or added, but excluding deleted files. */ - std::set files; + std::map files; + + /* The deleted files. */ + std::set deletedFiles; /* The submodules listed in .gitmodules of this workdir. */ std::vector submodules; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a6883a2d355..eec134980c3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -685,7 +685,7 @@ struct GitInputScheme : InputScheme if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) - repoInfo.workdirInfo.files.insert(submodule.path); + repoInfo.workdirInfo.files.emplace(submodule.path, GitRepo::WorkdirInfo::State::Clean); auto repo = GitRepo::openRepo(repoInfo.url, false, false); @@ -793,10 +793,34 @@ struct GitInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { + auto makeFingerprint = [&](const Hash & rev) + { + return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); + }; + if (auto rev = input.getRev()) - return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); - else + return makeFingerprint(*rev); + else { + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal && repoInfo.workdirInfo.headRev) { + /* Calculate a fingerprint that takes into account the + deleted and modified/added files. */ + HashSink hashSink{HashAlgorithm::SHA512}; + for (auto & file : repoInfo.workdirInfo.files) + if (file.second == GitRepo::WorkdirInfo::State::Dirty) { + writeString("modified:", hashSink); + writeString(file.first.abs(), hashSink); + readFile(std::filesystem::path(repoInfo.url) + file.first.abs(), hashSink); + } + for (auto & file : repoInfo.workdirInfo.deletedFiles) { + writeString("deleted:", hashSink); + writeString(file.abs(), hashSink); + } + return makeFingerprint(*repoInfo.workdirInfo.headRev) + + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); + } return std::nullopt; + } } bool isLocked(const Input & input) const override From d044a05197c1d4066ef0c0b67ff0461ee3d5fa6c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Dec 2024 15:50:39 +0100 Subject: [PATCH 82/93] Don't fingerprint dirty repos with submodules for now Fixes nixpkgsLibTests. --- src/libfetchers/git.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index eec134980c3..d47e731f14e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "finally.hh" #include "fetch-settings.hh" #include "json-utils.hh" +#include "archive.hh" #include #include @@ -802,7 +803,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (repoInfo.isLocal && repoInfo.workdirInfo.headRev) { + if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -810,7 +811,7 @@ struct GitInputScheme : InputScheme if (file.second == GitRepo::WorkdirInfo::State::Dirty) { writeString("modified:", hashSink); writeString(file.first.abs(), hashSink); - readFile(std::filesystem::path(repoInfo.url) + file.first.abs(), hashSink); + dumpPath(repoInfo.url + "/" + file.first.abs(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); From f469bc2ae4a221d0397775b2663ba90a2dff0b9c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 13:17:31 +0100 Subject: [PATCH 83/93] Cache result of Input::getFingerprint() The fingerprint calculation can be expensive (especially for dirty Git trees) so we need to cache it. --- src/libfetchers/fetchers.cc | 14 +++++++++++--- src/libfetchers/fetchers.hh | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e15a460d0e4..b105c252a30 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -113,7 +113,15 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) std::optional Input::getFingerprint(ref store) const { - return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; + if (!scheme) return std::nullopt; + + if (cachedFingerprint) return *cachedFingerprint; + + auto fingerprint = scheme->getFingerprint(store, *this); + + cachedFingerprint = fingerprint; + + return fingerprint; } ParsedURL Input::toURL() const @@ -307,7 +315,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto accessor = makeStorePathAccessor(store, storePath); - accessor->fingerprint = scheme->getFingerprint(store, *this); + accessor->fingerprint = getFingerprint(store); return {accessor, *this}; } catch (Error & e) { @@ -318,7 +326,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); assert(!accessor->fingerprint); - accessor->fingerprint = scheme->getFingerprint(store, result); + accessor->fingerprint = result.getFingerprint(store); return {accessor, std::move(result)}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ff04d5551ad..841a4404163 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -46,6 +46,11 @@ struct Input */ std::optional parent; + /** + * Cached result of getFingerprint(). + */ + mutable std::optional> cachedFingerprint; + public: /** * Create an `Input` from a URL. From 7ba933e989b9baf56c2b542c7788b38f6d9ccb50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 13:56:03 +0100 Subject: [PATCH 84/93] Cache calls to GitRepo::getWorkdirInfo() A command like `nix flake metadata` was causing about 4 calls to getWorkdirInfo(), which is slow for large repos (even when they're not dirty). --- src/libfetchers/git-utils.cc | 14 ++++++++++++++ src/libfetchers/git-utils.hh | 2 ++ src/libfetchers/git.cc | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index bd578685763..15fa540b3ac 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -5,6 +5,7 @@ #include "signals.hh" #include "users.hh" #include "fs-sink.hh" +#include "sync.hh" #include #include @@ -1276,4 +1277,17 @@ ref getTarballCache() return GitRepo::openRepo(repoDir, true, true); } +GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) +{ + static Sync> _cache; + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo(); + _cache.lock()->emplace(path, workdirInfo); + return workdirInfo; +} + } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 12cee5db107..2db64e43823 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -70,6 +70,8 @@ struct GitRepo virtual WorkdirInfo getWorkdirInfo() = 0; + static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d47e731f14e..a584542d147 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -431,7 +431,7 @@ struct GitInputScheme : InputScheme // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); + repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url); return repoInfo; } From b9f60faab50f79c9d5fd4d11d1bf06feeb62edc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 14:51:41 +0100 Subject: [PATCH 85/93] Fix macOS build --- src/libfetchers/git-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 15fa540b3ac..a41546c44eb 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1279,7 +1279,7 @@ ref getTarballCache() GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + static Sync> _cache; { auto cache(_cache.lock()); auto i = cache->find(path); From 33852ead6b01b58cd66a474b6615595244dc56d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 15:31:19 +0100 Subject: [PATCH 86/93] Optimisation --- src/libfetchers/git-utils.cc | 21 ++++++--------------- src/libfetchers/git-utils.hh | 7 ++++--- src/libfetchers/git.cc | 13 ++++++------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index a41546c44eb..b54416b1062 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -438,11 +438,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && !(statusFlags & GIT_STATUS_WT_DELETED)) - info.files.emplace(CanonPath(path), - statusFlags == GIT_STATUS_CURRENT - ? WorkdirInfo::State::Clean - : WorkdirInfo::State::Dirty); - else + { + info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.dirtyFiles.insert(CanonPath(path)); + } else info.deletedFiles.insert(CanonPath(path)); if (statusFlags != GIT_STATUS_CURRENT) info.isDirty = true; @@ -1208,15 +1208,6 @@ ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore } } -template -std::set getKeys(const std::map & c) -{ - std::set res; - for (auto & i : c) - res.insert(i.first); - return res; -} - ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); @@ -1229,7 +1220,7 @@ ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool export ? makeEmptySourceAccessor() : AllowListSourceAccessor::create( makeFSSourceAccessor(path), - std::set { getKeys(wd.files) }, + std::set { wd.files }, std::move(makeNotAllowedError)).cast(); if (exportIgnore) return make_ref(self, fileAccessor, std::nullopt); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 2db64e43823..ff115143fc7 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -55,11 +55,12 @@ struct GitRepo in the repo yet. */ std::optional headRev; - enum State { Clean, Dirty }; - /* All files in the working directory that are unchanged, modified or added, but excluding deleted files. */ - std::map files; + std::set files; + + /* All modified or added files. */ + std::set dirtyFiles; /* The deleted files. */ std::set deletedFiles; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a584542d147..c73f537652d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -686,7 +686,7 @@ struct GitInputScheme : InputScheme if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) - repoInfo.workdirInfo.files.emplace(submodule.path, GitRepo::WorkdirInfo::State::Clean); + repoInfo.workdirInfo.files.insert(submodule.path); auto repo = GitRepo::openRepo(repoInfo.url, false, false); @@ -807,12 +807,11 @@ struct GitInputScheme : InputScheme /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; - for (auto & file : repoInfo.workdirInfo.files) - if (file.second == GitRepo::WorkdirInfo::State::Dirty) { - writeString("modified:", hashSink); - writeString(file.first.abs(), hashSink); - dumpPath(repoInfo.url + "/" + file.first.abs(), hashSink); - } + for (auto & file : repoInfo.workdirInfo.dirtyFiles) { + writeString("modified:", hashSink); + writeString(file.abs(), hashSink); + dumpPath(repoInfo.url + "/" + file.abs(), hashSink); + } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); writeString(file.abs(), hashSink); From 757ea706449925636a99dfbcb09a09c62b1de319 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Dec 2024 13:13:39 +0100 Subject: [PATCH 87/93] Add a test --- tests/functional/flakes/flakes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index de37ae1b781..6c466a0c7cb 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision) echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) From b33814093100ca15a2afdc1be207fcd4d1328a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 11:49:43 +0100 Subject: [PATCH 88/93] fix missing includes in various headers --- src/libexpr/lexer-helpers.hh | 8 ++++++++ src/libexpr/print-options.hh | 1 + src/libstore/path-regex.hh | 2 ++ src/libutil/callback.hh | 1 + src/libutil/config-impl.hh | 1 + src/libutil/regex-combinators.hh | 1 + src/libutil/unix/signals-impl.hh | 1 + src/nix/self-exe.hh | 2 ++ 8 files changed, 17 insertions(+) diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index caba6e18f48..b4a166f46a8 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -1,5 +1,13 @@ #pragma once +#include + +// inluding the generated headers twice leads to errors +#ifndef BISON_HEADER +#include "lexer-tab.hh" +#include "parser-tab.hh" +#endif + namespace nix::lexer::internal { void initLoc(YYLTYPE * loc); diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh index 080ba26b86c..9ad54e5323c 100644 --- a/src/libexpr/print-options.hh +++ b/src/libexpr/print-options.hh @@ -5,6 +5,7 @@ */ #include +#include namespace nix { diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 56c2cfc1db5..e34a305c5f9 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + namespace nix { diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index 26c386d8078..c2cada2f682 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index c3aa61ddb95..94c2cb2e4b8 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -13,6 +13,7 @@ */ #include "config.hh" +#include "args.hh" namespace nix { diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 87d6aa6785f..8a7b2c2c01e 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -2,6 +2,7 @@ ///@file #include +#include namespace nix::regex { diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh index 2193922be4c..037416e7d6b 100644 --- a/src/libutil/unix/signals-impl.hh +++ b/src/libutil/unix/signals-impl.hh @@ -14,6 +14,7 @@ #include "error.hh" #include "logging.hh" #include "ansicolor.hh" +#include "signals.hh" #include #include diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh index 3161553ecfa..91e260f0b79 100644 --- a/src/nix/self-exe.hh +++ b/src/nix/self-exe.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include namespace nix { From b9bbdbeb0b5ec0bb3231d1e04d1825fdc1f6c4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 11:50:03 +0100 Subject: [PATCH 89/93] ca-fd-leak: add missing c casts --- tests/nixos/ca-fd-leak/sender.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c index 75e54fc8f52..8356b2479a6 100644 --- a/tests/nixos/ca-fd-leak/sender.c +++ b/tests/nixos/ca-fd-leak/sender.c @@ -49,8 +49,8 @@ int main(int argc, char **argv) { msg.msg_controllen = CMSG_SPACE(sizeof(int)); // Write a single null byte too. - msg.msg_iov = malloc(sizeof(struct iovec)); - msg.msg_iov[0].iov_base = ""; + msg.msg_iov = (struct iovec*) malloc(sizeof(struct iovec)); + msg.msg_iov[0].iov_base = (void*) ""; msg.msg_iov[0].iov_len = 1; msg.msg_iovlen = 1; From eda331e53fc6322922f467263ded0b69c28dd427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 12:43:28 +0100 Subject: [PATCH 90/93] libutil/windows: only define headers/code if we are actually on windows all those includes otherwise break linters. --- src/libexpr/lexer-helpers.hh | 4 ++-- src/libstore/windows/pathlocks.cc | 11 +++++++---- src/libutil/windows/environment-variables.cc | 4 +++- src/libutil/windows/file-descriptor.cc | 2 ++ src/libutil/windows/file-system.cc | 2 ++ src/libutil/windows/muxable-pipe.cc | 12 +++++++----- src/libutil/windows/os-string.cc | 4 ++++ src/libutil/windows/processes.cc | 4 ++++ src/libutil/windows/users.cc | 2 ++ src/libutil/windows/windows-async-pipe.cc | 4 ++++ src/libutil/windows/windows-async-pipe.hh | 2 ++ src/libutil/windows/windows-error.cc | 2 ++ src/libutil/windows/windows-error.hh | 2 ++ 13 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index b4a166f46a8..d40f7b874e8 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -4,8 +4,8 @@ // inluding the generated headers twice leads to errors #ifndef BISON_HEADER -#include "lexer-tab.hh" -#include "parser-tab.hh" +# include "lexer-tab.hh" +# include "parser-tab.hh" #endif namespace nix::lexer::internal { diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 00761a8c33c..197f5a1c46d 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -2,10 +2,12 @@ #include "pathlocks.hh" #include "signals.hh" #include "util.hh" -#include -#include -#include -#include "windows-error.hh" + +#ifdef WIN32 +# include +# include +# include +# include "windows-error.hh" namespace nix { @@ -154,3 +156,4 @@ FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view w } } +#endif diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 5ce8a139537..308a432e4d8 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" -#include "processenv.h" +#ifdef WIN32 +# include "processenv.h" namespace nix { @@ -43,3 +44,4 @@ int setEnvOs(const OsString & name, const OsString & value) } } +#endif diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 7b8a712e879..71f53ccb886 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,6 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" +#ifdef WIN32 #include #include #include @@ -152,3 +153,4 @@ Path windows::handleToPath(HANDLE handle) { #endif } +#endif diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index b15355efe88..53271cef3da 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,5 +1,6 @@ #include "file-system.hh" +#ifdef WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) @@ -15,3 +16,4 @@ Descriptor openDirectory(const std::filesystem::path & path) } } +#endif diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 91a321f7cfa..af7e987e9f2 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,9 +1,10 @@ -#include -#include "windows-error.hh" +#ifdef WIN32 +# include +# include "windows-error.hh" -#include "logging.hh" -#include "util.hh" -#include "muxable-pipe.hh" +# include "logging.hh" +# include "util.hh" +# include "muxable-pipe.hh" namespace nix { @@ -68,3 +69,4 @@ void MuxablePipePollState::iterate( } } +#endif diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 7507f9030da..26ad9cba0c8 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,6 +7,8 @@ #include "file-path-impl.hh" #include "util.hh" +#ifdef WIN32 + namespace nix { std::string os_string_to_string(PathViewNG::string_view path) @@ -22,3 +24,5 @@ std::filesystem::path::string_type string_to_os_string(std::string_view s) } } + +#endif diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 7f34c5632e8..e69f1ed45a5 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,6 +23,8 @@ #include #include +#ifdef WIN32 + #define WIN32_LEAN_AND_MEAN #include @@ -386,3 +388,5 @@ int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * } } + +#endif diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index db6c42df381..2780e45f4a1 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" +#ifdef WIN32 #define WIN32_LEAN_AND_MEAN #include @@ -50,3 +51,4 @@ bool isRootUser() { } } +#endif diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 4fa57ca3672..13b56351029 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,6 +1,8 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" +#ifdef WIN32 + namespace nix::windows { void AsyncPipe::createAsyncPipe(HANDLE iocp) @@ -47,3 +49,5 @@ void AsyncPipe::close() } } + +#endif diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 8f554e403f3..277336ed76a 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,6 +2,7 @@ ///@file #include "file-descriptor.hh" +#ifdef WIN32 namespace nix::windows { @@ -25,3 +26,4 @@ public: }; } +#endif diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index aead4af2378..4cf4274dae7 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,5 +1,6 @@ #include "windows-error.hh" +#ifdef WIN32 #include #define WIN32_LEAN_AND_MEAN #include @@ -29,3 +30,4 @@ std::string WinError::renderError(DWORD lastError) } } +#endif diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 624b4c4cb0b..4e48ee8599c 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#ifdef WIN32 #include #include "error.hh" @@ -49,3 +50,4 @@ private: }; } +#endif From 3392a96901bfde94142fbfffd9d7d12549522ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 12:07:35 +0000 Subject: [PATCH 91/93] ca-fd-leak: fix various unsafe c handling --- tests/nixos/ca-fd-leak/smuggler.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/nixos/ca-fd-leak/smuggler.c b/tests/nixos/ca-fd-leak/smuggler.c index 82acf37e68e..3f89af5bbe6 100644 --- a/tests/nixos/ca-fd-leak/smuggler.c +++ b/tests/nixos/ca-fd-leak/smuggler.c @@ -16,7 +16,7 @@ int main(int argc, char **argv) { struct sockaddr_un data; data.sun_family = AF_UNIX; data.sun_path[0] = 0; - strcpy(data.sun_path + 1, argv[1]); + strncpy(data.sun_path + 1, argv[1], sizeof(data.sun_path) - 1); int res = bind(sock, (const struct sockaddr *)&data, offsetof(struct sockaddr_un, sun_path) + strlen(argv[1]) @@ -57,10 +57,11 @@ int main(int argc, char **argv) { // Wait for a second connection, which will tell us that the build is // done a = accept(sock, 0, 0); + if (a < 0) perror("accept"); fprintf(stderr, "%s\n", "Got a second connection, rewriting the file"); // Write a new content to the file if (ftruncate(smuggling_fd, 0)) perror("ftruncate"); - char * new_content = "Pwned\n"; + const char * new_content = "Pwned\n"; int written_bytes = write(smuggling_fd, new_content, strlen(new_content)); if (written_bytes != strlen(new_content)) perror("write"); } From 6848154b2e657e64a592b60f00b68db8d911b89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 12:47:20 +0000 Subject: [PATCH 92/93] regex-combinators: fix linter error when adding string_view and string --- src/libutil/regex-combinators.hh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 8a7b2c2c01e..75ccd4e6cf0 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -3,6 +3,7 @@ #include #include +#include namespace nix::regex { @@ -11,22 +12,23 @@ namespace nix::regex { static inline std::string either(std::string_view a, std::string_view b) { - return std::string { a } + "|" + b; + std::stringstream ss; + ss << a << "|" << b; + return ss.str(); } static inline std::string group(std::string_view a) { - return std::string { "(" } + a + ")"; -} - -static inline std::string many(std::string_view a) -{ - return std::string { "(?:" } + a + ")*"; + std::stringstream ss; + ss << "(" << a << ")"; + return ss.str(); } static inline std::string list(std::string_view a) { - return std::string { a } + many(group("," + a)); + std::stringstream ss; + ss << a << "(," << a << ")*"; + return ss.str(); } } From 5cd5391dd8055bc0876204b6b38274f8c821c751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 20:41:39 +0100 Subject: [PATCH 93/93] checks: re-enable perlBindings --- flake.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index a5e68609eda..44a764dd7b8 100644 --- a/flake.nix +++ b/flake.nix @@ -201,11 +201,7 @@ # Some perl dependencies are broken on i686-linux. # Since the support is only best-effort there, disable the perl # bindings - - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable this. - #perlBindings = self.hydraJobs.perlBindings.${system}; + perlBindings = self.hydraJobs.perlBindings.${system}; } # Add "passthru" tests // flatMapAttrs ({