diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index ba5667a4305..90eaa1a73fe 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -116,9 +116,12 @@ let storeInfo = commandInfo.stores; inherit inlineHTML; }; + hasInfix = infix: content: + builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content); in optionalString (details ? doc) ( - if match ".*@store-types@.*" details.doc != null + # An alternate implementation with builtins.match stack overflowed on some systems. + if hasInfix "@store-types@" details.doc then help-stores else details.doc ); diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md new file mode 100644 index 00000000000..05023697c96 --- /dev/null +++ b/doc/manual/rl-next/repl-doc-renders-doc-comments.md @@ -0,0 +1,53 @@ +--- +synopsis: "`nix-repl`'s `:doc` shows documentation comments" +significance: significant +issues: +- 3904 +- 10771 +prs: +- 1652 +- 9054 +- 11072 +--- + +`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. +This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + +Example: + +``` +nix-repl> :doc lib.toFunction +Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + +Inputs + + v + + : Any value + +Examples + + :::{.example} + +## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: +``` + +Known limitations: +- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. +- Some extensions to markdown are not yet supported, as you can see in the example above. + +We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. + +[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index a96ba997bed..3949164d503 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -91,7 +91,7 @@ A environment variables that Google Test accepts are also worth knowing: Putting the two together, one might run ```bash -GTEST_BREIF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v ``` for short but comprensive output. diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 46b3e136324..66bfcf609b1 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -143,6 +143,7 @@ ''^src/libstore/common-protocol-impl\.hh$'' ''^src/libstore/common-protocol\.cc$'' ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' ''^src/libstore/content-address\.cc$'' ''^src/libstore/content-address\.hh$'' ''^src/libstore/daemon\.cc$'' @@ -215,7 +216,6 @@ ''^src/libstore/serve-protocol\.hh$'' ''^src/libstore/sqlite\.cc$'' ''^src/libstore/sqlite\.hh$'' - ''^src/libstore/ssh-store-config\.hh$'' ''^src/libstore/ssh-store\.cc$'' ''^src/libstore/ssh\.cc$'' ''^src/libstore/ssh\.hh$'' diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 34b3449718d..73ba9cd586f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -11,11 +11,28 @@ versionSuffix, }: +let + prevStdenv = stdenv; +in + let inherit (pkgs) lib; root = ../.; + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. resolvePath = p: p; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 582e6d623d0..600fc7ee2b7 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -16,6 +16,7 @@ #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d7e..905e70f32c9 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) - -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE - -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0eefb..dc78d3e599d 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 74d146c66c5..67fef190920 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -132,7 +133,7 @@ ref EvalCommand::getEvalState() #else std::make_shared( #endif - lookupPath, getEvalStore(), evalSettings, getStore()) + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 62745b6815f..470a25c4ec9 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,3 +1,4 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" @@ -7,6 +8,7 @@ #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" @@ -16,6 +18,10 @@ namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + EvalSettings evalSettings { settings.readOnlyMode, { @@ -24,7 +30,7 @@ EvalSettings evalSettings { [](ref store, std::string_view rest) { experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. - auto flakeRef = parseFlakeRef(std::string { rest }, {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); auto storePath = flakeRef.resolve(store).fetchTree(store).first; return store->toRealPath(storePath); @@ -35,6 +41,12 @@ EvalSettings evalSettings { static GlobalConfig::Register rEvalSettings(&evalSettings); + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + CompatibilitySettings compatibilitySettings {}; static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); @@ -171,8 +183,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -230,7 +242,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 8d303ee7c13..c62365b32e2 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -11,17 +11,32 @@ namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; struct EvalSettings; struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + /** * @todo Get rid of global setttings variables */ extern EvalSettings evalSettings; +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + /** * Settings that control behaviors that have changed since Nix 2.3. */ diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d42fa7aaccc..8796ad5ba79 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -210,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c140d..8e0a232ef8a 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6835c512c1c..0fe956ec023 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -289,10 +292,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s if (v2.type() == nAttrs) { for (auto & i : *v2.attrs()) { - std::string name = state->symbols[i.name]; + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -534,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -851,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -873,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 420cce1db89..b285c8a9ae5 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -73,7 +73,7 @@ static int listPossibleCallback(char * s, char *** avp) { auto possible = curRepl->completePrefix(s); - if (possible.size() > (INT_MAX / sizeof(char *))) + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) throw Error("too many completions"); int ac = 0; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index ce1c5af6997..b5d0816dd2c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include "repl-interacter.hh" #include "repl.hh" @@ -10,8 +9,6 @@ #include "shared.hh" #include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" @@ -29,12 +26,16 @@ #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -616,6 +617,38 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + + } else { + evalString(arg, v); + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -633,6 +666,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -690,14 +736,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk index 51b02562e12..227a4095b30 100644 --- a/src/libexpr-c/local.mk +++ b/src/libexpr-c/local.mk @@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ $(INCLUDE_libstore) $(INCLUDE_libstorec) \ $(INCLUDE_libexpr) $(INCLUDE_libexprc) -libexprc_LIBS = libutil libutilc libstore libstorec libexpr +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr libexprc_LDFLAGS += $(THREAD_LDFLAGS) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 13e0f5f3a9f..547453f8f93 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -1,12 +1,10 @@ #include -#include #include #include -#include "config.hh" #include "eval.hh" +#include "eval-gc.hh" #include "globals.hh" -#include "util.hh" #include "eval-settings.hh" #include "nix_api_expr.h" @@ -112,12 +110,14 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c static_cast(alignof(EvalState))); auto * p2 = static_cast(p); new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, .settings = nix::EvalSettings{ nix::settings.readOnlyMode, }, .state = nix::EvalState( nix::LookupPath::parse(lookupPath), store->ptr, + p2->fetchSettings, p2->settings), }; loadConfFile(p2->settings); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index d4ccffd29d6..12f24b6eb07 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,6 +1,7 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include "fetch-settings.hh" #include "eval.hh" #include "eval-settings.hh" #include "attr-set.hh" @@ -8,6 +9,7 @@ struct EvalState { + nix::fetchers::Settings fetchSettings; nix::EvalSettings settings; nix::EvalState state; }; diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 3c3dd6ca99d..3565092a4a6 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -115,7 +115,7 @@ class NixCExternalValue : public nix::ExternalValueBase /** * Compare to another value of the same type. */ - virtual bool operator==(const ExternalValueBase & b) const override + virtual bool operator==(const ExternalValueBase & b) const noexcept override { if (!desc.equal) { return false; diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 2f2f99617e2..cb5d9ee8928 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -383,7 +383,7 @@ nix_value * nix_get_attr_byidx( try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); + *name = state->state.symbols[a.name].c_str(); nix_gc_incref(nullptr, a.value); state->state.forceValue(*a.value, nix::noPos); return as_nix_value_ptr(a.value); @@ -399,7 +399,7 @@ nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalSt try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - return ((const std::string &) (state->state.symbols[a.name])).c_str(); + return state->state.symbols[a.name].c_str(); } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9ad201b63ba..d61d9363070 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -76,7 +76,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!a) { std::set attrNames; for (auto & attr : *v->attrs()) - attrNames.insert(state.symbols[attr.name]); + attrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(attrNames, attr); throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index ba798196d3c..4df9a1acdc9 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -5,7 +5,6 @@ #include "symbol-table.hh" #include -#include namespace nix { @@ -28,9 +27,9 @@ struct Attr Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name), pos(pos), value(value) { }; Attr() { }; - bool operator < (const Attr & a) const + auto operator <=> (const Attr & a) const { - return name < a.name; + return name <=> a.name; } }; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2630c34d563..e573fe88499 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -225,7 +225,7 @@ struct AttrDb (key.first) (symbols[key.second]) (AttrType::ListOfStrings) - (concatStringsSep("\t", l)).exec(); + (dropEmptyInitThenConcatStringsSep("\t", l)).exec(); return state->db.getLastInsertedRowId(); }); @@ -435,12 +435,12 @@ std::vector AttrCursor::getAttrPath(Symbol name) const std::string AttrCursor::getAttrPathStr() const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath())); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath())); } std::string AttrCursor::getAttrPathStr(Symbol name) const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); } Value & AttrCursor::forceValue() @@ -484,7 +484,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) auto attrNames = getAttrs(); std::set strAttrNames; for (auto & name : attrNames) - strAttrNames.insert(root->state.symbols[name]); + strAttrNames.insert(std::string(root->state.symbols[name])); return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index fe48e054bf2..2f604647331 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -1,7 +1,5 @@ #pragma once -#include - #include "error.hh" #include "pos-idx.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 6b7b52cef32..e2151aa7fc1 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,5 +1,4 @@ #include "users.hh" -#include "config-global.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 191dde21aa9..b915ba530a0 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -2,6 +2,7 @@ ///@file #include "config.hh" +#include "ref.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2a08621231f..4f01d0a62d3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,6 @@ #include "eval.hh" +#include "eval-gc.hh" #include "eval-settings.hh" -#include "hash.hh" #include "primops.hh" #include "print-options.hh" #include "exit.hh" @@ -16,7 +16,6 @@ #include "print.hh" #include "filtering-source-accessor.hh" #include "memory-source-accessor.hh" -#include "signals.hh" #include "gc-small-vector.hh" #include "url.hh" #include "fetch-to-store.hh" @@ -24,7 +23,6 @@ #include "parser-tab.hh" #include -#include #include #include #include @@ -51,6 +49,8 @@ #endif +#include "strings-inline.hh" + using json = nlohmann::json; namespace nix { @@ -217,9 +217,11 @@ static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, const EvalSettings & settings, std::shared_ptr buildStore) - : settings{settings} + : fetchSettings{fetchSettings} + , settings{settings} , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -557,6 +559,54 @@ std::optional EvalState::getDoc(Value & v) .doc = doc, }; } + if (v.isLambda()) { + auto exprLambda = v.payload.lambda.fun; + + std::stringstream s(std::ios_base::out); + std::string name; + auto pos = positions[exprLambda->getPos()]; + std::string docStr; + + if (exprLambda->name) { + name = symbols[exprLambda->name]; + } + + if (exprLambda->docComment) { + docStr = exprLambda->docComment.getInnerText(positions); + } + + if (name.empty()) { + s << "Function "; + } + else { + s << "Function `" << name << "`"; + if (pos) + s << "\\\n … " ; + else + s << "\\\n"; + } + if (pos) { + s << "defined at " << pos; + } + if (!docStr.empty()) { + s << "\n\n"; + } + + s << docStr; + + s << '\0'; // for making a c string below + std::string ss = s.str(); + + return Doc { + .pos = pos, + .name = name, + .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... + .args = {}, + .doc = + // FIXME: this leaks; make the field std::string? + strdup(ss.data()), + }; + } return {}; } @@ -633,11 +683,11 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. for (auto & j : *env.values[0]->attrs()) - vm[st[j.name]] = j.value; + vm.insert_or_assign(std::string(st[j.name]), j.value); } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) - vm[st[i.first]] = env.values[i.second]; + vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]); } } } @@ -1011,7 +1061,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) if (!e) e = parseExprFromFile(resolvedPath); - fileParseCache[resolvedPath] = e; + fileParseCache.emplace(resolvedPath, e); try { auto dts = debugRepl @@ -1034,8 +1084,8 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) throw; } - fileEvalCache[resolvedPath] = v; - if (path != resolvedPath) fileEvalCache[path] = v; + fileEvalCache.emplace(resolvedPath, v); + if (path != resolvedPath) fileEvalCache.emplace(path, v); } @@ -1338,7 +1388,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) if (!(j = vAttrs->attrs()->get(name))) { std::set allAttrNames; for (auto & attr : *vAttrs->attrs()) - allAttrNames.insert(state.symbols[attr.name]); + allAttrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); state.error("attribute '%1%' missing", state.symbols[name]) .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); @@ -1365,6 +1415,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) v = *vAttrs; } +Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs) +{ + Value vTmp; + Symbol name = getName(attrPath[attrPath.size() - 1], state, env); + + if (attrPath.size() == 1) { + e->eval(state, env, vTmp); + } else { + ExprSelect init(*this); + init.attrPath.pop_back(); + init.eval(state, env, vTmp); + } + attrs = vTmp; + return name; +} + void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) { @@ -1496,7 +1562,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) - formalNames.insert(symbols[formal.name]); + formalNames.insert(std::string(symbols[formal.name])); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); error("function '%1%' called with unexpected argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), @@ -2826,13 +2892,37 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols); + DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath + DocCommentMap *docComments = &tmpDocComments; + + if (auto sourcePath = std::get_if(&origin)) { + auto [it, _] = positionToDocComment.try_emplace(*sourcePath); + docComments = &it->second; + } + + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols); result->bindVars(*this, staticEnv); return result; } +DocComment EvalState::getDocCommentForPos(PosIdx pos) +{ + auto pos2 = positions[pos]; + auto path = pos2.getSourcePath(); + if (!path) + return {}; + + auto table = positionToDocComment.find(*path); + if (table == positionToDocComment.end()) + return {}; + + auto it = table->second.find(pos); + if (it == table->second.end()) + return {}; + return it->second; +} std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { @@ -2842,7 +2932,7 @@ std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & } -bool ExternalValueBase::operator==(const ExternalValueBase & b) const +bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept { return false; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e45358055ed..d376046ae30 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,21 +3,21 @@ #include "attr-set.hh" #include "eval-error.hh" -#include "eval-gc.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "position.hh" +#include "pos-table.hh" #include "source-accessor.hh" #include "search-path.hh" #include "repl-exit-status.hh" +#include "ref.hh" #include #include -#include -#include #include namespace nix { @@ -30,6 +30,7 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +namespace fetchers { struct Settings; } struct EvalSettings; class EvalState; class StorePath; @@ -43,7 +44,7 @@ namespace eval_cache { /** * Function that implements a primop. */ -typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v); /** * Info about a primitive operation, and its implementation @@ -84,7 +85,7 @@ struct PrimOp /** * Implementation of the primop. */ - std::function::type> fun; + std::function fun; /** * Optional experimental for this to be gated on. @@ -129,6 +130,8 @@ struct Constant typedef std::map ValMap; #endif +typedef std::map DocCommentMap; + struct Env { Env * up; @@ -162,6 +165,7 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const fetchers::Settings & fetchSettings; const EvalSettings & settings; SymbolTable symbols; PosTable positions; @@ -305,15 +309,15 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - Sync> srcToStore; + Sync> srcToStore; /** * A cache from path names to parse trees. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileParseCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileParseCache; #else - typedef std::map FileParseCache; + typedef std::unordered_map FileParseCache; #endif FileParseCache fileParseCache; @@ -321,12 +325,18 @@ private: * A cache from path names to values. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileEvalCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileEvalCache; #else - typedef std::map FileEvalCache; + typedef std::unordered_map FileEvalCache; #endif FileEvalCache fileEvalCache; + /** + * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. + * Grouped by file. + */ + std::map positionToDocComment; + LookupPath lookupPath; std::map> lookupPathResolved; @@ -353,6 +363,7 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); @@ -768,6 +779,8 @@ public: std::string_view pathArg, PosIdx pos); + DocComment getDocCommentForPos(PosIdx pos); + private: /** diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 8967334237a..7041a3932ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -342,9 +342,9 @@ std::optional getDerivation(EvalState & state, Value & v, } -static std::string addToPath(const std::string & s1, const std::string & s2) +static std::string addToPath(const std::string & s1, std::string_view s2) { - return s1.empty() ? s2 : s1 + "." + s2; + return s1.empty() ? std::string(s2) : s1 + "." + s2; } diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc new file mode 100644 index 00000000000..d9eeb73e269 --- /dev/null +++ b/src/libexpr/lexer-helpers.cc @@ -0,0 +1,30 @@ +#include "lexer-tab.hh" +#include "lexer-helpers.hh" +#include "parser-tab.hh" + +void nix::lexer::internal::initLoc(YYLTYPE * loc) +{ + loc->beginOffset = loc->endOffset = 0; +} + +void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) +{ + loc->stash(); + + LexerState & lexerState = *yyget_extra(yyscanner); + + if (lexerState.docCommentDistance == 1) { + // Preceding token was a doc comment. + ParserLocation doc; + doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset; + ParserLocation docEnd; + docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset; + DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; + PosIdx locPos = lexerState.at(*loc); + lexerState.positionToDocComment.emplace(locPos, docComment); + } + lexerState.docCommentDistance++; + + loc->beginOffset = loc->endOffset; + loc->endOffset += len; +} diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh new file mode 100644 index 00000000000..caba6e18f48 --- /dev/null +++ b/src/libexpr/lexer-helpers.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix::lexer::internal { + +void initLoc(YYLTYPE * loc); + +void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len); + +} // namespace nix::lexer diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 8c0f9d1f2f7..58401be8e71 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -5,7 +5,7 @@ %option stack %option nodefault %option nounput noyy_top_state - +%option extra-type="::nix::LexerState *" %s DEFAULT %x STRING @@ -14,6 +14,10 @@ %x INPATH_SLASH %x PATH_START +%top { +#include "parser-tab.hh" // YYSTYPE +#include "parser-state.hh" +} %{ #ifdef __clang__ @@ -22,27 +26,18 @@ #include "nixexpr.hh" #include "parser-tab.hh" - -using namespace nix; +#include "lexer-helpers.hh" namespace nix { - -#define CUR_POS state->at(*yylloc) - -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 0; - loc->first_column = loc->last_column = 0; + struct LexerState; } -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) -{ - loc->stash(); +using namespace nix; +using namespace nix::lexer::internal; - loc->first_column = loc->last_column; - loc->last_column += len; -} +namespace nix { +#define CUR_POS state->at(*yylloc) // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. @@ -79,7 +74,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); +#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng); #define PUSH_STATE(state) yy_push_state(state, yyscanner) #define POP_STATE() yy_pop_state(yyscanner) @@ -279,9 +274,32 @@ or { return OR_KW; } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } {URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; } -[ \t\r\n]+ /* eat up whitespace */ -\#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ +%{ +// Doc comment rule +// +// \/\*\* /** +// [^/*] reject /**/ (empty comment) and /*** +// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule +// ( )* zero or more non-ending sequences +// \* end(1) +// \/ end(2) +%} +\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { + LexerState & lexerState = *yyget_extra(yyscanner); + lexerState.docCommentDistance = 0; + lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset; + lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset; +} + + +%{ +// The following rules have docCommentDistance-- +// This compensates for the docCommentDistance++ which happens by default to +// make all the other rules invalidate the doc comment. +%} +[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; } +\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; } +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; } {ANY} { /* Don't return a negative number, as this will cause diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index e65cf545f2d..fa90e7b41a3 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -139,6 +139,7 @@ sources = files( 'function-trace.cc', 'get-drvs.cc', 'json-to-value.cc', + 'lexer-helpers.cc', 'nixexpr.cc', 'paths.cc', 'primops.cc', @@ -165,6 +166,7 @@ headers = [config_h] + files( 'gc-small-vector.hh', 'get-drvs.hh', 'json-to-value.hh', + # internal: 'lexer-helpers.hh', 'nixexpr.hh', 'parser-state.hh', 'pos-idx.hh', diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 44198a25238..93c8bdef6b6 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -1,5 +1,4 @@ #include "nixexpr.hh" -#include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" #include "util.hh" @@ -8,6 +7,8 @@ #include #include +#include "strings-inline.hh" + namespace nix { unsigned long Expr::nrExprs = 0; @@ -582,6 +583,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const return fmt("%1% at %2%", id, state.positions[pos]); } +void ExprLambda::setDocComment(DocComment docComment) { + // RFC 145 specifies that the innermost doc comment wins. + // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + if (!this->docComment) { + this->docComment = docComment; + + // Curried functions are defined by putting a function directly + // in the body of another function. To render docs for those, we + // need to propagate the doc comment to the innermost function. + // + // If we have our own comment, we've already propagated it, so this + // belongs in the same conditional. + body->setDocComment(docComment); + } +}; + /* Position table. */ @@ -626,4 +643,22 @@ size_t SymbolTable::totalSize() const return n; } +std::string DocComment::getInnerText(const PosTable & positions) const { + auto beginPos = positions[begin]; + auto endPos = positions[end]; + auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or(""); + + // Strip "/**" and "*/" + constexpr size_t prefixLen = 3; + constexpr size_t suffixLen = 2; + std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); + if (docStr.empty()) + return {}; + // Turn the now missing "/**" into indentation + docStr = " " + docStr; + // Strip indentation (for the whole, potentially multi-line string) + docStr = stripIndentation(docStr); + return docStr; +} + } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e37e3bdd153..1bcc962c56a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -6,21 +6,58 @@ #include "value.hh" #include "symbol-table.hh" -#include "error.hh" -#include "position.hh" #include "eval-error.hh" #include "pos-idx.hh" -#include "pos-table.hh" namespace nix { - -struct Env; -struct Value; class EvalState; +class PosTable; +struct Env; struct ExprWith; struct StaticEnv; +struct Value; + +/** + * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md) + * + * Note that this does not implement the following: + * - argument attribute names ("formals"): TBD + * - argument names: these are internal to the function and their names may not be optimal for documentation + * - function arity (degree of currying or number of ':'s): + * - Functions returning partially applied functions have a higher arity + * than can be determined locally and without evaluation. + * We do not want to present false data. + * - Some functions should be thought of as transformations of other + * functions. For instance `overlay -> overlay -> overlay` is the simplest + * way to understand `composeExtensions`, but its implementation looks like + * `f: g: final: prev: <...>`. The parameters `final` and `prev` are part + * of the overlay concept, while distracting from the function's purpose. + */ +struct DocComment { + /** + * Start of the comment, including the opening, ie `/` and `**`. + */ + PosIdx begin; + + /** + * Position right after the final asterisk and `/` that terminate the comment. + */ + PosIdx end; + + /** + * Whether the comment is set. + * + * A `DocComment` is small enough that it makes sense to pass by value, and + * therefore baking optionality into it is also useful, to avoiding the memory + * overhead of `std::optional`. + */ + operator bool() const { return static_cast(begin); } + + std::string getInnerText(const PosTable & positions) const; + +}; /** * An attribute path is a sequence of attribute names. @@ -57,6 +94,7 @@ struct Expr virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol name); + virtual void setDocComment(DocComment docComment) { }; virtual PosIdx getPos() const { return noPos; } }; @@ -159,6 +197,17 @@ struct ExprSelect : Expr ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } + + /** + * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl. + * + * @param[out] v The attribute set that should contain the last attribute name (if it exists). + * @return The last attribute name in `attrPath` + * + * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. + */ + Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); + COMMON_METHODS }; @@ -281,6 +330,8 @@ struct ExprLambda : Expr Symbol arg; Formals * formals; Expr * body; + DocComment docComment; + ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body) : pos(pos), arg(arg), formals(formals), body(body) { @@ -293,6 +344,7 @@ struct ExprLambda : Expr std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } PosIdx getPos() const override { return pos; } + virtual void setDocComment(DocComment docComment) override; COMMON_METHODS }; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index cff6282faa0..983a17a2e98 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "eval.hh" namespace nix { @@ -20,25 +22,59 @@ struct StringToken struct ParserLocation { - int first_line, first_column; - int last_line, last_column; + int beginOffset; + int endOffset; // backup to recover from yyless(0) - int stashed_first_column, stashed_last_column; + int stashedBeginOffset, stashedEndOffset; void stash() { - stashed_first_column = first_column; - stashed_last_column = last_column; + stashedBeginOffset = beginOffset; + stashedEndOffset = endOffset; } void unstash() { - first_column = stashed_first_column; - last_column = stashed_last_column; + beginOffset = stashedBeginOffset; + endOffset = stashedEndOffset; } + + /** Latest doc comment position, or 0. */ + int doc_comment_first_column, doc_comment_last_column; +}; + +struct LexerState +{ + /** + * Tracks the distance to the last doc comment, in terms of lexer tokens. + * + * The lexer sets this to 0 when reading a doc comment, and increments it + * for every matched rule; see `lexer-helpers.cc`. + * Whitespace and comment rules decrement the distance, so that they result + * in a net 0 change in distance. + */ + int docCommentDistance = std::numeric_limits::max(); + + /** + * The location of the last doc comment. + * + * (stashing fields are not used) + */ + ParserLocation lastDocCommentLoc; + + /** + * @brief Maps some positions to a DocComment, where the comment is relevant to the location. + */ + std::map & positionToDocComment; + + PosTable & positions; + PosTable::Origin origin; + + PosIdx at(const ParserLocation & loc); }; struct ParserState { + const LexerState & lexerState; SymbolTable & symbols; PosTable & positions; Expr * result; @@ -270,9 +306,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, return new ExprConcatStrings(pos, true, es2); } +inline PosIdx LexerState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.beginOffset); +} + inline PosIdx ParserState::at(const ParserLocation & loc) { - return positions.add(origin, loc.first_column); + return positions.add(origin, loc.beginOffset); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 709a4532a85..452d265bcbf 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,8 +31,25 @@ #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) +// For efficiency, we only track offsets; not line,column coordinates +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \ + (Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \ + } \ + else \ + { \ + (Current).beginOffset = (Current).endOffset = \ + YYRHSLOC (Rhs, 0).endOffset; \ + } \ + while (0) + namespace nix { +typedef std::map DocCommentMap; + Expr * parseExprFromBuf( char * text, size_t length, @@ -41,6 +58,7 @@ Expr * parseExprFromBuf( SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -65,8 +83,7 @@ using namespace nix; void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { - loc->first_column = loc->last_column; - loc->first_line = loc->last_line; + loc->beginOffset = loc->endOffset; } throw ParseError({ .msg = HintFmt(error), @@ -74,6 +91,14 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * }); } +#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos)) +static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) { + auto it = lexerState.positionToDocComment.find(start); + if (it != lexerState.positionToDocComment.end()) { + lambda->setDocComment(it->second); + } +} + %} @@ -119,6 +144,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS + %right IMPL %left OR %left AND @@ -140,18 +166,28 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); } + { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); + $$ = me; + SET_DOC_POS(me, @1); + } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); } + { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + $$ = me; + SET_DOC_POS(me, @1); + } | '{' formals '}' '@' ID ':' expr_function { auto arg = state->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + $$ = me; + SET_DOC_POS(me, @1); } | ID '@' '{' formals '}' ':' expr_function { auto arg = state->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + $$ = me; + SET_DOC_POS(me, @1); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -312,7 +348,22 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } + : binds attrpath '=' expr ';' { + $$ = $1; + + auto pos = state->at(@2); + auto exprPos = state->at(@4); + { + auto it = state->lexerState.positionToDocComment.find(pos); + if (it != state->lexerState.positionToDocComment.end()) { + $4->setDocComment(it->second); + state->lexerState.positionToDocComment.emplace(exprPos, it->second); + } + } + + state->addAttr($$, std::move(*$2), $4, pos); + delete $2; + } | binds INHERIT attrs ';' { $$ = $1; for (auto & [i, iPos] : *$3) { @@ -431,21 +482,28 @@ Expr * parseExprFromBuf( SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols) { yyscan_t scanner; + LexerState lexerState { + .positionToDocComment = docComments, + .positions = positions, + .origin = positions.addOrigin(origin, length), + }; ParserState state { + .lexerState = lexerState, .symbols = symbols, .positions = positions, .basePath = basePath, - .origin = positions.addOrigin(origin, length), + .origin = lexerState.origin, .rootFS = rootFS, .s = astSymbols, .settings = settings, }; - yylex_init(&scanner); + yylex_init_extra(&lexerState, &scanner); Finally _destroy([&] { yylex_destroy(scanner); }); yy_scan_buffer(text, length, scanner); diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh index e94fd85c673..e1349156022 100644 --- a/src/libexpr/pos-idx.hh +++ b/src/libexpr/pos-idx.hh @@ -28,20 +28,15 @@ public: return id > 0; } - bool operator<(const PosIdx other) const + auto operator<=>(const PosIdx other) const { - return id < other.id; + return id <=> other.id; } bool operator==(const PosIdx other) const { return id == other.id; } - - bool operator!=(const PosIdx other) const - { - return id != other.id; - } }; inline PosIdx noPos = {}; diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh index 8a0a3ba86fa..ba2b91cf35e 100644 --- a/src/libexpr/pos-table.hh +++ b/src/libexpr/pos-table.hh @@ -1,10 +1,8 @@ #pragma once -#include -#include +#include #include -#include "chunked-vector.hh" #include "pos-idx.hh" #include "position.hh" #include "sync.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 134363e1ab2..127823d4944 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,4 +1,3 @@ -#include "archive.hh" #include "derivations.hh" #include "downstream-placeholder.hh" #include "eval-inline.hh" @@ -1225,7 +1224,7 @@ static void derivationStrictInternal( for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; - const std::string & key = state.symbols[i->name]; + auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { @@ -1309,7 +1308,7 @@ static void derivationStrictInternal( if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); + jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context)); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, pos, context_below); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 7b5f4193a23..64e3abf2db4 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs.insert_or_assign("name", std::string(name)); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto [storePath, input2] = input.fetchToStore(state.store); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 567b73f9a1b..6a7accad7f6 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,7 +85,7 @@ static void fetchTree( Value & v, const FetchTreeParams & params = FetchTreeParams{} ) { - fetchers::Input input; + fetchers::Input input { state.fetchSettings }; NixStringContext context; std::optional type; if (params.isFetchGit) type = "git"; @@ -148,7 +148,7 @@ static void fetchTree( "attribute 'name' isn’t supported in call to 'fetchTree'" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { auto url = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to the fetcher", @@ -161,13 +161,13 @@ static void fetchTree( if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { attrs.emplace("exportIgnore", Explicit{true}); } - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) state.error( "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(state.fetchSettings, url); } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 967a186dd59..c7a3563b040 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -30,9 +30,9 @@ public: return *s == s2; } - operator const std::string & () const + const char * c_str() const { - return *s; + return s->c_str(); } operator const std::string_view () const @@ -41,6 +41,11 @@ public: } friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); + + bool empty() const + { + return s->empty(); + } }; /** @@ -62,9 +67,8 @@ public: explicit operator bool() const { return id > 0; } - bool operator<(const Symbol other) const { return id < other.id; } + auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } - bool operator!=(const Symbol other) const { return id != other.id; } }; /** diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 936ecf07826..f8cc056161e 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict, out = json::object(); for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { try { - out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore); + out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); } catch (Error & e) { e.addTrace(state.positions[a->pos], HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1de8cdf848d..9734ebec498 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -9,7 +9,7 @@ namespace nix { -static XMLAttrs singletonAttrs(const std::string & name, const std::string & value) +static XMLAttrs singletonAttrs(const std::string & name, std::string_view value) { XMLAttrs attrs; attrs[name] = value; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 208cab21d60..1f4d72d3926 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include "symbol-table.hh" @@ -112,7 +111,7 @@ class ExternalValueBase * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ - virtual bool operator ==(const ExternalValueBase & b) const; + virtual bool operator ==(const ExternalValueBase & b) const noexcept; /** * Print the value as JSON. Defaults to unconvertable, i.e. throws an error diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index 21c42567cb5..c7ed4c7af08 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,14 +1,9 @@ #include "fetch-settings.hh" -#include "config-global.hh" -namespace nix { +namespace nix::fetchers { -FetchSettings::FetchSettings() +Settings::Settings() { } -FetchSettings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - } diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 629967697da..f7cb34a0220 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -9,11 +9,11 @@ #include -namespace nix { +namespace nix::fetchers { -struct FetchSettings : public Config +struct Settings : public Config { - FetchSettings(); + Settings(); Setting accessTokens{this, {}, "access-tokens", R"( @@ -84,9 +84,14 @@ struct FetchSettings : public Config `narHash` attribute is specified, e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. )"}; -}; -// FIXME: don't use a global variable. -extern FetchSettings fetchSettings; + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )", + {}, true, Xp::Flakes}; +}; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 29496067832..dee1f687b2e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -35,9 +35,11 @@ nlohmann::json dumpRegisterInputSchemeInfo() { return res; } -Input Input::fromURL(const std::string & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const std::string & url, bool requireTree) { - return fromURL(parseURL(url), requireTree); + return fromURL(settings, parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -49,10 +51,12 @@ static void fixupInput(Input & input) input.getLastModified(); } -Input Input::fromURL(const ParsedURL & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) { for (auto & [_, inputScheme] : *inputSchemes) { - auto res = inputScheme->inputFromURL(url, requireTree); + auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; @@ -64,7 +68,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) throw Error("input '%s' is unsupported", url.url); } -Input Input::fromAttrs(Attrs && attrs) +Input Input::fromAttrs(const Settings & settings, Attrs && attrs) { auto schemeName = ({ auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); @@ -78,7 +82,7 @@ Input Input::fromAttrs(Attrs && attrs) // but not all of them. Doing this is to support those other // operations which are supposed to be robust on // unknown/uninterpretable inputs. - Input input; + Input input { settings }; input.attrs = attrs; fixupInput(input); return input; @@ -99,7 +103,7 @@ Input Input::fromAttrs(Attrs && attrs) if (name != "type" && allowedAttrs.count(name) == 0) throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(settings, attrs); if (!res) return raw(); res->scheme = inputScheme; fixupInput(*res); @@ -146,7 +150,7 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::operator ==(const Input & other) const +bool Input::operator ==(const Input & other) const noexcept { return attrs == other.attrs; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f9a..a5f9bdcc6d5 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -11,12 +11,16 @@ #include #include +#include "ref.hh" + namespace nix { class Store; class StorePath; struct SourceAccessor; } namespace nix::fetchers { struct InputScheme; +struct Settings; + /** * The `Input` object is generated by a specific fetcher, based on * user-supplied information, and contains @@ -28,6 +32,12 @@ struct Input { friend struct InputScheme; + const Settings * settings; + + Input(const Settings & settings) + : settings{&settings} + { } + std::shared_ptr scheme; // note: can be null Attrs attrs; @@ -42,16 +52,22 @@ public: * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromURL(const std::string & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree = true); /** * Create an `Input` from a an `Attrs`. * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromAttrs(Attrs && attrs); + static Input fromAttrs( + const Settings & settings, + Attrs && attrs); ParsedURL toURL() const; @@ -73,7 +89,7 @@ public: */ bool isLocked() const; - bool operator ==(const Input & other) const; + bool operator ==(const Input & other) const noexcept; bool contains(const Input & other) const; @@ -146,9 +162,13 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; + virtual std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + virtual std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const = 0; /** * What is the name of the scheme? diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 500064ceedc..ecc71ae471d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -115,10 +115,10 @@ git_oid hashToOID(const Hash & hash) return oid; } -Object lookupObject(git_repository * repo, const git_oid & oid) +Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t type = GIT_OBJECT_ANY) { Object obj; - if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + if (git_object_lookup(Setter(obj), repo, &oid, type)) { auto err = git_error_last(); throw Error("getting Git object '%s': %s", oid, err->message); } @@ -909,6 +909,61 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); } + void createHardlink(const CanonPath & path, const CanonPath & target) override + { + std::vector pathComponents; + for (auto & c : path) + pathComponents.emplace_back(c); + + if (!prepareDirs(pathComponents, false)) return; + + // We can't just look up the path from the start of the root, since + // some parent directories may not have finished yet, so we compute + // a relative path that helps us find the right git_tree_builder or object. + auto relTarget = CanonPath(path).parent()->makeRelative(target); + + auto dir = pendingDirs.rbegin(); + + // For each ../ component at the start, go up one directory. + // CanonPath::makeRelative() always puts all .. elements at the start, + // so they're all handled by this loop: + std::string_view relTargetLeft(relTarget); + while (hasPrefix(relTargetLeft, "../")) { + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + ++dir; + relTargetLeft = relTargetLeft.substr(3); + } + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + + // Look up the remainder of the target, starting at the + // top-most `git_treebuilder`. + std::variant curDir{dir->builder.get()}; + Object tree; // needed to keep `entry` alive + const git_tree_entry * entry = nullptr; + + for (auto & c : CanonPath(relTargetLeft)) { + if (auto builder = std::get_if(&curDir)) { + assert(*builder); + if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } else if (auto oid = std::get_if(&curDir)) { + tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); + if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } + } + + assert(entry); + + addToTree(*pathComponents.rbegin(), + *git_tree_entry_id(entry), + git_tree_entry_filemode(entry)); + } + Hash sync() override { updateBuilders({}); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 29d79955480..495916f6283 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -7,7 +7,7 @@ namespace nix { namespace fetchers { struct PublicKey; } -struct GitFileSystemObjectSink : FileSystemObjectSink +struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { /** * Flush builder and return a final Git hash. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 184c1383e58..076c757c5f5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -164,7 +164,9 @@ static const Hash nullRev{HashAlgorithm::SHA1}; struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -190,7 +192,7 @@ struct GitInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } @@ -222,7 +224,9 @@ struct GitInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { for (auto & [name, _] : attrs) if (name == "verifyCommit" @@ -238,7 +242,7 @@ struct GitInputScheme : InputScheme throw BadURL("invalid Git branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; auto url = fixGitURL(getStrAttr(attrs, "url")); parseURL(url); @@ -366,13 +370,13 @@ struct GitInputScheme : InputScheme /* URL of the repo, or its path if isLocal. Never a `file` URL. */ std::string url; - void warnDirty() const + void warnDirty(const Settings & settings) const { if (workdirInfo.isDirty) { - if (!fetchSettings.allowDirty) + if (!settings.allowDirty) throw Error("Git tree '%s' is dirty", url); - if (fetchSettings.warnDirty) + if (settings.warnDirty) warn("Git tree '%s' is dirty", url); } } @@ -653,7 +657,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); attrs.insert_or_assign("submodules", Explicit{ true }); attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -711,7 +715,7 @@ struct GitInputScheme : InputScheme // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out // attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -743,7 +747,7 @@ struct GitInputScheme : InputScheme verifyCommit(input, repo); } else { - repoInfo.warnDirty(); + repoInfo.warnDirty(*input.settings); if (repoInfo.workdirInfo.headRev) { input.attrs.insert_or_assign("dirtyRev", diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ddb41e63f9f..2968d2df2a4 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -31,7 +31,9 @@ struct GitArchiveInputScheme : InputScheme { virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const fetchers::Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != schemeName()) return {}; @@ -90,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme if (ref && rev) throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); @@ -119,12 +121,14 @@ struct GitArchiveInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const fetchers::Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -168,18 +172,20 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const std::string & host) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host) const { - auto tokens = fetchSettings.accessTokens.get(); + auto tokens = settings.accessTokens.get(); if (auto token = get(tokens, host)) return *token; return {}; } - Headers makeHeadersWithAuthTokens(const std::string & host) const + Headers makeHeadersWithAuthTokens( + const fetchers::Settings & settings, + const std::string & host) const { Headers headers; - auto accessToken = getAccessToken(host); + auto accessToken = getAccessToken(settings, host); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -295,7 +301,7 @@ struct GitArchiveInputScheme : InputScheme locking. FIXME: in the future, we may want to require a Git tree hash instead of a NAR hash. */ return input.getRev().has_value() - && (fetchSettings.trustTarballsFromGitForges || + && (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value()); } @@ -352,7 +358,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getOwner(input), getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -369,7 +375,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -389,7 +395,7 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) const override { auto host = getHost(input); - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -426,7 +432,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -456,7 +462,7 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } @@ -464,7 +470,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -496,7 +502,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); std::string refUri; if (ref == "HEAD") { @@ -543,14 +549,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); - Input::fromURL(fmt("git+https://%s/%s/%s", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index ba507863138..2e5cd82c78d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -8,7 +8,9 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; @@ -41,7 +43,7 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -65,13 +67,15 @@ struct IndirectInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 198795caa23..3feb3cb19ba 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -45,7 +45,9 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -68,7 +70,7 @@ struct MercurialInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } std::string_view schemeName() const override @@ -88,7 +90,9 @@ struct MercurialInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { parseURL(getStrAttr(attrs, "url")); @@ -97,7 +101,7 @@ struct MercurialInputScheme : InputScheme throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -182,10 +186,10 @@ struct MercurialInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!fetchSettings.allowDirty) + if (!input.settings->allowDirty) throw Error("Mercurial tree '%s' is unclean", actualUrl); - if (fetchSettings.warnDirty) + if (input.settings->warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 68958d55971..fca0df84b10 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -7,14 +7,16 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("path", url.path); @@ -54,11 +56,13 @@ struct PathInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "path"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 52cbac5e0a0..3c893c8ea33 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,7 +1,7 @@ +#include "fetch-settings.hh" #include "registry.hh" #include "tarball.hh" #include "users.hh" -#include "config-global.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -11,12 +11,13 @@ namespace nix::fetchers { std::shared_ptr Registry::read( + const Settings & settings, const Path & path, RegistryType type) { - auto registry = std::make_shared(type); + auto registry = std::make_shared(settings, type); if (!pathExists(path)) - return std::make_shared(type); + return std::make_shared(settings, type); try { @@ -36,8 +37,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = Input::fromAttrs(jsonToAttrs(i["from"])), - .to = Input::fromAttrs(std::move(toAttrs)), + .from = Input::fromAttrs(settings, jsonToAttrs(i["from"])), + .to = Input::fromAttrs(settings, std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -106,10 +107,10 @@ static Path getSystemRegistryPath() return settings.nixConfDir + "/registry.json"; } -static std::shared_ptr getSystemRegistry() +static std::shared_ptr getSystemRegistry(const Settings & settings) { static auto systemRegistry = - Registry::read(getSystemRegistryPath(), Registry::System); + Registry::read(settings, getSystemRegistryPath(), Registry::System); return systemRegistry; } @@ -118,25 +119,24 @@ Path getUserRegistryPath() return getConfigDir() + "/nix/registry.json"; } -std::shared_ptr getUserRegistry() +std::shared_ptr getUserRegistry(const Settings & settings) { static auto userRegistry = - Registry::read(getUserRegistryPath(), Registry::User); + Registry::read(settings, getUserRegistryPath(), Registry::User); return userRegistry; } -std::shared_ptr getCustomRegistry(const Path & p) +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p) { static auto customRegistry = - Registry::read(p, Registry::Custom); + Registry::read(settings, p, Registry::Custom); return customRegistry; } -static std::shared_ptr flagRegistry = - std::make_shared(Registry::Flag); - -std::shared_ptr getFlagRegistry() +std::shared_ptr getFlagRegistry(const Settings & settings) { + static auto flagRegistry = + std::make_shared(settings, Registry::Flag); return flagRegistry; } @@ -145,30 +145,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs) { - flagRegistry->add(from, to, extraAttrs); + getFlagRegistry(*from.settings)->add(from, to, extraAttrs); } -struct RegistrySettings : Config -{ - Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - R"( - Path or URI of the global flake registry. - - When empty, disables the global flake registry. - )", - {}, true, Xp::Flakes}; -}; - -RegistrySettings registrySettings; - -static GlobalConfig::Register rRegistrySettings(®istrySettings); - -static std::shared_ptr getGlobalRegistry(ref store) +static std::shared_ptr getGlobalRegistry(const Settings & settings, ref store) { static auto reg = [&]() { - auto path = registrySettings.flakeRegistry.get(); + auto path = settings.flakeRegistry.get(); if (path == "") { - return std::make_shared(Registry::Global); // empty registry + return std::make_shared(settings, Registry::Global); // empty registry } if (!hasPrefix(path, "/")) { @@ -178,19 +163,19 @@ static std::shared_ptr getGlobalRegistry(ref store) path = store->toRealPath(storePath); } - return Registry::read(path, Registry::Global); + return Registry::read(settings, path, Registry::Global); }(); return reg; } -Registries getRegistries(ref store) +Registries getRegistries(const Settings & settings, ref store) { Registries registries; - registries.push_back(getFlagRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getSystemRegistry()); - registries.push_back(getGlobalRegistry(store)); + registries.push_back(getFlagRegistry(settings)); + registries.push_back(getUserRegistry(settings)); + registries.push_back(getSystemRegistry(settings)); + registries.push_back(getGlobalRegistry(settings, store)); return registries; } @@ -207,7 +192,7 @@ std::pair lookupInRegistries( n++; if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); - for (auto & registry : getRegistries(store)) { + for (auto & registry : getRegistries(*input.settings, store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index f57ab1e6b59..0d68ac395e9 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -10,6 +10,8 @@ namespace nix::fetchers { struct Registry { + const Settings & settings; + enum RegistryType { Flag = 0, User = 1, @@ -29,11 +31,13 @@ struct Registry std::vector entries; - Registry(RegistryType type) - : type(type) + Registry(const Settings & settings, RegistryType type) + : settings{settings} + , type{type} { } static std::shared_ptr read( + const Settings & settings, const Path & path, RegistryType type); void write(const Path & path); @@ -48,13 +52,13 @@ struct Registry typedef std::vector> Registries; -std::shared_ptr getUserRegistry(); +std::shared_ptr getUserRegistry(const Settings & settings); -std::shared_ptr getCustomRegistry(const Path & p); +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p); Path getUserRegistryPath(); -Registries getRegistries(ref store); +Registries getRegistries(const Settings & settings, ref store); void overrideRegistry( const Input & from, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index aa8ff652f07..55db3eafbc1 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -2,12 +2,10 @@ #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" -#include "globals.hh" #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" #include "types.hh" -#include "split.hh" #include "store-path-accessor.hh" #include "store-api.hh" #include "git-utils.hh" @@ -214,12 +212,14 @@ struct CurlInputScheme : InputScheme static const std::set specialParams; - std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & _url, bool requireTree) const override { if (!isValidURL(_url, requireTree)) return std::nullopt; - Input input; + Input input{settings}; auto url = _url; @@ -267,9 +267,11 @@ struct CurlInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { - Input input; + Input input{settings}; input.attrs = attrs; //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); @@ -349,7 +351,7 @@ struct TarballInputScheme : CurlInputScheme result.accessor->setPathDisplay("«" + input.to_string() + "»"); if (result.immutableUrl) { - auto immutableInput = Input::fromURL(*result.immutableUrl); + auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl); // FIXME: would be nice to support arbitrary flakerefs // here, e.g. git flakes. if (immutableInput.getType() != "tarball") diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index ba0dfd6230a..d9bdd123d58 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -1,11 +1,12 @@ #pragma once -#include "types.hh" -#include "path.hh" -#include "hash.hh" - #include +#include "hash.hh" +#include "path.hh" +#include "ref.hh" +#include "types.hh" + namespace nix { class Store; struct SourceAccessor; diff --git a/src/libflake/flake-settings.cc b/src/libflake/flake-settings.cc deleted file mode 100644 index ba97e0ce723..00000000000 --- a/src/libflake/flake-settings.cc +++ /dev/null @@ -1,12 +0,0 @@ -#include "flake-settings.hh" -#include "config-global.hh" - -namespace nix { - -FlakeSettings::FlakeSettings() {} - -FlakeSettings flakeSettings; - -static GlobalConfig::Register rFlakeSettings(&flakeSettings); - -} diff --git a/src/libflake/flake/config.cc b/src/libflake/flake/config.cc index 49859535939..e526cdddfa3 100644 --- a/src/libflake/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -1,6 +1,6 @@ #include "users.hh" #include "config-global.hh" -#include "flake-settings.hh" +#include "flake/settings.hh" #include "flake.hh" #include @@ -30,7 +30,7 @@ static void writeTrustedList(const TrustedList & trustedList) writeFile(path, nlohmann::json(trustedList).dump()); } -void ConfigFile::apply() +void ConfigFile::apply(const Settings & flakeSettings) { std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; @@ -47,11 +47,11 @@ void ConfigFile::apply() else if (auto* b = std::get_if>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil + valueS = dropEmptyInitThenConcatStringsSep(" ", *ss); // FIXME: evil else assert(false); - if (!whitelist.count(baseName) && !nix::flakeSettings.acceptFlakeConfig) { + if (!whitelist.count(baseName) && !flakeSettings.acceptFlakeConfig) { bool trusted = false; auto trustedList = readTrustedList(); auto tlname = get(trustedList, name); diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 6f47b599229..627dfe83093 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -9,7 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" -#include "flake-settings.hh" +#include "flake/settings.hh" #include "value-to-json.hh" #include "local-fs-store.hh" @@ -98,7 +98,7 @@ static std::map parseFlakeInputs( const std::optional & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, + std::string_view inputName, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -164,7 +164,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (attrs.count("type")) try { - input.ref = FlakeRef::fromAttrs(attrs); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, attrs); } catch (Error & e) { e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; @@ -174,11 +174,11 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(state.fetchSettings, *url, baseDir, true, input.isFlake); } if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}}); return input; } @@ -244,7 +244,7 @@ static Flake readFlake( for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(state.symbols[formal.name], FlakeInput { - .ref = parseFlakeRef(state.symbols[formal.name]) + .ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name])) }); } } @@ -329,16 +329,19 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } -static LockFile readLockFile(const SourcePath & lockFilePath) +static LockFile readLockFile( + const fetchers::Settings & fetchSettings, + const SourcePath & lockFilePath) { return lockFilePath.pathExists() - ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath)) + ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) : LockFile(); } /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) @@ -347,21 +350,22 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); if (lockFlags.applyNixConfig) { - flake.config.apply(); + flake.config.apply(settings); state.store->setOptions(); } try { - if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } auto oldLockFile = readLockFile( + state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -597,7 +601,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -660,7 +664,7 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { - if (fetchSettings.warnDirty) + if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) @@ -692,7 +696,7 @@ LockedFlake lockFlake( if (lockFlags.commitLockFile) { std::string cm; - cm = flakeSettings.commitLockFileSummary.get(); + cm = settings.commitLockFileSummary.get(); if (cm == "") { cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); @@ -800,45 +804,48 @@ void callFlake(EvalState & state, state.callFunction(*vTmp1, vOverrides, vRes, noPos); } -static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) +void initLib(const Settings & settings) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (state.settings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); - - callFlake(state, - lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !state.settings.pureEval && flakeSettings.useRegistries, - .allowUnlocked = !state.settings.pureEval, - }), - v); -} - -static RegisterPrimOp r2({ - .name = "__getFlake", - .args = {"args"}, - .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: - - ```nix - (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix - ``` - - Unless impure evaluation is allowed (`--impure`), the flake reference - must be "locked", e.g. contain a Git revision or content hash. An - example of an unlocked usage is: + auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value * * args, Value & v) + { + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + auto flakeRef = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); + + callFlake(state, + lockFlake(settings, state, flakeRef, + LockFlags { + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }), + v); + }; - ```nix - (builtins.getFlake "github:edolstra/dwarffs").rev - ``` - )", - .fun = prim_getFlake, - .experimentalFeature = Xp::Flakes, -}); + RegisterPrimOp::primOps->push_back({ + .name = "__getFlake", + .args = {"args"}, + .doc = R"( + Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + + ```nix + (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + ``` + + Unless impure evaluation is allowed (`--impure`), the flake reference + must be "locked", e.g. contain a Git revision or content hash. An + example of an unlocked usage is: + + ```nix + (builtins.getFlake "github:edolstra/dwarffs").rev + ``` + )", + .fun = prim_getFlake, + .experimentalFeature = Xp::Flakes, + }); +} static void prim_parseFlakeRef( EvalState & state, @@ -848,7 +855,7 @@ static void prim_parseFlakeRef( { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.parseFlakeRef")); - auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto attrs = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true).toAttrs(); auto binds = state.buildBindings(attrs.size()); for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); @@ -913,7 +920,7 @@ static void prim_flakeRefToString( showType(*attr.value)).debugThrow(); } } - auto flakeRef = FlakeRef::fromAttrs(attrs); + auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); v.mkString(flakeRef.to_string()); } diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 1ba085f0f46..cce17009ce3 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -12,6 +12,16 @@ class EvalState; namespace flake { +struct Settings; + +/** + * Initialize `libnixflake` + * + * So far, this registers the `builtins.getFlake` primop, which depends + * on the choice of `flake:Settings`. + */ +void initLib(const Settings & settings); + struct FlakeInput; typedef std::map FlakeInputs; @@ -57,7 +67,7 @@ struct ConfigFile std::map settings; - void apply(); + void apply(const Settings & settings); }; /** @@ -194,6 +204,7 @@ struct LockFlags }; LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 6e4aad64d28..a57fce9f319 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -36,11 +36,6 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -bool FlakeRef::operator ==(const FlakeRef & other) const -{ - return input == other.input && subdir == other.subdir; -} - FlakeRef FlakeRef::resolve(ref store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); @@ -48,28 +43,32 @@ FlakeRef FlakeRef::resolve(ref store) const } FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; } std::optional maybeParseFlakeRef( - const std::string & url, const std::optional & baseDir) + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir) { try { - return parseFlakeRef(url, baseDir); + return parseFlakeRef(fetchSettings, url, baseDir); } catch (Error &) { return {}; } } std::pair parsePathFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -166,7 +165,7 @@ std::pair parsePathFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")), fragment); } @@ -185,13 +184,14 @@ std::pair parsePathFlakeRefWithFragment( attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); }; /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ -std::optional> parseFlakeIdRef( +static std::optional> parseFlakeIdRef( + const fetchers::Settings & fetchSettings, const std::string & url, bool isFlake ) @@ -213,7 +213,7 @@ std::optional> parseFlakeIdRef( }; return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -221,6 +221,7 @@ std::optional> parseFlakeIdRef( } std::optional> parseURLFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool isFlake @@ -236,7 +237,7 @@ std::optional> parseURLFlakeRef( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = fetchers::Input::fromURL(parsedURL, isFlake); + auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); input.parent = baseDir; return std::make_pair( @@ -245,6 +246,7 @@ std::optional> parseURLFlakeRef( } std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -254,31 +256,34 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - if (auto res = parseFlakeIdRef(url, isFlake)) { + if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; - } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); } } std::optional> maybeParseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir) { try { - return parseFlakeRefWithFragment(url, baseDir); + return parseFlakeRefWithFragment(fetchSettings, url, baseDir); } catch (Error & e) { return {}; } } -FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +FlakeRef FlakeRef::fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs) { auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::Input::fromAttrs(fetchSettings, std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } @@ -289,13 +294,16 @@ std::pair FlakeRef::fetchTree(ref store) const } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, + std::string { prefix }, baseDir, allowMissing, isFlake); return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 04c812ed099..1064538a72a 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -1,14 +1,12 @@ #pragma once ///@file +#include + #include "types.hh" -#include "hash.hh" #include "fetchers.hh" #include "outputs-spec.hh" -#include -#include - namespace nix { class Store; @@ -48,7 +46,7 @@ struct FlakeRef */ Path subdir; - bool operator==(const FlakeRef & other) const; + bool operator ==(const FlakeRef & other) const = default; FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) @@ -61,7 +59,9 @@ struct FlakeRef FlakeRef resolve(ref store) const; - static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + static FlakeRef fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs); std::pair fetchTree(ref store) const; }; @@ -72,6 +72,7 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -81,12 +82,15 @@ FlakeRef parseFlakeRef( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional maybeParseFlake( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -96,12 +100,15 @@ std::pair parseFlakeRefWithFragment( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional> maybeParseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index d252214dd2b..80f14ff6fa3 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -1,6 +1,7 @@ +#include + #include "lockfile.hh" #include "store-api.hh" -#include "url-parts.hh" #include #include @@ -8,9 +9,12 @@ #include #include +#include "strings.hh" + namespace nix::flake { -FlakeRef getFlakeRef( +static FlakeRef getFlakeRef( + const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) @@ -26,15 +30,17 @@ FlakeRef getFlakeRef( attrs.insert_or_assign(k.first, k.second); } } - return FlakeRef::fromAttrs(attrs); + return FlakeRef::fromAttrs(fetchSettings, attrs); } throw Error("attribute '%s' missing in lock file", attr); } -LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(json, "original", nullptr)) +LockedNode::LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) @@ -84,7 +90,9 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return doFind(root, path, visited); } -LockFile::LockFile(std::string_view contents, std::string_view path) +LockFile::LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path) { auto json = nlohmann::json::parse(contents); @@ -113,7 +121,7 @@ LockFile::LockFile(std::string_view contents, std::string_view path) auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(*jsonNode2); + auto input = make_ref(fetchSettings, *jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } @@ -243,11 +251,6 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON().first == other.toJSON().first; } -bool LockFile::operator !=(const LockFile & other) const -{ - return !(*this == other); -} - InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh index 7e62e6d0970..841931c1119 100644 --- a/src/libflake/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -45,7 +45,9 @@ struct LockedNode : Node : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } - LockedNode(const nlohmann::json & json); + LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json); StorePath computeStorePath(Store & store) const; }; @@ -55,7 +57,9 @@ struct LockFile ref root = make_ref(); LockFile() {}; - LockFile(std::string_view contents, std::string_view path); + LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path); typedef std::map, std::string> KeyMap; @@ -70,9 +74,6 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; - // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 - // that is still the default on aarch64-linux) - bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); diff --git a/src/libflake/flake/settings.cc b/src/libflake/flake/settings.cc new file mode 100644 index 00000000000..6a0294e6229 --- /dev/null +++ b/src/libflake/flake/settings.cc @@ -0,0 +1,7 @@ +#include "flake/settings.hh" + +namespace nix::flake { + +Settings::Settings() {} + +} diff --git a/src/libflake/flake-settings.hh b/src/libflake/flake/settings.hh similarity index 86% rename from src/libflake/flake-settings.hh rename to src/libflake/flake/settings.hh index f97c175e8a3..fee247a7d2f 100644 --- a/src/libflake/flake-settings.hh +++ b/src/libflake/flake/settings.hh @@ -10,11 +10,11 @@ #include -namespace nix { +namespace nix::flake { -struct FlakeSettings : public Config +struct Settings : public Config { - FlakeSettings(); + Settings(); Setting useRegistries{ this, @@ -47,7 +47,4 @@ struct FlakeSettings : public Config Xp::Flakes}; }; -// TODO: don't use a global variable. -extern FlakeSettings flakeSettings; - } diff --git a/src/libflake/meson.build b/src/libflake/meson.build index d3c3d3079cf..38d70c678a5 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -42,21 +42,21 @@ add_project_arguments( subdir('build-utils-meson/diagnostics') sources = files( - 'flake-settings.cc', 'flake/config.cc', 'flake/flake.cc', 'flake/flakeref.cc', - 'flake/url-name.cc', 'flake/lockfile.cc', + 'flake/settings.cc', + 'flake/url-name.cc', ) include_dirs = [include_directories('.')] headers = files( - 'flake-settings.hh', 'flake/flake.hh', 'flake/flakeref.hh', 'flake/lockfile.hh', + 'flake/settings.hh', 'flake/url-name.hh', ) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index fc55fe3f1b2..a224f8d92a4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -8,7 +8,6 @@ #include "signals.hh" #include -#include #include #include @@ -23,6 +22,8 @@ #include +#include "exit.hh" +#include "strings.hh" namespace nix { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index aa44e1321fa..712b404d3e1 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -8,13 +8,9 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" -#include "exit.hh" #include -#include - - namespace nix { int handleExceptions(const std::string & programName, std::function fun); diff --git a/src/libstore/auth-tunnel.hh b/src/libstore/auth-tunnel.hh index 333172bee24..e53cd2475a6 100644 --- a/src/libstore/auth-tunnel.hh +++ b/src/libstore/auth-tunnel.hh @@ -2,6 +2,7 @@ #include "file-descriptor.hh" #include "worker-protocol.hh" +#include "ref.hh" #include diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 18f519c5c61..96cbfd62fff 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -2,17 +2,7 @@ namespace nix { -GENERATE_CMP_EXT( - , - BuildResult, - me->status, - me->errorMsg, - me->timesBuilt, - me->isNonDeterministic, - me->builtOutputs, - me->startTime, - me->stopTime, - me->cpuUser, - me->cpuSystem); +bool BuildResult::operator==(const BuildResult &) const noexcept = default; +std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; } diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 3636ad3a4c3..8c66cfeb353 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,7 +3,6 @@ #include "realisation.hh" #include "derived-path.hh" -#include "comparator.hh" #include #include @@ -101,7 +100,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; - DECLARE_CMP(BuildResult); + bool operator ==(const BuildResult &) const noexcept; + std::strong_ordering operator <=>(const BuildResult &) const noexcept; bool success() { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 64b8495e1bb..010f905d640 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -32,8 +32,18 @@ #include +#include "strings.hh" + namespace nix { +Goal::Co DerivationGoal::init() { + if (useDerivation) { + co_return getDerivation(); + } else { + co_return haveDerivation(); + } +} + DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) @@ -42,7 +52,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -63,7 +72,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -107,13 +115,9 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, std::move(ex)); -} - - -void DerivationGoal::work() -{ - (this->*state)(); + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); } void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) @@ -137,7 +141,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() +Goal::Co DerivationGoal::getDerivation() { trace("init"); @@ -145,23 +149,22 @@ void DerivationGoal::getDerivation() exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; + co_return loadDerivation(); } addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - state = &DerivationGoal::loadDerivation; + co_await Suspend{}; + co_return loadDerivation(); } -void DerivationGoal::loadDerivation() +Goal::Co DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; + co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); } /* `drvPath' should already be a root, but let's be on the safe @@ -183,11 +186,11 @@ void DerivationGoal::loadDerivation() } assert(drv); - haveDerivation(); + co_return haveDerivation(); } -void DerivationGoal::haveDerivation() +Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); @@ -215,8 +218,7 @@ void DerivationGoal::haveDerivation() }); } - gaveUpOnSubstitution(); - return; + co_return gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -238,8 +240,7 @@ void DerivationGoal::haveDerivation() /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths @@ -266,24 +267,21 @@ void DerivationGoal::haveDerivation() } } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return outputsSubstitutionTried(); } -void DerivationGoal::outputsSubstitutionTried() +Goal::Co DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, {}, + co_return done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); - return; } /* If the substitutes form an incomplete closure, then we should @@ -317,32 +315,29 @@ void DerivationGoal::outputsSubstitutionTried() if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - haveDerivation(); - return; + co_return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); if (buildMode == bmNormal && allValid) { - done(BuildResult::Substituted, std::move(validOutputs)); - return; + co_return done(BuildResult::Substituted, std::move(validOutputs)); } if (buildMode == bmRepair && allValid) { - repairClosure(); - return; + co_return repairClosure(); } if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); + co_return gaveUpOnSubstitution(); } /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() +Goal::Co DerivationGoal::gaveUpOnSubstitution() { /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -403,14 +398,12 @@ void DerivationGoal::gaveUpOnSubstitution() addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return inputsRealised(); } -void DerivationGoal::repairClosure() +Goal::Co DerivationGoal::repairClosure() { assert(!drv->type().isImpure()); @@ -464,41 +457,39 @@ void DerivationGoal::repairClosure() } if (waitees.empty()) { - done(BuildResult::AlreadyValid, assertPathValidity()); - return; + co_return done(BuildResult::AlreadyValid, assertPathValidity()); + } else { + co_await Suspend{}; + co_return closureRepaired(); } - - state = &DerivationGoal::closureRepaired; } -void DerivationGoal::closureRepaired() +Goal::Co DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid, assertPathValidity()); + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } -void DerivationGoal::inputsRealised() +Goal::Co DerivationGoal::inputsRealised() { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, {}, Error( + co_return done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); - return; } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - haveDerivation(); - return; + co_return haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -564,8 +555,8 @@ void DerivationGoal::inputsRealised() pathResolved, wantedOutputs, buildMode); addWaitee(resolvedDrvGoal); - state = &DerivationGoal::resolvedFinished; - return; + co_await Suspend{}; + co_return resolvedFinished(); } std::function::ChildNode &)> accumInputPaths; @@ -629,8 +620,9 @@ void DerivationGoal::inputsRealised() /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryToBuild(); } void DerivationGoal::started() @@ -655,7 +647,7 @@ void DerivationGoal::started() worker.updateProgress(); } -void DerivationGoal::tryToBuild() +Goal::Co DerivationGoal::tryToBuild() { trace("trying to build"); @@ -691,7 +683,8 @@ void DerivationGoal::tryToBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryToBuild(); } actLock.reset(); @@ -708,8 +701,7 @@ void DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -735,9 +727,9 @@ void DerivationGoal::tryToBuild() EOF from the hook. */ actLock.reset(); buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; started(); - return; + co_await Suspend{}; + co_return buildDone(); case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ @@ -746,7 +738,8 @@ void DerivationGoal::tryToBuild() fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); case rpDecline: /* We should do it ourselves. */ break; @@ -755,11 +748,12 @@ void DerivationGoal::tryToBuild() actLock.reset(); - state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryLocalBuild(); } -void DerivationGoal::tryLocalBuild() { +Goal::Co DerivationGoal::tryLocalBuild() { throw Error( R"( Unable to build with a primary store that isn't a local store; @@ -936,7 +930,7 @@ void runPostBuildHook( }); } -void DerivationGoal::buildDone() +Goal::Co DerivationGoal::buildDone() { trace("build done"); @@ -1031,7 +1025,7 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); - done(BuildResult::Built, std::move(builtOutputs)); + co_return done(BuildResult::Built, std::move(builtOutputs)); } catch (BuildError & e) { outputLocks.unlock(); @@ -1056,12 +1050,11 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, std::move(e)); - return; + co_return done(st, {}, std::move(e)); } } -void DerivationGoal::resolvedFinished() +Goal::Co DerivationGoal::resolvedFinished() { trace("resolved derivation finished"); @@ -1129,7 +1122,7 @@ void DerivationGoal::resolvedFinished() if (status == BuildResult::AlreadyValid) status = BuildResult::ResolvesToAlreadyValid; - done(status, std::move(builtOutputs)); + co_return done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() @@ -1323,7 +1316,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); - done( + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done( BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); @@ -1529,7 +1524,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -void DerivationGoal::done( +Goal::Done DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional ex) @@ -1566,7 +1561,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 04f13aedd17..ad3d9ca2acf 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -194,9 +194,6 @@ struct DerivationGoal : public Goal */ std::optional derivationType; - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - BuildMode buildMode; std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; @@ -227,8 +224,6 @@ struct DerivationGoal : public Goal std::string key() override; - void work() override; - /** * Add wanted outputs to an already existing derivation goal. */ @@ -237,18 +232,19 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - virtual void tryLocalBuild(); - void buildDone(); + Co init() override; + Co getDerivation(); + Co loadDerivation(); + Co haveDerivation(); + Co outputsSubstitutionTried(); + Co gaveUpOnSubstitution(); + Co closureRepaired(); + Co inputsRealised(); + Co tryToBuild(); + virtual Co tryLocalBuild(); + Co buildDone(); - void resolvedFinished(); + Co resolvedFinished(); /** * Is the build hook willing to perform the build? @@ -329,11 +325,11 @@ struct DerivationGoal : public Goal */ virtual void killChild(); - void repairClosure(); + Co repairClosure(); void started(); - void done( + Done done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 13a07e4ea38..02284d93c1a 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -14,146 +14,135 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -void DrvOutputSubstitutionGoal::init() +Goal::Co DrvOutputSubstitutionGoal::init() { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - amDone(ecSuccess); - return; + co_return amDone(ecSuccess); } - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - tryNext(); -} - -void DrvOutputSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + + bool substituterFailed = false; + + for (auto sub : subs) { + trace("trying next substituter"); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + auto outPipe = std::make_shared(); + #ifndef _WIN32 + outPipe->create(); + #else + outPipe->createAsyncPipe(worker.ioport.get()); + #endif + + auto promise = std::make_shared>>(); + + sub->queryRealisation( + id, + { [outPipe(outPipe), promise(promise)](std::future> res) { + try { + Finally updateStats([&]() { outPipe->writeSide.close(); }); + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), { + #ifndef _WIN32 + outPipe->readSide.get() + #else + &outPipe + #endif + }, true, false); + + co_await Suspend{}; + + worker.childTerminated(this); + + /* + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ + std::shared_ptr outputInfo; + + try { + outputInfo = promise->get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; } - return; - } - - sub = subs.front(); - subs.pop_front(); - - // FIXME: Make async - // outputInfo = sub->queryRealisation(id); - - /* The callback of the curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - downloadState = std::make_shared(); -#ifndef _WIN32 - downloadState->outPipe.create(); -#else - downloadState->outPipe.createAsyncPipe(worker.ioport.get()); -#endif - - sub->queryRealisation( - id, - { [downloadState(downloadState)](std::future> res) { - try { - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); - downloadState->promise.set_value(res.get()); - } catch (...) { - downloadState->promise.set_exception(std::current_exception()); + if (!outputInfo) continue; + + bool failed = false; + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + failed = true; + break; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); } - } }); - - worker.childStarted(shared_from_this(), { -#ifndef _WIN32 - downloadState->outPipe.readSide.get() -#else - &downloadState->outPipe -#endif - }, true, false); - - state = &DrvOutputSubstitutionGoal::realisationFetched; -} + } -void DrvOutputSubstitutionGoal::realisationFetched() -{ - worker.childTerminated(this); + if (failed) continue; - try { - outputInfo = downloadState->promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - substituterFailed = true; + co_return realisationFetched(outputInfo, sub); } - if (!outputInfo) { - return tryNext(); - } + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->getUri(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath) - ); - tryNext(); - return; - } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); - } + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); } + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); +} + +Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr outputInfo, nix::ref sub) { addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); - if (waitees.empty()) outPathValid(); - else state = &DrvOutputSubstitutionGoal::outPathValid; -} + if (!waitees.empty()) co_await Suspend{}; -void DrvOutputSubstitutionGoal::outPathValid() -{ - assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; + co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); } worker.store.registerDrvOutput(*outputInfo); - finished(); -} -void DrvOutputSubstitutionGoal::finished() -{ trace("finished"); - amDone(ecSuccess); + co_return amDone(ecSuccess); } std::string DrvOutputSubstitutionGoal::key() @@ -163,14 +152,9 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -void DrvOutputSubstitutionGoal::work() -{ - (this->*state)(); -} - void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) { - if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 6967ca84ff8..80705492662 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -27,52 +27,19 @@ class DrvOutputSubstitutionGoal : public Goal { */ DrvOutput id; - /** - * The realisation corresponding to the given output id. - * Will be filled once we can get it. - */ - std::shared_ptr outputInfo; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - struct DownloadState - { - MuxablePipe outPipe; - std::promise> promise; - }; - - std::shared_ptr downloadState; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - public: DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); typedef void (DrvOutputSubstitutionGoal::*GoalState)(); GoalState state; - void init(); - void tryNext(); - void realisationFetched(); - void outPathValid(); - void finished(); + Co init() override; + Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); void timedOut(Error && ex) override { abort(); }; std::string key() override; - void work() override; void handleEOF(Descriptor fd) override; JobCategory jobCategory() const override { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 784f618c1a0..4c1373bfaff 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -4,6 +4,7 @@ # include "derivation-goal.hh" #endif #include "local-store.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index f8db9828076..9a16da14555 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -3,6 +3,97 @@ namespace nix { +using Co = nix::Goal::Co; +using promise_type = nix::Goal::promise_type; +using handle_type = nix::Goal::handle_type; +using Suspend = nix::Goal::Suspend; + +Co::Co(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +void Co::operator=(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +Co::~Co() { + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } +} + +Co promise_type::get_return_object() { + auto handle = handle_type::from_promise(*this); + return Co{handle}; +}; + +std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept { + auto& p = h.promise(); + auto goal = p.goal; + assert(goal); + goal->trace("in final_awaiter"); + auto c = std::move(p.continuation); + + if (c) { + // We still have a continuation, i.e. work to do. + // We assert that the goal is still busy. + assert(goal->exitCode == ecBusy); + assert(goal->top_co); // Goal must have an active coroutine. + assert(goal->top_co->handle == h); // The active coroutine must be us. + assert(p.alive); // We must not have been destructed. + + // we move continuation to the top, + // note: previous top_co is actually h, so by moving into it, + // we're calling the destructor on h, DON'T use h and p after this! + + // We move our continuation into `top_co`, i.e. the marker for the active continuation. + // By doing this we destruct the old `top_co`, i.e. us, so `h` can't be used anymore. + // Be careful not to access freed memory! + goal->top_co = std::move(c); + + // We resume `top_co`. + return goal->top_co->handle; + } else { + // We have no continuation, i.e. no more work to do, + // so the goal must not be busy anymore. + assert(goal->exitCode != ecBusy); + + // We reset `top_co` for good measure. + p.goal->top_co = {}; + + // We jump to the noop coroutine, which doesn't do anything and immediately suspends. + // This passes control back to the caller of goal.work(). + return std::noop_coroutine(); + } +} + +void promise_type::return_value(Co&& next) { + goal->trace("return_value(Co&&)"); + // Save old continuation. + auto old_continuation = std::move(continuation); + // We set next as our continuation. + continuation = std::move(next); + // We set next's goal, and thus it must not have one already. + assert(!continuation->handle.promise().goal); + continuation->handle.promise().goal = goal; + // Nor can next have a continuation, as we set it to our old one. + assert(!continuation->handle.promise().continuation); + continuation->handle.promise().continuation = std::move(old_continuation); +} + +std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) { + assert(handle); // we must be a valid coroutine + auto& p = handle.promise(); + assert(!p.continuation); // we must have no continuation + assert(!p.goal); // we must not have a goal yet + auto goal = caller.promise().goal; + assert(goal); + p.goal = goal; + p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller) + goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this! + return p.goal->top_co->handle; // we execute ourselves +} bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { std::string s1 = a->key(); @@ -75,10 +166,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } } - -void Goal::amDone(ExitCode result, std::optional ex) +Goal::Done Goal::amDone(ExitCode result, std::optional ex) { trace("done"); + assert(top_co); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; @@ -98,6 +189,13 @@ void Goal::amDone(ExitCode result, std::optional ex) worker.removeGoal(shared_from_this()); cleanup(); + + // We drop the continuation. + // In `final_awaiter` this will signal that there is no more work to be done. + top_co->handle.promise().continuation = {}; + + // won't return to caller because of logic in final_awaiter + return Done{}; } @@ -106,4 +204,16 @@ void Goal::trace(std::string_view s) debug("%1%: %2%", name, s); } +void Goal::work() +{ + assert(top_co); + assert(top_co->handle); + assert(top_co->handle.promise().alive); + top_co->handle.resume(); + // We either should be in a state where we can be work()-ed again, + // or we should be done. + assert(top_co || exitCode != ecBusy); +} + + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 0d9b828e1d6..162c392d066 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,10 +1,11 @@ #pragma once ///@file -#include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include + namespace nix { /** @@ -103,9 +104,263 @@ protected: * Build result. */ BuildResult buildResult; - public: + /** + * Suspend our goal and wait until we get @ref work()-ed again. + * `co_await`-able by @ref Co. + */ + struct Suspend {}; + + /** + * Return from the current coroutine and suspend our goal + * if we're not busy anymore, or jump to the next coroutine + * set to be executed/resumed. + */ + struct Return {}; + + /** + * `co_return`-ing this will end the goal. + * If you're not inside a coroutine, you can safely discard this. + */ + struct [[nodiscard]] Done { + private: + Done(){} + + friend Goal; + }; + + // forward declaration of promise_type, see below + struct promise_type; + + /** + * Handle to coroutine using @ref Co and @ref promise_type. + */ + using handle_type = std::coroutine_handle; + + /** + * C++20 coroutine wrapper for use in goal logic. + * Coroutines are functions that use `co_await`/`co_return` (and `co_yield`, but not supported by @ref Co). + * + * @ref Co is meant to be used by methods of subclasses of @ref Goal. + * The main functionality provided by `Co` is + * - `co_await Suspend{}`: Suspends the goal. + * - `co_await f()`: Waits until `f()` finishes. + * - `co_return f()`: Tail-calls `f()`. + * - `co_return Return{}`: Ends coroutine. + * + * The idea is that you implement the goal logic using coroutines, + * and do the core thing a goal can do, suspension, when you have + * children you're waiting for. + * Coroutines allow you to resume the work cleanly. + * + * @note Brief explanation of C++20 coroutines: + * When you `Co f()`, a `std::coroutine_handle` is created, + * alongside its @ref promise_type. + * There are suspension points at the beginning of the coroutine, + * at every `co_await`, and at the final (possibly implicit) `co_return`. + * Once suspended, you can resume the `std::coroutine_handle` by doing `coroutine_handle.resume()`. + * Suspension points are implemented by passing a struct to the compiler + * that implements `await_sus`pend. + * `await_suspend` can either say "cancel suspension", in which case execution resumes, + * "suspend", in which case control is passed back to the caller of `coroutine_handle.resume()` + * or the place where the coroutine function is initially executed in the case of the initial + * suspension, or `await_suspend` can specify another coroutine to jump to, which is + * how tail calls are implemented. + * + * @note Resources: + * - https://lewissbaker.github.io/ + * - https://www.chiark.greenend.org.uk/~sgtatham/quasiblog/coroutines-c++20/ + * - https://www.scs.stanford.edu/~dm/blog/c++-coroutines.html + * + * @todo Allocate explicitly on stack since HALO thing doesn't really work, + * specifically, there's no way to uphold the requirements when trying to do + * tail-calls without using a trampoline AFAICT. + * + * @todo Support returning data natively + */ + struct [[nodiscard]] Co { + /** + * The underlying handle. + */ + handle_type handle; + + explicit Co(handle_type handle) : handle(handle) {}; + void operator=(Co&&); + Co(Co&& rhs); + ~Co(); + + bool await_ready() { return false; }; + /** + * When we `co_await` another @ref Co-returning coroutine, + * we tell the caller of `caller_coroutine.resume()` to switch to our coroutine (@ref handle). + * To make sure we return to the original coroutine, we set it as the continuation of our + * coroutine. In @ref promise_type::final_awaiter we check if it's set and if so we return to it. + * + * To explain in more understandable terms: + * When we `co_await Co_returning_function()`, this function is called on the resultant @ref Co of + * the _called_ function, and C++ automatically passes the caller in. + * + * `goal` field of @ref promise_type is also set here by copying it from the caller. + */ + std::coroutine_handle<> await_suspend(handle_type handle); + void await_resume() {}; + }; + + /** + * Used on initial suspend, does the same as @ref std::suspend_always, + * but asserts that everything has been set correctly. + */ + struct InitialSuspend { + /** + * Handle of coroutine that does the + * initial suspend + */ + handle_type handle; + + bool await_ready() { return false; }; + void await_suspend(handle_type handle_) { + handle = handle_; + } + void await_resume() { + assert(handle); + assert(handle.promise().goal); // goal must be set + assert(handle.promise().goal->top_co); // top_co of goal must be set + assert(handle.promise().goal->top_co->handle == handle); // top_co of goal must be us + } + }; + + /** + * Promise type for coroutines defined using @ref Co. + * Attached to coroutine handle. + */ + struct promise_type { + /** + * Either this is who called us, or it is who we will tail-call. + * It is what we "jump" to once we are done. + */ + std::optional continuation; + + /** + * The goal that we're a part of. + * Set either in @ref Co::await_suspend or in constructor of @ref Goal. + */ + Goal* goal = nullptr; + + /** + * Is set to false when destructed to ensure we don't use a + * destructed coroutine by accident + */ + bool alive = true; + + /** + * The awaiter used by @ref final_suspend. + */ + struct final_awaiter { + bool await_ready() noexcept { return false; }; + /** + * Here we execute our continuation, by passing it back to the caller. + * C++ compiler will create code that takes that and executes it promptly. + * `h` is the handle for the coroutine that is finishing execution, + * thus it must be destroyed. + */ + std::coroutine_handle<> await_suspend(handle_type h) noexcept; + void await_resume() noexcept { assert(false); }; + }; + + /** + * Called by compiler generated code to construct the @ref Co + * that is returned from a @ref Co-returning coroutine. + */ + Co get_return_object(); + + /** + * Called by compiler generated code before body of coroutine. + * We use this opportunity to set the @ref goal field + * and `top_co` field of @ref Goal. + */ + InitialSuspend initial_suspend() { return {}; }; + + /** + * Called on `co_return`. Creates @ref final_awaiter which + * either jumps to continuation or suspends goal. + */ + final_awaiter final_suspend() noexcept { return {}; }; + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Return) {} + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Done) {} + + /** + * When "returning" another coroutine, what happens is that + * we set it as our own continuation, thus once the final suspend + * happens, we transfer control to it. + * The original continuation we had is set as the continuation + * of the coroutine passed in. + * @ref final_suspend is called after this, and @ref final_awaiter will + * pass control off to @ref continuation. + * + * If we already have a continuation, that continuation is set as + * the continuation of the new continuation. Thus, the continuation + * passed to @ref return_value must not have a continuation set. + */ + void return_value(Co&&); + + /** + * If an exception is thrown inside a coroutine, + * we re-throw it in the context of the "resumer" of the continuation. + */ + void unhandled_exception() { throw; }; + + /** + * Allows awaiting a @ref Co. + */ + Co&& await_transform(Co&& co) { return static_cast(co); } + + /** + * Allows awaiting a @ref Suspend. + * Always suspends. + */ + std::suspend_always await_transform(Suspend) { return {}; }; + }; + + /** + * The coroutine being currently executed. + * MUST be updated when switching the coroutine being executed. + * This is used both for memory management and to resume the last + * coroutine executed. + * Destroying this should destroy all coroutines created for this goal. + */ + std::optional top_co; + + /** + * The entry point for the goal + */ + virtual Co init() = 0; + + /** + * Wrapper around @ref init since virtual functions + * can't be used in constructors. + */ + inline Co init_wrapper(); + + /** + * Signals that the goal is done. + * `co_return` the result. If you're not inside a coroutine, you can ignore + * the return value safely. + */ + Done amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } + /** * Project a `BuildResult` with just the information that pertains * to the given request. @@ -124,15 +379,20 @@ public: std::optional ex; Goal(Worker & worker, DerivedPath path) - : worker(worker) - { } + : worker(worker), top_co(init_wrapper()) + { + // top_co shouldn't have a goal already, should be nullptr. + assert(!top_co->handle.promise().goal); + // we set it such that top_co can pass it down to its subcoroutines. + top_co->handle.promise().goal = this; + } virtual ~Goal() { trace("goal destroyed"); } - virtual void work() = 0; + void work(); void addWaitee(GoalPtr waitee); @@ -164,10 +424,6 @@ public: virtual std::string key() = 0; - void amDone(ExitCode result, std::optional ex = {}); - - virtual void cleanup() { } - /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory @@ -178,3 +434,12 @@ public: void addToWeakGoals(WeakGoals & goals, GoalPtr p); } + +template +struct std::coroutine_traits { + using promise_type = nix::Goal::promise_type; +}; + +nix::Goal::Co nix::Goal::init_wrapper() { + co_return init(); +} diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 0be3d1e8d98..7deeb47487d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,7 @@ #include "nar-info.hh" #include "finally.hh" #include "signals.hh" +#include namespace nix { @@ -12,7 +13,6 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); @@ -25,7 +25,7 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done( +Goal::Done PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional errorMsg) @@ -35,17 +35,11 @@ void PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - amDone(result); + return amDone(result); } -void PathSubstitutionGoal::work() -{ - (this->*state)(); -} - - -void PathSubstitutionGoal::init() +Goal::Co PathSubstitutionGoal::init() { trace("init"); @@ -53,152 +47,135 @@ void PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - done(ecSuccess, BuildResult::AlreadyValid); - return; + co_return done(ecSuccess, BuildResult::AlreadyValid); } if (settings.readOnlyMode) throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + bool substituterFailed = false; -void PathSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); + for (auto sub : subs) { + trace("trying next substituter"); - cleanup(); + cleanup(); - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - done( - substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; - return; - } + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr info; - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - ContentAddressWithReferences::withoutRefs(*ca)); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; + if (ca) { + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + continue; } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; + + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + } catch (InvalidPath &) { + continue; + } catch (SubstituterDisabled & e) { + if (settings.tryFallback) continue; + else throw e; + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + continue; + } else throw e; } - throw; - } - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + continue; + } } - } - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; - worker.updateProgress(); + worker.updateProgress(); - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) - { - warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", - worker.store.printStorePath(storePath), sub->getUri()); - tryNext(); - return; + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + { + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + worker.store.printStorePath(storePath), sub->getUri()); + continue; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); + + if (!waitees.empty()) co_await Suspend{}; + + // FIXME: consider returning boolean instead of passing in reference + bool out = false; // is mutated by tryToRun + co_await tryToRun(subPath ? *subPath : storePath, sub, info, out); + substituterFailed = substituterFailed || out; } - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + /* None left. Terminate this goal and let someone else deal + with it. */ - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &PathSubstitutionGoal::referencesValid; + worker.failedSubstitutions++; + worker.updateProgress(); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } -void PathSubstitutionGoal::referencesValid() +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) { trace("all references realised"); if (nrFailed > 0) { - done( + co_return done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, BuildResult::DependencyFailed, fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); - return; } for (auto & i : info->references) if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); -} - + co_await Suspend{}; -void PathSubstitutionGoal::tryToRun() -{ trace("trying to run"); /* Make sure that we are allowed to start a substitution. Note that even @@ -206,10 +183,10 @@ void PathSubstitutionGoal::tryToRun() prevents infinite waiting. */ if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { worker.waitForBuildSlot(shared_from_this()); - return; + co_await Suspend{}; } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); #ifndef _WIN32 @@ -218,9 +195,9 @@ void PathSubstitutionGoal::tryToRun() outPipe.createAsyncPipe(worker.ioport.get()); #endif - promise = std::promise(); + auto promise = std::promise(); - thr = std::thread([this]() { + thr = std::thread([this, &promise, &subPath, &sub]() { try { ReceiveInterrupts receiveInterrupts; @@ -231,7 +208,7 @@ void PathSubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -247,12 +224,8 @@ void PathSubstitutionGoal::tryToRun() #endif }, true, false); - state = &PathSubstitutionGoal::finished; -} - + co_await Suspend{}; -void PathSubstitutionGoal::finished() -{ trace("substitute finished"); thr.join(); @@ -274,10 +247,7 @@ void PathSubstitutionGoal::finished() substituterFailed = true; } - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; + co_return Return{}; } worker.markContentsGood(storePath); @@ -295,23 +265,19 @@ void PathSubstitutionGoal::finished() worker.doneDownloadSize += fileSize; } + assert(maintainExpectedNar); worker.doneNarSize += maintainExpectedNar->delta; maintainExpectedNar.reset(); worker.updateProgress(); - done(ecSuccess, BuildResult::Substituted); -} - - -void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data) -{ + co_return done(ecSuccess, BuildResult::Substituted); } void PathSubstitutionGoal::handleEOF(Descriptor fd) { - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1a051fc1fd5..86e4f542382 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,14 +1,16 @@ #pragma once ///@file +#include "worker.hh" #include "store-api.hh" #include "goal.hh" #include "muxable-pipe.hh" +#include +#include +#include namespace nix { -class Worker; - struct PathSubstitutionGoal : public Goal { /** @@ -17,30 +19,9 @@ struct PathSubstitutionGoal : public Goal StorePath storePath; /** - * The path the substituter refers to the path as. This will be - * different when the stores have different names. - */ - std::optional subPath; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - - /** - * Path info returned by the substituter's query info operation. + * Whether to try to repair a valid path. */ - std::shared_ptr info; + RepairFlag repair; /** * Pipe for the substituter's standard output. @@ -52,31 +33,15 @@ struct PathSubstitutionGoal : public Goal */ std::thread thr; - std::promise promise; - - /** - * Whether to try to repair a valid path. - */ - RepairFlag repair; - - /** - * Location where we're downloading the substitute. Differs from - * storePath when doing a repair. - */ - Path destPath; - std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (PathSubstitutionGoal::*GoalState)(); - GoalState state; - /** * Content address for recomputing store path */ std::optional ca; - void done( + Done done( ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); @@ -96,22 +61,18 @@ public: return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - void work() override; - /** * The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); + Co init() override; + Co gotInfo(); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co finished(); /** * Callback used by the worker to write to the log. */ - void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleChildOutput(Descriptor fd, std::string_view data) override {}; void handleEOF(Descriptor fd) override; /* Called by destructor, can't be overridden */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 8a5d6de725f..7fc41b1214a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -337,31 +337,27 @@ void Worker::run(const Goals & _topGoals) /* Wait for input. */ if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); - else { - if (awake.empty() && 0U == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error( - R"( - Unable to start any build; - either increase '--max-jobs' or enable remote builds. - - For more information run 'man nix.conf' and search for '/machines'. - )" - ); - else - throw Error( - R"( - Unable to start any build; - remote machines may not have all required system features. - - For more information run 'man nix.conf' and search for '/machines'. - )" - ); + else if (awake.empty() && 0U == settings.maxBuildJobs) { + if (getMachines().empty()) + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + else + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. - } - assert(!awake.empty()); - } + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + + } else assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal diff --git a/src/libstore/ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc similarity index 96% rename from src/libstore/ssh-store-config.cc rename to src/libstore/common-ssh-store-config.cc index e81a9487449..05332b9bb5c 100644 --- a/src/libstore/ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -1,6 +1,6 @@ #include -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "ssh.hh" namespace nix { diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh similarity index 100% rename from src/libstore/ssh-store-config.hh rename to src/libstore/common-ssh-store-config.hh diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6cc3b7cd9e7..bb515013a04 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -73,6 +73,7 @@ struct ContentAddressMethod Raw raw; + bool operator ==(const ContentAddressMethod &) const = default; auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -159,6 +160,7 @@ struct ContentAddress */ Hash hash; + bool operator ==(const ContentAddress &) const = default; auto operator <=>(const ContentAddress &) const = default; /** @@ -218,7 +220,9 @@ struct StoreReferences */ size_t size() const; - auto operator <=>(const StoreReferences &) const = default; + bool operator ==(const StoreReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -235,7 +239,9 @@ struct TextInfo */ StorePathSet references; - auto operator <=>(const TextInfo &) const = default; + bool operator ==(const TextInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -255,7 +261,9 @@ struct FixedOutputInfo */ StoreReferences references; - auto operator <=>(const FixedOutputInfo &) const = default; + bool operator ==(const FixedOutputInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -272,7 +280,9 @@ struct ContentAddressWithReferences Raw raw; - auto operator <=>(const ContentAddressWithReferences &) const = default; + bool operator ==(const ContentAddressWithReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const ContentAddressWithReferences &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6dfcc408c33..8f9c7185168 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -10,6 +10,8 @@ #include #include +#include "strings-inline.hh" + namespace nix { std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 522523e4597..58e5328a5eb 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -8,13 +8,11 @@ #include "repair-flag.hh" #include "derived-path-map.hh" #include "sync.hh" -#include "comparator.hh" #include "variant-wrapper.hh" #include #include - namespace nix { struct StoreDirConfig; @@ -33,7 +31,8 @@ struct DerivationOutput { StorePath path; - GENERATE_CMP(InputAddressed, me->path); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -57,7 +56,8 @@ struct DerivationOutput */ StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - GENERATE_CMP(CAFixed, me->ca); + bool operator == (const CAFixed &) const = default; + auto operator <=> (const CAFixed &) const = default; }; /** @@ -77,7 +77,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashAlgo); + bool operator == (const CAFloating &) const = default; + auto operator <=> (const CAFloating &) const = default; }; /** @@ -85,7 +86,8 @@ struct DerivationOutput * isn't known yet. */ struct Deferred { - GENERATE_CMP(Deferred); + bool operator == (const Deferred &) const = default; + auto operator <=> (const Deferred &) const = default; }; /** @@ -104,7 +106,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashAlgo); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -117,7 +120,8 @@ struct DerivationOutput Raw raw; - GENERATE_CMP(DerivationOutput, me->raw); + bool operator == (const DerivationOutput &) const = default; + auto operator <=> (const DerivationOutput &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); @@ -178,7 +182,8 @@ struct DerivationType { */ bool deferred; - GENERATE_CMP(InputAddressed, me->deferred); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -202,7 +207,8 @@ struct DerivationType { */ bool fixed; - GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed); + bool operator == (const ContentAddressed &) const = default; + auto operator <=> (const ContentAddressed &) const = default; }; /** @@ -212,7 +218,8 @@ struct DerivationType { * type, but has some restrictions on its usage. */ struct Impure { - GENERATE_CMP(Impure); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -223,7 +230,8 @@ struct DerivationType { Raw raw; - GENERATE_CMP(DerivationType, me->raw); + bool operator == (const DerivationType &) const = default; + auto operator <=> (const DerivationType &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationType); @@ -313,14 +321,9 @@ struct BasicDerivation static std::string_view nameFromPath(const StorePath & storePath); - GENERATE_CMP(BasicDerivation, - me->outputs, - me->inputSrcs, - me->platform, - me->builder, - me->args, - me->env, - me->name); + bool operator == (const BasicDerivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BasicDerivation &) const = default; }; class Store; @@ -378,9 +381,9 @@ struct Derivation : BasicDerivation const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - GENERATE_CMP(Derivation, - static_cast(*me), - me->inputDrvs); + bool operator == (const Derivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const Derivation &) const = default; }; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 4c1ea417a36..c97d52773eb 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -54,17 +54,18 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single namespace nix { -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode, - me->value, - me->childMap); +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept = default; -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>, - me->map); +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept = default; +#endif +template struct DerivedPathMap>::ChildNode; template struct DerivedPathMap>; }; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 393cdedf747..bd60fe88710 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -47,7 +47,11 @@ struct DerivedPathMap { */ Map childMap; - DECLARE_CMP(ChildNode); + bool operator == (const ChildNode &) const noexcept; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // decltype(std::declval() <=> std::declval()) + // operator <=> (const ChildNode &) const noexcept; }; /** @@ -60,7 +64,10 @@ struct DerivedPathMap { */ Map map; - DECLARE_CMP(DerivedPathMap); + bool operator == (const DerivedPathMap &) const = default; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // auto operator <=> (const DerivedPathMap &) const noexcept; /** * Find the node for `k`, creating it if needed. @@ -83,14 +90,21 @@ struct DerivedPathMap { ChildNode * findSlot(const SingleDerivedPath & k); }; +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept; + +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept; + +template<> +inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +#endif -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::, - DerivedPathMap>); -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode::, - DerivedPathMap>::ChildNode); +extern template struct DerivedPathMap>::ChildNode; +extern template struct DerivedPathMap>; } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index a7b40432108..1eef881de0c 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,6 +1,7 @@ #include "derived-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,26 +9,32 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) - -CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) - -CMP(SingleDerivedPath, DerivedPathBuilt, outputs) - -#undef CMP -#undef CMP_ONE +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleDerivedPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + DerivedPathBuilt ::, + DerivedPathBuilt, + *me->drvPath, + me->outputs); +GENERATE_ONE_CMP( + , + bool, + DerivedPathBuilt ::, + <, + DerivedPathBuilt, + *me->drvPath, + me->outputs); nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b238f844e56..4ba3fb37d4c 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,8 +3,8 @@ #include "path.hh" #include "outputs-spec.hh" -#include "comparator.hh" #include "config.hh" +#include "ref.hh" #include @@ -31,7 +31,8 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - GENERATE_CMP(DerivedPathOpaque, me->path); + bool operator == (const DerivedPathOpaque &) const = default; + auto operator <=> (const DerivedPathOpaque &) const = default; }; struct SingleDerivedPath; @@ -78,7 +79,8 @@ struct SingleDerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(SingleDerivedPathBuilt); + bool operator == (const SingleDerivedPathBuilt &) const noexcept; + std::strong_ordering operator <=> (const SingleDerivedPathBuilt &) const noexcept; }; using _SingleDerivedPathRaw = std::variant< @@ -108,6 +110,9 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { return static_cast(*this); } + bool operator == (const SingleDerivedPath &) const = default; + auto operator <=> (const SingleDerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). @@ -201,7 +206,9 @@ struct DerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(DerivedPathBuilt); + bool operator == (const DerivedPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const DerivedPathBuilt &) const noexcept; }; using _DerivedPathRaw = std::variant< @@ -230,6 +237,10 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + bool operator == (const DerivedPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const DerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 36259b7006d..049db08825b 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,13 +1,15 @@ #pragma once ///@file -#include "types.hh" -#include "hash.hh" -#include "config.hh" - #include #include +#include "logging.hh" +#include "types.hh" +#include "ref.hh" +#include "config.hh" +#include "serialise.hh" + namespace nix { namespace auth { class Authenticator; } diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index ab1059fb1ec..020f770b07a 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "store-api.hh" +#include +#include "store-api.hh" namespace nix { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7e1d7ea6d67..4eabf6054dc 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -35,6 +35,8 @@ #include #endif +#include "strings.hh" + namespace nix { @@ -82,7 +84,7 @@ Settings::Settings() Strings ss; for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); - builders = concatStringsSep(" ", ss); + builders = concatStringsSep("\n", ss); } #if defined(__linux__) && defined(SANDBOX_SHELL) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 9664b126edc..eac360a4f7a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,5 +1,5 @@ #include "legacy-ssh-store.hh" -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -15,6 +15,15 @@ namespace nix { +LegacySSHStoreConfig::LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) +{ +} + std::string LegacySSHStoreConfig::doc() { return @@ -35,7 +44,7 @@ LegacySSHStore::LegacySSHStore( const Params & params) : StoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(params) + , LegacySSHStoreConfig(scheme, host, params) , Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index db49188ec9d..f2665189828 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "store-api.hh" #include "ssh.hh" #include "callback.hh" @@ -13,6 +13,11 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; + LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2b4e01eb3ba..82b70ff2135 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -51,6 +51,8 @@ #include +#include "strings.hh" + namespace nix { diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 2a55c445667..983652d5f8b 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" +#include "ref.hh" #include "store-reference.hh" namespace nix { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 5324b2a1fd7..297bf71870c 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -162,6 +162,7 @@ sources = files( 'builtins/fetchurl.cc', 'builtins/unpack-channel.cc', 'common-protocol.cc', + 'common-ssh-store-config.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', @@ -206,7 +207,6 @@ sources = files( 'serve-protocol-connection.cc', 'serve-protocol.cc', 'sqlite.cc', - 'ssh-store-config.cc', 'ssh-store.cc', 'ssh.cc', 'store-api.cc', @@ -233,6 +233,7 @@ headers = [config_h] + files( 'builtins/buildenv.hh', 'common-protocol-impl.hh', 'common-protocol.hh', + 'common-ssh-store-config.hh', 'content-address.hh', 'daemon.hh', 'derivations.hh', @@ -272,11 +273,11 @@ headers = [config_h] + files( 'remote-store.hh', 's3-binary-cache-store.hh', 's3.hh', + 'ssh-store.hh', 'serve-protocol-connection.hh', 'serve-protocol-impl.hh', 'serve-protocol.hh', 'sqlite.hh', - 'ssh-store-config.hh', 'ssh.hh', 'store-api.hh', 'store-cast.hh', diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index cc3f4884f51..bcc02206bc9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,3 +1,5 @@ +#include + #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" @@ -8,6 +10,7 @@ #include "callback.hh" #include "closure.hh" #include "filetransfer.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b1079b027d6..9a541bb7776 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -3,7 +3,6 @@ #include #include -#include #include @@ -74,7 +73,10 @@ struct NarAccessor : public SourceAccessor NarMember & createMember(const CanonPath & path, NarMember member) { size_t level = 0; - for (auto _ : path) ++level; + for (auto _ : path) { + (void)_; + ++level; + } while (parents.size() > level) parents.pop(); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 07beb8acb0b..288f618d5d7 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -7,6 +7,8 @@ #include #include +#include "strings.hh" + namespace nix { static const char * schema = R"sql( diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 0d219a48929..2442a7b09a3 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,18 +1,10 @@ #include "globals.hh" #include "nar-info.hh" #include "store-api.hh" +#include "strings.hh" namespace nix { -GENERATE_CMP_EXT( - , - NarInfo, - me->url, - me->compression, - me->fileHash, - me->fileSize, - static_cast(*me)); - NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index fd538a7cd9f..561c9a86364 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -24,7 +24,9 @@ struct NarInfo : ValidPathInfo NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - DECLARE_CMP(NarInfo); + bool operator ==(const NarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + //auto operator <=>(const NarInfo &) const = default; std::string to_string(const Store & store) const; diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 21c06922379..86788a87e72 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -5,6 +5,7 @@ #include "regex-combinators.hh" #include "outputs-spec.hh" #include "path-regex.hh" +#include "strings-inline.hh" namespace nix { diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 1ef99a5fc67..30d15311d0a 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -6,9 +6,7 @@ #include #include -#include "comparator.hh" #include "json-impls.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -60,7 +58,11 @@ struct OutputsSpec { Raw raw; - GENERATE_CMP(OutputsSpec, me->raw); + bool operator == (const OutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const OutputsSpec & other) const { + return raw < other.raw; + } MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); @@ -99,7 +101,9 @@ struct ExtendedOutputsSpec { Raw raw; - GENERATE_CMP(ExtendedOutputsSpec, me->raw); + bool operator == (const ExtendedOutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const ExtendedOutputsSpec &) const; MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5c27182b720..6e87e60f446 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,11 +3,14 @@ #include "path-info.hh" #include "store-api.hh" #include "json-utils.hh" +#include "comparator.hh" +#include "strings.hh" namespace nix { GENERATE_CMP_EXT( , + std::weak_ordering, UnkeyedValidPathInfo, me->deriver, me->narHash, @@ -19,12 +22,6 @@ GENERATE_CMP_EXT( me->sigs, me->ca); -GENERATE_CMP_EXT( - , - ValidPathInfo, - me->path, - static_cast(*me)); - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b6dc0855d75..71f1476a672 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,17 +32,47 @@ struct SubstitutablePathInfo using SubstitutablePathInfos = std::map; +/** + * Information about a store object. + * + * See `store/store-object` and `protocols/json/store-object-info` in + * the Nix manual + */ struct UnkeyedValidPathInfo { + /** + * Path to derivation that produced this store object, if known. + */ std::optional deriver; + /** * \todo document this */ Hash narHash; + + /** + * Other store objects this store object referes to. + */ StorePathSet references; + + /** + * When this store object was registered in the store that contains + * it, if known. + */ time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id = 0; // internal use only + + /** + * 0 = unknown + */ + uint64_t narSize = 0; + + /** + * internal use only: SQL primary key for on-disk store objects with + * `LocalStore`. + * + * @todo Remove, layer violation + */ + uint64_t id = 0; /** * Whether the path is ultimately trusted, that is, it's a @@ -75,7 +105,12 @@ struct UnkeyedValidPathInfo UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; - DECLARE_CMP(UnkeyedValidPathInfo); + bool operator == (const UnkeyedValidPathInfo &) const noexcept; + + /** + * @todo return `std::strong_ordering` once `id` is removed + */ + std::weak_ordering operator <=> (const UnkeyedValidPathInfo &) const noexcept; virtual ~UnkeyedValidPathInfo() { } @@ -95,7 +130,8 @@ struct UnkeyedValidPathInfo struct ValidPathInfo : UnkeyedValidPathInfo { StorePath path; - DECLARE_CMP(ValidPathInfo); + bool operator == (const ValidPathInfo &) const = default; + auto operator <=> (const ValidPathInfo &) const = default; /** * Return a fingerprint of the store path to be used in binary @@ -135,6 +171,9 @@ struct ValidPathInfo : UnkeyedValidPathInfo { */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + /** + * References as store path basenames, including a self reference if it has one. + */ Strings shortRefs() const; ValidPathInfo(const ValidPathInfo & other) = default; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 026e376471f..161d023d14c 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,7 +1,9 @@ +#include + #include "path-with-outputs.hh" #include "store-api.hh" +#include "strings.hh" -#include namespace nix { diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index 67bc5dc6eed..6f3b177acd2 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -10,7 +10,6 @@ #include "serve-protocol.hh" #include "length-prefixed-protocol-helper.hh" -#include "store-api.hh" namespace nix { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 7ad934b738b..b21c22d7e08 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,7 +1,5 @@ -#include "ssh-store-config.hh" -#include "store-api.hh" +#include "ssh-store.hh" #include "local-fs-store.hh" -#include "remote-store.hh" #include "remote-store-connection.hh" #include "source-accessor.hh" #include "archive.hh" @@ -12,23 +10,22 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) { - using RemoteStoreConfig::RemoteStoreConfig; - using CommonSSHStoreConfig::CommonSSHStoreConfig; - - const Setting remoteProgram{this, {"nix-daemon"}, "remote-program", - "Path to the `nix-daemon` executable on the remote machine."}; - - const std::string name() override { return "Experimental SSH Store"; } +} - std::string doc() override - { - return - #include "ssh-store.md" - ; - } -}; +std::string SSHStoreConfig::doc() +{ + return + #include "ssh-store.md" + ; +} class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { @@ -41,7 +38,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore : StoreConfig(params) , RemoteStoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) + , SSHStoreConfig(scheme, host, params) , Store(params) , RemoteStore(params) , master(createSSHMaster( diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh new file mode 100644 index 00000000000..6ef2219a2e9 --- /dev/null +++ b/src/libstore/ssh-store.hh @@ -0,0 +1,28 @@ +#pragma once +///@file + +#include "common-ssh-store-config.hh" +#include "store-api.hh" +#include "remote-store.hh" + +namespace nix { + +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + + const Setting remoteProgram{ + this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + + const std::string name() override + { + return "Experimental SSH Store"; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8e630ea5071..b10f06e6b80 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -22,6 +22,8 @@ #include #include +#include "strings.hh" + using json = nlohmann::json; namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a5effb4c131..749d7ea098d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -18,14 +18,10 @@ #include #include -#include #include -#include -#include #include #include #include -#include namespace nix { diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh index e99335c0d57..459cea9c20e 100644 --- a/src/libstore/store-reference.hh +++ b/src/libstore/store-reference.hh @@ -71,7 +71,6 @@ struct StoreReference Params params; bool operator==(const StoreReference & rhs) const = default; - auto operator<=>(const StoreReference & rhs) const = default; /** * Render the whole store reference as a URI, including parameters. diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index dfc20879881..d73d86ff27f 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -3,6 +3,7 @@ #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index aafd7663f00..4e4151b4c2a 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -66,6 +66,8 @@ #include #include +#include "strings.hh" + namespace nix { void handleDiffHook( @@ -177,7 +179,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() +Goal::Co LocalDerivationGoal::tryLocalBuild() { #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); @@ -185,10 +187,10 @@ void LocalDerivationGoal::tryLocalBuild() unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { - state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); } assert(derivationType); @@ -242,7 +244,8 @@ void LocalDerivationGoal::tryLocalBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryLocalBuild(); } } @@ -257,15 +260,13 @@ void LocalDerivationGoal::tryLocalBuild() outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, std::move(e)); - return; + co_return done(BuildResult::InputRejected, {}, std::move(e)); } - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - started(); + co_await Suspend{}; + // after EOF on child + co_return buildDone(); } static void chmod_(const Path & path, mode_t mode) diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index 523e633148f..84d176b1895 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -206,7 +206,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - void tryLocalBuild() override; + Goal::Co tryLocalBuild() override; /** * Start building a derivation. diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index 8057aa13e1b..29f4b2cb31c 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -1,12 +1,13 @@ +#include +#include +#include + #include "user-lock.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" #include "users.hh" -#include -#include - namespace nix { #if __linux__ diff --git a/src/libstore/unix/user-lock.hh b/src/libstore/unix/user-lock.hh index 1c268e1fbd5..a7caf8518f3 100644 --- a/src/libstore/unix/user-lock.hh +++ b/src/libstore/unix/user-lock.hh @@ -1,10 +1,8 @@ #pragma once ///@file -#include "types.hh" - -#include - +#include +#include #include namespace nix { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c202facdfea..d58f4b4aeaf 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -57,8 +57,7 @@ void Completions::add(std::string completion, std::string description) }); } -bool Completion::operator<(const Completion & other) const -{ return completion < other.completion || (completion == other.completion && description < other.description); } +auto Completion::operator<=>(const Completion & other) const noexcept = default; std::string completionMarker = "___COMPLETE___"; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7759b74a93c..c0236ee3d72 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include #include #include #include @@ -11,6 +10,7 @@ #include "types.hh" #include "experimental-features.hh" +#include "ref.hh" namespace nix { @@ -380,7 +380,7 @@ struct Completion { std::string completion; std::string description; - bool operator<(const Completion & other) const; + auto operator<=>(const Completion & other) const noexcept; }; /** diff --git a/src/libutil/auth.hh b/src/libutil/auth.hh index 8d094a19ddb..ee1354a16b9 100644 --- a/src/libutil/auth.hh +++ b/src/libutil/auth.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "config.hh" +#include "ref.hh" namespace nix::auth { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 27f048697e5..03db6378a82 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,6 +1,7 @@ #include "canon-path.hh" #include "util.hh" #include "file-path-impl.hh" +#include "strings-inline.hh" namespace nix { diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 8f5a1c2793d..f84347dc458 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -169,7 +169,7 @@ public: * a directory is always followed directly by its children. For * instance, 'foo' < 'foo/bar' < 'foo!'. */ - bool operator < (const CanonPath & x) const + auto operator <=> (const CanonPath & x) const { auto i = path.begin(); auto j = x.path.begin(); @@ -178,10 +178,9 @@ public: if (c_i == '/') c_i = 0; auto c_j = *j; if (c_j == '/') c_j = 0; - if (c_i < c_j) return true; - if (c_i > c_j) return false; + if (auto cmp = c_i <=> c_j; cmp != 0) return cmp; } - return i == path.end() && j != x.path.end(); + return (i != path.end()) <=> (j != x.path.end()); } /** diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index cbc2bb4fd72..34ba6f4534c 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,17 +1,8 @@ #pragma once ///@file -#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, ==, my_type) -#define DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, <, my_type) -#define DECLARE_NEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, !=, my_type) - -#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ +#define GENERATE_ONE_CMP(PRE, RET, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE RET QUAL operator COMPARATOR(const MY_TYPE & other) const noexcept { \ __VA_OPT__(const MY_TYPE * me = this;) \ auto fields1 = std::tie( __VA_ARGS__ ); \ __VA_OPT__(me = &other;) \ @@ -19,30 +10,9 @@ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) -#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) -#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) - -/** - * Declare comparison methods without defining them. - */ -#define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(,,my_type) \ - DECLARE_LEQ(,,my_type) \ - DECLARE_NEQ(,,my_type) - -/** - * @param prefix This is for something before each declaration like - * `template`. - * - * @param my_type the type are defining operators for. - */ -#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ - DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_NEQ(prefix, qualification, my_type) + GENERATE_ONE_CMP(prefix, bool, qualification, ==, my_type, args) +#define GENERATE_SPACESHIP(prefix, ret, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, ret, qualification, <=>, my_type, args) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -55,15 +25,19 @@ * will generate comparison operators semantically equivalent to: * * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; + * auto operator<=>(const ClassName& other) const noexcept { + * if (auto cmp = field1 <=> other.field1; cmp != 0) + * return cmp; + * if (auto cmp = field2 <=> other.field2; cmp != 0) + * return cmp; + * ... + * return 0; * } * ``` */ #define GENERATE_CMP(args...) \ GENERATE_EQUAL(,,args) \ - GENERATE_LEQ(,,args) \ - GENERATE_NEQ(,,args) + GENERATE_SPACESHIP(,auto,,args) /** * @param prefix This is for something before each declaration like @@ -71,7 +45,6 @@ * * @param my_type the type are defining operators for. */ -#define GENERATE_CMP_EXT(prefix, my_type, args...) \ +#define GENERATE_CMP_EXT(prefix, ret, my_type, args...) \ GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ - GENERATE_LEQ(prefix, my_type ::, my_type, args) \ - GENERATE_NEQ(prefix, my_type ::, my_type, args) + GENERATE_SPACESHIP(prefix, ret, my_type ::, my_type, args) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 907ca7fc149..726e5091e54 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + namespace nix { Config::Config(StringMap initials) @@ -114,7 +116,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p if (tokens.empty()) continue; if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto include = false; auto ignoreMissing = false; @@ -127,7 +129,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p if (include) { if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { try { @@ -143,7 +145,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p } if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); std::string name = std::move(tokens[0]); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e01f064484c..33c391963f3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -46,27 +46,22 @@ std::ostream & operator <<(std::ostream & os, const HintFmt & hf) /** * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. */ -inline bool operator<(const Trace& lhs, const Trace& rhs) +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs) { // `std::shared_ptr` does not have value semantics for its comparison // functions, so we need to check for nulls and compare the dereferenced // values here. if (lhs.pos != rhs.pos) { - if (!lhs.pos) - return true; - if (!rhs.pos) - return false; - if (*lhs.pos != *rhs.pos) - return *lhs.pos < *rhs.pos; + if (auto cmp = bool{lhs.pos} <=> bool{rhs.pos}; cmp != 0) + return cmp; + if (auto cmp = *lhs.pos <=> *rhs.pos; cmp != 0) + return cmp; } // This formats a freshly formatted hint string and then throws it away, which // shouldn't be much of a problem because it only runs when pos is equal, and this function is // used for trace printing, which is infrequent. - return lhs.hint.str() < rhs.hint.str(); + return lhs.hint.str() <=> rhs.hint.str(); } -inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } -inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } -inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 4b08a045e32..d7fe902d6c0 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -16,16 +16,12 @@ */ #include "suggestions.hh" -#include "ref.hh" -#include "types.hh" #include "fmt.hh" #include #include #include -#include #include -#include #include #include @@ -79,10 +75,7 @@ struct Trace { TracePrint print = TracePrint::Default; }; -inline bool operator<(const Trace& lhs, const Trace& rhs); -inline bool operator> (const Trace& lhs, const Trace& rhs); -inline bool operator<=(const Trace& lhs, const Trace& rhs); -inline bool operator>=(const Trace& lhs, const Trace& rhs); +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs); struct ErrorInfo { Verbosity level; @@ -127,6 +120,8 @@ protected: public: BaseError(const BaseError &) = default; + BaseError& operator=(const BaseError &) = default; + BaseError& operator=(BaseError &&) = default; template BaseError(unsigned int status, const Args & ... args) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 4731111dc9d..57ec07fe160 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "error.hh" #include "json-utils.hh" #include "types.hh" diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index 4c7218f1941..ec42d3d3493 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -2,8 +2,6 @@ ///@file #include "source-accessor.hh" -#include "fs-sink.hh" -#include "util.hh" namespace nix { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index f75851bbd58..9042e3a5e67 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -23,6 +23,8 @@ # include #endif +#include "strings-inline.hh" + namespace fs = std::filesystem; namespace nix { diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index cf7d34d2273..de165fb7d5f 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "serialise.hh" #include "source-accessor.hh" #include "file-system.hh" @@ -41,6 +40,19 @@ struct FileSystemObjectSink virtual void createSymlink(const CanonPath & path, const std::string & target) = 0; }; +/** + * An extension of `FileSystemObjectSink` that supports file types + * that are not supported by Nix's FSO model. + */ +struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink +{ + /** + * Create a hard link. The target must be the path of a previously + * encountered file relative to the root of the FSO. + */ + virtual void createHardlink(const CanonPath & path, const CanonPath & target) = 0; +}; + /** * Recursively copy file system objects from the source into the sink. */ diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 2190cc5509c..1dbdb733582 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -39,7 +39,8 @@ struct TreeEntry Mode mode; Hash hash; - GENERATE_CMP(TreeEntry, me->mode, me->hash); + bool operator ==(const TreeEntry &) const = default; + auto operator <=>(const TreeEntry &) const = default; }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7064e96e61c..35b913e4249 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -41,7 +41,7 @@ Hash::Hash(HashAlgorithm algo) : algo(algo) } -bool Hash::operator == (const Hash & h2) const +bool Hash::operator == (const Hash & h2) const noexcept { if (hashSize != h2.hashSize) return false; for (unsigned int i = 0; i < hashSize; i++) @@ -50,7 +50,7 @@ bool Hash::operator == (const Hash & h2) const } -std::strong_ordering Hash::operator <=> (const Hash & h) const +std::strong_ordering Hash::operator <=> (const Hash & h) const noexcept { if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ef96d08c978..dc95b9f2f9b 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -88,12 +88,12 @@ public: /** * Check whether two hashes are equal. */ - bool operator == (const Hash & h2) const; + bool operator == (const Hash & h2) const noexcept; /** * Compare how two hashes are ordered. */ - std::strong_ordering operator <=> (const Hash & h2) const; + std::strong_ordering operator <=> (const Hash & h2) const noexcept; /** * Returns the length of a base-16 representation of this hash. diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 9e81132e354..250f9209972 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "error.hh" #include "config.hh" diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index cd5146c8913..012a388c0e7 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -15,11 +15,15 @@ struct MemorySourceAccessor : virtual SourceAccessor * defining what a "file system object" is in Nix. */ struct File { + bool operator == (const File &) const noexcept; + std::strong_ordering operator <=> (const File &) const noexcept; + struct Regular { bool executable = false; std::string contents; - GENERATE_CMP(Regular, me->executable, me->contents); + bool operator == (const Regular &) const = default; + auto operator <=> (const Regular &) const = default; }; struct Directory { @@ -27,13 +31,16 @@ struct MemorySourceAccessor : virtual SourceAccessor std::map> contents; - GENERATE_CMP(Directory, me->contents); + bool operator == (const Directory &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + bool operator < (const Directory &) const noexcept; }; struct Symlink { std::string target; - GENERATE_CMP(Symlink, me->target); + bool operator == (const Symlink &) const = default; + auto operator <=> (const Symlink &) const = default; }; using Raw = std::variant; @@ -41,14 +48,15 @@ struct MemorySourceAccessor : virtual SourceAccessor MAKE_WRAPPER_CONSTRUCTOR(File); - GENERATE_CMP(File, me->raw); - Stat lstat() const; }; File root { File::Directory {} }; - GENERATE_CMP(MemorySourceAccessor, me->root); + bool operator == (const MemorySourceAccessor &) const noexcept = default; + bool operator < (const MemorySourceAccessor & other) const noexcept { + return root < other.root; + } std::string readFile(const CanonPath & path) override; bool pathExists(const CanonPath & path) override; @@ -72,6 +80,20 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; + +inline bool MemorySourceAccessor::File::Directory::operator == ( + const MemorySourceAccessor::File::Directory &) const noexcept = default; +inline bool MemorySourceAccessor::File::Directory::operator < ( + const MemorySourceAccessor::File::Directory & other) const noexcept +{ + return contents < other.contents; +} + +inline bool MemorySourceAccessor::File::operator == ( + const MemorySourceAccessor::File &) const noexcept = default; +inline std::strong_ordering MemorySourceAccessor::File::operator <=> ( + const MemorySourceAccessor::File &) const noexcept = default; + /** * Write to a `MemorySourceAccessor` at the given path */ diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ac2b8353674..fbfcbe67c42 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -148,6 +148,7 @@ sources = files( 'signature/signer.cc', 'source-accessor.cc', 'source-path.cc', + 'strings.cc', 'suggestions.cc', 'tarfile.cc', 'terminal.cc', @@ -215,6 +216,8 @@ headers = [config_h] + files( 'source-accessor.hh', 'source-path.hh', 'split.hh', + 'strings.hh', + 'strings-inline.hh', 'suggestions.hh', 'sync.hh', 'tarfile.hh', diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 724e560b7a4..3289dbe8bf0 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -17,12 +17,6 @@ Pos::operator std::shared_ptr() const return std::make_shared(&*this); } -bool Pos::operator<(const Pos &rhs) const -{ - return std::forward_as_tuple(line, column, origin) - < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin); -} - std::optional Pos::getCodeLines() const { if (line == 0) @@ -116,4 +110,50 @@ void Pos::LinesIterator::bump(bool atFirst) input.remove_prefix(eol); } +std::optional Pos::getSnippetUpTo(const Pos & end) const { + assert(this->origin == end.origin); + + if (end.line < this->line) + return std::nullopt; + + if (auto source = getSource()) { + + auto firstLine = LinesIterator(*source); + for (auto i = 1; i < this->line; ++i) { + ++firstLine; + } + + auto lastLine = LinesIterator(*source); + for (auto i = 1; i < end.line; ++i) { + ++lastLine; + } + + LinesIterator linesEnd; + + std::string result; + for (auto i = firstLine; i != linesEnd; ++i) { + auto firstColumn = i == firstLine ? (this->column ? this->column - 1 : 0) : 0; + if (firstColumn > i->size()) + firstColumn = i->size(); + + auto lastColumn = i == lastLine ? (end.column ? end.column - 1 : 0) : std::numeric_limits::max(); + if (lastColumn < firstColumn) + lastColumn = firstColumn; + if (lastColumn > i->size()) + lastColumn = i->size(); + + result += i->substr(firstColumn, lastColumn - firstColumn); + + if (i == lastLine) { + break; + } else { + result += '\n'; + } + } + return result; + } + return std::nullopt; +} + + } diff --git a/src/libutil/position.hh b/src/libutil/position.hh index 9bdf3b4b5db..25217069c14 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -7,6 +7,7 @@ #include #include +#include #include "source-path.hh" @@ -22,21 +23,17 @@ struct Pos struct Stdin { ref source; - bool operator==(const Stdin & rhs) const + bool operator==(const Stdin & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const Stdin & rhs) const - { return *source != *rhs.source; } - bool operator<(const Stdin & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const Stdin & rhs) const noexcept + { return *source <=> *rhs.source; } }; struct String { ref source; - bool operator==(const String & rhs) const + bool operator==(const String & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const String & rhs) const - { return *source != *rhs.source; } - bool operator<(const String & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const String & rhs) const noexcept + { return *source <=> *rhs.source; } }; typedef std::variant Origin; @@ -65,8 +62,16 @@ struct Pos std::optional getCodeLines() const; bool operator==(const Pos & rhs) const = default; - bool operator!=(const Pos & rhs) const = default; - bool operator<(const Pos & rhs) const; + auto operator<=>(const Pos & rhs) const = default; + + std::optional getSnippetUpTo(const Pos & end) const; + + /** + * Get the SourcePath, if the source was loaded from a file. + */ + std::optional getSourcePath() const { + return *std::get_if(&origin); + } struct LinesIterator { using difference_type = size_t; diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 03aa642739c..016fdd74a39 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,9 +1,7 @@ #pragma once ///@file -#include #include -#include #include namespace nix { @@ -89,9 +87,9 @@ public: return p != other.p; } - bool operator < (const ref & other) const + auto operator <=> (const ref & other) const { - return p < other.p; + return p <=> other.p; } private: diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index cc8db01f59c..b16960d4a4c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -4,6 +4,7 @@ #include "canon-path.hh" #include "hash.hh" +#include "ref.hh" namespace nix { @@ -151,9 +152,9 @@ struct SourceAccessor : std::enable_shared_from_this return number == x.number; } - bool operator < (const SourceAccessor & x) const + auto operator <=> (const SourceAccessor & x) const { - return number < x.number; + return number <=> x.number; } void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index 023b5ed4b91..759d3c35579 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -47,19 +47,14 @@ SourcePath SourcePath::operator / (const CanonPath & x) const SourcePath SourcePath::operator / (std::string_view c) const { return {accessor, path / c}; } -bool SourcePath::operator==(const SourcePath & x) const +bool SourcePath::operator==(const SourcePath & x) const noexcept { return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); } -bool SourcePath::operator!=(const SourcePath & x) const +std::strong_ordering SourcePath::operator<=>(const SourcePath & x) const noexcept { - return std::tie(*accessor, path) != std::tie(*x.accessor, x.path); -} - -bool SourcePath::operator<(const SourcePath & x) const -{ - return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); + return std::tie(*accessor, path) <=> std::tie(*x.accessor, x.path); } std::ostream & operator<<(std::ostream & str, const SourcePath & path) diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 83ec6295de1..1e96b72e52d 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -9,6 +9,8 @@ #include "canon-path.hh" #include "source-accessor.hh" +#include // for boost::hash_combine + namespace nix { /** @@ -103,9 +105,8 @@ struct SourcePath */ SourcePath operator / (std::string_view c) const; - bool operator==(const SourcePath & x) const; - bool operator!=(const SourcePath & x) const; - bool operator<(const SourcePath & x) const; + bool operator==(const SourcePath & x) const noexcept; + std::strong_ordering operator<=>(const SourcePath & x) const noexcept; /** * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. @@ -115,8 +116,21 @@ struct SourcePath { return {accessor, accessor->resolveSymlinks(path, mode)}; } + + friend class std::hash; }; std::ostream & operator << (std::ostream & str, const SourcePath & path); } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::SourcePath & s) const noexcept + { + std::size_t hash = 0; + hash_combine(hash, s.accessor->number, s.path); + return hash; + } +}; diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh new file mode 100644 index 00000000000..10c1b19e688 --- /dev/null +++ b/src/libutil/strings-inline.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "strings.hh" + +namespace nix { + +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + bool tail = false; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) { + if (tail) + size += sep.size(); + size += std::string_view(s).size(); + tail = true; + } + std::string s; + s.reserve(size); + tail = false; + for (auto & i : ss) { + if (tail) + s += sep; + s += i; + tail = true; + } + return s; +} + +} // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 00000000000..7ec618bf449 --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,19 @@ +#include + +#include "strings-inline.hh" +#include "util.hh" + +namespace nix { + +template std::string concatStringsSep(std::string_view, const Strings &); +template std::string concatStringsSep(std::string_view, const StringSet &); +template std::string concatStringsSep(std::string_view, const std::vector &); + +typedef std::string_view strings_2[2]; +template std::string concatStringsSep(std::string_view, const strings_2 &); +typedef std::string_view strings_3[3]; +template std::string concatStringsSep(std::string_view, const strings_3 &); +typedef std::string_view strings_4[4]; +template std::string concatStringsSep(std::string_view, const strings_4 &); + +} // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 00000000000..3b112c4096d --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nix { + +/** + * Concatenate the given strings with a separator between the elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string concatStringsSep(std::string_view, const std::list &); +extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const std::vector &); + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index e67e986fb59..84c8e296f17 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -38,8 +38,8 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - std::set allMatches, - std::string query) + const std::set & allMatches, + std::string_view query) { std::set res; for (const auto & possibleMatch : allMatches) { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 9abf5ee5fad..e39ab400c0d 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "types.hh" #include @@ -20,7 +19,8 @@ public: std::string to_string() const; - GENERATE_CMP(Suggestion, me->distance, me->suggestion) + bool operator ==(const Suggestion &) const = default; + auto operator <=>(const Suggestion &) const = default; }; class Suggestions { @@ -35,8 +35,8 @@ public: ) const; static Suggestions bestMatches ( - std::set allMatches, - std::string query + const std::set & allMatches, + std::string_view query ); Suggestions& operator+=(const Suggestions & other); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f0df527b378..2e323629512 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -174,7 +174,7 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink) { time_t lastModified = 0; @@ -195,7 +195,12 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin lastModified = std::max(lastModified, archive_entry_mtime(entry)); - switch (archive_entry_filetype(entry)) { + if (auto target = archive_entry_hardlink(entry)) { + parseSink.createHardlink(cpath, CanonPath(target)); + continue; + } + + switch (auto type = archive_entry_filetype(entry)) { case AE_IFDIR: parseSink.createDirectory(cpath); @@ -232,7 +237,7 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin } default: - throw Error("file '%s' in tarball has unsupported file type", path); + throw Error("file '%s' in tarball has unsupported file type %d", path, type); } } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 705d211e437..0517177dbe6 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -41,6 +41,6 @@ void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink); } diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh index 902e759456d..7ff05a487c3 100644 --- a/src/libutil/terminal.hh +++ b/src/libutil/terminal.hh @@ -1,7 +1,8 @@ #pragma once ///@file -#include "types.hh" +#include +#include namespace nix { /** diff --git a/src/libutil/types.hh b/src/libutil/types.hh index c86f52175d5..325e3ea7351 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,12 +1,10 @@ #pragma once ///@file -#include "ref.hh" #include #include #include -#include #include #include #include diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f4178f87fd9..bcbe9ea4eb2 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -132,7 +132,7 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } -bool ParsedURL::operator ==(const ParsedURL & other) const +bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return scheme == other.scheme diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 6cd06e53d17..738ee9f82e6 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -18,7 +18,7 @@ struct ParsedURL std::string to_string() const; - bool operator ==(const ParsedURL & other) const; + bool operator ==(const ParsedURL & other) const noexcept; /** * Remove `.` and `..` path elements. diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 23682ff7ca5..83b42a5283e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,8 @@ #include #include +#include "strings.hh" + namespace nix { void initLibUtil(); @@ -33,13 +35,26 @@ template C tokenizeString(std::string_view s, std::string_view separato /** - * Concatenate the given strings with a separator between the - * elements. + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. */ template -std::string concatStringsSep(const std::string_view sep, const C & ss) +[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + // need a cast to string_view since this is also called with Symbols for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); std::string s; @@ -360,4 +375,18 @@ inline std::string operator + (std::string_view s1, const char * s2) return s; } +/** + * hash_combine() from Boost. Hash several hashable values together + * into a single hash. + */ +inline void hash_combine(std::size_t & seed) { } + +template +inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed<<6) + (seed>>2); + hash_combine(seed, rest...); +} + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d37b16bdca0..4d373ef8d4e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -286,7 +286,7 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c72378cd5f4..5e170c99d37 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, evalSettings)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index a4bfeebbbdb..c4854951124 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index d0840a02e5e..f073074e851 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,13 +2,11 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" -#include "build-result.hh" #include "store-cast.hh" #include "local-fs-store.hh" #include "log-store.hh" #include "serve-protocol.hh" #include "serve-protocol-connection.hh" -#include "serve-protocol-impl.hh" #include "shared.hh" #include "graphml.hh" #include "legacy.hh" @@ -23,12 +21,14 @@ #include #include -#include #include #include #include +#include "build-result.hh" +#include "exit.hh" +#include "serve-protocol-impl.hh" namespace nix_store { diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 554c3654022..7d9aa771124 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -76,7 +76,9 @@ struct CmdBundle : InstallableValueCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = + parseFlakeRefWithFragmentAndExtendedOutputsSpec( + fetchSettings, bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 9575bf33887..09d14073386 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -1,6 +1,7 @@ #include #include "command.hh" +#include "exit.hh" #include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6bd3dc9efc6..7cc0965a9a8 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -19,6 +19,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct DevelopSettings : Config diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c7c37b66fbe..46c94b211f9 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -6,6 +6,8 @@ #include +#include "strings.hh" + namespace nix { struct Info diff --git a/src/nix/env.cc b/src/nix/env.cc index 021c47cbbd0..bc4e1b5e583 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -1,6 +1,9 @@ +#include +#include + #include "command.hh" #include "run.hh" -#include +#include "strings.hh" using namespace nix; @@ -90,6 +93,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment } } + // TODO: split losslessly; empty means . auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); auto unixPathString = concatStringsSep(":", unixPath); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 84c659023a5..3f9f8f99b06 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -19,9 +19,10 @@ #include "users.hh" #include -#include #include +#include "strings-inline.hh" + using namespace nix; using namespace nix::flake; using json = nlohmann::json; @@ -48,19 +49,19 @@ class FlakeCommand : virtual Args, public MixFlakeOptions FlakeRef getFlakeRef() { - return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, absPath(".")); //FIXME } LockedFlake lockFlake() { - return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } std::vector getFlakeRefsForCompletion() override { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), absPath(".")) }; } }; @@ -165,7 +166,7 @@ struct CmdFlakeLock : FlakeCommand }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { auto pos = vFlake.determinePos(noPos); state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); @@ -393,15 +394,15 @@ struct CmdFlakeCheck : FlakeCommand || (hasPrefix(name, "_") && name.substr(1) == expected); }; - auto checkSystemName = [&](const std::string & system, const PosIdx pos) { + auto checkSystemName = [&](std::string_view system, const PosIdx pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; - auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + auto checkSystemType = [&](std::string_view system, const PosIdx pos) { if (!checkAllSystems && system != localSystem) { - omittedSystems.insert(system); + omittedSystems.insert(std::string(system)); return false; } else { return true; @@ -450,7 +451,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkOverlay = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking overlay '%s'", attrPath)); @@ -469,7 +470,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkModule = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS module '%s'", attrPath)); @@ -480,9 +481,9 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + checkHydraJobs = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath)); @@ -523,7 +524,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkTemplate = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking template '%s'", attrPath)); @@ -579,7 +580,7 @@ struct CmdFlakeCheck : FlakeCommand enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vOutput, const PosIdx pos) { + [&](std::string_view name, Value & vOutput, const PosIdx pos) { Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); @@ -603,7 +604,7 @@ struct CmdFlakeCheck : FlakeCommand if (name == "checks") { state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; + std::string_view attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); @@ -803,6 +804,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("some errors were encountered during the evaluation"); if (!omittedSystems.empty()) { + // TODO: empty system is not visible; render all as nix strings? warn( "The check omitted these incompatible systems: %s\n" "Use '--all-systems' to check all.", @@ -848,7 +850,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); - auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( + fetchSettings, templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), diff --git a/src/nix/main.cc b/src/nix/main.cc index d6a8e4e31c5..80e88336f8c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,9 +1,8 @@ -#include - #include "args/root.hh" #include "current-process.hh" #include "command.hh" #include "common-args.hh" +#include "eval-gc.hh" #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" @@ -19,6 +18,7 @@ #include "users.hh" #include "network-proxy.hh" #include "eval-cache.hh" +#include "flake/flake.hh" #include "auth.hh" #include @@ -42,6 +42,8 @@ extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); #endif +#include "strings.hh" + namespace nix { enum struct AliasStatus { @@ -243,7 +245,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), evalSettings); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -363,6 +365,7 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + flake::initLib(flakeSettings); #if __linux__ if (isRootUser()) { @@ -423,7 +426,7 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), evalSettings); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto builtinsJson = nlohmann::json::object(); for (auto & builtin : *state.baseEnv.values[0]->attrs()) { auto b = nlohmann::json::object(); @@ -434,7 +437,7 @@ void mainWrapped(int argc, char * * argv) b["doc"] = trim(stripIndentation(primOp->doc)); if (primOp->experimentalFeature) b["experimental-feature"] = primOp->experimentalFeature; - builtinsJson[state.symbols[builtin.name]] = std::move(b); + builtinsJson.emplace(state.symbols[builtin.name], std::move(b)); } for (auto & [name, info] : state.constantInfos) { auto b = nlohmann::json::object(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 47f9baee505..e7cfb6e7a68 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + using namespace nix; using nlohmann::json; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index c6e60a53be6..db7d9e4efe6 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -195,7 +195,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store, evalSettings); + auto state = std::make_unique(myArgs.lookupPath, store, fetchSettings, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c89b8c9bde6..1096f4386be 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -17,6 +17,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct ProfileElementSource @@ -27,7 +29,9 @@ struct ProfileElementSource std::string attrPath; ExtendedOutputsSpec outputs; - bool operator < (const ProfileElementSource & other) const + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const ProfileElementSource & other) const + auto operator < (const ProfileElementSource & other) const { return std::tuple(originalRef.to_string(), attrPath, outputs) < @@ -56,7 +60,7 @@ struct ProfileElement StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); + return dropEmptyInitThenConcatStringsSep(", ", names); } /** @@ -154,8 +158,8 @@ struct ProfileManifest } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), + parseFlakeRef(fetchSettings, e[sOriginalUrl]), + parseFlakeRef(fetchSettings, e[sUrl]), e["attrPath"], e["outputs"].get() }; diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 8124292406e..ee45162302c 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -33,9 +33,9 @@ class RegistryCommand : virtual Args { if (registry) return registry; if (registry_path.empty()) { - registry = fetchers::getUserRegistry(); + registry = fetchers::getUserRegistry(fetchSettings); } else { - registry = fetchers::getCustomRegistry(registry_path); + registry = fetchers::getCustomRegistry(fetchSettings, registry_path); } return registry; } @@ -68,7 +68,7 @@ struct CmdRegistryList : StoreCommand { using namespace fetchers; - auto registries = getRegistries(store); + auto registries = getRegistries(fetchSettings, store); for (auto & registry : registries) { for (auto & entry : registry->entries) { @@ -109,8 +109,8 @@ struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand void run() override { - auto fromRef = parseFlakeRef(fromUrl); - auto toRef = parseFlakeRef(toUrl); + auto fromRef = parseFlakeRef(fetchSettings, fromUrl); + auto toRef = parseFlakeRef(fetchSettings, toUrl); auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; @@ -144,7 +144,7 @@ struct CmdRegistryRemove : RegistryCommand, Command void run() override { auto registry = getRegistry(); - registry->remove(parseFlakeRef(url).input); + registry->remove(parseFlakeRef(fetchSettings, url).input); registry->write(getRegistryPath()); } }; @@ -185,8 +185,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand { if (locked.empty()) locked = url; auto registry = getRegistry(); - auto ref = parseFlakeRef(url); - auto lockedRef = parseFlakeRef(locked); + auto ref = parseFlakeRef(fetchSettings, url); + auto lockedRef = parseFlakeRef(fetchSettings, locked); registry->remove(ref.input); auto resolved = lockedRef.resolve(store).input.getAccessor(store).second; if (!resolved.isLocked()) diff --git a/src/nix/search.cc b/src/nix/search.cc index 97ef1375ed2..7f8504d3f1e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -15,6 +15,8 @@ #include #include +#include "strings.hh" + using namespace nix; using json = nlohmann::json; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 29297253bda..7b3357700e8 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -147,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store, evalSettings); + auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2a0cbd19ffb..124a05bed2c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,14 +1,14 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" #include "thread-pool.hh" -#include "references.hh" #include "signals.hh" #include "keys.hh" #include +#include "exit.hh" + using namespace nix; struct CmdVerify : StorePathsCommand diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh index 913fdd9673d..5ca125ab5bc 100644 --- a/tests/functional/characterisation/framework.sh +++ b/tests/functional/characterisation/framework.sh @@ -63,7 +63,7 @@ function characterisationTestExit() { echo '' echo 'You can rerun this test with:' echo '' - echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test" + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.sh.test" echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 86cd6f458d0..4f5bb36aae9 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash source common.sh +source characterisation/framework.sh testDir="$PWD" cd "$TEST_ROOT" @@ -73,21 +74,31 @@ testReplResponseGeneral () { local grepMode commands expectedResponse response grepMode="$1"; shift commands="$1"; shift - expectedResponse="$1"; shift - response="$(nix repl "$@" <<< "$commands" | stripColors)" - echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ - || fail "repl command set: + # Expected response can contain newlines. + # grep can't handle multiline patterns, so replace newlines with TEST_NEWLINE + # in both expectedResponse and response. + # awk ORS always adds a trailing record separator, so we strip it with sed. + expectedResponse="$(printf '%s' "$1" | awk 1 ORS=TEST_NEWLINE | sed 's/TEST_NEWLINE$//')"; shift + # We don't need to strip trailing record separator here, since extra data is ok. + response="$(nix repl "$@" <<< "$commands" 2>&1 | stripColors | awk 1 ORS=TEST_NEWLINE)" + printf '%s' "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ + || fail "$(echo "repl command set: $commands does not respond with: +--- $expectedResponse +--- but with: +--- $response -" +--- + +" | sed 's/TEST_NEWLINE/\n/g')" } testReplResponse () { @@ -189,7 +200,7 @@ testReplResponseNoRegex ' let x = { y = { a = 1; }; inherit x; }; in x ' \ '{ - x = { ... }; + x = «repeated»; y = { ... }; } ' @@ -241,6 +252,45 @@ testReplResponseNoRegex ' ' \ '{ x = «repeated»; - y = { a = 1 }; + y = { a = 1; }; } ' + +# TODO: move init to characterisation/framework.sh +badDiff=0 +badExitCode=0 + +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" + +runRepl () { + + # That is right, we are also filtering out the testdir _without underscores_. + # This is crazy, but without it, GHA will fail to run the tests, showing paths + # _with_ underscores in the set -x log, but _without_ underscores in the + # supposed nix repl output. I have looked in a number of places, but I cannot + # find a mechanism that could cause this to happen. + local testDirNoUnderscores + testDirNoUnderscores="${testDir//_/}" + + # TODO: pass arguments to nix repl; see lang.sh + nix repl 2>&1 \ + | stripColors \ + | sed \ + -e "s@$testDir@/path/to/tests/functional@g" \ + -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ + -e "s@$nixVersion@@g" \ + -e "s@Added [0-9]* variables@Added variables@g" \ + | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ + ; +} + +for test in $(cd "$testDir/repl"; echo *.in); do + test="$(basename "$test" .in)" + in="$testDir/repl/$test.in" + actual="$testDir/repl/$test.actual" + expected="$testDir/repl/$test.expected" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + diffAndAcceptInner "$test" "$actual" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/repl/characterisation/empty b/tests/functional/repl/characterisation/empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected new file mode 100644 index 00000000000..c10c171e1e8 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -0,0 +1,24 @@ +Nix +Type :? for help. +Added variables. + +Function curriedArgs + … defined at + /path/to/tests/functional/repl/doc-comments.nix:48:5 + + A documented function. + + +"Note that users may not expect this to behave as it currently does" + +Function curriedArgs + … defined at + /path/to/tests/functional/repl/doc-comments.nix:50:5 + + The function returned by applying once + +"This won't produce documentation, because we can't actually document arbitrary values" + +error: value does not have documentation + + diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in new file mode 100644 index 00000000000..8dbbfc37052 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -0,0 +1,7 @@ +:l doc-comments.nix +:doc curriedArgs +x = curriedArgs 1 +"Note that users may not expect this to behave as it currently does" +:doc x +"This won't produce documentation, because we can't actually document arbitrary values" +:doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected new file mode 100644 index 00000000000..704c0050bd8 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. +Added variables. + +"Note that this is not yet complete" + +Function documentedFormals + … defined at + /path/to/tests/functional/repl/doc-comments.nix:57:5 + + Finds x + + diff --git a/tests/functional/repl/doc-comment-formals.in b/tests/functional/repl/doc-comment-formals.in new file mode 100644 index 00000000000..e32fb8ab1a3 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.in @@ -0,0 +1,3 @@ +:l doc-comments.nix +"Note that this is not yet complete" +:doc documentedFormals diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected new file mode 100644 index 00000000000..5ec465a96c9 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.expected @@ -0,0 +1,8 @@ +Nix +Type :? for help. +Function defined at + /path/to/tests/functional/repl/doc-comment-function.nix:2:1 + + A doc comment for a file that only contains a function + + diff --git a/tests/functional/repl/doc-comment-function.in b/tests/functional/repl/doc-comment-function.in new file mode 100644 index 00000000000..8f3c1388a09 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.in @@ -0,0 +1 @@ +:doc import ./doc-comment-function.nix diff --git a/tests/functional/repl/doc-comment-function.nix b/tests/functional/repl/doc-comment-function.nix new file mode 100644 index 00000000000..cdd2413476f --- /dev/null +++ b/tests/functional/repl/doc-comment-function.nix @@ -0,0 +1,3 @@ +/** A doc comment for a file that only contains a function */ +{ ... }: +{ } diff --git a/tests/functional/repl/doc-comments.nix b/tests/functional/repl/doc-comments.nix new file mode 100644 index 00000000000..e91ee0b513d --- /dev/null +++ b/tests/functional/repl/doc-comments.nix @@ -0,0 +1,60 @@ +{ + /** + Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + + ```nix + multiply 2 3 + => 6 + ``` + */ + multiply = x: y: x * y; + + /**👈 precisely this wide 👉*/ + measurement = x: x; + + floatedIn = /** This also works. */ + x: y: x; + + compact=/**boom*/x: x; + + # https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + /** Ignore!!! */ + unambiguous = + /** Very close */ + x: x; + + /** Firmly rigid. */ + constant = true; + + /** Immovably fixed. */ + lib.version = "9000"; + + /** Unchangeably constant. */ + lib.attr.empty = { }; + + lib.attr.undocumented = { }; + + nonStrict = /** My syntax is not strict, but I'm strict anyway. */ x: x; + strict = /** I don't have to be strict, but I am anyway. */ { ... }: null; + # Note that pre and post are the same here. I just had to name them somehow. + strictPre = /** Here's one way to do this */ a@{ ... }: a; + strictPost = /** Here's another way to do this */ { ... }@a: a; + + # TODO + + /** You won't see this. */ + curriedArgs = + /** A documented function. */ + x: + /** The function returned by applying once */ + y: + /** A function body performing summation of two items */ + x + y; + + /** Documented formals (but you won't see this comment) */ + documentedFormals = + /** Finds x */ + { /** The x attribute */ + x + }: x; +} diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected new file mode 100644 index 00000000000..4b05b653c45 --- /dev/null +++ b/tests/functional/repl/doc-compact.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function compact + … defined at + /path/to/tests/functional/repl/doc-comments.nix:18:20 + + boom + + diff --git a/tests/functional/repl/doc-compact.in b/tests/functional/repl/doc-compact.in new file mode 100644 index 00000000000..c87c4e7abbe --- /dev/null +++ b/tests/functional/repl/doc-compact.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc compact diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected new file mode 100644 index 00000000000..c6655833376 --- /dev/null +++ b/tests/functional/repl/doc-constant.expected @@ -0,0 +1,102 @@ +Nix +Type :? for help. +Added variables. + +error: value does not have documentation + +Attribute version + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:30:3 + + Immovably fixed. + +Attribute empty + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:33:3 + + Unchangeably constant. + +error: + … while evaluating the attribute 'attr.undocument' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocument' missing + at «string»:1:1: + 1| lib.attr.undocument + | ^ + Did you mean undocumented? + +Attribute constant + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:27:3 + + Firmly rigid. + +Attribute version + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:30:3 + + Immovably fixed. + +Attribute empty + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:33:3 + + Unchangeably constant. + +Attribute undocumented + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:35:3 + + No documentation found. + +error: undefined variable 'missing' + at «string»:1:1: + 1| missing + | ^ + +error: undefined variable 'constanz' + at «string»:1:1: + 1| constanz + | ^ + +error: undefined variable 'missing' + at «string»:1:1: + 1| missing.attr + | ^ + +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing + | ^ + +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing.attr + | ^ + +error: + … while evaluating the attribute 'attr.undocumental' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocumental' missing + at «string»:1:1: + 1| lib.attr.undocumental + | ^ + Did you mean undocumented? + + diff --git a/tests/functional/repl/doc-constant.in b/tests/functional/repl/doc-constant.in new file mode 100644 index 00000000000..9c0dde5e154 --- /dev/null +++ b/tests/functional/repl/doc-constant.in @@ -0,0 +1,15 @@ +:l doc-comments.nix +:doc constant +:doc lib.version +:doc lib.attr.empty +:doc lib.attr.undocument +:doc (import ./doc-comments.nix).constant +:doc (import ./doc-comments.nix).lib.version +:doc (import ./doc-comments.nix).lib.attr.empty +:doc (import ./doc-comments.nix).lib.attr.undocumented +:doc missing +:doc constanz +:doc missing.attr +:doc lib.missing +:doc lib.missing.attr +:doc lib.attr.undocumental diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected new file mode 100644 index 00000000000..30f13572586 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function floatedIn + … defined at + /path/to/tests/functional/repl/doc-comments.nix:16:5 + + This also works. + + diff --git a/tests/functional/repl/doc-floatedIn.in b/tests/functional/repl/doc-floatedIn.in new file mode 100644 index 00000000000..97c12408e63 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc floatedIn diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected new file mode 100644 index 00000000000..43f483ce943 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -0,0 +1,29 @@ +Nix +Type :? for help. +Added variables. + +Function nonStrict + … defined at + /path/to/tests/functional/repl/doc-comments.nix:37:70 + + My syntax is not strict, but I'm strict anyway. + +Function strict + … defined at + /path/to/tests/functional/repl/doc-comments.nix:38:63 + + I don't have to be strict, but I am anyway. + +Function strictPre + … defined at + /path/to/tests/functional/repl/doc-comments.nix:40:48 + + Here's one way to do this + +Function strictPost + … defined at + /path/to/tests/functional/repl/doc-comments.nix:41:53 + + Here's another way to do this + + diff --git a/tests/functional/repl/doc-lambda-flavors.in b/tests/functional/repl/doc-lambda-flavors.in new file mode 100644 index 00000000000..760c99636a6 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.in @@ -0,0 +1,5 @@ +:l doc-comments.nix +:doc nonStrict +:doc strict +:doc strictPre +:doc strictPost diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected new file mode 100644 index 00000000000..8598aaedbc7 --- /dev/null +++ b/tests/functional/repl/doc-measurement.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function measurement + … defined at + /path/to/tests/functional/repl/doc-comments.nix:13:17 + + 👈 precisely this wide 👉 + + diff --git a/tests/functional/repl/doc-measurement.in b/tests/functional/repl/doc-measurement.in new file mode 100644 index 00000000000..fecd5f9d2f0 --- /dev/null +++ b/tests/functional/repl/doc-measurement.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc measurement diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected new file mode 100644 index 00000000000..db82af91fc6 --- /dev/null +++ b/tests/functional/repl/doc-multiply.expected @@ -0,0 +1,15 @@ +Nix +Type :? for help. +Added variables. + +Function multiply + … defined at + /path/to/tests/functional/repl/doc-comments.nix:10:14 + + Perform arithmetic multiplication. It's kind of like + repeated addition, very neat. + + | multiply 2 3 + | => 6 + + diff --git a/tests/functional/repl/doc-multiply.in b/tests/functional/repl/doc-multiply.in new file mode 100644 index 00000000000..bffc6696fd4 --- /dev/null +++ b/tests/functional/repl/doc-multiply.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc multiply diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected new file mode 100644 index 00000000000..825aa1ee170 --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function unambiguous + … defined at + /path/to/tests/functional/repl/doc-comments.nix:24:5 + + Very close + + diff --git a/tests/functional/repl/doc-unambiguous.in b/tests/functional/repl/doc-unambiguous.in new file mode 100644 index 00000000000..8282a5cb95c --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc unambiguous diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index f999b7a10cb..ab357ac7895 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -71,3 +71,15 @@ test_tarball() { test_tarball '' cat test_tarball .xz xz test_tarball .gz gzip + +# Test hard links. +# All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. +# We don't preserve the hard links, because that's an optimization we think is not worth the complexity, +# so we only make sure that the contents are copied correctly. +path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .storePath)" +[[ $(cat "$path/a/b/foo") = bar ]] +[[ $(cat "$path/a/b/xyzzy") = bar ]] +[[ $(cat "$path/a/yyy") = bar ]] +[[ $(cat "$path/a/zzz") = bar ]] +[[ $(cat "$path/c/aap") = bar ]] +[[ $(cat "$path/fnord") = bar ]] diff --git a/tests/functional/tree.tar.gz b/tests/functional/tree.tar.gz new file mode 100644 index 00000000000..f1f1d996d84 Binary files /dev/null and b/tests/functional/tree.tar.gz differ diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index eacbf0d5cac..54a4a0e9e3b 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -4,12 +4,14 @@ #include #include +#include "fetch-settings.hh" #include "value.hh" #include "nixexpr.hh" +#include "nixexpr.hh" #include "eval.hh" +#include "eval-gc.hh" #include "eval-inline.hh" #include "eval-settings.hh" -#include "store-api.hh" #include "tests/libstore.hh" @@ -24,7 +26,7 @@ namespace nix { protected: LibExprTest() : LibStoreTest() - , state({}, store, evalSettings, nullptr) + , state({}, store, fetchSettings, evalSettings, nullptr) { evalSettings.nixPath = {}; } @@ -43,6 +45,7 @@ namespace nix { } bool readOnlyMode = true; + fetchers::Settings fetchSettings{}; EvalSettings evalSettings{readOnlyMode}; EvalState state; }; diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc new file mode 100644 index 00000000000..d3547ec6a31 --- /dev/null +++ b/tests/unit/libfetchers/git-utils.cc @@ -0,0 +1,112 @@ +#include "git-utils.hh" +#include "file-system.hh" +#include "gmock/gmock.h" +#include +#include +#include +#include +#include "fs-sink.hh" +#include "serialise.hh" + +namespace nix { + +class GitUtilsTest : public ::testing::Test +{ + // We use a single repository for all tests. + Path tmpDir; + std::unique_ptr delTmpDir; + +public: + void SetUp() override + { + tmpDir = createTempDir(); + delTmpDir = std::make_unique(tmpDir, true); + + // Create the repo with libgit2 + git_libgit2_init(); + git_repository * repo = nullptr; + auto r = git_repository_init(&repo, tmpDir.c_str(), 0); + ASSERT_EQ(r, 0); + git_repository_free(repo); + } + + void TearDown() override + { + // Destroy the AutoDelete, triggering removal + // not AutoDelete::reset(), which would cancel the deletion. + delTmpDir.reset(); + } + + ref openRepo() + { + return GitRepo::openRepo(tmpDir, true, false); + } +}; + +void writeString(CreateRegularFileSink & fileSink, std::string contents, bool executable) +{ + if (executable) + fileSink.isExecutable(); + fileSink.preallocateContents(contents.size()); + fileSink(contents); +} + +TEST_F(GitUtilsTest, sink_basic) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + // TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory + // here + // The sync method does not document this behavior, should probably renamed because it's not very + // general, and I can't imagine that "non-conventional" archives or any other source to be handled by + // this sink. + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + sink->createRegularFile(CanonPath("foo-1.1/bye"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "thanks for all the fish", false); + }); + sink->createSymlink(CanonPath("foo-1.1/bye-link"), "bye"); + sink->createDirectory(CanonPath("foo-1.1/empty")); + sink->createDirectory(CanonPath("foo-1.1/links")); + sink->createHardlink(CanonPath("foo-1.1/links/foo"), CanonPath("foo-1.1/hello")); + + // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); + + auto result = sink->sync(); + auto accessor = repo->getAccessor(result, false); + auto entries = accessor->readDirectory(CanonPath::root); + ASSERT_EQ(entries.size(), 5); + ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); + ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); + ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); + ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0); + ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); +}; + +TEST_F(GitUtilsTest, sink_hardlink) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + + try { + sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); + FAIL() << "Expected an exception"; + } catch (const nix::Error & e) { + ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + } +}; + +} // namespace nix diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk index 286a590304a..30aa142a5e2 100644 --- a/tests/unit/libfetchers/local.mk +++ b/tests/unit/libfetchers/local.mk @@ -29,7 +29,7 @@ libfetchers-tests_LIBS = \ libstore-test-support libutil-test-support \ libfetchers libstore libutil -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) ifdef HOST_WINDOWS # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space diff --git a/tests/unit/libflake/flakeref.cc b/tests/unit/libflake/flakeref.cc index 2b7809b938d..d704a26d36c 100644 --- a/tests/unit/libflake/flakeref.cc +++ b/tests/unit/libflake/flakeref.cc @@ -1,5 +1,6 @@ #include +#include "fetch-settings.hh" #include "flake/flakeref.hh" namespace nix { @@ -11,8 +12,9 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(to_string, doesntReencodeUrl) { + fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; - auto flakeref = parseFlakeRef(s); + auto flakeref = parseFlakeRef(fetchSettings, s); auto parsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc new file mode 100644 index 00000000000..eb31a240804 --- /dev/null +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "legacy-ssh-store.hh" + +namespace nix { + +TEST(LegacySSHStore, constructConfig) +{ + LegacySSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +} diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 90e7d3047c7..41b2fb0abc2 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -58,6 +58,7 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'legacy-ssh-store.cc', 'machines.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', @@ -67,6 +68,7 @@ sources = files( 'path.cc', 'references.cc', 'serve-protocol.cc', + 'ssh-store.cc', 'store-reference.cc', 'worker-protocol.cc', ) diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 7637cb366aa..9e9c6303d90 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -26,9 +26,9 @@ static UnkeyedValidPathInfo makeEmpty() }; } -static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) +static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) { - UnkeyedValidPathInfo info = ValidPathInfo { + ValidPathInfo info = ValidPathInfo { store, "foo", FixedOutputInfo { @@ -57,6 +57,9 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo } return info; } +static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) { + return makeFullKeyed(store, includeImpureInfo); +} #define JSON_TEST(STEM, OBJ, PURE) \ TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ @@ -86,4 +89,13 @@ JSON_TEST(empty_impure, makeEmpty(), true) JSON_TEST(pure, makeFull(*store, false), false) JSON_TEST(impure, makeFull(*store, true), true) +TEST_F(PathInfoTest, PathInfo_full_shortRefs) { + ValidPathInfo it = makeFullKeyed(*store, true); + // it.references = unkeyed.references; + auto refs = it.shortRefs(); + ASSERT_EQ(refs.size(), 2); + ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"); } + +} // namespace nix diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc new file mode 100644 index 00000000000..7010ad157dd --- /dev/null +++ b/tests/unit/libstore/ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "ssh-store.hh" + +namespace nix { + +TEST(SSHStore, constructConfig) +{ + SSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +} diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc new file mode 100644 index 00000000000..122a09dcb32 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc @@ -0,0 +1,34 @@ +#include +#include "tracing-file-system-object-sink.hh" + +namespace nix::test { + +void TracingFileSystemObjectSink::createDirectory(const CanonPath & path) +{ + std::cerr << "createDirectory(" << path << ")\n"; + sink.createDirectory(path); +} + +void TracingFileSystemObjectSink::createRegularFile( + const CanonPath & path, std::function fn) +{ + std::cerr << "createRegularFile(" << path << ")\n"; + sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { + // We could wrap this and trace about the chunks of data and such + fn(crf); + }); +} + +void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target) +{ + std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; + sink.createSymlink(path, target); +} + +void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target) +{ + std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; + sink.createHardlink(path, target); +} + +} // namespace nix::test diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh new file mode 100644 index 00000000000..895ac366405 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh @@ -0,0 +1,41 @@ +#pragma once +#include "fs-sink.hh" + +namespace nix::test { + +/** + * A `FileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingFileSystemObjectSink : public virtual FileSystemObjectSink +{ + FileSystemObjectSink & sink; +public: + TracingFileSystemObjectSink(FileSystemObjectSink & sink) + : sink(sink) + { + } + + void createDirectory(const CanonPath & path) override; + + void createRegularFile(const CanonPath & path, std::function fn) override; + + void createSymlink(const CanonPath & path, const std::string & target) override; +}; + +/** + * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink +{ + ExtendedFileSystemObjectSink & sink; +public: + TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink) + : TracingFileSystemObjectSink(sink) + , sink(sink) + { + } + + void createHardlink(const CanonPath & path, const CanonPath & target) override; +}; + +} diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index a0125d023ba..3d01d980609 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -229,7 +229,7 @@ TEST_F(GitTest, both_roundrip) { mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular); - ASSERT_EQ(*files, *files2); + ASSERT_EQ(files->root, files2->root); } TEST(GitLsRemote, parseSymrefLineWithReference) { diff --git a/tests/unit/libutil/position.cc b/tests/unit/libutil/position.cc new file mode 100644 index 00000000000..484ecc2479b --- /dev/null +++ b/tests/unit/libutil/position.cc @@ -0,0 +1,122 @@ +#include + +#include "position.hh" + +namespace nix { + +inline Pos::Origin makeStdin(std::string s) +{ + return Pos::Stdin{make_ref(s)}; +} + +TEST(Position, getSnippetUpTo_0) +{ + Pos::Origin o = makeStdin(""); + Pos p(1, 1, o); + ASSERT_EQ(p.getSnippetUpTo(p), ""); +} +TEST(Position, getSnippetUpTo_1) +{ + Pos::Origin o = makeStdin("x"); + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 0, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 99, o); + Pos end(99, 0, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + + // "x" might be preferable, but we only care about not crashing for invalid inputs + ASSERT_EQ(start.getSnippetUpTo(end), ""); + + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + Pos start(1, 1, o); + Pos end(1, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 1, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } +} +TEST(Position, getSnippetUpTo_2) +{ + Pos::Origin o = makeStdin("asdf\njkl\nqwer"); + { + Pos start(1, 1, o); + Pos end(1, 2, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "a"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + + // nullopt? I feel like changing the column handling would just make it more fragile + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 2, o); + Pos end(1, 3, o); + ASSERT_EQ(start.getSnippetUpTo(end), "s"); + } + { + Pos start(1, 2, o); + Pos end(2, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\nj"); + } + { + Pos start(1, 2, o); + Pos end(3, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl\nq"); + } + { + Pos start(1, 2, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl"); + } + { + Pos start(1, 4, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "f\njkl"); + } + { + Pos start(1, 5, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); + } + { + Pos start(1, 6, o); // invalid: starting column past last "line character", ie at the newline + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); // jkl might be acceptable for this invalid start position + } + { + Pos start(1, 1, o); + Pos end(2, 0, o); // invalid + ASSERT_EQ(start.getSnippetUpTo(end), "asdf\n"); + } +} + +TEST(Position, example_1) +{ + Pos::Origin o = makeStdin(" unambiguous = \n /** Very close */\n x: x;\n# ok\n"); + Pos start(2, 5, o); + Pos end(2, 22, o); + ASSERT_EQ(start.getSnippetUpTo(end), "/** Very close */"); +} + +} // namespace nix diff --git a/tests/unit/libutil/references.cc b/tests/unit/libutil/references.cc index a517d9aa175..c3efa6d5101 100644 --- a/tests/unit/libutil/references.cc +++ b/tests/unit/libutil/references.cc @@ -15,7 +15,7 @@ struct RewriteParams { strRewrites.insert(from + "->" + to); return os << "OriginalString: " << bar.originalString << std::endl << - "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Rewrites: " << dropEmptyInitThenConcatStringsSep(",", strRewrites) << std::endl << "Expected result: " << bar.finalString; } }; diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc new file mode 100644 index 00000000000..47a20770e2e --- /dev/null +++ b/tests/unit/libutil/strings.cc @@ -0,0 +1,83 @@ +#include + +#include "strings.hh" + +namespace nix { + +using Strings = std::vector; + +/* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(concatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, justOne) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +TEST(concatStringsSep, emptyString) +{ + Strings strings; + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ","); +} + +TEST(concatStringsSep, threeEmptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ",,"); +} + +TEST(concatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); +} + +TEST(concatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); +} + +TEST(concatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +} // namespace nix diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 9be4a400d02..8a3ca8561e5 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -227,32 +227,32 @@ namespace nix { } /* ---------------------------------------------------------------------------- - * concatStringsSep + * dropEmptyInitThenConcatStringsSep * --------------------------------------------------------------------------*/ - TEST(concatStringsSep, buildCommaSeparatedString) { + TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) { Strings strings; strings.push_back("this"); strings.push_back("is"); strings.push_back("great"); - ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); } - TEST(concatStringsSep, buildStringWithEmptySeparator) { + TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) { Strings strings; strings.push_back("this"); strings.push_back("is"); strings.push_back("great"); - ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); } - TEST(concatStringsSep, buildSingleString) { + TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) { Strings strings; strings.push_back("this"); - ASSERT_EQ(concatStringsSep(",", strings), "this"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this"); } /* ----------------------------------------------------------------------------