From 35bdb9cee7dbb7f8733cab1b1fe327f525496773 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Jun 2024 16:05:57 +0200 Subject: [PATCH 01/70] Support hard links in tarballs Fixes #10395. --- src/libfetchers/git-utils.cc | 48 +++++++++++++++++++++++++++++++++-- src/libfetchers/git-utils.hh | 2 +- src/libutil/fs-sink.hh | 13 ++++++++++ src/libutil/tarfile.cc | 11 +++++--- src/libutil/tarfile.hh | 2 +- tests/functional/tarball.sh | 9 +++++++ tests/functional/tree.tar.gz | Bin 0 -> 298 bytes 7 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 tests/functional/tree.tar.gz diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 2ea1e15ed8b..32b35931a6f 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -115,10 +115,10 @@ git_oid hashToOID(const Hash & hash) return oid; } -Object lookupObject(git_repository * repo, const git_oid & oid) +Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t type = GIT_OBJECT_ANY) { Object obj; - if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + if (git_object_lookup(Setter(obj), repo, &oid, type)) { auto err = git_error_last(); throw Error("getting Git object '%s': %s", oid, err->message); } @@ -909,6 +909,50 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); } + void createHardlink(const Path & path, const CanonPath & target) override + { + auto pathComponents = tokenizeString>(path, "/"); + if (!prepareDirs(pathComponents, false)) return; + + auto relTarget = CanonPath(path).parent()->makeRelative(target); + + auto dir = pendingDirs.rbegin(); + + // For each ../ component at the start, go up one directory. + std::string_view relTargetLeft(relTarget); + while (hasPrefix(relTargetLeft, "../")) { + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s'", target); + ++dir; + relTargetLeft = relTargetLeft.substr(3); + } + + // Look up the remainder of the target, starting at the + // top-most `git_treebuilder`. + std::variant curDir{dir->builder.get()}; + Object tree; // needed to keep `entry` alive + const git_tree_entry * entry = nullptr; + + for (auto & c : CanonPath(relTargetLeft)) { + if (auto builder = std::get_if(&curDir)) { + if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s'", target); + curDir = *git_tree_entry_id(entry); + } else if (auto oid = std::get_if(&curDir)) { + tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); + if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s'", target); + curDir = *git_tree_entry_id(entry); + } + } + + assert(entry); + + addToTree(*pathComponents.rbegin(), + *git_tree_entry_id(entry), + git_tree_entry_filemode(entry)); + } + Hash sync() override { updateBuilders({}); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 29d79955480..495916f6283 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -7,7 +7,7 @@ namespace nix { namespace fetchers { struct PublicKey; } -struct GitFileSystemObjectSink : FileSystemObjectSink +struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { /** * Flush builder and return a final Git hash. diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index ae577819a25..994f1996075 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -41,6 +41,19 @@ struct FileSystemObjectSink virtual void createSymlink(const Path & path, const std::string & target) = 0; }; +/** + * An extension of `FileSystemObjectSink` that supports file types + * that are not supported by Nix's FSO model. + */ +struct ExtendedFileSystemObjectSink : FileSystemObjectSink +{ + /** + * Create a hard link. The target must be the path of a previously + * encountered file relative to the root of the FSO. + */ + virtual void createHardlink(const Path & path, const CanonPath & target) = 0; +}; + /** * Recursively copy file system objects from the source into the sink. */ diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 6bb2bd2f32b..e45cfaf85b4 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -163,7 +163,7 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink) { time_t lastModified = 0; @@ -183,7 +183,12 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin lastModified = std::max(lastModified, archive_entry_mtime(entry)); - switch (archive_entry_filetype(entry)) { + if (auto target = archive_entry_hardlink(entry)) { + parseSink.createHardlink(path, CanonPath(target)); + continue; + } + + switch (auto type = archive_entry_filetype(entry)) { case AE_IFDIR: parseSink.createDirectory(path); @@ -220,7 +225,7 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin } default: - throw Error("file '%s' in tarball has unsupported file type", path); + throw Error("file '%s' in tarball has unsupported file type %d", path, type); } } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 705d211e437..0517177dbe6 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -41,6 +41,6 @@ void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink); } diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index ce162ddcef0..5d4749eb24d 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -59,3 +59,12 @@ test_tarball() { test_tarball '' cat test_tarball .xz xz test_tarball .gz gzip + +# Test hard links. +path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .storePath)" +[[ $(cat "$path/a/b/foo") = bar ]] +[[ $(cat "$path/a/b/xyzzy") = bar ]] +[[ $(cat "$path/a/yyy") = bar ]] +[[ $(cat "$path/a/zzz") = bar ]] +[[ $(cat "$path/c/aap") = bar ]] +[[ $(cat "$path/fnord") = bar ]] diff --git a/tests/functional/tree.tar.gz b/tests/functional/tree.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f1f1d996d8494c930ede3d0b66574c8e6efb014c GIT binary patch literal 298 zcmV+_0oDE=iwFP!000001MQdHYQ!KAMtzjLLC2rb=W)00RcVTwLX)R&bY%;LZb_DL z3;oWG1caG*bVjF~(vy;fRswSwbzrLB+POM5ly=@4Vr!jOq%~Qs1{Th%@_wFT9tM@t z%W=FpFXeNOg!(cS|50`aZ1K;Ui+|$+{P&>wUzSBKMiJ~UzJKswt0k+hC6HKZ9E)eQ}53c@Cj`>+G#*Y3^#PAOQ00000000000KmO`0`*Phd;ll_0G_ItkN^Mx literal 0 HcmV?d00001 From bd37a70d8f0ed43c21ce3cf746e9a20bede221ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Jun 2024 19:39:42 +0200 Subject: [PATCH 02/70] Update tests/functional/tarball.sh Co-authored-by: Robert Hensing --- tests/functional/tarball.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 5d4749eb24d..86b8ef2f59b 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -61,6 +61,9 @@ test_tarball .xz xz test_tarball .gz gzip # Test hard links. +# All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. +# We don't preserve the hard links, because that's an optimization we think is not worth the complexity, +# so we only make sure that the contents are copied correctly. path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .storePath)" [[ $(cat "$path/a/b/foo") = bar ]] [[ $(cat "$path/a/b/xyzzy") = bar ]] From efd4bf653325ce2a3f243923f39092e77af29fe3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Jun 2024 14:41:35 +0200 Subject: [PATCH 03/70] Update src/libfetchers/git-utils.cc Co-authored-by: Robert Hensing --- src/libfetchers/git-utils.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 32b35931a6f..ca02fbc8974 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -914,11 +914,16 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink auto pathComponents = tokenizeString>(path, "/"); if (!prepareDirs(pathComponents, false)) return; + // We can't just look up the path from the start of the root, since + // some parent directories may not have finished yet, so we compute + // a relative path that helps us find the right git_tree_builder or object. auto relTarget = CanonPath(path).parent()->makeRelative(target); auto dir = pendingDirs.rbegin(); // For each ../ component at the start, go up one directory. + // CanonPath::makeRelative() always puts all .. elements at the start, + // so they're all handled by this loop: std::string_view relTargetLeft(relTarget); while (hasPrefix(relTargetLeft, "../")) { if (dir == pendingDirs.rend()) From 992912f3b4a0eb8aaa7f27e5cb67e351b6ab07ac Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 1 Jul 2024 17:12:34 +0200 Subject: [PATCH 04/70] test-support: Add TracingFileSystemObjectSink --- src/libutil/fs-sink.hh | 2 +- .../tests/tracing-file-system-object-sink.cc | 33 +++++++++++++++ .../tests/tracing-file-system-object-sink.hh | 41 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc create mode 100644 tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 994f1996075..32dcb2f0122 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -45,7 +45,7 @@ struct FileSystemObjectSink * An extension of `FileSystemObjectSink` that supports file types * that are not supported by Nix's FSO model. */ -struct ExtendedFileSystemObjectSink : FileSystemObjectSink +struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink { /** * Create a hard link. The target must be the path of a previously diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc new file mode 100644 index 00000000000..737e022130a --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc @@ -0,0 +1,33 @@ +#include +#include "tracing-file-system-object-sink.hh" + +namespace nix::test { + +void TracingFileSystemObjectSink::createDirectory(const Path & path) +{ + std::cerr << "createDirectory(" << path << ")\n"; + sink.createDirectory(path); +} + +void TracingFileSystemObjectSink::createRegularFile(const Path & path, std::function fn) +{ + std::cerr << "createRegularFile(" << path << ")\n"; + sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { + // We could wrap this and trace about the chunks of data and such + fn(crf); + }); +} + +void TracingFileSystemObjectSink::createSymlink(const Path & path, const std::string & target) +{ + std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; + sink.createSymlink(path, target); +} + +void TracingExtendedFileSystemObjectSink::createHardlink(const Path & path, const CanonPath & target) +{ + std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; + sink.createHardlink(path, target); +} + +} // namespace nix::test diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh new file mode 100644 index 00000000000..9527b0be3e9 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh @@ -0,0 +1,41 @@ +#pragma once +#include "fs-sink.hh" + +namespace nix::test { + +/** + * A `FileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingFileSystemObjectSink : public virtual FileSystemObjectSink +{ + FileSystemObjectSink & sink; +public: + TracingFileSystemObjectSink(FileSystemObjectSink & sink) + : sink(sink) + { + } + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path, std::function fn); + + void createSymlink(const Path & path, const std::string & target); +}; + +/** + * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink +{ + ExtendedFileSystemObjectSink & sink; +public: + TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink) + : TracingFileSystemObjectSink(sink) + , sink(sink) + { + } + + void createHardlink(const Path & path, const CanonPath & target); +}; + +} From 1fac22b16e216e3ab8850d542be587eb379605b6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 1 Jul 2024 17:13:48 +0200 Subject: [PATCH 05/70] GitFileSystemObjectSink: Add path context to some messages --- src/libfetchers/git-utils.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ca02fbc8974..c41cfe6834f 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -927,7 +927,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink std::string_view relTargetLeft(relTarget); while (hasPrefix(relTargetLeft, "../")) { if (dir == pendingDirs.rend()) - throw Error("invalid hard link target '%s'", target); + throw Error("invalid hard link target '%s' for path '%s'", target, path); ++dir; relTargetLeft = relTargetLeft.substr(3); } @@ -940,13 +940,14 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink for (auto & c : CanonPath(relTargetLeft)) { if (auto builder = std::get_if(&curDir)) { + assert(*builder); if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) - throw Error("cannot find hard link target '%s'", target); + throw Error("cannot find hard link target '%s' for path '%s'", target, path); curDir = *git_tree_entry_id(entry); } else if (auto oid = std::get_if(&curDir)) { tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) - throw Error("cannot find hard link target '%s'", target); + throw Error("cannot find hard link target '%s' for path '%s'", target, path); curDir = *git_tree_entry_id(entry); } } From a409c1a882e65878476f58d033321eb05e163ab5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 1 Jul 2024 17:22:26 +0200 Subject: [PATCH 06/70] Start unit testing GitFileSystemObjectSink --- tests/unit/libfetchers/git-utils.cc | 90 +++++++++++++++++++++++++++++ tests/unit/libfetchers/local.mk | 2 +- 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/unit/libfetchers/git-utils.cc diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc new file mode 100644 index 00000000000..c007d1c3c96 --- /dev/null +++ b/tests/unit/libfetchers/git-utils.cc @@ -0,0 +1,90 @@ +#include "git-utils.hh" +#include "file-system.hh" +#include "gmock/gmock.h" +#include +#include +#include +#include +#include "fs-sink.hh" +#include "serialise.hh" + +namespace nix { + +class GitUtilsTest : public ::testing::Test +{ + // We use a single repository for all tests. + Path tmpDir; + std::unique_ptr delTmpDir; + +public: + void SetUp() override + { + tmpDir = createTempDir(); + delTmpDir = std::make_unique(tmpDir, true); + + // Create the repo with libgit2 + git_libgit2_init(); + git_repository * repo = nullptr; + auto r = git_repository_init(&repo, tmpDir.c_str(), 0); + ASSERT_EQ(r, 0); + git_repository_free(repo); + } + + void TearDown() override + { + // Destroy the AutoDelete, triggering removal + // not AutoDelete::reset(), which would cancel the deletion. + delTmpDir.reset(); + } + + ref openRepo() + { + return GitRepo::openRepo(tmpDir, true, false); + } +}; + +void writeString(CreateRegularFileSink & fileSink, std::string contents, bool executable) +{ + if (executable) + fileSink.isExecutable(); + fileSink.preallocateContents(contents.size()); + fileSink(contents); +} + +TEST_F(GitUtilsTest, sink_basic) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + // TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory + // here + // The sync method does not document this behavior, should probably renamed because it's not very + // general, and I can't imagine that "non-conventional" archives or any other source to be handled by + // this sink. + + sink->createDirectory("foo-1.1"); + + sink->createRegularFile( + "foo-1.1/hello", [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); }); + sink->createRegularFile("foo-1.1/bye", [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "thanks for all the fish", false); + }); + sink->createSymlink("foo-1.1/bye-link", "bye"); + sink->createDirectory("foo-1.1/empty"); + sink->createDirectory("foo-1.1/links"); + sink->createHardlink("foo-1.1/links/foo", CanonPath("foo-1.1/hello")); + + // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); + + auto result = sink->sync(); + auto accessor = repo->getAccessor(result, false); + auto entries = accessor->readDirectory(CanonPath::root); + ASSERT_EQ(entries.size(), 5); + ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); + ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); + ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); + ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0); + ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); +}; + +} // namespace nix diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk index d576d28f382..f4e56a50107 100644 --- a/tests/unit/libfetchers/local.mk +++ b/tests/unit/libfetchers/local.mk @@ -29,7 +29,7 @@ libfetchers-tests_LIBS = \ libstore-test-support libutil-test-support \ libfetchers libstore libutil -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) ifdef HOST_WINDOWS # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space From f0329568b5afeddd03db4b969489e19a1bd2ee97 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 1 Jul 2024 17:14:27 +0200 Subject: [PATCH 07/70] GitFileSystemObjectSink: catch an overflow --- src/libfetchers/git-utils.cc | 2 ++ tests/unit/libfetchers/git-utils.cc | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index c41cfe6834f..64fd39aedce 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -931,6 +931,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink ++dir; relTargetLeft = relTargetLeft.substr(3); } + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); // Look up the remainder of the target, starting at the // top-most `git_treebuilder`. diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index c007d1c3c96..3c06b593a67 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -87,4 +87,24 @@ TEST_F(GitUtilsTest, sink_basic) ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); }; +TEST_F(GitUtilsTest, sink_hardlink) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory("foo-1.1"); + + sink->createRegularFile( + "foo-1.1/hello", [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); }); + + try { + sink->createHardlink("foo-1.1/link", CanonPath("hello")); + FAIL() << "Expected an exception"; + } catch (const nix::Error & e) { + ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + } +}; + } // namespace nix From 4fd8f19ecfd8ced21c0f43bb3f3e3567d1d38bcd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 11 Jul 2024 12:14:48 +0200 Subject: [PATCH 08/70] Fix build to use CanonPath in new FSO sinks --- src/libfetchers/git-utils.cc | 7 +++-- src/libutil/fs-sink.hh | 2 +- src/libutil/tarfile.cc | 2 +- tests/unit/libfetchers/git-utils.cc | 26 ++++++++++--------- .../tests/tracing-file-system-object-sink.cc | 9 ++++--- .../tests/tracing-file-system-object-sink.hh | 8 +++--- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index a88bdc8b691..ecc71ae471d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -909,9 +909,12 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); } - void createHardlink(const Path & path, const CanonPath & target) override + void createHardlink(const CanonPath & path, const CanonPath & target) override { - auto pathComponents = tokenizeString>(path, "/"); + std::vector pathComponents; + for (auto & c : path) + pathComponents.emplace_back(c); + if (!prepareDirs(pathComponents, false)) return; // We can't just look up the path from the start of the root, since diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index e5e24007326..774c0d942bd 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -51,7 +51,7 @@ struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink * Create a hard link. The target must be the path of a previously * encountered file relative to the root of the FSO. */ - virtual void createHardlink(const Path & path, const CanonPath & target) = 0; + virtual void createHardlink(const CanonPath & path, const CanonPath & target) = 0; }; /** diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f3b2f55b51e..2e323629512 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -196,7 +196,7 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & lastModified = std::max(lastModified, archive_entry_mtime(entry)); if (auto target = archive_entry_hardlink(entry)) { - parseSink.createHardlink(path, CanonPath(target)); + parseSink.createHardlink(cpath, CanonPath(target)); continue; } diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index 3c06b593a67..d3547ec6a31 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -62,17 +62,18 @@ TEST_F(GitUtilsTest, sink_basic) // general, and I can't imagine that "non-conventional" archives or any other source to be handled by // this sink. - sink->createDirectory("foo-1.1"); + sink->createDirectory(CanonPath("foo-1.1")); - sink->createRegularFile( - "foo-1.1/hello", [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); }); - sink->createRegularFile("foo-1.1/bye", [](CreateRegularFileSink & fileSink) { + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + sink->createRegularFile(CanonPath("foo-1.1/bye"), [](CreateRegularFileSink & fileSink) { writeString(fileSink, "thanks for all the fish", false); }); - sink->createSymlink("foo-1.1/bye-link", "bye"); - sink->createDirectory("foo-1.1/empty"); - sink->createDirectory("foo-1.1/links"); - sink->createHardlink("foo-1.1/links/foo", CanonPath("foo-1.1/hello")); + sink->createSymlink(CanonPath("foo-1.1/bye-link"), "bye"); + sink->createDirectory(CanonPath("foo-1.1/empty")); + sink->createDirectory(CanonPath("foo-1.1/links")); + sink->createHardlink(CanonPath("foo-1.1/links/foo"), CanonPath("foo-1.1/hello")); // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); @@ -92,13 +93,14 @@ TEST_F(GitUtilsTest, sink_hardlink) auto repo = openRepo(); auto sink = repo->getFileSystemObjectSink(); - sink->createDirectory("foo-1.1"); + sink->createDirectory(CanonPath("foo-1.1")); - sink->createRegularFile( - "foo-1.1/hello", [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); }); + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); try { - sink->createHardlink("foo-1.1/link", CanonPath("hello")); + sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc index 737e022130a..122a09dcb32 100644 --- a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc @@ -3,13 +3,14 @@ namespace nix::test { -void TracingFileSystemObjectSink::createDirectory(const Path & path) +void TracingFileSystemObjectSink::createDirectory(const CanonPath & path) { std::cerr << "createDirectory(" << path << ")\n"; sink.createDirectory(path); } -void TracingFileSystemObjectSink::createRegularFile(const Path & path, std::function fn) +void TracingFileSystemObjectSink::createRegularFile( + const CanonPath & path, std::function fn) { std::cerr << "createRegularFile(" << path << ")\n"; sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { @@ -18,13 +19,13 @@ void TracingFileSystemObjectSink::createRegularFile(const Path & path, std::func }); } -void TracingFileSystemObjectSink::createSymlink(const Path & path, const std::string & target) +void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target) { std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; sink.createSymlink(path, target); } -void TracingExtendedFileSystemObjectSink::createHardlink(const Path & path, const CanonPath & target) +void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target) { std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; sink.createHardlink(path, target); diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh index 9527b0be3e9..895ac366405 100644 --- a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh @@ -15,11 +15,11 @@ public: { } - void createDirectory(const Path & path) override; + void createDirectory(const CanonPath & path) override; - void createRegularFile(const Path & path, std::function fn); + void createRegularFile(const CanonPath & path, std::function fn) override; - void createSymlink(const Path & path, const std::string & target); + void createSymlink(const CanonPath & path, const std::string & target) override; }; /** @@ -35,7 +35,7 @@ public: { } - void createHardlink(const Path & path, const CanonPath & target); + void createHardlink(const CanonPath & path, const CanonPath & target) override; }; } From 0395ff9bd39fb966b69abc76ae9e1f0f2dd1c7ca Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 11 Jul 2024 15:01:38 +0200 Subject: [PATCH 09/70] packaging: Set darwinMinVersion to fix x86_64-darwin build Ported from https://github.com/NixOS/nixpkgs/pull/326172 Co-authored-by: Emily --- packaging/dependencies.nix | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 34b3449718d..73ba9cd586f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -11,11 +11,28 @@ versionSuffix, }: +let + prevStdenv = stdenv; +in + let inherit (pkgs) lib; root = ../.; + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. resolvePath = p: p; From 87323a5689f4789d9fc25271a16ba57c57f76392 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jul 2024 16:21:27 +0200 Subject: [PATCH 10/70] Remove unused InstallableFlake::getFlakeOutputs() --- src/libcmd/installable-flake.cc | 14 -------------- src/libcmd/installable-flake.hh | 2 -- 2 files changed, 16 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d42fa7aaccc..899919550e6 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c140d..30240a35ae3 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -52,8 +52,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; From 61080554ab03201b2c70c127f6d97dcfd76d6058 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Jun 2024 16:33:41 +0200 Subject: [PATCH 11/70] SymbolStr: Remove std::string conversion This refactoring allows the symbol table to be stored as something other than std::strings. --- src/libcmd/installables.cc | 4 ++-- src/libexpr-c/nix_api_value.cc | 4 ++-- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval.cc | 8 ++++---- src/libexpr/get-drvs.cc | 4 ++-- src/libexpr/primops.cc | 4 ++-- src/libexpr/symbol-table.hh | 4 ++-- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 2 +- src/libflake/flake/flake.cc | 6 +++--- src/libutil/suggestions.cc | 4 ++-- src/libutil/suggestions.hh | 4 ++-- src/nix/flake.cc | 22 +++++++++++----------- src/nix/main.cc | 2 +- 15 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6835c512c1c..0c9e69fe88e 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -289,10 +289,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s if (v2.type() == nAttrs) { for (auto & i : *v2.attrs()) { - std::string name = state->symbols[i.name]; + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 2f2f99617e2..cb5d9ee8928 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -383,7 +383,7 @@ nix_value * nix_get_attr_byidx( try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); + *name = state->state.symbols[a.name].c_str(); nix_gc_incref(nullptr, a.value); state->state.forceValue(*a.value, nix::noPos); return as_nix_value_ptr(a.value); @@ -399,7 +399,7 @@ nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalSt try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - return ((const std::string &) (state->state.symbols[a.name])).c_str(); + return state->state.symbols[a.name].c_str(); } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9ad201b63ba..d61d9363070 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -76,7 +76,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!a) { std::set attrNames; for (auto & attr : *v->attrs()) - attrNames.insert(state.symbols[attr.name]); + attrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(attrNames, attr); throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2630c34d563..46dd3691c17 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -484,7 +484,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) auto attrNames = getAttrs(); std::set strAttrNames; for (auto & name : attrNames) - strAttrNames.insert(root->state.symbols[name]); + strAttrNames.insert(std::string(root->state.symbols[name])); return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2a08621231f..efca9dd2f3f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -633,11 +633,11 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. for (auto & j : *env.values[0]->attrs()) - vm[st[j.name]] = j.value; + vm.insert_or_assign(std::string(st[j.name]), j.value); } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) - vm[st[i.first]] = env.values[i.second]; + vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]); } } } @@ -1338,7 +1338,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) if (!(j = vAttrs->attrs()->get(name))) { std::set allAttrNames; for (auto & attr : *vAttrs->attrs()) - allAttrNames.insert(state.symbols[attr.name]); + allAttrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); state.error("attribute '%1%' missing", state.symbols[name]) .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); @@ -1496,7 +1496,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) - formalNames.insert(symbols[formal.name]); + formalNames.insert(std::string(symbols[formal.name])); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); error("function '%1%' called with unexpected argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 8967334237a..7041a3932ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -342,9 +342,9 @@ std::optional getDerivation(EvalState & state, Value & v, } -static std::string addToPath(const std::string & s1, const std::string & s2) +static std::string addToPath(const std::string & s1, std::string_view s2) { - return s1.empty() ? s2 : s1 + "." + s2; + return s1.empty() ? std::string(s2) : s1 + "." + s2; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 134363e1ab2..bcde0bb1274 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1225,7 +1225,7 @@ static void derivationStrictInternal( for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; - const std::string & key = state.symbols[i->name]; + auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { @@ -1309,7 +1309,7 @@ static void derivationStrictInternal( if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); + jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context)); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, pos, context_below); diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 967a186dd59..5c2821492be 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -30,9 +30,9 @@ public: return *s == s2; } - operator const std::string & () const + const char * c_str() const { - return *s; + return s->c_str(); } operator const std::string_view () const diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 936ecf07826..f8cc056161e 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict, out = json::object(); for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { try { - out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore); + out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); } catch (Error & e) { e.addTrace(state.positions[a->pos], HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1de8cdf848d..9734ebec498 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -9,7 +9,7 @@ namespace nix { -static XMLAttrs singletonAttrs(const std::string & name, const std::string & value) +static XMLAttrs singletonAttrs(const std::string & name, std::string_view value) { XMLAttrs attrs; attrs[name] = value; diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 6f47b599229..eb083fcee0b 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -98,7 +98,7 @@ static std::map parseFlakeInputs( const std::optional & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, + std::string_view inputName, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -178,7 +178,7 @@ static FlakeInput parseFlakeInput(EvalState & state, } if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", std::string(inputName)}}); return input; } @@ -244,7 +244,7 @@ static Flake readFlake( for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(state.symbols[formal.name], FlakeInput { - .ref = parseFlakeRef(state.symbols[formal.name]) + .ref = parseFlakeRef(std::string(state.symbols[formal.name])) }); } } diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index e67e986fb59..84c8e296f17 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -38,8 +38,8 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - std::set allMatches, - std::string query) + const std::set & allMatches, + std::string_view query) { std::set res; for (const auto & possibleMatch : allMatches) { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 9abf5ee5fad..17d1d69c16a 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -35,8 +35,8 @@ public: ) const; static Suggestions bestMatches ( - std::set allMatches, - std::string query + const std::set & allMatches, + std::string_view query ); Suggestions& operator+=(const Suggestions & other); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 84c659023a5..b65c7f59dcd 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -165,7 +165,7 @@ struct CmdFlakeLock : FlakeCommand }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { auto pos = vFlake.determinePos(noPos); state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); @@ -393,15 +393,15 @@ struct CmdFlakeCheck : FlakeCommand || (hasPrefix(name, "_") && name.substr(1) == expected); }; - auto checkSystemName = [&](const std::string & system, const PosIdx pos) { + auto checkSystemName = [&](std::string_view system, const PosIdx pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; - auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + auto checkSystemType = [&](std::string_view system, const PosIdx pos) { if (!checkAllSystems && system != localSystem) { - omittedSystems.insert(system); + omittedSystems.insert(std::string(system)); return false; } else { return true; @@ -450,7 +450,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkOverlay = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking overlay '%s'", attrPath)); @@ -469,7 +469,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkModule = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS module '%s'", attrPath)); @@ -480,9 +480,9 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + checkHydraJobs = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath)); @@ -523,7 +523,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkTemplate = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking template '%s'", attrPath)); @@ -579,7 +579,7 @@ struct CmdFlakeCheck : FlakeCommand enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vOutput, const PosIdx pos) { + [&](std::string_view name, Value & vOutput, const PosIdx pos) { Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); @@ -603,7 +603,7 @@ struct CmdFlakeCheck : FlakeCommand if (name == "checks") { state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; + std::string_view attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); diff --git a/src/nix/main.cc b/src/nix/main.cc index c90bb25a7d3..77505235185 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -429,7 +429,7 @@ void mainWrapped(int argc, char * * argv) b["doc"] = trim(stripIndentation(primOp->doc)); if (primOp->experimentalFeature) b["experimental-feature"] = primOp->experimentalFeature; - builtinsJson[state.symbols[builtin.name]] = std::move(b); + builtinsJson.emplace(state.symbols[builtin.name], std::move(b)); } for (auto & [name, info] : state.constantInfos) { auto b = nlohmann::json::object(); From f070d68c32463fab9972361e1874c9c270ec672a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 14:25:16 +0200 Subject: [PATCH 12/70] Add BaseError assignment operators The move assignment was implicitly generated and used in src/libstore/build/goal.cc:90:22: 90 | this->ex = std::move(*ex); Clang warns about this generated method being deprecated, so making them explicit fixes the warning. --- src/libutil/error.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 4b08a045e32..1fe98077e1d 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -127,6 +127,8 @@ protected: public: BaseError(const BaseError &) = default; + BaseError& operator=(const BaseError &) = default; + BaseError& operator=(BaseError &&) = default; template BaseError(unsigned int status, const Args & ... args) From 3fc77f281ef3def1f997c959687cba04660dd27d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Jul 2024 13:37:30 -0400 Subject: [PATCH 13/70] No global settings in `libnixfetchers` and `libnixflake` Progress on #5638 There are still a global fetcher and eval settings, but they are pushed down into `libnixcmd`, which is a lot less bad a place for this sort of thing. Continuing process pioneered in 52bfccf8d8112ba738e6fc9e4891f85b6b864566. --- src/libcmd/command.cc | 2 +- src/libcmd/common-eval-args.cc | 20 +++- src/libcmd/common-eval-args.hh | 15 +++ src/libcmd/installable-flake.cc | 3 +- src/libcmd/installable-flake.hh | 3 +- src/libcmd/installables.cc | 19 +-- src/libcmd/repl.cc | 4 +- src/libexpr-c/local.mk | 2 +- src/libexpr-c/nix_api_expr.cc | 2 + src/libexpr-c/nix_api_expr_internal.h | 2 + src/libexpr/eval-settings.cc | 1 - src/libexpr/eval.cc | 4 +- src/libexpr/eval.hh | 7 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 8 +- src/libfetchers/fetch-settings.cc | 9 +- src/libfetchers/fetch-settings.hh | 17 ++- src/libfetchers/fetchers.cc | 18 +-- src/libfetchers/fetchers.hh | 28 ++++- src/libfetchers/git.cc | 24 ++-- src/libfetchers/github.cc | 42 ++++--- src/libfetchers/indirect.cc | 12 +- src/libfetchers/mercurial.cc | 16 ++- src/libfetchers/path.cc | 12 +- src/libfetchers/registry.cc | 67 ++++------- src/libfetchers/registry.hh | 14 ++- src/libfetchers/tarball.cc | 14 ++- src/libflake/flake-settings.cc | 12 -- src/libflake/flake/config.cc | 6 +- src/libflake/flake/flake.cc | 111 ++++++++++-------- src/libflake/flake/flake.hh | 13 +- src/libflake/flake/flakeref.cc | 43 ++++--- src/libflake/flake/flakeref.hh | 15 ++- src/libflake/flake/lockfile.cc | 19 +-- src/libflake/flake/lockfile.hh | 8 +- src/libflake/flake/settings.cc | 7 ++ .../{flake-settings.hh => flake/settings.hh} | 9 +- src/libflake/meson.build | 6 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/bundle.cc | 4 +- src/nix/flake.cc | 9 +- src/nix/main.cc | 6 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 4 +- src/nix/registry.cc | 16 +-- src/nix/upgrade-nix.cc | 2 +- tests/unit/libexpr-support/tests/libexpr.hh | 5 +- tests/unit/libflake/flakeref.cc | 4 +- 50 files changed, 402 insertions(+), 272 deletions(-) delete mode 100644 src/libflake/flake-settings.cc create mode 100644 src/libflake/flake/settings.cc rename src/libflake/{flake-settings.hh => flake/settings.hh} (86%) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 74d146c66c5..e0e5f089025 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -132,7 +132,7 @@ ref EvalCommand::getEvalState() #else std::make_shared( #endif - lookupPath, getEvalStore(), evalSettings, getStore()) + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 62745b6815f..470a25c4ec9 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,3 +1,4 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" @@ -7,6 +8,7 @@ #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" @@ -16,6 +18,10 @@ namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + EvalSettings evalSettings { settings.readOnlyMode, { @@ -24,7 +30,7 @@ EvalSettings evalSettings { [](ref store, std::string_view rest) { experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. - auto flakeRef = parseFlakeRef(std::string { rest }, {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); auto storePath = flakeRef.resolve(store).fetchTree(store).first; return store->toRealPath(storePath); @@ -35,6 +41,12 @@ EvalSettings evalSettings { static GlobalConfig::Register rEvalSettings(&evalSettings); + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + CompatibilitySettings compatibilitySettings {}; static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); @@ -171,8 +183,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -230,7 +242,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 8d303ee7c13..c62365b32e2 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -11,17 +11,32 @@ namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; struct EvalSettings; struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + /** * @todo Get rid of global setttings variables */ extern EvalSettings evalSettings; +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + /** * Settings that control behaviors that have changed since Nix 2.3. */ diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 899919550e6..8796ad5ba79 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -196,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 30240a35ae3..8e0a232ef8a 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -78,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 0c9e69fe88e..417e1509498 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -129,7 +129,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -170,14 +170,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -338,10 +339,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -403,7 +405,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -534,7 +536,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -851,6 +854,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -873,6 +877,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index ce1c5af6997..661785335b1 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -690,14 +690,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk index 51b02562e12..227a4095b30 100644 --- a/src/libexpr-c/local.mk +++ b/src/libexpr-c/local.mk @@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ $(INCLUDE_libstore) $(INCLUDE_libstorec) \ $(INCLUDE_libexpr) $(INCLUDE_libexprc) -libexprc_LIBS = libutil libutilc libstore libstorec libexpr +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr libexprc_LDFLAGS += $(THREAD_LDFLAGS) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 13e0f5f3a9f..34e9b2744ac 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -112,12 +112,14 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c static_cast(alignof(EvalState))); auto * p2 = static_cast(p); new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, .settings = nix::EvalSettings{ nix::settings.readOnlyMode, }, .state = nix::EvalState( nix::LookupPath::parse(lookupPath), store->ptr, + p2->fetchSettings, p2->settings), }; loadConfFile(p2->settings); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index d4ccffd29d6..12f24b6eb07 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,6 +1,7 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include "fetch-settings.hh" #include "eval.hh" #include "eval-settings.hh" #include "attr-set.hh" @@ -8,6 +9,7 @@ struct EvalState { + nix::fetchers::Settings fetchSettings; nix::EvalSettings settings; nix::EvalState state; }; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 6b7b52cef32..e2151aa7fc1 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,5 +1,4 @@ #include "users.hh" -#include "config-global.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index efca9dd2f3f..9eb3d972cbf 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -217,9 +217,11 @@ static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, const EvalSettings & settings, std::shared_ptr buildStore) - : settings{settings} + : fetchSettings{fetchSettings} + , settings{settings} , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e45358055ed..5df3e92beda 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -30,6 +30,7 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +namespace fetchers { struct Settings; } struct EvalSettings; class EvalState; class StorePath; @@ -43,7 +44,7 @@ namespace eval_cache { /** * Function that implements a primop. */ -typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v); /** * Info about a primitive operation, and its implementation @@ -84,7 +85,7 @@ struct PrimOp /** * Implementation of the primop. */ - std::function::type> fun; + std::function fun; /** * Optional experimental for this to be gated on. @@ -162,6 +163,7 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const fetchers::Settings & fetchSettings; const EvalSettings & settings; SymbolTable symbols; PosTable positions; @@ -353,6 +355,7 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 7b5f4193a23..64e3abf2db4 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs.insert_or_assign("name", std::string(name)); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto [storePath, input2] = input.fetchToStore(state.store); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 567b73f9a1b..6a7accad7f6 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,7 +85,7 @@ static void fetchTree( Value & v, const FetchTreeParams & params = FetchTreeParams{} ) { - fetchers::Input input; + fetchers::Input input { state.fetchSettings }; NixStringContext context; std::optional type; if (params.isFetchGit) type = "git"; @@ -148,7 +148,7 @@ static void fetchTree( "attribute 'name' isn’t supported in call to 'fetchTree'" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { auto url = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to the fetcher", @@ -161,13 +161,13 @@ static void fetchTree( if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { attrs.emplace("exportIgnore", Explicit{true}); } - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) state.error( "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(state.fetchSettings, url); } } diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index 21c42567cb5..c7ed4c7af08 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,14 +1,9 @@ #include "fetch-settings.hh" -#include "config-global.hh" -namespace nix { +namespace nix::fetchers { -FetchSettings::FetchSettings() +Settings::Settings() { } -FetchSettings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - } diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 629967697da..f7cb34a0220 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -9,11 +9,11 @@ #include -namespace nix { +namespace nix::fetchers { -struct FetchSettings : public Config +struct Settings : public Config { - FetchSettings(); + Settings(); Setting accessTokens{this, {}, "access-tokens", R"( @@ -84,9 +84,14 @@ struct FetchSettings : public Config `narHash` attribute is specified, e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. )"}; -}; -// FIXME: don't use a global variable. -extern FetchSettings fetchSettings; + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )", + {}, true, Xp::Flakes}; +}; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 29496067832..59e77621c31 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -35,9 +35,11 @@ nlohmann::json dumpRegisterInputSchemeInfo() { return res; } -Input Input::fromURL(const std::string & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const std::string & url, bool requireTree) { - return fromURL(parseURL(url), requireTree); + return fromURL(settings, parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -49,10 +51,12 @@ static void fixupInput(Input & input) input.getLastModified(); } -Input Input::fromURL(const ParsedURL & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) { for (auto & [_, inputScheme] : *inputSchemes) { - auto res = inputScheme->inputFromURL(url, requireTree); + auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; @@ -64,7 +68,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) throw Error("input '%s' is unsupported", url.url); } -Input Input::fromAttrs(Attrs && attrs) +Input Input::fromAttrs(const Settings & settings, Attrs && attrs) { auto schemeName = ({ auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); @@ -78,7 +82,7 @@ Input Input::fromAttrs(Attrs && attrs) // but not all of them. Doing this is to support those other // operations which are supposed to be robust on // unknown/uninterpretable inputs. - Input input; + Input input { settings }; input.attrs = attrs; fixupInput(input); return input; @@ -99,7 +103,7 @@ Input Input::fromAttrs(Attrs && attrs) if (name != "type" && allowedAttrs.count(name) == 0) throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(settings, attrs); if (!res) return raw(); res->scheme = inputScheme; fixupInput(*res); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f9a..34d3bafac39 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -17,6 +17,8 @@ namespace nix::fetchers { struct InputScheme; +struct Settings; + /** * The `Input` object is generated by a specific fetcher, based on * user-supplied information, and contains @@ -28,6 +30,12 @@ struct Input { friend struct InputScheme; + const Settings * settings; + + Input(const Settings & settings) + : settings{&settings} + { } + std::shared_ptr scheme; // note: can be null Attrs attrs; @@ -42,16 +50,22 @@ public: * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromURL(const std::string & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree = true); /** * Create an `Input` from a an `Attrs`. * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromAttrs(Attrs && attrs); + static Input fromAttrs( + const Settings & settings, + Attrs && attrs); ParsedURL toURL() const; @@ -146,9 +160,13 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; + virtual std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + virtual std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const = 0; /** * What is the name of the scheme? diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 184c1383e58..076c757c5f5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -164,7 +164,9 @@ static const Hash nullRev{HashAlgorithm::SHA1}; struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -190,7 +192,7 @@ struct GitInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } @@ -222,7 +224,9 @@ struct GitInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { for (auto & [name, _] : attrs) if (name == "verifyCommit" @@ -238,7 +242,7 @@ struct GitInputScheme : InputScheme throw BadURL("invalid Git branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; auto url = fixGitURL(getStrAttr(attrs, "url")); parseURL(url); @@ -366,13 +370,13 @@ struct GitInputScheme : InputScheme /* URL of the repo, or its path if isLocal. Never a `file` URL. */ std::string url; - void warnDirty() const + void warnDirty(const Settings & settings) const { if (workdirInfo.isDirty) { - if (!fetchSettings.allowDirty) + if (!settings.allowDirty) throw Error("Git tree '%s' is dirty", url); - if (fetchSettings.warnDirty) + if (settings.warnDirty) warn("Git tree '%s' is dirty", url); } } @@ -653,7 +657,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); attrs.insert_or_assign("submodules", Explicit{ true }); attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -711,7 +715,7 @@ struct GitInputScheme : InputScheme // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out // attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -743,7 +747,7 @@ struct GitInputScheme : InputScheme verifyCommit(input, repo); } else { - repoInfo.warnDirty(); + repoInfo.warnDirty(*input.settings); if (repoInfo.workdirInfo.headRev) { input.attrs.insert_or_assign("dirtyRev", diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ddb41e63f9f..2968d2df2a4 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -31,7 +31,9 @@ struct GitArchiveInputScheme : InputScheme { virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const fetchers::Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != schemeName()) return {}; @@ -90,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme if (ref && rev) throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); @@ -119,12 +121,14 @@ struct GitArchiveInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const fetchers::Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -168,18 +172,20 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const std::string & host) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host) const { - auto tokens = fetchSettings.accessTokens.get(); + auto tokens = settings.accessTokens.get(); if (auto token = get(tokens, host)) return *token; return {}; } - Headers makeHeadersWithAuthTokens(const std::string & host) const + Headers makeHeadersWithAuthTokens( + const fetchers::Settings & settings, + const std::string & host) const { Headers headers; - auto accessToken = getAccessToken(host); + auto accessToken = getAccessToken(settings, host); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -295,7 +301,7 @@ struct GitArchiveInputScheme : InputScheme locking. FIXME: in the future, we may want to require a Git tree hash instead of a NAR hash. */ return input.getRev().has_value() - && (fetchSettings.trustTarballsFromGitForges || + && (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value()); } @@ -352,7 +358,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getOwner(input), getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -369,7 +375,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -389,7 +395,7 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) const override { auto host = getHost(input); - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -426,7 +432,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -456,7 +462,7 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } @@ -464,7 +470,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -496,7 +502,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); std::string refUri; if (ref == "HEAD") { @@ -543,14 +549,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); - Input::fromURL(fmt("git+https://%s/%s/%s", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index ba507863138..2e5cd82c78d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -8,7 +8,9 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; @@ -41,7 +43,7 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -65,13 +67,15 @@ struct IndirectInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 198795caa23..3feb3cb19ba 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -45,7 +45,9 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -68,7 +70,7 @@ struct MercurialInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } std::string_view schemeName() const override @@ -88,7 +90,9 @@ struct MercurialInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { parseURL(getStrAttr(attrs, "url")); @@ -97,7 +101,7 @@ struct MercurialInputScheme : InputScheme throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -182,10 +186,10 @@ struct MercurialInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!fetchSettings.allowDirty) + if (!input.settings->allowDirty) throw Error("Mercurial tree '%s' is unclean", actualUrl); - if (fetchSettings.warnDirty) + if (input.settings->warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 68958d55971..fca0df84b10 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -7,14 +7,16 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("path", url.path); @@ -54,11 +56,13 @@ struct PathInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "path"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 52cbac5e0a0..3c893c8ea33 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,7 +1,7 @@ +#include "fetch-settings.hh" #include "registry.hh" #include "tarball.hh" #include "users.hh" -#include "config-global.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -11,12 +11,13 @@ namespace nix::fetchers { std::shared_ptr Registry::read( + const Settings & settings, const Path & path, RegistryType type) { - auto registry = std::make_shared(type); + auto registry = std::make_shared(settings, type); if (!pathExists(path)) - return std::make_shared(type); + return std::make_shared(settings, type); try { @@ -36,8 +37,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = Input::fromAttrs(jsonToAttrs(i["from"])), - .to = Input::fromAttrs(std::move(toAttrs)), + .from = Input::fromAttrs(settings, jsonToAttrs(i["from"])), + .to = Input::fromAttrs(settings, std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -106,10 +107,10 @@ static Path getSystemRegistryPath() return settings.nixConfDir + "/registry.json"; } -static std::shared_ptr getSystemRegistry() +static std::shared_ptr getSystemRegistry(const Settings & settings) { static auto systemRegistry = - Registry::read(getSystemRegistryPath(), Registry::System); + Registry::read(settings, getSystemRegistryPath(), Registry::System); return systemRegistry; } @@ -118,25 +119,24 @@ Path getUserRegistryPath() return getConfigDir() + "/nix/registry.json"; } -std::shared_ptr getUserRegistry() +std::shared_ptr getUserRegistry(const Settings & settings) { static auto userRegistry = - Registry::read(getUserRegistryPath(), Registry::User); + Registry::read(settings, getUserRegistryPath(), Registry::User); return userRegistry; } -std::shared_ptr getCustomRegistry(const Path & p) +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p) { static auto customRegistry = - Registry::read(p, Registry::Custom); + Registry::read(settings, p, Registry::Custom); return customRegistry; } -static std::shared_ptr flagRegistry = - std::make_shared(Registry::Flag); - -std::shared_ptr getFlagRegistry() +std::shared_ptr getFlagRegistry(const Settings & settings) { + static auto flagRegistry = + std::make_shared(settings, Registry::Flag); return flagRegistry; } @@ -145,30 +145,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs) { - flagRegistry->add(from, to, extraAttrs); + getFlagRegistry(*from.settings)->add(from, to, extraAttrs); } -struct RegistrySettings : Config -{ - Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - R"( - Path or URI of the global flake registry. - - When empty, disables the global flake registry. - )", - {}, true, Xp::Flakes}; -}; - -RegistrySettings registrySettings; - -static GlobalConfig::Register rRegistrySettings(®istrySettings); - -static std::shared_ptr getGlobalRegistry(ref store) +static std::shared_ptr getGlobalRegistry(const Settings & settings, ref store) { static auto reg = [&]() { - auto path = registrySettings.flakeRegistry.get(); + auto path = settings.flakeRegistry.get(); if (path == "") { - return std::make_shared(Registry::Global); // empty registry + return std::make_shared(settings, Registry::Global); // empty registry } if (!hasPrefix(path, "/")) { @@ -178,19 +163,19 @@ static std::shared_ptr getGlobalRegistry(ref store) path = store->toRealPath(storePath); } - return Registry::read(path, Registry::Global); + return Registry::read(settings, path, Registry::Global); }(); return reg; } -Registries getRegistries(ref store) +Registries getRegistries(const Settings & settings, ref store) { Registries registries; - registries.push_back(getFlagRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getSystemRegistry()); - registries.push_back(getGlobalRegistry(store)); + registries.push_back(getFlagRegistry(settings)); + registries.push_back(getUserRegistry(settings)); + registries.push_back(getSystemRegistry(settings)); + registries.push_back(getGlobalRegistry(settings, store)); return registries; } @@ -207,7 +192,7 @@ std::pair lookupInRegistries( n++; if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); - for (auto & registry : getRegistries(store)) { + for (auto & registry : getRegistries(*input.settings, store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index f57ab1e6b59..0d68ac395e9 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -10,6 +10,8 @@ namespace nix::fetchers { struct Registry { + const Settings & settings; + enum RegistryType { Flag = 0, User = 1, @@ -29,11 +31,13 @@ struct Registry std::vector entries; - Registry(RegistryType type) - : type(type) + Registry(const Settings & settings, RegistryType type) + : settings{settings} + , type{type} { } static std::shared_ptr read( + const Settings & settings, const Path & path, RegistryType type); void write(const Path & path); @@ -48,13 +52,13 @@ struct Registry typedef std::vector> Registries; -std::shared_ptr getUserRegistry(); +std::shared_ptr getUserRegistry(const Settings & settings); -std::shared_ptr getCustomRegistry(const Path & p); +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p); Path getUserRegistryPath(); -Registries getRegistries(ref store); +Registries getRegistries(const Settings & settings, ref store); void overrideRegistry( const Input & from, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index aa8ff652f07..e049dffeb29 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -214,12 +214,14 @@ struct CurlInputScheme : InputScheme static const std::set specialParams; - std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & _url, bool requireTree) const override { if (!isValidURL(_url, requireTree)) return std::nullopt; - Input input; + Input input{settings}; auto url = _url; @@ -267,9 +269,11 @@ struct CurlInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { - Input input; + Input input{settings}; input.attrs = attrs; //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); @@ -349,7 +353,7 @@ struct TarballInputScheme : CurlInputScheme result.accessor->setPathDisplay("«" + input.to_string() + "»"); if (result.immutableUrl) { - auto immutableInput = Input::fromURL(*result.immutableUrl); + auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl); // FIXME: would be nice to support arbitrary flakerefs // here, e.g. git flakes. if (immutableInput.getType() != "tarball") diff --git a/src/libflake/flake-settings.cc b/src/libflake/flake-settings.cc deleted file mode 100644 index ba97e0ce723..00000000000 --- a/src/libflake/flake-settings.cc +++ /dev/null @@ -1,12 +0,0 @@ -#include "flake-settings.hh" -#include "config-global.hh" - -namespace nix { - -FlakeSettings::FlakeSettings() {} - -FlakeSettings flakeSettings; - -static GlobalConfig::Register rFlakeSettings(&flakeSettings); - -} diff --git a/src/libflake/flake/config.cc b/src/libflake/flake/config.cc index 49859535939..4e00d5c9368 100644 --- a/src/libflake/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -1,6 +1,6 @@ #include "users.hh" #include "config-global.hh" -#include "flake-settings.hh" +#include "flake/settings.hh" #include "flake.hh" #include @@ -30,7 +30,7 @@ static void writeTrustedList(const TrustedList & trustedList) writeFile(path, nlohmann::json(trustedList).dump()); } -void ConfigFile::apply() +void ConfigFile::apply(const Settings & flakeSettings) { std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; @@ -51,7 +51,7 @@ void ConfigFile::apply() else assert(false); - if (!whitelist.count(baseName) && !nix::flakeSettings.acceptFlakeConfig) { + if (!whitelist.count(baseName) && !flakeSettings.acceptFlakeConfig) { bool trusted = false; auto trustedList = readTrustedList(); auto tlname = get(trustedList, name); diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index eb083fcee0b..627dfe83093 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -9,7 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" -#include "flake-settings.hh" +#include "flake/settings.hh" #include "value-to-json.hh" #include "local-fs-store.hh" @@ -164,7 +164,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (attrs.count("type")) try { - input.ref = FlakeRef::fromAttrs(attrs); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, attrs); } catch (Error & e) { e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; @@ -174,11 +174,11 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(state.fetchSettings, *url, baseDir, true, input.isFlake); } if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", std::string(inputName)}}); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}}); return input; } @@ -244,7 +244,7 @@ static Flake readFlake( for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(state.symbols[formal.name], FlakeInput { - .ref = parseFlakeRef(std::string(state.symbols[formal.name])) + .ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name])) }); } } @@ -329,16 +329,19 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } -static LockFile readLockFile(const SourcePath & lockFilePath) +static LockFile readLockFile( + const fetchers::Settings & fetchSettings, + const SourcePath & lockFilePath) { return lockFilePath.pathExists() - ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath)) + ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) : LockFile(); } /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) @@ -347,21 +350,22 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); if (lockFlags.applyNixConfig) { - flake.config.apply(); + flake.config.apply(settings); state.store->setOptions(); } try { - if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } auto oldLockFile = readLockFile( + state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -597,7 +601,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -660,7 +664,7 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { - if (fetchSettings.warnDirty) + if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) @@ -692,7 +696,7 @@ LockedFlake lockFlake( if (lockFlags.commitLockFile) { std::string cm; - cm = flakeSettings.commitLockFileSummary.get(); + cm = settings.commitLockFileSummary.get(); if (cm == "") { cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); @@ -800,45 +804,48 @@ void callFlake(EvalState & state, state.callFunction(*vTmp1, vOverrides, vRes, noPos); } -static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) +void initLib(const Settings & settings) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (state.settings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); - - callFlake(state, - lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !state.settings.pureEval && flakeSettings.useRegistries, - .allowUnlocked = !state.settings.pureEval, - }), - v); -} - -static RegisterPrimOp r2({ - .name = "__getFlake", - .args = {"args"}, - .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: - - ```nix - (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix - ``` - - Unless impure evaluation is allowed (`--impure`), the flake reference - must be "locked", e.g. contain a Git revision or content hash. An - example of an unlocked usage is: + auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value * * args, Value & v) + { + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + auto flakeRef = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); + + callFlake(state, + lockFlake(settings, state, flakeRef, + LockFlags { + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }), + v); + }; - ```nix - (builtins.getFlake "github:edolstra/dwarffs").rev - ``` - )", - .fun = prim_getFlake, - .experimentalFeature = Xp::Flakes, -}); + RegisterPrimOp::primOps->push_back({ + .name = "__getFlake", + .args = {"args"}, + .doc = R"( + Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + + ```nix + (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + ``` + + Unless impure evaluation is allowed (`--impure`), the flake reference + must be "locked", e.g. contain a Git revision or content hash. An + example of an unlocked usage is: + + ```nix + (builtins.getFlake "github:edolstra/dwarffs").rev + ``` + )", + .fun = prim_getFlake, + .experimentalFeature = Xp::Flakes, + }); +} static void prim_parseFlakeRef( EvalState & state, @@ -848,7 +855,7 @@ static void prim_parseFlakeRef( { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.parseFlakeRef")); - auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto attrs = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true).toAttrs(); auto binds = state.buildBindings(attrs.size()); for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); @@ -913,7 +920,7 @@ static void prim_flakeRefToString( showType(*attr.value)).debugThrow(); } } - auto flakeRef = FlakeRef::fromAttrs(attrs); + auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); v.mkString(flakeRef.to_string()); } diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 1ba085f0f46..cce17009ce3 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -12,6 +12,16 @@ class EvalState; namespace flake { +struct Settings; + +/** + * Initialize `libnixflake` + * + * So far, this registers the `builtins.getFlake` primop, which depends + * on the choice of `flake:Settings`. + */ +void initLib(const Settings & settings); + struct FlakeInput; typedef std::map FlakeInputs; @@ -57,7 +67,7 @@ struct ConfigFile std::map settings; - void apply(); + void apply(const Settings & settings); }; /** @@ -194,6 +204,7 @@ struct LockFlags }; LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 6e4aad64d28..941790e0c38 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -48,28 +48,32 @@ FlakeRef FlakeRef::resolve(ref store) const } FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; } std::optional maybeParseFlakeRef( - const std::string & url, const std::optional & baseDir) + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir) { try { - return parseFlakeRef(url, baseDir); + return parseFlakeRef(fetchSettings, url, baseDir); } catch (Error &) { return {}; } } std::pair parsePathFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -166,7 +170,7 @@ std::pair parsePathFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")), fragment); } @@ -185,13 +189,14 @@ std::pair parsePathFlakeRefWithFragment( attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); }; /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ -std::optional> parseFlakeIdRef( +static std::optional> parseFlakeIdRef( + const fetchers::Settings & fetchSettings, const std::string & url, bool isFlake ) @@ -213,7 +218,7 @@ std::optional> parseFlakeIdRef( }; return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -221,6 +226,7 @@ std::optional> parseFlakeIdRef( } std::optional> parseURLFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool isFlake @@ -236,7 +242,7 @@ std::optional> parseURLFlakeRef( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = fetchers::Input::fromURL(parsedURL, isFlake); + auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); input.parent = baseDir; return std::make_pair( @@ -245,6 +251,7 @@ std::optional> parseURLFlakeRef( } std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -254,31 +261,34 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - if (auto res = parseFlakeIdRef(url, isFlake)) { + if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; - } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); } } std::optional> maybeParseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir) { try { - return parseFlakeRefWithFragment(url, baseDir); + return parseFlakeRefWithFragment(fetchSettings, url, baseDir); } catch (Error & e) { return {}; } } -FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +FlakeRef FlakeRef::fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs) { auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::Input::fromAttrs(fetchSettings, std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } @@ -289,13 +299,16 @@ std::pair FlakeRef::fetchTree(ref store) const } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, + std::string { prefix }, baseDir, allowMissing, isFlake); return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 04c812ed099..ad9b582c556 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -61,7 +61,9 @@ struct FlakeRef FlakeRef resolve(ref store) const; - static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + static FlakeRef fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs); std::pair fetchTree(ref store) const; }; @@ -72,6 +74,7 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -81,12 +84,15 @@ FlakeRef parseFlakeRef( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional maybeParseFlake( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -96,12 +102,15 @@ std::pair parseFlakeRefWithFragment( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional> maybeParseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index d252214dd2b..c94692d6e08 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -10,7 +10,8 @@ namespace nix::flake { -FlakeRef getFlakeRef( +static FlakeRef getFlakeRef( + const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) @@ -26,15 +27,17 @@ FlakeRef getFlakeRef( attrs.insert_or_assign(k.first, k.second); } } - return FlakeRef::fromAttrs(attrs); + return FlakeRef::fromAttrs(fetchSettings, attrs); } throw Error("attribute '%s' missing in lock file", attr); } -LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(json, "original", nullptr)) +LockedNode::LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) @@ -84,7 +87,9 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return doFind(root, path, visited); } -LockFile::LockFile(std::string_view contents, std::string_view path) +LockFile::LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path) { auto json = nlohmann::json::parse(contents); @@ -113,7 +118,7 @@ LockFile::LockFile(std::string_view contents, std::string_view path) auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(*jsonNode2); + auto input = make_ref(fetchSettings, *jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh index 7e62e6d0970..e269b6c39ec 100644 --- a/src/libflake/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -45,7 +45,9 @@ struct LockedNode : Node : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } - LockedNode(const nlohmann::json & json); + LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json); StorePath computeStorePath(Store & store) const; }; @@ -55,7 +57,9 @@ struct LockFile ref root = make_ref(); LockFile() {}; - LockFile(std::string_view contents, std::string_view path); + LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path); typedef std::map, std::string> KeyMap; diff --git a/src/libflake/flake/settings.cc b/src/libflake/flake/settings.cc new file mode 100644 index 00000000000..6a0294e6229 --- /dev/null +++ b/src/libflake/flake/settings.cc @@ -0,0 +1,7 @@ +#include "flake/settings.hh" + +namespace nix::flake { + +Settings::Settings() {} + +} diff --git a/src/libflake/flake-settings.hh b/src/libflake/flake/settings.hh similarity index 86% rename from src/libflake/flake-settings.hh rename to src/libflake/flake/settings.hh index f97c175e8a3..fee247a7d2f 100644 --- a/src/libflake/flake-settings.hh +++ b/src/libflake/flake/settings.hh @@ -10,11 +10,11 @@ #include -namespace nix { +namespace nix::flake { -struct FlakeSettings : public Config +struct Settings : public Config { - FlakeSettings(); + Settings(); Setting useRegistries{ this, @@ -47,7 +47,4 @@ struct FlakeSettings : public Config Xp::Flakes}; }; -// TODO: don't use a global variable. -extern FlakeSettings flakeSettings; - } diff --git a/src/libflake/meson.build b/src/libflake/meson.build index d3c3d3079cf..38d70c678a5 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -42,21 +42,21 @@ add_project_arguments( subdir('build-utils-meson/diagnostics') sources = files( - 'flake-settings.cc', 'flake/config.cc', 'flake/flake.cc', 'flake/flakeref.cc', - 'flake/url-name.cc', 'flake/lockfile.cc', + 'flake/settings.cc', + 'flake/url-name.cc', ) include_dirs = [include_directories('.')] headers = files( - 'flake-settings.hh', 'flake/flake.hh', 'flake/flakeref.hh', 'flake/lockfile.hh', + 'flake/settings.hh', 'flake/url-name.hh', ) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d37b16bdca0..4d373ef8d4e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -286,7 +286,7 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c72378cd5f4..5e170c99d37 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, evalSettings)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index a4bfeebbbdb..c4854951124 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 554c3654022..7d9aa771124 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -76,7 +76,9 @@ struct CmdBundle : InstallableValueCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = + parseFlakeRefWithFragmentAndExtendedOutputsSpec( + fetchSettings, bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b65c7f59dcd..cfbe58712be 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -48,19 +48,19 @@ class FlakeCommand : virtual Args, public MixFlakeOptions FlakeRef getFlakeRef() { - return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, absPath(".")); //FIXME } LockedFlake lockFlake() { - return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } std::vector getFlakeRefsForCompletion() override { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), absPath(".")) }; } }; @@ -848,7 +848,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); - auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( + fetchSettings, templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), diff --git a/src/nix/main.cc b/src/nix/main.cc index 77505235185..44e69f58876 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -19,6 +19,7 @@ #include "users.hh" #include "network-proxy.hh" #include "eval-cache.hh" +#include "flake/flake.hh" #include #include @@ -242,7 +243,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), evalSettings); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -362,6 +363,7 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + flake::initLib(flakeSettings); #if __linux__ if (isRootUser()) { @@ -418,7 +420,7 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), evalSettings); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto builtinsJson = nlohmann::json::object(); for (auto & builtin : *state.baseEnv.values[0]->attrs()) { auto b = nlohmann::json::object(); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index c6e60a53be6..db7d9e4efe6 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -195,7 +195,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store, evalSettings); + auto state = std::make_unique(myArgs.lookupPath, store, fetchSettings, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c89b8c9bde6..bb6424c4f13 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -154,8 +154,8 @@ struct ProfileManifest } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), + parseFlakeRef(fetchSettings, e[sOriginalUrl]), + parseFlakeRef(fetchSettings, e[sUrl]), e["attrPath"], e["outputs"].get() }; diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 8124292406e..ee45162302c 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -33,9 +33,9 @@ class RegistryCommand : virtual Args { if (registry) return registry; if (registry_path.empty()) { - registry = fetchers::getUserRegistry(); + registry = fetchers::getUserRegistry(fetchSettings); } else { - registry = fetchers::getCustomRegistry(registry_path); + registry = fetchers::getCustomRegistry(fetchSettings, registry_path); } return registry; } @@ -68,7 +68,7 @@ struct CmdRegistryList : StoreCommand { using namespace fetchers; - auto registries = getRegistries(store); + auto registries = getRegistries(fetchSettings, store); for (auto & registry : registries) { for (auto & entry : registry->entries) { @@ -109,8 +109,8 @@ struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand void run() override { - auto fromRef = parseFlakeRef(fromUrl); - auto toRef = parseFlakeRef(toUrl); + auto fromRef = parseFlakeRef(fetchSettings, fromUrl); + auto toRef = parseFlakeRef(fetchSettings, toUrl); auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; @@ -144,7 +144,7 @@ struct CmdRegistryRemove : RegistryCommand, Command void run() override { auto registry = getRegistry(); - registry->remove(parseFlakeRef(url).input); + registry->remove(parseFlakeRef(fetchSettings, url).input); registry->write(getRegistryPath()); } }; @@ -185,8 +185,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand { if (locked.empty()) locked = url; auto registry = getRegistry(); - auto ref = parseFlakeRef(url); - auto lockedRef = parseFlakeRef(locked); + auto ref = parseFlakeRef(fetchSettings, url); + auto lockedRef = parseFlakeRef(fetchSettings, locked); registry->remove(ref.input); auto resolved = lockedRef.resolve(store).input.getAccessor(store).second; if (!resolved.isLocked()) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 29297253bda..7b3357700e8 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -147,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store, evalSettings); + auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index eacbf0d5cac..c85545e6c9d 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -4,8 +4,10 @@ #include #include +#include "fetch-settings.hh" #include "value.hh" #include "nixexpr.hh" +#include "nixexpr.hh" #include "eval.hh" #include "eval-inline.hh" #include "eval-settings.hh" @@ -24,7 +26,7 @@ namespace nix { protected: LibExprTest() : LibStoreTest() - , state({}, store, evalSettings, nullptr) + , state({}, store, fetchSettings, evalSettings, nullptr) { evalSettings.nixPath = {}; } @@ -43,6 +45,7 @@ namespace nix { } bool readOnlyMode = true; + fetchers::Settings fetchSettings{}; EvalSettings evalSettings{readOnlyMode}; EvalState state; }; diff --git a/tests/unit/libflake/flakeref.cc b/tests/unit/libflake/flakeref.cc index 2b7809b938d..d704a26d36c 100644 --- a/tests/unit/libflake/flakeref.cc +++ b/tests/unit/libflake/flakeref.cc @@ -1,5 +1,6 @@ #include +#include "fetch-settings.hh" #include "flake/flakeref.hh" namespace nix { @@ -11,8 +12,9 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(to_string, doesntReencodeUrl) { + fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; - auto flakeref = parseFlakeRef(s); + auto flakeref = parseFlakeRef(fetchSettings, s); auto parsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; From 8df041cbc6686eebda7efced8dab256838cff900 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 15:37:54 +0200 Subject: [PATCH 14/70] Solve unused header warnings reported by clangd --- src/libcmd/repl.cc | 4 +--- src/libexpr-c/nix_api_expr.cc | 4 +--- src/libexpr/attr-set.hh | 1 - src/libexpr/eval-error.hh | 2 -- src/libexpr/eval-settings.hh | 1 + src/libexpr/eval.cc | 4 +--- src/libexpr/eval.hh | 6 +++--- src/libexpr/nixexpr.cc | 1 - src/libexpr/nixexpr.hh | 3 --- src/libexpr/pos-table.hh | 4 +--- src/libexpr/primops.cc | 1 - src/libexpr/value.hh | 1 - src/libfetchers/fetchers.hh | 2 ++ src/libfetchers/tarball.cc | 2 -- src/libfetchers/tarball.hh | 9 +++++---- src/libflake/flake/flakeref.hh | 6 ++---- src/libflake/flake/lockfile.cc | 3 ++- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 4 ---- src/libstore/derivations.hh | 1 - src/libstore/derived-path.hh | 1 + src/libstore/filetransfer.hh | 10 ++++++---- src/libstore/gc-store.hh | 3 ++- src/libstore/machines.hh | 2 +- src/libstore/misc.cc | 2 ++ src/libstore/nar-accessor.cc | 1 - src/libstore/serve-protocol-impl.hh | 1 - src/libstore/store-api.hh | 4 ---- src/libstore/unix/user-lock.cc | 7 ++++--- src/libstore/unix/user-lock.hh | 6 ++---- src/libutil/args.hh | 2 +- src/libutil/error.hh | 4 ---- src/libutil/file-content-address.hh | 2 -- src/libutil/fs-sink.hh | 1 - src/libutil/logging.hh | 1 - src/libutil/ref.hh | 2 -- src/libutil/source-accessor.hh | 1 + src/libutil/terminal.hh | 3 ++- src/libutil/types.hh | 2 -- src/nix-store/nix-store.cc | 6 +++--- src/nix/config-check.cc | 1 + src/nix/env.cc | 4 +++- src/nix/flake.cc | 1 - src/nix/main.cc | 3 +-- src/nix/verify.cc | 4 ++-- tests/unit/libexpr-support/tests/libexpr.hh | 2 +- 46 files changed, 53 insertions(+), 84 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index ce1c5af6997..dc5a311e7fc 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include "repl-interacter.hh" #include "repl.hh" @@ -10,8 +9,6 @@ #include "shared.hh" #include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" @@ -29,6 +26,7 @@ #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 13e0f5f3a9f..2494c653c00 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -1,12 +1,10 @@ #include -#include #include #include -#include "config.hh" #include "eval.hh" +#include "eval-gc.hh" #include "globals.hh" -#include "util.hh" #include "eval-settings.hh" #include "nix_api_expr.h" diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index ba798196d3c..9d10783d980 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -5,7 +5,6 @@ #include "symbol-table.hh" #include -#include namespace nix { diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index fe48e054bf2..2f604647331 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -1,7 +1,5 @@ #pragma once -#include - #include "error.hh" #include "pos-idx.hh" diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 191dde21aa9..b915ba530a0 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -2,6 +2,7 @@ ///@file #include "config.hh" +#include "ref.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index efca9dd2f3f..1da9346931e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,6 @@ #include "eval.hh" +#include "eval-gc.hh" #include "eval-settings.hh" -#include "hash.hh" #include "primops.hh" #include "print-options.hh" #include "exit.hh" @@ -16,7 +16,6 @@ #include "print.hh" #include "filtering-source-accessor.hh" #include "memory-source-accessor.hh" -#include "signals.hh" #include "gc-small-vector.hh" #include "url.hh" #include "fetch-to-store.hh" @@ -24,7 +23,6 @@ #include "parser-tab.hh" #include -#include #include #include #include diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e45358055ed..e5818a501e3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,21 +3,21 @@ #include "attr-set.hh" #include "eval-error.hh" -#include "eval-gc.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "position.hh" +#include "pos-table.hh" #include "source-accessor.hh" #include "search-path.hh" #include "repl-exit-status.hh" +#include "ref.hh" #include #include -#include -#include #include namespace nix { diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 44198a25238..81638916550 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -1,5 +1,4 @@ #include "nixexpr.hh" -#include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" #include "util.hh" diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e37e3bdd153..5152e31196e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -6,11 +6,8 @@ #include "value.hh" #include "symbol-table.hh" -#include "error.hh" -#include "position.hh" #include "eval-error.hh" #include "pos-idx.hh" -#include "pos-table.hh" namespace nix { diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh index 8a0a3ba86fa..ba2b91cf35e 100644 --- a/src/libexpr/pos-table.hh +++ b/src/libexpr/pos-table.hh @@ -1,10 +1,8 @@ #pragma once -#include -#include +#include #include -#include "chunked-vector.hh" #include "pos-idx.hh" #include "position.hh" #include "sync.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bcde0bb1274..127823d4944 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,4 +1,3 @@ -#include "archive.hh" #include "derivations.hh" #include "downstream-placeholder.hh" #include "eval-inline.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 208cab21d60..1f7b75f6eb4 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include "symbol-table.hh" diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f9a..6126b263c9f 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -11,6 +11,8 @@ #include #include +#include "ref.hh" + namespace nix { class Store; class StorePath; struct SourceAccessor; } namespace nix::fetchers { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index aa8ff652f07..4425575bdb5 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -2,12 +2,10 @@ #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" -#include "globals.hh" #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" #include "types.hh" -#include "split.hh" #include "store-path-accessor.hh" #include "store-api.hh" #include "git-utils.hh" diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index ba0dfd6230a..d9bdd123d58 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -1,11 +1,12 @@ #pragma once -#include "types.hh" -#include "path.hh" -#include "hash.hh" - #include +#include "hash.hh" +#include "path.hh" +#include "ref.hh" +#include "types.hh" + namespace nix { class Store; struct SourceAccessor; diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 04c812ed099..54794ccab49 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -1,14 +1,12 @@ #pragma once ///@file +#include + #include "types.hh" -#include "hash.hh" #include "fetchers.hh" #include "outputs-spec.hh" -#include -#include - namespace nix { class Store; diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index d252214dd2b..83ca2c678e3 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -1,6 +1,7 @@ +#include + #include "lockfile.hh" #include "store-api.hh" -#include "url-parts.hh" #include #include diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index fc55fe3f1b2..fee4d0c1e11 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -8,7 +8,6 @@ #include "signals.hh" #include -#include #include #include @@ -23,6 +22,7 @@ #include +#include "exit.hh" namespace nix { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index aa44e1321fa..712b404d3e1 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -8,13 +8,9 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" -#include "exit.hh" #include -#include - - namespace nix { int handleExceptions(const std::string & programName, std::function fun); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 522523e4597..9b82e5d12d8 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -14,7 +14,6 @@ #include #include - namespace nix { struct StoreDirConfig; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b238f844e56..af66ed4f0d0 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -5,6 +5,7 @@ #include "outputs-spec.hh" #include "comparator.hh" #include "config.hh" +#include "ref.hh" #include diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1c271cbec03..1f5b4ab934c 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,13 +1,15 @@ #pragma once ///@file -#include "types.hh" -#include "hash.hh" -#include "config.hh" - #include #include +#include "logging.hh" +#include "types.hh" +#include "ref.hh" +#include "config.hh" +#include "serialise.hh" + namespace nix { struct FileTransferSettings : Config diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index ab1059fb1ec..020f770b07a 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "store-api.hh" +#include +#include "store-api.hh" namespace nix { diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 2a55c445667..983652d5f8b 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" +#include "ref.hh" #include "store-reference.hh" namespace nix { diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index cc3f4884f51..dd0efbe1910 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,3 +1,5 @@ +#include + #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b1079b027d6..6376efbf4c7 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -3,7 +3,6 @@ #include #include -#include #include diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index 67bc5dc6eed..6f3b177acd2 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -10,7 +10,6 @@ #include "serve-protocol.hh" #include "length-prefixed-protocol-helper.hh" -#include "store-api.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a5effb4c131..749d7ea098d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -18,14 +18,10 @@ #include #include -#include #include -#include -#include #include #include #include -#include namespace nix { diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index 8057aa13e1b..29f4b2cb31c 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -1,12 +1,13 @@ +#include +#include +#include + #include "user-lock.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" #include "users.hh" -#include -#include - namespace nix { #if __linux__ diff --git a/src/libstore/unix/user-lock.hh b/src/libstore/unix/user-lock.hh index 1c268e1fbd5..a7caf8518f3 100644 --- a/src/libstore/unix/user-lock.hh +++ b/src/libstore/unix/user-lock.hh @@ -1,10 +1,8 @@ #pragma once ///@file -#include "types.hh" - -#include - +#include +#include #include namespace nix { diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7759b74a93c..f6e74e67efd 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include #include #include #include @@ -11,6 +10,7 @@ #include "types.hh" #include "experimental-features.hh" +#include "ref.hh" namespace nix { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1fe98077e1d..8cc8fb303bc 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -16,16 +16,12 @@ */ #include "suggestions.hh" -#include "ref.hh" -#include "types.hh" #include "fmt.hh" #include #include #include -#include #include -#include #include #include diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index 4c7218f1941..ec42d3d3493 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -2,8 +2,6 @@ ///@file #include "source-accessor.hh" -#include "fs-sink.hh" -#include "util.hh" namespace nix { diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 774c0d942bd..de165fb7d5f 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "serialise.hh" #include "source-accessor.hh" #include "file-system.hh" diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 9e81132e354..250f9209972 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "error.hh" #include "config.hh" diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 03aa642739c..8318251bde2 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,9 +1,7 @@ #pragma once ///@file -#include #include -#include #include namespace nix { diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index cc8db01f59c..32ab3685d28 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -4,6 +4,7 @@ #include "canon-path.hh" #include "hash.hh" +#include "ref.hh" namespace nix { diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh index 902e759456d..7ff05a487c3 100644 --- a/src/libutil/terminal.hh +++ b/src/libutil/terminal.hh @@ -1,7 +1,8 @@ #pragma once ///@file -#include "types.hh" +#include +#include namespace nix { /** diff --git a/src/libutil/types.hh b/src/libutil/types.hh index c86f52175d5..325e3ea7351 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,12 +1,10 @@ #pragma once ///@file -#include "ref.hh" #include #include #include -#include #include #include #include diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index d0840a02e5e..f073074e851 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,13 +2,11 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" -#include "build-result.hh" #include "store-cast.hh" #include "local-fs-store.hh" #include "log-store.hh" #include "serve-protocol.hh" #include "serve-protocol-connection.hh" -#include "serve-protocol-impl.hh" #include "shared.hh" #include "graphml.hh" #include "legacy.hh" @@ -23,12 +21,14 @@ #include #include -#include #include #include #include +#include "build-result.hh" +#include "exit.hh" +#include "serve-protocol-impl.hh" namespace nix_store { diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 9575bf33887..09d14073386 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -1,6 +1,7 @@ #include #include "command.hh" +#include "exit.hh" #include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" diff --git a/src/nix/env.cc b/src/nix/env.cc index 021c47cbbd0..bc9cd91ad81 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -1,6 +1,8 @@ +#include +#include + #include "command.hh" #include "run.hh" -#include using namespace nix; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b65c7f59dcd..896425c5ac2 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -19,7 +19,6 @@ #include "users.hh" #include -#include #include using namespace nix; diff --git a/src/nix/main.cc b/src/nix/main.cc index 77505235185..ec3e2a4067f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,9 +1,8 @@ -#include - #include "args/root.hh" #include "current-process.hh" #include "command.hh" #include "common-args.hh" +#include "eval-gc.hh" #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2a0cbd19ffb..124a05bed2c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,14 +1,14 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" #include "thread-pool.hh" -#include "references.hh" #include "signals.hh" #include "keys.hh" #include +#include "exit.hh" + using namespace nix; struct CmdVerify : StorePathsCommand diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index eacbf0d5cac..bfb425bfff5 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -7,9 +7,9 @@ #include "value.hh" #include "nixexpr.hh" #include "eval.hh" +#include "eval-gc.hh" #include "eval-inline.hh" #include "eval-settings.hh" -#include "store-api.hh" #include "tests/libstore.hh" From 27eaeebc4164341afdbbede8bf33ec71ce866a5b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 15:38:17 +0200 Subject: [PATCH 15/70] nar-accessor.cc: Silence unused variable warning --- src/libstore/nar-accessor.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 6376efbf4c7..9a541bb7776 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -73,7 +73,10 @@ struct NarAccessor : public SourceAccessor NarMember & createMember(const CanonPath & path, NarMember member) { size_t level = 0; - for (auto _ : path) ++level; + for (auto _ : path) { + (void)_; + ++level; + } while (parents.size() > level) parents.pop(); From 51a12b38bda241623aa12a400c066f0b3a95606a Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Fri, 12 Jul 2024 09:17:31 -0400 Subject: [PATCH 16/70] Fix stackoverflow during doc generation On some systems, previous usage of `match` may cause a stackoverflow (presumably due to the large size of the match result). Avoid this by (ab)using `replaceStrings` to test for containment without using regexes, thereby avoiding the issue. The causal configuration seems to be the stack size hard limit, which e.g. Amazon Linux sets, whereas most Linux distros leave unlimited. Match the fn name to similar fn in nixpkgs.lib, but different implementation that does not use `match`. This impl gives perhaps unexpected results when the needle is `""`, but the scope of this is narrow and that case is a bit odd anyway. This makes for some duplication-of-work as we do a different `replaceStrings` if this one is true, but this only runs during doc generation at build time so has no runtime impact. See https://github.com/NixOS/nix/issues/11085 for details. --- doc/manual/generate-manpage.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index ba5667a4305..90eaa1a73fe 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -116,9 +116,12 @@ let storeInfo = commandInfo.stores; inherit inlineHTML; }; + hasInfix = infix: content: + builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content); in optionalString (details ? doc) ( - if match ".*@store-types@.*" details.doc != null + # An alternate implementation with builtins.match stack overflowed on some systems. + if hasInfix "@store-types@" details.doc then help-stores else details.doc ); From 11a6db5993426940f9746623005114e735658c9d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 17:37:27 +0200 Subject: [PATCH 17/70] Remove unused operator<=>'s that darwin can't generate It was complaining *a lot*, with dozens of MB of logs. --- src/libstore/content-address.hh | 8 -------- src/libstore/store-reference.hh | 1 - 2 files changed, 9 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6cc3b7cd9e7..270e04789eb 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -217,8 +217,6 @@ struct StoreReferences * iff self is true. */ size_t size() const; - - auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -234,8 +232,6 @@ struct TextInfo * disallowed */ StorePathSet references; - - auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -254,8 +250,6 @@ struct FixedOutputInfo * References to other store objects or this one. */ StoreReferences references; - - auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -272,8 +266,6 @@ struct ContentAddressWithReferences Raw raw; - auto operator <=>(const ContentAddressWithReferences &) const = default; - MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); /** diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh index e99335c0d57..459cea9c20e 100644 --- a/src/libstore/store-reference.hh +++ b/src/libstore/store-reference.hh @@ -71,7 +71,6 @@ struct StoreReference Params params; bool operator==(const StoreReference & rhs) const = default; - auto operator<=>(const StoreReference & rhs) const = default; /** * Render the whole store reference as a URI, including parameters. From cdc23b67a66ff3daf495b8d1e5b199095e01bac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Jun 2024 19:12:36 +0200 Subject: [PATCH 18/70] Provide std::hash --- src/libexpr/eval.cc | 6 +++--- src/libexpr/eval.hh | 10 +++++----- src/libutil/source-path.hh | 11 +++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 72ae159cb9d..584828a097e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1011,7 +1011,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) if (!e) e = parseExprFromFile(resolvedPath); - fileParseCache[resolvedPath] = e; + fileParseCache.emplace(resolvedPath, e); try { auto dts = debugRepl @@ -1034,8 +1034,8 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) throw; } - fileEvalCache[resolvedPath] = v; - if (path != resolvedPath) fileEvalCache[path] = v; + fileEvalCache.emplace(resolvedPath, v); + if (path != resolvedPath) fileEvalCache.emplace(path, v); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7dbf61c5dfe..d368bc04943 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -307,15 +307,15 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - Sync> srcToStore; + Sync> srcToStore; /** * A cache from path names to parse trees. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileParseCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileParseCache; #else - typedef std::map FileParseCache; + typedef std::unordered_map FileParseCache; #endif FileParseCache fileParseCache; @@ -323,9 +323,9 @@ private: * A cache from path names to values. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileEvalCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileEvalCache; #else - typedef std::map FileEvalCache; + typedef std::unordered_map FileEvalCache; #endif FileEvalCache fileEvalCache; diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 83ec6295de1..94174412787 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -115,8 +115,19 @@ struct SourcePath { return {accessor, accessor->resolveSymlinks(path, mode)}; } + + friend class std::hash; }; std::ostream & operator << (std::ostream & str, const SourcePath & path); } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::SourcePath & s) const noexcept + { + return std::hash{}(s.path); + } +}; From bc83b9dc1fbf52ca7aabee275264a8fb850bbb9b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 May 2024 18:46:38 -0400 Subject: [PATCH 19/70] Remove `comparator.hh` and switch to `<=>` in a bunch of places Known behavior changes: - `MemorySourceAccessor`'s comparison operators no longer forget to compare the `SourceAccessor` base class. Progress on #10832 What remains for that issue is hopefully much easier! --- src/libcmd/built-path.cc | 43 +++++++++----------- src/libcmd/built-path.hh | 16 ++++++-- src/libexpr-c/nix_api_external.cc | 2 +- src/libexpr/attr-set.hh | 4 +- src/libexpr/eval.cc | 2 +- src/libexpr/pos-idx.hh | 9 +---- src/libexpr/symbol-table.hh | 3 +- src/libexpr/value.hh | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/fetchers.hh | 2 +- src/libflake/flake/flakeref.cc | 5 --- src/libflake/flake/flakeref.hh | 2 +- src/libflake/flake/lockfile.cc | 5 --- src/libflake/flake/lockfile.hh | 3 -- src/libstore/build-result.cc | 14 +------ src/libstore/build-result.hh | 4 +- src/libstore/content-address.hh | 18 +++++++++ src/libstore/derivations.hh | 48 +++++++++++----------- src/libstore/derived-path-map.cc | 19 ++++----- src/libstore/derived-path-map.hh | 34 +++++++++++----- src/libstore/derived-path.cc | 47 ++++++++++++---------- src/libstore/derived-path.hh | 18 +++++++-- src/libstore/nar-info.cc | 9 ----- src/libstore/nar-info.hh | 4 +- src/libstore/outputs-spec.hh | 12 ++++-- src/libstore/path-info.cc | 8 +--- src/libstore/path-info.hh | 44 +++++++++++++++++++-- src/libutil/args.cc | 3 +- src/libutil/args.hh | 2 +- src/libutil/canon-path.hh | 7 ++-- src/libutil/comparator.hh | 57 +++++++-------------------- src/libutil/error.cc | 17 +++----- src/libutil/error.hh | 5 +-- src/libutil/experimental-features.hh | 1 - src/libutil/git.hh | 3 +- src/libutil/hash.cc | 4 +- src/libutil/hash.hh | 4 +- src/libutil/memory-source-accessor.hh | 34 +++++++++++++--- src/libutil/position.cc | 6 --- src/libutil/position.hh | 19 ++++----- src/libutil/ref.hh | 4 +- src/libutil/source-accessor.hh | 4 +- src/libutil/source-path.cc | 11 ++---- src/libutil/source-path.hh | 5 +-- src/libutil/suggestions.hh | 4 +- src/libutil/url.cc | 2 +- src/libutil/url.hh | 2 +- src/nix/profile.cc | 4 +- tests/unit/libutil/git.cc | 2 +- 49 files changed, 304 insertions(+), 275 deletions(-) diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d7e..905e70f32c9 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) - -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE - -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0eefb..dc78d3e599d 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 3c3dd6ca99d..3565092a4a6 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -115,7 +115,7 @@ class NixCExternalValue : public nix::ExternalValueBase /** * Compare to another value of the same type. */ - virtual bool operator==(const ExternalValueBase & b) const override + virtual bool operator==(const ExternalValueBase & b) const noexcept override { if (!desc.equal) { return false; diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 9d10783d980..4df9a1acdc9 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -27,9 +27,9 @@ struct Attr Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name), pos(pos), value(value) { }; Attr() { }; - bool operator < (const Attr & a) const + auto operator <=> (const Attr & a) const { - return name < a.name; + return name <=> a.name; } }; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 72ae159cb9d..2ede55de7b6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2842,7 +2842,7 @@ std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & } -bool ExternalValueBase::operator==(const ExternalValueBase & b) const +bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept { return false; } diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh index e94fd85c673..e1349156022 100644 --- a/src/libexpr/pos-idx.hh +++ b/src/libexpr/pos-idx.hh @@ -28,20 +28,15 @@ public: return id > 0; } - bool operator<(const PosIdx other) const + auto operator<=>(const PosIdx other) const { - return id < other.id; + return id <=> other.id; } bool operator==(const PosIdx other) const { return id == other.id; } - - bool operator!=(const PosIdx other) const - { - return id != other.id; - } }; inline PosIdx noPos = {}; diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 5c2821492be..b85725e1234 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -62,9 +62,8 @@ public: explicit operator bool() const { return id > 0; } - bool operator<(const Symbol other) const { return id < other.id; } + auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } - bool operator!=(const Symbol other) const { return id != other.id; } }; /** diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f7b75f6eb4..1f4d72d3926 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -111,7 +111,7 @@ class ExternalValueBase * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ - virtual bool operator ==(const ExternalValueBase & b) const; + virtual bool operator ==(const ExternalValueBase & b) const noexcept; /** * Print the value as JSON. Defaults to unconvertable, i.e. throws an error diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 59e77621c31..dee1f687b2e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -150,7 +150,7 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::operator ==(const Input & other) const +bool Input::operator ==(const Input & other) const noexcept { return attrs == other.attrs; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 185987fec73..a5f9bdcc6d5 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -89,7 +89,7 @@ public: */ bool isLocked() const; - bool operator ==(const Input & other) const; + bool operator ==(const Input & other) const noexcept; bool contains(const Input & other) const; diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 941790e0c38..a57fce9f319 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -36,11 +36,6 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -bool FlakeRef::operator ==(const FlakeRef & other) const -{ - return input == other.input && subdir == other.subdir; -} - FlakeRef FlakeRef::resolve(ref store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 974cf55a4bf..1064538a72a 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -46,7 +46,7 @@ struct FlakeRef */ Path subdir; - bool operator==(const FlakeRef & other) const; + bool operator ==(const FlakeRef & other) const = default; FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index 28afba5fe2e..792dda740f2 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -249,11 +249,6 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON().first == other.toJSON().first; } -bool LockFile::operator !=(const LockFile & other) const -{ - return !(*this == other); -} - InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh index e269b6c39ec..841931c1119 100644 --- a/src/libflake/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -74,9 +74,6 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; - // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 - // that is still the default on aarch64-linux) - bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 18f519c5c61..96cbfd62fff 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -2,17 +2,7 @@ namespace nix { -GENERATE_CMP_EXT( - , - BuildResult, - me->status, - me->errorMsg, - me->timesBuilt, - me->isNonDeterministic, - me->builtOutputs, - me->startTime, - me->stopTime, - me->cpuUser, - me->cpuSystem); +bool BuildResult::operator==(const BuildResult &) const noexcept = default; +std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; } diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 3636ad3a4c3..8c66cfeb353 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,7 +3,6 @@ #include "realisation.hh" #include "derived-path.hh" -#include "comparator.hh" #include #include @@ -101,7 +100,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; - DECLARE_CMP(BuildResult); + bool operator ==(const BuildResult &) const noexcept; + std::strong_ordering operator <=>(const BuildResult &) const noexcept; bool success() { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 270e04789eb..bb515013a04 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -73,6 +73,7 @@ struct ContentAddressMethod Raw raw; + bool operator ==(const ContentAddressMethod &) const = default; auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -159,6 +160,7 @@ struct ContentAddress */ Hash hash; + bool operator ==(const ContentAddress &) const = default; auto operator <=>(const ContentAddress &) const = default; /** @@ -217,6 +219,10 @@ struct StoreReferences * iff self is true. */ size_t size() const; + + bool operator ==(const StoreReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -232,6 +238,10 @@ struct TextInfo * disallowed */ StorePathSet references; + + bool operator ==(const TextInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -250,6 +260,10 @@ struct FixedOutputInfo * References to other store objects or this one. */ StoreReferences references; + + bool operator ==(const FixedOutputInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -266,6 +280,10 @@ struct ContentAddressWithReferences Raw raw; + bool operator ==(const ContentAddressWithReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const ContentAddressWithReferences &) const = default; + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); /** diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9b82e5d12d8..58e5328a5eb 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -8,7 +8,6 @@ #include "repair-flag.hh" #include "derived-path-map.hh" #include "sync.hh" -#include "comparator.hh" #include "variant-wrapper.hh" #include @@ -32,7 +31,8 @@ struct DerivationOutput { StorePath path; - GENERATE_CMP(InputAddressed, me->path); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -56,7 +56,8 @@ struct DerivationOutput */ StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - GENERATE_CMP(CAFixed, me->ca); + bool operator == (const CAFixed &) const = default; + auto operator <=> (const CAFixed &) const = default; }; /** @@ -76,7 +77,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashAlgo); + bool operator == (const CAFloating &) const = default; + auto operator <=> (const CAFloating &) const = default; }; /** @@ -84,7 +86,8 @@ struct DerivationOutput * isn't known yet. */ struct Deferred { - GENERATE_CMP(Deferred); + bool operator == (const Deferred &) const = default; + auto operator <=> (const Deferred &) const = default; }; /** @@ -103,7 +106,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashAlgo); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -116,7 +120,8 @@ struct DerivationOutput Raw raw; - GENERATE_CMP(DerivationOutput, me->raw); + bool operator == (const DerivationOutput &) const = default; + auto operator <=> (const DerivationOutput &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); @@ -177,7 +182,8 @@ struct DerivationType { */ bool deferred; - GENERATE_CMP(InputAddressed, me->deferred); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -201,7 +207,8 @@ struct DerivationType { */ bool fixed; - GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed); + bool operator == (const ContentAddressed &) const = default; + auto operator <=> (const ContentAddressed &) const = default; }; /** @@ -211,7 +218,8 @@ struct DerivationType { * type, but has some restrictions on its usage. */ struct Impure { - GENERATE_CMP(Impure); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -222,7 +230,8 @@ struct DerivationType { Raw raw; - GENERATE_CMP(DerivationType, me->raw); + bool operator == (const DerivationType &) const = default; + auto operator <=> (const DerivationType &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationType); @@ -312,14 +321,9 @@ struct BasicDerivation static std::string_view nameFromPath(const StorePath & storePath); - GENERATE_CMP(BasicDerivation, - me->outputs, - me->inputSrcs, - me->platform, - me->builder, - me->args, - me->env, - me->name); + bool operator == (const BasicDerivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BasicDerivation &) const = default; }; class Store; @@ -377,9 +381,9 @@ struct Derivation : BasicDerivation const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - GENERATE_CMP(Derivation, - static_cast(*me), - me->inputDrvs); + bool operator == (const Derivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const Derivation &) const = default; }; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 4c1ea417a36..c97d52773eb 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -54,17 +54,18 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single namespace nix { -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode, - me->value, - me->childMap); +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept = default; -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>, - me->map); +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept = default; +#endif +template struct DerivedPathMap>::ChildNode; template struct DerivedPathMap>; }; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 393cdedf747..bd60fe88710 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -47,7 +47,11 @@ struct DerivedPathMap { */ Map childMap; - DECLARE_CMP(ChildNode); + bool operator == (const ChildNode &) const noexcept; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // decltype(std::declval() <=> std::declval()) + // operator <=> (const ChildNode &) const noexcept; }; /** @@ -60,7 +64,10 @@ struct DerivedPathMap { */ Map map; - DECLARE_CMP(DerivedPathMap); + bool operator == (const DerivedPathMap &) const = default; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // auto operator <=> (const DerivedPathMap &) const noexcept; /** * Find the node for `k`, creating it if needed. @@ -83,14 +90,21 @@ struct DerivedPathMap { ChildNode * findSlot(const SingleDerivedPath & k); }; +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept; + +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept; + +template<> +inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +#endif -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::, - DerivedPathMap>); -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode::, - DerivedPathMap>::ChildNode); +extern template struct DerivedPathMap>::ChildNode; +extern template struct DerivedPathMap>; } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index a7b40432108..1eef881de0c 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,6 +1,7 @@ #include "derived-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,26 +9,32 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) - -CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) - -CMP(SingleDerivedPath, DerivedPathBuilt, outputs) - -#undef CMP -#undef CMP_ONE +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleDerivedPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + DerivedPathBuilt ::, + DerivedPathBuilt, + *me->drvPath, + me->outputs); +GENERATE_ONE_CMP( + , + bool, + DerivedPathBuilt ::, + <, + DerivedPathBuilt, + *me->drvPath, + me->outputs); nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index af66ed4f0d0..4ba3fb37d4c 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,7 +3,6 @@ #include "path.hh" #include "outputs-spec.hh" -#include "comparator.hh" #include "config.hh" #include "ref.hh" @@ -32,7 +31,8 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - GENERATE_CMP(DerivedPathOpaque, me->path); + bool operator == (const DerivedPathOpaque &) const = default; + auto operator <=> (const DerivedPathOpaque &) const = default; }; struct SingleDerivedPath; @@ -79,7 +79,8 @@ struct SingleDerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(SingleDerivedPathBuilt); + bool operator == (const SingleDerivedPathBuilt &) const noexcept; + std::strong_ordering operator <=> (const SingleDerivedPathBuilt &) const noexcept; }; using _SingleDerivedPathRaw = std::variant< @@ -109,6 +110,9 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { return static_cast(*this); } + bool operator == (const SingleDerivedPath &) const = default; + auto operator <=> (const SingleDerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). @@ -202,7 +206,9 @@ struct DerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(DerivedPathBuilt); + bool operator == (const DerivedPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const DerivedPathBuilt &) const noexcept; }; using _DerivedPathRaw = std::variant< @@ -231,6 +237,10 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + bool operator == (const DerivedPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const DerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 0d219a48929..3e0a754f981 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,15 +4,6 @@ namespace nix { -GENERATE_CMP_EXT( - , - NarInfo, - me->url, - me->compression, - me->fileHash, - me->fileSize, - static_cast(*me)); - NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index fd538a7cd9f..561c9a86364 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -24,7 +24,9 @@ struct NarInfo : ValidPathInfo NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - DECLARE_CMP(NarInfo); + bool operator ==(const NarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + //auto operator <=>(const NarInfo &) const = default; std::string to_string(const Store & store) const; diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 1ef99a5fc67..30d15311d0a 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -6,9 +6,7 @@ #include #include -#include "comparator.hh" #include "json-impls.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -60,7 +58,11 @@ struct OutputsSpec { Raw raw; - GENERATE_CMP(OutputsSpec, me->raw); + bool operator == (const OutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const OutputsSpec & other) const { + return raw < other.raw; + } MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); @@ -99,7 +101,9 @@ struct ExtendedOutputsSpec { Raw raw; - GENERATE_CMP(ExtendedOutputsSpec, me->raw); + bool operator == (const ExtendedOutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const ExtendedOutputsSpec &) const; MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5c27182b720..51ed5fc62e3 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,11 +3,13 @@ #include "path-info.hh" #include "store-api.hh" #include "json-utils.hh" +#include "comparator.hh" namespace nix { GENERATE_CMP_EXT( , + std::weak_ordering, UnkeyedValidPathInfo, me->deriver, me->narHash, @@ -19,12 +21,6 @@ GENERATE_CMP_EXT( me->sigs, me->ca); -GENERATE_CMP_EXT( - , - ValidPathInfo, - me->path, - static_cast(*me)); - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b6dc0855d75..caefa7975a2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,17 +32,47 @@ struct SubstitutablePathInfo using SubstitutablePathInfos = std::map; +/** + * Information about a store object. + * + * See `store/store-object` and `protocols/json/store-object-info` in + * the Nix manual + */ struct UnkeyedValidPathInfo { + /** + * Path to derivation that produced this store object, if known. + */ std::optional deriver; + /** * \todo document this */ Hash narHash; + + /** + * Other store objects this store object referes to. + */ StorePathSet references; + + /** + * When this store object was registered in the store that contains + * it, if known. + */ time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id = 0; // internal use only + + /** + * 0 = unknown + */ + uint64_t narSize = 0; + + /** + * internal use only: SQL primary key for on-disk store objects with + * `LocalStore`. + * + * @todo Remove, layer violation + */ + uint64_t id = 0; /** * Whether the path is ultimately trusted, that is, it's a @@ -75,7 +105,12 @@ struct UnkeyedValidPathInfo UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; - DECLARE_CMP(UnkeyedValidPathInfo); + bool operator == (const UnkeyedValidPathInfo &) const noexcept; + + /** + * @todo return `std::strong_ordering` once `id` is removed + */ + std::weak_ordering operator <=> (const UnkeyedValidPathInfo &) const noexcept; virtual ~UnkeyedValidPathInfo() { } @@ -95,7 +130,8 @@ struct UnkeyedValidPathInfo struct ValidPathInfo : UnkeyedValidPathInfo { StorePath path; - DECLARE_CMP(ValidPathInfo); + bool operator == (const ValidPathInfo &) const = default; + auto operator <=> (const ValidPathInfo &) const = default; /** * Return a fingerprint of the store path to be used in binary diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c202facdfea..d58f4b4aeaf 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -57,8 +57,7 @@ void Completions::add(std::string completion, std::string description) }); } -bool Completion::operator<(const Completion & other) const -{ return completion < other.completion || (completion == other.completion && description < other.description); } +auto Completion::operator<=>(const Completion & other) const noexcept = default; std::string completionMarker = "___COMPLETE___"; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index f6e74e67efd..c0236ee3d72 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -380,7 +380,7 @@ struct Completion { std::string completion; std::string description; - bool operator<(const Completion & other) const; + auto operator<=>(const Completion & other) const noexcept; }; /** diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 8f5a1c2793d..f84347dc458 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -169,7 +169,7 @@ public: * a directory is always followed directly by its children. For * instance, 'foo' < 'foo/bar' < 'foo!'. */ - bool operator < (const CanonPath & x) const + auto operator <=> (const CanonPath & x) const { auto i = path.begin(); auto j = x.path.begin(); @@ -178,10 +178,9 @@ public: if (c_i == '/') c_i = 0; auto c_j = *j; if (c_j == '/') c_j = 0; - if (c_i < c_j) return true; - if (c_i > c_j) return false; + if (auto cmp = c_i <=> c_j; cmp != 0) return cmp; } - return i == path.end() && j != x.path.end(); + return (i != path.end()) <=> (j != x.path.end()); } /** diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index cbc2bb4fd72..34ba6f4534c 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,17 +1,8 @@ #pragma once ///@file -#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, ==, my_type) -#define DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, <, my_type) -#define DECLARE_NEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, !=, my_type) - -#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ +#define GENERATE_ONE_CMP(PRE, RET, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE RET QUAL operator COMPARATOR(const MY_TYPE & other) const noexcept { \ __VA_OPT__(const MY_TYPE * me = this;) \ auto fields1 = std::tie( __VA_ARGS__ ); \ __VA_OPT__(me = &other;) \ @@ -19,30 +10,9 @@ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) -#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) -#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) - -/** - * Declare comparison methods without defining them. - */ -#define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(,,my_type) \ - DECLARE_LEQ(,,my_type) \ - DECLARE_NEQ(,,my_type) - -/** - * @param prefix This is for something before each declaration like - * `template`. - * - * @param my_type the type are defining operators for. - */ -#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ - DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_NEQ(prefix, qualification, my_type) + GENERATE_ONE_CMP(prefix, bool, qualification, ==, my_type, args) +#define GENERATE_SPACESHIP(prefix, ret, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, ret, qualification, <=>, my_type, args) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -55,15 +25,19 @@ * will generate comparison operators semantically equivalent to: * * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; + * auto operator<=>(const ClassName& other) const noexcept { + * if (auto cmp = field1 <=> other.field1; cmp != 0) + * return cmp; + * if (auto cmp = field2 <=> other.field2; cmp != 0) + * return cmp; + * ... + * return 0; * } * ``` */ #define GENERATE_CMP(args...) \ GENERATE_EQUAL(,,args) \ - GENERATE_LEQ(,,args) \ - GENERATE_NEQ(,,args) + GENERATE_SPACESHIP(,auto,,args) /** * @param prefix This is for something before each declaration like @@ -71,7 +45,6 @@ * * @param my_type the type are defining operators for. */ -#define GENERATE_CMP_EXT(prefix, my_type, args...) \ +#define GENERATE_CMP_EXT(prefix, ret, my_type, args...) \ GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ - GENERATE_LEQ(prefix, my_type ::, my_type, args) \ - GENERATE_NEQ(prefix, my_type ::, my_type, args) + GENERATE_SPACESHIP(prefix, ret, my_type ::, my_type, args) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e01f064484c..33c391963f3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -46,27 +46,22 @@ std::ostream & operator <<(std::ostream & os, const HintFmt & hf) /** * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. */ -inline bool operator<(const Trace& lhs, const Trace& rhs) +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs) { // `std::shared_ptr` does not have value semantics for its comparison // functions, so we need to check for nulls and compare the dereferenced // values here. if (lhs.pos != rhs.pos) { - if (!lhs.pos) - return true; - if (!rhs.pos) - return false; - if (*lhs.pos != *rhs.pos) - return *lhs.pos < *rhs.pos; + if (auto cmp = bool{lhs.pos} <=> bool{rhs.pos}; cmp != 0) + return cmp; + if (auto cmp = *lhs.pos <=> *rhs.pos; cmp != 0) + return cmp; } // This formats a freshly formatted hint string and then throws it away, which // shouldn't be much of a problem because it only runs when pos is equal, and this function is // used for trace printing, which is infrequent. - return lhs.hint.str() < rhs.hint.str(); + return lhs.hint.str() <=> rhs.hint.str(); } -inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } -inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } -inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 8cc8fb303bc..d7fe902d6c0 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -75,10 +75,7 @@ struct Trace { TracePrint print = TracePrint::Default; }; -inline bool operator<(const Trace& lhs, const Trace& rhs); -inline bool operator> (const Trace& lhs, const Trace& rhs); -inline bool operator<=(const Trace& lhs, const Trace& rhs); -inline bool operator>=(const Trace& lhs, const Trace& rhs); +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs); struct ErrorInfo { Verbosity level; diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 1da2a3ff55d..6ffbc0c1028 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "error.hh" #include "json-utils.hh" #include "types.hh" diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 2190cc5509c..1dbdb733582 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -39,7 +39,8 @@ struct TreeEntry Mode mode; Hash hash; - GENERATE_CMP(TreeEntry, me->mode, me->hash); + bool operator ==(const TreeEntry &) const = default; + auto operator <=>(const TreeEntry &) const = default; }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7064e96e61c..35b913e4249 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -41,7 +41,7 @@ Hash::Hash(HashAlgorithm algo) : algo(algo) } -bool Hash::operator == (const Hash & h2) const +bool Hash::operator == (const Hash & h2) const noexcept { if (hashSize != h2.hashSize) return false; for (unsigned int i = 0; i < hashSize; i++) @@ -50,7 +50,7 @@ bool Hash::operator == (const Hash & h2) const } -std::strong_ordering Hash::operator <=> (const Hash & h) const +std::strong_ordering Hash::operator <=> (const Hash & h) const noexcept { if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ef96d08c978..dc95b9f2f9b 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -88,12 +88,12 @@ public: /** * Check whether two hashes are equal. */ - bool operator == (const Hash & h2) const; + bool operator == (const Hash & h2) const noexcept; /** * Compare how two hashes are ordered. */ - std::strong_ordering operator <=> (const Hash & h2) const; + std::strong_ordering operator <=> (const Hash & h2) const noexcept; /** * Returns the length of a base-16 representation of this hash. diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index cd5146c8913..012a388c0e7 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -15,11 +15,15 @@ struct MemorySourceAccessor : virtual SourceAccessor * defining what a "file system object" is in Nix. */ struct File { + bool operator == (const File &) const noexcept; + std::strong_ordering operator <=> (const File &) const noexcept; + struct Regular { bool executable = false; std::string contents; - GENERATE_CMP(Regular, me->executable, me->contents); + bool operator == (const Regular &) const = default; + auto operator <=> (const Regular &) const = default; }; struct Directory { @@ -27,13 +31,16 @@ struct MemorySourceAccessor : virtual SourceAccessor std::map> contents; - GENERATE_CMP(Directory, me->contents); + bool operator == (const Directory &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + bool operator < (const Directory &) const noexcept; }; struct Symlink { std::string target; - GENERATE_CMP(Symlink, me->target); + bool operator == (const Symlink &) const = default; + auto operator <=> (const Symlink &) const = default; }; using Raw = std::variant; @@ -41,14 +48,15 @@ struct MemorySourceAccessor : virtual SourceAccessor MAKE_WRAPPER_CONSTRUCTOR(File); - GENERATE_CMP(File, me->raw); - Stat lstat() const; }; File root { File::Directory {} }; - GENERATE_CMP(MemorySourceAccessor, me->root); + bool operator == (const MemorySourceAccessor &) const noexcept = default; + bool operator < (const MemorySourceAccessor & other) const noexcept { + return root < other.root; + } std::string readFile(const CanonPath & path) override; bool pathExists(const CanonPath & path) override; @@ -72,6 +80,20 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; + +inline bool MemorySourceAccessor::File::Directory::operator == ( + const MemorySourceAccessor::File::Directory &) const noexcept = default; +inline bool MemorySourceAccessor::File::Directory::operator < ( + const MemorySourceAccessor::File::Directory & other) const noexcept +{ + return contents < other.contents; +} + +inline bool MemorySourceAccessor::File::operator == ( + const MemorySourceAccessor::File &) const noexcept = default; +inline std::strong_ordering MemorySourceAccessor::File::operator <=> ( + const MemorySourceAccessor::File &) const noexcept = default; + /** * Write to a `MemorySourceAccessor` at the given path */ diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 724e560b7a4..573efeeb2d8 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -17,12 +17,6 @@ Pos::operator std::shared_ptr() const return std::make_shared(&*this); } -bool Pos::operator<(const Pos &rhs) const -{ - return std::forward_as_tuple(line, column, origin) - < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin); -} - std::optional Pos::getCodeLines() const { if (line == 0) diff --git a/src/libutil/position.hh b/src/libutil/position.hh index 9bdf3b4b5db..f8f34419b7a 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -22,21 +22,17 @@ struct Pos struct Stdin { ref source; - bool operator==(const Stdin & rhs) const + bool operator==(const Stdin & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const Stdin & rhs) const - { return *source != *rhs.source; } - bool operator<(const Stdin & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const Stdin & rhs) const noexcept + { return *source <=> *rhs.source; } }; struct String { ref source; - bool operator==(const String & rhs) const + bool operator==(const String & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const String & rhs) const - { return *source != *rhs.source; } - bool operator<(const String & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const String & rhs) const noexcept + { return *source <=> *rhs.source; } }; typedef std::variant Origin; @@ -65,8 +61,7 @@ struct Pos std::optional getCodeLines() const; bool operator==(const Pos & rhs) const = default; - bool operator!=(const Pos & rhs) const = default; - bool operator<(const Pos & rhs) const; + auto operator<=>(const Pos & rhs) const = default; struct LinesIterator { using difference_type = size_t; diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 8318251bde2..016fdd74a39 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -87,9 +87,9 @@ public: return p != other.p; } - bool operator < (const ref & other) const + auto operator <=> (const ref & other) const { - return p < other.p; + return p <=> other.p; } private: diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 32ab3685d28..b16960d4a4c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -152,9 +152,9 @@ struct SourceAccessor : std::enable_shared_from_this return number == x.number; } - bool operator < (const SourceAccessor & x) const + auto operator <=> (const SourceAccessor & x) const { - return number < x.number; + return number <=> x.number; } void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index 023b5ed4b91..759d3c35579 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -47,19 +47,14 @@ SourcePath SourcePath::operator / (const CanonPath & x) const SourcePath SourcePath::operator / (std::string_view c) const { return {accessor, path / c}; } -bool SourcePath::operator==(const SourcePath & x) const +bool SourcePath::operator==(const SourcePath & x) const noexcept { return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); } -bool SourcePath::operator!=(const SourcePath & x) const +std::strong_ordering SourcePath::operator<=>(const SourcePath & x) const noexcept { - return std::tie(*accessor, path) != std::tie(*x.accessor, x.path); -} - -bool SourcePath::operator<(const SourcePath & x) const -{ - return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); + return std::tie(*accessor, path) <=> std::tie(*x.accessor, x.path); } std::ostream & operator<<(std::ostream & str, const SourcePath & path) diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 83ec6295de1..faf0a5a3185 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -103,9 +103,8 @@ struct SourcePath */ SourcePath operator / (std::string_view c) const; - bool operator==(const SourcePath & x) const; - bool operator!=(const SourcePath & x) const; - bool operator<(const SourcePath & x) const; + bool operator==(const SourcePath & x) const noexcept; + std::strong_ordering operator<=>(const SourcePath & x) const noexcept; /** * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 17d1d69c16a..e39ab400c0d 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "types.hh" #include @@ -20,7 +19,8 @@ public: std::string to_string() const; - GENERATE_CMP(Suggestion, me->distance, me->suggestion) + bool operator ==(const Suggestion &) const = default; + auto operator <=>(const Suggestion &) const = default; }; class Suggestions { diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f4178f87fd9..bcbe9ea4eb2 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -132,7 +132,7 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } -bool ParsedURL::operator ==(const ParsedURL & other) const +bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return scheme == other.scheme diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 6cd06e53d17..738ee9f82e6 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -18,7 +18,7 @@ struct ParsedURL std::string to_string() const; - bool operator ==(const ParsedURL & other) const; + bool operator ==(const ParsedURL & other) const noexcept; /** * Remove `.` and `..` path elements. diff --git a/src/nix/profile.cc b/src/nix/profile.cc index bb6424c4f13..78532a2eccc 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -27,7 +27,9 @@ struct ProfileElementSource std::string attrPath; ExtendedOutputsSpec outputs; - bool operator < (const ProfileElementSource & other) const + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const ProfileElementSource & other) const + auto operator < (const ProfileElementSource & other) const { return std::tuple(originalRef.to_string(), attrPath, outputs) < diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index a0125d023ba..3d01d980609 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -229,7 +229,7 @@ TEST_F(GitTest, both_roundrip) { mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular); - ASSERT_EQ(*files, *files2); + ASSERT_EQ(files->root, files2->root); } TEST(GitLsRemote, parseSymrefLineWithReference) { From 1a8defd06f6b9fb1867680a053dd23dfccd5df50 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 22:09:27 +0200 Subject: [PATCH 20/70] Refactor: rename C++ concatStringsSep -> dropEmptyInitThenConcatStringsSep --- src/build-remote/build-remote.cc | 8 ++++---- src/libcmd/command.cc | 2 +- src/libcmd/installables.cc | 4 ++-- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 6 +++--- src/libflake/flake/config.cc | 2 +- src/libflake/flake/lockfile.cc | 4 ++-- src/libmain/shared.cc | 6 +++--- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/build/entry-points.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/local-store.cc | 8 ++++---- src/libstore/misc.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 2 +- src/libstore/outputs-spec.cc | 2 +- src/libstore/path-info.cc | 2 +- src/libstore/path-with-outputs.cc | 2 +- src/libstore/store-api.cc | 2 +- src/libstore/unix/build/hook-instance.cc | 2 +- src/libstore/unix/build/local-derivation-goal.cc | 8 ++++---- src/libutil/config.cc | 8 ++++---- src/libutil/util.hh | 4 ++-- src/nix/develop.cc | 2 +- src/nix/diff-closures.cc | 4 ++-- src/nix/env.cc | 2 +- src/nix/flake.cc | 10 +++++----- src/nix/main.cc | 6 +++--- src/nix/path-info.cc | 2 +- src/nix/profile.cc | 6 +++--- src/nix/search.cc | 4 ++-- tests/unit/libutil/references.cc | 2 +- tests/unit/libutil/tests.cc | 14 +++++++------- 33 files changed, 70 insertions(+), 70 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 582e6d623d0..1c3ce930a78 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -206,15 +206,15 @@ static int main_build_remote(int argc, char * * argv) error % drvstr % neededSystem - % concatStringsSep(", ", requiredFeatures) + % dropEmptyInitThenConcatStringsSep(", ", requiredFeatures) % machines.size(); for (auto & m : machines) error - % concatStringsSep(", ", m.systemTypes) + % dropEmptyInitThenConcatStringsSep(", ", m.systemTypes) % m.maxJobs - % concatStringsSep(", ", m.supportedFeatures) - % concatStringsSep(", ", m.mandatoryFeatures); + % dropEmptyInitThenConcatStringsSep(", ", m.supportedFeatures) + % dropEmptyInitThenConcatStringsSep(", ", m.mandatoryFeatures); printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str()); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index e0e5f089025..891a01f91fc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -42,7 +42,7 @@ void NixMultiCommand::run() for (auto & [name, _] : commands) subCommandTextLines.insert(fmt("- `%s`", name)); std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n", - commandName, concatStringsSep("\n", subCommandTextLines)); + commandName, dropEmptyInitThenConcatStringsSep("\n", subCommandTextLines)); throw UsageError(renderMarkdownToTerminal(markdownError)); } command->second->run(); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 417e1509498..1f6ee1e2394 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -374,7 +374,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions.add(flakeRefS + "#" + prefixRoot + dropEmptyInitThenConcatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -630,7 +630,7 @@ static void throwBuildErrors( } failedPaths.insert(failedResult->path.to_string(store)); } - throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + throw Error("build of %s failed", dropEmptyInitThenConcatStringsSep(", ", quoteStrings(failedPaths))); } } } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 3dd19ce39b7..1d39ef167c4 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -625,7 +625,7 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` " - + concatStringsSep(" ", args) + "\n\n"; + + dropEmptyInitThenConcatStringsSep(" ", args) + "\n\n"; } markdown += stripIndentation(doc->doc); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 46dd3691c17..e573fe88499 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -225,7 +225,7 @@ struct AttrDb (key.first) (symbols[key.second]) (AttrType::ListOfStrings) - (concatStringsSep("\t", l)).exec(); + (dropEmptyInitThenConcatStringsSep("\t", l)).exec(); return state->db.getLastInsertedRowId(); }); @@ -435,12 +435,12 @@ std::vector AttrCursor::getAttrPath(Symbol name) const std::string AttrCursor::getAttrPathStr() const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath())); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath())); } std::string AttrCursor::getAttrPathStr(Symbol name) const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); } Value & AttrCursor::forceValue() diff --git a/src/libflake/flake/config.cc b/src/libflake/flake/config.cc index 4e00d5c9368..e526cdddfa3 100644 --- a/src/libflake/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -47,7 +47,7 @@ void ConfigFile::apply(const Settings & flakeSettings) else if (auto* b = std::get_if>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil + valueS = dropEmptyInitThenConcatStringsSep(" ", *ss); // FIXME: evil else assert(false); diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index 792dda740f2..f0e22a75a4b 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -61,7 +61,7 @@ static std::shared_ptr doFind(const ref& root, const InputPath & pat std::vector cycle; std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); cycle.push_back(printInputPath(path)); - throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + throw Error("follow cycle detected: [%s]", dropEmptyInitThenConcatStringsSep(" -> ", cycle)); } visited.push_back(path); @@ -367,7 +367,7 @@ void check(); std::string printInputPath(const InputPath & path) { - return concatStringsSep("/", path); + return dropEmptyInitThenConcatStringsSep("/", path); } } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index fee4d0c1e11..681c1039fb1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -301,11 +301,11 @@ void printVersion(const std::string & programName) #endif cfg.push_back("signed-caches"); std::cout << "System type: " << settings.thisSystem << "\n"; - std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; - std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; + std::cout << "Additional system types: " << dropEmptyInitThenConcatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; + std::cout << "Features: " << dropEmptyInitThenConcatStringsSep(", ", cfg) << "\n"; std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; std::cout << "User configuration files: " << - concatStringsSep(":", settings.nixUserConfFiles) + dropEmptyInitThenConcatStringsSep(":", settings.nixUserConfFiles) << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 64b8495e1bb..c0a78434903 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -895,7 +895,7 @@ void runPostBuildHook( std::map hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))); + hookEnvironment.emplace("OUT_PATHS", chomp(dropEmptyInitThenConcatStringsSep(" ", store.printStorePathSet(outputPaths)))); hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue()); struct LogSink : Sink { @@ -1505,7 +1505,7 @@ std::pair DerivationGoal::checkPathValidity() if (!wantedOutputsLeft.empty()) throw Error("derivation '%s' does not have wanted outputs %s", worker.store.printStorePath(drvPath), - concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); + dropEmptyInitThenConcatStringsSep(", ", quoteStrings(wantedOutputsLeft))); bool allValid = true; for (auto & [_, status] : initialOutputs) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 784f618c1a0..8bf7ad35dce 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -42,7 +42,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); + throw Error(worker.failingExitStatus(), "build of %s failed", dropEmptyInitThenConcatStringsSep(", ", quoteStrings(failed))); } } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7e1d7ea6d67..5f5b4f89ba7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -82,7 +82,7 @@ Settings::Settings() Strings ss; for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); - builders = concatStringsSep(" ", ss); + builders = dropEmptyInitThenConcatStringsSep(" ", ss); } #if defined(__linux__) && defined(SANDBOX_SHELL) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2b4e01eb3ba..8764b88b76b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -656,7 +656,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput.use() - (concatStringsSep(" ", combinedSignatures)) + (dropEmptyInitThenConcatStringsSep(" ", combinedSignatures)) (info.id.strHash()) (info.id.outputName) .exec(); @@ -675,7 +675,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) - (concatStringsSep(" ", info.signatures)) + (dropEmptyInitThenConcatStringsSep(" ", info.signatures)) .exec(); } for (auto & [outputId, depPath] : info.dependentRealisations) { @@ -729,7 +729,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) - (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (dropEmptyInitThenConcatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = state.db.getLastInsertedRowId(); @@ -833,7 +833,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narSize, info.narSize != 0) (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) - (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (dropEmptyInitThenConcatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index dd0efbe1910..179e5478c9f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -464,7 +464,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) if (!outputsLeft.empty()) throw Error("derivation '%s' does not have an outputs %s", store.printStorePath(drvPath), - concatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); + dropEmptyInitThenConcatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); return outputMap; } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 07beb8acb0b..42d17223706 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -337,9 +337,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) (info->narHash.to_string(HashFormat::Nix32, true)) (info->narSize) - (concatStringsSep(" ", info->shortRefs())) + (dropEmptyInitThenConcatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) - (concatStringsSep(" ", info->sigs)) + (dropEmptyInitThenConcatStringsSep(" ", info->sigs)) (renderContentAddress(info->ca)) (time(0)).exec(); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3e0a754f981..577466f555c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -111,7 +111,7 @@ std::string NarInfo::to_string(const Store & store) const res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; - res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + res += "References: " + dropEmptyInitThenConcatStringsSep(" ", shortRefs()) + "\n"; if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 21c06922379..4ed8f95ae51 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -83,7 +83,7 @@ std::string OutputsSpec::to_string() const return "*"; }, [&](const OutputsSpec::Names & outputNames) -> std::string { - return concatStringsSep(",", outputNames); + return dropEmptyInitThenConcatStringsSep(",", outputNames); }, }, raw); } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 51ed5fc62e3..a13bb8bef8a 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -30,7 +30,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(HashFormat::Nix32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + dropEmptyInitThenConcatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 026e376471f..5fa38d5d9f9 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -9,7 +9,7 @@ std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) - : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); + : store.printStorePath(path) + "!" + dropEmptyInitThenConcatStringsSep(",", outputs); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 05c4e1c5e8f..6904996b589 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1208,7 +1208,7 @@ std::string StoreDirConfig::showPaths(const StorePathSet & paths) std::string showPaths(const PathSet & paths) { - return concatStringsSep(", ", quoteStrings(paths)); + return dropEmptyInitThenConcatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index dfc20879881..ba6c3a91261 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -8,7 +8,7 @@ namespace nix { HookInstance::HookInstance() { - debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); + debug("starting build hook '%s'", dropEmptyInitThenConcatStringsSep(" ", settings.buildHook.get())); auto buildHookArgs = settings.buildHook.get(); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index d5a3e0034f9..2ab4334e923 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -496,10 +496,10 @@ void LocalDerivationGoal::startBuilder() if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + dropEmptyInitThenConcatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); + dropEmptyInitThenConcatStringsSep(", ", worker.store.systemFeatures)); /* Create a temporary directory where the build will take place. */ @@ -840,7 +840,7 @@ void LocalDerivationGoal::startBuilder() /* Run the builder. */ printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); + printMsg(lvlChatty, "using builder args '%1%'", dropEmptyInitThenConcatStringsSep(" ", drv->args)); for (auto & i : drv->env) printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); @@ -1063,7 +1063,7 @@ void LocalDerivationGoal::startBuilder() e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", worker.store.printStorePath(drvPath), statusToString(status), - concatStringsSep("|", msgs)); + dropEmptyInitThenConcatStringsSep("|", msgs)); throw; } }(); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 907ca7fc149..81ec9a4c3bc 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -152,7 +152,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p parsedContents.push_back({ std::move(name), - concatStringsSep(" ", Strings(i, tokens.end())), + dropEmptyInitThenConcatStringsSep(" ", Strings(i, tokens.end())), }); }; } @@ -318,7 +318,7 @@ template<> void BaseSetting::appendOrSet(Strings newValue, bool append) template<> std::string BaseSetting::to_string() const { - return concatStringsSep(" ", value); + return dropEmptyInitThenConcatStringsSep(" ", value); } template<> StringSet BaseSetting::parse(const std::string & str) const @@ -334,7 +334,7 @@ template<> void BaseSetting::appendOrSet(StringSet newValue, bool app template<> std::string BaseSetting::to_string() const { - return concatStringsSep(" ", value); + return dropEmptyInitThenConcatStringsSep(" ", value); } template<> std::set BaseSetting>::parse(const std::string & str) const @@ -362,7 +362,7 @@ template<> std::string BaseSetting>::to_string() c StringSet stringifiedXpFeatures; for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); - return concatStringsSep(" ", stringifiedXpFeatures); + return dropEmptyInitThenConcatStringsSep(" ", stringifiedXpFeatures); } template<> StringMap BaseSetting::parse(const std::string & str) const diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 23682ff7ca5..66cef62edf5 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -37,7 +37,7 @@ template C tokenizeString(std::string_view s, std::string_view separato * elements. */ template -std::string concatStringsSep(const std::string_view sep, const C & ss) +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { size_t size = 0; // need a cast to string_view since this is also called with Symbols @@ -56,7 +56,7 @@ auto concatStrings(Parts && ... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { std::string_view views[sizeof...(parts)] = { parts... }; - return concatStringsSep({}, views); + return dropEmptyInitThenConcatStringsSep({}, views); } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6bd3dc9efc6..807c75d4a23 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -608,7 +608,7 @@ struct CmdDevelop : Common, MixEnvironment std::vector args; for (auto s : command) args.push_back(shellEscape(s)); - script += fmt("exec %s\n", concatStringsSep(" ", args)); + script += fmt("exec %s\n", dropEmptyInitThenConcatStringsSep(" ", args)); } else { diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c7c37b66fbe..2042133968a 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -49,7 +49,7 @@ std::string showVersions(const std::set & versions) std::set versions2; for (auto & version : versions) versions2.insert(version.empty() ? "ε" : version); - return concatStringsSep(", ", versions2); + return dropEmptyInitThenConcatStringsSep(", ", versions2); } void printClosureDiff( @@ -97,7 +97,7 @@ void printClosureDiff( items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); if (showDelta) items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); - logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); + logger->cout("%s%s: %s", indent, name, dropEmptyInitThenConcatStringsSep(", ", items)); } } } diff --git a/src/nix/env.cc b/src/nix/env.cc index bc9cd91ad81..7cc019c1dca 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -94,7 +94,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); - auto unixPathString = concatStringsSep(":", unixPath); + auto unixPathString = dropEmptyInitThenConcatStringsSep(":", unixPath); setEnv("PATH", unixPathString.c_str()); Strings args; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cb73778b388..3ee2b18384b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -805,7 +805,7 @@ struct CmdFlakeCheck : FlakeCommand warn( "The check omitted these incompatible systems: %s\n" "Use '--all-systems' to check all.", - concatStringsSep(", ", omittedSystems) + dropEmptyInitThenConcatStringsSep(", ", omittedSystems) ); }; }; @@ -1211,7 +1211,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto attrPathS = state->symbols.resolve(attrPath); Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); + fmt("evaluating '%s'", dropEmptyInitThenConcatStringsSep(".", attrPathS))); try { auto recurse = [&]() @@ -1291,7 +1291,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON if (!json) logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", dropEmptyInitThenConcatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) @@ -1315,13 +1315,13 @@ struct CmdFlakeShow : FlakeCommand, MixJSON if (!json) logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--legacy' to show)", dropEmptyInitThenConcatStringsSep(".", attrPathS))); } } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { if (!json) logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", dropEmptyInitThenConcatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) diff --git a/src/nix/main.cc b/src/nix/main.cc index e39f79f1fab..aa4ced6232a 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -185,7 +185,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto & info = i->second; if (info.status == AliasStatus::Deprecated) { warn("'%s' is a deprecated alias for '%s'", - arg, concatStringsSep(" ", info.replacement)); + arg, dropEmptyInitThenConcatStringsSep(" ", info.replacement)); } pos = args.erase(pos); for (auto j = info.replacement.rbegin(); j != info.replacement.rend(); ++j) @@ -238,7 +238,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs lowdown. */ static void showHelp(std::vector subcommand, NixArgs & toplevel) { - auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); + auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", dropEmptyInitThenConcatStringsSep("-", subcommand)); evalSettings.restrictEval = false; evalSettings.pureEval = false; @@ -273,7 +273,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto attr = vRes->attrs()->get(state.symbols.create(mdName + ".md")); if (!attr) - throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); + throw UsageError("Nix has no subcommand '%s'", dropEmptyInitThenConcatStringsSep("", subcommand)); auto markdown = state.forceString(*attr->value, noPos, "while evaluating the lowdown help text"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 47f9baee505..2383fbe080a 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -185,7 +185,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (info->ultimate) ss.push_back("ultimate"); if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); - std::cout << concatStringsSep(" ", ss); + std::cout << dropEmptyInitThenConcatStringsSep(" ", ss); } std::cout << std::endl; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 78532a2eccc..c21eb3040b7 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -58,7 +58,7 @@ struct ProfileElement StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); + return dropEmptyInitThenConcatStringsSep(", ", names); } /** @@ -472,7 +472,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile originalConflictingFilePath, newConflictingFilePath, originalEntryName, - concatStringsSep(" ", newConflictingRefs), + dropEmptyInitThenConcatStringsSep(" ", newConflictingRefs), conflictError.priority, conflictError.priority - 1, conflictError.priority + 1 @@ -813,7 +813,7 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); } - logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + logger->cout("Store paths: %s", dropEmptyInitThenConcatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } } diff --git a/src/nix/search.cc b/src/nix/search.cc index 97ef1375ed2..d709774ad7c 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -96,7 +96,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON auto attrPathS = state->symbols.resolve(attrPath); Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); + fmt("evaluating '%s'", dropEmptyInitThenConcatStringsSep(".", attrPathS))); try { auto recurse = [&]() { @@ -115,7 +115,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr; auto description = aDescription ? aDescription->getString() : ""; std::replace(description.begin(), description.end(), '\n', ' '); - auto attrPath2 = concatStringsSep(".", attrPathS); + auto attrPath2 = dropEmptyInitThenConcatStringsSep(".", attrPathS); std::vector attrPathMatches; std::vector descriptionMatches; diff --git a/tests/unit/libutil/references.cc b/tests/unit/libutil/references.cc index a517d9aa175..c3efa6d5101 100644 --- a/tests/unit/libutil/references.cc +++ b/tests/unit/libutil/references.cc @@ -15,7 +15,7 @@ struct RewriteParams { strRewrites.insert(from + "->" + to); return os << "OriginalString: " << bar.originalString << std::endl << - "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Rewrites: " << dropEmptyInitThenConcatStringsSep(",", strRewrites) << std::endl << "Expected result: " << bar.finalString; } }; diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 9be4a400d02..8a3ca8561e5 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -227,32 +227,32 @@ namespace nix { } /* ---------------------------------------------------------------------------- - * concatStringsSep + * dropEmptyInitThenConcatStringsSep * --------------------------------------------------------------------------*/ - TEST(concatStringsSep, buildCommaSeparatedString) { + TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) { Strings strings; strings.push_back("this"); strings.push_back("is"); strings.push_back("great"); - ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); } - TEST(concatStringsSep, buildStringWithEmptySeparator) { + TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) { Strings strings; strings.push_back("this"); strings.push_back("is"); strings.push_back("great"); - ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); } - TEST(concatStringsSep, buildSingleString) { + TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) { Strings strings; strings.push_back("this"); - ASSERT_EQ(concatStringsSep(",", strings), "this"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this"); } /* ---------------------------------------------------------------------------- From 79eb0adf9db942609b22d55ff764d1e759453543 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 22:33:14 +0200 Subject: [PATCH 21/70] dropEmptyInitThenConcatStringSep: Check that we don't drop... ... initial empty strings. The tests pass, which is encouraging. --- src/libexpr/symbol-table.hh | 5 +++++ src/libutil/util.hh | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index b85725e1234..c7a3563b040 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -41,6 +41,11 @@ public: } friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); + + bool empty() const + { + return s->empty(); + } }; /** diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 66cef62edf5..c545afd9e35 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -40,6 +40,13 @@ template std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { size_t size = 0; + + for (auto & i : ss) { + // Make sure we don't rely on the empty item ignoring behavior + assert(!i.empty()); + break; + } + // need a cast to string_view since this is also called with Symbols for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); std::string s; From a681d354e717a549595e9b687567dc7d05eaf29d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 23:02:08 +0200 Subject: [PATCH 22/70] Add fresh concatStringsSep without bug The buggy version was previously renamed to dropEmptyInitThenConcatStringsSep --- src/libutil/meson.build | 3 ++ src/libutil/strings-inline.hh | 31 +++++++++++++ src/libutil/strings.cc | 12 +++++ src/libutil/strings.hh | 21 +++++++++ tests/unit/libutil/strings.cc | 83 +++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 src/libutil/strings-inline.hh create mode 100644 src/libutil/strings.cc create mode 100644 src/libutil/strings.hh create mode 100644 tests/unit/libutil/strings.cc diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ac2b8353674..fbfcbe67c42 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -148,6 +148,7 @@ sources = files( 'signature/signer.cc', 'source-accessor.cc', 'source-path.cc', + 'strings.cc', 'suggestions.cc', 'tarfile.cc', 'terminal.cc', @@ -215,6 +216,8 @@ headers = [config_h] + files( 'source-accessor.hh', 'source-path.hh', 'split.hh', + 'strings.hh', + 'strings-inline.hh', 'suggestions.hh', 'sync.hh', 'tarfile.hh', diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh new file mode 100644 index 00000000000..10c1b19e688 --- /dev/null +++ b/src/libutil/strings-inline.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "strings.hh" + +namespace nix { + +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + bool tail = false; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) { + if (tail) + size += sep.size(); + size += std::string_view(s).size(); + tail = true; + } + std::string s; + s.reserve(size); + tail = false; + for (auto & i : ss) { + if (tail) + s += sep; + s += i; + tail = true; + } + return s; +} + +} // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 00000000000..15937d41570 --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,12 @@ +#include + +#include "strings-inline.hh" +#include "util.hh" + +namespace nix { + +template std::string concatStringsSep(std::string_view, const Strings &); +template std::string concatStringsSep(std::string_view, const StringSet &); +template std::string concatStringsSep(std::string_view, const std::vector &); + +} // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 00000000000..3b112c4096d --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nix { + +/** + * Concatenate the given strings with a separator between the elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string concatStringsSep(std::string_view, const std::list &); +extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const std::vector &); + +} diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc new file mode 100644 index 00000000000..47a20770e2e --- /dev/null +++ b/tests/unit/libutil/strings.cc @@ -0,0 +1,83 @@ +#include + +#include "strings.hh" + +namespace nix { + +using Strings = std::vector; + +/* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(concatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, justOne) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +TEST(concatStringsSep, emptyString) +{ + Strings strings; + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ","); +} + +TEST(concatStringsSep, threeEmptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ",,"); +} + +TEST(concatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); +} + +TEST(concatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); +} + +TEST(concatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +} // namespace nix From ea966a70fcae93538f41ff413fb9cec852054b3c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 23:06:32 +0200 Subject: [PATCH 23/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: diagnostics and docs These are non-critical, so their behavior is ok to change. Dropping empty items is not needed and usually not expected. --- src/build-remote/build-remote.cc | 9 +++++---- src/libcmd/command.cc | 7 ++++--- src/libcmd/installables.cc | 4 +++- src/libcmd/repl.cc | 4 +++- src/libflake/flake/lockfile.cc | 6 ++++-- src/libmain/shared.cc | 7 ++++--- src/libstore/build/entry-points.cc | 3 ++- src/libstore/misc.cc | 1 + src/libstore/unix/build/hook-instance.cc | 3 ++- 9 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1c3ce930a78..600fc7ee2b7 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -16,6 +16,7 @@ #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -206,15 +207,15 @@ static int main_build_remote(int argc, char * * argv) error % drvstr % neededSystem - % dropEmptyInitThenConcatStringsSep(", ", requiredFeatures) + % concatStringsSep(", ", requiredFeatures) % machines.size(); for (auto & m : machines) error - % dropEmptyInitThenConcatStringsSep(", ", m.systemTypes) + % concatStringsSep(", ", m.systemTypes) % m.maxJobs - % dropEmptyInitThenConcatStringsSep(", ", m.supportedFeatures) - % dropEmptyInitThenConcatStringsSep(", ", m.mandatoryFeatures); + % concatStringsSep(", ", m.supportedFeatures) + % concatStringsSep(", ", m.mandatoryFeatures); printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str()); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 891a01f91fc..67fef190920 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -42,7 +43,7 @@ void NixMultiCommand::run() for (auto & [name, _] : commands) subCommandTextLines.insert(fmt("- `%s`", name)); std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n", - commandName, dropEmptyInitThenConcatStringsSep("\n", subCommandTextLines)); + commandName, concatStringsSep("\n", subCommandTextLines)); throw UsageError(renderMarkdownToTerminal(markdownError)); } command->second->run(); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1f6ee1e2394..406e4bfd8ca 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -630,7 +632,7 @@ static void throwBuildErrors( } failedPaths.insert(failedResult->path.to_string(store)); } - throw Error("build of %s failed", dropEmptyInitThenConcatStringsSep(", ", quoteStrings(failedPaths))); + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); } } } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 1d39ef167c4..37a34e3de0e 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -33,6 +33,8 @@ #include #endif +#include "strings.hh" + namespace nix { /** @@ -625,7 +627,7 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` " - + dropEmptyInitThenConcatStringsSep(" ", args) + "\n\n"; + + concatStringsSep(" ", args) + "\n\n"; } markdown += stripIndentation(doc->doc); diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index f0e22a75a4b..80f14ff6fa3 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -9,6 +9,8 @@ #include #include +#include "strings.hh" + namespace nix::flake { static FlakeRef getFlakeRef( @@ -61,7 +63,7 @@ static std::shared_ptr doFind(const ref& root, const InputPath & pat std::vector cycle; std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); cycle.push_back(printInputPath(path)); - throw Error("follow cycle detected: [%s]", dropEmptyInitThenConcatStringsSep(" -> ", cycle)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); } visited.push_back(path); @@ -367,7 +369,7 @@ void check(); std::string printInputPath(const InputPath & path) { - return dropEmptyInitThenConcatStringsSep("/", path); + return concatStringsSep("/", path); } } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 681c1039fb1..a224f8d92a4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -23,6 +23,7 @@ #include #include "exit.hh" +#include "strings.hh" namespace nix { @@ -301,11 +302,11 @@ void printVersion(const std::string & programName) #endif cfg.push_back("signed-caches"); std::cout << "System type: " << settings.thisSystem << "\n"; - std::cout << "Additional system types: " << dropEmptyInitThenConcatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; - std::cout << "Features: " << dropEmptyInitThenConcatStringsSep(", ", cfg) << "\n"; + std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; + std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; std::cout << "User configuration files: " << - dropEmptyInitThenConcatStringsSep(":", settings.nixUserConfFiles) + concatStringsSep(":", settings.nixUserConfFiles) << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 8bf7ad35dce..4c1373bfaff 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -4,6 +4,7 @@ # include "derivation-goal.hh" #endif #include "local-store.hh" +#include "strings.hh" namespace nix { @@ -42,7 +43,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", dropEmptyInitThenConcatStringsSep(", ", quoteStrings(failed))); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 179e5478c9f..fbfa15f51a4 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -10,6 +10,7 @@ #include "callback.hh" #include "closure.hh" #include "filetransfer.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index ba6c3a91261..d73d86ff27f 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -3,12 +3,13 @@ #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" +#include "strings.hh" namespace nix { HookInstance::HookInstance() { - debug("starting build hook '%s'", dropEmptyInitThenConcatStringsSep(" ", settings.buildHook.get())); + debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); auto buildHookArgs = settings.buildHook.get(); From 39878c89798a95a423395ccfbc64d49f0fad92b9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 Jul 2024 23:08:23 +0200 Subject: [PATCH 24/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: preserve empty attr The empty attribute name should not be dropped from attribute paths. Rendering attribute paths with concatStringsSep is lossy and wrong, but this is just a first improvement while dealing with the dropEmptyInitThenConcatStringsSep problem. --- src/libcmd/installables.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 406e4bfd8ca..0fe956ec023 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -376,7 +376,8 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions.add(flakeRefS + "#" + prefixRoot + dropEmptyInitThenConcatStringsSep(".", evalState->symbols.resolve(attrPath2))); + // FIXME: handle names with dots + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } From 3f37785afd002a36ecd5070b16c59902eba9cf88 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 00:07:42 +0200 Subject: [PATCH 25/70] NIX_REMOTE_SYSTEMS: actually support multiple :-separated entries Bug not reported in 6 years, but here you go. Also it is safe to switch to normal concatStringsSep behavior because tokenizeString does not produce empty items. --- src/libstore/globals.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5f5b4f89ba7..4eabf6054dc 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -35,6 +35,8 @@ #include #endif +#include "strings.hh" + namespace nix { @@ -82,7 +84,7 @@ Settings::Settings() Strings ss; for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); - builders = dropEmptyInitThenConcatStringsSep(" ", ss); + builders = concatStringsSep("\n", ss); } #if defined(__linux__) && defined(SANDBOX_SHELL) From 75dde71ff97fcafeeefa08a5c9adba3414693ea2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 00:11:14 +0200 Subject: [PATCH 26/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: sigs are non-empty The sigs field is produced by tokenizeStrings, which does not return empty strings. --- src/libstore/local-store.cc | 10 ++++++---- src/libstore/nar-info-disk-cache.cc | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8764b88b76b..82b70ff2135 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -51,6 +51,8 @@ #include +#include "strings.hh" + namespace nix { @@ -656,7 +658,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput.use() - (dropEmptyInitThenConcatStringsSep(" ", combinedSignatures)) + (concatStringsSep(" ", combinedSignatures)) (info.id.strHash()) (info.id.outputName) .exec(); @@ -675,7 +677,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) - (dropEmptyInitThenConcatStringsSep(" ", info.signatures)) + (concatStringsSep(" ", info.signatures)) .exec(); } for (auto & [outputId, depPath] : info.dependentRealisations) { @@ -729,7 +731,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) - (dropEmptyInitThenConcatStringsSep(" ", info.sigs), !info.sigs.empty()) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = state.db.getLastInsertedRowId(); @@ -833,7 +835,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narSize, info.narSize != 0) (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) - (dropEmptyInitThenConcatStringsSep(" ", info.sigs), !info.sigs.empty()) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 42d17223706..c75043237ee 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -7,6 +7,8 @@ #include #include +#include "strings.hh" + namespace nix { static const char * schema = R"sql( @@ -339,7 +341,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (info->narSize) (dropEmptyInitThenConcatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) - (dropEmptyInitThenConcatStringsSep(" ", info->sigs)) + (concatStringsSep(" ", info->sigs)) (renderContentAddress(info->ca)) (time(0)).exec(); From 608a425550b3c90de39d4a668a504debcc8c53de Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 00:13:58 +0200 Subject: [PATCH 27/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: diag --- src/libstore/misc.cc | 2 +- src/libstore/unix/build/local-derivation-goal.cc | 6 ++++-- src/nix/diff-closures.cc | 2 +- src/nix/profile.cc | 4 +++- src/nix/search.cc | 4 +++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index fbfa15f51a4..bcc02206bc9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -465,7 +465,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) if (!outputsLeft.empty()) throw Error("derivation '%s' does not have an outputs %s", store.printStorePath(drvPath), - dropEmptyInitThenConcatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); return outputMap; } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 2ab4334e923..523fe07e799 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -64,6 +64,8 @@ #include #include +#include "strings.hh" + namespace nix { void handleDiffHook( @@ -840,7 +842,7 @@ void LocalDerivationGoal::startBuilder() /* Run the builder. */ printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - printMsg(lvlChatty, "using builder args '%1%'", dropEmptyInitThenConcatStringsSep(" ", drv->args)); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); for (auto & i : drv->env) printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); @@ -1063,7 +1065,7 @@ void LocalDerivationGoal::startBuilder() e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", worker.store.printStorePath(drvPath), statusToString(status), - dropEmptyInitThenConcatStringsSep("|", msgs)); + concatStringsSep("|", msgs)); throw; } }(); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 2042133968a..9e5a7e4c3a8 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -97,7 +97,7 @@ void printClosureDiff( items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); if (showDelta) items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); - logger->cout("%s%s: %s", indent, name, dropEmptyInitThenConcatStringsSep(", ", items)); + logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); } } } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c21eb3040b7..548a793d85c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -17,6 +17,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct ProfileElementSource @@ -472,7 +474,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile originalConflictingFilePath, newConflictingFilePath, originalEntryName, - dropEmptyInitThenConcatStringsSep(" ", newConflictingRefs), + concatStringsSep(" ", newConflictingRefs), conflictError.priority, conflictError.priority - 1, conflictError.priority + 1 diff --git a/src/nix/search.cc b/src/nix/search.cc index d709774ad7c..7c46f9fecdd 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -15,6 +15,8 @@ #include #include +#include "strings.hh" + using namespace nix; using json = nlohmann::json; @@ -96,7 +98,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON auto attrPathS = state->symbols.resolve(attrPath); Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", dropEmptyInitThenConcatStringsSep(".", attrPathS))); + fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); try { auto recurse = [&]() { From d3e49ac881654529f8de3e2a3b39f5840d8ed669 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 00:50:39 +0200 Subject: [PATCH 28/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: shortRefs are not empty --- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 3 ++- src/libstore/path-info.hh | 3 +++ tests/unit/libstore/path-info.cc | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c75043237ee..288f618d5d7 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -339,7 +339,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) (info->narHash.to_string(HashFormat::Nix32, true)) (info->narSize) - (dropEmptyInitThenConcatStringsSep(" ", info->shortRefs())) + (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) (renderContentAddress(info->ca)) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 577466f555c..2442a7b09a3 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,6 +1,7 @@ #include "globals.hh" #include "nar-info.hh" #include "store-api.hh" +#include "strings.hh" namespace nix { @@ -111,7 +112,7 @@ std::string NarInfo::to_string(const Store & store) const res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; - res += "References: " + dropEmptyInitThenConcatStringsSep(" ", shortRefs()) + "\n"; + res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index caefa7975a2..71f1476a672 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -171,6 +171,9 @@ struct ValidPathInfo : UnkeyedValidPathInfo { */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + /** + * References as store path basenames, including a self reference if it has one. + */ Strings shortRefs() const; ValidPathInfo(const ValidPathInfo & other) = default; diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 7637cb366aa..9e9c6303d90 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -26,9 +26,9 @@ static UnkeyedValidPathInfo makeEmpty() }; } -static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) +static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) { - UnkeyedValidPathInfo info = ValidPathInfo { + ValidPathInfo info = ValidPathInfo { store, "foo", FixedOutputInfo { @@ -57,6 +57,9 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo } return info; } +static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) { + return makeFullKeyed(store, includeImpureInfo); +} #define JSON_TEST(STEM, OBJ, PURE) \ TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ @@ -86,4 +89,13 @@ JSON_TEST(empty_impure, makeEmpty(), true) JSON_TEST(pure, makeFull(*store, false), false) JSON_TEST(impure, makeFull(*store, true), true) +TEST_F(PathInfoTest, PathInfo_full_shortRefs) { + ValidPathInfo it = makeFullKeyed(*store, true); + // it.references = unkeyed.references; + auto refs = it.shortRefs(); + ASSERT_EQ(refs.size(), 2); + ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"); } + +} // namespace nix From 49d100ba8b5d65c6f2df909e53ec92ba279cfc4d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:00:06 +0200 Subject: [PATCH 29/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: output name empty not feasible I don't think it's completely impossible, but I can't construct one easily as derivationStrict seems to (re)tokenize the outputs attribute, dropping the empty output. It's not a scenario we have to account for here. --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/outputs-spec.cc | 3 ++- src/libstore/path-with-outputs.cc | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c0a78434903..99d9ccedaa7 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1505,7 +1505,7 @@ std::pair DerivationGoal::checkPathValidity() if (!wantedOutputsLeft.empty()) throw Error("derivation '%s' does not have wanted outputs %s", worker.store.printStorePath(drvPath), - dropEmptyInitThenConcatStringsSep(", ", quoteStrings(wantedOutputsLeft))); + concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); bool allValid = true; for (auto & [_, status] : initialOutputs) { diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 4ed8f95ae51..86788a87e72 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -5,6 +5,7 @@ #include "regex-combinators.hh" #include "outputs-spec.hh" #include "path-regex.hh" +#include "strings-inline.hh" namespace nix { @@ -83,7 +84,7 @@ std::string OutputsSpec::to_string() const return "*"; }, [&](const OutputsSpec::Names & outputNames) -> std::string { - return dropEmptyInitThenConcatStringsSep(",", outputNames); + return concatStringsSep(",", outputNames); }, }, raw); } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 5fa38d5d9f9..161d023d14c 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,7 +1,9 @@ +#include + #include "path-with-outputs.hh" #include "store-api.hh" +#include "strings.hh" -#include namespace nix { @@ -9,7 +11,7 @@ std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) - : store.printStorePath(path) + "!" + dropEmptyInitThenConcatStringsSep(",", outputs); + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); } From f1966e22d9e959b6885bd9270d2789b484690aa8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:02:30 +0200 Subject: [PATCH 30/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: store paths are not empty --- src/libstore/build/derivation-goal.cc | 4 +++- src/libstore/path-info.cc | 3 ++- src/libstore/store-api.cc | 4 +++- src/nix/profile.cc | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 99d9ccedaa7..f795b05a167 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -32,6 +32,8 @@ #include +#include "strings.hh" + namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, @@ -895,7 +897,7 @@ void runPostBuildHook( std::map hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(dropEmptyInitThenConcatStringsSep(" ", store.printStorePathSet(outputPaths)))); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))); hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue()); struct LogSink : Sink { diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index a13bb8bef8a..6e87e60f446 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "json-utils.hh" #include "comparator.hh" +#include "strings.hh" namespace nix { @@ -30,7 +31,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(HashFormat::Nix32, true) + ";" + std::to_string(narSize) + ";" - + dropEmptyInitThenConcatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6904996b589..2c4dee518b4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -22,6 +22,8 @@ #include #include +#include "strings.hh" + using json = nlohmann::json; namespace nix { @@ -1208,7 +1210,7 @@ std::string StoreDirConfig::showPaths(const StorePathSet & paths) std::string showPaths(const PathSet & paths) { - return dropEmptyInitThenConcatStringsSep(", ", quoteStrings(paths)); + return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 548a793d85c..1096f4386be 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -815,7 +815,7 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); } - logger->cout("Store paths: %s", dropEmptyInitThenConcatStringsSep(" ", store->printStorePathSet(element.storePaths))); + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } } From e64643bf6372af1ee7f56ac602f25564201df7f0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:12:28 +0200 Subject: [PATCH 31/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: feature should not be empty (System) features are unlikely to be empty strings, but when they come in through structuredAttrs, they probably can. I don't think this means we should drop them, but most likely they will be dropped after this because next time they'll be parsed with tokenizeString. TODO: We should forbid empty features. --- src/libstore/unix/build/local-derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 523fe07e799..c3a65e34b57 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -498,10 +498,10 @@ void LocalDerivationGoal::startBuilder() if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, - dropEmptyInitThenConcatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - dropEmptyInitThenConcatStringsSep(", ", worker.store.systemFeatures)); + concatStringsSep(", ", worker.store.systemFeatures)); /* Create a temporary directory where the build will take place. */ From 3b77f134515866f28cc7b7ddb06274fccc5766f9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:16:33 +0200 Subject: [PATCH 32/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: experimental features do not render as empty strings --- src/libutil/config.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 81ec9a4c3bc..25bfe462fe5 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + namespace nix { Config::Config(StringMap initials) @@ -362,7 +364,7 @@ template<> std::string BaseSetting>::to_string() c StringSet stringifiedXpFeatures; for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); - return dropEmptyInitThenConcatStringsSep(" ", stringifiedXpFeatures); + return concatStringsSep(" ", stringifiedXpFeatures); } template<> StringMap BaseSetting::parse(const std::string & str) const From 837c3612d40c1e33be94080b9b2063c3ca0795ed Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:20:20 +0200 Subject: [PATCH 33/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: escaped shell args are never empty --- src/nix/develop.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 807c75d4a23..7cc0965a9a8 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -19,6 +19,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct DevelopSettings : Config @@ -608,7 +610,7 @@ struct CmdDevelop : Common, MixEnvironment std::vector args; for (auto s : command) args.push_back(shellEscape(s)); - script += fmt("exec %s\n", dropEmptyInitThenConcatStringsSep(" ", args)); + script += fmt("exec %s\n", concatStringsSep(" ", args)); } else { From 4b34feb4c2fded8b4ca0968f33f0e34514520b1a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:22:51 +0200 Subject: [PATCH 34/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: system string should not be empty --- src/nix/flake.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3ee2b18384b..1ed071ec80c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -21,6 +21,8 @@ #include #include +#include "strings-inline.hh" + using namespace nix; using namespace nix::flake; using json = nlohmann::json; @@ -802,10 +804,11 @@ struct CmdFlakeCheck : FlakeCommand throw Error("some errors were encountered during the evaluation"); if (!omittedSystems.empty()) { + // TODO: empty system is not visible; render all as nix strings? warn( "The check omitted these incompatible systems: %s\n" "Use '--all-systems' to check all.", - dropEmptyInitThenConcatStringsSep(", ", omittedSystems) + concatStringsSep(", ", omittedSystems) ); }; }; From 0480bfe50bf1deb3b5c93761f7fbba1bc59ef059 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:23:44 +0200 Subject: [PATCH 35/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: do not drop attributes with empty names Empty attributes are probably not well supported, but the least we could do is leave a hint. Attribute path rendering and parsing should be done according to Nix expression syntax in my opinion. --- src/nix/flake.cc | 8 ++++---- src/nix/search.cc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1ed071ec80c..3f9f8f99b06 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1214,7 +1214,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto attrPathS = state->symbols.resolve(attrPath); Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", dropEmptyInitThenConcatStringsSep(".", attrPathS))); + fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); try { auto recurse = [&]() @@ -1294,7 +1294,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON if (!json) logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--all-systems' to show)", dropEmptyInitThenConcatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) @@ -1318,13 +1318,13 @@ struct CmdFlakeShow : FlakeCommand, MixJSON if (!json) logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--legacy' to show)", dropEmptyInitThenConcatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { if (!json) logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--all-systems' to show)", dropEmptyInitThenConcatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) diff --git a/src/nix/search.cc b/src/nix/search.cc index 7c46f9fecdd..7f8504d3f1e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -117,7 +117,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr; auto description = aDescription ? aDescription->getString() : ""; std::replace(description.begin(), description.end(), '\n', ' '); - auto attrPath2 = dropEmptyInitThenConcatStringsSep(".", attrPathS); + auto attrPath2 = concatStringsSep(".", attrPathS); std::vector attrPathMatches; std::vector descriptionMatches; From 062672b022a35fb0251e37cc0f428778d82fb934 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:27:59 +0200 Subject: [PATCH 36/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: CLI commands are not empty --- src/nix/main.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index aa4ced6232a..21d364a1818 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -41,6 +41,8 @@ extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); #endif +#include "strings.hh" + namespace nix { enum struct AliasStatus { @@ -185,7 +187,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto & info = i->second; if (info.status == AliasStatus::Deprecated) { warn("'%s' is a deprecated alias for '%s'", - arg, dropEmptyInitThenConcatStringsSep(" ", info.replacement)); + arg, concatStringsSep(" ", info.replacement)); } pos = args.erase(pos); for (auto j = info.replacement.rbegin(); j != info.replacement.rend(); ++j) From d9043021dfc4b7238f538c8064a3587a6a8d473d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:29:15 +0200 Subject: [PATCH 37/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: break nix help "" "" "" build Garbage in, error out. Experimental CLI. Zero derivations given. --- src/nix/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 21d364a1818..00ad6fe2c97 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -240,7 +240,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs lowdown. */ static void showHelp(std::vector subcommand, NixArgs & toplevel) { - auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", dropEmptyInitThenConcatStringsSep("-", subcommand)); + auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); evalSettings.restrictEval = false; evalSettings.pureEval = false; @@ -275,7 +275,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto attr = vRes->attrs()->get(state.symbols.create(mdName + ".md")); if (!attr) - throw UsageError("Nix has no subcommand '%s'", dropEmptyInitThenConcatStringsSep("", subcommand)); + throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); auto markdown = state.forceString(*attr->value, noPos, "while evaluating the lowdown help text"); From cf3c5cd189b5818e837f308507db3e92a81d66d0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:34:41 +0200 Subject: [PATCH 38/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: showVersions version is not empty --- src/nix/diff-closures.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 9e5a7e4c3a8..46c94b211f9 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -6,6 +6,8 @@ #include +#include "strings.hh" + namespace nix { struct Info @@ -49,7 +51,7 @@ std::string showVersions(const std::set & versions) std::set versions2; for (auto & version : versions) versions2.insert(version.empty() ? "ε" : version); - return dropEmptyInitThenConcatStringsSep(", ", versions2); + return concatStringsSep(", ", versions2); } void printClosureDiff( From 0fe3525223976e1c0ad047cc706da66208214fb5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:39:06 +0200 Subject: [PATCH 39/70] illegal configuration line -> syntax error in configuration line The law has nothing to do with this, although I do feel like a badass when I mess with the config. I'm a conf artist. --- src/libutil/config.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 25bfe462fe5..b3e2d1a482e 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -116,7 +116,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p if (tokens.empty()) continue; if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto include = false; auto ignoreMissing = false; @@ -129,7 +129,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p if (include) { if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { try { @@ -145,7 +145,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p } if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); std::string name = std::move(tokens[0]); From 4029426ca8e8d48a74222d2058ff152d7fd36027 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:41:49 +0200 Subject: [PATCH 40/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: tokens from tokenizeString are not empty --- src/libutil/config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index b3e2d1a482e..6d929b7f77d 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -154,7 +154,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p parsedContents.push_back({ std::move(name), - dropEmptyInitThenConcatStringsSep(" ", Strings(i, tokens.end())), + concatStringsSep(" ", Strings(i, tokens.end())), }); }; } From 9ca42d5da20cbbb76cd2f35f2a442d70505bf862 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:43:11 +0200 Subject: [PATCH 41/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: setting value was already harmed Considering that `value` was probably parsed with tokenizeString prior, it's unlikely to contain empty strings, and we have no reason to remove them either. --- src/libutil/config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 6d929b7f77d..726e5091e54 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -320,7 +320,7 @@ template<> void BaseSetting::appendOrSet(Strings newValue, bool append) template<> std::string BaseSetting::to_string() const { - return dropEmptyInitThenConcatStringsSep(" ", value); + return concatStringsSep(" ", value); } template<> StringSet BaseSetting::parse(const std::string & str) const @@ -336,7 +336,7 @@ template<> void BaseSetting::appendOrSet(StringSet newValue, bool app template<> std::string BaseSetting::to_string() const { - return dropEmptyInitThenConcatStringsSep(" ", value); + return concatStringsSep(" ", value); } template<> std::set BaseSetting>::parse(const std::string & str) const From 76b2d5ef3ddd876fbe825f5804f1331dfcf2ecfe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:49:34 +0200 Subject: [PATCH 42/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: PATH handling It's still wrong, but one step closer to correct. Not that anyone should use "" or "." in their PATH, but that is not for us to intervene. --- src/nix/env.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/env.cc b/src/nix/env.cc index 7cc019c1dca..bc4e1b5e583 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -3,6 +3,7 @@ #include "command.hh" #include "run.hh" +#include "strings.hh" using namespace nix; @@ -92,9 +93,10 @@ struct CmdShell : InstallablesCommand, MixEnvironment } } + // TODO: split losslessly; empty means . auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); - auto unixPathString = dropEmptyInitThenConcatStringsSep(":", unixPath); + auto unixPathString = concatStringsSep(":", unixPath); setEnv("PATH", unixPathString.c_str()); Strings args; From 6b2c277c363f59b0e200882644deb5d0e658ec20 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 01:51:50 +0200 Subject: [PATCH 43/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: sigs are not empty ... but if they are, I'd like to see at least a hint of it so that I'd know to fix it. --- src/nix/path-info.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 2383fbe080a..e7cfb6e7a68 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + using namespace nix; using nlohmann::json; @@ -185,7 +187,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (info->ultimate) ss.push_back("ultimate"); if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); - std::cout << dropEmptyInitThenConcatStringsSep(" ", ss); + std::cout << concatStringsSep(" ", ss); } std::cout << std::endl; From 1c97718146264de4c4bc6e1694774da5d5b31188 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 13 Jul 2024 02:16:13 +0200 Subject: [PATCH 44/70] dropEmptyInitThenConcatStringsSep: Allow it to drop items again It's usually harmless, if it occurs at all. --- src/libutil/util.hh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c545afd9e35..b653bf115ef 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -41,11 +41,13 @@ std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const { size_t size = 0; - for (auto & i : ss) { - // Make sure we don't rely on the empty item ignoring behavior - assert(!i.empty()); - break; - } + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } // need a cast to string_view since this is also called with Symbols for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); From d40fdb5711e8f7c7dff88f611d5bc26d37ba79e9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 Jul 2024 11:50:14 +0200 Subject: [PATCH 45/70] dropEmptyInitThenConcatStringsSep: Update doc and deprecate --- src/libutil/util.hh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b653bf115ef..971ecf63b1d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -33,10 +33,14 @@ template C tokenizeString(std::string_view s, std::string_view separato /** - * Concatenate the given strings with a separator between the - * elements. + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. */ template +[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { size_t size = 0; From 97e01107ecf4d5cea97bd20423409ba2a112612c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 Jul 2024 11:51:53 +0200 Subject: [PATCH 46/70] dropEmptyInitThenConcatStringsSep -> concatStringSep: empty separator When the separator is empty, no difference is observable. Note that concatStringsSep has centralized definitions. This adds the required definitions. Alternatively, `strings-inline.hh` could be included at call sites. --- src/libutil/strings.cc | 7 +++++++ src/libutil/util.hh | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 15937d41570..7ec618bf449 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -9,4 +9,11 @@ template std::string concatStringsSep(std::string_view, const Strings &); template std::string concatStringsSep(std::string_view, const StringSet &); template std::string concatStringsSep(std::string_view, const std::vector &); +typedef std::string_view strings_2[2]; +template std::string concatStringsSep(std::string_view, const strings_2 &); +typedef std::string_view strings_3[3]; +template std::string concatStringsSep(std::string_view, const strings_3 &); +typedef std::string_view strings_4[4]; +template std::string concatStringsSep(std::string_view, const strings_4 &); + } // namespace nix diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 971ecf63b1d..877d1527945 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,8 @@ #include #include +#include "strings.hh" + namespace nix { void initLibUtil(); @@ -69,7 +71,7 @@ auto concatStrings(Parts && ... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { std::string_view views[sizeof...(parts)] = { parts... }; - return dropEmptyInitThenConcatStringsSep({}, views); + return concatStringsSep({}, views); } From 7e604f716cd3a4bcd47407bda27874262212dcbc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 14 Jul 2024 12:19:55 +0200 Subject: [PATCH 47/70] concatStrings: Give compiler access to definition for inlining ... at call sites that are may be in the hot path. I do not know how clever the compiler gets at these sites. My primary concern is to not regress performance and I am confident that this achieves it the easy way. --- src/libexpr/eval.cc | 2 ++ src/libexpr/nixexpr.cc | 2 ++ src/libstore/derivations.cc | 2 ++ src/libutil/canon-path.cc | 1 + src/libutil/file-system.cc | 2 ++ 5 files changed, 9 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2ede55de7b6..a4cf2e8c85d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -49,6 +49,8 @@ #endif +#include "strings-inline.hh" + using json = nlohmann::json; namespace nix { diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 81638916550..c1ffe3435e8 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -7,6 +7,8 @@ #include #include +#include "strings-inline.hh" + namespace nix { unsigned long Expr::nrExprs = 0; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6dfcc408c33..8f9c7185168 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -10,6 +10,8 @@ #include #include +#include "strings-inline.hh" + namespace nix { std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 27f048697e5..03db6378a82 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,6 +1,7 @@ #include "canon-path.hh" #include "util.hh" #include "file-path-impl.hh" +#include "strings-inline.hh" namespace nix { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index f75851bbd58..9042e3a5e67 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -23,6 +23,8 @@ # include #endif +#include "strings-inline.hh" + namespace fs = std::filesystem; namespace nix { From 550b3479cf066b9e5a435d1cb42240f88021b31f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jul 2024 15:46:30 +0200 Subject: [PATCH 48/70] Include the accessor in the SourcePath hash --- src/libutil/source-path.hh | 6 +++++- src/libutil/util.hh | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 94174412787..d310231a7c9 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -9,6 +9,8 @@ #include "canon-path.hh" #include "source-accessor.hh" +#include // for boost::hash_combine + namespace nix { /** @@ -128,6 +130,8 @@ struct std::hash { std::size_t operator()(const nix::SourcePath & s) const noexcept { - return std::hash{}(s.path); + std::size_t hash = 0; + hash_combine(hash, s.accessor->number, s.path); + return hash; } }; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 23682ff7ca5..dd113984355 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -360,4 +360,18 @@ inline std::string operator + (std::string_view s1, const char * s2) return s; } +/** + * hash_combine() from Boost. Hash several hashable values together + * into a single hash. + */ +inline void hash_combine(std::size_t & seed) { } + +template +inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed<<6) + (seed>>2); + hash_combine(seed, rest...); +} + } From 63f520fd00ec86f63a0e036f0be52cac822b2ac1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jul 2024 11:49:18 +0200 Subject: [PATCH 49/70] doc/testing: Typo --- doc/manual/src/contributing/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index a96ba997bed..3949164d503 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -91,7 +91,7 @@ A environment variables that Google Test accepts are also worth knowing: Putting the two together, one might run ```bash -GTEST_BREIF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v ``` for short but comprensive output. From e5af7cbeb9fdf5cff5f8cb25fbb99dccf13e47af Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jul 2024 11:43:07 +0200 Subject: [PATCH 50/70] libutil: Add Pos::getSnippetUpTo(Pos) --- src/libutil/position.cc | 46 +++++++++++++ src/libutil/position.hh | 2 + tests/unit/libutil/position.cc | 120 +++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 tests/unit/libutil/position.cc diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 573efeeb2d8..508816d850b 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -110,4 +110,50 @@ void Pos::LinesIterator::bump(bool atFirst) input.remove_prefix(eol); } +std::string Pos::getSnippetUpTo(const Pos & end) const { + assert(this->origin == end.origin); + + if (end.line < this->line) + return ""; + + if (auto source = getSource()) { + + auto firstLine = LinesIterator(*source); + for (auto i = 1; i < this->line; ++i) { + ++firstLine; + } + + auto lastLine = LinesIterator(*source); + for (auto i = 1; i < end.line; ++i) { + ++lastLine; + } + + LinesIterator linesEnd; + + std::string result; + for (auto i = firstLine; i != linesEnd; ++i) { + auto firstColumn = i == firstLine ? (this->column ? this->column - 1 : 0) : 0; + if (firstColumn > i->size()) + firstColumn = i->size(); + + auto lastColumn = i == lastLine ? (end.column ? end.column - 1 : 0) : std::numeric_limits::max(); + if (lastColumn < firstColumn) + lastColumn = firstColumn; + if (lastColumn > i->size()) + lastColumn = i->size(); + + result += i->substr(firstColumn, lastColumn - firstColumn); + + if (i == lastLine) { + break; + } else { + result += '\n'; + } + } + return result; + } + return ""; +} + + } diff --git a/src/libutil/position.hh b/src/libutil/position.hh index f8f34419b7a..729f2a52383 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -63,6 +63,8 @@ struct Pos bool operator==(const Pos & rhs) const = default; auto operator<=>(const Pos & rhs) const = default; + std::string getSnippetUpTo(const Pos & end) const; + struct LinesIterator { using difference_type = size_t; using value_type = std::string_view; diff --git a/tests/unit/libutil/position.cc b/tests/unit/libutil/position.cc new file mode 100644 index 00000000000..d38d2c538b6 --- /dev/null +++ b/tests/unit/libutil/position.cc @@ -0,0 +1,120 @@ +#include + +#include "position.hh" + +namespace nix { + +inline Pos::Origin makeStdin(std::string s) +{ + return Pos::Stdin{make_ref(s)}; +} + +TEST(Position, getSnippetUpTo_0) +{ + Pos::Origin o = makeStdin(""); + Pos p(1, 1, o); + ASSERT_EQ(p.getSnippetUpTo(p), ""); +} +TEST(Position, getSnippetUpTo_1) +{ + Pos::Origin o = makeStdin("x"); + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 0, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 99, o); + Pos end(99, 0, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + + // "x" might be preferable, but we only care about not crashing for invalid inputs + ASSERT_EQ(start.getSnippetUpTo(end), ""); + + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 1, o); + Pos end(1, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 1, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } +} +TEST(Position, getSnippetUpTo_2) +{ + Pos::Origin o = makeStdin("asdf\njkl\nqwer"); + { + Pos start(1, 1, o); + Pos end(1, 2, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "a"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 2, o); + Pos end(1, 3, o); + ASSERT_EQ(start.getSnippetUpTo(end), "s"); + } + { + Pos start(1, 2, o); + Pos end(2, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\nj"); + } + { + Pos start(1, 2, o); + Pos end(3, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl\nq"); + } + { + Pos start(1, 2, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl"); + } + { + Pos start(1, 4, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "f\njkl"); + } + { + Pos start(1, 5, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); + } + { + Pos start(1, 6, o); // invalid: starting column past last "line character", ie at the newline + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); // jkl might be acceptable for this invalid start position + } + { + Pos start(1, 1, o); + Pos end(2, 0, o); // invalid + ASSERT_EQ(start.getSnippetUpTo(end), "asdf\n"); + } +} + +TEST(Position, example_1) +{ + Pos::Origin o = makeStdin(" unambiguous = \n /** Very close */\n x: x;\n# ok\n"); + Pos start(2, 5, o); + Pos end(2, 22, o); + ASSERT_EQ(start.getSnippetUpTo(end), "/** Very close */"); +} + +} // namespace nix From 7fae378835534d2a17818fd7cd91aae91320aa03 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 8 Jul 2024 17:39:26 +0200 Subject: [PATCH 51/70] Track doc comments and render them in :doc --- .../rl-next/repl-doc-renders-doc-comments.md | 53 ++++++++++++++++ src/libexpr/eval.cc | 61 +++++++++++++++++++ src/libexpr/lexer.l | 56 +++++++++++++++-- src/libexpr/nixexpr.cc | 14 +++++ src/libexpr/nixexpr.hh | 53 +++++++++++++++- src/libexpr/parser-state.hh | 41 +++++++++++++ src/libexpr/parser.y | 51 +++++++++++++--- tests/functional/repl.sh | 30 +++++++++ tests/functional/repl/characterisation/empty | 0 .../repl/doc-comment-function.expected | 8 +++ tests/functional/repl/doc-comment-function.in | 1 + .../functional/repl/doc-comment-function.nix | 3 + tests/functional/repl/doc-comments.nix | 57 +++++++++++++++++ tests/functional/repl/doc-compact.expected | 11 ++++ tests/functional/repl/doc-compact.in | 2 + tests/functional/repl/doc-floatedIn.expected | 11 ++++ tests/functional/repl/doc-floatedIn.in | 2 + .../repl/doc-lambda-flavors.expected | 29 +++++++++ tests/functional/repl/doc-lambda-flavors.in | 5 ++ .../functional/repl/doc-measurement.expected | 11 ++++ tests/functional/repl/doc-measurement.in | 2 + tests/functional/repl/doc-multiply.expected | 15 +++++ tests/functional/repl/doc-multiply.in | 2 + .../functional/repl/doc-unambiguous.expected | 11 ++++ tests/functional/repl/doc-unambiguous.in | 2 + 25 files changed, 515 insertions(+), 16 deletions(-) create mode 100644 doc/manual/rl-next/repl-doc-renders-doc-comments.md create mode 100644 tests/functional/repl/characterisation/empty create mode 100644 tests/functional/repl/doc-comment-function.expected create mode 100644 tests/functional/repl/doc-comment-function.in create mode 100644 tests/functional/repl/doc-comment-function.nix create mode 100644 tests/functional/repl/doc-comments.nix create mode 100644 tests/functional/repl/doc-compact.expected create mode 100644 tests/functional/repl/doc-compact.in create mode 100644 tests/functional/repl/doc-floatedIn.expected create mode 100644 tests/functional/repl/doc-floatedIn.in create mode 100644 tests/functional/repl/doc-lambda-flavors.expected create mode 100644 tests/functional/repl/doc-lambda-flavors.in create mode 100644 tests/functional/repl/doc-measurement.expected create mode 100644 tests/functional/repl/doc-measurement.in create mode 100644 tests/functional/repl/doc-multiply.expected create mode 100644 tests/functional/repl/doc-multiply.in create mode 100644 tests/functional/repl/doc-unambiguous.expected create mode 100644 tests/functional/repl/doc-unambiguous.in diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md new file mode 100644 index 00000000000..c9213a88c2b --- /dev/null +++ b/doc/manual/rl-next/repl-doc-renders-doc-comments.md @@ -0,0 +1,53 @@ +--- +synopsis: "`nix-repl`'s `:doc` shows documentation comments" +significance: significant +issues: +- 3904 +- 10771 +prs: +- 1652 +- 9054 +- 11072 +--- + +`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. +This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + +Example: + +``` +nix-repl> :doc lib.toFunction +Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + +Inputs + + v + + : Any value + +Examples + + :::{.example} + +## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: +``` + +Known limitations: +- It currently only works for functions. We plan to extend this to attributes, which may contain arbitrary values. +- Some extensions to markdown are not yet supported, as you can see in the example above. + +We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. + +[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a4cf2e8c85d..bd7bac0f189 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -559,6 +559,67 @@ std::optional EvalState::getDoc(Value & v) .doc = doc, }; } + if (v.isLambda()) { + auto exprLambda = v.payload.lambda.fun; + + std::stringstream s(std::ios_base::out); + std::string name; + auto pos = positions[exprLambda->getPos()]; + std::string docStr; + + if (exprLambda->name) { + name = symbols[exprLambda->name]; + } + + if (exprLambda->docComment) { + auto begin = positions[exprLambda->docComment.begin]; + auto end = positions[exprLambda->docComment.end]; + auto docCommentStr = begin.getSnippetUpTo(end); + + // Strip "/**" and "*/" + constexpr size_t prefixLen = 3; + constexpr size_t suffixLen = 2; + docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); + if (docStr.empty()) + return {}; + // Turn the now missing "/**" into indentation + docStr = " " + docStr; + // Strip indentation (for the whole, potentially multi-line string) + docStr = stripIndentation(docStr); + } + + if (name.empty()) { + s << "Function "; + } + else { + s << "Function **" << name << "**"; + if (pos) + s << "\\\n … " ; + else + s << "\\\n"; + } + if (pos) { + s << "defined at " << pos; + } + if (!docStr.empty()) { + s << "\n\n"; + } + + s << docStr; + + s << '\0'; // for making a c string below + std::string ss = s.str(); + + return Doc { + .pos = pos, + .name = name, + .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... + .args = {}, + .doc = + // FIXME: this leaks; make the field std::string? + strdup(ss.data()), + }; + } return {}; } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 8c0f9d1f2f7..459f1d6f382 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -5,7 +5,7 @@ %option stack %option nodefault %option nounput noyy_top_state - +%option extra-type="::nix::LexerState *" %s DEFAULT %x STRING @@ -23,6 +23,12 @@ #include "nixexpr.hh" #include "parser-tab.hh" +// !!! FIXME !!! +#define YY_EXTRA_TYPE ::nix::LexerState * +int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner); +YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner ); +#undef YY_EXTRA_TYPE + using namespace nix; namespace nix { @@ -35,10 +41,24 @@ static void initLoc(YYLTYPE * loc) loc->first_column = loc->last_column = 0; } -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) +static void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) { loc->stash(); + LexerState & lexerState = *yyget_extra(yyscanner); + + if (lexerState.docCommentDistance == 1) { + // Preceding token was a doc comment. + ParserLocation doc; + doc.first_column = lexerState.lastDocCommentLoc.first_column; + ParserLocation docEnd; + docEnd.first_column = lexerState.lastDocCommentLoc.last_column; + DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; + PosIdx locPos = lexerState.at(*loc); + lexerState.positionToDocComment.emplace(locPos, docComment); + } + lexerState.docCommentDistance++; + loc->first_column = loc->last_column; loc->last_column += len; } @@ -79,7 +99,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); +#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng); #define PUSH_STATE(state) yy_push_state(state, yyscanner) #define POP_STATE() yy_pop_state(yyscanner) @@ -279,9 +299,33 @@ or { return OR_KW; } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } {URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; } -[ \t\r\n]+ /* eat up whitespace */ -\#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ +%{ +// Doc comment rule +// +// \/\*\* /** +// [^/*] reject /**/ (empty comment) and /*** +// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule +// ( )* zero or more non-ending sequences +// \* end(1) +// \/ end(2) +%} +\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { + LexerState & lexerState = *yyget_extra(yyscanner); + lexerState.docCommentDistance = 0; + lexerState.lastDocCommentLoc.first_line = yylloc->first_line; + lexerState.lastDocCommentLoc.first_column = yylloc->first_column; + lexerState.lastDocCommentLoc.last_column = yylloc->last_column; +} + + +%{ +// The following rules have docCommentDistance-- +// This compensates for the docCommentDistance++ which happens by default to +// make all the other rules invalidate the doc comment. +%} +[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; } +\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; } +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; } {ANY} { /* Don't return a negative number, as this will cause diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index c1ffe3435e8..6e705c314a3 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -583,6 +583,20 @@ std::string ExprLambda::showNamePos(const EvalState & state) const return fmt("%1% at %2%", id, state.positions[pos]); } +void ExprLambda::setDocComment(DocComment docComment) { + if (!this->docComment) { + this->docComment = docComment; + + // Curried functions are defined by putting a function directly + // in the body of another function. To render docs for those, we + // need to propagate the doc comment to the innermost function. + // + // If we have our own comment, we've already propagated it, so this + // belongs in the same conditional. + body->setDocComment(docComment); + } +}; + /* Position table. */ diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5152e31196e..913f46ff97b 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -11,13 +11,56 @@ namespace nix { - -struct Env; -struct Value; class EvalState; +class PosTable; +struct Env; struct ExprWith; struct StaticEnv; +struct Value; + +/** + * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md) + * + * Note that this does not implement the following: + * - argument attribute names ("formals"): TBD + * - argument names: these are internal to the function and their names may not be optimal for documentation + * - function arity (degree of currying or number of ':'s): + * - Functions returning partially applied functions have a higher arity + * than can be determined locally and without evaluation. + * We do not want to present false data. + * - Some functions should be thought of as transformations of other + * functions. For instance `overlay -> overlay -> overlay` is the simplest + * way to understand `composeExtensions`, but its implementation looks like + * `f: g: final: prev: <...>`. The parameters `final` and `prev` are part + * of the overlay concept, while distracting from the function's purpose. + */ +struct DocComment { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcomment" // "nested comment start" is intentional + + /** + * Start of the comment, including `/**`. + */ + PosIdx begin; + +#pragma GCC diagnostic pop + + /** + * Position right after the final asterisk and `/` that terminate the comment. + */ + PosIdx end; + + /** + * Whether the comment is set. + * + * A `DocComment` is small enough that it makes sense to pass by value, and + * therefore baking optionality into it is also useful, to avoiding the memory + * overhead of `std::optional`. + */ + operator bool() const { return static_cast(begin); } + +}; /** * An attribute path is a sequence of attribute names. @@ -54,6 +97,7 @@ struct Expr virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol name); + virtual void setDocComment(DocComment docComment) { }; virtual PosIdx getPos() const { return noPos; } }; @@ -278,6 +322,8 @@ struct ExprLambda : Expr Symbol arg; Formals * formals; Expr * body; + DocComment docComment; + ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body) : pos(pos), arg(arg), formals(formals), body(body) { @@ -290,6 +336,7 @@ struct ExprLambda : Expr std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } PosIdx getPos() const override { return pos; } + virtual void setDocComment(DocComment docComment) override; COMMON_METHODS }; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index cff6282faa0..8dc91046864 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "eval.hh" namespace nix { @@ -35,10 +37,44 @@ struct ParserLocation first_column = stashed_first_column; last_column = stashed_last_column; } + + /** Latest doc comment position, or 0. */ + int doc_comment_first_line, doc_comment_first_column, doc_comment_last_column; +}; + +struct LexerState +{ + /** + * Tracks the distance to the last doc comment, in terms of lexer tokens. + * + * The lexer sets this to 0 when reading a doc comment, and increments it + * for every matched rule; see `lexer-helpers.cc`. + * Whitespace and comment rules decrement the distance, so that they result + * in a net 0 change in distance. + */ + int docCommentDistance = std::numeric_limits::max(); + + /** + * The location of the last doc comment. + * + * (stashing fields are not used) + */ + ParserLocation lastDocCommentLoc; + + /** + * @brief Maps some positions to a DocComment, where the comment is relevant to the location. + */ + std::map positionToDocComment; + + PosTable & positions; + PosTable::Origin origin; + + PosIdx at(const ParserLocation & loc); }; struct ParserState { + const LexerState & lexerState; SymbolTable & symbols; PosTable & positions; Expr * result; @@ -270,6 +306,11 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, return new ExprConcatStrings(pos, true, es2); } +inline PosIdx LexerState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.first_column); +} + inline PosIdx ParserState::at(const ParserLocation & loc) { return positions.add(origin, loc.first_column); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 709a4532a85..3c1bf95c882 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -74,6 +74,14 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * }); } +#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos)) +static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) { + auto it = lexerState.positionToDocComment.find(start); + if (it != lexerState.positionToDocComment.end()) { + lambda->setDocComment(it->second); + } +} + %} @@ -119,6 +127,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS + %right IMPL %left OR %left AND @@ -140,18 +149,28 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); } + { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); + $$ = me; + SET_DOC_POS(me, @1); + } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); } + { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + $$ = me; + SET_DOC_POS(me, @1); + } | '{' formals '}' '@' ID ':' expr_function { auto arg = state->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + $$ = me; + SET_DOC_POS(me, @1); } | ID '@' '{' formals '}' ':' expr_function { auto arg = state->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + $$ = me; + SET_DOC_POS(me, @1); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -312,7 +331,20 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } + : binds attrpath '=' expr ';' { + $$ = $1; + + auto pos = state->at(@2); + { + auto it = state->lexerState.positionToDocComment.find(pos); + if (it != state->lexerState.positionToDocComment.end()) { + $4->setDocComment(it->second); + } + } + + state->addAttr($$, std::move(*$2), $4, pos); + delete $2; + } | binds INHERIT attrs ';' { $$ = $1; for (auto & [i, iPos] : *$3) { @@ -435,17 +467,22 @@ Expr * parseExprFromBuf( const Expr::AstSymbols & astSymbols) { yyscan_t scanner; + LexerState lexerState { + .positions = positions, + .origin = positions.addOrigin(origin, length), + }; ParserState state { + .lexerState = lexerState, .symbols = symbols, .positions = positions, .basePath = basePath, - .origin = positions.addOrigin(origin, length), + .origin = lexerState.origin, .rootFS = rootFS, .s = astSymbols, .settings = settings, }; - yylex_init(&scanner); + yylex_init_extra(&lexerState, &scanner); Finally _destroy([&] { yylex_destroy(scanner); }); yy_scan_buffer(text, length, scanner); diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 86cd6f458d0..d356fe09625 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash source common.sh +source characterisation/framework.sh testDir="$PWD" cd "$TEST_ROOT" @@ -244,3 +245,32 @@ testReplResponseNoRegex ' y = { a = 1 }; } ' + +# TODO: move init to characterisation/framework.sh +badDiff=0 +badExitCode=0 + +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" + +runRepl () { + # TODO: pass arguments to nix repl; see lang.sh + nix repl 2>&1 \ + | stripColors \ + | sed \ + -e "s@$testDir@/path/to/tests/functional@g" \ + -e "s@$nixVersion@@g" \ + -e "s@Added [0-9]* variables@Added variables@g" \ + | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ + ; +} + +for test in $(cd "$testDir/repl"; echo *.in); do + test="$(basename "$test" .in)" + in="$testDir/repl/$test.in" + actual="$testDir/repl/$test.actual" + expected="$testDir/repl/$test.expected" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + diffAndAcceptInner "$test" "$actual" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/repl/characterisation/empty b/tests/functional/repl/characterisation/empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected new file mode 100644 index 00000000000..5ec465a96c9 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.expected @@ -0,0 +1,8 @@ +Nix +Type :? for help. +Function defined at + /path/to/tests/functional/repl/doc-comment-function.nix:2:1 + + A doc comment for a file that only contains a function + + diff --git a/tests/functional/repl/doc-comment-function.in b/tests/functional/repl/doc-comment-function.in new file mode 100644 index 00000000000..8f3c1388a09 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.in @@ -0,0 +1 @@ +:doc import ./doc-comment-function.nix diff --git a/tests/functional/repl/doc-comment-function.nix b/tests/functional/repl/doc-comment-function.nix new file mode 100644 index 00000000000..cdd2413476f --- /dev/null +++ b/tests/functional/repl/doc-comment-function.nix @@ -0,0 +1,3 @@ +/** A doc comment for a file that only contains a function */ +{ ... }: +{ } diff --git a/tests/functional/repl/doc-comments.nix b/tests/functional/repl/doc-comments.nix new file mode 100644 index 00000000000..a98f1c688f5 --- /dev/null +++ b/tests/functional/repl/doc-comments.nix @@ -0,0 +1,57 @@ +{ + /** + Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + + ```nix + multiply 2 3 + => 6 + ``` + */ + multiply = x: y: x * y; + + /**👈 precisely this wide 👉*/ + measurement = x: x; + + floatedIn = /** This also works. */ + x: y: x; + + compact=/**boom*/x: x; + + /** Ignore!!! */ + unambiguous = + /** Very close */ + x: x; + + /** Firmly rigid. */ + constant = true; + + /** Immovably fixed. */ + lib.version = "9000"; + + /** Unchangeably constant. */ + lib.attr.empty = { }; + + lib.attr.undocumented = { }; + + nonStrict = /** My syntax is not strict, but I'm strict anyway. */ x: x; + strict = /** I don't have to be strict, but I am anyway. */ { ... }: null; + # Note that pre and post are the same here. I just had to name them somehow. + strictPre = /** Here's one way to do this */ a@{ ... }: a; + strictPost = /** Here's another way to do this */ { ... }@a: a; + + # TODO + + # /** This returns a documented function. */ + # documentedArgs = + # /** x */ + # x: + # /** y */ + # y: + # /** x + y */ + # x + y; + + # /** Documented formals */ + # documentedFormals = + # /** x */ + # x: x; +} diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected new file mode 100644 index 00000000000..4b05b653c45 --- /dev/null +++ b/tests/functional/repl/doc-compact.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function compact + … defined at + /path/to/tests/functional/repl/doc-comments.nix:18:20 + + boom + + diff --git a/tests/functional/repl/doc-compact.in b/tests/functional/repl/doc-compact.in new file mode 100644 index 00000000000..c87c4e7abbe --- /dev/null +++ b/tests/functional/repl/doc-compact.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc compact diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected new file mode 100644 index 00000000000..30f13572586 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function floatedIn + … defined at + /path/to/tests/functional/repl/doc-comments.nix:16:5 + + This also works. + + diff --git a/tests/functional/repl/doc-floatedIn.in b/tests/functional/repl/doc-floatedIn.in new file mode 100644 index 00000000000..97c12408e63 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc floatedIn diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected new file mode 100644 index 00000000000..4ac52f39e4b --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -0,0 +1,29 @@ +Nix +Type :? for help. +Added variables. + +Function nonStrict + … defined at + /path/to/tests/functional/repl/doc-comments.nix:36:70 + + My syntax is not strict, but I'm strict anyway. + +Function strict + … defined at + /path/to/tests/functional/repl/doc-comments.nix:37:63 + + I don't have to be strict, but I am anyway. + +Function strictPre + … defined at + /path/to/tests/functional/repl/doc-comments.nix:39:48 + + Here's one way to do this + +Function strictPost + … defined at + /path/to/tests/functional/repl/doc-comments.nix:40:53 + + Here's another way to do this + + diff --git a/tests/functional/repl/doc-lambda-flavors.in b/tests/functional/repl/doc-lambda-flavors.in new file mode 100644 index 00000000000..760c99636a6 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.in @@ -0,0 +1,5 @@ +:l doc-comments.nix +:doc nonStrict +:doc strict +:doc strictPre +:doc strictPost diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected new file mode 100644 index 00000000000..8598aaedbc7 --- /dev/null +++ b/tests/functional/repl/doc-measurement.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function measurement + … defined at + /path/to/tests/functional/repl/doc-comments.nix:13:17 + + 👈 precisely this wide 👉 + + diff --git a/tests/functional/repl/doc-measurement.in b/tests/functional/repl/doc-measurement.in new file mode 100644 index 00000000000..fecd5f9d2f0 --- /dev/null +++ b/tests/functional/repl/doc-measurement.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc measurement diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected new file mode 100644 index 00000000000..db82af91fc6 --- /dev/null +++ b/tests/functional/repl/doc-multiply.expected @@ -0,0 +1,15 @@ +Nix +Type :? for help. +Added variables. + +Function multiply + … defined at + /path/to/tests/functional/repl/doc-comments.nix:10:14 + + Perform arithmetic multiplication. It's kind of like + repeated addition, very neat. + + | multiply 2 3 + | => 6 + + diff --git a/tests/functional/repl/doc-multiply.in b/tests/functional/repl/doc-multiply.in new file mode 100644 index 00000000000..bffc6696fd4 --- /dev/null +++ b/tests/functional/repl/doc-multiply.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc multiply diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected new file mode 100644 index 00000000000..0f89a6f641e --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function unambiguous + … defined at + /path/to/tests/functional/repl/doc-comments.nix:23:5 + + Very close + + diff --git a/tests/functional/repl/doc-unambiguous.in b/tests/functional/repl/doc-unambiguous.in new file mode 100644 index 00000000000..8282a5cb95c --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc unambiguous From e68234c4f991071c0c7ff05852975a74d6a21771 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jul 2024 15:39:24 +0200 Subject: [PATCH 52/70] libexpr: Rearrange lexer files so that yylex_init_extra can be found --- src/libexpr/lexer-helpers.cc | 31 ++++++++++++++++++++++++++ src/libexpr/lexer-helpers.hh | 9 ++++++++ src/libexpr/lexer.l | 43 ++++++++---------------------------- src/libexpr/meson.build | 2 ++ 4 files changed, 51 insertions(+), 34 deletions(-) create mode 100644 src/libexpr/lexer-helpers.cc create mode 100644 src/libexpr/lexer-helpers.hh diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc new file mode 100644 index 00000000000..87c514c71cc --- /dev/null +++ b/src/libexpr/lexer-helpers.cc @@ -0,0 +1,31 @@ +#include "lexer-tab.hh" +#include "lexer-helpers.hh" +#include "parser-tab.hh" + +void nix::lexer::internal::initLoc(YYLTYPE * loc) +{ + loc->first_line = loc->last_line = 0; + loc->first_column = loc->last_column = 0; +} + +void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) +{ + loc->stash(); + + LexerState & lexerState = *yyget_extra(yyscanner); + + if (lexerState.docCommentDistance == 1) { + // Preceding token was a doc comment. + ParserLocation doc; + doc.first_column = lexerState.lastDocCommentLoc.first_column; + ParserLocation docEnd; + docEnd.first_column = lexerState.lastDocCommentLoc.last_column; + DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; + PosIdx locPos = lexerState.at(*loc); + lexerState.positionToDocComment.emplace(locPos, docComment); + } + lexerState.docCommentDistance++; + + loc->first_column = loc->last_column; + loc->last_column += len; +} diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh new file mode 100644 index 00000000000..caba6e18f48 --- /dev/null +++ b/src/libexpr/lexer-helpers.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix::lexer::internal { + +void initLoc(YYLTYPE * loc); + +void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len); + +} // namespace nix::lexer diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 459f1d6f382..7d4e6963fc9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -14,6 +14,10 @@ %x INPATH_SLASH %x PATH_START +%top { +#include "parser-tab.hh" // YYSTYPE +#include "parser-state.hh" +} %{ #ifdef __clang__ @@ -22,48 +26,19 @@ #include "nixexpr.hh" #include "parser-tab.hh" +#include "lexer-helpers.hh" -// !!! FIXME !!! -#define YY_EXTRA_TYPE ::nix::LexerState * -int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner); -YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner ); -#undef YY_EXTRA_TYPE +namespace nix { + struct LexerState; +} using namespace nix; +using namespace nix::lexer::internal; namespace nix { #define CUR_POS state->at(*yylloc) -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 0; - loc->first_column = loc->last_column = 0; -} - -static void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) -{ - loc->stash(); - - LexerState & lexerState = *yyget_extra(yyscanner); - - if (lexerState.docCommentDistance == 1) { - // Preceding token was a doc comment. - ParserLocation doc; - doc.first_column = lexerState.lastDocCommentLoc.first_column; - ParserLocation docEnd; - docEnd.first_column = lexerState.lastDocCommentLoc.last_column; - DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; - PosIdx locPos = lexerState.at(*loc); - lexerState.positionToDocComment.emplace(locPos, docComment); - } - lexerState.docCommentDistance++; - - loc->first_column = loc->last_column; - loc->last_column += len; -} - - // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index e65cf545f2d..fa90e7b41a3 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -139,6 +139,7 @@ sources = files( 'function-trace.cc', 'get-drvs.cc', 'json-to-value.cc', + 'lexer-helpers.cc', 'nixexpr.cc', 'paths.cc', 'primops.cc', @@ -165,6 +166,7 @@ headers = [config_h] + files( 'gc-small-vector.hh', 'get-drvs.hh', 'json-to-value.hh', + # internal: 'lexer-helpers.hh', 'nixexpr.hh', 'parser-state.hh', 'pos-idx.hh', From 491b9cf4154370df3da04977696dda3f053725f1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jul 2024 19:26:22 +0200 Subject: [PATCH 53/70] Refactor: extract DocComment::getInnerText(PosTable) --- src/libexpr/eval.cc | 15 +-------------- src/libexpr/nixexpr.cc | 18 ++++++++++++++++++ src/libexpr/nixexpr.hh | 2 ++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bd7bac0f189..f8282c4000d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -572,20 +572,7 @@ std::optional EvalState::getDoc(Value & v) } if (exprLambda->docComment) { - auto begin = positions[exprLambda->docComment.begin]; - auto end = positions[exprLambda->docComment.end]; - auto docCommentStr = begin.getSnippetUpTo(end); - - // Strip "/**" and "*/" - constexpr size_t prefixLen = 3; - constexpr size_t suffixLen = 2; - docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); - if (docStr.empty()) - return {}; - // Turn the now missing "/**" into indentation - docStr = " " + docStr; - // Strip indentation (for the whole, potentially multi-line string) - docStr = stripIndentation(docStr); + docStr = exprLambda->docComment.getInnerText(positions); } if (name.empty()) { diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 6e705c314a3..5479dd79e86 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -641,4 +641,22 @@ size_t SymbolTable::totalSize() const return n; } +std::string DocComment::getInnerText(const PosTable & positions) const { + auto beginPos = positions[begin]; + auto endPos = positions[end]; + auto docCommentStr = beginPos.getSnippetUpTo(endPos); + + // Strip "/**" and "*/" + constexpr size_t prefixLen = 3; + constexpr size_t suffixLen = 2; + std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); + if (docStr.empty()) + return {}; + // Turn the now missing "/**" into indentation + docStr = " " + docStr; + // Strip indentation (for the whole, potentially multi-line string) + docStr = stripIndentation(docStr); + return docStr; +} + } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 913f46ff97b..803b5ed8fd1 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -60,6 +60,8 @@ struct DocComment { */ operator bool() const { return static_cast(begin); } + std::string getInnerText(const PosTable & positions) const; + }; /** From d4f576b0b281265b58bb497108f8d063e2a0f06a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jul 2024 19:25:45 +0200 Subject: [PATCH 54/70] nix repl: Render docs for attributes --- .../rl-next/repl-doc-renders-doc-comments.md | 2 +- src/libcmd/repl.cc | 41 +++++++ src/libexpr/eval.cc | 42 +++++++- src/libexpr/eval.hh | 10 ++ src/libexpr/nixexpr.hh | 11 ++ src/libexpr/parser-state.hh | 2 +- src/libexpr/parser.y | 7 ++ src/libutil/position.hh | 8 ++ tests/functional/repl/doc-constant.expected | 102 ++++++++++++++++++ tests/functional/repl/doc-constant.in | 15 +++ 10 files changed, 237 insertions(+), 3 deletions(-) create mode 100644 tests/functional/repl/doc-constant.expected create mode 100644 tests/functional/repl/doc-constant.in diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md index c9213a88c2b..05023697c96 100644 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ b/doc/manual/rl-next/repl-doc-renders-doc-comments.md @@ -45,7 +45,7 @@ Examples ``` Known limitations: -- It currently only works for functions. We plan to extend this to attributes, which may contain arbitrary values. +- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. - Some extensions to markdown are not yet supported, as you can see in the example above. We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 37a34e3de0e..fb8106c4602 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -27,6 +27,7 @@ #include "local-fs-store.hh" #include "print.hh" #include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -616,6 +617,33 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // Trigger the normal error + evalString(arg, v); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + + } else { + evalString(arg, v); + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -633,6 +661,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f8282c4000d..81c64ba71ee 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1415,6 +1415,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) v = *vAttrs; } +Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs) +{ + Value vTmp; + Symbol name = getName(attrPath[attrPath.size() - 1], state, env); + + if (attrPath.size() == 1) { + e->eval(state, env, vTmp); + } else { + ExprSelect init(*this); + init.attrPath.pop_back(); + init.eval(state, env, vTmp); + } + attrs = vTmp; + return name; +} + void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) { @@ -2876,13 +2892,37 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols); + DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath + DocCommentMap *docComments = &tmpDocComments; + + if (auto sourcePath = std::get_if(&origin)) { + auto [it, _] = positionToDocComment.try_emplace(*sourcePath); + docComments = &it->second; + } + + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols); result->bindVars(*this, staticEnv); return result; } +DocComment EvalState::getDocCommentForPos(PosIdx pos) +{ + auto pos2 = positions[pos]; + auto path = pos2.getSourcePath(); + if (!path) + return {}; + + auto table = positionToDocComment.find(*path); + if (table == positionToDocComment.end()) + return {}; + + auto it = table->second.find(pos); + if (it == table->second.end()) + return {}; + return it->second; +} std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7dbf61c5dfe..3918fb092c4 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -130,6 +130,8 @@ struct Constant typedef std::map ValMap; #endif +typedef std::map DocCommentMap; + struct Env { Env * up; @@ -329,6 +331,12 @@ private: #endif FileEvalCache fileEvalCache; + /** + * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. + * Grouped by file. + */ + std::map positionToDocComment; + LookupPath lookupPath; std::map> lookupPathResolved; @@ -771,6 +779,8 @@ public: std::string_view pathArg, PosIdx pos); + DocComment getDocCommentForPos(PosIdx pos); + private: /** diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 803b5ed8fd1..4c4c8af1948 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -202,6 +202,17 @@ struct ExprSelect : Expr ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } + + /** + * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl. + * + * @param[out] v The attribute set that should contain the last attribute name (if it exists). + * @return The last attribute name in `attrPath` + * + * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. + */ + Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); + COMMON_METHODS }; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 8dc91046864..1df64e73d57 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -64,7 +64,7 @@ struct LexerState /** * @brief Maps some positions to a DocComment, where the comment is relevant to the location. */ - std::map positionToDocComment; + std::map & positionToDocComment; PosTable & positions; PosTable::Origin origin; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 3c1bf95c882..a25c6dd8769 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -33,6 +33,8 @@ namespace nix { +typedef std::map DocCommentMap; + Expr * parseExprFromBuf( char * text, size_t length, @@ -41,6 +43,7 @@ Expr * parseExprFromBuf( SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -335,10 +338,12 @@ binds $$ = $1; auto pos = state->at(@2); + auto exprPos = state->at(@4); { auto it = state->lexerState.positionToDocComment.find(pos); if (it != state->lexerState.positionToDocComment.end()) { $4->setDocComment(it->second); + state->lexerState.positionToDocComment.emplace(exprPos, it->second); } } @@ -463,11 +468,13 @@ Expr * parseExprFromBuf( SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols) { yyscan_t scanner; LexerState lexerState { + .positionToDocComment = docComments, .positions = positions, .origin = positions.addOrigin(origin, length), }; diff --git a/src/libutil/position.hh b/src/libutil/position.hh index 729f2a52383..aba263fdfe4 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -7,6 +7,7 @@ #include #include +#include #include "source-path.hh" @@ -65,6 +66,13 @@ struct Pos std::string getSnippetUpTo(const Pos & end) const; + /** + * Get the SourcePath, if the source was loaded from a file. + */ + std::optional getSourcePath() const { + return *std::get_if(&origin); + } + struct LinesIterator { using difference_type = size_t; using value_type = std::string_view; diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected new file mode 100644 index 00000000000..9aca0617872 --- /dev/null +++ b/tests/functional/repl/doc-constant.expected @@ -0,0 +1,102 @@ +Nix +Type :? for help. +Added variables. + +error: value does not have documentation + +Attribute version + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:29:3 + + Immovably fixed. + +Attribute empty + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:32:3 + + Unchangeably constant. + +error: + … while evaluating the attribute 'attr.undocument' + at /path/to/tests/functional/repl/doc-comments.nix:32:3: + 31| /** Unchangeably constant. */ + 32| lib.attr.empty = { }; + | ^ + 33| + + error: attribute 'undocument' missing + at «string»:1:1: + 1| lib.attr.undocument + | ^ + Did you mean undocumented? + +Attribute constant + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:26:3 + + Firmly rigid. + +Attribute version + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:29:3 + + Immovably fixed. + +Attribute empty + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:32:3 + + Unchangeably constant. + +Attribute undocumented + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:34:3 + + No documentation found. + +error: undefined variable 'missing' + at «string»:1:1: + 1| missing + | ^ + +error: undefined variable 'constanz' + at «string»:1:1: + 1| constanz + | ^ + +error: undefined variable 'missing' + at «string»:1:1: + 1| missing.attr + | ^ + +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing + | ^ + +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing.attr + | ^ + +error: + … while evaluating the attribute 'attr.undocumental' + at /path/to/tests/functional/repl/doc-comments.nix:32:3: + 31| /** Unchangeably constant. */ + 32| lib.attr.empty = { }; + | ^ + 33| + + error: attribute 'undocumental' missing + at «string»:1:1: + 1| lib.attr.undocumental + | ^ + Did you mean undocumented? + + diff --git a/tests/functional/repl/doc-constant.in b/tests/functional/repl/doc-constant.in new file mode 100644 index 00000000000..9c0dde5e154 --- /dev/null +++ b/tests/functional/repl/doc-constant.in @@ -0,0 +1,15 @@ +:l doc-comments.nix +:doc constant +:doc lib.version +:doc lib.attr.empty +:doc lib.attr.undocument +:doc (import ./doc-comments.nix).constant +:doc (import ./doc-comments.nix).lib.version +:doc (import ./doc-comments.nix).lib.attr.empty +:doc (import ./doc-comments.nix).lib.attr.undocumented +:doc missing +:doc constanz +:doc missing.attr +:doc lib.missing +:doc lib.missing.attr +:doc lib.attr.undocumental From cef11b23e89b71f81bc8c4b4b47c3ac87a1a8315 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 9 Jul 2024 19:27:36 +0200 Subject: [PATCH 55/70] Add missing .sh in _NIX_TEST_ACCEPT=1 message --- src/libexpr/eval.cc | 2 +- tests/functional/characterisation/framework.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 81c64ba71ee..c309e7e98f7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -579,7 +579,7 @@ std::optional EvalState::getDoc(Value & v) s << "Function "; } else { - s << "Function **" << name << "**"; + s << "Function `" << name << "`"; if (pos) s << "\\\n … " ; else diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh index 913fdd9673d..5ca125ab5bc 100644 --- a/tests/functional/characterisation/framework.sh +++ b/tests/functional/characterisation/framework.sh @@ -63,7 +63,7 @@ function characterisationTestExit() { echo '' echo 'You can rerun this test with:' echo '' - echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test" + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.sh.test" echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' From f9243eca75b0847a3984721564cf098baa185698 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 10 Jul 2024 00:48:23 +0200 Subject: [PATCH 56/70] tests/functional/repl.sh: Work around GHA failure --- tests/functional/repl.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index d356fe09625..c1f265df308 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -253,11 +253,21 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" runRepl () { + + # That is right, we are also filtering out the testdir _without underscores_. + # This is crazy, but without it, GHA will fail to run the tests, showing paths + # _with_ underscores in the set -x log, but _without_ underscores in the + # supposed nix repl output. I have looked in a number of places, but I cannot + # find a mechanism that could cause this to happen. + local testDirNoUnderscores + testDirNoUnderscores="${testDir//_/}" + # TODO: pass arguments to nix repl; see lang.sh nix repl 2>&1 \ | stripColors \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ + -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ -e "s@$nixVersion@@g" \ -e "s@Added [0-9]* variables@Added variables@g" \ | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ From 77e9f9ee82db9c77ef67f3916df746383ff451c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 11 Jul 2024 12:58:20 +0200 Subject: [PATCH 57/70] libexpr: Get rid of unused line tracking fields --- src/libexpr/lexer-helpers.cc | 1 - src/libexpr/lexer.l | 1 - src/libexpr/parser-state.hh | 6 +++--- src/libexpr/parser.y | 16 +++++++++++++++- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index 87c514c71cc..f7d1017c361 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -4,7 +4,6 @@ void nix::lexer::internal::initLoc(YYLTYPE * loc) { - loc->first_line = loc->last_line = 0; loc->first_column = loc->last_column = 0; } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 7d4e6963fc9..b1b87b96a33 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -287,7 +287,6 @@ or { return OR_KW; } \/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { LexerState & lexerState = *yyget_extra(yyscanner); lexerState.docCommentDistance = 0; - lexerState.lastDocCommentLoc.first_line = yylloc->first_line; lexerState.lastDocCommentLoc.first_column = yylloc->first_column; lexerState.lastDocCommentLoc.last_column = yylloc->last_column; } diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 1df64e73d57..3e801c13ab6 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -22,8 +22,8 @@ struct StringToken struct ParserLocation { - int first_line, first_column; - int last_line, last_column; + int first_column; + int last_column; // backup to recover from yyless(0) int stashed_first_column, stashed_last_column; @@ -39,7 +39,7 @@ struct ParserLocation } /** Latest doc comment position, or 0. */ - int doc_comment_first_line, doc_comment_first_column, doc_comment_last_column; + int doc_comment_first_column, doc_comment_last_column; }; struct LexerState diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index a25c6dd8769..76190c0faae 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,6 +31,21 @@ #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) +// For efficiency, we only track offsets; not line,column coordinates +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (0) + namespace nix { typedef std::map DocCommentMap; @@ -69,7 +84,6 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * { if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { loc->first_column = loc->last_column; - loc->first_line = loc->last_line; } throw ParseError({ .msg = HintFmt(error), From 71cb8bf509cf0e180e19230350318e2b453e6dbe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 11 Jul 2024 13:06:39 +0200 Subject: [PATCH 58/70] libexpr: Rename "column" fields to offset ... because that's what they are. --- src/libexpr/lexer-helpers.cc | 10 +++++----- src/libexpr/lexer.l | 4 ++-- src/libexpr/parser-state.hh | 18 +++++++++--------- src/libexpr/parser.y | 10 +++++----- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index f7d1017c361..d9eeb73e269 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -4,7 +4,7 @@ void nix::lexer::internal::initLoc(YYLTYPE * loc) { - loc->first_column = loc->last_column = 0; + loc->beginOffset = loc->endOffset = 0; } void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) @@ -16,15 +16,15 @@ void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const ch if (lexerState.docCommentDistance == 1) { // Preceding token was a doc comment. ParserLocation doc; - doc.first_column = lexerState.lastDocCommentLoc.first_column; + doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset; ParserLocation docEnd; - docEnd.first_column = lexerState.lastDocCommentLoc.last_column; + docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset; DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; PosIdx locPos = lexerState.at(*loc); lexerState.positionToDocComment.emplace(locPos, docComment); } lexerState.docCommentDistance++; - loc->first_column = loc->last_column; - loc->last_column += len; + loc->beginOffset = loc->endOffset; + loc->endOffset += len; } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index b1b87b96a33..58401be8e71 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -287,8 +287,8 @@ or { return OR_KW; } \/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { LexerState & lexerState = *yyget_extra(yyscanner); lexerState.docCommentDistance = 0; - lexerState.lastDocCommentLoc.first_column = yylloc->first_column; - lexerState.lastDocCommentLoc.last_column = yylloc->last_column; + lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset; + lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset; } diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 3e801c13ab6..983a17a2e98 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -22,20 +22,20 @@ struct StringToken struct ParserLocation { - int first_column; - int last_column; + int beginOffset; + int endOffset; // backup to recover from yyless(0) - int stashed_first_column, stashed_last_column; + int stashedBeginOffset, stashedEndOffset; void stash() { - stashed_first_column = first_column; - stashed_last_column = last_column; + stashedBeginOffset = beginOffset; + stashedEndOffset = endOffset; } void unstash() { - first_column = stashed_first_column; - last_column = stashed_last_column; + beginOffset = stashedBeginOffset; + endOffset = stashedEndOffset; } /** Latest doc comment position, or 0. */ @@ -308,12 +308,12 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, inline PosIdx LexerState::at(const ParserLocation & loc) { - return positions.add(origin, loc.first_column); + return positions.add(origin, loc.beginOffset); } inline PosIdx ParserState::at(const ParserLocation & loc) { - return positions.add(origin, loc.first_column); + return positions.add(origin, loc.beginOffset); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 76190c0faae..452d265bcbf 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -36,13 +36,13 @@ do \ if (N) \ { \ - (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ - (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \ + (Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \ } \ else \ { \ - (Current).first_column = (Current).last_column = \ - YYRHSLOC (Rhs, 0).last_column; \ + (Current).beginOffset = (Current).endOffset = \ + YYRHSLOC (Rhs, 0).endOffset; \ } \ while (0) @@ -83,7 +83,7 @@ using namespace nix; void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { - loc->first_column = loc->last_column; + loc->beginOffset = loc->endOffset; } throw ParseError({ .msg = HintFmt(error), From 8a855296f555fdb5a334acb5f482f3c781e87657 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 14:27:09 +0200 Subject: [PATCH 59/70] tests/function/repl: Characterise the missing doc comment behavior --- src/libexpr/nixexpr.cc | 2 ++ .../repl/doc-comment-curried-args.expected | 24 +++++++++++++++ .../repl/doc-comment-curried-args.in | 7 +++++ .../repl/doc-comment-formals.expected | 13 +++++++++ tests/functional/repl/doc-comment-formals.in | 3 ++ tests/functional/repl/doc-comments.nix | 29 ++++++++++--------- tests/functional/repl/doc-constant.expected | 28 +++++++++--------- .../repl/doc-lambda-flavors.expected | 8 ++--- .../functional/repl/doc-unambiguous.expected | 2 +- 9 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 tests/functional/repl/doc-comment-curried-args.expected create mode 100644 tests/functional/repl/doc-comment-curried-args.in create mode 100644 tests/functional/repl/doc-comment-formals.expected create mode 100644 tests/functional/repl/doc-comment-formals.in diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 5479dd79e86..d54f0b684b5 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -584,6 +584,8 @@ std::string ExprLambda::showNamePos(const EvalState & state) const } void ExprLambda::setDocComment(DocComment docComment) { + // RFC 145 specifies that the innermost doc comment wins. + // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement if (!this->docComment) { this->docComment = docComment; diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected new file mode 100644 index 00000000000..c10c171e1e8 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -0,0 +1,24 @@ +Nix +Type :? for help. +Added variables. + +Function curriedArgs + … defined at + /path/to/tests/functional/repl/doc-comments.nix:48:5 + + A documented function. + + +"Note that users may not expect this to behave as it currently does" + +Function curriedArgs + … defined at + /path/to/tests/functional/repl/doc-comments.nix:50:5 + + The function returned by applying once + +"This won't produce documentation, because we can't actually document arbitrary values" + +error: value does not have documentation + + diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in new file mode 100644 index 00000000000..8dbbfc37052 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -0,0 +1,7 @@ +:l doc-comments.nix +:doc curriedArgs +x = curriedArgs 1 +"Note that users may not expect this to behave as it currently does" +:doc x +"This won't produce documentation, because we can't actually document arbitrary values" +:doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected new file mode 100644 index 00000000000..704c0050bd8 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. +Added variables. + +"Note that this is not yet complete" + +Function documentedFormals + … defined at + /path/to/tests/functional/repl/doc-comments.nix:57:5 + + Finds x + + diff --git a/tests/functional/repl/doc-comment-formals.in b/tests/functional/repl/doc-comment-formals.in new file mode 100644 index 00000000000..e32fb8ab1a3 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.in @@ -0,0 +1,3 @@ +:l doc-comments.nix +"Note that this is not yet complete" +:doc documentedFormals diff --git a/tests/functional/repl/doc-comments.nix b/tests/functional/repl/doc-comments.nix index a98f1c688f5..e91ee0b513d 100644 --- a/tests/functional/repl/doc-comments.nix +++ b/tests/functional/repl/doc-comments.nix @@ -17,6 +17,7 @@ compact=/**boom*/x: x; + # https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement /** Ignore!!! */ unambiguous = /** Very close */ @@ -41,17 +42,19 @@ # TODO - # /** This returns a documented function. */ - # documentedArgs = - # /** x */ - # x: - # /** y */ - # y: - # /** x + y */ - # x + y; - - # /** Documented formals */ - # documentedFormals = - # /** x */ - # x: x; + /** You won't see this. */ + curriedArgs = + /** A documented function. */ + x: + /** The function returned by applying once */ + y: + /** A function body performing summation of two items */ + x + y; + + /** Documented formals (but you won't see this comment) */ + documentedFormals = + /** Finds x */ + { /** The x attribute */ + x + }: x; } diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index 9aca0617872..c6655833376 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -7,24 +7,24 @@ error: value does not have documentation Attribute version … defined at - /path/to/tests/functional/repl/doc-comments.nix:29:3 + /path/to/tests/functional/repl/doc-comments.nix:30:3 Immovably fixed. Attribute empty … defined at - /path/to/tests/functional/repl/doc-comments.nix:32:3 + /path/to/tests/functional/repl/doc-comments.nix:33:3 Unchangeably constant. error: … while evaluating the attribute 'attr.undocument' - at /path/to/tests/functional/repl/doc-comments.nix:32:3: - 31| /** Unchangeably constant. */ - 32| lib.attr.empty = { }; + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; | ^ - 33| + 34| error: attribute 'undocument' missing at «string»:1:1: @@ -35,28 +35,28 @@ error: Attribute constant … defined at - /path/to/tests/functional/repl/doc-comments.nix:26:3 + /path/to/tests/functional/repl/doc-comments.nix:27:3 Firmly rigid. Attribute version … defined at - /path/to/tests/functional/repl/doc-comments.nix:29:3 + /path/to/tests/functional/repl/doc-comments.nix:30:3 Immovably fixed. Attribute empty … defined at - /path/to/tests/functional/repl/doc-comments.nix:32:3 + /path/to/tests/functional/repl/doc-comments.nix:33:3 Unchangeably constant. Attribute undocumented … defined at - /path/to/tests/functional/repl/doc-comments.nix:34:3 + /path/to/tests/functional/repl/doc-comments.nix:35:3 No documentation found. @@ -87,11 +87,11 @@ error: attribute 'missing' missing error: … while evaluating the attribute 'attr.undocumental' - at /path/to/tests/functional/repl/doc-comments.nix:32:3: - 31| /** Unchangeably constant. */ - 32| lib.attr.empty = { }; + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; | ^ - 33| + 34| error: attribute 'undocumental' missing at «string»:1:1: diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 4ac52f39e4b..43f483ce943 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -4,25 +4,25 @@ Added variables. Function nonStrict … defined at - /path/to/tests/functional/repl/doc-comments.nix:36:70 + /path/to/tests/functional/repl/doc-comments.nix:37:70 My syntax is not strict, but I'm strict anyway. Function strict … defined at - /path/to/tests/functional/repl/doc-comments.nix:37:63 + /path/to/tests/functional/repl/doc-comments.nix:38:63 I don't have to be strict, but I am anyway. Function strictPre … defined at - /path/to/tests/functional/repl/doc-comments.nix:39:48 + /path/to/tests/functional/repl/doc-comments.nix:40:48 Here's one way to do this Function strictPost … defined at - /path/to/tests/functional/repl/doc-comments.nix:40:53 + /path/to/tests/functional/repl/doc-comments.nix:41:53 Here's another way to do this diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 0f89a6f641e..825aa1ee170 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -4,7 +4,7 @@ Added variables. Function unambiguous … defined at - /path/to/tests/functional/repl/doc-comments.nix:23:5 + /path/to/tests/functional/repl/doc-comments.nix:24:5 Very close From 6bbd493d49fa58aa128a2419db5b1139d5c7a944 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 15:08:41 +0200 Subject: [PATCH 60/70] libcmd/repl-interacter: INT_MAX -> numeric_limits --- src/libcmd/repl-interacter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 420cce1db89..b285c8a9ae5 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -73,7 +73,7 @@ static int listPossibleCallback(char * s, char *** avp) { auto possible = curRepl->completePrefix(s); - if (possible.size() > (INT_MAX / sizeof(char *))) + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) throw Error("too many completions"); int ac = 0; From 131b6ccc716d1807f4ea91bc3fc1239ff7775648 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 18:04:33 +0200 Subject: [PATCH 61/70] nixexpr.hh: Avoid the warning and pragmas --- src/libexpr/nixexpr.hh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4c4c8af1948..1bcc962c56a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -36,16 +36,11 @@ struct Value; */ struct DocComment { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcomment" // "nested comment start" is intentional - /** - * Start of the comment, including `/**`. + * Start of the comment, including the opening, ie `/` and `**`. */ PosIdx begin; -#pragma GCC diagnostic pop - /** * Position right after the final asterisk and `/` that terminate the comment. */ From 21817473e8035e2231c083c46184724057a30d7f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 19:04:37 +0200 Subject: [PATCH 62/70] Doc comments: use std::unordered_map Co-authored-by: Eelco Dolstra --- src/libexpr/eval.hh | 4 ++-- src/libexpr/parser-state.hh | 2 +- src/libexpr/parser.y | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3918fb092c4..d9a3e80bc8c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -130,7 +130,7 @@ struct Constant typedef std::map ValMap; #endif -typedef std::map DocCommentMap; +typedef std::unordered_map DocCommentMap; struct Env { @@ -335,7 +335,7 @@ private: * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. * Grouped by file. */ - std::map positionToDocComment; + std::unordered_map positionToDocComment; LookupPath lookupPath; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 983a17a2e98..5a7bcb7175d 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -64,7 +64,7 @@ struct LexerState /** * @brief Maps some positions to a DocComment, where the comment is relevant to the location. */ - std::map & positionToDocComment; + std::unordered_map & positionToDocComment; PosTable & positions; PosTable::Origin origin; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 452d265bcbf..8ea176b2461 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -48,7 +48,7 @@ namespace nix { -typedef std::map DocCommentMap; +typedef std::unordered_map DocCommentMap; Expr * parseExprFromBuf( char * text, From ac89df815d90eec38935f6c238df8811bd907cf9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 19:23:23 +0200 Subject: [PATCH 63/70] libcmd/repl.cc: Explain evalString call and defend --- src/libcmd/repl.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index fb8106c4602..b5d0816dd2c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -632,8 +632,13 @@ ProcessLineResult NixRepl::processLine(std::string line) assert(attrs); auto attr = attrs->get(name); if (!attr) { - // Trigger the normal error + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing evalString(arg, v); + assert(false); } if (attr->pos) { fallbackPos = attr->pos; From 6a125e65d0f2dac90cdc69f16be0a4bd888c7a2f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 19:33:56 +0200 Subject: [PATCH 64/70] Revert "Doc comments: use std::unordered_map" hash isn't implemented yet, and I can't cherry-pick a bug-free commit yet. This reverts commit 95529f31e3bbda99111c5ce98a33484dc6e7a462. --- src/libexpr/eval.hh | 4 ++-- src/libexpr/parser-state.hh | 2 +- src/libexpr/parser.y | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d9a3e80bc8c..3918fb092c4 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -130,7 +130,7 @@ struct Constant typedef std::map ValMap; #endif -typedef std::unordered_map DocCommentMap; +typedef std::map DocCommentMap; struct Env { @@ -335,7 +335,7 @@ private: * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. * Grouped by file. */ - std::unordered_map positionToDocComment; + std::map positionToDocComment; LookupPath lookupPath; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 5a7bcb7175d..983a17a2e98 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -64,7 +64,7 @@ struct LexerState /** * @brief Maps some positions to a DocComment, where the comment is relevant to the location. */ - std::unordered_map & positionToDocComment; + std::map & positionToDocComment; PosTable & positions; PosTable::Origin origin; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8ea176b2461..452d265bcbf 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -48,7 +48,7 @@ namespace nix { -typedef std::unordered_map DocCommentMap; +typedef std::map DocCommentMap; Expr * parseExprFromBuf( char * text, From ce31a0457f1e2e32d751200bb0a964d4a3e8f253 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 19:55:01 +0200 Subject: [PATCH 65/70] Use HintFmt for doc comments --- src/libcmd/repl.cc | 6 +++--- src/libexpr/eval.cc | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd2c..a555fcfccee 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -668,12 +668,12 @@ ProcessLineResult NixRepl::processLine(std::string line) logger->cout(trim(renderMarkdownToTerminal(markdown))); } else if (fallbackPos) { std::stringstream ss; - ss << "Attribute `" << fallbackName << "`\n\n"; - ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + ss << HintFmt("Attribute '%1%'", fallbackName) << "\n\n"; + ss << HintFmt(" … defined at %1%", state->positions[fallbackPos]) << "\n\n"; if (fallbackDoc) { ss << fallbackDoc.getInnerText(state->positions); } else { - ss << "No documentation found.\n\n"; + ss << HintFmt("No documentation found.") << "\n\n"; } auto markdown = ss.str(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c309e7e98f7..dd3677e39b2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -576,17 +576,17 @@ std::optional EvalState::getDoc(Value & v) } if (name.empty()) { - s << "Function "; + s << HintFmt("Function "); } else { - s << "Function `" << name << "`"; + s << HintFmt("Function '%s'", name); if (pos) s << "\\\n … " ; else s << "\\\n"; } if (pos) { - s << "defined at " << pos; + s << HintFmt("defined at %1%", pos); } if (!docStr.empty()) { s << "\n\n"; From 03d33703ef60ec40d8c376e4d935e991fc176294 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 19:55:05 +0200 Subject: [PATCH 66/70] Revert "Use HintFmt for doc comments" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unfortunately these don't render correctly, because they go into the markdown renderer, instead of the terminal. ``` nix-repl> :doc lib.version Attribute '[35;1mversion[0m' … defined at [35;1m/home/user/h/nixpkgs/lib/default.nix:73:40[0m ``` We could switch that to go direct to the terminal, but then we should do the same for the primops, to get a consistent look. Reverting for now. This reverts commit 3413e0338cbee1c7734d5cb614b5325e51815cde. --- src/libcmd/repl.cc | 6 +++--- src/libexpr/eval.cc | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index a555fcfccee..b5d0816dd2c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -668,12 +668,12 @@ ProcessLineResult NixRepl::processLine(std::string line) logger->cout(trim(renderMarkdownToTerminal(markdown))); } else if (fallbackPos) { std::stringstream ss; - ss << HintFmt("Attribute '%1%'", fallbackName) << "\n\n"; - ss << HintFmt(" … defined at %1%", state->positions[fallbackPos]) << "\n\n"; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; if (fallbackDoc) { ss << fallbackDoc.getInnerText(state->positions); } else { - ss << HintFmt("No documentation found.") << "\n\n"; + ss << "No documentation found.\n\n"; } auto markdown = ss.str(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dd3677e39b2..c309e7e98f7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -576,17 +576,17 @@ std::optional EvalState::getDoc(Value & v) } if (name.empty()) { - s << HintFmt("Function "); + s << "Function "; } else { - s << HintFmt("Function '%s'", name); + s << "Function `" << name << "`"; if (pos) s << "\\\n … " ; else s << "\\\n"; } if (pos) { - s << HintFmt("defined at %1%", pos); + s << "defined at " << pos; } if (!docStr.empty()) { s << "\n\n"; From 61a4d3d45c91cb19a114796846e5af014f59a6b6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 15 Jul 2024 20:08:41 +0200 Subject: [PATCH 67/70] getSnippetUpTo: Return optional This makes it possible to certain discern failures from empty snippets, which I think is an ok review comment. Maybe it should do so for swapped column indexes too, but I'm not sure. I don't think it matters in the grand scheme. We don't even have a real use case for `nullopt` now anyway. Since we don't have a use case, I'm not applying this logic to higher level functions yet. --- src/libexpr/nixexpr.cc | 2 +- src/libutil/position.cc | 6 +++--- src/libutil/position.hh | 2 +- tests/unit/libutil/position.cc | 8 +++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d54f0b684b5..93c8bdef6b6 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -646,7 +646,7 @@ size_t SymbolTable::totalSize() const std::string DocComment::getInnerText(const PosTable & positions) const { auto beginPos = positions[begin]; auto endPos = positions[end]; - auto docCommentStr = beginPos.getSnippetUpTo(endPos); + auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or(""); // Strip "/**" and "*/" constexpr size_t prefixLen = 3; diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 508816d850b..3289dbe8bf0 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -110,11 +110,11 @@ void Pos::LinesIterator::bump(bool atFirst) input.remove_prefix(eol); } -std::string Pos::getSnippetUpTo(const Pos & end) const { +std::optional Pos::getSnippetUpTo(const Pos & end) const { assert(this->origin == end.origin); if (end.line < this->line) - return ""; + return std::nullopt; if (auto source = getSource()) { @@ -152,7 +152,7 @@ std::string Pos::getSnippetUpTo(const Pos & end) const { } return result; } - return ""; + return std::nullopt; } diff --git a/src/libutil/position.hh b/src/libutil/position.hh index aba263fdfe4..25217069c14 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -64,7 +64,7 @@ struct Pos bool operator==(const Pos & rhs) const = default; auto operator<=>(const Pos & rhs) const = default; - std::string getSnippetUpTo(const Pos & end) const; + std::optional getSnippetUpTo(const Pos & end) const; /** * Get the SourcePath, if the source was loaded from a file. diff --git a/tests/unit/libutil/position.cc b/tests/unit/libutil/position.cc index d38d2c538b6..484ecc2479b 100644 --- a/tests/unit/libutil/position.cc +++ b/tests/unit/libutil/position.cc @@ -25,7 +25,7 @@ TEST(Position, getSnippetUpTo_1) ASSERT_EQ(start.getSnippetUpTo(start), ""); ASSERT_EQ(start.getSnippetUpTo(end), "x"); ASSERT_EQ(end.getSnippetUpTo(end), ""); - ASSERT_EQ(end.getSnippetUpTo(start), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); } { // NOTE: line and column are actually 1-based indexes @@ -37,7 +37,7 @@ TEST(Position, getSnippetUpTo_1) ASSERT_EQ(start.getSnippetUpTo(end), ""); ASSERT_EQ(end.getSnippetUpTo(end), ""); - ASSERT_EQ(end.getSnippetUpTo(start), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); } { Pos start(1, 1, o); @@ -53,7 +53,7 @@ TEST(Position, getSnippetUpTo_1) ASSERT_EQ(start.getSnippetUpTo(start), ""); ASSERT_EQ(start.getSnippetUpTo(end), "x"); ASSERT_EQ(end.getSnippetUpTo(end), ""); - ASSERT_EQ(end.getSnippetUpTo(start), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); } } TEST(Position, getSnippetUpTo_2) @@ -65,6 +65,8 @@ TEST(Position, getSnippetUpTo_2) ASSERT_EQ(start.getSnippetUpTo(start), ""); ASSERT_EQ(start.getSnippetUpTo(end), "a"); ASSERT_EQ(end.getSnippetUpTo(end), ""); + + // nullopt? I feel like changing the column handling would just make it more fragile ASSERT_EQ(end.getSnippetUpTo(start), ""); } { From 1bec90e3c4792289b960875d050ec65fd273a780 Mon Sep 17 00:00:00 2001 From: Goldstein Date: Mon, 15 Jul 2024 23:11:26 +0300 Subject: [PATCH 68/70] tests/functional/repl.sh: fail test on wrong stdout Previous test implementation assumed that grep supports newlines in patterns. It doesn't, so tests spuriously passed, even though some tests outputs were broken. This patches output (and expected output) before grepping, so there're no newlines in pattern. --- tests/functional/repl.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index c1f265df308..4f5bb36aae9 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -74,21 +74,31 @@ testReplResponseGeneral () { local grepMode commands expectedResponse response grepMode="$1"; shift commands="$1"; shift - expectedResponse="$1"; shift - response="$(nix repl "$@" <<< "$commands" | stripColors)" - echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ - || fail "repl command set: + # Expected response can contain newlines. + # grep can't handle multiline patterns, so replace newlines with TEST_NEWLINE + # in both expectedResponse and response. + # awk ORS always adds a trailing record separator, so we strip it with sed. + expectedResponse="$(printf '%s' "$1" | awk 1 ORS=TEST_NEWLINE | sed 's/TEST_NEWLINE$//')"; shift + # We don't need to strip trailing record separator here, since extra data is ok. + response="$(nix repl "$@" <<< "$commands" 2>&1 | stripColors | awk 1 ORS=TEST_NEWLINE)" + printf '%s' "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ + || fail "$(echo "repl command set: $commands does not respond with: +--- $expectedResponse +--- but with: +--- $response -" +--- + +" | sed 's/TEST_NEWLINE/\n/g')" } testReplResponse () { @@ -190,7 +200,7 @@ testReplResponseNoRegex ' let x = { y = { a = 1; }; inherit x; }; in x ' \ '{ - x = { ... }; + x = «repeated»; y = { ... }; } ' @@ -242,7 +252,7 @@ testReplResponseNoRegex ' ' \ '{ x = «repeated»; - y = { a = 1 }; + y = { a = 1; }; } ' @@ -256,7 +266,7 @@ runRepl () { # That is right, we are also filtering out the testdir _without underscores_. # This is crazy, but without it, GHA will fail to run the tests, showing paths - # _with_ underscores in the set -x log, but _without_ underscores in the + # _with_ underscores in the set -x log, but _without_ underscores in the # supposed nix repl output. I have looked in a number of places, but I cannot # find a mechanism that could cause this to happen. local testDirNoUnderscores From 846869da0ed0580beb7f827b303fef9a8386de37 Mon Sep 17 00:00:00 2001 From: Las Safin Date: Mon, 15 Jul 2024 21:49:15 +0100 Subject: [PATCH 69/70] Make goals use C++20 coroutines (#11005) undefined --- src/libstore/build/derivation-goal.cc | 141 +++++---- src/libstore/build/derivation-goal.hh | 32 +- .../build/drv-output-substitution-goal.cc | 208 ++++++------- .../build/drv-output-substitution-goal.hh | 37 +-- src/libstore/build/goal.cc | 114 ++++++- src/libstore/build/goal.hh | 283 +++++++++++++++++- src/libstore/build/substitution-goal.cc | 238 +++++++-------- src/libstore/build/substitution-goal.hh | 63 +--- src/libstore/build/worker.cc | 44 ++- .../unix/build/local-derivation-goal.cc | 19 +- .../unix/build/local-derivation-goal.hh | 2 +- 11 files changed, 709 insertions(+), 472 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f795b05a167..010f905d640 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -36,6 +36,14 @@ namespace nix { +Goal::Co DerivationGoal::init() { + if (useDerivation) { + co_return getDerivation(); + } else { + co_return haveDerivation(); + } +} + DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) @@ -44,7 +52,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -65,7 +72,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -109,13 +115,9 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, std::move(ex)); -} - - -void DerivationGoal::work() -{ - (this->*state)(); + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); } void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) @@ -139,7 +141,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() +Goal::Co DerivationGoal::getDerivation() { trace("init"); @@ -147,23 +149,22 @@ void DerivationGoal::getDerivation() exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; + co_return loadDerivation(); } addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - state = &DerivationGoal::loadDerivation; + co_await Suspend{}; + co_return loadDerivation(); } -void DerivationGoal::loadDerivation() +Goal::Co DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; + co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); } /* `drvPath' should already be a root, but let's be on the safe @@ -185,11 +186,11 @@ void DerivationGoal::loadDerivation() } assert(drv); - haveDerivation(); + co_return haveDerivation(); } -void DerivationGoal::haveDerivation() +Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); @@ -217,8 +218,7 @@ void DerivationGoal::haveDerivation() }); } - gaveUpOnSubstitution(); - return; + co_return gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -240,8 +240,7 @@ void DerivationGoal::haveDerivation() /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths @@ -268,24 +267,21 @@ void DerivationGoal::haveDerivation() } } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return outputsSubstitutionTried(); } -void DerivationGoal::outputsSubstitutionTried() +Goal::Co DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, {}, + co_return done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); - return; } /* If the substitutes form an incomplete closure, then we should @@ -319,32 +315,29 @@ void DerivationGoal::outputsSubstitutionTried() if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - haveDerivation(); - return; + co_return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); if (buildMode == bmNormal && allValid) { - done(BuildResult::Substituted, std::move(validOutputs)); - return; + co_return done(BuildResult::Substituted, std::move(validOutputs)); } if (buildMode == bmRepair && allValid) { - repairClosure(); - return; + co_return repairClosure(); } if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); + co_return gaveUpOnSubstitution(); } /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() +Goal::Co DerivationGoal::gaveUpOnSubstitution() { /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -405,14 +398,12 @@ void DerivationGoal::gaveUpOnSubstitution() addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return inputsRealised(); } -void DerivationGoal::repairClosure() +Goal::Co DerivationGoal::repairClosure() { assert(!drv->type().isImpure()); @@ -466,41 +457,39 @@ void DerivationGoal::repairClosure() } if (waitees.empty()) { - done(BuildResult::AlreadyValid, assertPathValidity()); - return; + co_return done(BuildResult::AlreadyValid, assertPathValidity()); + } else { + co_await Suspend{}; + co_return closureRepaired(); } - - state = &DerivationGoal::closureRepaired; } -void DerivationGoal::closureRepaired() +Goal::Co DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid, assertPathValidity()); + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } -void DerivationGoal::inputsRealised() +Goal::Co DerivationGoal::inputsRealised() { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, {}, Error( + co_return done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); - return; } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - haveDerivation(); - return; + co_return haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -566,8 +555,8 @@ void DerivationGoal::inputsRealised() pathResolved, wantedOutputs, buildMode); addWaitee(resolvedDrvGoal); - state = &DerivationGoal::resolvedFinished; - return; + co_await Suspend{}; + co_return resolvedFinished(); } std::function::ChildNode &)> accumInputPaths; @@ -631,8 +620,9 @@ void DerivationGoal::inputsRealised() /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryToBuild(); } void DerivationGoal::started() @@ -657,7 +647,7 @@ void DerivationGoal::started() worker.updateProgress(); } -void DerivationGoal::tryToBuild() +Goal::Co DerivationGoal::tryToBuild() { trace("trying to build"); @@ -693,7 +683,8 @@ void DerivationGoal::tryToBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryToBuild(); } actLock.reset(); @@ -710,8 +701,7 @@ void DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -737,9 +727,9 @@ void DerivationGoal::tryToBuild() EOF from the hook. */ actLock.reset(); buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; started(); - return; + co_await Suspend{}; + co_return buildDone(); case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ @@ -748,7 +738,8 @@ void DerivationGoal::tryToBuild() fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); case rpDecline: /* We should do it ourselves. */ break; @@ -757,11 +748,12 @@ void DerivationGoal::tryToBuild() actLock.reset(); - state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryLocalBuild(); } -void DerivationGoal::tryLocalBuild() { +Goal::Co DerivationGoal::tryLocalBuild() { throw Error( R"( Unable to build with a primary store that isn't a local store; @@ -938,7 +930,7 @@ void runPostBuildHook( }); } -void DerivationGoal::buildDone() +Goal::Co DerivationGoal::buildDone() { trace("build done"); @@ -1033,7 +1025,7 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); - done(BuildResult::Built, std::move(builtOutputs)); + co_return done(BuildResult::Built, std::move(builtOutputs)); } catch (BuildError & e) { outputLocks.unlock(); @@ -1058,12 +1050,11 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, std::move(e)); - return; + co_return done(st, {}, std::move(e)); } } -void DerivationGoal::resolvedFinished() +Goal::Co DerivationGoal::resolvedFinished() { trace("resolved derivation finished"); @@ -1131,7 +1122,7 @@ void DerivationGoal::resolvedFinished() if (status == BuildResult::AlreadyValid) status = BuildResult::ResolvesToAlreadyValid; - done(status, std::move(builtOutputs)); + co_return done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() @@ -1325,7 +1316,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); - done( + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done( BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); @@ -1531,7 +1524,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -void DerivationGoal::done( +Goal::Done DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional ex) @@ -1568,7 +1561,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 04f13aedd17..ad3d9ca2acf 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -194,9 +194,6 @@ struct DerivationGoal : public Goal */ std::optional derivationType; - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - BuildMode buildMode; std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; @@ -227,8 +224,6 @@ struct DerivationGoal : public Goal std::string key() override; - void work() override; - /** * Add wanted outputs to an already existing derivation goal. */ @@ -237,18 +232,19 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - virtual void tryLocalBuild(); - void buildDone(); + Co init() override; + Co getDerivation(); + Co loadDerivation(); + Co haveDerivation(); + Co outputsSubstitutionTried(); + Co gaveUpOnSubstitution(); + Co closureRepaired(); + Co inputsRealised(); + Co tryToBuild(); + virtual Co tryLocalBuild(); + Co buildDone(); - void resolvedFinished(); + Co resolvedFinished(); /** * Is the build hook willing to perform the build? @@ -329,11 +325,11 @@ struct DerivationGoal : public Goal */ virtual void killChild(); - void repairClosure(); + Co repairClosure(); void started(); - void done( + Done done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 13a07e4ea38..02284d93c1a 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -14,146 +14,135 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -void DrvOutputSubstitutionGoal::init() +Goal::Co DrvOutputSubstitutionGoal::init() { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - amDone(ecSuccess); - return; + co_return amDone(ecSuccess); } - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - tryNext(); -} - -void DrvOutputSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + + bool substituterFailed = false; + + for (auto sub : subs) { + trace("trying next substituter"); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + auto outPipe = std::make_shared(); + #ifndef _WIN32 + outPipe->create(); + #else + outPipe->createAsyncPipe(worker.ioport.get()); + #endif + + auto promise = std::make_shared>>(); + + sub->queryRealisation( + id, + { [outPipe(outPipe), promise(promise)](std::future> res) { + try { + Finally updateStats([&]() { outPipe->writeSide.close(); }); + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), { + #ifndef _WIN32 + outPipe->readSide.get() + #else + &outPipe + #endif + }, true, false); + + co_await Suspend{}; + + worker.childTerminated(this); + + /* + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ + std::shared_ptr outputInfo; + + try { + outputInfo = promise->get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; } - return; - } - - sub = subs.front(); - subs.pop_front(); - - // FIXME: Make async - // outputInfo = sub->queryRealisation(id); - - /* The callback of the curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - downloadState = std::make_shared(); -#ifndef _WIN32 - downloadState->outPipe.create(); -#else - downloadState->outPipe.createAsyncPipe(worker.ioport.get()); -#endif - - sub->queryRealisation( - id, - { [downloadState(downloadState)](std::future> res) { - try { - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); - downloadState->promise.set_value(res.get()); - } catch (...) { - downloadState->promise.set_exception(std::current_exception()); + if (!outputInfo) continue; + + bool failed = false; + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + failed = true; + break; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); } - } }); - - worker.childStarted(shared_from_this(), { -#ifndef _WIN32 - downloadState->outPipe.readSide.get() -#else - &downloadState->outPipe -#endif - }, true, false); - - state = &DrvOutputSubstitutionGoal::realisationFetched; -} + } -void DrvOutputSubstitutionGoal::realisationFetched() -{ - worker.childTerminated(this); + if (failed) continue; - try { - outputInfo = downloadState->promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - substituterFailed = true; + co_return realisationFetched(outputInfo, sub); } - if (!outputInfo) { - return tryNext(); - } + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->getUri(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath) - ); - tryNext(); - return; - } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); - } + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); } + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); +} + +Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr outputInfo, nix::ref sub) { addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); - if (waitees.empty()) outPathValid(); - else state = &DrvOutputSubstitutionGoal::outPathValid; -} + if (!waitees.empty()) co_await Suspend{}; -void DrvOutputSubstitutionGoal::outPathValid() -{ - assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; + co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); } worker.store.registerDrvOutput(*outputInfo); - finished(); -} -void DrvOutputSubstitutionGoal::finished() -{ trace("finished"); - amDone(ecSuccess); + co_return amDone(ecSuccess); } std::string DrvOutputSubstitutionGoal::key() @@ -163,14 +152,9 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -void DrvOutputSubstitutionGoal::work() -{ - (this->*state)(); -} - void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) { - if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 6967ca84ff8..80705492662 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -27,52 +27,19 @@ class DrvOutputSubstitutionGoal : public Goal { */ DrvOutput id; - /** - * The realisation corresponding to the given output id. - * Will be filled once we can get it. - */ - std::shared_ptr outputInfo; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - struct DownloadState - { - MuxablePipe outPipe; - std::promise> promise; - }; - - std::shared_ptr downloadState; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - public: DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); typedef void (DrvOutputSubstitutionGoal::*GoalState)(); GoalState state; - void init(); - void tryNext(); - void realisationFetched(); - void outPathValid(); - void finished(); + Co init() override; + Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); void timedOut(Error && ex) override { abort(); }; std::string key() override; - void work() override; void handleEOF(Descriptor fd) override; JobCategory jobCategory() const override { diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index f8db9828076..9a16da14555 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -3,6 +3,97 @@ namespace nix { +using Co = nix::Goal::Co; +using promise_type = nix::Goal::promise_type; +using handle_type = nix::Goal::handle_type; +using Suspend = nix::Goal::Suspend; + +Co::Co(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +void Co::operator=(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +Co::~Co() { + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } +} + +Co promise_type::get_return_object() { + auto handle = handle_type::from_promise(*this); + return Co{handle}; +}; + +std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept { + auto& p = h.promise(); + auto goal = p.goal; + assert(goal); + goal->trace("in final_awaiter"); + auto c = std::move(p.continuation); + + if (c) { + // We still have a continuation, i.e. work to do. + // We assert that the goal is still busy. + assert(goal->exitCode == ecBusy); + assert(goal->top_co); // Goal must have an active coroutine. + assert(goal->top_co->handle == h); // The active coroutine must be us. + assert(p.alive); // We must not have been destructed. + + // we move continuation to the top, + // note: previous top_co is actually h, so by moving into it, + // we're calling the destructor on h, DON'T use h and p after this! + + // We move our continuation into `top_co`, i.e. the marker for the active continuation. + // By doing this we destruct the old `top_co`, i.e. us, so `h` can't be used anymore. + // Be careful not to access freed memory! + goal->top_co = std::move(c); + + // We resume `top_co`. + return goal->top_co->handle; + } else { + // We have no continuation, i.e. no more work to do, + // so the goal must not be busy anymore. + assert(goal->exitCode != ecBusy); + + // We reset `top_co` for good measure. + p.goal->top_co = {}; + + // We jump to the noop coroutine, which doesn't do anything and immediately suspends. + // This passes control back to the caller of goal.work(). + return std::noop_coroutine(); + } +} + +void promise_type::return_value(Co&& next) { + goal->trace("return_value(Co&&)"); + // Save old continuation. + auto old_continuation = std::move(continuation); + // We set next as our continuation. + continuation = std::move(next); + // We set next's goal, and thus it must not have one already. + assert(!continuation->handle.promise().goal); + continuation->handle.promise().goal = goal; + // Nor can next have a continuation, as we set it to our old one. + assert(!continuation->handle.promise().continuation); + continuation->handle.promise().continuation = std::move(old_continuation); +} + +std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) { + assert(handle); // we must be a valid coroutine + auto& p = handle.promise(); + assert(!p.continuation); // we must have no continuation + assert(!p.goal); // we must not have a goal yet + auto goal = caller.promise().goal; + assert(goal); + p.goal = goal; + p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller) + goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this! + return p.goal->top_co->handle; // we execute ourselves +} bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { std::string s1 = a->key(); @@ -75,10 +166,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } } - -void Goal::amDone(ExitCode result, std::optional ex) +Goal::Done Goal::amDone(ExitCode result, std::optional ex) { trace("done"); + assert(top_co); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; @@ -98,6 +189,13 @@ void Goal::amDone(ExitCode result, std::optional ex) worker.removeGoal(shared_from_this()); cleanup(); + + // We drop the continuation. + // In `final_awaiter` this will signal that there is no more work to be done. + top_co->handle.promise().continuation = {}; + + // won't return to caller because of logic in final_awaiter + return Done{}; } @@ -106,4 +204,16 @@ void Goal::trace(std::string_view s) debug("%1%: %2%", name, s); } +void Goal::work() +{ + assert(top_co); + assert(top_co->handle); + assert(top_co->handle.promise().alive); + top_co->handle.resume(); + // We either should be in a state where we can be work()-ed again, + // or we should be done. + assert(top_co || exitCode != ecBusy); +} + + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 0d9b828e1d6..162c392d066 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,10 +1,11 @@ #pragma once ///@file -#include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include + namespace nix { /** @@ -103,9 +104,263 @@ protected: * Build result. */ BuildResult buildResult; - public: + /** + * Suspend our goal and wait until we get @ref work()-ed again. + * `co_await`-able by @ref Co. + */ + struct Suspend {}; + + /** + * Return from the current coroutine and suspend our goal + * if we're not busy anymore, or jump to the next coroutine + * set to be executed/resumed. + */ + struct Return {}; + + /** + * `co_return`-ing this will end the goal. + * If you're not inside a coroutine, you can safely discard this. + */ + struct [[nodiscard]] Done { + private: + Done(){} + + friend Goal; + }; + + // forward declaration of promise_type, see below + struct promise_type; + + /** + * Handle to coroutine using @ref Co and @ref promise_type. + */ + using handle_type = std::coroutine_handle; + + /** + * C++20 coroutine wrapper for use in goal logic. + * Coroutines are functions that use `co_await`/`co_return` (and `co_yield`, but not supported by @ref Co). + * + * @ref Co is meant to be used by methods of subclasses of @ref Goal. + * The main functionality provided by `Co` is + * - `co_await Suspend{}`: Suspends the goal. + * - `co_await f()`: Waits until `f()` finishes. + * - `co_return f()`: Tail-calls `f()`. + * - `co_return Return{}`: Ends coroutine. + * + * The idea is that you implement the goal logic using coroutines, + * and do the core thing a goal can do, suspension, when you have + * children you're waiting for. + * Coroutines allow you to resume the work cleanly. + * + * @note Brief explanation of C++20 coroutines: + * When you `Co f()`, a `std::coroutine_handle` is created, + * alongside its @ref promise_type. + * There are suspension points at the beginning of the coroutine, + * at every `co_await`, and at the final (possibly implicit) `co_return`. + * Once suspended, you can resume the `std::coroutine_handle` by doing `coroutine_handle.resume()`. + * Suspension points are implemented by passing a struct to the compiler + * that implements `await_sus`pend. + * `await_suspend` can either say "cancel suspension", in which case execution resumes, + * "suspend", in which case control is passed back to the caller of `coroutine_handle.resume()` + * or the place where the coroutine function is initially executed in the case of the initial + * suspension, or `await_suspend` can specify another coroutine to jump to, which is + * how tail calls are implemented. + * + * @note Resources: + * - https://lewissbaker.github.io/ + * - https://www.chiark.greenend.org.uk/~sgtatham/quasiblog/coroutines-c++20/ + * - https://www.scs.stanford.edu/~dm/blog/c++-coroutines.html + * + * @todo Allocate explicitly on stack since HALO thing doesn't really work, + * specifically, there's no way to uphold the requirements when trying to do + * tail-calls without using a trampoline AFAICT. + * + * @todo Support returning data natively + */ + struct [[nodiscard]] Co { + /** + * The underlying handle. + */ + handle_type handle; + + explicit Co(handle_type handle) : handle(handle) {}; + void operator=(Co&&); + Co(Co&& rhs); + ~Co(); + + bool await_ready() { return false; }; + /** + * When we `co_await` another @ref Co-returning coroutine, + * we tell the caller of `caller_coroutine.resume()` to switch to our coroutine (@ref handle). + * To make sure we return to the original coroutine, we set it as the continuation of our + * coroutine. In @ref promise_type::final_awaiter we check if it's set and if so we return to it. + * + * To explain in more understandable terms: + * When we `co_await Co_returning_function()`, this function is called on the resultant @ref Co of + * the _called_ function, and C++ automatically passes the caller in. + * + * `goal` field of @ref promise_type is also set here by copying it from the caller. + */ + std::coroutine_handle<> await_suspend(handle_type handle); + void await_resume() {}; + }; + + /** + * Used on initial suspend, does the same as @ref std::suspend_always, + * but asserts that everything has been set correctly. + */ + struct InitialSuspend { + /** + * Handle of coroutine that does the + * initial suspend + */ + handle_type handle; + + bool await_ready() { return false; }; + void await_suspend(handle_type handle_) { + handle = handle_; + } + void await_resume() { + assert(handle); + assert(handle.promise().goal); // goal must be set + assert(handle.promise().goal->top_co); // top_co of goal must be set + assert(handle.promise().goal->top_co->handle == handle); // top_co of goal must be us + } + }; + + /** + * Promise type for coroutines defined using @ref Co. + * Attached to coroutine handle. + */ + struct promise_type { + /** + * Either this is who called us, or it is who we will tail-call. + * It is what we "jump" to once we are done. + */ + std::optional continuation; + + /** + * The goal that we're a part of. + * Set either in @ref Co::await_suspend or in constructor of @ref Goal. + */ + Goal* goal = nullptr; + + /** + * Is set to false when destructed to ensure we don't use a + * destructed coroutine by accident + */ + bool alive = true; + + /** + * The awaiter used by @ref final_suspend. + */ + struct final_awaiter { + bool await_ready() noexcept { return false; }; + /** + * Here we execute our continuation, by passing it back to the caller. + * C++ compiler will create code that takes that and executes it promptly. + * `h` is the handle for the coroutine that is finishing execution, + * thus it must be destroyed. + */ + std::coroutine_handle<> await_suspend(handle_type h) noexcept; + void await_resume() noexcept { assert(false); }; + }; + + /** + * Called by compiler generated code to construct the @ref Co + * that is returned from a @ref Co-returning coroutine. + */ + Co get_return_object(); + + /** + * Called by compiler generated code before body of coroutine. + * We use this opportunity to set the @ref goal field + * and `top_co` field of @ref Goal. + */ + InitialSuspend initial_suspend() { return {}; }; + + /** + * Called on `co_return`. Creates @ref final_awaiter which + * either jumps to continuation or suspends goal. + */ + final_awaiter final_suspend() noexcept { return {}; }; + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Return) {} + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Done) {} + + /** + * When "returning" another coroutine, what happens is that + * we set it as our own continuation, thus once the final suspend + * happens, we transfer control to it. + * The original continuation we had is set as the continuation + * of the coroutine passed in. + * @ref final_suspend is called after this, and @ref final_awaiter will + * pass control off to @ref continuation. + * + * If we already have a continuation, that continuation is set as + * the continuation of the new continuation. Thus, the continuation + * passed to @ref return_value must not have a continuation set. + */ + void return_value(Co&&); + + /** + * If an exception is thrown inside a coroutine, + * we re-throw it in the context of the "resumer" of the continuation. + */ + void unhandled_exception() { throw; }; + + /** + * Allows awaiting a @ref Co. + */ + Co&& await_transform(Co&& co) { return static_cast(co); } + + /** + * Allows awaiting a @ref Suspend. + * Always suspends. + */ + std::suspend_always await_transform(Suspend) { return {}; }; + }; + + /** + * The coroutine being currently executed. + * MUST be updated when switching the coroutine being executed. + * This is used both for memory management and to resume the last + * coroutine executed. + * Destroying this should destroy all coroutines created for this goal. + */ + std::optional top_co; + + /** + * The entry point for the goal + */ + virtual Co init() = 0; + + /** + * Wrapper around @ref init since virtual functions + * can't be used in constructors. + */ + inline Co init_wrapper(); + + /** + * Signals that the goal is done. + * `co_return` the result. If you're not inside a coroutine, you can ignore + * the return value safely. + */ + Done amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } + /** * Project a `BuildResult` with just the information that pertains * to the given request. @@ -124,15 +379,20 @@ public: std::optional ex; Goal(Worker & worker, DerivedPath path) - : worker(worker) - { } + : worker(worker), top_co(init_wrapper()) + { + // top_co shouldn't have a goal already, should be nullptr. + assert(!top_co->handle.promise().goal); + // we set it such that top_co can pass it down to its subcoroutines. + top_co->handle.promise().goal = this; + } virtual ~Goal() { trace("goal destroyed"); } - virtual void work() = 0; + void work(); void addWaitee(GoalPtr waitee); @@ -164,10 +424,6 @@ public: virtual std::string key() = 0; - void amDone(ExitCode result, std::optional ex = {}); - - virtual void cleanup() { } - /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory @@ -178,3 +434,12 @@ public: void addToWeakGoals(WeakGoals & goals, GoalPtr p); } + +template +struct std::coroutine_traits { + using promise_type = nix::Goal::promise_type; +}; + +nix::Goal::Co nix::Goal::init_wrapper() { + co_return init(); +} diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 0be3d1e8d98..7deeb47487d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,7 @@ #include "nar-info.hh" #include "finally.hh" #include "signals.hh" +#include namespace nix { @@ -12,7 +13,6 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); @@ -25,7 +25,7 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done( +Goal::Done PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional errorMsg) @@ -35,17 +35,11 @@ void PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - amDone(result); + return amDone(result); } -void PathSubstitutionGoal::work() -{ - (this->*state)(); -} - - -void PathSubstitutionGoal::init() +Goal::Co PathSubstitutionGoal::init() { trace("init"); @@ -53,152 +47,135 @@ void PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - done(ecSuccess, BuildResult::AlreadyValid); - return; + co_return done(ecSuccess, BuildResult::AlreadyValid); } if (settings.readOnlyMode) throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + bool substituterFailed = false; -void PathSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); + for (auto sub : subs) { + trace("trying next substituter"); - cleanup(); + cleanup(); - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - done( - substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; - return; - } + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr info; - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - ContentAddressWithReferences::withoutRefs(*ca)); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; + if (ca) { + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + continue; } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; + + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + } catch (InvalidPath &) { + continue; + } catch (SubstituterDisabled & e) { + if (settings.tryFallback) continue; + else throw e; + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + continue; + } else throw e; } - throw; - } - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + continue; + } } - } - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; - worker.updateProgress(); + worker.updateProgress(); - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) - { - warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", - worker.store.printStorePath(storePath), sub->getUri()); - tryNext(); - return; + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + { + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + worker.store.printStorePath(storePath), sub->getUri()); + continue; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); + + if (!waitees.empty()) co_await Suspend{}; + + // FIXME: consider returning boolean instead of passing in reference + bool out = false; // is mutated by tryToRun + co_await tryToRun(subPath ? *subPath : storePath, sub, info, out); + substituterFailed = substituterFailed || out; } - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + /* None left. Terminate this goal and let someone else deal + with it. */ - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &PathSubstitutionGoal::referencesValid; + worker.failedSubstitutions++; + worker.updateProgress(); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } -void PathSubstitutionGoal::referencesValid() +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) { trace("all references realised"); if (nrFailed > 0) { - done( + co_return done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, BuildResult::DependencyFailed, fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); - return; } for (auto & i : info->references) if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); -} - + co_await Suspend{}; -void PathSubstitutionGoal::tryToRun() -{ trace("trying to run"); /* Make sure that we are allowed to start a substitution. Note that even @@ -206,10 +183,10 @@ void PathSubstitutionGoal::tryToRun() prevents infinite waiting. */ if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { worker.waitForBuildSlot(shared_from_this()); - return; + co_await Suspend{}; } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); #ifndef _WIN32 @@ -218,9 +195,9 @@ void PathSubstitutionGoal::tryToRun() outPipe.createAsyncPipe(worker.ioport.get()); #endif - promise = std::promise(); + auto promise = std::promise(); - thr = std::thread([this]() { + thr = std::thread([this, &promise, &subPath, &sub]() { try { ReceiveInterrupts receiveInterrupts; @@ -231,7 +208,7 @@ void PathSubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -247,12 +224,8 @@ void PathSubstitutionGoal::tryToRun() #endif }, true, false); - state = &PathSubstitutionGoal::finished; -} - + co_await Suspend{}; -void PathSubstitutionGoal::finished() -{ trace("substitute finished"); thr.join(); @@ -274,10 +247,7 @@ void PathSubstitutionGoal::finished() substituterFailed = true; } - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; + co_return Return{}; } worker.markContentsGood(storePath); @@ -295,23 +265,19 @@ void PathSubstitutionGoal::finished() worker.doneDownloadSize += fileSize; } + assert(maintainExpectedNar); worker.doneNarSize += maintainExpectedNar->delta; maintainExpectedNar.reset(); worker.updateProgress(); - done(ecSuccess, BuildResult::Substituted); -} - - -void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data) -{ + co_return done(ecSuccess, BuildResult::Substituted); } void PathSubstitutionGoal::handleEOF(Descriptor fd) { - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1a051fc1fd5..86e4f542382 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,14 +1,16 @@ #pragma once ///@file +#include "worker.hh" #include "store-api.hh" #include "goal.hh" #include "muxable-pipe.hh" +#include +#include +#include namespace nix { -class Worker; - struct PathSubstitutionGoal : public Goal { /** @@ -17,30 +19,9 @@ struct PathSubstitutionGoal : public Goal StorePath storePath; /** - * The path the substituter refers to the path as. This will be - * different when the stores have different names. - */ - std::optional subPath; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - - /** - * Path info returned by the substituter's query info operation. + * Whether to try to repair a valid path. */ - std::shared_ptr info; + RepairFlag repair; /** * Pipe for the substituter's standard output. @@ -52,31 +33,15 @@ struct PathSubstitutionGoal : public Goal */ std::thread thr; - std::promise promise; - - /** - * Whether to try to repair a valid path. - */ - RepairFlag repair; - - /** - * Location where we're downloading the substitute. Differs from - * storePath when doing a repair. - */ - Path destPath; - std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (PathSubstitutionGoal::*GoalState)(); - GoalState state; - /** * Content address for recomputing store path */ std::optional ca; - void done( + Done done( ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); @@ -96,22 +61,18 @@ public: return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - void work() override; - /** * The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); + Co init() override; + Co gotInfo(); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co finished(); /** * Callback used by the worker to write to the log. */ - void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleChildOutput(Descriptor fd, std::string_view data) override {}; void handleEOF(Descriptor fd) override; /* Called by destructor, can't be overridden */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 8a5d6de725f..7fc41b1214a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -337,31 +337,27 @@ void Worker::run(const Goals & _topGoals) /* Wait for input. */ if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); - else { - if (awake.empty() && 0U == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error( - R"( - Unable to start any build; - either increase '--max-jobs' or enable remote builds. - - For more information run 'man nix.conf' and search for '/machines'. - )" - ); - else - throw Error( - R"( - Unable to start any build; - remote machines may not have all required system features. - - For more information run 'man nix.conf' and search for '/machines'. - )" - ); + else if (awake.empty() && 0U == settings.maxBuildJobs) { + if (getMachines().empty()) + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + else + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. - } - assert(!awake.empty()); - } + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + + } else assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index c3a65e34b57..f968bbc5b7f 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -177,7 +177,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() +Goal::Co LocalDerivationGoal::tryLocalBuild() { #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); @@ -185,10 +185,10 @@ void LocalDerivationGoal::tryLocalBuild() unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { - state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); } assert(derivationType); @@ -242,7 +242,8 @@ void LocalDerivationGoal::tryLocalBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryLocalBuild(); } } @@ -257,15 +258,13 @@ void LocalDerivationGoal::tryLocalBuild() outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, std::move(e)); - return; + co_return done(BuildResult::InputRejected, {}, std::move(e)); } - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - started(); + co_await Suspend{}; + // after EOF on child + co_return buildDone(); } static void chmod_(const Path & path, mode_t mode) diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index 4bcf5c9d457..bf25cf2a60b 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -198,7 +198,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - void tryLocalBuild() override; + Goal::Co tryLocalBuild() override; /** * Start building a derivation. From 808082ea031126ac8738897c43f26240875c02a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Jul 2024 13:13:11 -0400 Subject: [PATCH 70/70] Ensure we can construct remote store configs in isolation Progress towards #10766 I thought that #10768 achieved, but when I went to use this stuff (in Hydra), turns out it did not. (Those `using FooConfig;` lines were not working --- they are so finicky!) This PR gets the job done, and adds some trivial unit tests to make sure I did what I intended. I had to add add a header to expose `SSHStoreConfig`, after which the preexisting `ssh-store-config.*` were very confusingly named files, so I renamed them to `common-ssh-store-config.hh` to match the type defined therein. --- maintainers/flake-module.nix | 2 +- ...e-config.cc => common-ssh-store-config.cc} | 2 +- ...e-config.hh => common-ssh-store-config.hh} | 0 src/libstore/legacy-ssh-store.cc | 13 +++++-- src/libstore/legacy-ssh-store.hh | 7 +++- src/libstore/meson.build | 5 +-- src/libstore/ssh-store.cc | 35 +++++++++---------- src/libstore/ssh-store.hh | 28 +++++++++++++++ tests/unit/libstore/legacy-ssh-store.cc | 26 ++++++++++++++ tests/unit/libstore/meson.build | 2 ++ tests/unit/libstore/ssh-store.cc | 26 ++++++++++++++ 11 files changed, 120 insertions(+), 26 deletions(-) rename src/libstore/{ssh-store-config.cc => common-ssh-store-config.cc} (96%) rename src/libstore/{ssh-store-config.hh => common-ssh-store-config.hh} (100%) create mode 100644 src/libstore/ssh-store.hh create mode 100644 tests/unit/libstore/legacy-ssh-store.cc create mode 100644 tests/unit/libstore/ssh-store.cc diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 46b3e136324..66bfcf609b1 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -143,6 +143,7 @@ ''^src/libstore/common-protocol-impl\.hh$'' ''^src/libstore/common-protocol\.cc$'' ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' ''^src/libstore/content-address\.cc$'' ''^src/libstore/content-address\.hh$'' ''^src/libstore/daemon\.cc$'' @@ -215,7 +216,6 @@ ''^src/libstore/serve-protocol\.hh$'' ''^src/libstore/sqlite\.cc$'' ''^src/libstore/sqlite\.hh$'' - ''^src/libstore/ssh-store-config\.hh$'' ''^src/libstore/ssh-store\.cc$'' ''^src/libstore/ssh\.cc$'' ''^src/libstore/ssh\.hh$'' diff --git a/src/libstore/ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc similarity index 96% rename from src/libstore/ssh-store-config.cc rename to src/libstore/common-ssh-store-config.cc index e81a9487449..05332b9bb5c 100644 --- a/src/libstore/ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -1,6 +1,6 @@ #include -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "ssh.hh" namespace nix { diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh similarity index 100% rename from src/libstore/ssh-store-config.hh rename to src/libstore/common-ssh-store-config.hh diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 9664b126edc..eac360a4f7a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,5 +1,5 @@ #include "legacy-ssh-store.hh" -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -15,6 +15,15 @@ namespace nix { +LegacySSHStoreConfig::LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) +{ +} + std::string LegacySSHStoreConfig::doc() { return @@ -35,7 +44,7 @@ LegacySSHStore::LegacySSHStore( const Params & params) : StoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(params) + , LegacySSHStoreConfig(scheme, host, params) , Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index db49188ec9d..f2665189828 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "store-api.hh" #include "ssh.hh" #include "callback.hh" @@ -13,6 +13,11 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; + LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 5324b2a1fd7..297bf71870c 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -162,6 +162,7 @@ sources = files( 'builtins/fetchurl.cc', 'builtins/unpack-channel.cc', 'common-protocol.cc', + 'common-ssh-store-config.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', @@ -206,7 +207,6 @@ sources = files( 'serve-protocol-connection.cc', 'serve-protocol.cc', 'sqlite.cc', - 'ssh-store-config.cc', 'ssh-store.cc', 'ssh.cc', 'store-api.cc', @@ -233,6 +233,7 @@ headers = [config_h] + files( 'builtins/buildenv.hh', 'common-protocol-impl.hh', 'common-protocol.hh', + 'common-ssh-store-config.hh', 'content-address.hh', 'daemon.hh', 'derivations.hh', @@ -272,11 +273,11 @@ headers = [config_h] + files( 'remote-store.hh', 's3-binary-cache-store.hh', 's3.hh', + 'ssh-store.hh', 'serve-protocol-connection.hh', 'serve-protocol-impl.hh', 'serve-protocol.hh', 'sqlite.hh', - 'ssh-store-config.hh', 'ssh.hh', 'store-api.hh', 'store-cast.hh', diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 7ad934b738b..b21c22d7e08 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,7 +1,5 @@ -#include "ssh-store-config.hh" -#include "store-api.hh" +#include "ssh-store.hh" #include "local-fs-store.hh" -#include "remote-store.hh" #include "remote-store-connection.hh" #include "source-accessor.hh" #include "archive.hh" @@ -12,23 +10,22 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) { - using RemoteStoreConfig::RemoteStoreConfig; - using CommonSSHStoreConfig::CommonSSHStoreConfig; - - const Setting remoteProgram{this, {"nix-daemon"}, "remote-program", - "Path to the `nix-daemon` executable on the remote machine."}; - - const std::string name() override { return "Experimental SSH Store"; } +} - std::string doc() override - { - return - #include "ssh-store.md" - ; - } -}; +std::string SSHStoreConfig::doc() +{ + return + #include "ssh-store.md" + ; +} class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { @@ -41,7 +38,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore : StoreConfig(params) , RemoteStoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) + , SSHStoreConfig(scheme, host, params) , Store(params) , RemoteStore(params) , master(createSSHMaster( diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh new file mode 100644 index 00000000000..6ef2219a2e9 --- /dev/null +++ b/src/libstore/ssh-store.hh @@ -0,0 +1,28 @@ +#pragma once +///@file + +#include "common-ssh-store-config.hh" +#include "store-api.hh" +#include "remote-store.hh" + +namespace nix { + +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + + const Setting remoteProgram{ + this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + + const std::string name() override + { + return "Experimental SSH Store"; + } + + std::string doc() override; +}; + +} diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc new file mode 100644 index 00000000000..eb31a240804 --- /dev/null +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "legacy-ssh-store.hh" + +namespace nix { + +TEST(LegacySSHStore, constructConfig) +{ + LegacySSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +} diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 90e7d3047c7..41b2fb0abc2 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -58,6 +58,7 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'legacy-ssh-store.cc', 'machines.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', @@ -67,6 +68,7 @@ sources = files( 'path.cc', 'references.cc', 'serve-protocol.cc', + 'ssh-store.cc', 'store-reference.cc', 'worker-protocol.cc', ) diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc new file mode 100644 index 00000000000..7010ad157dd --- /dev/null +++ b/tests/unit/libstore/ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "ssh-store.hh" + +namespace nix { + +TEST(SSHStore, constructConfig) +{ + SSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +}