diff --git a/doc/manual/rl-next/shallow-git-fetching-by-default.md b/doc/manual/rl-next/shallow-git-fetching-by-default.md new file mode 100644 index 00000000000..4d044f881d3 --- /dev/null +++ b/doc/manual/rl-next/shallow-git-fetching-by-default.md @@ -0,0 +1,12 @@ +--- +synopsis: "`fetchTree` now fetches git repositories shallowly by default" +prs: 10028 +--- + +`builtins.fetchTree` now clones git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + +Previously, the default behavior was to clone the full history of a specific tag or branch (eg. `ref`) and only afterwards extract the files of one specific revision. + +From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + +The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 9e73524a632..4c4e2bd6f5e 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -243,6 +243,7 @@ sub downloadFile { " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . "}\n"); diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b2637..b4e808d8e94 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fa462dc3348..a99a715770b 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -137,6 +137,11 @@ static void fetchTree( attrs.emplace("exportIgnore", Explicit{true}); } + // fetchTree should fetch git repos with shallow = true by default + if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) { + attrs.emplace("shallow", Explicit{true}); + } + if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) state.error( @@ -320,6 +325,8 @@ static RegisterPrimOp primop_fetchTree({ - `ref` (String, optional) + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. Default: `"HEAD"` @@ -333,8 +340,9 @@ static RegisterPrimOp primop_fetchTree({ - `shallow` (Bool, optional) Make a shallow clone when fetching the Git tree. + When this is enabled, the options `ref` and `allRefs` have no effect anymore. - Default: `false` + Default: `true` - `submodules` (Bool, optional) @@ -344,8 +352,11 @@ static RegisterPrimOp primop_fetchTree({ - `allRefs` (Bool, optional) - If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. - Otherwise, only the latest commit is fetched if it is not already cached. + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. + (Without setting this option, only `rev`s from the specified `ref` are supported). Default: `false` @@ -599,6 +610,8 @@ static RegisterPrimOp primop_fetchGit({ [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References + This option has no effect once `shallow` cloning is enabled. + By default, the `ref` value is prefixed with `refs/heads/`. As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. @@ -616,13 +629,15 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) Make a shallow clone when fetching the Git tree. - + When this is enabled, the options `ref` and `allRefs` have no effect anymore. - `allRefs` - Whether to fetch all references of the repository. - With this argument being true, it's possible to load a `rev` from *any* `ref` + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. (by default only `rev`s from the specified `ref` are supported). + This option has no effect once `shallow` cloning is enabled. + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index d4c9d65337b..2f2ed813848 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -50,21 +50,14 @@ bool Hash::operator == (const Hash & h2) const } -bool Hash::operator != (const Hash & h2) const +std::strong_ordering Hash::operator <=> (const Hash & h) const { - return !(*this == h2); -} - - -bool Hash::operator < (const Hash & h) const -{ - if (hashSize < h.hashSize) return true; - if (hashSize > h.hashSize) return false; + if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; + if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { - if (hash[i] < h.hash[i]) return true; - if (hash[i] > h.hash[i]) return false; + if (auto cmp = hash[i] <=> h.hash[i]; cmp != 0) return cmp; } - return false; + return std::strong_ordering::equivalent; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index e14aae43c08..ef96d08c978 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -86,19 +86,14 @@ private: public: /** - * Check whether two hash are equal. + * Check whether two hashes are equal. */ bool operator == (const Hash & h2) const; /** - * Check whether two hash are not equal. + * Compare how two hashes are ordered. */ - bool operator != (const Hash & h2) const; - - /** - * For sorting. - */ - bool operator < (const Hash & h) const; + std::strong_ordering operator <=> (const Hash & h2) const; /** * Returns the length of a base-16 representation of this hash. diff --git a/src/libutil/url.cc b/src/libutil/url.cc index c6561441d93..f4178f87fd9 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -171,16 +171,16 @@ std::string fixGitURL(const std::string & url) std::regex scpRegex("([^/]*)@(.*):(.*)"); if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); - else { - if (url.find("://") == std::string::npos) { - return (ParsedURL { - .scheme = "file", - .authority = "", - .path = url - }).to_string(); - } else - return url; + if (hasPrefix(url, "file:")) + return url; + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); } + return url; } // https://www.rfc-editor.org/rfc/rfc3986#section-3.1 diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh old mode 100644 new mode 100755 index 90c7abd3c69..0f8df1b91dc --- a/tests/functional/flakes/eval-cache.sh +++ b/tests/functional/flakes/eval-cache.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireGit diff --git a/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix b/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix new file mode 100644 index 00000000000..f635df1f879 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix @@ -0,0 +1,45 @@ +{ + description = "fetchTree fetches git repos shallowly by default"; + script = '' + # purge nix git cache to make sure we start with a clean slate + client.succeed("rm -rf ~/.cache/nix") + + # add two commits to the repo: + # - one with a large file (2M) + # - another one making the file small again + client.succeed(f""" + dd if=/dev/urandom of={repo.path}/thailand bs=1M count=2 \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + && echo 'ThaigerSprint' > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit2' \ + && {repo.git} push origin main + """) + + # memoize the revision + commit2_rev = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # construct the fetcher call + fetchGit_expr = f""" + builtins.fetchTree {{ + type = "git"; + url = "{repo.remote}"; + rev = "{commit2_rev}"; + }} + """ + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).outPath' + """) + + # check that the size of ~/.cache/nix is less than 1M + cache_size = client.succeed(""" + du -s ~/.cache/nix + """).strip().split()[0] + assert int(cache_size) < 1024, f"cache size is {cache_size}K which is larger than 1M" + ''; +}