Merge remote-tracking branch 'origin/master' into lazy-trees

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,24 +10,6 @@
 
 <!-- Large change: Provide instructions to reviewers how to read the diff. -->
 
-# Checklist for maintainers
-
-<!-- Contributors: please leave this as is -->
-
-Maintainers: tick if completed or explain if not relevant
-
- - [ ] agreed on idea
- - [ ] agreed on implementation strategy
- - [ ] tests, as appropriate
-   - functional tests - `tests/**.sh`
-   - unit tests - `src/*/tests`
-   - integration tests - `tests/nixos/*`
- - [ ] documentation in the manual
- - [ ] documentation in the internal API docs
- - [ ] code and comments are self-explanatory
- - [ ] commit message explains why the change was made
- - [ ] new feature or incompatible change: updated release notes
-
 # Priorities
 
 Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc).

.github/workflows/backport.yml

@@ -14,14 +14,14 @@ jobs:
     if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           # required to find all branches
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.3.1
+        uses: zeebe-io/backport-action@v1.4.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -17,10 +17,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v22
+    - uses: cachix/install-nix-action@v23
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -58,11 +58,11 @@ jobs:
     outputs:
       installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v22
+    - uses: cachix/install-nix-action@v23
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
@@ -82,9 +82,9 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v22
+    - uses: cachix/install-nix-action@v23
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -108,10 +108,10 @@ jobs:
       needs.check_secrets.outputs.docker == 'true'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v22
+    - uses: cachix/install-nix-action@v23
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
@@ -127,7 +127,7 @@ jobs:
     - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
     - run: docker tag nix:$NIX_VERSION nixos/nix:master
     - name: Login to Docker Hub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/hydra_status.yml

@@ -13,7 +13,7 @@ jobs:
     if: github.repository_owner == 'NixOS'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - run: bash scripts/check-hydra-status.sh

.gitignore

@@ -139,3 +139,6 @@ nix-rust/target
 result
 
 .vscode/
+
+# clangd and possibly more
+.cache/

CONTRIBUTING.md

@@ -30,8 +30,8 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 
 2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue.
 
-   Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly.
-   Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review.
+   Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly.
+   Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review.
 
 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests.
 
@@ -40,14 +40,27 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 4. Make your changes!
 
 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes.
-   * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes.
-   * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request).
    * Link related issues in your pull request to inform interested parties and future contributors about your change.
+   * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request).
      If your pull request closes one or multiple issues, note that in the description using `Closes: #<number>`, as it will then happen automatically when your change is merged.
+   * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes.
 
 6. Do not expect your pull request to be reviewed immediately.
    Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work.
 
+   Following this checklist will make the process smoother for everyone:
+
+   - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue
+   - [ ] Tests, as appropriate:
+     - Functional tests – [`tests/**.sh`](./tests)
+     - Unit tests – [`src/*/tests`](./src/)
+     - Integration tests – [`tests/nixos/*`](./tests/nixos)
+   - [ ] User documentation in the [manual](..doc/manual/src)
+   - [ ] API documentation in header files
+   - [ ] Code and comments are self-explanatory
+   - [ ] Commit message explains **why** the change was made
+   - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md)
+
 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams).
 
 ## Making changes to the Nix manual

doc/manual/generate-manpage.nix

@@ -98,7 +98,7 @@ let
                 (option ? labels)
                 (concatStringsSep " " (map (s: "*${s}*") option.labels));
             in trim ''
-              - `--${name}` ${shortName} ${labels}
+              - <span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}
 
                 ${option.description}
             '';

doc/manual/src/SUMMARY.md.in

@@ -100,6 +100,7 @@
   - [File System Object](architecture/file-system-object.md)
 - [Protocols](protocols/protocols.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
+  - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
 - [Contributing](contributing/contributing.md)
   - [Hacking](contributing/hacking.md)

doc/manual/src/advanced-topics/advanced-topics.md

@@ -1 +1 @@
-
+This section lists advanced topics related to builds and builds performance

doc/manual/src/command-ref/nix-env/install.md

@@ -30,7 +30,7 @@ a number of possible ways:
     derivation with the highest *priority* is used. A derivation can
     define a priority by declaring the `meta.priority` attribute. This
     attribute should be a number, with a higher value denoting a lower
-    priority. The default priority is `0`.
+    priority. The default priority is `5`.
 
     If there are multiple matching derivations with the same priority,
     then the derivation with the highest version will be installed.

doc/manual/src/command-ref/nix-prefetch-url.md

@@ -31,15 +31,18 @@ store already contains a file with the same hash and base name.
 Otherwise, the file is downloaded, and an error is signaled if the
 actual hash of the file does not match the specified hash.
 
-This command prints the hash on standard output. Additionally, if the
-option `--print-path` is used, the path of the downloaded file in the
-Nix store is also printed.
+This command prints the hash on standard output.
+The hash is printed using base-32 unless `--type md5` is specified,
+in which case it's printed using base-16.
+Additionally, if the option `--print-path` is used,
+the path of the downloaded file in the Nix store is also printed.
 
 # Options
 
   - `--type` *hashAlgo*\
-    Use the specified cryptographic hash algorithm, which can be one of
-    `md5`, `sha1`, `sha256`, and `sha512`.
+    Use the specified cryptographic hash algorithm,
+    which can be one of `md5`, `sha1`, `sha256`, and `sha512`.
+    The default is `sha256`.
 
   - `--print-path`\
     Print the store path of the downloaded file on standard output.

doc/manual/src/command-ref/nix-store/query.md

@@ -5,8 +5,8 @@
 # Synopsis
 
 `nix-store` {`--query` | `-q`}
-  {`--outputs` | `--requisites` | `-R` | `--references` |
-  `--referrers` | `--referrers-closure` | `--deriver` | `-d` |
+  {`--outputs` | `--requisites` | `-R` | `--references` | `--referrers` |
+  `--referrers-closure` | `--deriver` | `-d` | `--valid-derivers` |
   `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` |
   `--size` | `--roots`}
   [`--use-output`] [`-u`] [`--force-realise`] [`-f`]
@@ -82,13 +82,21 @@ symlink.
     in the Nix store that are dependent on *paths*.
 
   - `--deriver`; `-d`\
-    Prints the [deriver] of the store paths *paths*. If
+    Prints the [deriver] that was used to build the store paths *paths*. If
     the path has no deriver (e.g., if it is a source file), or if the
     deriver is not known (e.g., in the case of a binary-only
     deployment), the string `unknown-deriver` is printed.
+    The returned deriver is not guaranteed to exist in the local store, for
+    example when *paths* were substituted from a binary cache.
+    Use `--valid-derivers` instead to obtain valid paths only.
 
     [deriver]: ../../glossary.md#gloss-deriver
 
+  - `--valid-derivers`\
+    Prints a set of derivation files (`.drv`) which are supposed produce
+    said paths when realized. Might print nothing, for example for source paths
+    or paths subsituted from a binary cache.
+
   - `--graph`\
     Prints the references graph of the store paths *paths* in the format
     of the `dot` tool of AT\&T's [Graphviz

doc/manual/src/command-ref/nix-store/realise.md

@@ -1,40 +1,42 @@
 # Name
 
-`nix-store --realise` - realise specified store paths
+`nix-store --realise` - build or fetch store objects
 
 # Synopsis
 
 `nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`]
 
 # Description
 
-The operation `--realise` essentially “builds” the specified store
-paths. Realisation is a somewhat overloaded term:
-
-  - If the store path is a *derivation*, realisation ensures that the
-    output paths of the derivation are [valid] (i.e.,
-    the output path and its closure exist in the file system). This
-    can be done in several ways. First, it is possible that the
-    outputs are already valid, in which case we are done
-    immediately. Otherwise, there may be [substitutes]
-    that produce the outputs (e.g., by downloading them). Finally, the
-    outputs can be produced by running the build task described
-    by the derivation.
-
-  - If the store path is not a derivation, realisation ensures that the
-    specified path is valid (i.e., it and its closure exist in the file
-    system). If the path is already valid, we are done immediately.
-    Otherwise, the path and any missing paths in its closure may be
-    produced through substitutes. If there are no (successful)
-    substitutes, realisation fails.
 
+Each of *paths* is processed as follows:
+
+- If the path leads to a [store derivation]:
+  1. If it is not [valid], substitute the store derivation file itself.
+  2. Realise its [output paths]:
+    - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure].
+      - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database].
+    - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. <!-- TODO: Link to build process page #8888 -->
+- Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters].
+
+If no substitutes are available and no store derivation is given, realisation fails.
+
+[store paths]: @docroot@/glossary.md#gloss-store-path
 [valid]: @docroot@/glossary.md#gloss-validity
-[substitutes]: @docroot@/glossary.md#gloss-substitute
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
+[output paths]: @docroot@/glossary.md#gloss-output-path
+[store objects]: @docroot@/glossary.md#gloss-store-object
+[closure]: @docroot@/glossary.md#gloss-closure
+[substituters]: @docroot@/command-ref/conf-file.md#conf-substituters
+[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations
+[Nix database]: @docroot@/glossary.md#gloss-nix-database
+
+The resulting paths are printed on standard output.
+For non-derivation arguments, the argument itself is printed.
 
-The output path of each derivation is printed on standard output. (For
-non-derivations argument, the argument itself is printed.)
+{{#include ../status-build-failure.md}}
 
-The following flags are available:
+# Options
 
   - `--dry-run`\
     Print on standard error a description of what packages would be
@@ -54,8 +56,6 @@ The following flags are available:
     previous build, the new output path is left in
     `/nix/store/name.check.`
 
-{{#include ../status-build-failure.md}}
-
 {{#include ./opt-common.md}}
 
 {{#include ../opt-common.md}}
@@ -67,8 +67,6 @@ The following flags are available:
 This operation is typically used to build [store derivation]s produced by
 [`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md):
 
-[store derivation]: @docroot@/glossary.md#gloss-store-derivation
-
 ```console
 $ nix-store --realise $(nix-instantiate ./test.nix)
 /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1