From 4549773edf119c40f3285fd46b662a5ceec2e456 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 5 May 2024 11:33:00 +0200 Subject: [PATCH 01/12] nodePackages: enable overriding src and name attrs Signed-off-by: Sefa Eyeoglu --- pkgs/development/node-packages/node-env.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkgs/development/node-packages/node-env.nix b/pkgs/development/node-packages/node-env.nix index bc1e36628ac8a..5400fc63588ee 100644 --- a/pkgs/development/node-packages/node-env.nix +++ b/pkgs/development/node-packages/node-env.nix @@ -109,7 +109,7 @@ let ); # Recursively composes the dependencies of a package - composePackage = { name, packageName, src, dependencies ? [], ... }@args: + composePackage = { name, packageName, src, dependencies ? [], ... }: builtins.addErrorContext "while evaluating node package '${packageName}'" '' installPackage "${packageName}" "${src}" ${includeDependencies { inherit dependencies; }} @@ -194,7 +194,7 @@ let # dependencies in the package.json file to the versions that are actually # being used. - pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: + pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }: '' if [ -d "${packageName}" ] then @@ -496,7 +496,7 @@ let let extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" "meta" ]; in - stdenv.mkDerivation ({ + stdenv.mkDerivation (finalAttrs: { name = "${name}${if version == null then "" else "-${version}"}"; buildInputs = [ tarWrapper python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux @@ -508,8 +508,9 @@ let inherit dontStrip; # Stripping may fail a build for some package deployments inherit dontNpmInstall preRebuild unpackPhase buildPhase; - compositionScript = composePackage args; - pinpointDependenciesScript = pinpointDependenciesOfPackage args; + # TODO: enable overriding dependencies too? + compositionScript = composePackage { inherit packageName dependencies; inherit (finalAttrs) name src; }; + pinpointDependenciesScript = pinpointDependenciesOfPackage { inherit packageName dependencies production; }; passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; From 53af969d621c29357788baf5642ed28ada3b5f00 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Fri, 17 May 2024 11:09:12 +0200 Subject: [PATCH 02/12] pnpm: init at 9.1.1 Signed-off-by: Sefa Eyeoglu --- pkgs/development/tools/pnpm/default.nix | 20 ++++++++++ pkgs/development/tools/pnpm/generic.nix | 50 +++++++++++++++++++++++++ pkgs/top-level/all-packages.nix | 4 ++ 3 files changed, 74 insertions(+) create mode 100644 pkgs/development/tools/pnpm/default.nix create mode 100644 pkgs/development/tools/pnpm/generic.nix diff --git a/pkgs/development/tools/pnpm/default.nix b/pkgs/development/tools/pnpm/default.nix new file mode 100644 index 0000000000000..6fe9bc5cbe4f7 --- /dev/null +++ b/pkgs/development/tools/pnpm/default.nix @@ -0,0 +1,20 @@ +{ lib, callPackage }: +let + inherit (lib) mapAttrs' nameValuePair; + + variants = { + "8" = { + version = "8.15.8"; + hash = "sha256-aR/hdu6pqKgN8g5JdvPftEoEhBzriFY4/iomF0+B5l4="; + }; + "9" = { + version = "9.1.1"; + hash = "sha256-lVHoA9y3oYOf31QWFTqEQGDHvOATIYzoI0EFMlBKwQs="; + }; + }; + + callPnpm = variant: callPackage ./generic.nix {inherit (variant) version hash;}; + + mkPnpm = versionSuffix: variant: nameValuePair "pnpm_${versionSuffix}" (callPnpm variant); +in +mapAttrs' mkPnpm variants diff --git a/pkgs/development/tools/pnpm/generic.nix b/pkgs/development/tools/pnpm/generic.nix new file mode 100644 index 0000000000000..81fd5c2967dde --- /dev/null +++ b/pkgs/development/tools/pnpm/generic.nix @@ -0,0 +1,50 @@ +{ + lib, + stdenvNoCC, + fetchurl, + nodejs, + testers, + withNode ? true, + + version, + hash, +}: + +stdenvNoCC.mkDerivation (finalAttrs: { + pname = "pnpm"; + inherit version; + + src = fetchurl { + url = "https://registry.npmjs.org/pnpm/-/pnpm-${finalAttrs.version}.tgz"; + inherit hash; + }; + + buildInputs = lib.optionals withNode [ nodejs ]; + + installPhase = '' + runHook preInstall + + install -d $out/{bin,libexec} + cp -R . $out/libexec/pnpm + ln -s $out/libexec/pnpm/bin/pnpm.cjs $out/bin/pnpm + ln -s $out/libexec/pnpm/bin/pnpx.cjs $out/bin/pnpx + + runHook postInstall + ''; + + passthru = { + tests.version = lib.optionalAttrs withNode ( + testers.testVersion { package = finalAttrs.finalPackage; } + ); + }; + + meta = with lib; { + description = "Fast, disk space efficient package manager for JavaScript"; + homepage = "https://pnpm.io/"; + changelog = "https://github.com/pnpm/pnpm/releases/tag/v${finalAttrs.version}"; + license = licenses.mit; + maintainers = with maintainers; [ Scrumplex ]; + platforms = platforms.all; + mainProgram = "pnpm"; + }; +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 9607bb6e591ab..38b3978aabc32 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -11888,6 +11888,10 @@ with pkgs; pngquant = callPackage ../tools/graphics/pngquant { }; + inherit (callPackages ../development/tools/pnpm { }) + pnpm_8 pnpm_9; + pnpm = pnpm_9; + po4a = perlPackages.Po4a; poac = callPackage ../development/tools/poac { From 74f5ff78bfb0caae8dff6554b57dab7ca5ea9ca4 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Thu, 22 Feb 2024 23:22:44 +0100 Subject: [PATCH 03/12] pnpm.fetchDeps: init Signed-off-by: Sefa Eyeoglu --- .../javascript.section.md | 63 ++++++++++++++ .../tools/pnpm/fetch-deps/default.nix | 83 +++++++++++++++++++ .../tools/pnpm/fetch-deps/pnpm-config-hook.sh | 40 +++++++++ pkgs/development/tools/pnpm/generic.nix | 11 ++- 4 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 pkgs/development/tools/pnpm/fetch-deps/default.nix create mode 100644 pkgs/development/tools/pnpm/fetch-deps/pnpm-config-hook.sh diff --git a/doc/languages-frameworks/javascript.section.md b/doc/languages-frameworks/javascript.section.md index f706f92c6691f..5421ca0258b1b 100644 --- a/doc/languages-frameworks/javascript.section.md +++ b/doc/languages-frameworks/javascript.section.md @@ -310,6 +310,69 @@ See `node2nix` [docs](https://github.com/svanderburg/node2nix) for more info. - `node2nix` has some [bugs](https://github.com/svanderburg/node2nix/issues/238) related to working with lock files from npm distributed with `nodejs_16`. - `node2nix` does not like missing packages from npm. If you see something like `Cannot resolve version: vue-loader-v16@undefined` then you might want to try another tool. The package might have been pulled off of npm. +### pnpm {#javascript-pnpm} + +Pnpm is available as the top-level package `pnpm`. Additionally, there are variants pinned to certain major versions, like `pnpm_8` and `pnpm_9`, which support different sets of lock file versions. + +When packaging an application that includes a `pnpm-lock.yaml`, you need to fetch the pnpm store for that project using a fixed-output-derivation. The functions `pnpm_8.fetchDeps` and `pnpm_9.fetchDeps` can create this pnpm store derivation. In conjunction, the setup hooks `pnpm_8.configHook` and `pnpm_9.configHook` will prepare the build environment to install the prefetched dependencies store. Here is an example for a package that contains a `package.json` and a `pnpm-lock.yaml` files using the above `pnpm_` attributes: + +```nix +{ + stdenv, + nodejs, + # This is pinned as { pnpm = pnpm_9; } + pnpm +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "foo"; + version = "0-unstable-1980-01-01"; + + src = ...; + + nativeBuildInputs = [ + nodejs + pnpm.configHook + ]; + + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src; + hash = "..."; + }; +}) +``` + +NOTE: It is highly recommended to use a pinned version of pnpm (i.e. `pnpm_8` or `pnpm_9`), to increase future reproducibility. It might also be required to use an older version, if the package needs support for a certain lock file version. + +In case you are patching `package.json` or `pnpm-lock.yaml`, make sure to pass `finalAttrs.patches` to the function as well (i.e. `inherit (finalAttrs) patches`. + +#### Dealing with `sourceRoot` {#javascript-pnpm-sourceRoot} + +If the pnpm project is in a subdirectory, you can just define `sourceRoot` or `setSourceRoot` for `fetchDeps`. Note, that projects using `pnpm-workspace.yaml` are currently not supported, and will probably not work using this approach. +If `sourceRoot` is different between the parent derivation and `fetchDeps`, you will have to set `pnpmRoot` to effectively be the same location as it is in `fetchDeps`. + +Assuming the following directory structure, we can define `sourceRoot` and `pnpmRoot` as follows: + +``` +. +├── frontend +│   ├── ... +│   ├── package.json +│   └── pnpm-lock.yaml +└── ... +``` + +```nix + ... + pnpmDeps = pnpm.fetchDeps { + ... + sourceRoot = "${finalAttrs.src.name}/frontend"; + }; + + # by default the working directory is the extracted source + pnpmRoot = "frontend"; +``` + ### yarn2nix {#javascript-yarn2nix} #### Preparation {#javascript-yarn2nix-preparation} diff --git a/pkgs/development/tools/pnpm/fetch-deps/default.nix b/pkgs/development/tools/pnpm/fetch-deps/default.nix new file mode 100644 index 0000000000000..3a74f4cfe3894 --- /dev/null +++ b/pkgs/development/tools/pnpm/fetch-deps/default.nix @@ -0,0 +1,83 @@ +{ + stdenvNoCC, + fetchurl, + jq, + moreutils, + cacert, + makeSetupHook, + pnpm, +}: +{ + fetchDeps = + { + src, + hash ? "", + pname, + ... + }@args: + let + args' = builtins.removeAttrs args [ + "hash" + "pname" + ]; + hash' = + if hash != "" then + { outputHash = hash; } + else + { + outputHash = ""; + outputHashAlgo = "sha256"; + }; + in + stdenvNoCC.mkDerivation ( + args' + // { + name = "${pname}-pnpm-deps"; + + nativeBuildInputs = [ + jq + moreutils + pnpm + cacert + ]; + + installPhase = '' + runHook preInstall + + export HOME=$(mktemp -d) + pnpm config set store-dir $out + # Some packages produce platform dependent outputs. We do not want to cache those in the global store + pnpm config set side-effects-cache false + # As we pin pnpm versions, we don't really care about updates + pnpm config set update-notifier false + # pnpm is going to warn us about using --force + # --force allows us to fetch all dependencies including ones that aren't meant for our host platform + pnpm install --frozen-lockfile --ignore-script --force + + runHook postInstall + ''; + + fixupPhase = '' + runHook preFixup + + # Remove timestamp and sort the json files + rm -rf $out/v3/tmp + for f in $(find $out -name "*.json"); do + jq --sort-keys "del(.. | .checkedAt?)" $f | sponge $f + done + + runHook postFixup + ''; + + dontConfigure = true; + dontBuild = true; + outputHashMode = "recursive"; + } + // hash' + ); + + configHook = makeSetupHook { + name = "pnpm-config-hook"; + propagatedBuildInputs = [ pnpm ]; + } ./pnpm-config-hook.sh; +} diff --git a/pkgs/development/tools/pnpm/fetch-deps/pnpm-config-hook.sh b/pkgs/development/tools/pnpm/fetch-deps/pnpm-config-hook.sh new file mode 100644 index 0000000000000..fc0c47dbb70b6 --- /dev/null +++ b/pkgs/development/tools/pnpm/fetch-deps/pnpm-config-hook.sh @@ -0,0 +1,40 @@ +# shellcheck shell=bash + +pnpmConfigHook() { + echo "Executing pnpmConfigHook" + + if [ -n "${pnpmRoot-}" ]; then + pushd "$pnpmRoot" + fi + + if [ -z "${pnpmDeps-}" ]; then + echo "Error: 'pnpmDeps' must be set when using pnpmConfigHook." + exit 1 + fi + + echo "Configuring pnpm store" + + export HOME=$(mktemp -d) + export STORE_PATH=$(mktemp -d) + + cp -Tr "$pnpmDeps" "$STORE_PATH" + chmod -R +w "$STORE_PATH" + + pnpm config set store-dir "$STORE_PATH" + + echo "Installing dependencies" + + pnpm install --offline --frozen-lockfile --ignore-script + + echo "Patching scripts" + + patchShebangs node_modules/{*,.*} + + if [ -n "${pnpmRoot-}" ]; then + popd + fi + + echo "Finished pnpmConfigHook" +} + +postConfigureHooks+=(pnpmConfigHook) diff --git a/pkgs/development/tools/pnpm/generic.nix b/pkgs/development/tools/pnpm/generic.nix index 81fd5c2967dde..c53dfd78f673a 100644 --- a/pkgs/development/tools/pnpm/generic.nix +++ b/pkgs/development/tools/pnpm/generic.nix @@ -1,6 +1,7 @@ { lib, stdenvNoCC, + callPackages, fetchurl, nodejs, testers, @@ -8,9 +9,7 @@ version, hash, -}: - -stdenvNoCC.mkDerivation (finalAttrs: { +}: stdenvNoCC.mkDerivation (finalAttrs: { pname = "pnpm"; inherit version; @@ -32,7 +31,11 @@ stdenvNoCC.mkDerivation (finalAttrs: { runHook postInstall ''; - passthru = { + passthru = let + fetchDepsAttrs = callPackages ./fetch-deps { pnpm = finalAttrs.finalPackage; }; + in { + inherit (fetchDepsAttrs) fetchDeps configHook; + tests.version = lib.optionalAttrs withNode ( testers.testVersion { package = finalAttrs.finalPackage; } ); From 728d48285e5c17ed9764bc7067b9cece3ae81f15 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 2 Jun 2024 19:55:18 +0200 Subject: [PATCH 04/12] pnpm.fetchDeps: add serve script This script allows users to reuse the cached dependencies outside of derivations. Signed-off-by: Sefa Eyeoglu --- .../tools/pnpm/fetch-deps/default.nix | 12 ++++++-- .../tools/pnpm/fetch-deps/serve.nix | 30 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/tools/pnpm/fetch-deps/serve.nix diff --git a/pkgs/development/tools/pnpm/fetch-deps/default.nix b/pkgs/development/tools/pnpm/fetch-deps/default.nix index 3a74f4cfe3894..83972abf8b08a 100644 --- a/pkgs/development/tools/pnpm/fetch-deps/default.nix +++ b/pkgs/development/tools/pnpm/fetch-deps/default.nix @@ -1,6 +1,7 @@ { stdenvNoCC, fetchurl, + callPackage, jq, moreutils, cacert, @@ -29,7 +30,7 @@ outputHashAlgo = "sha256"; }; in - stdenvNoCC.mkDerivation ( + stdenvNoCC.mkDerivation (finalAttrs: ( args' // { name = "${pname}-pnpm-deps"; @@ -69,12 +70,19 @@ runHook postFixup ''; + passthru = { + serve = callPackage ./serve.nix { + inherit pnpm; + pnpmDeps = finalAttrs.finalPackage; + }; + }; + dontConfigure = true; dontBuild = true; outputHashMode = "recursive"; } // hash' - ); + )); configHook = makeSetupHook { name = "pnpm-config-hook"; diff --git a/pkgs/development/tools/pnpm/fetch-deps/serve.nix b/pkgs/development/tools/pnpm/fetch-deps/serve.nix new file mode 100644 index 0000000000000..a44022d841dc5 --- /dev/null +++ b/pkgs/development/tools/pnpm/fetch-deps/serve.nix @@ -0,0 +1,30 @@ +{ writeShellApplication, pnpm, pnpmDeps }: +writeShellApplication { + name = "serve-pnpm-store"; + + runtimeInputs = [ + pnpm + ]; + + text = '' + storePath=$(mktemp -d) + + clean() { + echo "Cleaning up temporary store at '$storePath'..." + + rm -rf "$storePath" + } + + echo "Copying pnpm store '${pnpmDeps}' to temporary store..." + + cp -Tr "${pnpmDeps}" "$storePath" + chmod -R +w "$storePath" + + echo "Run 'pnpm install --store-dir \"$storePath\"' to install packages from this store." + + trap clean EXIT + + pnpm server start \ + --store-dir "$storePath" + ''; +} From 444b9863a94dbc5f0040e6a615f5c0c2b15e57f2 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Fri, 17 May 2024 11:17:35 +0200 Subject: [PATCH 05/12] vesktop: use pnpm.fetchDeps Signed-off-by: Sefa Eyeoglu --- pkgs/by-name/ve/vesktop/package.nix | 84 +++-------------------------- pkgs/top-level/all-packages.nix | 2 + 2 files changed, 10 insertions(+), 76 deletions(-) diff --git a/pkgs/by-name/ve/vesktop/package.nix b/pkgs/by-name/ve/vesktop/package.nix index 9e45802d392eb..d133fe56ca12c 100644 --- a/pkgs/by-name/ve/vesktop/package.nix +++ b/pkgs/by-name/ve/vesktop/package.nix @@ -1,7 +1,6 @@ { lib, stdenv, - stdenvNoCC, fetchFromGitHub, substituteAll, makeWrapper, @@ -10,13 +9,11 @@ vencord, electron, libicns, - jq, - moreutils, - cacert, - nodePackages, pipewire, libpulseaudio, autoPatchelfHook, + pnpm, + nodejs, withTTS ? true, # Enables the use of vencord from nixpkgs instead of # letting vesktop manage it's own version @@ -33,66 +30,17 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-cZOyydwpIW9Xq716KVi1RGtSlgVnOP3w8vXDwouS70E="; }; - # NOTE: This requires pnpm 8.10.0 or newer - # https://github.com/pnpm/pnpm/pull/7214 - pnpmDeps = - assert lib.versionAtLeast nodePackages.pnpm.version "8.10.0"; - stdenvNoCC.mkDerivation { - pname = "${finalAttrs.pname}-pnpm-deps"; - inherit (finalAttrs) - src - version - patches - ELECTRON_SKIP_BINARY_DOWNLOAD - ; - - nativeBuildInputs = [ - cacert - jq - moreutils - nodePackages.pnpm - ]; - - # inspired by https://github.com/NixOS/nixpkgs/blob/763e59ffedb5c25774387bf99bc725df5df82d10/pkgs/applications/misc/pot/default.nix#L56 - # and based on https://github.com/NixOS/nixpkgs/pull/290715 - installPhase = '' - runHook preInstall - - export HOME=$(mktemp -d) - pnpm config set store-dir $out - # Some packages produce platform dependent outputs. We do not want to cache those in the global store - pnpm config set side-effects-cache false - # pnpm is going to warn us about using --force - # --force allows us to fetch all dependencies including ones that aren't meant for our host platform - pnpm install --force --frozen-lockfile --ignore-script - - ''; - - fixupPhase = '' - runHook preFixup - - # Remove timestamp and sort the json files - rm -rf $out/v3/tmp - for f in $(find $out -name "*.json"); do - sed -i -E -e 's/"checkedAt":[0-9]+,//g' $f - jq --sort-keys . $f | sponge $f - done - - runHook postFixup - ''; - - dontConfigure = true; - dontBuild = true; - outputHashMode = "recursive"; - outputHash = "sha256-PogE8uf3W5cKSCqFHMz7FOvT7ONUP4FiFWGBgtk3UC8="; - }; + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src patches; + hash = "sha256-PogE8uf3W5cKSCqFHMz7FOvT7ONUP4FiFWGBgtk3UC8="; + }; nativeBuildInputs = [ autoPatchelfHook copyDesktopItems makeWrapper - nodePackages.pnpm - nodePackages.nodejs + nodejs + pnpm.configHook ]; buildInputs = [ @@ -110,22 +58,6 @@ stdenv.mkDerivation (finalAttrs: { ELECTRON_SKIP_BINARY_DOWNLOAD = 1; - configurePhase = '' - runHook preConfigure - - export HOME=$(mktemp -d) - export STORE_PATH=$(mktemp -d) - - cp -Tr "$pnpmDeps" "$STORE_PATH" - chmod -R +w "$STORE_PATH" - - pnpm config set store-dir "$STORE_PATH" - pnpm install --frozen-lockfile --ignore-script --offline - patchShebangs node_modules/{*,.*} - - runHook postConfigure - ''; - buildPhase = '' runHook preBuild diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 38b3978aabc32..bef4d9234dcd5 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -24773,6 +24773,8 @@ with pkgs; vencord-web-extension = callPackage ../by-name/ve/vencord/package.nix { buildWebExtension = true; }; + vesktop = callPackage ../by-name/ve/vesktop/package.nix { pnpm = pnpm_8; }; + vid-stab = callPackage ../development/libraries/vid-stab { inherit (llvmPackages) openmp; }; From 1ec98f0d1080443e28730e2db2de3ddd0a5f1817 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 5 May 2024 12:05:02 +0200 Subject: [PATCH 06/12] pot: use pnpm.fetchDeps Signed-off-by: Sefa Eyeoglu --- pkgs/by-name/po/pot/package.nix | 65 ++++++++++----------------------- pkgs/top-level/all-packages.nix | 2 + 2 files changed, 21 insertions(+), 46 deletions(-) diff --git a/pkgs/by-name/po/pot/package.nix b/pkgs/by-name/po/pot/package.nix index 0547e72f48198..3f461b4b33411 100644 --- a/pkgs/by-name/po/pot/package.nix +++ b/pkgs/by-name/po/pot/package.nix @@ -1,76 +1,49 @@ { lib , stdenv -, stdenvNoCC , rustPlatform , fetchFromGitHub +, nodejs +, pnpm , wrapGAppsHook3 , cargo , rustc , cargo-tauri , pkg-config -, nodePackages , esbuild , buildGoModule -, jq -, moreutils , libayatana-appindicator , gtk3 , webkitgtk , libsoup , openssl , xdotool -, cacert }: -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "pot"; version = "2.7.9"; src = fetchFromGitHub { owner = "pot-app"; repo = "pot-desktop"; - rev = version; + rev = finalAttrs.version; hash = "sha256-Y2gFLvRNBjOGxdpIeoY1CXEip0Ht73aymWIP5wuc9kU="; }; - sourceRoot = "${src.name}/src-tauri"; + sourceRoot = "${finalAttrs.src.name}/src-tauri"; postPatch = '' substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \ --replace "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1" ''; - pnpm-deps = stdenvNoCC.mkDerivation { - pname = "${pname}-pnpm-deps"; - inherit src version; - - nativeBuildInputs = [ - jq - moreutils - nodePackages.pnpm - cacert - ]; - - installPhase = '' - export HOME=$(mktemp -d) - pnpm config set store-dir $out - # use --ignore-script and --no-optional to avoid downloading binaries - # use --frozen-lockfile to avoid checking git deps - pnpm install --frozen-lockfile --no-optional --ignore-script - - # Remove timestamp and sort the json files - rm -rf $out/v3/tmp - for f in $(find $out -name "*.json"); do - sed -i -E -e 's/"checkedAt":[0-9]+,//g' $f - jq --sort-keys . $f | sponge $f - done - ''; - - dontFixup = true; - outputHashMode = "recursive"; - outputHash = "sha256-LuY5vh642DgSa91eUcA/AT+ovDcP9tZFE2dKyicCOeQ="; + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src; + hash = "sha256-nRRUX6CH3s1cEoI80gtRmu0ovXpIwS+h1rFJo8kw60E="; }; + pnpmRoot = ".."; + cargoDeps = rustPlatform.importCargoLock { lockFile = ./Cargo.lock; outputHashes = { @@ -84,8 +57,9 @@ stdenv.mkDerivation rec { cargo rustc cargo-tauri + nodejs + pnpm.configHook wrapGAppsHook3 - nodePackages.pnpm pkg-config ]; @@ -111,13 +85,13 @@ stdenv.mkDerivation rec { }); })}"; - preBuild = '' - export HOME=$(mktemp -d) - pnpm config set store-dir ${pnpm-deps} + preConfigure = '' + # pnpm.configHook has to write to .., as our sourceRoot is set to src-tauri + # TODO: move frontend into its own drv chmod +w .. - pnpm install --offline --frozen-lockfile --no-optional --ignore-script - chmod -R +w ../node_modules - pnpm rebuild + ''; + + preBuild = '' # Use cargo-tauri from nixpkgs instead of pnpm tauri from npm cargo tauri build -b deb ''; @@ -134,5 +108,4 @@ stdenv.mkDerivation rec { license = licenses.gpl3Only; maintainers = with maintainers; [ linsui ]; }; -} - +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index bef4d9234dcd5..7aa67527f4b4d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2913,6 +2913,8 @@ with pkgs; portfolio-filemanager = callPackage ../applications/file-managers/portfolio-filemanager { }; + pot = callPackage ../by-name/po/pot/package.nix { pnpm = pnpm_8; }; + potreeconverter = callPackage ../applications/graphics/potreeconverter { }; ranger = callPackage ../applications/file-managers/ranger { }; From 3b2e68d242a97d9cc5488b80773eb679ffb06dce Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 5 May 2024 23:19:28 +0200 Subject: [PATCH 07/12] vikunja: use pnpm.fetchDeps Signed-off-by: Sefa Eyeoglu --- pkgs/by-name/vi/vikunja/package.nix | 70 +++++------------------------ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 12 insertions(+), 60 deletions(-) diff --git a/pkgs/by-name/vi/vikunja/package.nix b/pkgs/by-name/vi/vikunja/package.nix index cc9891f3ccd15..26640e2ed09d8 100644 --- a/pkgs/by-name/vi/vikunja/package.nix +++ b/pkgs/by-name/vi/vikunja/package.nix @@ -1,4 +1,4 @@ -{ lib, fetchFromGitHub, stdenv, stdenvNoCC, nodePackages, buildGoModule, jq, mage, writeShellScriptBin, nixosTests, buildNpmPackage, moreutils, cacert }: +{ lib, fetchFromGitHub, stdenv, nodejs, pnpm, buildGoModule, mage, writeShellScriptBin, nixosTests }: let version = "0.23.0"; @@ -13,72 +13,20 @@ let pname = "vikunja-frontend"; inherit version src; - postPatch = '' - cd frontend - ''; - - pnpmDeps = stdenvNoCC.mkDerivation { - pname = "${finalAttrs.pname}-pnpm-deps"; - inherit (finalAttrs) src version; - - nativeBuildInputs = [ - jq - nodePackages.pnpm - moreutils - cacert - ]; - - pnpmPatch = builtins.toJSON { - pnpm.supportedArchitectures = { - os = [ "linux" ]; - cpu = [ "x64" "arm64" ]; - }; - }; - - postPatch = '' - cd frontend - mv package.json package.json.orig - jq --raw-output ". * $pnpmPatch" package.json.orig > package.json - ''; + sourceRoot = "${finalAttrs.src.name}/frontend"; - # https://github.com/NixOS/nixpkgs/blob/763e59ffedb5c25774387bf99bc725df5df82d10/pkgs/applications/misc/pot/default.nix#L56 - installPhase = '' - export HOME=$(mktemp -d) - - pnpm config set store-dir $out - pnpm install --frozen-lockfile --ignore-script - - rm -rf $out/v3/tmp - for f in $(find $out -name "*.json"); do - sed -i -E -e 's/"checkedAt":[0-9]+,//g' $f - jq --sort-keys . $f | sponge $f - done - ''; - - dontBuild = true; - dontFixup = true; - outputHashMode = "recursive"; - outputHash = { - x86_64-linux = "sha256-ybAkXe2/VhGZhr59ZQOcQ+SI2a204e8uPjyE40xUVwU="; - aarch64-linux = "sha256-2iURs6JtI/b2+CnLwhog1X5hSFFO6OmmgFRuTbMjH+k="; - }.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}"); + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src sourceRoot; + hash = "sha256-awQgOLkb46v2aWfw6yv+zGPoOnczalkzg02tBgMTyMY="; }; nativeBuildInputs = [ - nodePackages.pnpm - nodePackages.nodejs + nodejs + pnpm.configHook ]; doCheck = true; - preBuild = '' - export HOME=$(mktemp -d) - - pnpm config set store-dir ${finalAttrs.pnpmDeps} - pnpm install --offline --frozen-lockfile --ignore-script - patchShebangs node_modules/{*,.*} - ''; - postBuild = '' pnpm run build ''; @@ -121,8 +69,10 @@ buildGoModule { vendorHash = "sha256-d4AeQEAtPqMDe5a5aKhCe3i3pDXAMZJkJXxfcAFTx7A="; + inherit frontend; + prePatch = '' - cp -r ${frontend} frontend/dist + cp -r $frontend frontend/dist ''; postConfigure = '' diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7aa67527f4b4d..82ba984341b31 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -14077,6 +14077,8 @@ with pkgs; viking = callPackage ../applications/misc/viking { }; + vikunja = callPackage ../by-name/vi/vikunja/package.nix { pnpm = pnpm_8; }; + vim-vint = callPackage ../development/tools/vim-vint { }; vimer = callPackage ../tools/misc/vimer { }; From 18b124840feaf69c7cff3914d9258cc2a2b0ce25 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 5 May 2024 12:34:40 +0200 Subject: [PATCH 08/12] kiwitalk: use pnpm.fetchDeps Signed-off-by: Sefa Eyeoglu --- pkgs/by-name/ki/kiwitalk/package.nix | 60 +++++----------------------- pkgs/top-level/all-packages.nix | 2 + 2 files changed, 12 insertions(+), 50 deletions(-) diff --git a/pkgs/by-name/ki/kiwitalk/package.nix b/pkgs/by-name/ki/kiwitalk/package.nix index 2a581feb26842..23e87bdb43c3a 100644 --- a/pkgs/by-name/ki/kiwitalk/package.nix +++ b/pkgs/by-name/ki/kiwitalk/package.nix @@ -2,7 +2,6 @@ , fetchFromGitHub , copyDesktopItems , stdenv -, stdenvNoCC , rustc , rustPlatform , cargo @@ -12,20 +11,18 @@ , webkitgtk , pkg-config , makeDesktopItem -, jq -, moreutils -, nodePackages -, cacert +, pnpm +, nodejs }: -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "kiwitalk"; version = "0.5.1"; src = fetchFromGitHub { owner = "KiwiTalk"; repo = "KiwiTalk"; - rev = "v${version}"; + rev = "v${finalAttrs.version}"; hash = "sha256-Th8q+Zbc102fIk2v7O3OOeSriUV/ydz60QwxzmS7AY8="; }; @@ -34,43 +31,9 @@ stdenv.mkDerivation rec { --replace "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1" ''; - pnpm-deps = stdenvNoCC.mkDerivation { - pname = "${pname}-pnpm-deps"; - inherit src version; - - nativeBuildInputs = [ - jq - moreutils - nodePackages.pnpm - cacert - ]; - - installPhase = '' - export HOME=$(mktemp -d) - pnpm config set store-dir $out - # This version of the package has different versions of esbuild as a dependency. - # You can use the command below to get esbuild binaries for a specific platform and calculate hashes for that platforms. (linux, darwin for os, and x86, arm64, ia32 for cpu) - # cat package.json | jq '.pnpm.supportedArchitectures += { "os": ["linux"], "cpu": ["arm64"] }' | sponge package.json - pnpm install --frozen-lockfile --ignore-script - - # Remove timestamp and sort the json files. - rm -rf $out/v3/tmp - for f in $(find $out -name "*.json"); do - sed -i -E -e 's/"checkedAt":[0-9]+,//g' $f - jq --sort-keys . $f | sponge $f - done - ''; - - dontBuild = true; - dontFixup = true; - outputHashMode = "recursive"; - outputHash = { - x86_64-linux = "sha256-LJPjWNpVfdUu8F5BMhAzpTo/h6ax7lxY2EESHj5P390="; - aarch64-linux = "sha256-N1K4pV5rbWmO/KonvYegzBoWa6TYQIqhQyxH/sWjOJQ="; - i686-linux = "sha256-/Q7VZahYhLdKVFB25CanROYxD2etQOcRg+4bXZUMqTc="; - x86_64-darwin = "sha256-9biFAbFD7Bva7KPKztgCvcaoX8E6AlJBKkjlDQdP6Zw="; - aarch64-darwin = "sha256-to5Y0R9tm9b7jUQAK3eBylLhpu+w5oDd63FbBCBAvd8="; - }.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}"); + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src; + hash = "sha256-gf3vmKUta8KksUOxyhQS4UO6ycAJDfEicyXVGMW8+4c="; }; cargoDeps = rustPlatform.importCargoLock { @@ -86,7 +49,8 @@ stdenv.mkDerivation rec { cargo rustc cargo-tauri - nodePackages.pnpm + nodejs + pnpm.configHook copyDesktopItems pkg-config ]; @@ -98,10 +62,6 @@ stdenv.mkDerivation rec { ]; preBuild = '' - export HOME=$(mktemp -d) - pnpm config set store-dir ${pnpm-deps} - pnpm install --offline --frozen-lockfile --ignore-script - pnpm rebuild cargo tauri build -b deb ''; @@ -131,4 +91,4 @@ stdenv.mkDerivation rec { platforms = platforms.linux ++ platforms.darwin; mainProgram = "kiwi-talk"; }; - } +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 82ba984341b31..f50e956b77947 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -32126,6 +32126,8 @@ with pkgs; kitsas = libsForQt5.callPackage ../applications/office/kitsas { }; + kiwitalk = callPackage ../by-name/ki/kiwitalk/package.nix { pnpm = pnpm_8; }; + kiwix = libsForQt5.callPackage ../applications/misc/kiwix { }; kiwix-tools = callPackage ../applications/misc/kiwix/tools.nix { }; From 6fe1507ab21f6cb4d00f899440329c7a234bc01f Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 5 May 2024 12:47:24 +0200 Subject: [PATCH 09/12] gephgui-wry: use pnpm.fetchDeps Signed-off-by: Sefa Eyeoglu --- pkgs/applications/networking/geph/default.nix | 64 ++++++------------- pkgs/top-level/all-packages.nix | 2 +- 2 files changed, 20 insertions(+), 46 deletions(-) diff --git a/pkgs/applications/networking/geph/default.nix b/pkgs/applications/networking/geph/default.nix index 68cf0d61dacc4..07775d238d779 100644 --- a/pkgs/applications/networking/geph/default.nix +++ b/pkgs/applications/networking/geph/default.nix @@ -4,7 +4,8 @@ , fetchFromGitHub , buildGoModule , makeWrapper -, nodePackages +, nodejs +, pnpm , cacert , esbuild , jq @@ -48,7 +49,7 @@ in }; }; - gui = stdenvNoCC.mkDerivation rec { + gui = stdenvNoCC.mkDerivation (finalAttrs: { pname = "geph-gui"; inherit version; @@ -60,42 +61,11 @@ in fetchSubmodules = true; }; - pnpm-deps = stdenvNoCC.mkDerivation { - pname = "${pname}-pnpm-deps"; - inherit src version; - - sourceRoot = "${src.name}/gephgui-wry/gephgui"; - - nativeBuildInputs = [ - jq - moreutils - nodePackages.pnpm - cacert - ]; - - installPhase = '' - export HOME=$(mktemp -d) - pnpm config set store-dir $out - pnpm install --ignore-scripts - - # Remove timestamp and sort the json files - rm -rf $out/v3/tmp - for f in $(find $out -name "*.json"); do - sed -i -E -e 's/"checkedAt":[0-9]+,//g' $f - jq --sort-keys . $f | sponge $f - done - ''; - - dontFixup = true; - outputHashMode = "recursive"; - outputHash = "sha256-OKPx5xRI7DWd6m31nYx1biP0k6pcZ7fq7dfVlHda4O0="; - }; - gephgui-wry = rustPlatform.buildRustPackage { pname = "gephgui-wry"; - inherit version src; + inherit (finalAttrs) version src; - sourceRoot = "${src.name}/gephgui-wry"; + sourceRoot = "${finalAttrs.src.name}/gephgui-wry"; cargoLock = { lockFile = ./Cargo.lock; @@ -105,10 +75,17 @@ in }; }; + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src; + sourceRoot = "${finalAttrs.src.name}/gephgui-wry/gephgui"; + hash = "sha256-0MGlsLEgugQ1wEz07ROIwkanTa8PSKwIaxNahyS1014="; + }; + nativeBuildInputs = [ pkg-config - nodePackages.pnpm + pnpm.configHook makeWrapper + nodejs ]; buildInputs = [ @@ -132,22 +109,19 @@ in }); })}"; + pnpmRoot = "gephgui"; + preBuild = '' - cd gephgui - export HOME=$(mktemp -d) - pnpm config set store-dir ${pnpm-deps} - pnpm install --ignore-scripts --offline - chmod -R +w node_modules - pnpm rebuild + pushd gephgui pnpm build - cd .. + popd ''; }; dontBuild = true; installPhase = '' - install -Dt $out/bin ${gephgui-wry}/bin/gephgui-wry + install -Dt $out/bin ${finalAttrs.gephgui-wry}/bin/gephgui-wry install -d $out/share/icons/hicolor for i in '16' '32' '64' '128' '256' do @@ -163,5 +137,5 @@ in meta = geph-meta // { license = with lib.licenses; [ unfree ]; }; - }; + }); } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f50e956b77947..99e79372b4e49 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -20827,7 +20827,7 @@ with pkgs; gecode_6 = qt5.callPackage ../development/libraries/gecode { }; gecode = gecode_6; - geph = recurseIntoAttrs (callPackages ../applications/networking/geph { }); + geph = recurseIntoAttrs (callPackages ../applications/networking/geph { pnpm = pnpm_8; }); gephi = callPackage ../applications/science/misc/gephi { }; From b69250022976c1e91b27e2762cea0b417e087206 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sun, 5 May 2024 12:53:19 +0200 Subject: [PATCH 10/12] youtube-music: use pnpm.fetchDeps Signed-off-by: Sefa Eyeoglu --- .../audio/youtube-music/default.nix | 52 +++---------------- pkgs/top-level/all-packages.nix | 2 +- 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/pkgs/applications/audio/youtube-music/default.nix b/pkgs/applications/audio/youtube-music/default.nix index 642ba8e3fc0ff..32a94f4eb7ce4 100644 --- a/pkgs/applications/audio/youtube-music/default.nix +++ b/pkgs/applications/audio/youtube-music/default.nix @@ -4,12 +4,9 @@ , electron , python3 , stdenv -, stdenvNoCC , copyDesktopItems -, moreutils -, cacert -, jq -, nodePackages +, nodejs +, pnpm , makeDesktopItem }: @@ -24,54 +21,17 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-nxpctEG4XoxW6jOAxGdgTEYr6YnhFRR8+5HUQLxRJB0="; }; - pnpmDeps = stdenvNoCC.mkDerivation { - pname = "${finalAttrs.pname}-pnpm-deps"; - inherit (finalAttrs) src version ELECTRON_SKIP_BINARY_DOWNLOAD; - - nativeBuildInputs = [ jq moreutils nodePackages.pnpm cacert ]; - - installPhase = '' - export HOME=$(mktemp -d) - - pnpm config set store-dir $out - pnpm install --frozen-lockfile --ignore-script - - rm -rf $out/v3/tmp - for f in $(find $out -name "*.json"); do - sed -i -E -e 's/"checkedAt":[0-9]+,//g' $f - jq --sort-keys . $f | sponge $f - done - ''; - - dontBuild = true; - dontFixup = true; - outputHashMode = "recursive"; - outputHash = { - x86_64-linux = "sha256-bujlQxP6Lr3qPUDxYXKyb702ZJY/xbuCsu3wVDhcb+8="; - aarch64-linux = "sha256-0kyjjttpXpFVhdza5NAjGrRn++qc/N5/u2dQl7VufLE="; - x86_64-darwin = "sha256-Q37QJt/mhfpSguOlkJGKFTCrIOrpbG3OBwaD/Bg09Us="; - aarch64-darwin = "sha256-wbfjzoGa/6vIlOOVX3bKNQ2uxzph3WSofo3MGXqA6yQ="; - }.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}"); + pnpmDeps = pnpm.fetchDeps { + inherit (finalAttrs) pname version src; + hash = "sha256-8oeloQYiwUy+GDG4R+XtiynT+8Fad4WYFWTO1KANZKQ="; }; - nativeBuildInputs = [ makeWrapper python3 nodePackages.pnpm nodePackages.nodejs ] + nativeBuildInputs = [ makeWrapper python3 nodejs pnpm.configHook ] ++ lib.optionals (!stdenv.isDarwin) [ copyDesktopItems ]; ELECTRON_SKIP_BINARY_DOWNLOAD = 1; - preBuild = '' - export HOME=$(mktemp -d) - export STORE_PATH=$(mktemp -d) - - cp -Tr "$pnpmDeps" "$STORE_PATH" - chmod -R +w "$STORE_PATH" - - pnpm config set store-dir "$STORE_PATH" - pnpm install --offline --frozen-lockfile --ignore-script - patchShebangs node_modules/{*,.*} - ''; - postBuild = lib.optionalString stdenv.isDarwin '' cp -R ${electron}/Applications/Electron.app Electron.app chmod -R u+w Electron.app diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 99e79372b4e49..a90c5d034bcd2 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -35833,7 +35833,7 @@ with pkgs; youtube-dl-light = with python3Packages; toPythonApplication youtube-dl-light; - youtube-music = callPackage ../applications/audio/youtube-music { }; + youtube-music = callPackage ../applications/audio/youtube-music { pnpm = pnpm_8; }; youtube-tui = callPackage ../applications/video/youtube-tui { inherit (darwin.apple_sdk.frameworks) CoreFoundation Security AppKit; From 711884c56608d76285b681c5024818e5ddc76d05 Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Mon, 3 Jun 2024 16:02:45 +0300 Subject: [PATCH 11/12] pnpm_{8,9}: remove uneeded binary files from src --- pkgs/development/tools/pnpm/generic.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/development/tools/pnpm/generic.nix b/pkgs/development/tools/pnpm/generic.nix index c53dfd78f673a..5560f4aee6ebc 100644 --- a/pkgs/development/tools/pnpm/generic.nix +++ b/pkgs/development/tools/pnpm/generic.nix @@ -17,6 +17,11 @@ url = "https://registry.npmjs.org/pnpm/-/pnpm-${finalAttrs.version}.tgz"; inherit hash; }; + # Remove binary files from src, we don't need them, and this way we make sure + # our distribution is free of binaryNativeCode + preConfigure = '' + rm -r dist/reflink.*node dist/vendor + ''; buildInputs = lib.optionals withNode [ nodejs ]; From c704c029566aaf525d949b54b8a8d85bb6aab29b Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Mon, 3 Jun 2024 16:06:26 +0300 Subject: [PATCH 12/12] doc/javascript: pnpm: mention lack of monorepos/workspaces support --- doc/languages-frameworks/javascript.section.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/languages-frameworks/javascript.section.md b/doc/languages-frameworks/javascript.section.md index 5421ca0258b1b..76db9d0007ce5 100644 --- a/doc/languages-frameworks/javascript.section.md +++ b/doc/languages-frameworks/javascript.section.md @@ -348,6 +348,8 @@ In case you are patching `package.json` or `pnpm-lock.yaml`, make sure to pass ` #### Dealing with `sourceRoot` {#javascript-pnpm-sourceRoot} +NOTE: Nixpkgs pnpm tooling doesn't support building projects with a `pnpm-workspace.yaml`, or building monorepos. It maybe possible to use `pnpm.fetchDeps` for these projects, but it may be hard or impossible to produce a binary from such projects ([an example attempt](https://github.com/NixOS/nixpkgs/pull/290715#issuecomment-2144543728)). + If the pnpm project is in a subdirectory, you can just define `sourceRoot` or `setSourceRoot` for `fetchDeps`. Note, that projects using `pnpm-workspace.yaml` are currently not supported, and will probably not work using this approach. If `sourceRoot` is different between the parent derivation and `fetchDeps`, you will have to set `pnpmRoot` to effectively be the same location as it is in `fetchDeps`.