From 004606373b5045afc2aeda8adf5bbcb04e69066f Mon Sep 17 00:00:00 2001 From: oddlama Date: Fri, 28 Jul 2023 23:20:05 +0200 Subject: [PATCH 1/3] immich: init at 1.79.1 --- pkgs/servers/immich/default.nix | 287 +++++++++++++++++++++++++ pkgs/servers/immich/sources.json | 15 ++ pkgs/servers/immich/update.sh | 40 ++++ pkgs/tools/misc/immich-cli/default.nix | 27 --- pkgs/top-level/all-packages.nix | 8 +- 5 files changed, 348 insertions(+), 29 deletions(-) create mode 100644 pkgs/servers/immich/default.nix create mode 100644 pkgs/servers/immich/sources.json create mode 100755 pkgs/servers/immich/update.sh delete mode 100644 pkgs/tools/misc/immich-cli/default.nix diff --git a/pkgs/servers/immich/default.nix b/pkgs/servers/immich/default.nix new file mode 100644 index 0000000000000..6009b3e4b9f34 --- /dev/null +++ b/pkgs/servers/immich/default.nix @@ -0,0 +1,287 @@ +{ lib +, pkgs +, buildNpmPackage +, fetchFromGitHub +, fetchPypi +, python +, nodejs +, nixosTests +# build-time deps +, pkg-config +, makeWrapper +, cmake +# runtime deps +, ffmpeg +, imagemagick +, libraw +, vips +}: +let + buildNpmPackage' = buildNpmPackage.override { inherit nodejs; }; + sources = lib.importJSON ./sources.json; + inherit (sources) version; + + src = fetchFromGitHub { + owner = "immich-app"; + repo = "immich"; + rev = "v${version}"; + inherit (sources) hash; + }; + + cli = buildNpmPackage' { + pname = "immich-cli"; + inherit version; + src = "${src}/cli"; + inherit (sources.components.cli) npmDepsHash; + + nativeBuildInputs = [ + makeWrapper + ]; + + installPhase = '' + runHook preInstall + + npm config delete cache + npm prune --omit=dev --omit=optional + + mkdir -p $out + mv package.json package-lock.json node_modules dist $out/ + + makeWrapper ${nodejs}/bin/node $out/bin/cli --add-flags $out/dist/index.js + + runHook postInstall + ''; + }; + + web = buildNpmPackage' { + pname = "immich-web"; + inherit version; + src = "${src}/web"; + inherit (sources.components.web) npmDepsHash; + + nativeBuildInputs = [ + makeWrapper + ]; + + installPhase = '' + runHook preInstall + + npm config delete cache + npm prune --omit=dev --omit=optional + + mkdir -p $out + mv package.json package-lock.json node_modules build $out/ + + makeWrapper ${nodejs}/bin/node $out/bin/web --add-flags $out/build/index.js + + runHook postInstall + ''; + }; + + # TODO should be converted to top-level package + open-clip-torch = python.pkgs.buildPythonApplication rec { + pname = "open-clip-torch"; + version = "2.20.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "mlfoundations"; + repo = "open_clip"; + rev = "v${version}"; + hash = "sha256-Ca4oi2LqleIFAGBJB7YIi4nXe2XhOP6ErDFXgXtJLxM="; + }; + + nativeBuildInputs = [ + python.pkgs.setuptools + python.pkgs.wheel + ]; + + propagatedBuildInputs = with python.pkgs; [ + ftfy + huggingface-hub + protobuf + regex + sentencepiece + timm + torch + torchvision + tqdm + ]; + + pythonImportsCheck = [ "open_clip" ]; + + meta = with lib; { + description = "An open source implementation of CLIP"; + homepage = "https://github.com/mlfoundations/open_clip"; + license = licenses.mit; + maintainers = with maintainers; [ oddlama ]; + mainProgram = "open-clip-torch"; + }; + }; + + # TODO should be converted to top-level package + clip-server = python.pkgs.buildPythonApplication rec { + pname = "clip-server"; + version = "0.8.2"; + pyproject = true; + + src = fetchPypi { + inherit pname version; + hash = "sha256-yhBLXqbV3U1JAGPlfih1bxxBzATHzyGQdNVGKI4DKg8="; + }; + + nativeBuildInputs = [ + python.pkgs.setuptools + python.pkgs.wheel + ]; + + pythonImportsCheck = [ "clip_server" ]; + + meta = with lib; { + description = "Embed images and sentences into fixed-length vectors via CLIP"; + homepage = "https://pypi.org/project/clip-server/"; + license = licenses.asl20; + maintainers = with maintainers; [ oddlama ]; + mainProgram = "clip-server"; + }; + }; + + machine-learning = python.pkgs.buildPythonApplication { + pname = "immich-machine-learning"; + inherit version; + src = "${src}/machine-learning"; + format = "pyproject"; + + postPatch = '' + rm poetry.lock + + # opencv is named differently, also remove development dependencies not needed at runtime + substituteInPlace pyproject.toml \ + --replace 'opencv-python-headless = "^4.7.0.72"' "" \ + --replace 'pydantic = "^1.10.8"' "" + + # XXX: Remove these once nix packages reach compatible versions + substituteInPlace pyproject.toml \ + --replace 'pillow = "^9.5.0"' 'pillow = "^10.0.0"' + + # XXX: These can be removed once opencv4 reaches 4.8.0 + substituteInPlace app/models/facial_recognition.py \ + --replace ": cv2.Mat" "" + substituteInPlace app/test_main.py \ + --replace ": cv2.Mat" "" + substituteInPlace app/main.py \ + --replace ": cv2.Mat" "" \ + --replace "-> cv2.Mat" "" + ''; + + nativeBuildInputs = with python.pkgs; [ + pythonRelaxDepsHook + poetry-core + cython + makeWrapper + ]; + + propagatedBuildInputs = with python.pkgs; [ + aiocache + fastapi + optimum + torchvision + rich + ftfy + open-clip-torch + clip-server + python-multipart + orjson + safetensors + gunicorn + insightface + onnxruntime + opencv4 + pillow + sentence-transformers + torch + transformers + uvicorn + ] ++ python.pkgs.uvicorn.optional-dependencies.standard; + + nativeCheckInputs = with python.pkgs; [ + pytestCheckHook + pytest-asyncio + pytest-mock + ]; + + postInstall = '' + mkdir -p $out/share + cp log_conf.json $out/share + makeWrapper ${python.pkgs.gunicorn}/bin/gunicorn $out/bin/machine-learning \ + --prefix PYTHONPATH : "$PYTHONPATH" \ + --add-flags "app.main:app -k uvicorn.workers.UvicornWorker \ + -w \"\$MACHINE_LEARNING_WORKERS\" \ + -b \"\$MACHINE_LEARNING_HOST:\$MACHINE_LEARNING_PORT\" \ + -t \"\$MACHINE_LEARNING_WORKER_TIMEOUT\" \ + --log-config-json $out/share/log_conf.json" + ''; + + preCheck = '' + export TRANSFORMERS_CACHE=/tmp + ''; + + passthru = { + inherit python; + }; + }; +in +buildNpmPackage' { + pname = "immich"; + inherit version; + src = "${src}/server"; + inherit (sources.components.server) npmDepsHash; + + nativeBuildInputs = [ + pkg-config + python + makeWrapper + ]; + + buildInputs = [ + ffmpeg + imagemagick + libraw + vips # Required for sharp + ]; + + # Required because vips tries to write to the cache dir + makeCacheWritable = true; + # TODO not working prePatch = '' + # TODO not working export npm_config_libvips_local_prebuilds="/tmp" + # TODO not working ''; + + installPhase = '' + runHook preInstall + + npm config delete cache + npm prune --omit=dev --omit=optional + + mkdir -p $out + mv package.json package-lock.json node_modules dist $out/ + + makeWrapper ${nodejs}/bin/node $out/bin/admin-cli --add-flags $out/dist/main --add-flags cli + makeWrapper ${nodejs}/bin/node $out/bin/microservices --add-flags $out/dist/main --add-flags microservices + makeWrapper ${nodejs}/bin/node $out/bin/server --add-flags $out/dist/main --add-flags immich + + runHook postInstall + ''; + + passthru = { + inherit cli web machine-learning; + updateScript = ./update.sh; + }; + + meta = with lib; { + description = "Self-hosted photo and video backup solution"; + homepage = "https://immich.app/"; + license = licenses.mit; + maintainers = with maintainers; [ oddlama ]; + inherit (nodejs.meta) platforms; + }; +} diff --git a/pkgs/servers/immich/sources.json b/pkgs/servers/immich/sources.json new file mode 100644 index 0000000000000..9e035576b7d52 --- /dev/null +++ b/pkgs/servers/immich/sources.json @@ -0,0 +1,15 @@ +{ + "version": "1.79.1", + "hash": "sha256-IzBf2K7pAh4oFHtVdFOChKwZ8GjuRLKwzc2pJGmpvOE=", + "components": { + "cli": { + "npmDepsHash": "sha256-jCbJd0fC1sjMRisRMvVK3kLFdRAxkGbIy+3fL5ZRftQ=" + }, + "server": { + "npmDepsHash": "sha256-5Q5Rn+75aXWtaHUoiHZwnBv+04+Whz1ZFmGNWqtY35g=" + }, + "web": { + "npmDepsHash": "sha256-160wDIf7VGkLJuf/L/ewgxpW/HvISVdJsOGwuUJelDE=" + } + } +} diff --git a/pkgs/servers/immich/update.sh b/pkgs/servers/immich/update.sh new file mode 100755 index 0000000000000..5331bce026181 --- /dev/null +++ b/pkgs/servers/immich/update.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl jq prefetch-npm-deps nix-prefetch-git nix-prefetch-github coreutils + +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ -v GITHUB_TOKEN ]]; then + TOKEN_ARGS=(--header "Authorization: token $GITHUB_TOKEN") +fi + +old_version=$(jq -r ".version" sources.json || echo -n "0.0.1") +version=$(curl "${TOKEN_ARGS[@]}" -s "https://api.github.com/repos/immich-app/immich/releases/latest" | jq -r ".tag_name") +version="${version#v}" + +if [[ "$old_version" == "$version" ]]; then + echo "Already up to date!" + exit 0 +fi + +src_hash=$(nix-prefetch-github immich-app immich --rev "v${version}" | jq -r .hash) +upstream_src="https://raw.githubusercontent.com/immich-app/immich/v$version" + +sources_tmp="$(mktemp)" +cat < "$sources_tmp" +{ + "version": "$version", + "hash": "$src_hash", + "components": {} +} +EOF + +for npm_component in cli server web; do + hash=$(prefetch-npm-deps <(curl "${TOKEN_ARGS[@]}" -s "$upstream_src/$npm_component/package-lock.json")) + echo "$(jq --arg npm_component "$npm_component" \ + --arg hash "$hash" \ + '.components += {($npm_component): {npmDepsHash: $hash}}' \ + "$sources_tmp")" > "$sources_tmp" +done + +cp "$sources_tmp" sources.json diff --git a/pkgs/tools/misc/immich-cli/default.nix b/pkgs/tools/misc/immich-cli/default.nix deleted file mode 100644 index ec1483ccc0c9b..0000000000000 --- a/pkgs/tools/misc/immich-cli/default.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ lib -, buildNpmPackage -, fetchFromGitHub -}: - -buildNpmPackage rec { - pname = "immich-cli"; - version = "0.41.0"; - - src = fetchFromGitHub { - owner = "immich-app"; - repo = "CLI"; - rev = "v${version}"; - hash = "sha256-BpJNssNTJZASH5VTgTNJ0ILj0XucWvyn3Y7hQdfCEGQ="; - }; - - npmDepsHash = "sha256-GOYWPRAzV59iaX32I42dOOEv1niLiDIPagzQ/QBBbKc="; - - meta = { - changelog = "https://github.com/immich-app/CLI/releases/tag/${src.rev}"; - description = "CLI utilities for Immich to help upload images and videos"; - homepage = "https://github.com/immich-app/CLI"; - license = lib.licenses.mit; - mainProgram = "immich"; - maintainers = with lib.maintainers; [ felschr ]; - }; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 5473845050dd1..c5840069f08d7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1854,8 +1854,6 @@ with pkgs; hyperpotamus = callPackage ../tools/misc/hyperpotamus { }; - immich-cli = callPackage ../tools/misc/immich-cli { }; - inherit (callPackage ../tools/networking/ivpn/default.nix {}) ivpn ivpn-service; jobber = callPackage ../tools/system/jobber { }; @@ -26532,6 +26530,12 @@ with pkgs; hyp = callPackage ../servers/http/hyp { }; + immich = callPackage ../servers/immich { + nodejs = nodejs_18; + python = python311; + }; + immich-cli = immich.cli; + peering-manager = callPackage ../servers/web-apps/peering-manager { }; podgrab = callPackage ../servers/misc/podgrab { }; From 6db73f0732e50eee497cf040282b2db2bf01fa9a Mon Sep 17 00:00:00 2001 From: oddlama Date: Fri, 28 Jul 2023 23:22:22 +0200 Subject: [PATCH 2/3] nixos/immich: init module --- .../manual/release-notes/rl-2311.section.md | 2 + nixos/modules/module-list.nix | 1 + nixos/modules/services/web-apps/immich.nix | 575 ++++++++++++++++++ 3 files changed, 578 insertions(+) create mode 100644 nixos/modules/services/web-apps/immich.nix diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 37b840539ea4a..16c968a7abf46 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -72,6 +72,8 @@ - [osquery](https://www.osquery.io/), a SQL powered operating system instrumentation, monitoring, and analytics. +- [Immich](https://github.com/immich/immich), a self-hosted photo and video backup solution. Available as [services.immich](#opt-services.immich.enable). + - [ebusd](https://ebusd.eu), a daemon for handling communication with eBUS devices connected to a 2-wire bus system (“energy bus” used by numerous heating systems). Available as [services.ebusd](#opt-services.ebusd.enable). - [systemd-sysupdate](https://www.freedesktop.org/software/systemd/man/systemd-sysupdate.html), atomically updates the host OS, container images, portable service images or other sources. Available as [systemd.sysupdate](opt-systemd.sysupdate). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index ec6f410a48f68..48465308620d5 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1235,6 +1235,7 @@ ./services/web-apps/gotify-server.nix ./services/web-apps/gotosocial.nix ./services/web-apps/grocy.nix + ./services/web-apps/immich.nix ./services/web-apps/pixelfed.nix ./services/web-apps/guacamole-client.nix ./services/web-apps/guacamole-server.nix diff --git a/nixos/modules/services/web-apps/immich.nix b/nixos/modules/services/web-apps/immich.nix new file mode 100644 index 0000000000000..b534b6d987d05 --- /dev/null +++ b/nixos/modules/services/web-apps/immich.nix @@ -0,0 +1,575 @@ +{ + config, + lib, + pkgs, + ... +}: let + inherit + (lib) + hasAttr + hasPrefix + maintainers + mapAttrs + mkDefault + mkEnableOption + mkIf + mkMerge + mkOption + mkPackageOption + optional + optionalAttrs + optionalString + types + ; + + cfg = config.services.immich; + serverCfg = config.services.immich.server; + backendCfg = serverCfg.backend; + microservicesCfg = serverCfg.microservices; + webCfg = cfg.web; + mlCfg = cfg.machineLearning; + + isServerPostgresUnix = hasPrefix "/" serverCfg.postgres.host; + postgresEnv = + if isServerPostgresUnix + then { + # If passwordFile is given, this will be overwritten in ExecStart + DB_URL = "socket://${serverCfg.postgres.host}?dbname=${serverCfg.postgres.database}"; + } + else { + DB_HOSTNAME = serverCfg.postgres.host; + DB_PORT = toString serverCfg.postgres.port; + DB_DATABASE_NAME = serverCfg.postgres.database; + DB_USERNAME = serverCfg.postgres.username; + }; + + typesenseEnv = { + TYPESENSE_ENABLED = toString serverCfg.typesense.enable; + } // optionalAttrs serverCfg.typesense.enable { + TYPESENSE_HOST = serverCfg.typesense.host; + TYPESENSE_PORT = toString serverCfg.typesense.port; + TYPESENSE_PROTOCOL = serverCfg.typesense.protocol; + }; + + serverMachineLearningEnv = { + IMMICH_MACHINE_LEARNING_ENABLED = toString serverCfg.machineLearning.enable; + } // optionalAttrs serverCfg.machineLearning.enable { + IMMICH_MACHINE_LEARNING_URL = serverCfg.machineLearning.url; + }; + + # Don't start a redis instance if the user sets a custom redis connection + enableRedis = !hasAttr "REDIS_URL" serverCfg.extraConfig && !hasAttr "REDIS_SOCKET" serverCfg.extraConfig; + redisServerCfg = config.services.redis.servers.immich; + redisEnv = optionalAttrs enableRedis { + REDIS_SOCKET = redisServerCfg.unixSocket; + }; + + serverEnv = postgresEnv // typesenseEnv // serverMachineLearningEnv // redisEnv // { + NODE_ENV = "production"; + IMMICH_MEDIA_LOCATION = serverCfg.mediaDir; + }; + + serverStartWrapper = program: '' + set -euo pipefail + mkdir -p ${serverCfg.mediaDir} + + ${optionalString (serverCfg.postgres.passwordFile != null) ( + if isServerPostgresUnix + then ''export DB_URL="socket://${serverCfg.postgres.username}:$(cat ${serverCfg.postgres.passwordFile})@${serverCfg.postgres.host}?dbname=${serverCfg.postgres.database}"'' + else "export DB_PASSWORD=$(cat ${serverCfg.postgres.passwordFile})" + )} + + ${optionalString serverCfg.typesense.enable '' + export TYPESENSE_API_KEY=$(cat ${serverCfg.typesense.apiKeyFile}) + ''} + + exec ${program} + ''; + + commonServiceConfig = { + Restart = "on-failure"; + + # Hardening + CapabilityBoundingSet = ""; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateUsers = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateMounts = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + # Would re-mount paths ignored by temporary root + # TODO ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "@pkey" + ]; + UMask = "0077"; + }; + + serverServiceConfig = { + DynamicUser = true; + User = "immich"; + Group = "immich"; + SupplementaryGroups = optional enableRedis redisServerCfg.user; + + StateDirectory = "immich"; + StateDirectoryMode = "0750"; + WorkingDirectory = "/var/lib/immich"; + + MemoryDenyWriteExecute = false; # nodejs requires this. + EnvironmentFile = mkIf (serverCfg.environmentFile != null) serverCfg.environmentFile; + + TemporaryFileSystem = "/:ro"; + BindReadOnlyPaths = [ + "/nix/store" + "-/etc/resolv.conf" + "-/etc/nsswitch.conf" + "-/etc/hosts" + "-/etc/localtime" + "-/run/postgresql" + ] ++ optional enableRedis redisServerCfg.unixSocket; + }; +in { + options.services.immich = { + enable = mkEnableOption "immich" // { + description = '' + Enables immich which consists of a backend server, microservices, + machine-learning and web ui. You can disable or reconfigure components + individually using the subsections. + ''; + }; + + package = mkPackageOption pkgs "immich" {}; + + server = { + mediaDir = mkOption { + type = types.str; + default = "/var/lib/immich/media"; + description = "Directory used to store media files."; + }; + + backend = { + enable = mkEnableOption "immich backend server" // { + default = true; + }; + port = mkOption { + type = types.port; + default = 3001; + description = "Port to bind to."; + }; + + openFirewall = mkOption { + default = false; + type = types.bool; + description = "Whether to open the firewall for the specified port."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = {}; + example = { + LOG_LEVEL = "debug"; + }; + description = '' + Extra configuration options (environment variables). + Refer to [the documented variables](https://documentation.immich.app/docs/install/environment-variables) tagged with 'server' for available options. + ''; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Environment file as defined in systemd.exec(5). May be used to provide + additional secret variables to the service without adding them to the + world-readable Nix store. + ''; + }; + }; + + microservices = { + enable = mkEnableOption "immich microservices" // { + default = true; + }; + + port = mkOption { + type = types.port; + default = 3002; + description = "Port to bind to."; + }; + + openFirewall = mkOption { + default = false; + type = types.bool; + description = "Whether to open the firewall for the specified port."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = {}; + example = { + REVERSE_GEOCODING_PRECISION = 1; + }; + description = '' + Extra configuration options (environment variables). + Refer to [the documented variables](https://documentation.immich.app/docs/install/environment-variables) tagged with 'microservices' for available options. + ''; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Environment file as defined in systemd.exec(5). May be used to provide + additional secret variables to the service without adding them to the + world-readable Nix store. + ''; + }; + }; + + typesense = { + enable = mkEnableOption "typesense" // { + default = true; + }; + + host = mkOption { + type = types.str; + default = "127.0.0.1"; + example = "typesense.example.com"; + description = "Hostname/address of the typesense server to use."; + }; + + port = mkOption { + type = types.port; + default = 8108; + description = "The port of the typesense server to use."; + }; + + protocol = mkOption { + type = types.str; + default = "http"; + description = "The protocol to use when connecting to the typesense server."; + }; + + apiKeyFile = mkOption { + type = types.path; + description = "Sets the api key for authentication with typesense."; + }; + }; + + postgres = { + host = mkOption { + type = types.str; + default = "/run/postgresql"; + description = "Hostname/address of the postgres server to use. If an absolute path is given here, it will be interpreted as a unix socket path."; + }; + + port = mkOption { + type = types.port; + default = 5432; + description = "The port of the postgres server to use."; + }; + + username = mkOption { + type = types.str; + default = "immich"; + description = "The postgres username to use."; + }; + + passwordFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Sets the password for authentication with postgres. + May be unset when using socket authentication. + ''; + }; + + database = mkOption { + type = types.str; + default = "immich"; + description = "The postgres database to use."; + }; + }; + + machineLearning = { + enable = mkEnableOption "machine learning by utilizing the specified ML endpoint" // { + default = true; + }; + + url = mkOption { + type = types.str; + default = "http://127.0.0.1:3003"; + example = "https://immich-ml.internal.example.com"; + description = "The machine learning server endpoint to use."; + }; + }; + + extraConfig = mkOption { + type = types.attrs; + default = {}; + example = { + REDIS_SOCKET = "/run/custom-redis"; + }; + description = '' + Extra configuration options (environment variables) for both backend and microservices. + Refer to [the documented variables](https://documentation.immich.app/docs/install/environment-variables) tagged with both 'server' and 'microservices' for available options. + ''; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Environment file as defined in systemd.exec(5). May be used to provide + additional secret variables to the backend and microservices servers without + adding them to the world-readable Nix store. + ''; + }; + }; + + web = { + enable = mkEnableOption "immich web frontend" // { + default = true; + }; + + port = mkOption { + type = types.port; + default = 3000; + description = "Port to bind to."; + }; + + openFirewall = mkOption { + default = false; + type = types.bool; + description = "Whether to open the firewall for the specified port."; + }; + + serverUrl = mkOption { + type = types.str; + default = "http://127.0.0.1:3001"; + example = "https://immich-backend.internal.example.com"; + description = "The backend server url to use."; + }; + + apiUrlExternal = mkOption { + type = types.str; + default = "/web"; + description = "The api url to use for external requests."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = {}; + example = { + PUBLIC_LOGIN_PAGE_MESSAGE = "My awesome Immich instance!"; + }; + description = '' + Extra configuration options (environment variables). + Refer to [the documented variables](https://documentation.immich.app/docs/install/environment-variables) tagged with 'web' for available options. + ''; + }; + }; + + machineLearning = { + enable = mkEnableOption "immich machine-learning server" // { + default = true; + }; + + port = mkOption { + type = types.port; + default = 3003; + description = "Port to bind to."; + }; + + openFirewall = mkOption { + default = false; + type = types.bool; + description = "Whether to open the firewall for the specified port."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = {}; + example = { + MACHINE_LEARNING_MODEL_TTL = 600; + }; + description = '' + Extra configuration options (environment variables). + Refer to [the documented variables](https://documentation.immich.app/docs/install/environment-variables) tagged with 'machine learning' for available options. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = !isServerPostgresUnix -> serverCfg.postgres.passwordFile != null; + message = "A database password must be provided when unix sockets are not used."; + } + ]; + + networking.firewall.allowedTCPPorts = mkMerge [ + (mkIf (backendCfg.enable && backendCfg.openFirewall) [backendCfg.port]) + (mkIf (microservicesCfg.enable && microservicesCfg.openFirewall) [microservicesCfg.port]) + (mkIf (webCfg.enable && webCfg.openFirewall) [webCfg.port]) + (mkIf (mlCfg.enable && mlCfg.openFirewall) [mlCfg.port]) + ]; + + services.redis.servers.immich.enable = mkIf enableRedis true; + services.redis.vmOverCommit = mkIf enableRedis (mkDefault true); + + systemd.services.immich-server = mkIf backendCfg.enable { + description = "Immich backend server (Self-hosted photo and video backup solution)"; + after = [ + "network.target" + "typesense.service" + "postgresql.service" + "immich-machine-learning.service" + ] ++ optional enableRedis "redis-immich.service"; + wantedBy = ["multi-user.target"]; + + environment = serverEnv // { + SERVER_PORT = toString backendCfg.port; + } + // mapAttrs (_: toString) serverCfg.extraConfig + // mapAttrs (_: toString) backendCfg.extraConfig; + + script = serverStartWrapper "${cfg.package}/bin/server"; + serviceConfig = mkMerge [ + (commonServiceConfig // serverServiceConfig) + { + EnvironmentFile = mkIf (backendCfg.environmentFile != null) backendCfg.environmentFile; + } + ]; + }; + + systemd.services.immich-microservices = mkIf microservicesCfg.enable { + description = "Immich microservices (Self-hosted photo and video backup solution)"; + after = [ + "network.target" + "typesense.service" + "postgresql.service" + "immich-machine-learning.service" + ] ++ optional enableRedis "redis-immich.service"; + wantedBy = ["multi-user.target"]; + + environment = serverEnv // { + MICROSERVICES_PORT = toString microservicesCfg.port; + } + // mapAttrs (_: toString) serverCfg.extraConfig + // mapAttrs (_: toString) microservicesCfg.extraConfig; + + script = serverStartWrapper "${cfg.package}/bin/microservices"; + serviceConfig = mkMerge [ + (commonServiceConfig // serverServiceConfig) + { + EnvironmentFile = mkIf (microservicesCfg.environmentFile != null) microservicesCfg.environmentFile; + } + ]; + }; + + systemd.services.immich-web = mkIf webCfg.enable { + description = "Immich web (Self-hosted photo and video backup solution)"; + after = [ + "network.target" + "immich-server.service" + ]; + wantedBy = ["multi-user.target"]; + + environment = { + NODE_ENV = "production"; + PORT = toString webCfg.port; + IMMICH_SERVER_URL = webCfg.serverUrl; + IMMICH_API_URL_EXTERNAL = webCfg.apiUrlExternal; + } + // mapAttrs (_: toString) webCfg.extraConfig; + + script = '' + set -euo pipefail + export PUBLIC_IMMICH_SERVER_URL=$IMMICH_SERVER_URL + export PUBLIC_IMMICH_API_URL_EXTERNAL=$IMMICH_API_URL_EXTERNAL + exec ${cfg.package.web}/bin/web + ''; + serviceConfig = commonServiceConfig // { + DynamicUser = true; + User = "immich-web"; + Group = "immich-web"; + + MemoryDenyWriteExecute = false; # nodejs requires this. + + TemporaryFileSystem = "/:ro"; + BindReadOnlyPaths = [ + "/nix/store" + "-/etc/resolv.conf" + "-/etc/nsswitch.conf" + "-/etc/hosts" + "-/etc/localtime" + ]; + }; + }; + + systemd.services.immich-machine-learning = mkIf mlCfg.enable { + description = "Immich machine learning (Self-hosted photo and video backup solution)"; + after = ["network.target"]; + wantedBy = ["multi-user.target"]; + + environment = { + NODE_ENV = "production"; + MACHINE_LEARNING_PORT = toString mlCfg.port; + + MACHINE_LEARNING_CACHE_FOLDER = "/var/cache/immich-ml"; + TRANSFORMERS_CACHE = "/var/cache/immich-ml"; + } + // mapAttrs (_: toString) mlCfg.extraConfig; + + serviceConfig = commonServiceConfig // { + ExecStart = "${cfg.package.machine-learning}/bin/machine-learning"; + DynamicUser = true; + User = "immich-ml"; + Group = "immich-ml"; + + MemoryDenyWriteExecute = false; # onnxruntime_pybind11 requires this. + ProcSubset = "all"; # Needs /proc/cpuinfo + + CacheDirectory = "immich-ml"; + CacheDirectoryMode = "0700"; + + # TODO gpu access + + TemporaryFileSystem = "/:ro"; + BindReadOnlyPaths = [ + "/nix/store" + "-/etc/resolv.conf" + "-/etc/nsswitch.conf" + "-/etc/hosts" + "-/etc/localtime" + ]; + }; + }; + + meta.maintainers = with maintainers; [ oddlama ]; + }; +} From 531ddadb327c0c2a75642acfa7d19b5dcc50a8a7 Mon Sep 17 00:00:00 2001 From: oddlama Date: Fri, 28 Jul 2023 23:22:50 +0200 Subject: [PATCH 3/3] nixos/tests/immich: init tests --- nixos/tests/all-tests.nix | 1 + nixos/tests/web-apps/immich.nix | 69 +++++++++++++++++++++++++++++++++ pkgs/servers/immich/default.nix | 1 + 3 files changed, 71 insertions(+) create mode 100644 nixos/tests/web-apps/immich.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5b2e12a501bf4..8ffc0ae9414a2 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -374,6 +374,7 @@ in { i3wm = handleTest ./i3wm.nix {}; icingaweb2 = handleTest ./icingaweb2.nix {}; iftop = handleTest ./iftop.nix {}; + immich = handleTest ./web-apps/immich.nix {}; incron = handleTest ./incron.nix {}; influxdb = handleTest ./influxdb.nix {}; influxdb2 = handleTest ./influxdb2.nix {}; diff --git a/nixos/tests/web-apps/immich.nix b/nixos/tests/web-apps/immich.nix new file mode 100644 index 0000000000000..572598b212d3f --- /dev/null +++ b/nixos/tests/web-apps/immich.nix @@ -0,0 +1,69 @@ +import ../make-test-python.nix ({pkgs, ...}: let + typesenseApiKeyFile = pkgs.writeText "typesense-api-key" "12318551487654187654"; +in { + name = "immich-nixos"; + # TODO second machine test distributed setup + nodes.machine = { + self, + pkgs, + ... + }: { + # These tests need a little more juice + virtualisation.cores = 2; + virtualisation.memorySize = 2048; + + services.immich = { + enable = true; + server.typesense.apiKeyFile = typesenseApiKeyFile; + }; + + services.typesense = { + enable = true; + # In a real setup you should generate an api key for immich + # and not use the admin key! + apiKeyFile = typesenseApiKeyFile; + settings.server.api-address = "127.0.0.1"; + }; + + services.postgresql = { + enable = true; + identMap = '' + # ArbitraryMapName systemUser DBUser + superuser_map root postgres + superuser_map postgres postgres + # Let other names login as themselves + superuser_map /^(.*)$ \1 + ''; + authentication = pkgs.lib.mkOverride 10 '' + local sameuser all peer map=superuser_map + ''; + + ensureDatabases = [ "immich" ]; + ensureUsers = [ + { + name = "immich"; + ensurePermissions = { + "DATABASE immich" = "ALL PRIVILEGES"; + }; + } + ]; + }; + }; + + testScript = '' + start_all() + + # wait for our service to start + machine.wait_for_unit("immich-server.service") + # machine.wait_for_open_port(${toString 1234}) + # machine.succeed("curl --fail http://localhost:${toString 1234}/") + + machine.wait_for_open_port(${toString 3000}) + machine.wait_for_open_port(${toString 3001}) + machine.wait_for_open_port(${toString 3002}) + machine.wait_for_open_port(${toString 3003}) + # TODO microservices + # TODO machine learning + # TODO web + ''; +}) diff --git a/pkgs/servers/immich/default.nix b/pkgs/servers/immich/default.nix index 6009b3e4b9f34..3c2b14f4bbbd4 100644 --- a/pkgs/servers/immich/default.nix +++ b/pkgs/servers/immich/default.nix @@ -273,6 +273,7 @@ buildNpmPackage' { ''; passthru = { + tests = { inherit (nixosTests) immich; }; inherit cli web machine-learning; updateScript = ./update.sh; };