Merge pull request #244233 from oddlama/init-typesense-bin

nixos/doc/manual/release-notes/rl-2311.section.md

@@ -18,6 +18,8 @@
 
 - [GoToSocial](https://gotosocial.org/), an ActivityPub social network server, written in Golang. Available as [services.gotosocial](#opt-services.gotosocial.enable).
 
+- [Typesense](https://github.com/typesense/typesense), a fast, typo-tolerant search engine for building delightful search experiences. Available as [services.typesense](#opt-services.typesense.enable).
+
 - [Anuko Time Tracker](https://github.com/anuko/timetracker), a simple, easy to use, open source time tracking system. Available as [services.anuko-time-tracker](#opt-services.anuko-time-tracker.enable).
 
 - [sitespeed-io](https://sitespeed.io), a tool that can generate metrics (timings, diagnostics) for websites. Available as [services.sitespeed-io](#opt-services.sitespeed-io.enable).

nixos/modules/module-list.nix

@@ -1110,6 +1110,7 @@
   ./services/search/meilisearch.nix
   ./services/search/opensearch.nix
   ./services/search/qdrant.nix
+  ./services/search/typesense.nix
   ./services/security/aesmd.nix
   ./services/security/authelia.nix
   ./services/security/certmgr.nix

nixos/modules/services/search/typesense.nix

@@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }: let
+  inherit
+    (lib)
+    concatMapStringsSep
+    generators
+    mdDoc
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    optionalString
+    types
+    ;
+
+  cfg = config.services.typesense;
+  settingsFormatIni = pkgs.formats.ini {
+    listToValue = concatMapStringsSep " " (generators.mkValueStringDefault { });
+    mkKeyValue = generators.mkKeyValueDefault
+      {
+        mkValueString = v:
+          if v == null then ""
+          else generators.mkValueStringDefault { } v;
+      }
+      "=";
+  };
+  configFile = settingsFormatIni.generate "typesense.ini" cfg.settings;
+in {
+  options.services.typesense = {
+    enable = mkEnableOption "typesense";
+    package = mkPackageOption pkgs "typesense" {};
+
+    apiKeyFile = mkOption {
+      type = types.path;
+      description = ''
+        Sets the admin api key for typesense. Always use this option
+        instead of {option}`settings.server.api-key` to prevent the key
+        from being written to the world-readable nix store.
+      '';
+    };
+
+    settings = mkOption {
+      description = mdDoc "Typesense configuration. Refer to [the documentation](https://typesense.org/docs/0.24.1/api/server-configuration.html) for supported values.";
+      default = {};
+      type = types.submodule {
+        freeformType = settingsFormatIni.type;
+        options.server = {
+          data-dir = mkOption {
+            type = types.str;
+            default = "/var/lib/typesense";
+            description = mdDoc "Path to the directory where data will be stored on disk.";
+          };
+
+          api-address = mkOption {
+            type = types.str;
+            description = mdDoc "Address to which Typesense API service binds.";
+          };
+
+          api-port = mkOption {
+            type = types.port;
+            default = 8108;
+            description = mdDoc "Port on which the Typesense API service listens.";
+          };
+        };
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.typesense = {
+      description = "Typesense search engine";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      script = ''
+        export TYPESENSE_API_KEY=$(cat ${cfg.apiKeyFile})
+        exec ${cfg.package}/bin/typesense-server --config ${configFile}
+      '';
+
+      serviceConfig = {
+        Restart = "on-failure";
+        DynamicUser = true;
+        User = "typesense";
+        Group = "typesense";
+
+        StateDirectory = "typesense";
+        StateDirectoryMode = "0700";
+
+        # Hardening
+        CapabilityBoundingSet = "";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateUsers = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProcSubset = "pid";
+        ProtectSystem = "strict";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_UNIX"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+        UMask = "0077";
+      };
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -808,6 +808,7 @@ in {
   turbovnc-headless-server = handleTest ./turbovnc-headless-server.nix {};
   tuxguitar = handleTest ./tuxguitar.nix {};
   twingate = runTest ./twingate.nix;
+  typesense = handleTest ./typesense.nix {};
   ucarp = handleTest ./ucarp.nix {};
   udisks2 = handleTest ./udisks2.nix {};
   ulogd = handleTest ./ulogd.nix {};

nixos/tests/typesense.nix

@@ -0,0 +1,23 @@
+import ./make-test-python.nix ({ pkgs, ... }: let
+  testPort = 8108;
+in {
+  name = "typesense";
+  meta.maintainers = with pkgs.lib.maintainers; [ oddlama ];
+
+  nodes.machine = { ... }: {
+    services.typesense = {
+      enable = true;
+      apiKeyFile = pkgs.writeText "typesense-api-key" "dummy";
+      settings.server = {
+        api-port = testPort;
+        api-address = "0.0.0.0";
+      };
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("typesense.service")
+    machine.wait_for_open_port(${toString testPort})
+    assert machine.succeed("curl --fail http://localhost:${toString testPort}/health") == '{"ok":true}'
+  '';
+})

pkgs/servers/search/typesense/default.nix

@@ -0,0 +1,64 @@
+{ lib
+, stdenv
+, fetchurl
+, autoPatchelfHook
+, nixosTests
+}:
+let
+  inherit (stdenv.hostPlatform) system;
+  throwSystem = throw "Unsupported system: ${system}";
+
+  sources = lib.importJSON ./sources.json;
+  platform = sources.platforms.${system} or throwSystem;
+  inherit (sources) version;
+  inherit (platform) arch hash;
+in
+stdenv.mkDerivation {
+  pname = "typesense";
+  inherit version;
+  src = fetchurl {
+    url = "https://dl.typesense.org/releases/${version}/typesense-server-${version}-${arch}.tar.gz";
+    inherit hash;
+  };
+
+  nativeBuildInputs = [
+    autoPatchelfHook
+  ];
+
+  # The tar.gz contains no subdirectory
+  sourceRoot = ".";
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp $sourceRoot/typesense-server $out/bin
+  '';
+
+  passthru = {
+    tests = { inherit (nixosTests) typesense; };
+    updateScript = ./update.sh;
+  };
+
+  meta = with lib; {
+    homepage = "https://typesense.org";
+    description = "Typesense is a fast, typo-tolerant search engine for building delightful search experiences.";
+    license = licenses.gpl3;
+    # There has been an attempt at building this from source, which were deemed
+    # unfeasible at the time of writing this (July 2023) for the following reasons.
+    # - Pre 0.25 would have been possible, but typesense has switched to bazel for 0.25+,
+    #   so the build would break immediately next version
+    # - The new bazel build has many issues, only some of which were fixable:
+    #   - preBuild requires export LANG="C.UTF-8", since onxxruntime contains a
+    #     unicode file path that is handled incorrectly and otherwise leads to a build failure
+    #   - bazel downloads extensions to the build systems at build time which have
+    #     invalid shebangs that need to be fixed by patching rules_foreign_cc through
+    #     bazel (so a patch in nix that adds a patch to the bazel WORKSPACE)
+    #   - WORKSPACE has to be patched to use system cmake and ninja instead of downloaded toolchains
+    #   - The cmake dependencies that are pulled in via bazel at build time will
+    #     try to download stuff via cmake again, which is not possible in the sandbox.
+    #     This is where I stopped trying for now.
+    # XXX: retry once typesense has officially released their bazel based build.
+    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
+    platforms = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" ];
+    maintainers = with maintainers; [ oddlama ];
+  };
+}

pkgs/servers/search/typesense/sources.json

@@ -0,0 +1,17 @@
+{
+  "version": "0.24.1",
+  "platforms": {
+    "aarch64-linux": {
+      "arch": "linux-arm64",
+      "hash": "sha256-TI/bjGqyEZpGDq1F9MBaDypm5XDTlsw9OGd3lIn7JCI="
+    },
+    "x86_64-linux": {
+      "arch": "linux-amd64",
+      "hash": "sha256-bmvje439QYivV96fjnEXblYJnSk8C916OwVeK2n/QR8="
+    },
+    "x86_64-darwin": {
+      "arch": "darwin-amd64",
+      "hash": "sha256-24odPFqHWQoGXXXDLxvMDjCRu81Y+I5QOdK/KLdeH5o="
+    }
+  }
+}

pkgs/servers/search/typesense/update.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p curl jq nix-prefetch common-updater-scripts nix coreutils
+# shellcheck shell=bash
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+old_version=$(jq -r ".version" sources.json || echo -n "0.0.1")
+version=$(curl -s "https://api.github.com/repos/typesense/typesense/releases/latest" | jq -r ".tag_name")
+version="${version#v}"
+
+if [[ "$old_version" == "$version" ]]; then
+    echo "Already up to date!"
+    exit 0
+fi
+
+declare -A platforms=(
+    [aarch64-linux]="linux-arm64"
+    [x86_64-darwin]="darwin-amd64"
+    [x86_64-linux]="linux-amd64"
+)
+
+sources_tmp="$(mktemp)"
+cat <<EOF > "$sources_tmp"
+{
+  "version": "$version",
+  "platforms": {}
+}
+EOF
+
+for platform in "${!platforms[@]}"; do
+    arch="${platforms[$platform]}"
+    url="https://dl.typesense.org/releases/${version}/typesense-server-${version}-${arch}.tar.gz"
+    sha256hash="$(nix-prefetch-url --type sha256 "$url")"
+    hash="$(nix hash to-sri --type sha256 "$sha256hash")"
+    echo "$(jq --arg arch "$arch" \
+      --arg platform "$platform" \
+      --arg hash "$hash" \
+      '.platforms += {($platform): {arch: $arch, hash: $hash}}' \
+      "$sources_tmp")" > "$sources_tmp"
+done
+
+cp "$sources_tmp" sources.json

pkgs/top-level/all-packages.nix

@@ -13746,6 +13746,8 @@ with pkgs;
 
   tydra = callPackage ../tools/misc/tydra { };
 
+  typesense = callPackage ../servers/search/typesense { };
+
   typos = callPackage ../development/tools/typos { };
 
   typst = callPackage ../tools/typesetting/typst { };