kube-prometheus-stack.yaml

---
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
  name: kube-prometheus-stack
  namespace: monitoring
spec:
  interval: 15m
  chart:
    spec:
      chart: kube-prometheus-stack
      version: 66.7.1
      sourceRef:
        kind: HelmRepository
        name: prometheus-community-charts
        namespace: flux-system
  maxHistory: 3
  install:
    createNamespace: true
    remediation:
      retries: 3
  upgrade:
    cleanupOnFail: true
    remediation:
      retries: 3
  uninstall:
    keepHistory: false
  values:

    crds:
      enabled: true

    cleanPrometheusOperatorObjectNames: true

    defaultRules:
      disabled:
        HostHighCpuLoad: true
        NodeSystemSaturation: true

    alertmanager:
      config:
        global:
          resolve_timeout: 5m
        receivers:
        - name: 'null'
        - name: 'slack-notifications'
          slack_configs:
            - channel: '#notifications'
              icon_url: https://avatars3.githubusercontent.com/u/3380462
              username: 'Alertmanager'
              send_resolved: true
              title: |-
                [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ if ne .CommonAnnotations.summary ""}}{{ .CommonAnnotations.summary }} {{ else if ne .CommonAnnotations.message ""}}{{ .CommonAnnotations.message }} {{ else if ne .CommonAnnotations.description ""}}{{ .CommonAnnotations.description }} {{ else }}{{ .CommonLabels.alertname }}{{ end }}
              text: >-
                {{ range .Alerts -}}
                  *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}

                {{ if ne .Annotations.summary ""}}*Summary:* {{ .Annotations.summary }} {{ else if ne .Annotations.message ""}}*Message:* {{ .Annotations.message }} {{ else if ne .Annotations.description ""}}*Description:* {{ .Annotations.description }}{{ end }}

                *Details:*
                  {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
                  {{ end }}
                {{ end }}
        route:
          group_by: ["alertname", "job"]
          group_wait: 45s
          group_interval: 10m
          repeat_interval: 12h
          receiver: 'slack-notifications'
          routes:
            - receiver: "null" # quote
              matchers:
                - alertname =~ "Watchdog"
            - receiver: "null" # quote
              matchers:
                - alertname =~ "InfoInhibitor"
            - receiver: 'slack-notifications'
        inhibit_rules:
          - source_matchers:
              - severity = "critical"
            target_matchers:
              - severity = "warning"
            equal: ["alertname", "namespace"]

        templates: ["*.tmpl"]
      templateFiles:
        pagerduty-custom.tmpl: |-
          {{- define "pagerduty.custom.description" -}}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ if ne .CommonAnnotations.summary ""}}{{ .CommonAnnotations.summary }} {{ else if ne .CommonAnnotations.message ""}}{{ .CommonAnnotations.message }} {{ else if ne .CommonAnnotations.description ""}}{{ .CommonAnnotations.description }} {{ else }}{{ .CommonLabels.alertname }}{{ end }}{{- end -}}
      ingress:
        enabled: true
        pathType: Prefix
        ingressClassName: nginx-tailscale
        hosts:
        - prom-alert.eviljungle.com
        tls:
        - hosts:
          - prom-alert.eviljungle.com
      alertmanagerSpec:
        replicas: 1
        podAntiAffinity: hard
        storage:
          volumeClaimTemplate:
            spec:
              storageClassName: rook-ceph-block
              resources:
                requests:
                  storage: 1Gi

    grafana:
      enabled: false
      forceDeployDashboards: true
      sidecar:
        dashboards:
          annotations:
            grafana_folder: Kubernetes

    kubelet:
      enabled: true
      serviceMonitor:
        metricRelabelings:
          - action: replace
            sourceLabels: ["node"]
            targetLabel: instance

    kube-state-metrics:
      metricLabelsAllowlist: ["persistentvolumeclaims=[*]"]
      prometheus:
        monitor:
          enabled: true
          relabelings:
            - action: replace
              regex: (.*)
              replacement: $1
              sourceLabels: ["__meta_kubernetes_pod_node_name"]
              targetLabel: kubernetes_node

    kubeApiServer:
      enabled: true

    kubeControllerManager:
      enabled: false
      endpoints:
        - 10.2.0.30
      service:
        enabled: true
        port: 10257
        targetPort: 10257
      serviceMonitor:
        enabled: true
        https: true
        insecureSkipVerify: true

    kubeScheduler:
      enabled: false
      endpoints:
        - 10.2.0.30
      service:
        enabled: true
        port: 10259
        targetPort: 10259
      serviceMonitor:
        enabled: true
        https: true
        insecureSkipVerify: true

    kubeProxy:
      enabled: false
      endpoints:
        - 10.2.0.30

    kubeEtcd:
      enabled: false
      endpoints:
        - 10.2.0.30
      service:
        enabled: true
        port: 2381
        targetPort: 2381

    prometheusOperator:
      createCustomResource: true
      prometheusConfigReloader:
        resources:
          limits:
            cpu: 200m
      admissionWebhooks:
        enabled: false

    prometheus:
      ingress:
        enabled: true
        pathType: Prefix
        ingressClassName: nginx-tailscale
        hosts:
        - prom-server.eviljungle.com
        tls:
        - hosts:
          - prom-server.eviljungle.com
      thanosService:
        enabled: true
      thanosServiceMonitor:
        enabled: true
      prometheusSpec:
        replicas: 1
        replicaExternalLabelName: "replica"
        thanos:
          image: quay.io/thanos/thanos:v0.37.1
          version: v0.30.1
          objectStorageConfig:
            name: thanos-objstore-secret
            key: objstore.yml
        podAntiAffinity: hard
        ruleSelectorNilUsesHelmValues: false
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        probeSelectorNilUsesHelmValues: false
        retention: 2d
        retentionSize: 6GB
        enableAdminAPI: true
        walCompression: true
        storageSpec:
          volumeClaimTemplate:
            spec:
              storageClassName: rook-ceph-block
              resources:
                requests:
                  storage: 20Gi
        resources:
          requests:
            memory: 2000Mi
            cpu: 250m
          limits:
            memory: 5000Mi

        additionalScrapeConfigs:
        - job_name: 'ipmi'
          metrics_path: '/metrics'
          static_configs:
          - targets:
            - nas.home:9290
        - job_name: 'node'
          static_configs:
          - targets:
            - proxmox-b.home:9100
            - proxmox-c.home:9100
            - opnsense.home:9100
            - nas.home:9100
        - job_name: 'smartctl'
          metrics_path: '/metrics'
          static_configs:
          - targets:
            - nas.home:9633
        - job_name: 'upsc-exporter'
          metrics_path: '/metrics'
          static_configs:
          - targets:
            - proxmox-b.home:8081
            - proxmox-c.home:8081
        - job_name: 'zfs'
          metrics_path: '/metrics'
          static_configs:
          - targets:
            - nas.home:9134

    nodeExporter:
      enabled: true

    prometheus-node-exporter:
      prometheus:
        monitor:
          enabled: true
          # relabelings:
          #   - action: replace
          #     regex: (.*)
          #     replacement: $1
          #     sourceLabels:
          #       - __meta_kubernetes_pod_node_name
          #     targetLabel: instance
      tolerations:
      - key: "node-role.kubernetes.io/master"
        operator: "Exists"

  valuesFrom:
  - kind: Secret
    name: "kube-prometheus-stack-helm-values"
    optional: false