store: Pod anti affinity to distribute stores across nodes

thanos-io · Jun 5, 2020 · 29eb9b2 · 29eb9b2
1 parent 8818368
commit 29eb9b2
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 0 deletions.
diff --git a/examples/all/manifests/thanos-store-statefulSet-with-memcached.yaml b/examples/all/manifests/thanos-store-statefulSet-with-memcached.yaml
@@ -24,6 +24,24 @@ spec:
         app.kubernetes.io/name: thanos-store
         app.kubernetes.io/version: master-2020-05-24-079ad427
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-store
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-store
+              namespaces:
+              - thanos
+              topologyKey: kubernetes.io/hostname
+            weight: 100
       containers:
       - args:
         - store

diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml
@@ -24,6 +24,24 @@ spec:
         app.kubernetes.io/name: thanos-store
         app.kubernetes.io/version: master-2020-05-24-079ad427
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-store
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-store
+              namespaces:
+              - thanos
+              topologyKey: kubernetes.io/hostname
+            weight: 100
       containers:
       - args:
         - store

diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet
@@ -47,6 +47,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
     local container = sts.mixin.spec.template.spec.containersType;
     local containerEnv = container.envType;
     local containerVolumeMount = container.volumeMountsType;
+    local affinity = sts.mixin.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecutionType;
+    local matchExpression = affinity.mixin.podAffinityTerm.labelSelector.matchExpressionsType;
 
     local c =
       container.new('thanos-store', ts.config.image) +
@@ -93,6 +95,22 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
     sts.mixin.spec.template.spec.withVolumes([
       volume.fromEmptyDir('data'),
     ]) +
+    sts.mixin.spec.template.spec.affinity.podAntiAffinity.withPreferredDuringSchedulingIgnoredDuringExecution([
+      affinity.new() +
+      affinity.withWeight(100) +
+      affinity.mixin.podAffinityTerm.withNamespaces(ts.config.namespace) +
+      affinity.mixin.podAffinityTerm.withTopologyKey('kubernetes.io/hostname') +
+      affinity.mixin.podAffinityTerm.labelSelector.withMatchExpressions([
+        matchExpression.new() +
+        matchExpression.withKey('app.kubernetes.io/name') +
+        matchExpression.withOperator('In') +
+        matchExpression.withValues([ts.statefulSet.metadata.labels['app.kubernetes.io/name']]),
+        matchExpression.new() +
+        matchExpression.withKey('app.kubernetes.io/instance') +
+        matchExpression.withOperator('In') +
+        matchExpression.withValues([ts.statefulSet.metadata.labels['app.kubernetes.io/instance']]),
+      ]),
+    ]) +
     sts.mixin.spec.selector.withMatchLabels(ts.config.podLabelSelector) +
     {
       spec+: {

diff --git a/manifests/thanos-store-statefulSet.yaml b/manifests/thanos-store-statefulSet.yaml
@@ -24,6 +24,24 @@ spec:
         app.kubernetes.io/name: thanos-store
         app.kubernetes.io/version: v0.13.0-rc.0
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-store
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-store
+              namespaces:
+              - thanos
+              topologyKey: kubernetes.io/hostname
+            weight: 100
       containers:
       - args:
         - store