feat: Add tolerations to vulnerability scan jobs (#586)

Resolves: #538
aquasecurity · May 19, 2021 · edb7522 · edb7522
1 parent 277bb6b
commit edb7522
Show file tree

Hide file tree

Showing 8 changed files with 135 additions and 41 deletions.
diff --git a/docs/settings.md b/docs/settings.md
@@ -56,27 +56,28 @@ The following tables list available configuration settings with their default va
     used if `vulnerabilityReports.scanner` is set to `Trivy`). Check
     [integrations](./integrations/vulnerability-scanners/index.md) page to see example configuration settings for common use cases.
 
-| CONFIGMAP KEY                  | DEFAULT                                                | DESCRIPTION |
-| ------------------------------ | ------------------------------------------------------ | ----------- |
-| `vulnerabilityReports.scanner` | `Trivy`                                                | The name of the plugin that generates vulnerability reports. Either `Trivy` or `Aqua`. |
-| `configAuditReports.scanner`   | `Polaris`                                              | The name of the plugin that generates config audit reports. Either `Polaris` or `Conftest`. |
-| `trivy.httpProxy`              | N/A                                                    | The HTTP proxy used by Trivy to download the vulnerabilities database from GitHub. |
-| `trivy.httpsProxy`             | N/A                                                    | The HTTPS proxy used by Trivy to download the vulnerabilities database from GitHub. |
-| `trivy.noProxy`                | N/A                                                    | A comma separated list of IPs and domain names that are not subject to proxy settings. |
-| `trivy.severity`               | `UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL`                     | A comma separated list of severity levels reported by Trivy |
-| `trivy.imageRef`               | `docker.io/aquasec/trivy:0.16.0`                       | Trivy image reference |
-| `trivy.mode`                   | `Standalone`                                           | Trivy client mode. Either `Standalone` or `ClientServer`. Depending on the active mode other settings might be applicable or required. |
-| `trivy.serverURL`              | N/A                                                    | The endpoint URL of the Trivy server. Required in `ClientServer` mode. |
-| `trivy.serverTokenHeader`      | `Trivy-Token`                                          | The name of the HTTP header to send the authentication token to Trivy server. Only application in `ClientServer` mode when `trivy.serverToken` is specified. |
-| `trivy.insecureRegistry.<id>`  | N/A                                                    | The registry to which insecure connections are allowed. There can be multiple registries with different registry `<id>`. |
-| `aqua.imageRef`                | `docker.io/aquasec/scanner:5.3`                        | Aqua scanner image reference. The tag determines the version of the `scanner` binary executable and it must be compatible with version of Aqua console. |
-| `aqua.serverURL`               | N/A                                                    | The endpoint URL of Aqua management console |
-| `kube-bench.imageRef`          | `docker.io/aquasec/kube-bench:0.5.0`                   | kube-bench image reference |
-| `kube-hunter.imageRef`         | `docker.io/aquasec/kube-hunter:0.4.1`                  | kube-hunter image reference |
-| `kube-hunter.quick`            | `"false"`                                              | Whether to use kube-hunter's "quick" scanning mode (subnet 24). Set to `"true"` to enable. |
-| `polaris.imageRef`             | `quay.io/fairwinds/polaris:3.2`                        | Polaris image reference |
-| `polaris.config.yaml`          | [Check the default value here][default-polaris-config] | Polaris configuration file |
-| `conftest.imageRef`            | `docker.io/openpolicyagent/conftest:v0.25.0`           | Conftest image reference |
+| CONFIGMAP KEY                         | DEFAULT                                                                           | DESCRIPTION |
+| ------------------------------- | ------------------------------------------------------ | ----------- |
+| `vulnerabilityReports.scanner`  | `Trivy`                                                | The name of the plugin that generates vulnerability reports. Either `Trivy` or `Aqua`. |
+| `configAuditReports.scanner`    | `Polaris`                                              | The name of the plugin that generates config audit reports. Either `Polaris` or `Conftest`. |
+| `trivy.httpProxy`               | N/A                                                    | The HTTP proxy used by Trivy to download the vulnerabilities database from GitHub. |
+| `trivy.httpsProxy`              | N/A                                                    | The HTTPS proxy used by Trivy to download the vulnerabilities database from GitHub. |
+| `trivy.noProxy`                 | N/A                                                    | A comma separated list of IPs and domain names that are not subject to proxy settings. |
+| `trivy.severity`                | `UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL`                     | A comma separated list of severity levels reported by Trivy |
+| `trivy.imageRef`                | `docker.io/aquasec/trivy:0.16.0`                       | Trivy image reference |
+| `trivy.mode`                    | `Standalone`                                           | Trivy client mode. Either `Standalone` or `ClientServer`. Depending on the active mode other settings might be applicable or required. |
+| `trivy.serverURL`               | N/A                                                    | The endpoint URL of the Trivy server. Required in `ClientServer` mode. |
+| `trivy.serverTokenHeader`       | `Trivy-Token`                                          | The name of the HTTP header to send the authentication token to Trivy server. Only application in `ClientServer` mode when `trivy.serverToken` is specified. |
+| `trivy.insecureRegistry.<id>`   | N/A                                                    | The registry to which insecure connections are allowed. There can be multiple registries with different registry `<id>`. |
+| `aqua.imageRef`                 | `docker.io/aquasec/scanner:5.3`                        | Aqua scanner image reference. The tag determines the version of the `scanner` binary executable and it must be compatible with version of Aqua console. |
+| `aqua.serverURL`                | N/A                                                    | The endpoint URL of Aqua management console |
+| `kube-bench.imageRef`           | `docker.io/aquasec/kube-bench:0.5.0`                   | kube-bench image reference |
+| `kube-hunter.imageRef`          | `docker.io/aquasec/kube-hunter:0.4.1`                  | kube-hunter image reference |
+| `kube-hunter.quick`             | `"false"`                                              | Whether to use kube-hunter's "quick" scanning mode (subnet 24). Set to `"true"` to enable. |
+| `polaris.imageRef`              | `quay.io/fairwinds/polaris:3.2`                        | Polaris image reference |
+| `polaris.config.yaml`           | [Check the default value here][default-polaris-config] | Polaris configuration file |
+| `conftest.imageRef`             | `docker.io/openpolicyagent/conftest:v0.25.0`           | Conftest image reference |
+| `scanJob.tolerations`           |                                                        | JSON representation of the [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration) to be applied to the vulnerability scanner pods so that they can run on nodes with matching taints. Example: `'[{"key":"key1", "operator":"Equal", "value":"value1", "effect":"NoSchedule"}]'`|
 
 | SECRET KEY                  | DESCRIPTION |
 | --------------------------- | ----------- |

diff --git a/pkg/cmd/scan_vulnerabilities.go b/pkg/cmd/scan_vulnerabilities.go
@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-
 	"github.com/aquasecurity/starboard/pkg/plugin"
 	"github.com/aquasecurity/starboard/pkg/starboard"
 	"github.com/aquasecurity/starboard/pkg/vulnerabilityreport"
@@ -108,7 +107,7 @@ func ScanVulnerabilityReports(buildInfo starboard.BuildInfo, cf *genericclioptio
 		if err != nil {
 			return err
 		}
-		scanner := vulnerabilityreport.NewScanner(kubeClientset, kubeClient, opts, plugin)
+		scanner := vulnerabilityreport.NewScanner(starboardConfig, kubeClientset, kubeClient, opts, plugin)
 		reports, err := scanner.Scan(ctx, workload)
 		if err != nil {
 			return err

diff --git a/pkg/operator/controller/vulnerabilityreport.go b/pkg/operator/controller/vulnerabilityreport.go
@@ -38,6 +38,7 @@ type VulnerabilityReportReconciler struct {
 	kube.SecretsReader
 	vulnerabilityreport.Plugin
 	vulnerabilityreport.ReadWriter
+	starboard.ConfigData
 }
 
 func (r *VulnerabilityReportReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -215,6 +216,12 @@ func (r *VulnerabilityReportReconciler) submitScanJob(ctx context.Context, spec
 		return err
 	}
 
+	scanJobTolerations, err := r.ConfigData.GetScanJobTolerations()
+	if err != nil {
+		return err
+	}
+	templateSpec.Tolerations = append(templateSpec.Tolerations, scanJobTolerations...)
+
 	containerImagesAsJSON, err := images.AsJSON()
 	if err != nil {
 		return err

diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go
@@ -143,6 +143,7 @@ func Start(ctx context.Context, buildInfo starboard.BuildInfo, operatorConfig et
 		if err = (&controller.VulnerabilityReportReconciler{
 			Logger:         ctrl.Log.WithName("reconciler").WithName("vulnerabilityreport"),
 			Config:         operatorConfig,
+			ConfigData:     starboardConfig,
 			Client:         mgr.GetClient(),
 			ObjectResolver: objectResolver,
 			LimitChecker:   limitChecker,

diff --git a/pkg/starboard/config.go b/pkg/starboard/config.go
@@ -2,6 +2,7 @@ package starboard
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -296,6 +297,16 @@ func GetDefaultConftestConfig() map[string]string {
 	return map[string]string{}
 }
 
+func (c ConfigData) GetScanJobTolerations() ([]corev1.Toleration, error) {
+	scanJobTolerations := []corev1.Toleration{}
+	if c["scanJob.tolerations"] == "" {
+		return scanJobTolerations, nil
+	}
+	err := json.Unmarshal([]byte(c["scanJob.tolerations"]), &scanJobTolerations)
+
+	return scanJobTolerations, err
+}
+
 func (c ConfigData) GetVulnerabilityReportsScanner() (Scanner, error) {
 	var ok bool
 	var value string

diff --git a/pkg/starboard/config_test.go b/pkg/starboard/config_test.go
@@ -602,3 +602,70 @@ func TestConfigData_GetTrivyInsecureRegistries(t *testing.T) {
 		})
 	}
 }
+
+func TestGetScanJobTolerations(t *testing.T) {
+	testcases := []struct {
+		name        string
+		config      starboard.ConfigData
+		expected    []corev1.Toleration
+		expectError string
+	}{
+		{
+			name:     "no scanJob.tolerations in ConfigData",
+			config:   starboard.ConfigData{},
+			expected: []corev1.Toleration{},
+		},
+		{
+			name:        "scanJob.tolerations value is not json",
+			config:      starboard.ConfigData{"scanJob.tolerations": `lolwut`},
+			expected:    []corev1.Toleration{},
+			expectError: "invalid character 'l' looking for beginning of value",
+		},
+		{
+			name:     "empty JSON array",
+			config:   starboard.ConfigData{"scanJob.tolerations": `[]`},
+			expected: []corev1.Toleration{},
+		},
+		{
+			name: "one valid toleration",
+			config: starboard.ConfigData{
+				"scanJob.tolerations": `[{"key":"key1","operator":"Equal","value":"value1","effect":"NoSchedule"}]`},
+			expected: []corev1.Toleration{{
+				Key:      "key1",
+				Operator: "Equal",
+				Value:    "value1",
+				Effect:   "NoSchedule",
+			}},
+		},
+		{
+			name: "multiple valid tolerations",
+			config: starboard.ConfigData{
+				"scanJob.tolerations": `[{"key":"key1","operator":"Equal","value":"value1","effect":"NoSchedule"},
+					  {"key":"key2","operator":"Equal","value":"value2","effect":"NoSchedule"}]`},
+			expected: []corev1.Toleration{
+				{
+					Key:      "key1",
+					Operator: "Equal",
+					Value:    "value1",
+					Effect:   "NoSchedule",
+				},
+				{
+					Key:      "key2",
+					Operator: "Equal",
+					Value:    "value2",
+					Effect:   "NoSchedule",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		got, err := tc.config.GetScanJobTolerations()
+		if tc.expectError != "" {
+			assert.Error(t, err, "unexpected end of JSON input", tc.name)
+		} else {
+			assert.NoError(t, err, tc.name)
+		}
+		assert.Equal(t, tc.expected, got, tc.name)
+	}
+}
diff --git a/pkg/vulnerabilityreport/scanner.go b/pkg/vulnerabilityreport/scanner.go
@@ -3,7 +3,6 @@ package vulnerabilityreport
 import (
 	"context"
 	"fmt"
-
 	"github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1"
 	"github.com/aquasecurity/starboard/pkg/docker"
 	"github.com/aquasecurity/starboard/pkg/ext"
@@ -23,12 +22,13 @@ import (
 // Scanner is a template for running static vulnerability scanners that implement
 // the Plugin interface.
 type Scanner struct {
-	scheme         *runtime.Scheme
-	clientset      kubernetes.Interface
-	opts           kube.ScannerOpts
-	plugin         Plugin
-	objectResolver *kube.ObjectResolver
-	logsReader     kube.LogsReader
+	scheme          *runtime.Scheme
+	clientset       kubernetes.Interface
+	opts            kube.ScannerOpts
+	plugin          Plugin
+	objectResolver  *kube.ObjectResolver
+	logsReader      kube.LogsReader
+	starboardConfig starboard.ConfigData
 	ext.IDGenerator
 	kube.SecretsReader
 }
@@ -38,20 +38,22 @@ type Scanner struct {
 // which is performed by running a Kubernetes job, and knows how to convert logs
 // to instances of v1alpha1.VulnerabilityReport.
 func NewScanner(
+	starboardConfig starboard.ConfigData,
 	clientset kubernetes.Interface,
 	client client.Client,
 	opts kube.ScannerOpts,
 	plugin Plugin,
 ) *Scanner {
 	return &Scanner{
-		scheme:         client.Scheme(),
-		clientset:      clientset,
-		opts:           opts,
-		plugin:         plugin,
-		objectResolver: &kube.ObjectResolver{Client: client},
-		logsReader:     kube.NewLogsReader(clientset),
-		IDGenerator:    ext.NewGoogleUUIDGenerator(),
-		SecretsReader:  kube.NewSecretsReader(client),
+		scheme:          client.Scheme(),
+		clientset:       clientset,
+		opts:            opts,
+		plugin:          plugin,
+		objectResolver:  &kube.ObjectResolver{Client: client},
+		logsReader:      kube.NewLogsReader(clientset),
+		starboardConfig: starboardConfig,
+		IDGenerator:     ext.NewGoogleUUIDGenerator(),
+		SecretsReader:   kube.NewSecretsReader(client),
 	}
 }
 
@@ -62,6 +64,7 @@ func NewScanner(
 // to instances of v1alpha1.VulnerabilityReport by delegating such transformation
 // logic also to the Plugin.
 func (s *Scanner) Scan(ctx context.Context, workload kube.Object) ([]v1alpha1.VulnerabilityReport, error) {
+
 	klog.V(3).Infof("Getting Pod template for workload: %v", workload)
 
 	owner, err := s.objectResolver.GetObjectFromPartialObject(ctx, workload)
@@ -80,7 +83,7 @@ func (s *Scanner) Scan(ctx context.Context, workload kube.Object) ([]v1alpha1.Vu
 		return nil, err
 	}
 
-	job, secrets, err := s.prepareScanJob(workload, spec, credentials)
+	job, secrets, err := s.prepareScanJob(workload, spec, credentials, s.starboardConfig)
 	if err != nil {
 		return nil, fmt.Errorf("preparing scan job: %w", err)
 	}
@@ -115,12 +118,18 @@ func (s *Scanner) getCredentials(ctx context.Context, spec corev1.PodSpec, ns st
 	return kube.MapContainerNamesToDockerAuths(kube.GetContainerImagesFromPodSpec(spec), imagePullSecrets)
 }
 
-func (s *Scanner) prepareScanJob(workload kube.Object, spec corev1.PodSpec, credentials map[string]docker.Auth) (*batchv1.Job, []*corev1.Secret, error) {
+func (s *Scanner) prepareScanJob(workload kube.Object, spec corev1.PodSpec, credentials map[string]docker.Auth, starboardConfigData starboard.ConfigData) (*batchv1.Job, []*corev1.Secret, error) {
 	templateSpec, secrets, err := s.plugin.GetScanJobSpec(spec, credentials)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	scanJobTolerations, err := starboardConfigData.GetScanJobTolerations()
+	if err != nil {
+		return nil, nil, err
+	}
+	templateSpec.Tolerations = append(templateSpec.Tolerations, scanJobTolerations...)
+
 	templateSpec.ServiceAccountName = starboard.ServiceAccountName
 
 	containerImagesAsJSON, err := kube.GetContainerImagesFromPodSpec(spec).AsJSON()

diff --git a/pkg/vulnerabilityreport/scanner_test.go b/pkg/vulnerabilityreport/scanner_test.go