feat: Schedule kube-bench jobs on all nodes (#91)

Resolves: #45
aquasecurity · Jul 24, 2020 · a0ad1ad · a0ad1ad
1 parent 193c18b
commit a0ad1ad
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 7 deletions.
diff --git a/pkg/cmd/kube_bench.go b/pkg/cmd/kube_bench.go
@@ -2,14 +2,22 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"sync"
 
 	"github.com/aquasecurity/starboard/pkg/ext"
 	starboard "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned"
 	"github.com/aquasecurity/starboard/pkg/kubebench"
 	"github.com/aquasecurity/starboard/pkg/kubebench/crd"
 	"github.com/spf13/cobra"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
+)
+
+const (
+	masterNodeLabel = "node-role.kubernetes.io/master"
 )
 
 func NewKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command {
@@ -30,15 +38,38 @@ func NewKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command {
 			if err != nil {
 				return
 			}
-			report, node, err := kubebench.NewScanner(opts, kubernetesClientset).Scan(ctx)
+			starboardClientset, err := starboard.NewForConfig(config)
 			if err != nil {
 				return
 			}
-			starboardClientset, err := starboard.NewForConfig(config)
+			// List Nodes
+			nodeList, err := kubernetesClientset.CoreV1().Nodes().List(ctx, meta.ListOptions{})
 			if err != nil {
+				err = fmt.Errorf("list nodes: %w", err)
 				return
 			}
-			err = crd.NewWriter(ext.NewSystemClock(), starboardClientset).Write(ctx, report, node)
+			var wg sync.WaitGroup
+			wg.Add(len(nodeList.Items))
+			for _, nodeItem := range nodeList.Items {
+				target := "node"
+				if _, ok := nodeItem.Labels[masterNodeLabel]; ok {
+					target = "master"
+				}
+				nodeName := nodeItem.Name
+				go func() {
+					klog.V(3).Infof("Node name: %s Label:%s", nodeName, target)
+					report, node, err := kubebench.NewScanner(opts, kubernetesClientset).Scan(ctx, nodeName, target, &wg)
+
+					if err != nil {
+						klog.Warningf("Node name: %s Error NewScanner: %s", nodeName, err)
+					}
+					err = crd.NewWriter(ext.NewSystemClock(), starboardClientset).Write(ctx, report, node)
+					if err != nil {
+						klog.Warningf("Node name: %s Error NewWriter: %s", nodeName, err)
+					}
+				}()
+			}
+			wg.Wait()
 			return
 		},
 	}

diff --git a/pkg/kubebench/scanner.go b/pkg/kubebench/scanner.go
@@ -3,6 +3,7 @@ package kubebench
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"github.com/aquasecurity/starboard/pkg/scanners"
 
@@ -26,6 +27,7 @@ import (
 const (
 	kubeBenchVersion       = "0.3.0"
 	kubeBenchContainerName = "kube-bench"
+	masterNodeLabel        = "node-role.kubernetes.io/master"
 )
 
 var (
@@ -49,9 +51,10 @@ func NewScanner(opts kube.ScannerOpts, clientset kubernetes.Interface) *Scanner
 	}
 }
 
-func (s *Scanner) Scan(ctx context.Context) (report starboard.CISKubeBenchOutput, node *core.Node, err error) {
+func (s *Scanner) Scan(ctx context.Context, nodeName string, target string, wg *sync.WaitGroup) (report starboard.CISKubeBenchOutput, node *core.Node, err error) {
+	defer wg.Done()
 	// 1. Prepare descriptor for the Kubernetes Job which will run kube-bench
-	job := s.prepareKubeBenchJob()
+	job := s.prepareKubeBenchJob(nodeName, target)
 
 	// 2. Run the prepared Job and wait for its completion or failure
 	err = runner.New().Run(ctx, kube.NewRunnableJob(s.clientset, job))
@@ -100,10 +103,11 @@ func (s *Scanner) Scan(ctx context.Context) (report starboard.CISKubeBenchOutput
 	}
 
 	node, err = s.clientset.CoreV1().Nodes().Get(ctx, kubeBenchPod.Spec.NodeName, meta.GetOptions{})
+
 	return
 }
 
-func (s *Scanner) prepareKubeBenchJob() *batch.Job {
+func (s *Scanner) prepareKubeBenchJob(nodeName string, target string) *batch.Job {
 	return &batch.Job{
 		ObjectMeta: meta.ObjectMeta{
 			Name:      uuid.New().String(),
@@ -125,6 +129,7 @@ func (s *Scanner) prepareKubeBenchJob() *batch.Job {
 				Spec: core.PodSpec{
 					RestartPolicy: core.RestartPolicyNever,
 					HostPID:       true,
+					NodeName:      nodeName,
 					Volumes: []core.Volume{
 						{
 							Name: "var-lib-etcd",
@@ -173,7 +178,7 @@ func (s *Scanner) prepareKubeBenchJob() *batch.Job {
 							Image:                    kubeBenchContainerImage,
 							ImagePullPolicy:          core.PullIfNotPresent,
 							TerminationMessagePolicy: core.TerminationMessageFallbackToLogsOnError,
-							Command:                  []string{"kube-bench"},
+							Command:                  []string{"kube-bench", target},
 							Args:                     []string{"--json"},
 							Resources: core.ResourceRequirements{
 								Limits: core.ResourceList{