Skip to content

Commit

Permalink
fix(Conftest): Error: configmap references non-existent config key (#511
Browse files Browse the repository at this point in the history 
)

When we created a scan Job for config audits it referred directly
to the starboard-conftest-config ConfigMap. However, the ConfigMap
might have changed before the Job was run. When we removed a policy
from the ConfigMap the Job got stuck in the Pending state due to
non-existent config key error.

The solution is to copy policies to a temporary Secret and associate
it with the scan Job so we're agnostic to changes of the
starboard-conftest-config ConfigMap.

Resolves: #509

Signed-off-by: Daniel Pacak <[email protected]>
  • Loading branch information
@danielpacak
danielpacak authored Apr 20, 2021
1 parent 118f987 commit 0ccb209
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 133 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/google/go-containerregistry v0.1.1
github.com/google/uuid v1.1.1
github.com/hashicorp/go-version v1.2.0
github.com/onsi/ginkgo v1.15.2
github.com/onsi/ginkgo v1.16.1
github.com/onsi/gomega v1.11.0
github.com/spf13/cobra v1.1.1
github.com/spf13/pflag v1.0.5
Expand Down
6 changes: 4 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
Expand Down Expand Up @@ -592,8 +594,8 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.15.2 h1:l77YT15o814C2qVL47NOyjV/6RbaP7kKdrvZnxQ3Org=
github.com/onsi/ginkgo v1.15.2/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o=
github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand Down
212 changes: 104 additions & 108 deletions pkg/plugin/conftest/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package conftest

import (
"encoding/json"
"fmt"
"io"
"strings"

Expand All @@ -20,6 +21,11 @@ import (

const (
conftestContainerName = "conftest"
policyPrefix = "conftest.policy."
workloadKey = "starboard.workload.yaml"
defaultCheckCategory = "Security"
severityWarning = "WARNING"
severityDanger = "DANGER"
)

type Config interface {
Expand All @@ -32,8 +38,8 @@ type plugin struct {
config Config
}

// NewPlugin constructs a new configauditreport.Plugin, which is using an
// official Conftest container image to audit Kubernetes workloads.
// NewPlugin constructs a new configauditreport.Plugin, which is using
// the upstream Conftest container image to audit K8s workloads.
func NewPlugin(idGenerator ext.IDGenerator, clock ext.Clock, config Config) configauditreport.Plugin {
return &plugin{
idGenerator: idGenerator,
Expand All @@ -53,159 +59,149 @@ func (p *plugin) GetConfigHash(ctx starboard.PluginContext) (string, error) {
func (p *plugin) GetScanJobSpec(ctx starboard.PluginContext, obj client.Object) (corev1.PodSpec, []*corev1.Secret, error) {
imageRef, err := p.config.GetConftestImageRef()
if err != nil {
return corev1.PodSpec{}, nil, err
return corev1.PodSpec{}, nil, fmt.Errorf("getting image reference: %w", err)
}

var secrets []*corev1.Secret

workloadAsYAML, err := yaml.Marshal(obj)
config, err := ctx.GetConfig()
if err != nil {
return corev1.PodSpec{}, nil, err
}

secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: p.idGenerator.GenerateID(),
Namespace: ctx.GetNamespace(),
},
StringData: map[string]string{
"workload.yaml": string(workloadAsYAML),
},
return corev1.PodSpec{}, nil, fmt.Errorf("getting config: %w", err)
}

secrets = append(secrets, secret)

policies, err := p.getPolicies(ctx)
if err != nil {
return corev1.PodSpec{}, nil, err
}
policies := p.getPolicies(config)

var volumeMounts []corev1.VolumeMount
var volumeItems []corev1.KeyToPath

for _, control := range policies {
secretName := p.idGenerator.GenerateID()
secretData := make(map[string]string)

for policy, script := range policies {
policyName := strings.TrimPrefix(policy, policyPrefix)

// Copy policies so even if the starboard-conftest-config ConfigMap has changed
// before the scan Job is run, it won't fail with references to non-existent config key error.
secretData[policy] = script

volumeItems = append(volumeItems, corev1.KeyToPath{
Key: "conftest.policy." + control,
Path: control,
Key: policy,
Path: policyName,
})

volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "policies",
MountPath: "/project/policy/" + control,
SubPath: control,
Name: secretName,
MountPath: "/project/policy/" + policyName,
SubPath: policyName,
ReadOnly: true,
})

}

workloadAsYAML, err := yaml.Marshal(obj)
if err != nil {
return corev1.PodSpec{}, nil, fmt.Errorf("marshalling workload: %w", err)
}

secretData[workloadKey] = string(workloadAsYAML)

volumeItems = append(volumeItems, corev1.KeyToPath{
Key: workloadKey,
Path: "workload.yaml",
})

volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: secret.Name,
Name: secretName,
MountPath: "/project/workload.yaml",
SubPath: "workload.yaml",
ReadOnly: true,
})

return corev1.PodSpec{
ServiceAccountName: ctx.GetServiceAccountName(),
AutomountServiceAccountToken: pointer.BoolPtr(false),
RestartPolicy: corev1.RestartPolicyNever,
Affinity: starboard.LinuxNodeAffinity(),
Volumes: []corev1.Volume{
{
Name: "policies",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: starboard.GetPluginConfigMapName(ctx.GetName()),
ServiceAccountName: ctx.GetServiceAccountName(),
AutomountServiceAccountToken: pointer.BoolPtr(false),
RestartPolicy: corev1.RestartPolicyNever,
Affinity: starboard.LinuxNodeAffinity(),
Volumes: []corev1.Volume{
{
Name: secretName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secretName,
Items: volumeItems,
},
Items: volumeItems,
},
},
},
{
Name: secret.Name,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.Name,
Containers: []corev1.Container{
{
Name: conftestContainerName,
Image: imageRef,
ImagePullPolicy: corev1.PullIfNotPresent,
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("300m"),
corev1.ResourceMemory: resource.MustParse("300M"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("50M"),
},
},
},
},
},
Containers: []corev1.Container{
{
Name: conftestContainerName,
Image: imageRef,
ImagePullPolicy: corev1.PullIfNotPresent,
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("300m"),
corev1.ResourceMemory: resource.MustParse("300M"),
VolumeMounts: volumeMounts,
Command: []string{
"sh",
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("50M"),
// TODO Follow up with Conftest maintainers to allow returning 0 exit code in case of failures
Args: []string{
"-c",
"conftest test --output json --all-namespaces --policy /project/policy /project/workload.yaml || true",
},
},
VolumeMounts: volumeMounts,
Command: []string{
"sh",
},
// TODO Follow up with Conftest maintainers to allow returning 0 exit code in case of failures
Args: []string{
"-c",
"conftest test --output json --all-namespaces --policy /project/policy /project/workload.yaml || true",
},
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.BoolPtr(false),
AllowPrivilegeEscalation: pointer.BoolPtr(false),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"all"},
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.BoolPtr(false),
AllowPrivilegeEscalation: pointer.BoolPtr(false),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"all"},
},
ReadOnlyRootFilesystem: pointer.BoolPtr(true),
},
ReadOnlyRootFilesystem: pointer.BoolPtr(true),
},
},
},
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: pointer.Int64Ptr(1000),
RunAsGroup: pointer.Int64Ptr(1000),
SeccompProfile: &corev1.SeccompProfile{
Type: corev1.SeccompProfileTypeRuntimeDefault,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: pointer.Int64Ptr(1000),
RunAsGroup: pointer.Int64Ptr(1000),
SeccompProfile: &corev1.SeccompProfile{
Type: corev1.SeccompProfileTypeRuntimeDefault,
},
},
},
}, secrets, nil

}, []*corev1.Secret{{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: ctx.GetNamespace(),
},
StringData: secretData,
}}, nil
}

func (p *plugin) getPolicies(ctx starboard.PluginContext) ([]string, error) {
cm, err := ctx.GetConfig()
if err != nil {
return nil, err
}
func (p *plugin) getPolicies(cm *corev1.ConfigMap) map[string]string {
policies := make(map[string]string)

var policies []string

for key := range cm.Data {
if !strings.HasPrefix(key, "conftest.policy.") {
for key, value := range cm.Data {
if !strings.HasPrefix(key, policyPrefix) {
continue
}
if !strings.HasSuffix(key, ".rego") {
continue
}
policyName := strings.TrimPrefix(key, "conftest.policy.")
policies = append(policies, policyName)
policies[key] = value
}

return policies, nil
return policies
}

func (p *plugin) GetContainerName() string {
return conftestContainerName
}

const (
defaultCategory = "Security"
)

func (p *plugin) ParseConfigAuditReportData(logsReader io.ReadCloser) (v1alpha1.ConfigAuditResult, error) {
var checkResults []CheckResult
err := json.NewDecoder(logsReader).Decode(&checkResults)
Expand All @@ -222,9 +218,9 @@ func (p *plugin) ParseConfigAuditReportData(logsReader io.ReadCloser) (v1alpha1.
for _, warning := range cr.Warnings {
checks = append(checks, v1alpha1.Check{
ID: p.getPolicyTitleFromResult(warning),
Severity: "WARNING",
Severity: severityWarning,
Message: warning.Message,
Category: defaultCategory,
Category: defaultCheckCategory,
Success: false,
})
warningCount++
Expand All @@ -233,9 +229,9 @@ func (p *plugin) ParseConfigAuditReportData(logsReader io.ReadCloser) (v1alpha1.
for _, failure := range cr.Failures {
checks = append(checks, v1alpha1.Check{
ID: p.getPolicyTitleFromResult(failure),
Severity: "DANGER",
Severity: severityDanger,
Message: failure.Message,
Category: defaultCategory,
Category: defaultCheckCategory,
})
dangerCount++
}
Expand Down
Loading

0 comments on commit 0ccb209

Please sign in to comment.