From b601e4e36223f73528d3bfccd94861e550af7a68 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Tue, 8 Oct 2019 17:24:00 -0600 Subject: [PATCH] chore(Gopkg.toml): bump cnab-go dep; update some usage (#710) --- Gopkg.lock | 6 +- Gopkg.toml | 4 +- pkg/cnab/provider/parameters.go | 17 +---- pkg/manifest/runtime-manifest.go | 15 +--- .../deislabs/cnab-go/driver/docker/docker.go | 13 +++- .../cnab-go/driver/kubernetes/kubernetes.go | 76 +++++++++++++------ .../deislabs/cnab-go/packager/import.go | 17 +---- 7 files changed, 77 insertions(+), 71 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 4b84b6977..0cc073e92 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -266,7 +266,7 @@ version = "v1.1.1" [[projects]] - digest = "1:47771b0b7b6461b5d7e559eb547b81b0a7752d2a90be61af4dcb1dac1f71fe11" + digest = "1:b6893d684f9f344938f9e020a2be731e11592e65200e4cfb82425bc4956e1e79" name = "github.com/deislabs/cnab-go" packages = [ "action", @@ -287,8 +287,8 @@ "utils/crud", ] pruneopts = "NT" - revision = "525007ac513a763cb8fb96ec88f2aa101b791535" - version = "v0.5.0-beta1" + revision = "406808480de1f033040608d1924f96dcd662c2d0" + version = "v0.6.0-beta1" [[projects]] digest = "1:7a6852b35eb5bbc184561443762d225116ae630c26a7c4d90546619f1e7d2ad2" diff --git a/Gopkg.toml b/Gopkg.toml index e21191aaf..916337ee7 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,10 +1,10 @@ [[constraint]] name = "github.com/deislabs/cnab-go" - version = "v0.5.0-beta1" + version = "v0.6.0-beta1" [[override]] name = "github.com/deislabs/cnab-go" - version = "v0.5.0-beta1" + version = "v0.6.0-beta1" [[constraint]] diff --git a/pkg/cnab/provider/parameters.go b/pkg/cnab/provider/parameters.go index 2e0d6de79..abac286f7 100644 --- a/pkg/cnab/provider/parameters.go +++ b/pkg/cnab/provider/parameters.go @@ -38,7 +38,7 @@ func (d *Runtime) loadParameters(claim *claim.Claim, rawOverrides map[string]str } // If this parameter applies to the current action, set the override accordingly - if appliesToAction(action, param) { + if param.AppliesTo(action) { overrides[key] = value } else { // Otherwise, set to current parameter value on the claim, if exists @@ -60,7 +60,7 @@ func (d *Runtime) loadParameters(claim *claim.Claim, rawOverrides map[string]str for name, param := range bun.Parameters { if param.Required { if _, exists := rawOverrides[name]; !exists { - if !appliesToAction(action, param) { + if !param.AppliesTo(action) { overrides[name] = claim.Parameters[name] } } @@ -83,16 +83,3 @@ func (d *Runtime) getUnconvertedValueFromRaw(def *definition.Schema, key, rawVal } return rawValue, nil } - -// TODO: remove in favor of cnab-go logic: https://github.com/deislabs/cnab-go/pull/129 -func appliesToAction(action string, parameter bundle.Parameter) bool { - if len(parameter.ApplyTo) == 0 { - return true - } - for _, act := range parameter.ApplyTo { - if action == act { - return true - } - } - return false -} diff --git a/pkg/manifest/runtime-manifest.go b/pkg/manifest/runtime-manifest.go index f4c5aea02..f6e66dba4 100644 --- a/pkg/manifest/runtime-manifest.go +++ b/pkg/manifest/runtime-manifest.go @@ -214,7 +214,7 @@ func (m *RuntimeManifest) buildSourceData() (map[string]interface{}, error) { continue } for name, output := range bun.Outputs { - if !OutputAppliesTo(string(m.Action), output) { + if !output.AppliesTo(string(m.Action)) { continue } @@ -306,16 +306,3 @@ func (m *RuntimeManifest) Prepare() error { } return nil } - -// TODO: remove in favor of cnab-go logic: https://github.com/deislabs/cnab-go/pull/129 -func OutputAppliesTo(action string, output bundle.Output) bool { - if len(output.ApplyTo) == 0 { - return true - } - for _, act := range output.ApplyTo { - if action == act { - return true - } - } - return false -} diff --git a/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go b/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go index 5ee710cc9..5d96830ca 100644 --- a/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go +++ b/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go @@ -55,11 +55,20 @@ func (d *Driver) Config() map[string]string { "PULL_ALWAYS": "Always pull image, even if locally available (0|1)", "DOCKER_DRIVER_QUIET": "Make the Docker driver quiet (only print container stdout/stderr)", "OUTPUTS_MOUNT_PATH": "Absolute path to where Docker driver can create temporary directories to bundle outputs. Defaults to temp dir.", + "CLEANUP_CONTAINERS": "If true, the docker container will be destroyed when it finishes running. If false, it will not be destroyed. The supported values are true and false. Defaults to true.", } } // SetConfig sets Docker driver configuration func (d *Driver) SetConfig(settings map[string]string) { + // Set default and provide feedback on acceptable input values. + value, ok := settings["CLEANUP_CONTAINERS"] + if !ok { + settings["CLEANUP_CONTAINERS"] = "true" + } else if value != "true" && value != "false" { + fmt.Printf("CLEANUP_CONTAINERS environment variable has unexpected value %q. Supported values are 'true', 'false', or unset.", value) + } + d.config = settings } @@ -175,7 +184,9 @@ func (d *Driver) exec(op *driver.Operation) (driver.OperationResult, error) { return driver.OperationResult{}, fmt.Errorf("cannot create container: %v", err) } - defer cli.Client().ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}) + if d.config["CLEANUP_CONTAINERS"] == "true" { + defer cli.Client().ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}) + } tarContent, err := generateTar(op.Files) if err != nil { diff --git a/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go b/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go index 1d60cb735..30e5e360e 100644 --- a/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go +++ b/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go @@ -6,14 +6,13 @@ import ( "os" "path/filepath" "strings" + "time" // load credential helpers _ "k8s.io/client-go/plugin/pkg/client/auth" // Convert transitive deps to direct deps so that we can use constraints in our Gopkg.toml _ "github.com/Azure/go-autorest/autorest" - "github.com/deislabs/cnab-go/bundle" - "github.com/deislabs/cnab-go/driver" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -23,11 +22,15 @@ import ( coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + "github.com/deislabs/cnab-go/bundle" + "github.com/deislabs/cnab-go/driver" ) const ( k8sContainerName = "invocation" k8sFileSecretVolume = "files" + numBackoffLoops = 6 ) // Driver runs an invocation image in a Kubernetes cluster. @@ -70,7 +73,7 @@ func (k *Driver) Config() map[string]string { return map[string]string{ "KUBE_NAMESPACE": "Kubernetes namespace in which to run the invocation image", "SERVICE_ACCOUNT": "Kubernetes service account to be mounted by the invocation image (if empty, no service account token will be mounted)", - "KUBE_CONFIG": "Absolute path to the kubeconfig file", + "KUBECONFIG": "Absolute path to the kubeconfig file", "MASTER_URL": "Kubernetes master endpoint", } } @@ -82,7 +85,7 @@ func (k *Driver) SetConfig(settings map[string]string) { k.ServiceAccountName = settings["SERVICE_ACCOUNT"] var kubeconfig string - if kpath := settings["KUBE_CONFIG"]; kpath != "" { + if kpath := settings["KUBECONFIG"]; kpath != "" { kubeconfig = kpath } else if home := homeDir(); home != "" { kubeconfig = filepath.Join(home, ".kube", "config") @@ -234,22 +237,29 @@ func (k *Driver) Run(op *driver.Operation) (driver.OperationResult, error) { return driver.OperationResult{}, nil } - selector := metav1.ListOptions{ + // Create a selector to detect the job just created + jobSelector := metav1.ListOptions{ LabelSelector: labels.Set(job.ObjectMeta.Labels).String(), + FieldSelector: newSingleFieldSelector("metadata.name", job.ObjectMeta.Name), + } + + // Prevent detecting pods from prior jobs by adding the job name to the labels + podSelector := metav1.ListOptions{ + LabelSelector: newSingleFieldSelector("job-name", job.ObjectMeta.Name), } - return driver.OperationResult{}, k.watchJobStatusAndLogs(selector, op.Out) + return driver.OperationResult{}, k.watchJobStatusAndLogs(podSelector, jobSelector, op.Out) } -func (k *Driver) watchJobStatusAndLogs(selector metav1.ListOptions, out io.Writer) error { +func (k *Driver) watchJobStatusAndLogs(podSelector metav1.ListOptions, jobSelector metav1.ListOptions, out io.Writer) error { // Stream Pod logs in the background logsStreamingComplete := make(chan bool) - err := k.streamPodLogs(selector, out, logsStreamingComplete) + err := k.streamPodLogs(podSelector, out, logsStreamingComplete) if err != nil { return err } // Watch job events and exit on failure/success - watch, err := k.jobs.Watch(selector) + watch, err := k.jobs.Watch(jobSelector) if err != nil { return err } @@ -306,22 +316,36 @@ func (k *Driver) streamPodLogs(options metav1.ListOptions, out io.Writer, done c // The event was for a pod whose logs have already been streamed, so do nothing. continue } - req := k.pods.GetLogs(podName, &v1.PodLogOptions{ - Container: k8sContainerName, - Follow: true, - }) - reader, err := req.Stream() - // There was an error connecting to the pod, so continue the loop and attempt streaming - // logs again next time there is an event for the same pod. - if err != nil { - continue + + for i := 0; i < numBackoffLoops; i++ { + time.Sleep(time.Duration(i*i/2) * time.Second) + req := k.pods.GetLogs(podName, &v1.PodLogOptions{ + Container: k8sContainerName, + Follow: true, + }) + reader, err := req.Stream() + if err != nil { + // There was an error connecting to the pod, so continue the loop and attempt streaming + // the logs again. + continue + } + + // Block the loop until all logs from the pod have been processed. + bytesRead, err := io.Copy(out, reader) + reader.Close() + if err != nil { + continue + } + if bytesRead == 0 { + // There is a chance where we have connected to the pod, but it has yet to write something. + // In that case, we continue to to keep streaming until it does. + continue + } + // Set the pod to have successfully streamed data. + streamedLogs[podName] = true + break } - // We successfully connected to the pod, so mark it as having streamed logs. - streamedLogs[podName] = true - // Block the loop until all logs from the pod have been processed. - io.Copy(out, reader) - reader.Close() done <- true } }() @@ -377,6 +401,12 @@ func generateFileSecret(files map[string]string) (*v1.Secret, []v1.VolumeMount) return secret, mounts } +func newSingleFieldSelector(k, v string) string { + return labels.Set(map[string]string{ + k: v, + }).String() +} + func homeDir() string { if h := os.Getenv("HOME"); h != "" { return h diff --git a/vendor/github.com/deislabs/cnab-go/packager/import.go b/vendor/github.com/deislabs/cnab-go/packager/import.go index 624a8313e..5c97d617b 100644 --- a/vendor/github.com/deislabs/cnab-go/packager/import.go +++ b/vendor/github.com/deislabs/cnab-go/packager/import.go @@ -1,7 +1,6 @@ package packager import ( - "errors" "fmt" "os" "path/filepath" @@ -12,17 +11,11 @@ import ( "github.com/docker/docker/pkg/archive" ) -var ( - // ErrNoArtifactsDirectory indicates a missing artifacts/ directory - ErrNoArtifactsDirectory = errors.New("No artifacts/ directory found") -) - // Importer is responsible for importing a file type Importer struct { Source string Destination string Loader loader.BundleLoader - Verbose bool } // NewImporter creates a new secure *Importer @@ -30,20 +23,19 @@ type Importer struct { // source is the filesystem path to the archive. // destination is the directory to unpack the contents. // load is a loader.BundleLoader preconfigured for loading bundles. -func NewImporter(source, destination string, load loader.BundleLoader, verbose bool) (*Importer, error) { +func NewImporter(source, destination string, load loader.BundleLoader) *Importer { return &Importer{ Source: source, Destination: destination, Loader: load, - Verbose: verbose, - }, nil + } } // Import decompresses a bundle from Source (location of the compressed bundle) and properly places artifacts in the correct location(s) func (im *Importer) Import() error { _, _, err := im.Unzip() - // TODO: https://github.com/deislabs/duffle/issues/758 + // TODO: https://github.com/deislabs/cnab-go/issues/136 return err } @@ -66,8 +58,7 @@ func (im *Importer) Unzip() (string, *bundle.Bundle, error) { Compression: archive.Gzip, IncludeFiles: []string{"."}, IncludeSourceDir: true, - // Issue #416 - NoLchown: true, + NoLchown: true, } if err := archive.Untar(reader, dest, tarOptions); err != nil { return "", nil, fmt.Errorf("untar failed: %s", err)