Refactor claims plugin to be a storage plugin

Instead of having a credential plugin, and a claims plugin, etc this
refactors the plugin POC so that there is a single storage plugin with
the intent of being able to offload the entire porter home directory
contents (except the config file) to the plugins.

For example everything that we currently store today on the filesystem
could be stored in blob storage or a database.

This replies upon a unmerged branch with changes to the crud.Store
interface and other changes to support this.

I have also stubbed in a new plugin for secrets, though it isn't
completely plumbed through and will be completed in a separate PR. At
the moment all secrets, for example the values that we resolve
credentials from but could eventually be parameter values too among
other things, are coming from a host plugin embedded in porter.

CONTRIBUTING.md

@@ -391,7 +391,7 @@ fmt.Fprintln(p.Err, "DEBUG: loading plans from r2d2...")
 ```
 
 Most of the structs in Porter have an embedded
-`github.com/deislabs/porter/pkg/context.Context` struct. This has both `Out` and
+`get.porter.sh/porter/pkg/context.Context` struct. This has both `Out` and
 `Err` which represent stdout and stderr respectively. You should log to those
 instead of directly to stdout/stderr because that is how we capture output in
 our unit tests. That means use `fmt.Fprint*` instead of `fmt.Print*` so that you

Makefile

@@ -87,7 +87,7 @@ test-cli: clean-last-testrun build init-porter-home-for-ci
 
 init-porter-home-for-ci:
 	cp -R build/testdata/credentials $(PORTER_HOME)
-	sed -i 's|KUBECONFIGPATH|$(KUBECONFIG)|g' $(PORTER_HOME)/credentials/ci.yaml
+	sed -i 's|KUBECONFIGPATH|$(KUBECONFIG)|g' $(PORTER_HOME)/credentials/ci.json
 	cp -R build/testdata/bundles $(PORTER_HOME)
 
 .PHONY: docs

build/testdata/credentials/ci.json

@@ -0,0 +1,11 @@
+{
+  "name": "ci",
+  "credentials": [
+  {
+    "name": "kubeconfig",
+    "source": {
+      "path": "KUBECONFIGPATH"
+    }
+  }
+  ]
+}

go.mod

@@ -44,3 +44,5 @@ replace github.com/docker/docker => github.com/moby/moby v0.7.3-0.20190826074503
 replace golang.org/x/sys => golang.org/x/sys v0.0.0-20190830141801-acfa387b8d69
 
 replace github.com/hashicorp/go-plugin => github.com/carolynvs/go-plugin v1.0.1-acceptstdin
+
+replace github.com/cnabio/cnab-go => github.com/carolynvs/cnab-go v0.0.0-20200116210330-60b3c613dd4f

go.sum

@@ -77,6 +77,8 @@ github.com/bugsnag/bugsnag-go v1.5.0 h1:tP8hiPv1pGGW3LA6LKy5lW6WG+y9J2xWUdPd3WC4
 github.com/bugsnag/bugsnag-go v1.5.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA=
 github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/carolynvs/cnab-go v0.0.0-20200116210330-60b3c613dd4f h1:zo0hri1VK2RjOnGn7bLeBN9bqngnpRddesX95FV2DAE=
+github.com/carolynvs/cnab-go v0.0.0-20200116210330-60b3c613dd4f/go.mod h1:5c4uOP6ZppR4nUGtCMAElscRiYEUi44vNQwtSAvISXk=
 github.com/carolynvs/datetime-printer v0.2.0 h1:Td3FU4YGzx0OogCMhCmLBTUTDPQcq0xlgCeMhAKZmMc=
 github.com/carolynvs/datetime-printer v0.2.0/go.mod h1:p9W8ZUhmQUOVD5kiDuGXwRG65/nTkZWlLylY7s+Qw2k=
 github.com/carolynvs/go-plugin v1.0.1-acceptstdin h1:8JccOWqcZoqCILz191C0D6RTnz/DKfNcY8+T6F3/G9g=
@@ -95,10 +97,6 @@ github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiK
 github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo=
 github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4=
 github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo=
-github.com/cnabio/cnab-go v0.8.1-beta1 h1:+UCpSdayWYdNS5Cm2VRtiJ38hhsnvngRI5aiBg9GdoU=
-github.com/cnabio/cnab-go v0.8.1-beta1/go.mod h1:5c4uOP6ZppR4nUGtCMAElscRiYEUi44vNQwtSAvISXk=
-github.com/cnabio/cnab-go v0.8.2-beta1 h1:pJfXR9w0G1m3sFE3yp+ImAKeaBUwa2OviE9f+PnpMnM=
-github.com/cnabio/cnab-go v0.8.2-beta1/go.mod h1:5c4uOP6ZppR4nUGtCMAElscRiYEUi44vNQwtSAvISXk=
 github.com/containerd/cgroups v0.0.0-20200108155730-918ed86e29cc h1:Euk3pMvf7cw8tKEi8ZlMHCrnd5iBlz8hWBUVNL44hmQ=
 github.com/containerd/cgroups v0.0.0-20200108155730-918ed86e29cc/go.mod h1:6KyBUkSDshoWUZPkqlFXQzOMWNtlcJ1stduPAd2MRes=
 github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=

pkg/cnab/provider/action.go

@@ -62,7 +62,7 @@ func (d *Runtime) AddFiles(args ActionArguments) action.OperationConfigFunc {
 
 		// Add claim.json to file list as well, if exists
 		claimName := args.Claim
-		claim, err := d.instanceStorage.Read(claimName)
+		claim, err := d.claims.Read(claimName)
 		if err == nil {
 			claimBytes, err := yaml.Marshal(claim)
 			if err != nil {

pkg/cnab/provider/action_test.go

@@ -5,7 +5,8 @@ import (
 	"testing"
 
 	"get.porter.sh/porter/pkg/config"
-	instancestorage "get.porter.sh/porter/pkg/instance-storage"
+	"get.porter.sh/porter/pkg/credentials"
+	"get.porter.sh/porter/pkg/storage"
 	"github.com/cnabio/cnab-go/bundle"
 	"github.com/cnabio/cnab-go/driver"
 	"github.com/stretchr/testify/assert"
@@ -17,8 +18,9 @@ func TestAddReloccation(t *testing.T) {
 	require.NoError(t, err)
 
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	args := ActionArguments{
 		RelocationMapping: "/cnab/app/relocation-mapping.json",

pkg/cnab/provider/credentials.go

@@ -8,44 +8,31 @@ import (
 	"github.com/cnabio/cnab-go/credentials"
 )
 
-const (
-	// CredentialsDirectory represents the name of the directory where credentials are stored
-	CredentialsDirectory = "credentials"
-)
-
-func (d *Runtime) loadCredentials(b *bundle.Bundle, files []string) (map[string]string, error) {
-	// TODO: export back outta Compton
-
-	creds := map[string]string{}
-	if len(files) == 0 {
-		return creds, credentials.Validate(creds, b.Credentials)
+func (d *Runtime) loadCredentials(b *bundle.Bundle, creds []string) (credentials.Set, error) {
+	if len(creds) == 0 {
+		return nil, credentials.Validate(nil, b.Credentials)
 	}
 
 	// The strategy here is "last one wins". We loop through each credential file and
 	// calculate its credentials. Then we insert them into the creds map in the order
 	// in which they were supplied on the CLI.
-	for _, file := range files {
-		if !d.isPathy(file) {
-			credsPath, err := d.Config.GetCredentialsDir()
-			if err != nil {
-				return nil, err
-			}
-			file = filepath.Join(credsPath, file+".yaml")
-		}
-		cset, err := credentials.Load(file)
+	resolvedCredentials := credentials.Set{}
+	for _, name := range creds {
+		cset, err := d.credentials.Read(name)
 		if err != nil {
-			return creds, err
+			return nil, err
 		}
-		res, err := cset.Resolve()
+
+		rc, err := d.credentials.ResolveAll(cset)
 		if err != nil {
-			return res, err
+			return nil, err
 		}
 
-		for k, v := range res {
-			creds[k] = v
+		for k, v := range rc {
+			resolvedCredentials[k] = v
 		}
 	}
-	return creds, credentials.Validate(creds, b.Credentials)
+	return resolvedCredentials, credentials.Validate(resolvedCredentials, b.Credentials)
 }
 
 // isPathy checks to see if a name looks like a path.

pkg/cnab/provider/dockerdriver_test.go

@@ -4,15 +4,17 @@ import (
 	"testing"
 
 	"get.porter.sh/porter/pkg/config"
-	instancestorage "get.porter.sh/porter/pkg/instance-storage"
+	"get.porter.sh/porter/pkg/credentials"
+	"get.porter.sh/porter/pkg/storage"
 	"github.com/cnabio/cnab-go/driver/docker"
 	"github.com/stretchr/testify/require"
 )
 
 func TestNewDriver_Docker(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	driver, err := d.newDriver("docker", "myclaim", ActionArguments{})
 	require.NoError(t, err)

pkg/cnab/provider/install.go

@@ -69,7 +69,7 @@ func (d *Runtime) Install(args ActionArguments) error {
 	}
 
 	// ALWAYS write out a claim, even if the installation fails
-	err = d.instanceStorage.Store(*c)
+	err = d.claims.Save(*c)
 	if err != nil {
 		result = multierror.Append(result, errors.Wrap(err, "failed to record the installation for the bundle"))
 	}

pkg/cnab/provider/invoke.go

@@ -14,7 +14,7 @@ import (
 // is examined to see if the action is stateless. If the action is stateless, we create a new, temporary, claim
 // Returns a pointer to the claim, a flag to indicate if the claim is temporary, and an error if present.
 func (d *Runtime) getClaim(bun *bundle.Bundle, actionName, claimName string) (*claim.Claim, bool, error) {
-	c, err := d.instanceStorage.Read(claimName)
+	c, err := d.claims.Read(claimName)
 	if err != nil {
 		if bun != nil {
 			if action, ok := bun.Actions[actionName]; ok {
@@ -36,7 +36,7 @@ func (d *Runtime) getClaim(bun *bundle.Bundle, actionName, claimName string) (*c
 // a temporary claim
 func (d *Runtime) writeClaim(tempClaim bool, c *claim.Claim) error {
 	if !tempClaim {
-		return d.instanceStorage.Store(*c)
+		return d.claims.Save(*c)
 	}
 	return nil
 }

pkg/cnab/provider/invoke_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 
 	"get.porter.sh/porter/pkg/config"
-	instancestorage "get.porter.sh/porter/pkg/instance-storage"
+	"get.porter.sh/porter/pkg/credentials"
+	"get.porter.sh/porter/pkg/storage"
 	"github.com/cnabio/cnab-go/bundle"
 	"github.com/cnabio/cnab-go/claim"
-
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -28,14 +28,15 @@ func Test_ClaimWriting(t *testing.T) {
 	}
 
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewTestInstanceStorageProvider()
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credStorage)
 
 	eClaim, err := claim.New("exists")
 	require.NoError(t, err)
 	eClaim.Update(claim.ActionInstall, claim.StatusSuccess)
 
-	err = instanceStorage.Store(*eClaim)
+	err = claimStorage.Save(*eClaim)
 	require.NoError(t, err)
 
 	bun := &bundle.Bundle{
@@ -91,7 +92,7 @@ func Test_ClaimWriting(t *testing.T) {
 		err = d.writeClaim(temp, c)
 		assert.NoError(t, err)
 
-		fc, err := d.instanceStorage.Read(in.claim)
+		fc, err := d.claims.Read(in.claim)
 		if tc.want {
 			assert.NoErrorf(t, err, "expected claim for %s", tc.name)
 			assert.Equalf(t, in.action, fc.Result.Action, "expected action=%s for %s", in.action, tc.name)
@@ -137,10 +138,11 @@ func Test_ClaimLoading(t *testing.T) {
 	eClaim.Update(claim.ActionInstall, claim.StatusSuccess)
 
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewTestInstanceStorageProvider()
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credStorage)
 
-	err = instanceStorage.Store(*eClaim)
+	err = claimStorage.Save(*eClaim)
 	require.NoError(t, err)
 
 	tests := []test{

pkg/cnab/provider/parameters_test.go

@@ -5,8 +5,10 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"get.porter.sh/porter/pkg/credentials"
+
 	"get.porter.sh/porter/pkg/config"
-	instancestorage "get.porter.sh/porter/pkg/instance-storage"
+	"get.porter.sh/porter/pkg/storage"
 	"github.com/cnabio/cnab-go/bundle"
 	"github.com/cnabio/cnab-go/bundle/definition"
 	"github.com/cnabio/cnab-go/claim"
@@ -15,8 +17,9 @@ import (
 
 func Test_loadParameters_paramNotDefined(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	claim, err := claim.New("test")
 	require.NoError(t, err)
@@ -35,8 +38,9 @@ func Test_loadParameters_paramNotDefined(t *testing.T) {
 
 func Test_loadParameters_definitionNotDefined(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	claim, err := claim.New("test")
 	require.NoError(t, err)
@@ -59,8 +63,9 @@ func Test_loadParameters_definitionNotDefined(t *testing.T) {
 
 func Test_loadParameters_applyToClaimDefaults(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	claim, err := claim.New("test")
 	require.NoError(t, err)
@@ -123,8 +128,9 @@ func Test_loadParameters_applyToClaimDefaults(t *testing.T) {
 
 func Test_loadParameters_applyToBundleDefaults(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	claim, err := claim.New("test")
 	require.NoError(t, err)
@@ -158,8 +164,9 @@ func Test_loadParameters_applyToBundleDefaults(t *testing.T) {
 
 func Test_loadParameters_requiredButDoesNotApply(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	claim, err := claim.New("test")
 	require.NoError(t, err)
@@ -211,8 +218,9 @@ func Test_loadParameters_zeroValues(t *testing.T) {
 	for _, tc := range testcases {
 		t.Run(tc.paramType, func(t *testing.T) {
 			c := config.NewTestConfig(t)
-			instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-			d := NewRuntime(c.Config, instanceStorage)
+			claimStorage := storage.NewTestClaimProvider()
+			credentialStorage := credentials.NewTestCredentialProvider(t, c)
+			d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 			claim, err := claim.New("test")
 			require.NoError(t, err)
@@ -247,8 +255,9 @@ func Test_loadParameters_zeroValues(t *testing.T) {
 
 func Test_loadParameters_fileParameter(t *testing.T) {
 	c := config.NewTestConfig(t)
-	instanceStorage := instancestorage.NewPluggableInstanceStorage(c.Config)
-	d := NewRuntime(c.Config, instanceStorage)
+	claimStorage := storage.NewTestClaimProvider()
+	credentialStorage := credentials.NewTestCredentialProvider(t, c)
+	d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 	c.TestContext.AddTestFile("testdata/file-param", "/path/to/file")
 
@@ -355,8 +364,9 @@ func Test_Paramapalooza(t *testing.T) {
 			for _, tc := range testcases {
 				t.Run(tc.name, func(t *testing.T) {
 					c := config.NewTestConfig(t)
-					instanceStorage := instancestorage.NewTestInstanceStorageProvider()
-					d := NewRuntime(c.Config, instanceStorage)
+					claimStorage := storage.NewTestClaimProvider()
+					credentialStorage := credentials.NewTestCredentialProvider(t, c)
+					d := NewRuntime(c.Config, claimStorage, credentialStorage)
 
 					bun := &bundle.Bundle{
 						Name:          "mybuns",
@@ -430,7 +440,7 @@ func Test_Paramapalooza(t *testing.T) {
 						require.NoError(t, err)
 
 						claim.Bundle = bun
-						d.instanceStorage.Store(*claim)
+						d.claims.Save(*claim)
 					}
 
 					var err error
@@ -452,7 +462,7 @@ func Test_Paramapalooza(t *testing.T) {
 
 						if action != "uninstall" {
 							// Verify the updated param value on the generated claim
-							updatedClaim, err := d.instanceStorage.Read("test")
+							updatedClaim, err := d.claims.Read("test")
 							require.NoError(t, err)
 							require.Equal(t, tc.expectedVal, updatedClaim.Parameters["my-param"])
 						}