From f6e01cfda788b7e2765807163d312359bdb98d10 Mon Sep 17 00:00:00 2001 From: Jingfang Liu Date: Fri, 12 Apr 2019 15:41:41 -0700 Subject: [PATCH] add support for exec plugins --- bin/pre-commit.sh | 9 -- k8sdeps/kunstruct/kunstruct.go | 6 +- pkg/plugins/builtin/executable.go | 72 --------- pkg/plugins/compiler.go | 11 ++ pkg/plugins/executable.go | 140 ++++++++++++++++++ pkg/plugins/transformers.go | 40 ++--- pkg/target/generatorplugin_test.go | 3 +- pkg/target/transformerplugin_test.go | 46 ++++++ pkg/types/errors.go | 29 ++++ .../v1/ConfigMapGenerator | 6 +- .../someteam.example.com/v1/SedTransformer | 9 ++ 11 files changed, 263 insertions(+), 108 deletions(-) delete mode 100644 pkg/plugins/builtin/executable.go create mode 100644 pkg/plugins/executable.go create mode 100644 pkg/types/errors.go create mode 100755 plugins/someteam.example.com/v1/SedTransformer diff --git a/bin/pre-commit.sh b/bin/pre-commit.sh index ffa294c78d..9ea7f8e43c 100755 --- a/bin/pre-commit.sh +++ b/bin/pre-commit.sh @@ -11,14 +11,6 @@ cd "$base_dir" || { rc=0 -function buildPlugins { - go build \ - -buildmode plugin \ - -tags=plugin \ - -o ./pkg/plugins/builtin/executable.so \ - ./pkg/plugins/builtin/executable.go -} - function runTest { local name=$1 local result="SUCCESS" @@ -88,7 +80,6 @@ echo pwd=`pwd` echo " " echo "Beginning tests..." -runTest buildPlugins runTest testGoLangCILint runTest testGoTest diff --git a/k8sdeps/kunstruct/kunstruct.go b/k8sdeps/kunstruct/kunstruct.go index 9fa0b67eb3..c074311b36 100644 --- a/k8sdeps/kunstruct/kunstruct.go +++ b/k8sdeps/kunstruct/kunstruct.go @@ -19,7 +19,7 @@ package kunstruct import ( "encoding/json" - "fmt" + "sigs.k8s.io/kustomize/pkg/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -88,7 +88,7 @@ func (fs *UnstructAdapter) GetFieldValue(path string) (string, error) { if found || err != nil { return s, err } - return "", fmt.Errorf("no field named '%s'", path) + return "", types.NoFieldError{Field: path} } // GetStringSlice returns value at the given fieldpath. @@ -102,5 +102,5 @@ func (fs *UnstructAdapter) GetStringSlice(path string) ([]string, error) { if found || err != nil { return s, err } - return []string{}, fmt.Errorf("no field named '%s'", path) + return []string{}, types.NoFieldError{Field: path} } diff --git a/pkg/plugins/builtin/executable.go b/pkg/plugins/builtin/executable.go deleted file mode 100644 index f3114f4723..0000000000 --- a/pkg/plugins/builtin/executable.go +++ /dev/null @@ -1,72 +0,0 @@ -// +build plugin - -package main - -import ( - "bytes" - "os" - "os/exec" - "path/filepath" - - "github.com/ghodss/yaml" - "sigs.k8s.io/kustomize/pkg/ifc" - "sigs.k8s.io/kustomize/pkg/pgmconfig" - "sigs.k8s.io/kustomize/pkg/resmap" -) - -type plugin struct { - name string - input string - rf *resmap.Factory -} - -var KustomizePlugin plugin - -func (p *plugin) Config( - ldr ifc.Loader, rf *resmap.Factory, k ifc.Kunstructured) error { - dir := filepath.Join(pgmconfig.ConfigRoot(), "plugins") - id := k.GetGvk() - p.name = filepath.Join(dir, id.Group, id.Version, id.Kind) - content, err := yaml.Marshal(k) - if err != nil { - return err - } - p.input = string(content) - p.rf = rf - return nil -} - -func (p *plugin) Generate() (resmap.ResMap, error) { - return p.run(nil) -} - -func (p *plugin) Transformer(rm resmap.ResMap) error { - result, err := p.run(rm) - if err != nil { - return err - } - for id := range rm { - delete(rm, id) - } - for id, r := range result { - rm[id] = r - } - return nil -} - -func (p *plugin) run(rm resmap.ResMap) (resmap.ResMap, error) { - cmd := exec.Command(p.name, p.input) - cmd.Env = os.Environ() - if rm != nil { - content, err := rm.EncodeAsYaml() - if err != nil { - return nil, err - } - cmd.Stdin = bytes.NewReader(content) - } - output, err := cmd.Output() - if err != nil { - return nil, err - } - return p.rf.NewResMapFromBytes(output) -} diff --git a/pkg/plugins/compiler.go b/pkg/plugins/compiler.go index cfcd57c99e..22f0dc5a7a 100644 --- a/pkg/plugins/compiler.go +++ b/pkg/plugins/compiler.go @@ -17,7 +17,9 @@ import ( "fmt" "os" "os/exec" + "path" "path/filepath" + "runtime" "time" "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" @@ -51,6 +53,15 @@ func DefaultSrcRoot() (string, error) { } nope = append(nope, root) + // get the root kustomize source directory when + // GOPATH is not set + _, filename, _, _ := runtime.Caller(1) + root = path.Join(path.Dir(filename), "../..", plugin.PluginRoot) + if FileExists(root) { + return root, nil + } + nope = append(nope, root) + root = filepath.Join( pgmconfig.ConfigRoot(), plugin.PluginRoot) if FileExists(root) { diff --git a/pkg/plugins/executable.go b/pkg/plugins/executable.go new file mode 100644 index 0000000000..2d4013c705 --- /dev/null +++ b/pkg/plugins/executable.go @@ -0,0 +1,140 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugins + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "sigs.k8s.io/kustomize/pkg/types" + "strings" + + "github.com/ghodss/yaml" + "sigs.k8s.io/kustomize/pkg/ifc" + "sigs.k8s.io/kustomize/pkg/pgmconfig" + "sigs.k8s.io/kustomize/pkg/resmap" +) + +// ExecPlugin record the name and args of an executable +// It triggers the executable generator and transformer +type ExecPlugin struct { + // name of the executable + name string + + // one line of arguments for the executable + argOneLiner string + + // relative file path to a file + // Each line of this file is treated as one argument + argsFromFile string + + // resmap Factory to make resources + rf *resmap.Factory + + // loader to load files + ldr ifc.Loader +} + +func (p *ExecPlugin) Config( + ldr ifc.Loader, rf *resmap.Factory, k ifc.Kunstructured) error { + dir := filepath.Join(pgmconfig.ConfigRoot(), "plugins") + id := k.GetGvk() + p.name = filepath.Join(dir, id.Group, id.Version, id.Kind) + p.rf = rf + p.ldr = ldr + + var err error + p.argOneLiner, err = k.GetFieldValue("arg") + if err != nil && !isNoFieldError(err) { + return err + } + p.argsFromFile, err = k.GetFieldValue("file") + if err != nil && !isNoFieldError(err) { + return err + } + return nil +} + +func (p *ExecPlugin) Generate() (resmap.ResMap, error) { + args, err := p.getArgs() + if err != nil { + return nil, err + } + cmd := exec.Command(p.name, args...) + cmd.Env = os.Environ() + cmd.Stderr = os.Stderr + output, err := cmd.Output() + if err != nil { + return nil, err + } + return p.rf.NewResMapFromBytes(output) +} + +func (p *ExecPlugin) Transform(rm resmap.ResMap) error { + args, err := p.getArgs() + if err != nil { + return err + } + + for id, r := range rm { + content, err := yaml.Marshal(r.Kunstructured) + if err != nil { + return err + } + cmd := exec.Command(p.name, args...) + cmd.Env = os.Environ() + cmd.Stdin = bytes.NewReader(content) + cmd.Stderr = os.Stderr + output, err := cmd.Output() + if err != nil { + return err + } + tmpMap, err := p.rf.NewResMapFromBytes(output) + if err != nil { + return err + } + if len(tmpMap) != 1 { + return fmt.Errorf("Unable to put two resources into one") + } + for _, v := range tmpMap { + rm[id].Kunstructured = v.Kunstructured + } + } + return nil +} + +func (p *ExecPlugin) getArgs() ([]string, error) { + args := strings.Split(p.argOneLiner, " ") + if p.argsFromFile != "" { + content, err := p.ldr.Load(p.argsFromFile) + if err != nil { + return nil, err + } + args = append(args, strings.Split(string(content), "\n")...) + } + return args, nil +} + +func isNoFieldError(e error) bool { + _, ok := e.(types.NoFieldError) + if ok { + return true + } + return false +} diff --git a/pkg/plugins/transformers.go b/pkg/plugins/transformers.go index 661ccc2b79..3f2e60eac6 100644 --- a/pkg/plugins/transformers.go +++ b/pkg/plugins/transformers.go @@ -18,12 +18,10 @@ package plugins import ( "fmt" + "github.com/pkg/errors" "os" "path/filepath" "plugin" - "runtime" - - "github.com/pkg/errors" kplugin "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/resid" @@ -96,28 +94,30 @@ func loadAndConfigurePlugin( ldr ifc.Loader, rf *resmap.Factory, res *resource.Resource) (Configurable, error) { var fileName string + var c Configurable + exec := execPluginFileName(dir, id) if isExecAvailable(exec) { - _, f, _, _ := runtime.Caller(1) - fileName = filepath.Join(filepath.Dir(f), "builtin", "executable.so") + c = &ExecPlugin{} } else { fileName = goPluginFileName(dir, id) + goPlugin, err := plugin.Open(fileName) + if err != nil { + return nil, errors.Wrapf(err, "plugin %s fails to load", fileName) + } + symbol, err := goPlugin.Lookup(kplugin.PluginSymbol) + if err != nil { + return nil, errors.Wrapf( + err, "plugin %s doesn't have symbol %s", + fileName, kplugin.PluginSymbol) + } + var ok bool + c, ok = symbol.(Configurable) + if !ok { + return nil, fmt.Errorf("plugin %s not configurable", fileName) + } } - goPlugin, err := plugin.Open(fileName) - if err != nil { - return nil, errors.Wrapf(err, "plugin %s fails to load", fileName) - } - symbol, err := goPlugin.Lookup(kplugin.PluginSymbol) - if err != nil { - return nil, errors.Wrapf( - err, "plugin %s doesn't have symbol %s", - fileName, kplugin.PluginSymbol) - } - c, ok := symbol.(Configurable) - if !ok { - return nil, fmt.Errorf("plugin %s not configurable", fileName) - } - err = c.Config(ldr, rf, res) + err := c.Config(ldr, rf, res) if err != nil { return nil, errors.Wrapf(err, "plugin %s fails configuration", fileName) } diff --git a/pkg/target/generatorplugin_test.go b/pkg/target/generatorplugin_test.go index 5e4b2753ca..e71f17e181 100644 --- a/pkg/target/generatorplugin_test.go +++ b/pkg/target/generatorplugin_test.go @@ -128,7 +128,7 @@ type: Opaque `) } -func xTestConfigMapGenerator(t *testing.T) { +func TestConfigMapGenerator(t *testing.T) { tc := NewTestEnvController(t).Set() defer tc.Reset() @@ -146,6 +146,7 @@ apiVersion: someteam.example.com/v1 kind: ConfigMapGenerator metadata: name: some-random-name +arg: "admin secret" `) m, err := th.makeKustTarget().MakeCustomizedResMap() if err != nil { diff --git a/pkg/target/transformerplugin_test.go b/pkg/target/transformerplugin_test.go index 84a6bf2278..fa18ba61a5 100644 --- a/pkg/target/transformerplugin_test.go +++ b/pkg/target/transformerplugin_test.go @@ -98,6 +98,52 @@ spec: `) } +func TestSedTransformer(t *testing.T) { + tc := NewTestEnvController(t).Set() + defer tc.Reset() + + tc.BuildExecPlugin( + "someteam.example.com", "v1", "SedTransformer") + + th := NewKustTestHarnessWithPluginConfig( + t, "/app", plugin.ActivePluginConfig()) + th.writeK("/app", ` +transformers: +- sed-transformer.yaml + +configMapGenerator: +- name: test + literals: + - FOO=$FOO + - BAR=$BAR +`) + th.writeF("/app/sed-transformer.yaml", ` +apiVersion: someteam.example.com/v1 +kind: SedTransformer +metadata: + name: some-random-name +file: sed-input.txt +`) + th.writeF("/app/sed-input.txt", ` +s/$FOO/foo/g +s/$BAR/bar/g +`) + + m, err := th.makeKustTarget().MakeCustomizedResMap() + if err != nil { + t.Fatalf("Err: %v", err) + } + th.assertActualEqualsExpected(m, ` +apiVersion: v1 +data: + BAR: bar + FOO: foo +kind: ConfigMap +metadata: + name: test-k4bkhftttd +`) +} + func xTestTransformedTransformers(t *testing.T) { th := NewKustTestHarnessWithPluginConfig( t, "/app/overlay", plugin.ActivePluginConfig()) diff --git a/pkg/types/errors.go b/pkg/types/errors.go new file mode 100644 index 0000000000..a94ec12e93 --- /dev/null +++ b/pkg/types/errors.go @@ -0,0 +1,29 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" +) + +type NoFieldError struct { + Field string +} + +func (e NoFieldError) Error() string { + return fmt.Sprintf("no field named '%s'", e.Field) +} diff --git a/plugins/someteam.example.com/v1/ConfigMapGenerator b/plugins/someteam.example.com/v1/ConfigMapGenerator index d0df0adaa3..94613cb333 100755 --- a/plugins/someteam.example.com/v1/ConfigMapGenerator +++ b/plugins/someteam.example.com/v1/ConfigMapGenerator @@ -6,6 +6,6 @@ apiVersion: v1 metadata: name: example-configmap-test data: - username: admin - password: secret -" \ No newline at end of file + username: $1 + password: $2 +" diff --git a/plugins/someteam.example.com/v1/SedTransformer b/plugins/someteam.example.com/v1/SedTransformer new file mode 100755 index 0000000000..c48d4cb853 --- /dev/null +++ b/plugins/someteam.example.com/v1/SedTransformer @@ -0,0 +1,9 @@ +#!/bin/bash + +args="" + +for arg in $@; do + args="$args -e $arg" +done + +cat - | sed $args