From 93dbc1f23a786a6f1eed268a3b6477a3df0f2870 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Mon, 2 Sep 2019 10:50:36 +0800 Subject: [PATCH] Support setting command in go-getter plugin This allows one to use non-kustomization remote source --- examples/README.md | 40 +++--- examples/goGetterGeneratorPlugin.md | 126 ++++++++++++++++++ .../someteam.example.com/v1/gogetter/GoGetter | 16 ++- .../v1/gogetter/GoGetter_test.go | 64 ++++++++- 4 files changed, 221 insertions(+), 25 deletions(-) create mode 100644 examples/goGetterGeneratorPlugin.md diff --git a/examples/README.md b/examples/README.md index 9a73adf602f..051e62111d8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,59 +12,61 @@ Basic Usage * [configGenerations](configGeneration.md) - Rolling update when ConfigMapGenerator changes. - + * [combineConfigs](combineConfigs.md) - Mixing configuration data from different owners (e.g. devops/SRE and developers). - + * [generatorOptions](generatorOptions.md) - - Modifying behavior of all ConfigMap and Secret generators. + Modifying behavior of all ConfigMap and Secret generators. * [vars](wordpress/README.md) - Injecting k8s runtime data into container arguments (e.g. to point wordpress to a SQL service) by vars. - + * [image names and tags](image.md) - Updating image names and tags without applying a patch. - + * [remote target](remoteBuild.md) - Building a kustomization from a github URL - - * [json patch](jsonpatch.md) - Apply a json patch in a kustomization + + * [json patch](jsonpatch.md) - Apply a json patch in a kustomization * [patch multiple objects](patchMultipleObjects.md) - Apply a patch to multiple objects Advanced Usage - + - generator plugins: - + * [last mile helm](chart.md) - Make last mile modifications to a helm chart. - - * [secret generation](secretGeneratorPlugin.md) - Generating secrets from a plugin. + + * [secret generation](secretGeneratorPlugin.md) - Generating secrets from a plugin. + + * [remote sources](goGetterGeneratorPlugin.md) - Generating from remote sources. - transformer plugins: * [validation transformer](validationTransformer/README.md) - validate resources through a transformer - + - customize builtin transformer configurations - + * [transformer configs](transformerconfigs/README.md) - Customize transformer configurations - + Multi Variant Examples * [hello world](helloWorld/README.md) - Deploy multiple (differently configured) variants of a simple Hello World server. - + * [LDAP](ldap/README.md) - Deploy multiple (differently configured) variants of a LDAP server. - + * [springboot](springboot/README.md) - Create a Spring Boot application production configuration from scratch. * [mySql](mySql/README.md) - Create a MySQL production configuration from scratch. - + * [breakfast](breakfast.md) - Customize breakfast for Alice and Bob. - - * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base. + + * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base. diff --git a/examples/goGetterGeneratorPlugin.md b/examples/goGetterGeneratorPlugin.md new file mode 100644 index 00000000000..04c09063f13 --- /dev/null +++ b/examples/goGetterGeneratorPlugin.md @@ -0,0 +1,126 @@ +# Remote Sources + +Kustomize supports building a [remote target], but the URLs are limited to common [Git repository specs]. + +To extend the supported format, Kustomize has a [plugin] system that allows one to integrate third-party tools such as [hashicorp/go-getter] to "download things from a string URL suing a variety of protocols", extract the content and generated resources as part of kustomize build. + +[remote target]: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md +[Git repository specs]: https://github.com/kubernetes-sigs/kustomize/blob/master/pkg/git/repospec_test.go +[plugin]: ../docs/plugins +[hashicorp/go-getter]: https://github.com/hashicorp/go-getter + +## Make a place to work + + +```sh +DEMO_HOME=$(mktemp -d) +``` + +## Use a remote kustomize layer + +Define a kustomization representing your _local_ variant (aka environment). + +This could involve any number of kustomizations (see other examples), but in this case just add the name prefix `my-` to all resources: + + +```sh +cat <<'EOF' >$DEMO_HOME/my/kustomization.yaml +namePrefix: my- +resources: +- ../base +EOF +``` + +It refer a remote base defined as below: + + +```sh +cat <<'EOF' >$DEMO_HOME/base/kustomization.yaml +generators: +- goGetter.yaml +EOF +``` + +The base refers to a generator configuration file called `goGetter.yaml`. + +This file lets one specify the source URL, and other things like sub path in the package, defaulting to base directory, and command to run under the path, defaulting to `kustomize build`. + +Create the config file `goGetter.yaml`, specifying the arbitrarily chosen name _example_: + + +```sh +cat <<'EOF' >$DEMO_HOME/base/goGetter.yaml +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git +EOF +``` + +Because this particular YAML file is listed in the `generators:` stanza of a kustomization file, it is treated as the binding between a generator plugin - identified by the _apiVersion_ and _kind_ fields - and other fields that configure the plugin. + +Download the plugin to your `DEMO_HOME` and make it executable: + + +```sh +plugin=plugin/someteam.example.com/v1/gogetter/GoGetter +curl -s --create-dirs -o \ +"$DEMO_HOME/kustomize/$plugin" \ +"https://raw.githubusercontent.com/\ +kubernetes-sigs/kustomize/master/$plugin" + +chmod a+x $DEMO_HOME/kustomize/$plugin +``` + +Define a helper function to run kustomize with the correct environment and flags for plugins: + + +```sh +function kustomizeIt { + XDG_CONFIG_HOME=$DEMO_HOME \ + kustomize build --enable_alpha_plugins \ + $DEMO_HOME/$1 +} +``` + +Finally, build the local variant. Notice that all +resource names now have the `my-` prefix: + + +```sh +clear +kustomizeIt my +``` + +Compare local variant to remote base: + + +```sh +diff <(kustomizeIt my) <(kustomizeIt base) | more +``` + +To see the unmodified but extracted sources, run kustomize on the base. Every invocation here is re-downloading and re-build the package. + + +```sh +kustomizeIt base +``` + +## Use non-kustomize remote sources + +Sometimes the remote sources does not include `kustomization.yaml`. To use that in the plugin, set command to override the default build. + + +```sh +echo "command: cat resources.yaml" >>$DEMO_HOME/base/goGetter.yaml +``` + +Finally, built it + + +```sh +kustomizeIt my +``` + +and observe the transformation from remote kustomization.yaml is not included. diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter b/plugin/someteam.example.com/v1/gogetter/GoGetter index 958b29e4061..604c2dad240 100755 --- a/plugin/someteam.example.com/v1/gogetter/GoGetter +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter @@ -11,7 +11,8 @@ # metadata: # name: example # url: github.com/kustless/kustomize-examples.git -# # overlay: base # (optional) relative path in the package +# # subPath: base # (optional) relative path in the package +# # command: cat *.yaml # (optional) build command, default `kustomize build $subPath` # # download kustomize layes and build it to stdout # @@ -20,7 +21,7 @@ # # TODO: cache downloads -set -e +set -ex # YAML parsing function borrowed from ChartInflator function parseYaml { @@ -32,7 +33,8 @@ function parseYaml { local t=${v#"${v%%[![:space:]]*}"} # trim leading space if [ "$k" == "url" ]; then url=$t - elif [ "$k" == "overlay" ]; then overlay=$t + elif [ "$k" == "subPath" ]; then subPath=$t + elif [ "$k" == "command" ]; then command=$t fi done <"$file" } @@ -40,7 +42,13 @@ function parseYaml { TMP_DIR=$(mktemp -d) parseYaml $1 + +if [ -z "$command" ]; then + command="kustomize build" +fi + go-getter $url $TMP_DIR/got 2> /dev/null -kustomize build $TMP_DIR/got/$overlay + +(cd $TMP_DIR/got/$subPath; $command) /bin/rm -rf $TMP_DIR diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go b/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go index 226c4a128d0..c7dc9701214 100644 --- a/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go @@ -30,7 +30,7 @@ apiVersion: someteam.example.com/v1 kind: GoGetter metadata: name: example -url: github.com/kustless/kustomize-examples.git +url: github.com/kustless/kustomize-examples.git?ref=adef0a8 `) th.AssertActualEqualsExpected(m, ` @@ -40,6 +40,66 @@ data: enableRisky: "false" kind: ConfigMap metadata: - name: the-map + name: remote-cm +`) +} + +func TestGoGetterCommand(t *testing.T) { + tc := plugins_test.NewEnvForTest(t).Set() + defer tc.Reset() + + tc.BuildExecPlugin( + "someteam.example.com", "v1", "GoGetter") + + th := kusttest_test.NewKustTestPluginHarness(t, "/app") + + m := th.LoadAndRunGenerator(` +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git?ref=adef0a8 +command: cat resources.yaml +`) + + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +data: + altGreeting: Good Morning! + enableRisky: "false" +kind: ConfigMap +metadata: + name: cm +`) +} + + +func TestGoGetterSubPath(t *testing.T) { + tc := plugins_test.NewEnvForTest(t).Set() + defer tc.Reset() + + tc.BuildExecPlugin( + "someteam.example.com", "v1", "GoGetter") + + th := kusttest_test.NewKustTestPluginHarness(t, "/app") + + m := th.LoadAndRunGenerator(` +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git?ref=9ca07d2 +subPath: dev +command: kustomize build --enable_alpha_plugins +`) + + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +data: + altGreeting: Good Morning! + enableRisky: "false" +kind: ConfigMap +metadata: + name: dev-remote-cm `) }