From ed91bce275328d8873afcfdcb707a6939048985e Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Fri, 30 Aug 2019 11:01:42 +0800 Subject: [PATCH 1/3] Add example plugin for go-getter --- docs/plugins/README.md | 3 +- .../someteam.example.com/v1/gogetter/GoGetter | 46 +++++++++++++++++++ .../v1/gogetter/GoGetter_test.go | 45 ++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100755 plugin/someteam.example.com/v1/gogetter/GoGetter create mode 100644 plugin/someteam.example.com/v1/gogetter/GoGetter_test.go diff --git a/docs/plugins/README.md b/docs/plugins/README.md index 730897a0ed..f250da22b6 100644 --- a/docs/plugins/README.md +++ b/docs/plugins/README.md @@ -223,6 +223,7 @@ provided in the kustomization file). [helm chart inflator]: ../../plugin/someteam.example.com/v1/chartinflator [bashed config map]: ../../plugin/someteam.example.com/v1/bashedconfigmap [sed transformer]: ../../plugin/someteam.example.com/v1/sedtransformer +[hashicorp go-getter]: ../../plugin/someteam.example.com/v1/gogetter #### Examples @@ -230,7 +231,7 @@ provided in the kustomization file). * [bashed config map] - Super simple configMap generation from bash. * [sed transformer] - Define your unstructured edits using a plugin like this one. - + * [hashicorp go-getter] - Download kustomize layes and build it to generate resources A generator plugin accepts nothing on `stdin`, but emits generated resources to `stdout`. diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter b/plugin/someteam.example.com/v1/gogetter/GoGetter new file mode 100755 index 0000000000..958b29e406 --- /dev/null +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +# HashiCorp go-getter +# +# Reads a file like this +# +# apiVersion: someteam.example.com/v1 +# kind: GoGetter +# metadata: +# name: example +# url: github.com/kustless/kustomize-examples.git +# # overlay: base # (optional) relative path in the package +# +# download kustomize layes and build it to stdout +# +# Example execution: +# ./plugin/someteam.example.com/v1/gogetter/GoGetter configFile.yaml +# +# TODO: cache downloads + +set -e + +# YAML parsing function borrowed from ChartInflator +function parseYaml { + local file=$1 + while read -r line + do + local k=${line%:*} + local v=${line#*:} + local t=${v#"${v%%[![:space:]]*}"} # trim leading space + + if [ "$k" == "url" ]; then url=$t + elif [ "$k" == "overlay" ]; then overlay=$t + fi + done <"$file" +} + +TMP_DIR=$(mktemp -d) + +parseYaml $1 +go-getter $url $TMP_DIR/got 2> /dev/null +kustomize build $TMP_DIR/got/$overlay + +/bin/rm -rf $TMP_DIR diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go b/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go new file mode 100644 index 0000000000..226c4a128d --- /dev/null +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go @@ -0,0 +1,45 @@ +// +build notravis + +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Disabled on travis, because don't want to install go-getter on travis. + +package main_test + +import ( + "testing" + + kusttest_test "sigs.k8s.io/kustomize/v3/pkg/kusttest" + plugins_test "sigs.k8s.io/kustomize/v3/pkg/plugins/test" +) + +// This test requires having the go-getter binary on the PATH. +// +func TestGoGetter(t *testing.T) { + tc := plugins_test.NewEnvForTest(t).Set() + defer tc.Reset() + + tc.BuildExecPlugin( + "someteam.example.com", "v1", "GoGetter") + + th := kusttest_test.NewKustTestPluginHarness(t, "/app") + + m := th.LoadAndRunGenerator(` +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git +`) + + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +data: + altGreeting: Good Morning! + enableRisky: "false" +kind: ConfigMap +metadata: + name: the-map +`) +} From ed920afb2e35a2cd6ecec4eaa2d4ecac898eb475 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Mon, 2 Sep 2019 10:50:36 +0800 Subject: [PATCH 2/3] Support setting command in go-getter plugin This allows one to use non-kustomization remote source --- examples/README.md | 40 +++--- examples/goGetterGeneratorPlugin.md | 126 ++++++++++++++++++ .../someteam.example.com/v1/gogetter/GoGetter | 16 ++- .../v1/gogetter/GoGetter_test.go | 64 ++++++++- 4 files changed, 221 insertions(+), 25 deletions(-) create mode 100644 examples/goGetterGeneratorPlugin.md diff --git a/examples/README.md b/examples/README.md index 9a73adf602..051e62111d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,59 +12,61 @@ Basic Usage * [configGenerations](configGeneration.md) - Rolling update when ConfigMapGenerator changes. - + * [combineConfigs](combineConfigs.md) - Mixing configuration data from different owners (e.g. devops/SRE and developers). - + * [generatorOptions](generatorOptions.md) - - Modifying behavior of all ConfigMap and Secret generators. + Modifying behavior of all ConfigMap and Secret generators. * [vars](wordpress/README.md) - Injecting k8s runtime data into container arguments (e.g. to point wordpress to a SQL service) by vars. - + * [image names and tags](image.md) - Updating image names and tags without applying a patch. - + * [remote target](remoteBuild.md) - Building a kustomization from a github URL - - * [json patch](jsonpatch.md) - Apply a json patch in a kustomization + + * [json patch](jsonpatch.md) - Apply a json patch in a kustomization * [patch multiple objects](patchMultipleObjects.md) - Apply a patch to multiple objects Advanced Usage - + - generator plugins: - + * [last mile helm](chart.md) - Make last mile modifications to a helm chart. - - * [secret generation](secretGeneratorPlugin.md) - Generating secrets from a plugin. + + * [secret generation](secretGeneratorPlugin.md) - Generating secrets from a plugin. + + * [remote sources](goGetterGeneratorPlugin.md) - Generating from remote sources. - transformer plugins: * [validation transformer](validationTransformer/README.md) - validate resources through a transformer - + - customize builtin transformer configurations - + * [transformer configs](transformerconfigs/README.md) - Customize transformer configurations - + Multi Variant Examples * [hello world](helloWorld/README.md) - Deploy multiple (differently configured) variants of a simple Hello World server. - + * [LDAP](ldap/README.md) - Deploy multiple (differently configured) variants of a LDAP server. - + * [springboot](springboot/README.md) - Create a Spring Boot application production configuration from scratch. * [mySql](mySql/README.md) - Create a MySQL production configuration from scratch. - + * [breakfast](breakfast.md) - Customize breakfast for Alice and Bob. - - * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base. + + * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base. diff --git a/examples/goGetterGeneratorPlugin.md b/examples/goGetterGeneratorPlugin.md new file mode 100644 index 0000000000..e957d5a5ad --- /dev/null +++ b/examples/goGetterGeneratorPlugin.md @@ -0,0 +1,126 @@ +# Remote Sources + +Kustomize supports building a [remote target], but the URLs are limited to common [Git repository specs]. + +To extend the supported format, Kustomize has a [plugin] system that allows one to integrate third-party tools such as [hashicorp/go-getter] to "download things from a string URL suing a variety of protocols", extract the content and generated resources as part of kustomize build. + +[remote target]: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md +[Git repository specs]: https://github.com/kubernetes-sigs/kustomize/blob/master/pkg/git/repospec_test.go +[plugin]: ../docs/plugins +[hashicorp/go-getter]: https://github.com/hashicorp/go-getter + +## Make a place to work + + +```sh +DEMO_HOME=$(mktemp -d) +``` + +## Use a remote kustomize layer + +Define a kustomization representing your _local_ variant (aka environment). + +This could involve any number of kustomizations (see other examples), but in this case just add the name prefix `my-` to all resources: + + +```sh +cat <<'EOF' >$DEMO_HOME/my/kustomization.yaml +namePrefix: my- +resources: +- ../base +EOF +``` + +It refer a remote base defined as below: + + +```sh +cat <<'EOF' >$DEMO_HOME/base/kustomization.yaml +generators: +- goGetter.yaml +EOF +``` + +The base refers to a generator configuration file called `goGetter.yaml`. + +This file lets one specify the source URL, and other things like sub path in the package, defaulting to base directory, and command to run under the path, defaulting to `kustomize build`. + +Create the config file `goGetter.yaml`, specifying the arbitrarily chosen name _example_: + + +```sh +cat <<'EOF' >$DEMO_HOME/base/goGetter.yaml +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git +EOF +``` + +Because this particular YAML file is listed in the `generators:` stanza of a kustomization file, it is treated as the binding between a generator plugin - identified by the _apiVersion_ and _kind_ fields - and other fields that configure the plugin. + +Download the plugin to your `DEMO_HOME` and make it executable: + + +```sh +plugin=plugin/someteam.example.com/v1/gogetter/GoGetter +curl -s --create-dirs -o \ +"$DEMO_HOME/kustomize/$plugin" \ +"https://raw.githubusercontent.com/\ +kubernetes-sigs/kustomize/master/$plugin" + +chmod a+x $DEMO_HOME/kustomize/$plugin +``` + +Define a helper function to run kustomize with the correct environment and flags for plugins: + + +```sh +function kustomizeIt { + XDG_CONFIG_HOME=$DEMO_HOME \ + kustomize build --enable_alpha_plugins \ + $DEMO_HOME/$1 +} +``` + +Finally, build the local variant. Notice that all +resource names now have the `my-` prefix: + + +```sh +clear +kustomizeIt my +``` + +Compare local variant to remote base: + + +```sh +diff <(kustomizeIt my) <(kustomizeIt base) | more +``` + +To see the unmodified but extracted sources, run kustomize on the base. Every invocation here is re-downloading and re-build the package. + + +```sh +kustomizeIt base +``` + +## Use non-kustomize remote sources + +Sometimes the remote sources does not include `kustomization.yaml`. To use that in the plugin, set command to override the default build. + + +```sh +echo "command: cat resources.yaml" >>$DEMO_HOME/base/goGetter.yaml +``` + +Finally, built it + + +```sh +kustomizeIt my +``` + +and observe the transformation from remote kustomization.yaml is not included. diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter b/plugin/someteam.example.com/v1/gogetter/GoGetter index 958b29e406..604c2dad24 100755 --- a/plugin/someteam.example.com/v1/gogetter/GoGetter +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter @@ -11,7 +11,8 @@ # metadata: # name: example # url: github.com/kustless/kustomize-examples.git -# # overlay: base # (optional) relative path in the package +# # subPath: base # (optional) relative path in the package +# # command: cat *.yaml # (optional) build command, default `kustomize build $subPath` # # download kustomize layes and build it to stdout # @@ -20,7 +21,7 @@ # # TODO: cache downloads -set -e +set -ex # YAML parsing function borrowed from ChartInflator function parseYaml { @@ -32,7 +33,8 @@ function parseYaml { local t=${v#"${v%%[![:space:]]*}"} # trim leading space if [ "$k" == "url" ]; then url=$t - elif [ "$k" == "overlay" ]; then overlay=$t + elif [ "$k" == "subPath" ]; then subPath=$t + elif [ "$k" == "command" ]; then command=$t fi done <"$file" } @@ -40,7 +42,13 @@ function parseYaml { TMP_DIR=$(mktemp -d) parseYaml $1 + +if [ -z "$command" ]; then + command="kustomize build" +fi + go-getter $url $TMP_DIR/got 2> /dev/null -kustomize build $TMP_DIR/got/$overlay + +(cd $TMP_DIR/got/$subPath; $command) /bin/rm -rf $TMP_DIR diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go b/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go index 226c4a128d..c7dc970121 100644 --- a/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter_test.go @@ -30,7 +30,7 @@ apiVersion: someteam.example.com/v1 kind: GoGetter metadata: name: example -url: github.com/kustless/kustomize-examples.git +url: github.com/kustless/kustomize-examples.git?ref=adef0a8 `) th.AssertActualEqualsExpected(m, ` @@ -40,6 +40,66 @@ data: enableRisky: "false" kind: ConfigMap metadata: - name: the-map + name: remote-cm +`) +} + +func TestGoGetterCommand(t *testing.T) { + tc := plugins_test.NewEnvForTest(t).Set() + defer tc.Reset() + + tc.BuildExecPlugin( + "someteam.example.com", "v1", "GoGetter") + + th := kusttest_test.NewKustTestPluginHarness(t, "/app") + + m := th.LoadAndRunGenerator(` +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git?ref=adef0a8 +command: cat resources.yaml +`) + + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +data: + altGreeting: Good Morning! + enableRisky: "false" +kind: ConfigMap +metadata: + name: cm +`) +} + + +func TestGoGetterSubPath(t *testing.T) { + tc := plugins_test.NewEnvForTest(t).Set() + defer tc.Reset() + + tc.BuildExecPlugin( + "someteam.example.com", "v1", "GoGetter") + + th := kusttest_test.NewKustTestPluginHarness(t, "/app") + + m := th.LoadAndRunGenerator(` +apiVersion: someteam.example.com/v1 +kind: GoGetter +metadata: + name: example +url: github.com/kustless/kustomize-examples.git?ref=9ca07d2 +subPath: dev +command: kustomize build --enable_alpha_plugins +`) + + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +data: + altGreeting: Good Morning! + enableRisky: "false" +kind: ConfigMap +metadata: + name: dev-remote-cm `) } From 6a4150d199cf078a7fc369f372fcc8d30a5f46a6 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Thu, 5 Sep 2019 13:56:37 +0800 Subject: [PATCH 3/3] Amend go-getter plugin document according to comments --- examples/goGetterGeneratorPlugin.md | 42 +++++++++++-------- .../someteam.example.com/v1/gogetter/GoGetter | 2 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/examples/goGetterGeneratorPlugin.md b/examples/goGetterGeneratorPlugin.md index e957d5a5ad..16a347f174 100644 --- a/examples/goGetterGeneratorPlugin.md +++ b/examples/goGetterGeneratorPlugin.md @@ -11,9 +11,10 @@ To extend the supported format, Kustomize has a [plugin] system that allows one ## Make a place to work - + ```sh DEMO_HOME=$(mktemp -d) +mkdir -p $DEMO_HOME/base ``` ## Use a remote kustomize layer @@ -22,18 +23,18 @@ Define a kustomization representing your _local_ variant (aka environment). This could involve any number of kustomizations (see other examples), but in this case just add the name prefix `my-` to all resources: - + ```sh -cat <<'EOF' >$DEMO_HOME/my/kustomization.yaml +cat <<'EOF' >$DEMO_HOME/kustomization.yaml namePrefix: my- resources: -- ../base +- base/ EOF ``` It refer a remote base defined as below: - + ```sh cat <<'EOF' >$DEMO_HOME/base/kustomization.yaml generators: @@ -47,7 +48,7 @@ This file lets one specify the source URL, and other things like sub path in the Create the config file `goGetter.yaml`, specifying the arbitrarily chosen name _example_: - + ```sh cat <<'EOF' >$DEMO_HOME/base/goGetter.yaml apiVersion: someteam.example.com/v1 @@ -62,7 +63,7 @@ Because this particular YAML file is listed in the `generators:` stanza of a kus Download the plugin to your `DEMO_HOME` and make it executable: - + ```sh plugin=plugin/someteam.example.com/v1/gogetter/GoGetter curl -s --create-dirs -o \ @@ -75,7 +76,7 @@ chmod a+x $DEMO_HOME/kustomize/$plugin Define a helper function to run kustomize with the correct environment and flags for plugins: - + ```sh function kustomizeIt { XDG_CONFIG_HOME=$DEMO_HOME \ @@ -87,22 +88,27 @@ function kustomizeIt { Finally, build the local variant. Notice that all resource names now have the `my-` prefix: - + ```sh clear -kustomizeIt my +kustomizeIt ``` Compare local variant to remote base: - + ```sh -diff <(kustomizeIt my) <(kustomizeIt base) | more +diff <(kustomizeIt) <(kustomizeIt base) | more + +... +< name: my-remote-cm +--- +> name: remote-cm ``` -To see the unmodified but extracted sources, run kustomize on the base. Every invocation here is re-downloading and re-build the package. +To see the unmodified but extracted sources, run kustomize on the base. Every invocation here is re-downloading and re-building the package. - + ```sh kustomizeIt base ``` @@ -111,16 +117,16 @@ kustomizeIt base Sometimes the remote sources does not include `kustomization.yaml`. To use that in the plugin, set command to override the default build. - + ```sh echo "command: cat resources.yaml" >>$DEMO_HOME/base/goGetter.yaml ``` Finally, built it - + ```sh -kustomizeIt my +kustomizeIt ``` -and observe the transformation from remote kustomization.yaml is not included. +and observe the output includes raw `resources.yaml` instead of building result of remote `kustomization.yaml`. diff --git a/plugin/someteam.example.com/v1/gogetter/GoGetter b/plugin/someteam.example.com/v1/gogetter/GoGetter index 604c2dad24..efdd566d32 100755 --- a/plugin/someteam.example.com/v1/gogetter/GoGetter +++ b/plugin/someteam.example.com/v1/gogetter/GoGetter @@ -21,7 +21,7 @@ # # TODO: cache downloads -set -ex +set -e # YAML parsing function borrowed from ChartInflator function parseYaml {