ci(*): update int tests to use kind

- add Makefile.kind with kind-centric targets - update cli/int tests (and test bundles) to use local docker registry - update auto PR az pipeline to run cli/int tests - delete manual PR pipeline - add doc on using KinD w/ Porter Signed-off-by: Vaughn Dice <[email protected]>
getporter · Jul 17, 2020 · 15f30e9 · 15f30e9
1 parent 6879e18
commit 15f30e9
Show file tree

Hide file tree

Showing 13 changed files with 223 additions and 178 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,4 @@ workshop/**/Dockerfile
 
 .vscode
 .idea
+kind.config.yaml
diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@ PORTER_HOME = bin
 
 CLIENT_PLATFORM = $(shell go env GOOS)
 CLIENT_ARCH = $(shell go env GOARCH)
-CLIENT_GOPATH=$(shell go env GOPATH)
+CLIENT_GOPATH = $(shell go env GOPATH)
 RUNTIME_PLATFORM = linux
 RUNTIME_ARCH = amd64
 BASEURL_FLAG ?= 
@@ -125,6 +125,14 @@ publish-mixins:
 publish-images:
 	VERSION=$(VERSION) PERMALINK=$(PERMALINK) ./scripts/publish-images.sh
 
+start-local-docker-registry:
+	@docker run -d -p 5000:5000 --name registry registry:2
+
+stop-local-docker-registry:
+	@if $$(docker inspect registry > /dev/null 2>&1); then \
+		docker kill registry && docker rm registry ; \
+	fi
+
 # all-bundles loops through all items under the dir provided by the first argument
 # and if the item is a sub-directory containing a porter.yaml file,
 # runs the make target(s) provided by the second argument

diff --git a/Makefile.kind b/Makefile.kind
@@ -0,0 +1,50 @@
+SHELL           = /bin/bash
+CLIENT_PLATFORM = $(shell go env GOOS)
+CLIENT_ARCH     = $(shell go env GOARCH)
+
+# Determine host ip to populate kind config api server details
+# https://kind.sigs.k8s.io/docs/user/configuration/#api-server
+ifeq ($(CLIENT_PLATFORM),linux)
+export CURRENT_HOST_IP=$(shell hostname -i | awk '{print $1; exit}' | cut -d ' ' -f 1)
+else
+export CURRENT_HOST_IP=$(shell ifconfig en0 | awk '/inet / {print $2; }' | cut -d ' ' -f 2)
+endif
+
+## Create file definition for the kind cluster
+export KIND_CONFIG_FILE_NAME = kind.config.yaml
+define get_kind_config_file
+# Remove config file
+rm -rf ${KIND_CONFIG_FILE_NAME}
+# Define config file
+cat << EOF >> ${KIND_CONFIG_FILE_NAME}
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerAddress: "${CURRENT_HOST_IP}"
+  apiServerPort: 6443
+EOF
+endef
+export KIND_CLUSTER_FILE_CREATOR = $(value get_kind_config_file)
+
+KIND_VERSION ?= v0.8.1
+CLUSTER_NAME  = porter
+
+install-kind:
+	@curl -Lo /tmp/kind https://github.com/kubernetes-sigs/kind/releases/download/$(KIND_VERSION)/kind-$(CLIENT_PLATFORM)-$(CLIENT_ARCH)
+	@chmod +x /tmp/kind
+	@sudo mv /tmp/kind /usr/local/bin/kind
+
+create-kind-config-file:; @eval "$$KIND_CLUSTER_FILE_CREATOR"
+
+# TODO: remove need to manually set server value if darwin
+# https://github.com/kubernetes-sigs/kind/issues/1732
+create-kind-cluster: create-kind-config-file
+	@kind create cluster \
+		--name ${CLUSTER_NAME} \
+		--config ${KIND_CONFIG_FILE_NAME}
+	@if [[ "$(CLIENT_PLATFORM)" == "darwin" ]]; then \
+		kubectl config set clusters.kind-${CLUSTER_NAME}.server https://${CURRENT_HOST_IP}:6443 ; \
+	fi
+
+delete-kind-cluster:
+	@kind delete cluster --name ${CLUSTER_NAME}
diff --git a/build/azure-pipelines.pr-automatic.yml b/build/azure-pipelines.pr-automatic.yml
@@ -75,3 +75,45 @@ jobs:
       make build-bundle validate-bundle
     workingDirectory: '$(System.DefaultWorkingDirectory)'
     displayName: 'Validate Example Bundles'
+
+- job: integration_test
+  dependsOn: setup
+  condition: eq(dependencies.setup.outputs['BUILD.DOCS_ONLY'], 'false')
+  steps:
+  - task: GoTool@0
+    inputs:
+      version: '$(GOVERSION)'
+  - script: build/azure-pipelines.setup-go-workspace.sh
+    displayName: 'Set up the Go workspace'
+
+  - script: |
+      make build
+    workingDirectory: '$(System.DefaultWorkingDirectory)'
+    displayName: 'Build'
+
+  - script: |
+      trap 'make -f Makefile.kind delete-kind-cluster' EXIT
+      make -f Makefile.kind install-kind create-kind-cluster
+      make start-local-docker-registry test-integration stop-local-docker-registry
+    workingDirectory: '$(System.DefaultWorkingDirectory)'
+    displayName: 'Integration Test'
+
+- job: cli_test
+  dependsOn: setup
+  condition: eq(dependencies.setup.outputs['BUILD.DOCS_ONLY'], 'false')
+  steps:
+  - task: GoTool@0
+    inputs:
+      version: '$(GOVERSION)'
+  - script: build/azure-pipelines.setup-go-workspace.sh
+    displayName: 'Set up the Go workspace'
+
+  - script: |
+       make build
+    workingDirectory: '$(System.DefaultWorkingDirectory)'
+    displayName: 'Build'
+
+  - script: |
+      make start-local-docker-registry test-cli stop-local-docker-registry
+    workingDirectory: '$(System.DefaultWorkingDirectory)'
+    displayName: 'CLI Test'
diff --git a/build/azure-pipelines.pr-manual.yml b/build/azure-pipelines.pr-manual.yml
diff --git a/build/azure-pipelines.release.yml b/build/azure-pipelines.release.yml
@@ -42,21 +42,10 @@ stages:
     - script: build/azure-pipelines.setup-go-workspace.sh
       displayName: 'Set up the Go workspace'
 
-    - task: Docker@1
-      displayName: Docker Login
-      inputs:
-        containerRegistryType: Container Registry
-        dockerRegistryEndpoint: getporter-registry
-        command: login
-
-    - task: DownloadSecureFile@1
-      displayName: Download Kubeconfig
-      inputs:
-        secureFile: kubeconfig
-
     - script: |
-        export KUBECONFIG=$DOWNLOADSECUREFILE_SECUREFILEPATH
-        make test-integration
+        trap 'make -f Makefile.kind delete-kind-cluster' EXIT
+        make -f Makefile.kind install-kind create-kind-cluster
+        make start-local-docker-registry test-integration stop-local-docker-registry
       workingDirectory: '$(System.DefaultWorkingDirectory)'
       displayName: 'Integration Test'
 
@@ -68,26 +57,12 @@ stages:
     - script: build/azure-pipelines.setup-go-workspace.sh
       displayName: 'Set up the Go workspace'
 
-    - task: Docker@1
-      displayName: Docker Login
-      inputs:
-        containerRegistryType: Container Registry
-        dockerRegistryEndpoint: getporter-registry
-        command: login
-
-    - task: DownloadSecureFile@1
-      displayName: Download Kubeconfig
-      inputs:
-        secureFile: kubeconfig
-
     - script: make build
       workingDirectory: '$(System.DefaultWorkingDirectory)'
       displayName: 'Build'
 
     - script: |
-        export KUBECONFIG=$DOWNLOADSECUREFILE_SECUREFILEPATH
-        export REGISTRY=getporterci
-        make test-cli
+        make start-local-docker-registry test-cli stop-local-docker-registry
       workingDirectory: '$(System.DefaultWorkingDirectory)'
       displayName: 'CLI Test'
 

diff --git a/build/brigade.js b/build/brigade.js
@@ -1,4 +1,5 @@
 const { events, Job, Group } = require("brigadier");
+const { KindJob } = require("@brigadecore/brigade-utils");
 
 // **********************************************
 // Globals
@@ -18,7 +19,6 @@ events.on("issue_comment:edited", handleIssueComment);
 
 events.on("exec", (e, p) => {
   return Group.runAll([
-    verify(e, p),
     build(e, p),
     xbuild(e, p),
     testUnit(e, p),
@@ -28,6 +28,13 @@ events.on("exec", (e, p) => {
   ]);
 });
 
+events.on("test-integration", (e, p) => {
+  return Group.runAll([
+    testIntegration(e, p),
+    testCLI(e, p)
+  ]);
+})
+
 // Although a GH App will trigger 'check_suite:requested' on a push to main branch event,
 // it will not for a tag push, hence the need for this handler
 events.on("push", (e, p) => {
@@ -62,16 +69,6 @@ events.on("publish-examples", (e, p) => {
 
 // Important: each Job name below must only consist of lowercase
 // alphanumeric characters and hyphens, per K8s resource name restrictions
-function verify(e, p) {
-  var goBuild = new GoJob(`${projectName}-verify`);
-
-  goBuild.tasks.push(
-    "make verify"
-  );
-
-  return goBuild;
-}
-
 function build(e, p) {
   var goBuild = new GoJob(`${projectName}-build`);
 
@@ -120,53 +117,29 @@ function testUnit(e, p) {
 // TODO: we could refactor so that this job shares a mount with the build job above,
 // to remove the need of re-building before running test-cli
 function testIntegration(e, p) {
-  var goTest = new GoJob(`${projectName}-testintegration`);
-  // Enable Docker-in-Docker
-  goTest.enableDind();
-
-  goTest.env.kubeconfig = {
-    secretKeyRef: {
-      name: "porter-kubeconfig",
-      key: "kubeconfig"
-    }
-  };
-
-  // Set up kubeconfig, docker login, run tests
-  goTest.tasks.push(
-    "mkdir -p ${HOME}/.kube",
-    'echo "${kubeconfig}" > ${HOME}/.kube/config',
-    `docker login ${p.secrets.dockerhubRegistry} \
-      -u ${p.secrets.dockerhubUsername} \
-      -p ${p.secrets.dockerhubPassword}`,
-    `REGISTRY=${p.secrets.dockerhubOrg} make test-integration`
+  var testInt = new KindJob(`${projectName}-testintegration`, "vdice/go-dind:kind-v0.7.0");
+
+  testInt.tasks.push(
+    "mkdir -p /go/bin",
+    "cd /src",
+    "trap 'make -f Makefile.kind delete-kind-cluster' EXIT",
+    `make -f Makefile.kind create-kind-cluster`,
+    "make start-local-docker-registry test-integration stop-local-docker-registry"
   );
 
-  return goTest;
+  return testInt;
 }
 
 function testCLI(e, p) {
-  var goTest = new GoJob(`${projectName}-testcli`);
+  var testCLI = new GoJob(`${projectName}-testcli`);
   // Enable Docker-in-Docker
-  goTest.enableDind();
+  testCLI.enableDind();
 
-  goTest.env.kubeconfig = {
-    secretKeyRef: {
-      name: "porter-kubeconfig",
-      key: "kubeconfig"
-    }
-  };
-
-  // Set up kubeconfig, docker login, run tests
-  goTest.tasks.push(
-    "mkdir -p ${HOME}/.kube",
-    'echo "${kubeconfig}" > ${HOME}/.kube/config',
-    `docker login ${p.secrets.dockerhubRegistry} \
-    -u ${p.secrets.dockerhubUsername} \
-    -p ${p.secrets.dockerhubPassword}`,
-    `REGISTRY=${p.secrets.dockerhubOrg} make test-cli`
+  testCLI.tasks.push(
+    "make start-local-docker-registry test-cli stop-local-docker-registry"
   );
 
-  return goTest;
+  return testCLI;
 }
 
 function publishExamples(e, p) {
@@ -214,7 +187,6 @@ function publish(e, p) {
 // using data supplied by a corresponding GitHub webhook, say, on a
 // check_run:rerequested event (see runCheck below)
 checks = {
-  "verify": { runFunc: verify, description: "Verify" },
   "build": { runFunc: build, description: "Build" },
   "validate": { runFunc: validate, description: "Validate" },
   "crossplatformbuild": { runFunc: xbuild, description: "Cross-Platform Build" },
@@ -302,7 +274,7 @@ class GoJob extends Job {
     const gopath = "/go";
     const localPath = gopath + `/src/get.porter.sh/${projectName}`;
 
-    this.image = "quay.io/vdice/go-dind:v0.1.0";
+    this.image = "quay.io/vdice/go-dind:v0.1.2";
 
     this.tasks = [
       // Need to move the source into GOPATH so vendor/ works as desired.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,3 +10,4 @@ workshop/**/Dockerfile

		.vscode
		.idea
		kind.config.yaml