Merge pull request #707 from wongma7/ekse2e

Support e2e test EFS create on EKS clusters by finding EKS node subnets

Dockerfile

@@ -26,9 +26,9 @@ ADD . .
 ARG client_source=k8s
 ENV EFS_CLIENT_SOURCE=$client_source
 
-RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make aws-efs-csi-driver
+RUN OS=${TARGETOS} ARCH=${TARGETARCH} make $TARGETOS/$TARGETARCH
 
-FROM amazonlinux:2
+FROM amazonlinux:2 as linux-amazon
 RUN yum update -y
 # Install efs-utils from github by default. It can be overriden to `yum` with --build-arg when building the Docker image.
 # If value of `EFSUTILSSOURCE` build arg is overriden with `yum`, docker will install efs-utils from Amazon Linux 2's yum repo.

Makefile

@@ -13,29 +13,90 @@
 # limitations under the License.
 #
 
-PKG=github.com/kubernetes-sigs/aws-efs-csi-driver
-IMAGE?=amazon/aws-efs-csi-driver
 VERSION=v1.3.8
+
+PKG=github.com/kubernetes-sigs/aws-efs-csi-driver
 GIT_COMMIT?=$(shell git rev-parse HEAD)
 BUILD_DATE?=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
+
 EFS_CLIENT_SOURCE?=k8s
-IMAGE_PLATFORMS?=linux/arm64,linux/amd64
 LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} \
 		  -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} \
 		  -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE} \
 		  -X ${PKG}/pkg/driver.efsClientSource=${EFS_CLIENT_SOURCE}"
+
 GO111MODULE=on
 GOPROXY=direct
 GOPATH=$(shell go env GOPATH)
 GOOS=$(shell go env GOOS)
 
+REGISTRY?=amazon
+IMAGE?=$(REGISTRY)/aws-efs-csi-driver
+TAG?=$(GIT_COMMIT)
+
+OUTPUT_TYPE?=docker
+
+OS?=linux
+ARCH?=amd64
+OSVERSION?=amazon
+
+ALL_OS?=linux
+ALL_ARCH_linux?=amd64 arm64
+ALL_OSVERSION_linux?=amazon
+ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversion, ${ALL_OSVERSION_linux}, linux-$(arch)-${osversion}))
+
+ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}})
+
+PLATFORM?=linux/amd64,linux/arm64
+
+# split words on hyphen, access by 1-index
+word-hyphen = $(word $2,$(subst -, ,$1))
+
 .EXPORT_ALL_VARIABLES:
 
-.PHONY: aws-efs-csi-driver
-aws-efs-csi-driver:
-	mkdir -p bin
-	@echo GOARCH:${GOARCH}
-	CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags ${LDFLAGS} -o bin/aws-efs-csi-driver ./cmd/
+.PHONY: linux/$(ARCH) bin/aws-efs-csi-driver
+linux/$(ARCH): bin/aws-efs-csi-driver
+bin/aws-efs-csi-driver: | bin
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod=vendor -ldflags ${LDFLAGS} -o bin/aws-efs-csi-driver ./cmd/
+
+.PHONY: all
+all: all-image-docker
+
+.PHONY: all-push
+all-push:
+	docker buildx build \
+		--no-cache-filter=linux-amazon \
+		--platform=$(PLATFORM) \
+		--progress=plain \
+		--target=$(OS)-$(OSVERSION) \
+		--output=type=registry \
+		-t=$(IMAGE):$(TAG) \
+		.
+	touch $@
+
+.PHONY: all-image-docker
+all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux))
+
+sub-image-%:
+	$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image
+
+.PHONY: image
+image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION)
+.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION):
+	docker buildx build \
+		--no-cache-filter=linux-amazon \
+		--platform=$(OS)/$(ARCH) \
+		--progress=plain \
+		--target=$(OS)-$(OSVERSION) \
+		--output=type=$(OUTPUT_TYPE) \
+		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
+		.
+	touch $@
+
+
+.PHONY: clean
+clean:
+	rm -rf .*image-* bin/
 
 bin /tmp/helm:
 	@mkdir -p $@
@@ -45,13 +106,6 @@ bin/helm: | /tmp/helm bin
 	@tar -zxf /tmp/helm/helm.tar.gz -C bin --strip-components=1
 	@rm -rf /tmp/helm/*
 
-build-darwin:
-	mkdir -p bin/darwin/
-	CGO_ENABLED=0 GOOS=darwin go build -mod=vendor -ldflags ${LDFLAGS} -o bin/darwin/aws-efs-csi-driver ./cmd/
-
-run-darwin: build-darwin
-	bin/darwin/aws-efs-csi-driver --version
-
 .PHONY: verify
 verify:
 	./hack/verify-all
@@ -76,30 +130,6 @@ test-e2e-bin:
 	mkdir -p bin
 	CGO_ENABLED=0 GOOS=linux go test -mod=vendor -ldflags ${LDFLAGS} -c -o bin/test-e2e ./test/e2e/
 
-.PHONY: image
-image:
-	docker build -t $(IMAGE):master .
-
-.PHONY: image-multi-arch--push
-image-multi-arch-push:
-	docker buildx build \
-			  -t $(IMAGE):master \
-			  --platform=$(IMAGE_PLATFORMS) \
-			  --progress plain \
-			  --push .
-
-.PHONY: push
-push: image
-	docker push $(IMAGE):master
-
-.PHONY: image-release
-image-release:
-	docker build -t $(IMAGE):$(VERSION) .
-
-.PHONY: push-release
-push-release:
-	docker push $(IMAGE):$(VERSION)
-
 .PHONY: generate-kustomize
 generate-kustomize: bin/helm
 	cd charts/aws-efs-csi-driver && ../../bin/helm template kustomize . -s templates/csidriver.yaml > ../../deploy/kubernetes/base/csidriver.yaml

hack/e2e/ecr.sh

@@ -18,7 +18,8 @@ function ecr_build_and_push() {
     set -e
     loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
     aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
-    docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" .
+    IMAGE=${IMAGE_NAME} TAG=${IMAGE_TAG} OS=linux ARCH=amd64 OSVERSION=amazon make image
+    docker tag "${IMAGE_NAME}":"${IMAGE_TAG}"-linux-amd64-amazon "${IMAGE_NAME}":"${IMAGE_TAG}"
     docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
   fi
 }

hack/e2e/eksctl.sh

@@ -6,7 +6,7 @@ function eksctl_install() {
   INSTALL_PATH=${1}
   EKSCTL_VERSION=${2}
   if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then
-    EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz"
+    EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz"
     curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
     chmod +x "${INSTALL_PATH}"/eksctl
   fi
@@ -55,6 +55,9 @@ function eksctl_create_cluster() {
   fi
 
   loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"
+  # TODO: Workaround for https://github.com/weaveworks/eksctl/issues/5257
+  # Remove when eksctl releases a fix
+  sed -i 's/v1alpha1/v1beta1/g' ${KUBECONFIG}
 
   loudecho "Getting cluster ${CLUSTER_NAME}"
   ${BIN} get cluster "${CLUSTER_NAME}"
@@ -68,6 +71,7 @@ function eksctl_create_cluster() {
 
   if [[ "$WINDOWS" == true ]]; then
     ${BIN} create nodegroup \
+      --managed=false \
       --cluster="${CLUSTER_NAME}" \
       --node-ami-family=WindowsServer2019FullContainer \
       -n ng-windows \

hack/e2e/run.sh

@@ -42,7 +42,7 @@ REGION=${AWS_REGION:-us-west-2}
 ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
 FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
 NODE_COUNT=${NODE_COUNT:-3}
-INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large}
+INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
 
 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
 IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}}
@@ -57,7 +57,7 @@ KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
 KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml}
 
-EKSCTL_VERSION=${EKSCTL_VERSION:-0.56.0-rc.1}
+EKSCTL_VERSION=${EKSCTL_VERSION:-0.69.0}
 EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml}
 EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-}
 # Creates a windows node group. The windows ami doesn't (yet) install csi-proxy

hack/eksctl-patch.yaml

@@ -0,0 +1,9 @@
+iam:
+  vpcResourceControllerPolicy: true
+  withOIDC: true
+  serviceAccounts:
+    - metadata:
+        name: efs-csi-controller-sa
+        namespace: kube-system
+      wellKnownPolicies:
+        efsCSIController: true

hack/values_eksctl.yaml

@@ -0,0 +1,6 @@
+controller:
+  logLevel: 5
+  serviceAccount:
+    create: false # let eksctl create it
+node:
+  logLevel: 5