Merge pull request #355 from wongma7/ci-script-asdfff

Use aws-ebs-csi-driver/hack/e2e scripts for testing

Makefile

@@ -53,8 +53,14 @@ test:
 
 .PHONY: test-e2e
 test-e2e:
-	go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
-	TESTCONFIG=./tester/e2e-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
+	DRIVER_NAME=aws-efs-csi-driver \
+	CONTAINER_NAME=efs-plugin \
+	TEST_EXTRA_FLAGS='--cluster-name=$$CLUSTER_NAME' \
+	AWS_REGION=us-west-2 \
+	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
+	TEST_PATH=./test/e2e/... \
+	GINKGO_FOCUS="\[efs-csi\]" \
+	./hack/e2e/run.sh
 
 .PHONY: test-e2e-bin
 test-e2e-bin:

hack/additional-policies.yaml

@@ -0,0 +1,14 @@
+  additionalPolicies:
+    node: |
+      [
+        {
+          "Effect": "Allow",
+          "Action": [
+            "elasticfilesystem:CreateAccessPoint",
+            "elasticfilesystem:DeleteAccessPoint",
+            "elasticfilesystem:DescribeFileSystems",
+            "elasticfilesystem:DescribeAccessPoints"
+          ],
+          "Resource": "*"
+        }
+      ]

hack/e2e/.gitignore

@@ -0,0 +1 @@
+/csi-test-artifacts

hack/e2e/README.md

@@ -0,0 +1,47 @@
+# Usage
+
+run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up.
+
+See below for an example.
+
+KOPS_STATE_FILE is an S3 bucket you have write access to.
+
+TEST_ID is a token used for idempotency.
+
+For more details, see the script itself.
+
+For more examples, see the top-level Makefile.
+
+```
+TEST_PATH=./tests/e2e-migration/... \
+EBS_CHECK_MIGRATION=true \
+TEST_ID=18512 \
+CLEAN=false \
+KOPS_STATE_FILE=s3://mattwon \
+AWS_REGION=us-west-2 \
+AWS_AVAILABILITY_ZONES=us-west-2a \
+GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \
+GINKGO_NODES=1 \
+./hack/e2e/run.sh
+```
+
+# git read-tree
+
+Reference: https://stackoverflow.com/questions/23937436/add-subdirectory-of-remote-repo-with-git-subtree
+
+How to consume this directory by read-treeing the ebs repo:
+
+```
+git remote add ebs [email protected]:kubernetes-sigs/aws-ebs-csi-driver.git
+git fetch ebs
+git read-tree --prefix=hack/e2e/ -u ebs/master:hack/e2e
+```
+
+To commit changes and submit them as a PR back to the ebs repo:
+
+```
+git diff ebs/master:hack/e2e HEAD:hack/e2e > /tmp/hack_e2e.diff
+cd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver
+git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
+git commit
+```

hack/e2e/ebs.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -uo pipefail
+
+BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${BASE_DIR}"/util.sh
+
+function ebs_check_migration() {
+  loudecho "Checking migration"
+  # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
+  # Find the controller-manager log and read its metrics to verify
+  NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
+  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &
+
+  # Ensure port forwarding succeeded
+  n=0
+  until [ "$n" -ge 30 ]; do
+    set +e
+    HEALTHZ=$(curl -s 127.0.0.1:10252/healthz)
+    set -e
+    if [[ ${HEALTHZ} == "ok" ]]; then
+      loudecho "Port forwarding succeeded"
+      break
+    else
+      loudecho "Port forwarding is not yet ready"
+    fi
+    n=$((n + 1))
+    sleep 1
+  done
+  if [[ "$n" -eq 30 ]]; then
+    loudecho "Timed out waiting for port forward"
+    for PROC in $(jobs -p); do
+      kill "${PROC}"
+    done
+    return 1
+  fi
+
+  set +e
+  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
+  CSI_CALLED=${PIPESTATUS[1]}
+  set -e
+
+  set +e
+  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
+  INTREE_CALLED=${PIPESTATUS[1]}
+  set -e
+
+  for PROC in $(jobs -p); do
+    kill "${PROC}"
+  done
+
+  loudecho "CSI_CALLED: ${CSI_CALLED}"
+  loudecho "INTREE_CALLED: ${INTREE_CALLED}"
+
+  # if CSI was called and In-tree was not called, return 0/true/success
+  if [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then
+    echo 0
+  else
+    echo 1
+  fi
+}

hack/e2e/ecr.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -uo pipefail
+
+BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${BASE_DIR}"/util.sh
+
+function ecr_build_and_push() {
+  REGION=${1}
+  AWS_ACCOUNT_ID=${2}
+  IMAGE_NAME=${3}
+  IMAGE_TAG=${4}
+  set +e
+  if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
+    set -e
+    loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
+  else
+    set -e
+    loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
+    aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
+    docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" .
+    docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
+  fi
+}

hack/e2e/helm.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -uo pipefail
+
+function helm_install() {
+  INSTALL_PATH=${1}
+  if [[ ! -e ${INSTALL_PATH}/helm ]]; then
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+    chmod 700 get_helm.sh
+    export USE_SUDO=false
+    export HELM_INSTALL_DIR=${INSTALL_PATH}
+    ./get_helm.sh
+    rm get_helm.sh
+  fi
+}

hack/e2e/kops.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -uo pipefail
+
+OS_ARCH=$(go env GOOS)-amd64
+
+BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${BASE_DIR}"/util.sh
+
+function kops_install() {
+  INSTALL_PATH=${1}
+  KOPS_VERSION=${2}
+  if [[ ! -e ${INSTALL_PATH}/kops ]]; then
+    KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH}
+    curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops
+    chmod +x "${INSTALL_PATH}"/kops
+  fi
+}
+
+function kops_create_cluster() {
+  SSH_KEY_PATH=${1}
+  KOPS_STATE_FILE=${2}
+  CLUSTER_NAME=${3}
+  KOPS_BIN=${4}
+  ZONES=${5}
+  INSTANCE_TYPE=${6}
+  K8S_VERSION=${7}
+  TEST_DIR=${8}
+  KOPS_FEATURE_GATES_FILE=${10}
+  KOPS_ADDITIONAL_POLICIES_FILE=${11}
+
+  loudecho "Generating SSH key $SSH_KEY_PATH"
+  if [[ ! -e ${SSH_KEY_PATH} ]]; then
+    ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
+  fi
+
+  set +e
+  if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
+    set -e
+    loudecho "Updating cluster $CLUSTER_NAME"
+  else
+    set -e
+    loudecho "Creating cluster $CLUSTER_NAME"
+    ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
+      --zones "${ZONES}" \
+      --node-count=3 \
+      --node-size="${INSTANCE_TYPE}" \
+      --kubernetes-version="${K8S_VERSION}" \
+      --ssh-public-key="${SSH_KEY_PATH}".pub \
+      "${CLUSTER_NAME}"
+  fi
+
+  CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
+  ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
+  [ -r "$KOPS_FEATURE_GATES_FILE" ] && cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
+  [ -r "$KOPS_ADDITIONAL_POLICIES_FILE" ] && cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
+  ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
+  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
+
+  loudecho "Validating cluster $CLUSTER_NAME"
+  ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
+  return $?
+}
+
+function kops_delete_cluster() {
+  KOPS_BIN=${1}
+  CLUSTER_NAME=${2}
+  KOPS_STATE_FILE=${3}
+  loudecho "Deleting cluster ${CLUSTER_NAME}"
+  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+}