Run upstream e2e tests

Makefile

@@ -19,6 +19,10 @@ VERSION=0.1.0
 GIT_COMMIT?=$(shell git rev-parse HEAD)
 BUILD_DATE?=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE}"
+GO111MODULE=on
+GOPROXY=direct
+
+.EXPORT_ALL_VARIABLES:
 
 .PHONY: aws-efs-csi-driver
 aws-efs-csi-driver:
@@ -33,6 +37,10 @@ verify:
 test:
 	go test -v -race ./pkg/...
 
+.PHONY: test-e2e
+test-e2e:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c ./hack/run-e2e-test
+
 .PHONY: image
 image:
 	docker build -t $(IMAGE):latest .

hack/run-e2e-test

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+OS_ARCH=$(go env GOOS)-amd64
+TEST_ID=$RANDOM
+CLUSTER_NAME=test-cluster-$TEST_ID
+TEST_DIR=/tmp/efs-e2e-test
+BASE_DIR=$(dirname $0)
+REGION=${AWS_REGION-us-east-1}
+ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
+K8S_VERSION=${K8S_VERSION-1.14.1}
+
+echo "Testing in region: $REGION and zones: $ZONES"
+
+KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.14.0-alpha.3/kops-$OS_ARCH
+KOPS_PATH=$TEST_DIR/kops
+KOPS_STATE_FILE=s3://k8s-kops-csi-e2e
+
+# Download kops if not yet
+if [[ ! -e $KOPS_PATH ]]; then
+  mkdir -p $TEST_DIR 
+  echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
+  curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
+  chmod +x $KOPS_PATH
+fi
+
+# Push test driver image
+eval $(aws ecr get-login --region $REGION --no-include-email)
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+IMAGE_TAG=$TEST_ID
+IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-efs-csi-driver
+docker build -t $IMAGE_NAME:$IMAGE_TAG .
+docker push $IMAGE_NAME:$IMAGE_TAG
+
+set +e
+echo "Creating cluster $CLUSTER_NAME"
+CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml
+SSH_KEY_PATH=$TEST_DIR/id_rsa
+ssh-keygen -P csi-e2e -f $SSH_KEY_PATH
+
+$KOPS_PATH create cluster --state $KOPS_STATE_FILE \
+  --zones $ZONES \
+  --node-count=3 \
+  --kubernetes-version=$K8S_VERSION \
+  --ssh-public-key=$SSH_KEY_PATH.pub \
+  $CLUSTER_NAME.k8s.local
+$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH
+$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH
+$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes
+
+# Wait for cluster creation
+while [[ 1 ]]; do
+  $KOPS_PATH validate cluster --state $KOPS_STATE_FILE
+  ret=$?
+  if [[ $ret -eq 0 ]]; then
+    break
+  else
+    echo "Waiting cluster to be created"
+    sleep 30
+  fi
+done;
+
+
+echo "Deploying driver"
+cat deploy/kubernetes/manifest.yaml | sed s,amazon/aws-efs-csi-driver:latest,$IMAGE_NAME:$IMAGE_TAG, > $TEST_DIR/manifest.yaml
+kubectl apply -f $TEST_DIR/manifest.yaml
+
+echo "Creating EFS file system"
+aws efs create-file-system --creation-token $TEST_ID --tags Key=KubernetesCluster,Value=$CLUSTER_NAME.k8s.local --region $REGION
+FILE_SYSTEM_ID=$(aws efs describe-file-systems --creation-token $TEST_ID --region $REGION | jq -r '.FileSystems[0].FileSystemId')
+
+GROUP_ID=$(aws ec2 describe-security-groups --filters Name=group-name,Values=nodes.$CLUSTER_NAME.k8s.local --region $REGION | jq -r '.SecurityGroups[0].GroupId')
+aws ec2 authorize-security-group-ingress --group-id $GROUP_ID --protocol tcp --port 2049 --source-group $GROUP_ID --region $REGION
+
+for zone in ${ZONES//,/ }; do
+  SUBNET_NAME=${zone}.$CLUSTER_NAME.k8s.local
+  SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=$SUBNET_NAME --region $REGION | jq -r '.Subnets[0].SubnetId')
+  aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_ID --security-groups $GROUP_ID --region $REGION
+done
+
+# Run the test
+export KUBECONFIG=$HOME/.kube/config
+
+go test -v -timeout 0 ./test/e2e/... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \
+  -file-system-id=$FILE_SYSTEM_ID
+TEST_PASS=$?
+
+echo "Deleting EFS file system $FILE_SYSTEM_ID"
+MOUNT_TARGETS=$(aws efs describe-mount-targets --file-system-id $FILE_SYSTEM_ID --region $REGION)
+i=0
+for zone in ${ZONES//,/ }; do
+  echo "Deleting EFS mount target $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId')"
+  aws efs delete-mount-target --mount-target-id $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId') --region $REGION
+  ((i++))
+done
+i=0
+until aws efs delete-file-system --file-system-id $FILE_SYSTEM_ID --region $REGION || [ $i == 10 ]; do
+  echo "Deleting EFS file system $FILE_SYSTEM_ID"
+  sleep 5
+  ((i++))
+done;
+
+echo "Deleting cluster $CLUSTER_NAME"
+$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes
+
+rm -rf $TEST_DIR
+
+if [[ $TEST_PASS -ne 0 ]]; then
+  exit 1
+fi

test/e2e/README.md

@@ -0,0 +1,9 @@
+# Prerequisites
+- Amazon EFS file system
+- kubernetes 1.14+ cluster whose workers (preferably 2 or more) can mount the Amazon EFS file system
+
+# Run
+```sh
+go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \
+  -file-system-id=fs-c2a43e69
+```

test/e2e/e2e_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+)
+
+func init() {
+	framework.HandleFlags()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+	// PWD is test/e2e inside the git repo
+	testfiles.AddFileSource(testfiles.RootFileSource{Root: "../.."})
+}
+
+var fileSystemID = flag.String("file-system-id", "", "required. file system ID of the EFS file system to use for the test")
+
+func TestEFSCSI(t *testing.T) {
+	if *fileSystemID == "" {
+		log.Fatalf("file-system-id required. file system ID of the EFS file system to use for the test.")
+	}
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
+	var r []ginkgo.Reporter
+	if framework.TestContext.ReportDir != "" {
+		if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil {
+			log.Fatalf("Failed creating report directory: %v", err)
+		} else {
+			r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
+		}
+	}
+	log.Printf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "EFS CSI Suite", r)
+}
+
+type efsDriver struct {
+	driverInfo testsuites.DriverInfo
+}
+
+var _ testsuites.TestDriver = &efsDriver{}
+
+// TODO implement Inline (unless it's redundant) and DynamicPV
+// var _ testsuites.InlineVolumeTestDriver = &efsDriver{}
+var _ testsuites.PreprovisionedPVTestDriver = &efsDriver{}
+
+func InitEFSCSIDriver() testsuites.TestDriver {
+	return &efsDriver{
+		driverInfo: testsuites.DriverInfo{
+			Name:                 "efs.csi.aws.com",
+			SupportedFsType:      sets.NewString(""),
+			SupportedMountOption: sets.NewString("tls"),
+			Capabilities: map[testsuites.Capability]bool{
+				testsuites.CapPersistence: true,
+				testsuites.CapExec:        true,
+				testsuites.CapMultiPODs:   true,
+				testsuites.CapRWX:         true,
+			},
+		},
+	}
+}
+
+func (e *efsDriver) GetDriverInfo() *testsuites.DriverInfo {
+	return &e.driverInfo
+}
+
+func (e *efsDriver) SkipUnsupportedTest(testpatterns.TestPattern) {}
+
+func (e *efsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	ginkgo.By("Deploying EFS CSI driver")
+	cancelPodLogs := testsuites.StartPodLogs(f)
+
+	return &testsuites.PerTestConfig{
+			Driver:    e,
+			Prefix:    "efs",
+			Framework: f,
+		}, func() {
+			ginkgo.By("Cleaning up EFS CSI driver")
+			cancelPodLogs()
+		}
+}
+
+func (e *efsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+	return nil
+}
+
+func (e *efsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+	pvSource := v1.PersistentVolumeSource{
+		CSI: &v1.CSIPersistentVolumeSource{
+			Driver:       e.driverInfo.Name,
+			VolumeHandle: *fileSystemID,
+		},
+	}
+	return &pvSource, nil
+}
+
+// List of testSuites to be executed in below loop
+var csiTestSuites = []func() testsuites.TestSuite{
+	testsuites.InitVolumesTestSuite,
+	testsuites.InitVolumeIOTestSuite,
+	testsuites.InitVolumeModeTestSuite,
+	testsuites.InitSubPathTestSuite,
+	testsuites.InitProvisioningTestSuite,
+	//testsuites.InitSnapshottableTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
+}
+
+var _ = ginkgo.Describe("[efs-csi] EFS CSI", func() {
+	driver := InitEFSCSIDriver()
+	ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(driver), func() {
+		testsuites.DefineTestSuite(driver, csiTestSuites)
+	})
+})

test/e2e/go.mod

@@ -0,0 +1,86 @@
+module github.com/kubernetes-sigs/aws-efs-csi-driver/test/e2e
+
+go 1.12
+
+require (
+	github.com/Microsoft/go-winio v0.4.12
+	github.com/PuerkitoBio/purell v1.1.1
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578
+	github.com/beorn7/perks v1.0.0
+	github.com/davecgh/go-spew v1.1.1
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c
+	github.com/emicklei/go-restful v2.9.6+incompatible
+	github.com/evanphx/json-patch v3.0.0+incompatible
+	github.com/fatih/camelcase v1.0.0
+	github.com/fsnotify/fsnotify v1.4.7
+	github.com/ghodss/yaml v1.0.0
+	github.com/go-openapi/jsonpointer v0.19.2
+	github.com/go-openapi/jsonreference v0.19.2
+	github.com/go-openapi/spec v0.19.2
+	github.com/go-openapi/swag v0.19.4
+	github.com/gogo/protobuf v1.2.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/golang/protobuf v1.3.2
+	github.com/google/btree v1.0.0
+	github.com/google/go-cmp v0.3.0
+	github.com/google/gofuzz v1.0.0
+	github.com/google/uuid v1.1.1
+	github.com/googleapis/gnostic v0.3.0
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
+	github.com/hashicorp/golang-lru v0.5.1
+	github.com/hpcloud/tail v1.0.0
+	github.com/imdario/mergo v0.3.7
+	github.com/inconshreveable/mousetrap v1.0.0
+	github.com/json-iterator/go v1.1.6
+	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e
+	github.com/matttproud/golang_protobuf_extensions v1.0.1
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
+	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
+	github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709
+	github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99
+	github.com/peterbourgon/diskv v0.0.0-20190425080511-0be1b92a6df0
+	github.com/pkg/errors v0.8.1
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
+	github.com/prometheus/common v0.6.0
+	github.com/prometheus/procfs v0.0.3
+	github.com/spf13/afero v1.2.2
+	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.3
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
+	golang.org/x/net v0.0.0-20190628185345-da137c7871d7
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7
+	golang.org/x/text v0.3.2
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	google.golang.org/appengine v1.6.1
+	google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610
+	google.golang.org/grpc v1.22.0
+	gopkg.in/fsnotify.v1 v1.4.7
+	gopkg.in/inf.v0 v0.9.1
+	gopkg.in/square/go-jose.v2 v2.3.1
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/yaml.v2 v2.2.2
+	k8s.io/api v0.0.0-20190718183219-b59d8169aab5
+	k8s.io/apiextensions-apiserver v0.0.0-20190718185103-d1ef975d28ce
+	k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719
+	k8s.io/apiserver v0.0.0-20190718184206-a1aa83af71a7
+	k8s.io/cli-runtime v0.0.0-20190717024643-59adbd30f884
+	k8s.io/client-go v0.0.0-20190718183610-8e956561bbf5
+	k8s.io/cloud-provider v0.0.0-20190717025205-585d8110a88f
+	k8s.io/component-base v0.0.0-20190717023551-b4f50308a616
+	k8s.io/cri-api v0.0.0-20190711114129-0b0c4484b3d3
+	k8s.io/csi-translation-lib v0.0.0-20190717025251-2a445d2079de
+	k8s.io/klog v0.3.3
+	k8s.io/kube-openapi v0.0.0-20190718094010-3cf2ea392886
+	k8s.io/kubernetes v1.15.1
+	k8s.io/utils v0.0.0-20190712204705-3dccf664f023
+	sigs.k8s.io/kustomize v2.0.3+incompatible
+	sigs.k8s.io/yaml v1.1.0
+)