diff --git a/Makefile b/Makefile index 0e610cbed..9c935b106 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,10 @@ VERSION=0.1.0 GIT_COMMIT?=$(shell git rev-parse HEAD) BUILD_DATE?=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE}" +GO111MODULE=on +GOPROXY=direct + +.EXPORT_ALL_VARIABLES: .PHONY: aws-efs-csi-driver aws-efs-csi-driver: @@ -33,6 +37,10 @@ verify: test: go test -v -race ./pkg/... +.PHONY: test-e2e +test-e2e: + AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c ./hack/run-e2e-test + .PHONY: image image: docker build -t $(IMAGE):latest . diff --git a/hack/run-e2e-test b/hack/run-e2e-test new file mode 100755 index 000000000..73254e260 --- /dev/null +++ b/hack/run-e2e-test @@ -0,0 +1,125 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +OS_ARCH=$(go env GOOS)-amd64 +TEST_ID=$RANDOM +CLUSTER_NAME=test-cluster-$TEST_ID +TEST_DIR=/tmp/efs-e2e-test +BASE_DIR=$(dirname $0) +REGION=${AWS_REGION-us-east-1} +ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c} +K8S_VERSION=${K8S_VERSION-1.14.1} + +echo "Testing in region: $REGION and zones: $ZONES" + +KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.14.0-alpha.3/kops-$OS_ARCH +KOPS_PATH=$TEST_DIR/kops +KOPS_STATE_FILE=s3://k8s-kops-csi-e2e + +# Download kops if not yet +if [[ ! -e $KOPS_PATH ]]; then + mkdir -p $TEST_DIR + echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH" + curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH + chmod +x $KOPS_PATH +fi + +# Push test driver image +eval $(aws ecr get-login --region $REGION --no-include-email) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +IMAGE_TAG=$TEST_ID +IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-efs-csi-driver +docker build -t $IMAGE_NAME:$IMAGE_TAG . +docker push $IMAGE_NAME:$IMAGE_TAG + +set +e +echo "Creating cluster $CLUSTER_NAME" +CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml +SSH_KEY_PATH=$TEST_DIR/id_rsa +ssh-keygen -P csi-e2e -f $SSH_KEY_PATH + +$KOPS_PATH create cluster --state $KOPS_STATE_FILE \ + --zones $ZONES \ + --node-count=3 \ + --kubernetes-version=$K8S_VERSION \ + --ssh-public-key=$SSH_KEY_PATH.pub \ + $CLUSTER_NAME.k8s.local +$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH +$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH +$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes + +# Wait for cluster creation +while [[ 1 ]]; do + $KOPS_PATH validate cluster --state $KOPS_STATE_FILE + ret=$? + if [[ $ret -eq 0 ]]; then + break + else + echo "Waiting cluster to be created" + sleep 30 + fi +done; + + +echo "Deploying driver" +cat deploy/kubernetes/manifest.yaml | sed s,amazon/aws-efs-csi-driver:latest,$IMAGE_NAME:$IMAGE_TAG, > $TEST_DIR/manifest.yaml +kubectl apply -f $TEST_DIR/manifest.yaml + +echo "Creating EFS file system" +aws efs create-file-system --creation-token $TEST_ID --tags Key=KubernetesCluster,Value=$CLUSTER_NAME.k8s.local --region $REGION +FILE_SYSTEM_ID=$(aws efs describe-file-systems --creation-token $TEST_ID --region $REGION | jq -r '.FileSystems[0].FileSystemId') + +GROUP_ID=$(aws ec2 describe-security-groups --filters Name=group-name,Values=nodes.$CLUSTER_NAME.k8s.local --region $REGION | jq -r '.SecurityGroups[0].GroupId') +aws ec2 authorize-security-group-ingress --group-id $GROUP_ID --protocol tcp --port 2049 --source-group $GROUP_ID --region $REGION + +for zone in ${ZONES//,/ }; do + SUBNET_NAME=${zone}.$CLUSTER_NAME.k8s.local + SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=$SUBNET_NAME --region $REGION | jq -r '.Subnets[0].SubnetId') + aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_ID --security-groups $GROUP_ID --region $REGION +done + +# Run the test +export KUBECONFIG=$HOME/.kube/config + +go test -v -timeout 0 ./test/e2e/... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \ + -file-system-id=$FILE_SYSTEM_ID +TEST_PASS=$? + +echo "Deleting EFS file system $FILE_SYSTEM_ID" +MOUNT_TARGETS=$(aws efs describe-mount-targets --file-system-id $FILE_SYSTEM_ID --region $REGION) +i=0 +for zone in ${ZONES//,/ }; do + echo "Deleting EFS mount target $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId')" + aws efs delete-mount-target --mount-target-id $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId') --region $REGION + ((i++)) +done +i=0 +until aws efs delete-file-system --file-system-id $FILE_SYSTEM_ID --region $REGION || [ $i == 10 ]; do + echo "Deleting EFS file system $FILE_SYSTEM_ID" + sleep 5 + ((i++)) +done; + +echo "Deleting cluster $CLUSTER_NAME" +$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes + +rm -rf $TEST_DIR + +if [[ $TEST_PASS -ne 0 ]]; then + exit 1 +fi diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..0f2b847fb --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,9 @@ +# Prerequisites +- Amazon EFS file system +- kubernetes 1.14+ cluster whose workers (preferably 2 or more) can mount the Amazon EFS file system + +# Run +```sh +go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \ + -file-system-id=fs-c2a43e69 +``` \ No newline at end of file diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 000000000..130b323c2 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "flag" + "fmt" + "log" + "os" + "path" + "testing" + + "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/config" + "github.com/onsi/ginkgo/reporters" + "github.com/onsi/gomega" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" + "k8s.io/kubernetes/test/e2e/storage/testsuites" +) + +func init() { + framework.HandleFlags() + framework.AfterReadingAllFlags(&framework.TestContext) + // PWD is test/e2e inside the git repo + testfiles.AddFileSource(testfiles.RootFileSource{Root: "../.."}) +} + +var fileSystemID = flag.String("file-system-id", "", "required. file system ID of the EFS file system to use for the test") + +func TestEFSCSI(t *testing.T) { + if *fileSystemID == "" { + log.Fatalf("file-system-id required. file system ID of the EFS file system to use for the test.") + } + gomega.RegisterFailHandler(ginkgo.Fail) + + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins + var r []ginkgo.Reporter + if framework.TestContext.ReportDir != "" { + if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil { + log.Fatalf("Failed creating report directory: %v", err) + } else { + r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) + } + } + log.Printf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) + + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "EFS CSI Suite", r) +} + +type efsDriver struct { + driverInfo testsuites.DriverInfo +} + +var _ testsuites.TestDriver = &efsDriver{} + +// TODO implement Inline (unless it's redundant) and DynamicPV +// var _ testsuites.InlineVolumeTestDriver = &efsDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &efsDriver{} + +func InitEFSCSIDriver() testsuites.TestDriver { + return &efsDriver{ + driverInfo: testsuites.DriverInfo{ + Name: "efs.csi.aws.com", + SupportedFsType: sets.NewString(""), + SupportedMountOption: sets.NewString("tls"), + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapExec: true, + testsuites.CapMultiPODs: true, + testsuites.CapRWX: true, + }, + }, + } +} + +func (e *efsDriver) GetDriverInfo() *testsuites.DriverInfo { + return &e.driverInfo +} + +func (e *efsDriver) SkipUnsupportedTest(testpatterns.TestPattern) {} + +func (e *efsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) { + ginkgo.By("Deploying EFS CSI driver") + cancelPodLogs := testsuites.StartPodLogs(f) + + return &testsuites.PerTestConfig{ + Driver: e, + Prefix: "efs", + Framework: f, + }, func() { + ginkgo.By("Cleaning up EFS CSI driver") + cancelPodLogs() + } +} + +func (e *efsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume { + return nil +} + +func (e *efsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) { + pvSource := v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: e.driverInfo.Name, + VolumeHandle: *fileSystemID, + }, + } + return &pvSource, nil +} + +// List of testSuites to be executed in below loop +var csiTestSuites = []func() testsuites.TestSuite{ + testsuites.InitVolumesTestSuite, + testsuites.InitVolumeIOTestSuite, + testsuites.InitVolumeModeTestSuite, + testsuites.InitSubPathTestSuite, + testsuites.InitProvisioningTestSuite, + //testsuites.InitSnapshottableTestSuite, + testsuites.InitMultiVolumeTestSuite, +} + +var _ = ginkgo.Describe("[efs-csi] EFS CSI", func() { + driver := InitEFSCSIDriver() + ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(driver), func() { + testsuites.DefineTestSuite(driver, csiTestSuites) + }) +}) diff --git a/test/e2e/go.mod b/test/e2e/go.mod new file mode 100644 index 000000000..a4bcc1a4e --- /dev/null +++ b/test/e2e/go.mod @@ -0,0 +1,86 @@ +module github.com/kubernetes-sigs/aws-efs-csi-driver/test/e2e + +go 1.12 + +require ( + github.com/Microsoft/go-winio v0.4.12 + github.com/PuerkitoBio/purell v1.1.1 + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 + github.com/beorn7/perks v1.0.0 + github.com/davecgh/go-spew v1.1.1 + github.com/docker/distribution v2.7.1+incompatible + github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c + github.com/emicklei/go-restful v2.9.6+incompatible + github.com/evanphx/json-patch v3.0.0+incompatible + github.com/fatih/camelcase v1.0.0 + github.com/fsnotify/fsnotify v1.4.7 + github.com/ghodss/yaml v1.0.0 + github.com/go-openapi/jsonpointer v0.19.2 + github.com/go-openapi/jsonreference v0.19.2 + github.com/go-openapi/spec v0.19.2 + github.com/go-openapi/swag v0.19.4 + github.com/gogo/protobuf v1.2.1 + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 + github.com/golang/protobuf v1.3.2 + github.com/google/btree v1.0.0 + github.com/google/go-cmp v0.3.0 + github.com/google/gofuzz v1.0.0 + github.com/google/uuid v1.1.1 + github.com/googleapis/gnostic v0.3.0 + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 + github.com/hashicorp/golang-lru v0.5.1 + github.com/hpcloud/tail v1.0.0 + github.com/imdario/mergo v0.3.7 + github.com/inconshreveable/mousetrap v1.0.0 + github.com/json-iterator/go v1.1.6 + github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e + github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 + github.com/onsi/ginkgo v1.8.0 + github.com/onsi/gomega v1.5.0 + github.com/opencontainers/go-digest v1.0.0-rc1 + github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 + github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 + github.com/peterbourgon/diskv v0.0.0-20190425080511-0be1b92a6df0 + github.com/pkg/errors v0.8.1 + github.com/pmezard/go-difflib v1.0.0 + github.com/prometheus/client_golang v1.0.0 + github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 + github.com/prometheus/common v0.6.0 + github.com/prometheus/procfs v0.0.3 + github.com/spf13/afero v1.2.2 + github.com/spf13/cobra v0.0.5 + github.com/spf13/pflag v1.0.3 + github.com/stretchr/testify v1.3.0 + golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 + golang.org/x/net v0.0.0-20190628185345-da137c7871d7 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 + golang.org/x/text v0.3.2 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + google.golang.org/appengine v1.6.1 + google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 + google.golang.org/grpc v1.22.0 + gopkg.in/fsnotify.v1 v1.4.7 + gopkg.in/inf.v0 v0.9.1 + gopkg.in/square/go-jose.v2 v2.3.1 + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 + gopkg.in/yaml.v2 v2.2.2 + k8s.io/api v0.0.0-20190718183219-b59d8169aab5 + k8s.io/apiextensions-apiserver v0.0.0-20190718185103-d1ef975d28ce + k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719 + k8s.io/apiserver v0.0.0-20190718184206-a1aa83af71a7 + k8s.io/cli-runtime v0.0.0-20190717024643-59adbd30f884 + k8s.io/client-go v0.0.0-20190718183610-8e956561bbf5 + k8s.io/cloud-provider v0.0.0-20190717025205-585d8110a88f + k8s.io/component-base v0.0.0-20190717023551-b4f50308a616 + k8s.io/cri-api v0.0.0-20190711114129-0b0c4484b3d3 + k8s.io/csi-translation-lib v0.0.0-20190717025251-2a445d2079de + k8s.io/klog v0.3.3 + k8s.io/kube-openapi v0.0.0-20190718094010-3cf2ea392886 + k8s.io/kubernetes v1.15.1 + k8s.io/utils v0.0.0-20190712204705-3dccf664f023 + sigs.k8s.io/kustomize v2.0.3+incompatible + sigs.k8s.io/yaml v1.1.0 +)