diff --git a/.gitignore b/.gitignore index fc737de69ed13..965e6bb7fdea0 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,7 @@ openshift*.tar.gz # Ensure that openapi definitions are not ignored to ensure that # openshift/origin can vendor them. !pkg/generated/openapi/zz_generated.openapi.go + +# Ignore openshift-tests-extension +/.openshift-tests-extension +/k8s-tests* diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index fd111fea3cdae..356642b3ed923 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -80,6 +80,7 @@ kube::golang::server_targets() { cluster/gce/gci/mounter cmd/watch-termination openshift-hack/cmd/k8s-tests + openshift-hack/cmd/k8s-tests-ext ) echo "${targets[@]}" } diff --git a/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go b/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go new file mode 100644 index 0000000000000..8d82786410dab --- /dev/null +++ b/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go @@ -0,0 +1,94 @@ +package main + +import ( + "os" + + "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + utilflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/logs" + "k8s.io/kubernetes/openshift-hack/e2e/annotate/generated" + + // initialize framework extensions + _ "k8s.io/kubernetes/test/e2e/framework/debug/init" + _ "k8s.io/kubernetes/test/e2e/framework/metrics/init" +) + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) + + registry := e.NewRegistry() + kubeTestsExtension := e.NewExtension("openshift", "payload", "hyperkube") + registry.Register(kubeTestsExtension) + + // Carve up the kube tests into our openshift suites... + kubeTestsExtension.AddSuite(e.Suite{ + Name: "kubernetes/conformance/parallel", + Parents: []string{ + "openshift/conformance/parallel", + "openshift/conformance/parallel/minimal", + }, + Qualifiers: []string{`!labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance"`}, + }) + + kubeTestsExtension.AddSuite(e.Suite{ + Name: "kubernetes/conformance/serial", + Parents: []string{ + "openshift/conformance/serial", + "openshift/conformance/serial/minimal", + }, + Qualifiers: []string{`labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance"`}, + }) + + //FIXME(stbenjam): what other suites does k8s-test contribute to? + + specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() + if err != nil { + panic(err) + } + + // Initialization for kube ginkgo test framework needs to run before all tests execute + specs.AddBeforeAll(func() { + if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil { + panic(err) + } + }) + + // Annotations get appended to test names, these are additions to upstream + // tests for controlling skips, suite membership, etc. + // + // TODO: + // - Remove this annotation code, and migrate to Labels/Tags and + // the environmental skip code from the enhancement once its implemented. + // - Make sure to account for test renames + specs.Walk(func(spec *extensiontests.ExtensionTestSpec) { + if annotations, ok := generated.Annotations[spec.Name]; ok { + spec.Name += annotations + } + }) + + kubeTestsExtension.AddSpecs(specs) + + // Cobra stuff + root := &cobra.Command{ + Long: "Kubernetes tests extension for OpenShift", + } + + root.AddCommand( + cmd.DefaultExtensionCommands(registry)..., + ) + + if err := func() error { + return root.Execute() + }(); err != nil { + os.Exit(1) + } +} diff --git a/openshift-hack/cmd/k8s-tests-ext/provider.go b/openshift-hack/cmd/k8s-tests-ext/provider.go new file mode 100644 index 0000000000000..cdc948a45c652 --- /dev/null +++ b/openshift-hack/cmd/k8s-tests-ext/provider.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + kclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubernetes/openshift-hack/e2e" + conformancetestdata "k8s.io/kubernetes/test/conformance/testdata" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + "k8s.io/kubernetes/test/e2e/storage/external" + e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" + testfixtures "k8s.io/kubernetes/test/fixtures" + + // this appears to inexplicably auto-register global flags. + _ "k8s.io/kubernetes/test/e2e/storage/drivers" + + // these are loading important global flags that we need to get and set + _ "k8s.io/kubernetes/test/e2e" + _ "k8s.io/kubernetes/test/e2e/lifecycle" +) + +// copied directly from github.com/openshift/origin/cmd/openshift-tests/provider.go +// and github.com/openshift/origin/test/extended/util/test.go +func initializeTestFramework(provider string) error { + providerInfo := &ClusterConfiguration{} + if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil { + return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err) + } + if len(providerInfo.ProviderName) == 0 { + return fmt.Errorf("provider must be a JSON object with the 'type' key") + } + config := &ClusterConfiguration{} + if err := json.Unmarshal([]byte(provider), config); err != nil { + return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err) + } + + // update testContext with loaded config + testContext := &framework.TestContext + testContext.Provider = config.ProviderName + testContext.CloudConfig = framework.CloudConfig{ + ProjectID: config.ProjectID, + Region: config.Region, + Zone: config.Zone, + Zones: config.Zones, + NumNodes: config.NumNodes, + MultiMaster: config.MultiMaster, + MultiZone: config.MultiZone, + ConfigFile: config.ConfigFile, + } + testContext.AllowedNotReadyNodes = -1 + testContext.MinStartupPods = -1 + testContext.MaxNodesToGather = 0 + testContext.KubeConfig = os.Getenv("KUBECONFIG") + + // allow the CSI tests to access test data, but only briefly + // TODO: ideally CSI would not use any of these test methods + // var err error + // exutil.WithCleanup(func() { err = initCSITests(dryRun) }) + // TODO: for now I'm only initializing CSI directly, but we probably need that + // WithCleanup here as well + if err := initCSITests(); err != nil { + return err + } + + if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 { + os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts")) + } + + testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false" + testContext.VerifyServiceAccount = true + testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS()) + testfiles.AddFileSource(testfixtures.GetTestFixturesFS()) + testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS()) + testContext.KubectlPath = "kubectl" + // context.KubeConfig = KubeConfigPath() + testContext.KubeConfig = os.Getenv("KUBECONFIG") + + // "debian" is used when not set. At least GlusterFS tests need "custom". + // (There is no option for "rhel" or "centos".) + testContext.NodeOSDistro = "custom" + testContext.MasterOSDistro = "custom" + + // load and set the host variable for kubectl + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{}) + cfg, err := clientConfig.ClientConfig() + if err != nil { + return err + } + testContext.Host = cfg.Host + + // Ensure that Kube tests run privileged (like they do upstream) + testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) { + return e2e.CreateTestingNS(ctx, baseName, c, labels, true) + } + + gomega.RegisterFailHandler(ginkgo.Fail) + + framework.AfterReadingAllFlags(testContext) + testContext.DumpLogsOnFailure = true + + // these constants are taken from kube e2e and used by tests + testContext.IPFamily = "ipv4" + if config.HasIPv6 && !config.HasIPv4 { + testContext.IPFamily = "ipv6" + } + + testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR") + + return nil +} + +const ( + manifestEnvVar = "TEST_CSI_DRIVER_FILES" +) + +// copied directly from github.com/openshift/origin/cmd/openshift-tests/csi.go +// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES. +func initCSITests() error { + manifestList := os.Getenv(manifestEnvVar) + if manifestList != "" { + manifests := strings.Split(manifestList, ",") + for _, manifest := range manifests { + if err := external.AddDriverDefinition(manifest); err != nil { + return fmt.Errorf("failed to load manifest from %q: %s", manifest, err) + } + // Register the base dir of the manifest file as a file source. + // With this we can reference the CSI driver's storageClass + // in the manifest file (FromFile field). + testfiles.AddFileSource(testfiles.RootFileSource{ + Root: filepath.Dir(manifest), + }) + } + } + + return nil +} diff --git a/openshift-hack/cmd/k8s-tests-ext/types.go b/openshift-hack/cmd/k8s-tests-ext/types.go new file mode 100644 index 0000000000000..b43652499537d --- /dev/null +++ b/openshift-hack/cmd/k8s-tests-ext/types.go @@ -0,0 +1,47 @@ +package main + +// copied directly from github.com/openshift/origin/test/extended/util/cluster/cluster.go +type ClusterConfiguration struct { + ProviderName string `json:"type"` + + // These fields (and the "type" tag for ProviderName) chosen to match + // upstream's e2e.CloudConfig. + ProjectID string + Region string + Zone string + NumNodes int + MultiMaster bool + MultiZone bool + Zones []string + ConfigFile string + + // Disconnected is set for test jobs without external internet connectivity + Disconnected bool + + // SingleReplicaTopology is set for disabling disruptive tests or tests + // that require high availability + SingleReplicaTopology bool + + // NetworkPlugin is the "official" plugin name + NetworkPlugin string + // NetworkPluginMode is an optional sub-identifier for the NetworkPlugin. + // (Currently it is only used for OpenShiftSDN.) + NetworkPluginMode string `json:",omitempty"` + + // HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific, + // and dual-stack-specific tests are run + HasIPv4 bool + HasIPv6 bool + + // HasSCTP determines whether SCTP connectivity tests can be run in the cluster + HasSCTP bool + + // IsProxied determines whether we are accessing the cluster through an HTTP proxy + IsProxied bool + + // IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS) + IsIBMROKS bool + + // IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled + HasNoOptionalCapabilities bool +} diff --git a/openshift-hack/images/hyperkube/Dockerfile.rhel b/openshift-hack/images/hyperkube/Dockerfile.rhel index 5257653bc8d07..7df4e779e3389 100644 --- a/openshift-hack/images/hyperkube/Dockerfile.rhel +++ b/openshift-hack/images/hyperkube/Dockerfile.rhel @@ -1,11 +1,12 @@ FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder WORKDIR /go/src/k8s.io/kubernetes COPY . . -RUN make WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler cmd/kubelet cmd/watch-termination openshift-hack/cmd/k8s-tests' && \ +RUN make WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler cmd/kubelet cmd/watch-termination openshift-hack/cmd/k8s-tests openshift-hack/cmd/k8s-tests-ext' && \ mkdir -p /tmp/build && \ cp openshift-hack/images/hyperkube/hyperkube openshift-hack/images/hyperkube/kubensenter /tmp/build && \ - cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/{kube-apiserver,kube-controller-manager,kube-scheduler,kubelet,watch-termination,k8s-tests} \ - /tmp/build + cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/{kube-apiserver,kube-controller-manager,kube-scheduler,kubelet,watch-termination,k8s-tests,k8s-tests-ext} \ + /tmp/build && \ + gzip /tmp/build/k8s-tests-ext FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 RUN yum install -y --setopt=tsflags=nodocs --setopt=skip_missing_names_on_install=False iproute && yum clean all