From ed0b96e047728ec4e9d2086f0e0ecf9ccb4b8f38 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Mon, 7 Oct 2024 10:26:20 -0400 Subject: [PATCH] UPSTREAM: : add openshift-tests-extension compatible binary --- .gitignore | 4 + openshift-hack/cmd/k8s-tests-ext/k8s-tests.go | 74 +++++++++ openshift-hack/cmd/k8s-tests-ext/provider.go | 147 ++++++++++++++++++ openshift-hack/cmd/k8s-tests-ext/types.go | 47 ++++++ 4 files changed, 272 insertions(+) create mode 100644 openshift-hack/cmd/k8s-tests-ext/k8s-tests.go create mode 100644 openshift-hack/cmd/k8s-tests-ext/provider.go create mode 100644 openshift-hack/cmd/k8s-tests-ext/types.go diff --git a/.gitignore b/.gitignore index fc737de69ed13..965e6bb7fdea0 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,7 @@ openshift*.tar.gz # Ensure that openapi definitions are not ignored to ensure that # openshift/origin can vendor them. !pkg/generated/openapi/zz_generated.openapi.go + +# Ignore openshift-tests-extension +/.openshift-tests-extension +/k8s-tests* diff --git a/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go b/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go new file mode 100644 index 0000000000000..9adf7c1673875 --- /dev/null +++ b/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go @@ -0,0 +1,74 @@ +package main + +import ( + "math/rand" + "os" + "time" + + "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + utilflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/logs" + "k8s.io/kubernetes/openshift-hack/e2e/annotate/generated" + + // initialize framework extensions + _ "k8s.io/kubernetes/test/e2e/framework/debug/init" + _ "k8s.io/kubernetes/test/e2e/framework/metrics/init" +) + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + rand.Seed(time.Now().UTC().UnixNano()) + pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) + + // Create openshift-tests extension + registry := e.NewRegistry() + ext := e.NewExtension("openshift", "payload", "hyperkube") + ext.AddSuite(e.Suite{Name: "k8s", Parents: []string{"openshift/conformance/parallel"}}) + + // Cobra stuff + root := &cobra.Command{ + Long: "OpenShift Tests compatible wrapper", + } + + root.AddCommand( + cmd.DefaultExtensionCommands(registry)..., + ) + + specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() + if err != nil { + panic(err) + } + + // Initialization for kube ginkgo test framework needs to run before all tests execute + specs.AddBeforeAll(func() { + if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil { + panic(err) + } + }) + + // Annotations get appended to test names, these are additions to upstream + // tests for controlling skips, suite membership, etc. + // TODO: get rid of annotations + specs.Walk(func(spec *extensiontests.ExtensionTestSpec) { + if annotations, ok := generated.Annotations[spec.Name]; ok { + spec.Name += annotations + } + }) + + ext.AddSpecs(specs) + registry.Register(ext) + + if err := func() error { + return root.Execute() + }(); err != nil { + os.Exit(1) + } +} diff --git a/openshift-hack/cmd/k8s-tests-ext/provider.go b/openshift-hack/cmd/k8s-tests-ext/provider.go new file mode 100644 index 0000000000000..cdc948a45c652 --- /dev/null +++ b/openshift-hack/cmd/k8s-tests-ext/provider.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + kclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubernetes/openshift-hack/e2e" + conformancetestdata "k8s.io/kubernetes/test/conformance/testdata" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + "k8s.io/kubernetes/test/e2e/storage/external" + e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" + testfixtures "k8s.io/kubernetes/test/fixtures" + + // this appears to inexplicably auto-register global flags. + _ "k8s.io/kubernetes/test/e2e/storage/drivers" + + // these are loading important global flags that we need to get and set + _ "k8s.io/kubernetes/test/e2e" + _ "k8s.io/kubernetes/test/e2e/lifecycle" +) + +// copied directly from github.com/openshift/origin/cmd/openshift-tests/provider.go +// and github.com/openshift/origin/test/extended/util/test.go +func initializeTestFramework(provider string) error { + providerInfo := &ClusterConfiguration{} + if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil { + return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err) + } + if len(providerInfo.ProviderName) == 0 { + return fmt.Errorf("provider must be a JSON object with the 'type' key") + } + config := &ClusterConfiguration{} + if err := json.Unmarshal([]byte(provider), config); err != nil { + return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err) + } + + // update testContext with loaded config + testContext := &framework.TestContext + testContext.Provider = config.ProviderName + testContext.CloudConfig = framework.CloudConfig{ + ProjectID: config.ProjectID, + Region: config.Region, + Zone: config.Zone, + Zones: config.Zones, + NumNodes: config.NumNodes, + MultiMaster: config.MultiMaster, + MultiZone: config.MultiZone, + ConfigFile: config.ConfigFile, + } + testContext.AllowedNotReadyNodes = -1 + testContext.MinStartupPods = -1 + testContext.MaxNodesToGather = 0 + testContext.KubeConfig = os.Getenv("KUBECONFIG") + + // allow the CSI tests to access test data, but only briefly + // TODO: ideally CSI would not use any of these test methods + // var err error + // exutil.WithCleanup(func() { err = initCSITests(dryRun) }) + // TODO: for now I'm only initializing CSI directly, but we probably need that + // WithCleanup here as well + if err := initCSITests(); err != nil { + return err + } + + if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 { + os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts")) + } + + testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false" + testContext.VerifyServiceAccount = true + testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS()) + testfiles.AddFileSource(testfixtures.GetTestFixturesFS()) + testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS()) + testContext.KubectlPath = "kubectl" + // context.KubeConfig = KubeConfigPath() + testContext.KubeConfig = os.Getenv("KUBECONFIG") + + // "debian" is used when not set. At least GlusterFS tests need "custom". + // (There is no option for "rhel" or "centos".) + testContext.NodeOSDistro = "custom" + testContext.MasterOSDistro = "custom" + + // load and set the host variable for kubectl + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{}) + cfg, err := clientConfig.ClientConfig() + if err != nil { + return err + } + testContext.Host = cfg.Host + + // Ensure that Kube tests run privileged (like they do upstream) + testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) { + return e2e.CreateTestingNS(ctx, baseName, c, labels, true) + } + + gomega.RegisterFailHandler(ginkgo.Fail) + + framework.AfterReadingAllFlags(testContext) + testContext.DumpLogsOnFailure = true + + // these constants are taken from kube e2e and used by tests + testContext.IPFamily = "ipv4" + if config.HasIPv6 && !config.HasIPv4 { + testContext.IPFamily = "ipv6" + } + + testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR") + + return nil +} + +const ( + manifestEnvVar = "TEST_CSI_DRIVER_FILES" +) + +// copied directly from github.com/openshift/origin/cmd/openshift-tests/csi.go +// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES. +func initCSITests() error { + manifestList := os.Getenv(manifestEnvVar) + if manifestList != "" { + manifests := strings.Split(manifestList, ",") + for _, manifest := range manifests { + if err := external.AddDriverDefinition(manifest); err != nil { + return fmt.Errorf("failed to load manifest from %q: %s", manifest, err) + } + // Register the base dir of the manifest file as a file source. + // With this we can reference the CSI driver's storageClass + // in the manifest file (FromFile field). + testfiles.AddFileSource(testfiles.RootFileSource{ + Root: filepath.Dir(manifest), + }) + } + } + + return nil +} diff --git a/openshift-hack/cmd/k8s-tests-ext/types.go b/openshift-hack/cmd/k8s-tests-ext/types.go new file mode 100644 index 0000000000000..b43652499537d --- /dev/null +++ b/openshift-hack/cmd/k8s-tests-ext/types.go @@ -0,0 +1,47 @@ +package main + +// copied directly from github.com/openshift/origin/test/extended/util/cluster/cluster.go +type ClusterConfiguration struct { + ProviderName string `json:"type"` + + // These fields (and the "type" tag for ProviderName) chosen to match + // upstream's e2e.CloudConfig. + ProjectID string + Region string + Zone string + NumNodes int + MultiMaster bool + MultiZone bool + Zones []string + ConfigFile string + + // Disconnected is set for test jobs without external internet connectivity + Disconnected bool + + // SingleReplicaTopology is set for disabling disruptive tests or tests + // that require high availability + SingleReplicaTopology bool + + // NetworkPlugin is the "official" plugin name + NetworkPlugin string + // NetworkPluginMode is an optional sub-identifier for the NetworkPlugin. + // (Currently it is only used for OpenShiftSDN.) + NetworkPluginMode string `json:",omitempty"` + + // HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific, + // and dual-stack-specific tests are run + HasIPv4 bool + HasIPv6 bool + + // HasSCTP determines whether SCTP connectivity tests can be run in the cluster + HasSCTP bool + + // IsProxied determines whether we are accessing the cluster through an HTTP proxy + IsProxied bool + + // IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS) + IsIBMROKS bool + + // IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled + HasNoOptionalCapabilities bool +}