UPSTREAM: <carry>: add openshift-tests-extension compatible binary

.gitignore

@@ -131,3 +131,7 @@ openshift*.tar.gz
 # Ensure that openapi definitions are not ignored to ensure that
 # openshift/origin can vendor them.
 !pkg/generated/openapi/zz_generated.openapi.go
+
+# Ignore openshift-tests-extension
+/.openshift-tests-extension
+/k8s-tests*

hack/lib/golang.sh

@@ -80,6 +80,7 @@ kube::golang::server_targets() {
     cluster/gce/gci/mounter
     cmd/watch-termination
     openshift-hack/cmd/k8s-tests
+    openshift-hack/cmd/k8s-tests-ext
   )
   echo "${targets[@]}"
 }

openshift-hack/cmd/k8s-tests-ext/k8s-tests.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"os"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+
+	utilflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
+
+	// initialize framework extensions
+	_ "k8s.io/kubernetes/test/e2e/framework/debug/init"
+	_ "k8s.io/kubernetes/test/e2e/framework/metrics/init"
+)
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+
+	registry := e.NewRegistry()
+	kubeTestsExtension := e.NewExtension("openshift", "payload", "hyperkube")
+	registry.Register(kubeTestsExtension)
+
+	// Carve up the kube tests into our openshift suites...
+	kubeTestsExtension.AddSuite(e.Suite{
+		Name: "kubernetes/conformance/parallel",
+		Parents: []string{
+			"openshift/conformance/parallel",
+			"openshift/conformance/parallel/minimal",
+		},
+		Qualifiers: []string{`!labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance"`},
+	})
+
+	kubeTestsExtension.AddSuite(e.Suite{
+		Name: "kubernetes/conformance/serial",
+		Parents: []string{
+			"openshift/conformance/serial",
+			"openshift/conformance/serial/minimal",
+		},
+		Qualifiers: []string{`labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance"`},
+	})
+
+	//FIXME(stbenjam): what other suites does k8s-test contribute to?
+
+	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
+	if err != nil {
+		panic(err)
+	}
+
+	// Initialization for kube ginkgo test framework needs to run before all tests execute
+	specs.AddBeforeAll(func() {
+		if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil {
+			panic(err)
+		}
+	})
+
+	// Annotations get appended to test names, these are additions to upstream
+	// tests for controlling skips, suite membership, etc.
+	//
+	// TODO:
+	//		- Remove this annotation code, and migrate to Labels/Tags and
+	//		  the environmental skip code from the enhancement once its implemented.
+	//		- Make sure to account for test renames
+	specs.Walk(func(spec *extensiontests.ExtensionTestSpec) {
+		if annotations, ok := generated.Annotations[spec.Name]; ok {
+			spec.Name += annotations
+		}
+	})
+
+	kubeTestsExtension.AddSpecs(specs)
+
+	// Cobra stuff
+	root := &cobra.Command{
+		Long: "Kubernetes tests extension for OpenShift",
+	}
+
+	root.AddCommand(
+		cmd.DefaultExtensionCommands(registry)...,
+	)
+
+	if err := func() error {
+		return root.Execute()
+	}(); err != nil {
+		os.Exit(1)
+	}
+}

openshift-hack/cmd/k8s-tests-ext/provider.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	kclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/openshift-hack/e2e"
+	conformancetestdata "k8s.io/kubernetes/test/conformance/testdata"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/storage/external"
+	e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
+	testfixtures "k8s.io/kubernetes/test/fixtures"
+
+	// this appears to inexplicably auto-register global flags.
+	_ "k8s.io/kubernetes/test/e2e/storage/drivers"
+
+	// these are loading important global flags that we need to get and set
+	_ "k8s.io/kubernetes/test/e2e"
+	_ "k8s.io/kubernetes/test/e2e/lifecycle"
+)
+
+// copied directly from github.com/openshift/origin/cmd/openshift-tests/provider.go
+// and github.com/openshift/origin/test/extended/util/test.go
+func initializeTestFramework(provider string) error {
+	providerInfo := &ClusterConfiguration{}
+	if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
+		return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
+	}
+	if len(providerInfo.ProviderName) == 0 {
+		return fmt.Errorf("provider must be a JSON object with the 'type' key")
+	}
+	config := &ClusterConfiguration{}
+	if err := json.Unmarshal([]byte(provider), config); err != nil {
+		return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err)
+	}
+
+	// update testContext with loaded config
+	testContext := &framework.TestContext
+	testContext.Provider = config.ProviderName
+	testContext.CloudConfig = framework.CloudConfig{
+		ProjectID:   config.ProjectID,
+		Region:      config.Region,
+		Zone:        config.Zone,
+		Zones:       config.Zones,
+		NumNodes:    config.NumNodes,
+		MultiMaster: config.MultiMaster,
+		MultiZone:   config.MultiZone,
+		ConfigFile:  config.ConfigFile,
+	}
+	testContext.AllowedNotReadyNodes = -1
+	testContext.MinStartupPods = -1
+	testContext.MaxNodesToGather = 0
+	testContext.KubeConfig = os.Getenv("KUBECONFIG")
+
+	// allow the CSI tests to access test data, but only briefly
+	// TODO: ideally CSI would not use any of these test methods
+	// var err error
+	// exutil.WithCleanup(func() { err = initCSITests(dryRun) })
+	// TODO: for now I'm only initializing CSI directly, but we probably need that
+	// WithCleanup here as well
+	if err := initCSITests(); err != nil {
+		return err
+	}
+
+	if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 {
+		os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts"))
+	}
+
+	testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false"
+	testContext.VerifyServiceAccount = true
+	testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS())
+	testfiles.AddFileSource(testfixtures.GetTestFixturesFS())
+	testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS())
+	testContext.KubectlPath = "kubectl"
+	// context.KubeConfig = KubeConfigPath()
+	testContext.KubeConfig = os.Getenv("KUBECONFIG")
+
+	// "debian" is used when not set. At least GlusterFS tests need "custom".
+	// (There is no option for "rhel" or "centos".)
+	testContext.NodeOSDistro = "custom"
+	testContext.MasterOSDistro = "custom"
+
+	// load and set the host variable for kubectl
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{})
+	cfg, err := clientConfig.ClientConfig()
+	if err != nil {
+		return err
+	}
+	testContext.Host = cfg.Host
+
+	// Ensure that Kube tests run privileged (like they do upstream)
+	testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) {
+		return e2e.CreateTestingNS(ctx, baseName, c, labels, true)
+	}
+
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	framework.AfterReadingAllFlags(testContext)
+	testContext.DumpLogsOnFailure = true
+
+	// these constants are taken from kube e2e and used by tests
+	testContext.IPFamily = "ipv4"
+	if config.HasIPv6 && !config.HasIPv4 {
+		testContext.IPFamily = "ipv6"
+	}
+
+	testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR")
+
+	return nil
+}
+
+const (
+	manifestEnvVar = "TEST_CSI_DRIVER_FILES"
+)
+
+// copied directly from github.com/openshift/origin/cmd/openshift-tests/csi.go
+// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES.
+func initCSITests() error {
+	manifestList := os.Getenv(manifestEnvVar)
+	if manifestList != "" {
+		manifests := strings.Split(manifestList, ",")
+		for _, manifest := range manifests {
+			if err := external.AddDriverDefinition(manifest); err != nil {
+				return fmt.Errorf("failed to load manifest from %q: %s", manifest, err)
+			}
+			// Register the base dir of the manifest file as a file source.
+			// With this we can reference the CSI driver's storageClass
+			// in the manifest file (FromFile field).
+			testfiles.AddFileSource(testfiles.RootFileSource{
+				Root: filepath.Dir(manifest),
+			})
+		}
+	}
+
+	return nil
+}

openshift-hack/cmd/k8s-tests-ext/types.go

@@ -0,0 +1,47 @@
+package main
+
+// copied directly from github.com/openshift/origin/test/extended/util/cluster/cluster.go
+type ClusterConfiguration struct {
+	ProviderName string `json:"type"`
+
+	// These fields (and the "type" tag for ProviderName) chosen to match
+	// upstream's e2e.CloudConfig.
+	ProjectID   string
+	Region      string
+	Zone        string
+	NumNodes    int
+	MultiMaster bool
+	MultiZone   bool
+	Zones       []string
+	ConfigFile  string
+
+	// Disconnected is set for test jobs without external internet connectivity
+	Disconnected bool
+
+	// SingleReplicaTopology is set for disabling disruptive tests or tests
+	// that require high availability
+	SingleReplicaTopology bool
+
+	// NetworkPlugin is the "official" plugin name
+	NetworkPlugin string
+	// NetworkPluginMode is an optional sub-identifier for the NetworkPlugin.
+	// (Currently it is only used for OpenShiftSDN.)
+	NetworkPluginMode string `json:",omitempty"`
+
+	// HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific,
+	// and dual-stack-specific tests are run
+	HasIPv4 bool
+	HasIPv6 bool
+
+	// HasSCTP determines whether SCTP connectivity tests can be run in the cluster
+	HasSCTP bool
+
+	// IsProxied determines whether we are accessing the cluster through an HTTP proxy
+	IsProxied bool
+
+	// IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS)
+	IsIBMROKS bool
+
+	// IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled
+	HasNoOptionalCapabilities bool
+}

openshift-hack/images/hyperkube/Dockerfile.rhel

@@ -1,11 +1,12 @@
 FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder
 WORKDIR /go/src/k8s.io/kubernetes
 COPY . .
-RUN make WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler cmd/kubelet cmd/watch-termination openshift-hack/cmd/k8s-tests' && \
+RUN make WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler cmd/kubelet cmd/watch-termination openshift-hack/cmd/k8s-tests openshift-hack/cmd/k8s-tests-ext' && \
     mkdir -p /tmp/build && \
     cp openshift-hack/images/hyperkube/hyperkube openshift-hack/images/hyperkube/kubensenter /tmp/build && \
-    cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/{kube-apiserver,kube-controller-manager,kube-scheduler,kubelet,watch-termination,k8s-tests} \
-    /tmp/build
+    cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/{kube-apiserver,kube-controller-manager,kube-scheduler,kubelet,watch-termination,k8s-tests,k8s-tests-ext} \
+    /tmp/build && \
+    gzip /tmp/build/k8s-tests-ext
 
 FROM registry.ci.openshift.org/ocp/4.18:base-rhel9
 RUN yum install -y --setopt=tsflags=nodocs --setopt=skip_missing_names_on_install=False iproute && yum clean all