From 3fc816f952fe8c16654fdb747841e94a7f5a8f7a Mon Sep 17 00:00:00 2001 From: sadasilv Date: Fri, 15 Nov 2024 10:59:25 -0500 Subject: [PATCH 1/4] internal/utils/cluster_environment.go: Change the flavour focus away from microshift vs openshift The reason why we had the flavours as Openshift vs Microshift is that we assumed that if the DPU Operator is running on Openshift then it runs an immutable operating system, which in this case is CoreOS. Like wise if the operator is running on Microshift we assumed that the node is running classic RHEL (mutable). The assumptions are not wrong in our current use cases but microshift's target audience seems to be for immutable flavors of RHEL, so we should make this more robust by checking whether a system is Classic RHEL or if its an OSTREE-Based RHEL, since it gives us a better way to know if parts of the filesystem is writeble or if binaries can even be installed on it by the Operator. This would also add compatibility with running Microshift on RHEL Image Mode, RHEL for edge or RHEL CoreOS on the DPU. --- internal/utils/cluster_environment.go | 112 +++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/internal/utils/cluster_environment.go b/internal/utils/cluster_environment.go index 9065d5e4..fd74e3d3 100644 --- a/internal/utils/cluster_environment.go +++ b/internal/utils/cluster_environment.go @@ -3,6 +3,7 @@ package utils import ( "context" "fmt" + "os" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -22,32 +23,82 @@ func NewClusterEnvironment(client client.Client) *ClusterEnvironment { } type Flavour string +type FlavourSet map[Flavour]struct{} + +// Add a flavour to the set +func (f FlavourSet) Add(flavour Flavour) { + f[flavour] = struct{}{} +} + +// Remove a flavour from the set +func (f FlavourSet) Remove(flavour Flavour) { + delete(f, flavour) +} + +// Check if a flavour exists in the set +func (f FlavourSet) Contains(flavour Flavour) bool { + _, exists := f[flavour] + return exists +} + +// Convert the set to a slice +func (f FlavourSet) ToSlice() []Flavour { + keys := make([]Flavour, 0, len(f)) + for key := range f { + keys = append(keys, key) + } + return keys +} const ( - OpenShiftFlavour Flavour = "OpenShift" - MicroShiftFlavour Flavour = "MicroShift" - UnknownFlavour Flavour = "Unknown" + OpenShiftFlavour Flavour = "OpenShift" + MicroShiftFlavour Flavour = "MicroShift" + OstreeFlavour Flavour = "Ostree" + ClassicRhelFlavour Flavour = "ClassicRHEL" + UnknownFlavour Flavour = "Unknown" ) -func (ce *ClusterEnvironment) Flavour(ctx context.Context) (Flavour, error) { +func (ce *ClusterEnvironment) Flavours(ctx context.Context) (FlavourSet, error) { + detectedFlavours := FlavourSet{} + microShift, err := ce.isMicroShift(ctx) if err != nil { - return UnknownFlavour, err + return nil, err } if microShift { - return MicroShiftFlavour, nil + detectedFlavours.Add(MicroShiftFlavour) } openShift, err := ce.isOpenShift(ctx) if err != nil { - return UnknownFlavour, err + return nil, err } if openShift { - return OpenShiftFlavour, nil + detectedFlavours.Add(OpenShiftFlavour) + } + + ostree, err := ce.isOSTree() + if err != nil { + return nil, err + } + if ostree { + detectedFlavours.Add(OstreeFlavour) + } + + classic, err := ce.isClassicRHEL(ctx) + if err != nil { + return nil, err + } + if classic { + detectedFlavours.Add(ClassicRhelFlavour) } - return UnknownFlavour, nil -} + if len(detectedFlavours) == 0 { + detectedFlavours.Add(UnknownFlavour) + } + + return detectedFlavours, nil +} func (ce *ClusterEnvironment) isMicroShift(ctx context.Context) (bool, error) { cm := v1.ConfigMap{} cm.SetName("microshift-version") @@ -74,3 +125,44 @@ func (ce *ClusterEnvironment) isOpenShift(ctx context.Context) (bool, error) { } return true, nil } + +// isClassicRHEL checks if the OS running on the current node is classic RHEL. +func (ce *ClusterEnvironment) isClassicRHEL(ctx context.Context) (bool, error) { + // Retrieve the node name from the K8S_NODE environment variable + nodeName := os.Getenv("K8S_NODE") + if nodeName == "" { + return false, fmt.Errorf("K8S_NODE environment variable is not set") + } + + // Fetch the Node object using the Kubernetes client + node := &v1.Node{} + if err := ce.client.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { + return false, fmt.Errorf("failed to retrieve node information: %v", err) + } + + // Extract OSImage from the node's status + osImage := node.Status.NodeInfo.OSImage + fmt.Printf("Node OSImage: %s\n", osImage) + + isOstree, err := ce.isOSTree() + if err != nil { + return false, err + } + // Determine if the OS is classic RHEL + if osImage == "Red Hat Enterprise Linux" || (len(osImage) >= 3 && osImage[:3] == "RHEL") && !isOstree { + return true, nil // Classic RHEL detected + } + + return false, nil // Not classic RHEL (could be OSTree-based or something else) +} + +// isOSTree checks for the presence of the /run/ostree-booted file to determine if running on OSTree +func (ce *ClusterEnvironment) isOSTree() (bool, error) { + if _, err := os.Stat("/run/ostree-booted"); err != nil { + if os.IsNotExist(err) { + return false, nil // Not an OSTree-based OS + } + return false, fmt.Errorf("Failed to check OSTree status: %v", err) + } + return true, nil // OSTree-based OS detected +} From 05d6acbd25eeaa3a0fe82e69e4bea673fe7a60d0 Mon Sep 17 00:00:00 2001 From: sadasilv Date: Fri, 15 Nov 2024 11:01:03 -0500 Subject: [PATCH 2/4] internal/utils/path_manager.go: Match each path to its respective flavour OSTREE-Based immutable systems usually mount `/opt` as read only so we need it to use the path that the OpenShift flavour was using before. Likewise we let the ClassicRHEL flavour use the microshift path instead. --- internal/utils/path_manager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/utils/path_manager.go b/internal/utils/path_manager.go index c12e492d..62072d34 100644 --- a/internal/utils/path_manager.go +++ b/internal/utils/path_manager.go @@ -34,13 +34,13 @@ func (p *PathManager) PluginEndpointFilename() string { return filepath.Base(p.PluginEndpoint()) } -func (p *PathManager) CniPath(flavour Flavour) (string, error) { +func (p *PathManager) CniPath(flavours FlavourSet) (string, error) { // Some k8s cluster flavours use /var/lib (in the case of RHCOS based) // and some use /opt (in the case of RHEL based) - switch flavour { - case MicroShiftFlavour: + switch { + case flavours.Contains(ClassicRhelFlavour): return p.wrap("/opt/cni/bin/dpu-cni"), nil - case OpenShiftFlavour: + case flavours.Contains(OstreeFlavour): return p.wrap("/var/lib/cni/bin/dpu-cni"), nil default: return "", fmt.Errorf("unknown flavour") From ca9e8ef6f681398773363e7934fdb5229d16e018 Mon Sep 17 00:00:00 2001 From: sadasilv Date: Fri, 15 Nov 2024 11:01:49 -0500 Subject: [PATCH 3/4] 99.daemonset.yaml: Mount /run so we can interact with ostree-booted - Changed the name of the host-run that mounted the path `/var/run/netns` to match its functionality - Added a mountPath for `/var/run` and named it as host-run, so we have access to the `/run/ostree-booted` and check if it has been created or not. (Doing it this way lets us avoid any extra file creation that might give false positives for the `isOstree` function) --- internal/controller/bindata/daemon/99.daemonset.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/controller/bindata/daemon/99.daemonset.yaml b/internal/controller/bindata/daemon/99.daemonset.yaml index 531489bd..0762dc2e 100644 --- a/internal/controller/bindata/daemon/99.daemonset.yaml +++ b/internal/controller/bindata/daemon/99.daemonset.yaml @@ -49,11 +49,14 @@ spec: mountPath: /var/lib/cni - name: opt-cni-dir mountPath: /opt/cni - - name: host-run + - name: host-run-netns mountPath: /var/run/netns mountPropagation: Bidirectional - name: proc mountPath: /proc + - name: host-run + mountPath: /var/run + readOnly: true args: - --mode - {{.Mode}} @@ -70,9 +73,12 @@ spec: - name: opt-cni-dir hostPath: path: /opt/cni - - name: host-run + - name: host-run-netns hostPath: path: /var/run/netns - name: proc hostPath: path: /proc/ + - name: host-run + hostPath: + path: /var/run From d66af18c1121e80401bf99f1d6a7cf91d5a50d4a Mon Sep 17 00:00:00 2001 From: sadasilv Date: Fri, 22 Nov 2024 11:30:08 -0500 Subject: [PATCH 4/4] daemon.go: Change the Daemon to retrieve a set of Flavours instead of a single flavor - Retrieving a set of flavours lets us match cases well such as being able to differentiate if something is microshift + RHEL For Edge or Openshift + Classic RHEL. (Basically able to match multiple flavours) --- internal/daemon/daemon.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 91f58431..3a101c84 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -67,13 +67,13 @@ func NewDaemon(mode string, client client.Client, scheme *runtime.Scheme, vspIma func (d *Daemon) Run() { ce := utils.NewClusterEnvironment(d.client) - flavour, err := ce.Flavour(context.TODO()) + flavours, err := ce.Flavours(context.TODO()) if err != nil { d.log.Error(err, "Failed to get cluster flavour") return } - d.log.Info("Detected OpenShift", "flavour", flavour) - err = d.prepareCni(flavour) + d.log.Info("Detected Flavours", "flavours", flavours) + err = d.prepareCni(flavours) if err != nil { return } @@ -90,8 +90,8 @@ func (d *Daemon) Run() { daemon.ListenAndServe() } -func (d *Daemon) prepareCni(flavour utils.Flavour) error { - cniPath, err := d.pm.CniPath(flavour) +func (d *Daemon) prepareCni(flavours utils.FlavourSet) error { + cniPath, err := d.pm.CniPath(flavours) if err != nil { d.log.Error(err, "Failed to get cni path") return err