WIP: CSI Node daemonset

Signed-off-by: Juan Miguel Olmo Martínez <[email protected]>

config/rbac/role_binding.yaml

@@ -10,3 +10,16 @@ subjects:
 - kind: ServiceAccount
   name: controller-manager
   namespace: system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: topolvm-node-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: topolvm-node
+subjects:
+  - kind: ServiceAccount
+    name: topolvm-node
+    namespace: system

config/rbac/service_account.yaml

@@ -3,3 +3,9 @@ kind: ServiceAccount
 metadata:
   name: controller-manager
   namespace: system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: topolvm-node
+  namespace: system

controllers/defaults.go

@@ -38,6 +38,23 @@ var (
 	CsiProvisionerImage   = GetEnvOrDefault("CSI_PROVISIONER_IMAGE")
 	CsiLivenessProbeImage = GetEnvOrDefault("CSI_LIVENESSPROBE_IMAGE")
 	CsiResizerImage       = GetEnvOrDefault("CSI_RESIZER_IMAGE")
+
+	TopolvmControllerServiceAccount = "topolvm-controller"
+	AppAttr                         = "app.kubernetes.io/name"
+
+	// topoLVM Node
+	TopolvmNodeServiceAccount       = "topolvm-node"
+	TopolvmNodeDaemonsetName        = "topolvm-node"
+	CSIKubeletRootDir               = "/var/lib/kubelet/"
+	lvmdConfigCM                    = "lvmd_config"
+	NodeContainerName               = "topolvm-node"
+	TopolvmNodeContainerHealthzName = "healthz"
+
+	// topoLVM Node resource requests/limits
+	TopolvmNodeMemRequest = "250Mi"
+	TopolvmNodeMemLimit   = "250Mi"
+	TopolvmNodeCPURequest = "250m"
+	TopolvmNodeCPULimit   = "250m"
 )
 
 func GetEnvOrDefault(env string) string {

controllers/lvmcluster_controller.go

@@ -95,7 +95,9 @@ func (r *LVMClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 // errors returned by this will be updated in the reconcileSucceeded condition of the LVMCluster
 func (r *LVMClusterReconciler) reconcile(ctx context.Context, instance *lvmv1alpha1.LVMCluster) (ctrl.Result, error) {
-	resourceList := []resourceManager{}
+	resourceList := []resourceManager{
+		&topolvmNode{},
+	}
 
 	//The resource was deleted
 	if !instance.DeletionTimestamp.IsZero() {

controllers/topolvm_node.go

@@ -0,0 +1,265 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	lvmv1alpha1 "github.com/red-hat-storage/lvm-operator/api/v1alpha1"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	cutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"strings"
+)
+
+const (
+	topolvmNodeName = "topolvm-node"
+)
+
+type topolvmNode struct{}
+
+func (n topolvmNode) getName() string {
+	return topolvmNodeName
+}
+
+func (n topolvmNode) ensureCreated(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error {
+	nodeDaemonSet := getNodeDaemonSet(lvmCluster)
+	result, err := cutil.CreateOrUpdate(ctx, r.Client, nodeDaemonSet, func() error {
+		// make sure LVMCluster CR garbage collects this daemonset and also block owner removal
+		return cutil.SetControllerReference(lvmCluster, nodeDaemonSet, r.Scheme)
+	})
+
+	switch result {
+	case cutil.OperationResultCreated:
+		r.Log.Info(topolvmNodeName, "operation", result, "name", nodeDaemonSet.Name)
+	case cutil.OperationResultUpdated:
+		r.Log.Info(topolvmNodeName, "operation", result, "name", nodeDaemonSet.Name)
+	case cutil.OperationResultNone:
+		r.Log.Info(topolvmNodeName, "operation", result, "name", nodeDaemonSet.Name)
+	default:
+		r.Log.Error(err, fmt.Sprintf("%s reconcile failure", topolvmNodeName), "name", nodeDaemonSet.Name)
+		return err
+	}
+
+	return nil
+}
+
+// ensureDeleted should wait for the resources to be cleaned up
+func (n topolvmNode) ensureDeleted(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error {
+	NodeDaemonSet := &appsv1.DaemonSet{}
+	err := r.Client.Get(ctx, types.NamespacedName{Name: TopolvmNodeDaemonsetName, Namespace: lvmCluster.Namespace}, NodeDaemonSet)
+
+	if err != nil {
+		if errors.IsNotFound(err) {
+			r.Log.Info("topolvm node deleted", "TopolvmNode", NodeDaemonSet.Name)
+			return nil
+		}
+		r.Log.Error(err, "unable to retrieve topolvm node daemonset", "TopolvmNode", NodeDaemonSet.Name)
+		return err
+	}
+	return nil
+}
+
+// updateStatus should optionally update the CR's status about the health of the managed resource
+// each unit will have updateStatus called individually so
+// avoid status fields like lastHeartbeatTime and have a
+// status that changes only when the operands change.
+func (n topolvmNode) updateStatus(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error {
+	return nil
+}
+
+func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster) *v1.DaemonSet {
+	hostPathDirectory := corev1.HostPathDirectory
+	hostPathDirectoryOrCreateType := corev1.HostPathDirectoryOrCreate
+	storageMedium := corev1.StorageMediumMemory
+
+	volumes := []corev1.Volume{
+		{Name: "registration-dir",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: fmt.Sprintf("%splugins_registry/", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+					Type: &hostPathDirectory}}},
+		{Name: "node-plugin-dir",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: fmt.Sprintf("%splugins/topolvm.cybozu.com/node", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+					Type: &hostPathDirectoryOrCreateType}}},
+		{Name: "csi-plugin-dir",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: fmt.Sprintf("%splugins/kubernetes.io/csi", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+					Type: &hostPathDirectoryOrCreateType}}},
+		{Name: "pod-volumes-dir",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: fmt.Sprintf("%spods/", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+					Type: &hostPathDirectoryOrCreateType}}},
+		{Name: "lvmd-config-dir",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{Name: lvmdConfigCM}}}},
+		{Name: "lvmd-socket-dir",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{Medium: storageMedium}}},
+	}
+
+	containers := []corev1.Container{*getNodeContainer(), *getCsiRegistrarContainer(), *getLivenessProbeContainer()}
+
+	// TODO: Add the same node selector we will have in the lvmcluster CRD
+	nodeDaemonset := &v1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      TopolvmNodeDaemonsetName,
+			Namespace: lvmCluster.Namespace,
+		},
+		Spec: v1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					AppAttr: lvmCluster.Name,
+				},
+			},
+			UpdateStrategy: v1.DaemonSetUpdateStrategy{
+				Type: v1.RollingUpdateDaemonSetStrategyType,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: lvmCluster.Name,
+					Labels: map[string]string{
+						AppAttr: lvmCluster.Name,
+					},
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: TopolvmNodeServiceAccount,
+					Containers:         containers,
+					Volumes:            volumes,
+					HostPID:            true,
+					Tolerations:        []corev1.Toleration{{Operator: corev1.TolerationOpExists}},
+				},
+			},
+		},
+	}
+
+	return nodeDaemonset
+}
+
+func getNodeContainer() *corev1.Container {
+	privileged := true
+	runAsUser := int64(0)
+
+	command := []string{
+		"/topolvm-node",
+		"--lvmd-socket=/run/lvmd/lvmd.sock",
+	}
+
+	requirements := corev1.ResourceRequirements{
+		Limits: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse(TopolvmNodeCPULimit),
+			corev1.ResourceMemory: resource.MustParse(TopolvmNodeMemLimit),
+		},
+		Requests: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse(TopolvmNodeCPURequest),
+			corev1.ResourceMemory: resource.MustParse(TopolvmNodeMemRequest),
+		},
+	}
+
+	mountPropagationMode := corev1.MountPropagationBidirectional
+
+	volumeMounts := []corev1.VolumeMount{
+		{Name: "node-plugin-dir", MountPath: "/run/topolvm"},
+		{Name: "lvmd-socket-dir", MountPath: "/run/lvmd"},
+		{Name: "pod-volumes-dir",
+			MountPath:        fmt.Sprintf("%spods", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+			MountPropagation: &mountPropagationMode},
+		{Name: "csi-plugin-dir",
+			MountPath:        fmt.Sprintf("%splugins/kubernetes.io/csi", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+			MountPropagation: &mountPropagationMode},
+	}
+
+	env := []corev1.EnvVar{
+		{Name: "NODE_NAME",
+			ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}},
+	}
+
+	node := &corev1.Container{
+		Name:    NodeContainerName,
+		Image:   TopolvmCsiImage,
+		Command: command,
+		SecurityContext: &corev1.SecurityContext{
+			Privileged: &privileged,
+			RunAsUser:  &runAsUser,
+		},
+		Ports: []corev1.ContainerPort{{Name: TopolvmNodeContainerHealthzName,
+			ContainerPort: 9808,
+			Protocol:      corev1.ProtocolTCP}},
+		LivenessProbe: &corev1.Probe{
+			Handler: corev1.Handler{
+				HTTPGet: &corev1.HTTPGetAction{Path: "/healthz",
+					Port: intstr.FromString(TopolvmNodeContainerHealthzName)}},
+			FailureThreshold:    3,
+			InitialDelaySeconds: 10,
+			TimeoutSeconds:      3,
+			PeriodSeconds:       60},
+		Resources:    requirements,
+		Env:          env,
+		VolumeMounts: volumeMounts,
+	}
+	return node
+}
+
+func getCsiRegistrarContainer() *corev1.Container {
+	command := []string{
+		"/csi-node-driver-registrar",
+		"--csi-address=/run/topolvm/csi-topolvm.sock",
+		fmt.Sprintf("--kubelet-registration-path=%splugins/topolvm.cybozu.com/node/csi-topolvm.sock", getAbsoluteKubeletPath(CSIKubeletRootDir)),
+	}
+
+	volumeMounts := []corev1.VolumeMount{
+		{Name: "node-plugin-dir", MountPath: "/run/topolvm"},
+		{Name: "registration-dir", MountPath: "/registration"},
+	}
+
+	preStopCmd := []string{
+		"/bin/sh",
+		"-c",
+		"rm -rf /registration/topolvm.cybozu.com /registration/topolvm.cybozu.com-reg.sock",
+	}
+
+	csiRegistrar := &corev1.Container{
+		Name:         "csi-registrar",
+		Image:        CsiRegistrarImage,
+		Command:      command,
+		Lifecycle:    &corev1.Lifecycle{PreStop: &corev1.Handler{Exec: &corev1.ExecAction{Command: preStopCmd}}},
+		VolumeMounts: volumeMounts,
+	}
+	return csiRegistrar
+}
+
+func getLivenessProbeContainer() *corev1.Container {
+	command := []string{
+		"/livenessprobe",
+		"--csi-address=/run/topolvm/csi-topolvm.sock",
+	}
+
+	volumeMounts := []corev1.VolumeMount{
+		{Name: "node-plugin-dir", MountPath: "/run/topolvm"},
+	}
+
+	liveness := &corev1.Container{
+		Name:         "liveness-probe",
+		Image:        CsiLivenessProbeImage,
+		Command:      command,
+		VolumeMounts: volumeMounts,
+	}
+	return liveness
+}
+
+func getAbsoluteKubeletPath(name string) string {
+	if strings.HasSuffix(name, "/") {
+		return name
+	} else {
+		return name + "/"
+	}
+}