Add snapshot secret reference to group snapshot controller.

Fixes: kubernetes-csi#834
Signed-off-by: Manish <[email protected]>

pkg/common-controller/groupsnapshot_controller_helper.go

@@ -743,7 +743,7 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
 		TODO: Add PVC finalizer
 	*/
 
-	groupSnapshotClass, volumes, contentName, err := ctrl.getCreateGroupSnapshotInput(groupSnapshot)
+	groupSnapshotClass, volumes, contentName, snapshotterSecretRef, err := ctrl.getCreateGroupSnapshotInput(groupSnapshot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get input parameters to create group snapshot %s: %q", groupSnapshot.Name, err)
 	}
@@ -773,8 +773,15 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
 	}
 
 	/*
-		TODO: Add secret reference details
+		Add secret reference details
 	*/
+	if snapshotterSecretRef != nil {
+		klog.V(5).Infof("createGroupSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefName, groupSnapshotContent.Name)
+		metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefName, snapshotterSecretRef.Name)
+
+		klog.V(5).Infof("creategroupSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefNamespace, groupSnapshotContent.Name)
+		metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefNamespace, snapshotterSecretRef.Namespace)
+	}
 
 	var updateGroupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent
 	klog.V(5).Infof("volume group snapshot content %#v", groupSnapshotContent)
@@ -810,7 +817,7 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
 	return updateGroupSnapshotContent, nil
 }
 
-func (ctrl *csiSnapshotCommonController) getCreateGroupSnapshotInput(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotClass, []*v1.PersistentVolume, string, error) {
+func (ctrl *csiSnapshotCommonController) getCreateGroupSnapshotInput(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotClass, []*v1.PersistentVolume, string, *v1.SecretReference, error) {
 	className := groupSnapshot.Spec.VolumeGroupSnapshotClassName
 	klog.V(5).Infof("getCreateGroupSnapshotInput [%s]", groupSnapshot.Name)
 	var groupSnapshotClass *crdv1alpha1.VolumeGroupSnapshotClass
@@ -819,23 +826,31 @@ func (ctrl *csiSnapshotCommonController) getCreateGroupSnapshotInput(groupSnapsh
 		groupSnapshotClass, err = ctrl.getGroupSnapshotClass(*className)
 		if err != nil {
 			klog.Errorf("getCreateGroupSnapshotInput failed to getClassFromVolumeGroupSnapshot %s", err)
-			return nil, nil, "", err
+			return nil, nil, "", nil, err
 		}
 	} else {
 		klog.Errorf("failed to getCreateGroupSnapshotInput %s without a group snapshot class", groupSnapshot.Name)
-		return nil, nil, "", fmt.Errorf("failed to take group snapshot %s without a group snapshot class", groupSnapshot.Name)
+		return nil, nil, "", nil, fmt.Errorf("failed to take group snapshot %s without a group snapshot class", groupSnapshot.Name)
 	}
 
 	volumes, err := ctrl.getVolumesFromVolumeGroupSnapshot(groupSnapshot)
 	if err != nil {
 		klog.Errorf("getCreateGroupSnapshotInput failed to get PersistentVolume objects [%s]: Error: [%#v]", groupSnapshot.Name, err)
-		return nil, nil, "", err
+		return nil, nil, "", nil, err
 	}
 
 	// Create VolumeGroupSnapshotContent name
 	contentName := utils.GetDynamicSnapshotContentNameForGroupSnapshot(groupSnapshot)
 
-	return groupSnapshotClass, volumes, contentName, nil
+	// Get the secret reference
+	snapshotterSecretRef, err := utils.GetGroupSnapshotSecretReference(utils.SnapshotterSecretParams, groupSnapshotClass.Parameters, contentName, groupSnapshot)
+
+	klog.V(5).Infof("snapshotterSecretRef %+v", snapshotterSecretRef)
+	if err != nil {
+		return nil, nil, "", nil, err
+	}
+
+	return groupSnapshotClass, volumes, contentName, snapshotterSecretRef, nil
 }
 
 // syncGroupSnapshotContent deals with one key off the queue

pkg/sidecar-controller/groupsnapshot_helper.go

@@ -423,6 +423,11 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
 		creationTime = time.Now()
 	}
 
+	groupSnapshotSecret, err := utils.GetSecretReference(utils.SnapshotterSecretParams, class.Parameters, groupSnapshotContent.GetObjectMeta().GetName(), nil)
+	if err != nil {
+		klog.Errorf("Failed to get secret reference for group snapshot content %s: %v", groupSnapshotContent.Name, err)
+		return groupSnapshotContent, fmt.Errorf("failed to get secret reference for group snapshot content %s: %v", groupSnapshotContent.Name, err)
+	}
 	// Create individual snapshots and snapshot contents
 	var snapshotContentNames []string
 	for _, snapshot := range snapshots {
@@ -452,6 +457,13 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
 			},
 		}
 
+		if groupSnapshotSecret != nil {
+			klog.V(5).Infof("createGroupSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefName, volumeSnapshotContent.Name)
+			metav1.SetMetaDataAnnotation(&volumeSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefName, groupSnapshotSecret.Name)
+
+			klog.V(5).Infof("createGroupSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefNamespace, volumeSnapshotContent.Name)
+			metav1.SetMetaDataAnnotation(&volumeSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefNamespace, groupSnapshotSecret.Namespace)
+		}
 		label := make(map[string]string)
 		label["volumeGroupSnapshotName"] = groupSnapshotContent.Spec.VolumeGroupSnapshotRef.Name
 		volumeSnapshot := &crdv1.VolumeSnapshot{
@@ -498,7 +510,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
 
 func (ctrl *csiSnapshotSideCarController) getCSIGroupSnapshotInput(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) (*crdv1alpha1.VolumeGroupSnapshotClass, map[string]string, error) {
 	className := groupSnapshotContent.Spec.VolumeGroupSnapshotClassName
-	klog.V(5).Infof("getCSIGroupSnapshotInput for group snapshot content [%s]", groupSnapshotContent.Name)
+	klog.V(5).Infof("getCSIGroupSnapshotInput for group snapshot content [%+v]", groupSnapshotContent)
 	var class *crdv1alpha1.VolumeGroupSnapshotClass
 	var err error
 	if className != nil {
@@ -517,9 +529,13 @@ func (ctrl *csiSnapshotSideCarController) getCSIGroupSnapshotInput(groupSnapshot
 		klog.V(5).Infof("getCSISnapshotInput for groupSnapshotContent [%s]: no VolumeGroupSnapshotClassName provided for pre-provisioned group snapshot", groupSnapshotContent.Name)
 	}
 
-	// TODO: Resolve snapshotting secret credentials.
+	// Resolve snapshotting secret credentials.
+	snapshotterCredentials, err := ctrl.GetGroupCredentialsFromAnnotation(groupSnapshotContent)
+	if err != nil {
+		return nil, nil, err
+	}
 
-	return class, nil, nil
+	return class, snapshotterCredentials, nil
 }
 
 // getGroupSnapshotClass is a helper function to get group snapshot class from the class name.

pkg/sidecar-controller/snapshot_controller.go

@@ -28,6 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	klog "k8s.io/klog/v2"
 
+	crdv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumegroupsnapshot/v1alpha1"
 	crdv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
 	"github.com/kubernetes-csi/external-snapshotter/v7/pkg/utils"
 )
@@ -265,7 +266,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c
 				return content, fmt.Errorf("failed to get snapshot class %s for snapshot content %s: %v", *content.Spec.VolumeSnapshotClassName, content.Name, err)
 			}
 
-			snapshotterListSecretRef, err := utils.GetSecretReference(utils.SnapshotterListSecretParams, class.Parameters, content.GetObjectMeta().GetName(), nil)
+			snapshotterListSecretRef, err := utils.GetSecretReference(utils.SnapshotterSecretParams, class.Parameters, content.GetObjectMeta().GetName(), nil)
 			if err != nil {
 				klog.Errorf("Failed to get secret reference for snapshot content %s: %v", content.Name, err)
 				return content, fmt.Errorf("failed to get secret reference for snapshot content %s: %v", content.Name, err)
@@ -560,6 +561,36 @@ func (ctrl *csiSnapshotSideCarController) GetCredentialsFromAnnotation(content *
 	return snapshotterCredentials, nil
 }
 
+func (ctrl *csiSnapshotSideCarController) GetGroupCredentialsFromAnnotation(content *crdv1alpha1.VolumeGroupSnapshotContent) (map[string]string, error) {
+	// get secrets if VolumeSnapshotClass specifies it
+	var snapshotterCredentials map[string]string
+	var err error
+
+	// Check if annotation exists
+	if metav1.HasAnnotation(content.ObjectMeta, utils.AnnDeletionSecretRefName) && metav1.HasAnnotation(content.ObjectMeta, utils.AnnDeletionSecretRefNamespace) {
+		annDeletionSecretName := content.Annotations[utils.AnnDeletionSecretRefName]
+		annDeletionSecretNamespace := content.Annotations[utils.AnnDeletionSecretRefNamespace]
+
+		snapshotterSecretRef := &v1.SecretReference{}
+
+		if annDeletionSecretName == "" || annDeletionSecretNamespace == "" {
+			return nil, fmt.Errorf("cannot retrieve secrets for snapshot content %#v, err: secret name or namespace not specified", content.Name)
+		}
+
+		snapshotterSecretRef.Name = annDeletionSecretName
+		snapshotterSecretRef.Namespace = annDeletionSecretNamespace
+
+		snapshotterCredentials, err = utils.GetCredentials(ctrl.client, snapshotterSecretRef)
+		if err != nil {
+			// Continue with deletion, as the secret may have already been deleted.
+			klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err.Error())
+			return nil, fmt.Errorf("cannot get credentials for snapshot content %#v", content.Name)
+		}
+	}
+
+	return snapshotterCredentials, nil
+}
+
 // removeContentFinalizer removes the VolumeSnapshotContentFinalizer from a
 // content if there exists one.
 func (ctrl csiSnapshotSideCarController) removeContentFinalizer(content *crdv1.VolumeSnapshotContent) error {

pkg/utils/util.go

@@ -284,6 +284,7 @@ func verifyAndGetSecretNameAndNamespaceTemplate(secret secretParamsMap, snapshot
 		numNamespace++
 	}
 
+	klog.V(5).Infof("numName %v nameTemplate %v, namespaceTemplate %v", numName, nameTemplate, namespaceTemplate)
 	if numName != numNamespace {
 		// Not both 0 or both 1
 		return "", "", fmt.Errorf("either name and namespace for %s secrets specified, Both must be specified", secret.name)
@@ -376,6 +377,64 @@ func GetSecretReference(secretParams secretParamsMap, snapshotClassParams map[st
 	return ref, nil
 }
 
+// GetSecretReference for the group snapshot
+func GetGroupSnapshotSecretReference(secretParams secretParamsMap, snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1alpha1.VolumeGroupSnapshot) (*v1.SecretReference, error) {
+	nameTemplate, namespaceTemplate, err := verifyAndGetSecretNameAndNamespaceTemplate(secretParams, snapshotClassParams)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get name and namespace template from params: %v", err)
+	}
+	if nameTemplate == "" && namespaceTemplate == "" {
+		return nil, nil
+	}
+
+	ref := &v1.SecretReference{}
+
+	// Secret namespace template can make use of the VolumeGroupSnapshotContent name, VolumeGroupSnapshot name or namespace.
+	// Note that neither of those things are under the control of the VolumeGroupSnapshot user.
+	namespaceParams := map[string]string{"volumegroupsnapshotcontent.name": snapContentName}
+	// snapshot may be nil when resolving create/delete volumegroupsnapshot secret names because the
+	// snapshot may or may not exist at delete time
+	if snapshot != nil {
+		namespaceParams["volumegroupsnapshot.namespace"] = snapshot.Namespace
+	}
+
+	resolvedNamespace, err := resolveTemplate(namespaceTemplate, namespaceParams)
+	if err != nil {
+		return nil, fmt.Errorf("error resolving value %q: %v", namespaceTemplate, err)
+	}
+	klog.V(4).Infof("GetGroupSnapshotSecretReference namespaceTemplate %s, namespaceParams: %+v, resolved %s", namespaceTemplate, namespaceParams, resolvedNamespace)
+
+	if len(validation.IsDNS1123Label(resolvedNamespace)) > 0 {
+		if namespaceTemplate != resolvedNamespace {
+			return nil, fmt.Errorf("%q resolved to %q which is not a valid namespace name", namespaceTemplate, resolvedNamespace)
+		}
+		return nil, fmt.Errorf("%q is not a valid namespace name", namespaceTemplate)
+	}
+	ref.Namespace = resolvedNamespace
+
+	// Secret name template can make use of the VolumeGroupSnapshotContent name, VolumeGroupSnapshot name or namespace.
+	// Note that VolumeGroupSnapshot name and namespace are under the VolumeGroupSnapshot user's control.
+	nameParams := map[string]string{"volumegroupsnapshotcontent.name": snapContentName}
+	if snapshot != nil {
+		nameParams["volumegroupsnapshot.name"] = snapshot.Name
+		nameParams["volumegroupsnapshot.namespace"] = snapshot.Namespace
+	}
+	resolvedName, err := resolveTemplate(nameTemplate, nameParams)
+	if err != nil {
+		return nil, fmt.Errorf("error resolving value %q: %v", nameTemplate, err)
+	}
+	if len(validation.IsDNS1123Subdomain(resolvedName)) > 0 {
+		if nameTemplate != resolvedName {
+			return nil, fmt.Errorf("%q resolved to %q which is not a valid secret name", nameTemplate, resolvedName)
+		}
+		return nil, fmt.Errorf("%q is not a valid secret name", nameTemplate)
+	}
+	ref.Name = resolvedName
+
+	klog.V(4).Infof("GetGroupSnapshotSecretReference validated Secret: %+v", ref)
+	return ref, nil
+}
+
 // resolveTemplate resolves the template by checking if the value is missing for a key
 func resolveTemplate(template string, params map[string]string) (string, error) {
 	missingParams := sets.NewString()