From f766819c788ca65b3d7619d1e9988584e4637d20 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Fri, 5 Apr 2024 20:32:48 +0000 Subject: [PATCH 01/13] implement security settings --- api/v1alpha1/etcdcluster_types.go | 40 +++ api/v1alpha1/etcdcluster_webhook.go | 46 ++++ api/v1alpha1/etcdcluster_webhook_test.go | 95 +++++++ api/v1alpha1/zz_generated.deepcopy.go | 96 +++++++ .../crd/bases/etcd.aenix.io_etcdclusters.yaml | 41 +++ config/manager/kustomization.yaml | 5 +- .../certificates/ca-certificate-peer.yaml | 25 ++ .../certificates/ca-certificate-server.yaml | 25 ++ examples/certificates/ca-issuer-peer.yaml | 8 + examples/certificates/ca-issuer-server.yaml | 8 + .../client-server-certificate-peer.yaml | 30 +++ .../root-client-certificate-server.yaml | 14 + examples/certificates/self-signed-issuer.yaml | 7 + .../server-certificate-server.yaml | 32 +++ internal/controller/factory/statefulset.go | 250 ++++++++++++++---- .../controller/factory/statefulset_test.go | 24 +- 16 files changed, 676 insertions(+), 70 deletions(-) create mode 100644 examples/certificates/ca-certificate-peer.yaml create mode 100644 examples/certificates/ca-certificate-server.yaml create mode 100644 examples/certificates/ca-issuer-peer.yaml create mode 100644 examples/certificates/ca-issuer-server.yaml create mode 100644 examples/certificates/client-server-certificate-peer.yaml create mode 100644 examples/certificates/root-client-certificate-server.yaml create mode 100644 examples/certificates/self-signed-issuer.yaml create mode 100644 examples/certificates/server-certificate-server.yaml diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index adbbb3f4..27202b89 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -41,6 +41,9 @@ type EtcdClusterSpec struct { // +optional PodDisruptionBudgetTemplate *EmbeddedPodDisruptionBudget `json:"podDisruptionBudgetTemplate,omitempty"` Storage StorageSpec `json:"storage"` + // Security describes security settings of etcd (authentication, certificates, rbac) + // +optional + Security *SecuritySpec `json:"security,omitempty"` } const ( @@ -200,6 +203,43 @@ type StorageSpec struct { VolumeClaimTemplate EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"` } +// SecuritySpec defines security settings for etcd. +// +k8s:openapi-gen=true +type SecuritySpec struct { + // +optional + Peer *PeerSpec `json:"peer,omitempty"` + // +optional + ClientServer *ClientServerSpec `json:"clientServer,omitempty"` + // +optional + Rbac RbacSpec `json:"rbac,omitempty"` +} + +type PeerSpec struct { + // +optional + Ca SecretSpec `json:"ca,omitempty"` + // +optional + Cert SecretSpec `json:"cert,omitempty"` +} + +type ClientServerSpec struct { + // +optional + Ca SecretSpec `json:"ca,omitempty"` + // +optional + Cert SecretSpec `json:"cert,omitempty"` + // +optional + RootClientCert SecretSpec `json:"rootClientCert,omitempty"` +} + +type SecretSpec struct { + // +optional + SecretName string `json:"secretName,omitempty"` +} + +type RbacSpec struct { + // +optional + Enabled bool `json:"enabled,omitempty"` +} + // EmbeddedPersistentVolumeClaim is an embedded version of k8s.io/api/core/v1.PersistentVolumeClaim. // It contains TypeMeta and a reduced ObjectMeta. type EmbeddedPersistentVolumeClaim struct { diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 44d424ee..9d38d8f7 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -95,6 +95,11 @@ func (r *EtcdCluster) ValidateCreate() (admission.Warnings, error) { allErrors = append(allErrors, pdbErr...) } + securityErr := r.validateSecurity() + if securityErr != nil { + allErrors = append(allErrors, securityErr...) + } + if errOptions := validateOptions(r); errOptions != nil { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "options"), @@ -256,6 +261,47 @@ func (r *EtcdCluster) validatePdb() (admission.Warnings, field.ErrorList) { return warnings, nil } +func (r *EtcdCluster) validateSecurity() field.ErrorList { + + var allErrors field.ErrorList + + if r.Spec.Security == nil { + return nil + } + + security := r.Spec.Security + + if security.Peer != nil { + if (security.Peer.Ca.SecretName != "" && security.Peer.Cert.SecretName == "") || + (security.Peer.Ca.SecretName == "" && security.Peer.Cert.SecretName != "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security", "peer.ca.secretName", "peer.cert.secretName"), + security.Peer, + "both peer.ca.secretName and peer.cert.secretName must be filled or empty"), + ) + } + } + + if security.ClientServer != nil { + if (security.ClientServer.Ca.SecretName != "" && security.ClientServer.Cert.SecretName == "") || + (security.ClientServer.Ca.SecretName == "" && security.ClientServer.Cert.SecretName != "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security", "clientServer.ca.secretName", "clientServer.cert.secretName"), + security.ClientServer, + "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty"), + ) + } + } + + if len(allErrors) > 0 { + return allErrors + } + + return nil +} + func validateOptions(cluster *EtcdCluster) error { if len(cluster.Spec.Options) == 0 { return nil diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index ac176551..c850530a 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -130,6 +130,101 @@ var _ = Describe("EtcdCluster Webhook", func() { }) }) + Context("Validate Security", func() { + etcdCluster := &EtcdCluster{ + Spec: EtcdClusterSpec{ + Replicas: ptr.To(int32(3)), + Security: &SecuritySpec{}, + }, + } + It("Should admit enabled empty security", func() { + localCluster := etcdCluster.DeepCopy() + err := localCluster.validateSecurity() + Expect(err).To(BeNil()) + }) + + It("Should reject if only one secret in peer section is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.Peer = &PeerSpec{ + Ca: SecretSpec{ + SecretName: "test-peer-ca-cert", + }, + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "peer.ca.secretName", "peer.cert.secretName"), + localCluster.Spec.Security.Peer, + "both peer.ca.secretName and peer.cert.secretName must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + It("Should reject if only one secret in peer section is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.Peer = &PeerSpec{ + Cert: SecretSpec{ + SecretName: "test-peer-cert", + }, + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "peer.ca.secretName", "peer.cert.secretName"), + localCluster.Spec.Security.Peer, + "both peer.ca.secretName and peer.cert.secretName must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + It("Should reject if only one secret in peer section is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.ClientServer = &ClientServerSpec{ + Ca: SecretSpec{ + SecretName: "test-ca-server-cert", + }, + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "clientServer.ca.secretName", "clientServer.cert.secretName"), + localCluster.Spec.Security.ClientServer, + "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + It("Should reject if only one secret in peer section is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.ClientServer = &ClientServerSpec{ + Cert: SecretSpec{ + SecretName: "test-server-cert", + }, + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "clientServer.ca.secretName", "clientServer.cert.secretName"), + localCluster.Spec.Security.ClientServer, + "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + }) + Context("Validate PDB", func() { etcdCluster := &EtcdCluster{ Spec: EtcdClusterSpec{ diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index add1e7d7..a9cc47b6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,24 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientServerSpec) DeepCopyInto(out *ClientServerSpec) { + *out = *in + out.Ca = in.Ca + out.Cert = in.Cert + out.RootClientCert = in.RootClientCert +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientServerSpec. +func (in *ClientServerSpec) DeepCopy() *ClientServerSpec { + if in == nil { + return nil + } + out := new(ClientServerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EmbeddedObjectMetadata) DeepCopyInto(out *EmbeddedObjectMetadata) { *out = *in @@ -173,6 +191,11 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) { (*in).DeepCopyInto(*out) } in.Storage.DeepCopyInto(&out.Storage) + if in.Security != nil { + in, out := &in.Security, &out.Security + *out = new(SecuritySpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdClusterSpec. @@ -207,6 +230,23 @@ func (in *EtcdClusterStatus) DeepCopy() *EtcdClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerSpec) DeepCopyInto(out *PeerSpec) { + *out = *in + out.Ca = in.Ca + out.Cert = in.Cert +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerSpec. +func (in *PeerSpec) DeepCopy() *PeerSpec { + if in == nil { + return nil + } + out := new(PeerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodDisruptionBudgetSpec) DeepCopyInto(out *PodDisruptionBudgetSpec) { *out = *in @@ -329,6 +369,62 @@ func (in *PodTemplate) DeepCopy() *PodTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RbacSpec) DeepCopyInto(out *RbacSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RbacSpec. +func (in *RbacSpec) DeepCopy() *RbacSpec { + if in == nil { + return nil + } + out := new(RbacSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretSpec) DeepCopyInto(out *SecretSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSpec. +func (in *SecretSpec) DeepCopy() *SecretSpec { + if in == nil { + return nil + } + out := new(SecretSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecuritySpec) DeepCopyInto(out *SecuritySpec) { + *out = *in + if in.Peer != nil { + in, out := &in.Peer, &out.Peer + *out = new(PeerSpec) + **out = **in + } + if in.ClientServer != nil { + in, out := &in.ClientServer, &out.ClientServer + *out = new(ClientServerSpec) + **out = **in + } + out.Rbac = in.Rbac +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecuritySpec. +func (in *SecuritySpec) DeepCopy() *SecuritySpec { + if in == nil { + return nil + } + out := new(SecuritySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { *out = *in diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index 1b808fc3..79f459a2 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -4571,6 +4571,47 @@ spec: format: int32 minimum: 0 type: integer + security: + description: Security describes security settings of etcd (authentication, + certificates, rbac) + properties: + clientServer: + properties: + ca: + properties: + secretName: + type: string + type: object + cert: + properties: + secretName: + type: string + type: object + rootClientCert: + properties: + secretName: + type: string + type: object + type: object + peer: + properties: + ca: + properties: + secretName: + type: string + type: object + cert: + properties: + secretName: + type: string + type: object + type: object + rbac: + properties: + enabled: + type: boolean + type: object + type: object storage: description: |- StorageSpec defines the configured storage for a etcd members. diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 8335489f..40c41b8b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -2,4 +2,7 @@ resources: - manager.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -images: [] +images: +- name: ghcr.io/aenix-io/etcd-operator + newName: ghcr.io/aenix-io/etcd-operator + newTag: latest diff --git a/examples/certificates/ca-certificate-peer.yaml b/examples/certificates/ca-certificate-peer.yaml new file mode 100644 index 00000000..856bad5e --- /dev/null +++ b/examples/certificates/ca-certificate-peer.yaml @@ -0,0 +1,25 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-crname-crnamespace-peer + namespace: etcd-operator-system +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-crname-crnamespace-peer + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-secret-crname-crnamespace-peer + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer-crname-crnamespace + kind: Issuer + group: cert-manager.io diff --git a/examples/certificates/ca-certificate-server.yaml b/examples/certificates/ca-certificate-server.yaml new file mode 100644 index 00000000..8958bb98 --- /dev/null +++ b/examples/certificates/ca-certificate-server.yaml @@ -0,0 +1,25 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-crname-crnamespace-server + namespace: etcd-operator-system +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-crname-crnamespace-server + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-secret-crname-crnamespace-server + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer-crname-crnamespace + kind: Issuer + group: cert-manager.io diff --git a/examples/certificates/ca-issuer-peer.yaml b/examples/certificates/ca-issuer-peer.yaml new file mode 100644 index 00000000..d6732c1e --- /dev/null +++ b/examples/certificates/ca-issuer-peer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-crname-crnamespace-peer + namespace: etcd-operator-system +spec: + ca: + secretName: ca-secret-crname-crnamespace-peer diff --git a/examples/certificates/ca-issuer-server.yaml b/examples/certificates/ca-issuer-server.yaml new file mode 100644 index 00000000..7f40d1f7 --- /dev/null +++ b/examples/certificates/ca-issuer-server.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-crname-crnamespace-server + namespace: etcd-operator-system +spec: + ca: + secretName: ca-secret-crname-crnamespace-server diff --git a/examples/certificates/client-server-certificate-peer.yaml b/examples/certificates/client-server-certificate-peer.yaml new file mode 100644 index 00000000..9a5e2b09 --- /dev/null +++ b/examples/certificates/client-server-certificate-peer.yaml @@ -0,0 +1,30 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: certificate-crname-crnamespace-peer + namespace: etcd-operator-system +spec: + secretName: crname-crnamespace-peer-tls + isCA: false + usages: + - server auth + - client auth + - signing + - key encipherment + dnsNames: + - etcdcluster-sample-0 + - etcdcluster-sample-0.etcdcluster-sample + - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc + - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc.cluster.local + - etcdcluster-sample-1 + - etcdcluster-sample-1.etcdcluster-sample + - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc + - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc.cluster.local + - etcdcluster-sample-2 + - etcdcluster-sample-2.etcdcluster-sample + - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc + - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc.cluster.local + - localhost + - "127.0.0.1" + issuerRef: + name: ca-issuer-crname-crnamespace-peer diff --git a/examples/certificates/root-client-certificate-server.yaml b/examples/certificates/root-client-certificate-server.yaml new file mode 100644 index 00000000..740c359b --- /dev/null +++ b/examples/certificates/root-client-certificate-server.yaml @@ -0,0 +1,14 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: client-certificate-crname-crnamespace-peer +spec: + commonName: root + secretName: crname-crnamespace-client-peer-tls + usages: + - "signing" + - "key encipherment" + - "client auth" + issuerRef: + name: ca-issuer-crname-crnamespace-server + kind: Issuer diff --git a/examples/certificates/self-signed-issuer.yaml b/examples/certificates/self-signed-issuer.yaml new file mode 100644 index 00000000..028f4905 --- /dev/null +++ b/examples/certificates/self-signed-issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer-crname-crnamespace + namespace: etcd-operator-system +spec: + selfSigned: {} diff --git a/examples/certificates/server-certificate-server.yaml b/examples/certificates/server-certificate-server.yaml new file mode 100644 index 00000000..4ca9deb8 --- /dev/null +++ b/examples/certificates/server-certificate-server.yaml @@ -0,0 +1,32 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: certificate-crname-crnamespace-server + namespace: etcd-operator-system +spec: + secretName: crname-crnamespace-server-tls + isCA: false + usages: + - server auth + - signing + - key encipherment + dnsNames: + - etcdcluster-sample-0 + - etcdcluster-sample-0.etcdcluster-sample + - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc + - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc.cluster.local + - etcdcluster-sample-1 + - etcdcluster-sample-1.etcdcluster-sample + - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc + - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc.cluster.local + - etcdcluster-sample-2 + - etcdcluster-sample-2.etcdcluster-sample + - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc + - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc.cluster.local + - etcdcluster-sample-client + - etcdcluster-sample-client.etcd-operator-system.svc + - etcdcluster-sample-client.etcd-operator-system.svc.cluster.local + - localhost + - "127.0.0.1" + issuerRef: + name: ca-issuer-crname-crnamespace-server diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index a433643c..2a1b50e4 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -52,6 +52,20 @@ func CreateOrUpdateStatefulSet( podMetadata.Annotations = cluster.Spec.PodTemplate.Annotations + volumeClaimTemplates := []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: GetPVCName(cluster), + Labels: cluster.Spec.Storage.VolumeClaimTemplate.Labels, + Annotations: cluster.Spec.Storage.VolumeClaimTemplate.Annotations, + }, + Spec: cluster.Spec.Storage.VolumeClaimTemplate.Spec, + Status: cluster.Spec.Storage.VolumeClaimTemplate.Status, + }, + } + + volumes := generateVolumes(cluster) + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, @@ -81,54 +95,159 @@ func CreateOrUpdateStatefulSet( ServiceAccountName: cluster.Spec.PodTemplate.Spec.ServiceAccountName, ReadinessGates: cluster.Spec.PodTemplate.Spec.ReadinessGates, RuntimeClassName: cluster.Spec.PodTemplate.Spec.RuntimeClassName, + Volumes: volumes, }, }, + VolumeClaimTemplates: volumeClaimTemplates, }, } - statefulSet.Spec.Template.Spec.Volumes = cluster.Spec.PodTemplate.Spec.Volumes - dataVolumeIdx := slices.IndexFunc(statefulSet.Spec.Template.Spec.Volumes, func(volume corev1.Volume) bool { + + if err := ctrl.SetControllerReference(cluster, statefulSet, rscheme); err != nil { + return fmt.Errorf("cannot set controller reference: %w", err) + } + + return reconcileStatefulSet(ctx, rclient, cluster.Name, statefulSet) +} + +func generateVolumes(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Volume { + volumes := []corev1.Volume{} + + dataVolumeSource := corev1.VolumeSource{} + + if cluster.Spec.Storage.EmptyDir != nil { + dataVolumeSource = corev1.VolumeSource{EmptyDir: cluster.Spec.Storage.EmptyDir} + } else { + dataVolumeSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: GetPVCName(cluster), + }, + } + } + + dataVolumeIdx := slices.IndexFunc(cluster.Spec.PodTemplate.Spec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "data" }) if dataVolumeIdx == -1 { - dataVolumeIdx = len(statefulSet.Spec.Template.Spec.Volumes) - statefulSet.Spec.Template.Spec.Volumes = append( - statefulSet.Spec.Template.Spec.Volumes, - corev1.Volume{Name: "data"}, + dataVolumeIdx = len(cluster.Spec.PodTemplate.Spec.Volumes) + volumes = append( + volumes, + corev1.Volume{}, ) } - if cluster.Spec.Storage.EmptyDir != nil { - statefulSet.Spec.Template.Spec.Volumes[dataVolumeIdx] = corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{EmptyDir: cluster.Spec.Storage.EmptyDir}, - } - } else { - statefulSet.Spec.Template.Spec.Volumes[dataVolumeIdx] = corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: GetPVCName(cluster), + volumes[dataVolumeIdx] = corev1.Volume{ + Name: "data", + VolumeSource: dataVolumeSource, + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.Peer != nil { + volumes = append(volumes, + []corev1.Volume{ + { + Name: "ca-peer-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.Peer.Ca.SecretName, + }, + }, }, - }, + { + Name: "peer-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.Peer.Cert.SecretName, + }, + }, + }, + }...) + + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.ClientServer != nil { + volumes = append(volumes, + []corev1.Volume{ + { + Name: "ca-server-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.ClientServer.Ca.SecretName, + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.ClientServer.Cert.SecretName, + }, + }, + }, + }...) + } + + return volumes + +} + +func generateVolumeMounts(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.VolumeMount { + + volumeMounts := []corev1.VolumeMount{} + + for _, c := range cluster.Spec.PodTemplate.Spec.Containers { + if c.Name == "etcd" { + + volumeMounts = c.VolumeMounts + + mountIdx := slices.IndexFunc(volumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == "data" + }) + if mountIdx == -1 { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "data", + ReadOnly: false, + MountPath: "/var/run/etcd", + }) + } else { + volumeMounts[mountIdx].ReadOnly = false + volumeMounts[mountIdx].MountPath = "/var/run/etcd" + } } - statefulSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.Peer != nil { + + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ { - ObjectMeta: metav1.ObjectMeta{ - Name: GetPVCName(cluster), - Labels: cluster.Spec.Storage.VolumeClaimTemplate.Labels, - Annotations: cluster.Spec.Storage.VolumeClaimTemplate.Annotations, - }, - Spec: cluster.Spec.Storage.VolumeClaimTemplate.Spec, - Status: cluster.Spec.Storage.VolumeClaimTemplate.Status, + Name: "ca-peer-cert", + ReadOnly: true, + MountPath: "/etc/etcd/pki/peer/ca", }, - } + { + Name: "peer-cert", + ReadOnly: true, + MountPath: "/etc/etcd/pki/peer/cert", + }, + }...) } - if err := ctrl.SetControllerReference(cluster, statefulSet, rscheme); err != nil { - return fmt.Errorf("cannot set controller reference: %w", err) + if cluster.Spec.Security != nil && cluster.Spec.Security.ClientServer != nil { + + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ + { + Name: "ca-server-cert", + ReadOnly: true, + MountPath: "/etc/etcd/pki/server/ca", + }, + { + Name: "server-cert", + ReadOnly: true, + MountPath: "/etc/etcd/pki/server/cert", + }, + }...) } - return reconcileStatefulSet(ctx, rclient, cluster.Name, statefulSet) + return volumeMounts } func generateEtcdCommand() []string { @@ -138,17 +257,7 @@ func generateEtcdCommand() []string { } func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { - args := []string{ - "--name=$(POD_NAME)", - "--listen-peer-urls=https://0.0.0.0:2380", - // for first version disable TLS for client access - "--listen-client-urls=http://0.0.0.0:2379", - fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", cluster.Name), - "--data-dir=/var/run/etcd/default.etcd", - "--auto-tls", - "--peer-auto-tls", - fmt.Sprintf("--advertise-client-urls=http://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", cluster.Name), - } + args := []string{} for name, value := range cluster.Spec.Options { flag := "--" + name @@ -161,6 +270,45 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { args = append(args, fmt.Sprintf("%s=%s", flag, value)) } + peerTlsSettings := []string{"--peer-auto-tls"} + + if cluster.Spec.Security != nil && cluster.Spec.Security.Peer != nil { + peerTlsSettings = []string{ + "--peer-trusted-ca-file=/etc/etcd/pki/peer/ca/ca.crt", + "--peer-cert-file=/etc/etcd/pki/peer/cert/tls.crt", + "--peer-key-file=/etc/etcd/pki/peer/cert/tls.key", + "--peer-client-cert-auth", + } + } + + serverProtocol := "http" + + clientTlsSettings := []string{"--auto-tls"} + + if cluster.Spec.Security != nil && cluster.Spec.Security.ClientServer != nil { + clientTlsSettings = []string{ + "--trusted-ca-file=/etc/etcd/pki/server/ca/ca.crt", + "--cert-file=/etc/etcd/pki/server/cert/tls.crt", + "--key-file=/etc/etcd/pki/server/cert/tls.key", + "--client-cert-auth", + } + } + + serverProtocol = "https" + + args = append(args, []string{ + "--name=$(POD_NAME)", + "--listen-metrics-urls=http://0.0.0.0:2381", + "--listen-peer-urls=https://0.0.0.0:2380", + fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", serverProtocol), + fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", cluster.Name), + "--data-dir=/var/run/etcd/default.etcd", + fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", serverProtocol, cluster.Name), + }...) + + args = append(args, peerTlsSettings...) + args = append(args, clientTlsSettings...) + return args } @@ -210,19 +358,7 @@ func generateContainers(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Conta c.LivenessProbe = getLivenessProbe(c.LivenessProbe) c.ReadinessProbe = getReadinessProbe(c.ReadinessProbe) c.Env = mergeEnvs(c.Env, podEnv) - - mountIdx := slices.IndexFunc(c.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == "data" - }) - if mountIdx == -1 { - c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ - Name: "data", - MountPath: "/var/run/etcd", - }) - } else { - c.VolumeMounts[mountIdx].ReadOnly = false - c.VolumeMounts[mountIdx].MountPath = "/var/run/etcd" - } + c.VolumeMounts = generateVolumeMounts(cluster) } containers = append(containers, c) @@ -270,7 +406,7 @@ func getStartupProbe(probe *corev1.Probe) *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -283,7 +419,7 @@ func getReadinessProbe(probe *corev1.Probe) *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -296,7 +432,7 @@ func getLivenessProbe(probe *corev1.Probe) *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index 8618f0c2..163221ce 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -157,7 +157,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -173,7 +173,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -189,7 +189,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -259,7 +259,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -275,7 +275,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -353,7 +353,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -381,7 +381,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { PeriodSeconds: 3, })) }) - It("should correctly override partial changes ", func() { + It("should correctly override partial changes", func() { probe := getLivenessProbe(&v1.Probe{ InitialDelaySeconds: 7, PeriodSeconds: 3, @@ -390,7 +390,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, InitialDelaySeconds: 7, @@ -406,7 +406,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -443,7 +443,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, InitialDelaySeconds: 7, @@ -459,7 +459,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -496,7 +496,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, InitialDelaySeconds: 11, From 54be512acb9b96f8603c97a3a4668f5e781f43c8 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Sat, 6 Apr 2024 21:58:25 +0000 Subject: [PATCH 02/13] add tests --- api/v1alpha1/etcdcluster_webhook.go | 5 + internal/controller/factory/statefulset.go | 2 +- .../controller/factory/statefulset_test.go | 104 ++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 9d38d8f7..c12a0927 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -144,6 +144,11 @@ func (r *EtcdCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, er warnings = append(warnings, pdbWarnings...) } + securityErr := r.validateSecurity() + if securityErr != nil { + allErrors = append(allErrors, securityErr...) + } + if errOptions := validateOptions(r); errOptions != nil { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "options"), diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index 2a1b50e4..814cbb6c 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -290,7 +290,7 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { "--trusted-ca-file=/etc/etcd/pki/server/ca/ca.crt", "--cert-file=/etc/etcd/pki/server/cert/tls.crt", "--key-file=/etc/etcd/pki/server/cert/tls.key", - "--client-cert-auth", + "--client-cert-auth=false", } } diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index 163221ce..af3fc436 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -117,6 +118,24 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }, }, } + etcdcluster.Spec.Security = &etcdaenixiov1alpha1.SecuritySpec{ + ClientServer: &etcdaenixiov1alpha1.ClientServerSpec{ + Ca: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "server-ca-secret", + }, + Cert: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "server-cert-secret", + }, + }, + Peer: &etcdaenixiov1alpha1.PeerSpec{ + Ca: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "peer-ca-secret", + }, + Cert: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "peer-cert-secret", + }, + }, + } sts := &appsv1.StatefulSet{} err := CreateOrUpdateStatefulSet(ctx, etcdcluster, k8sClient, k8sClient.Scheme()) @@ -200,6 +219,45 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { })) }) + By("Checking generated security volumes", func() { + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "ca-peer-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "peer-ca-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "peer-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "peer-cert-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "ca-server-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "server-ca-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "server-cert-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + }) + By("Deleting the statefulset", func() { Expect(k8sClient.Delete(ctx, sts)).To(Succeed()) }) @@ -639,5 +697,51 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { } } }) + It("should generate security volumes mounts", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security = &etcdaenixiov1alpha1.SecuritySpec{ + ClientServer: &etcdaenixiov1alpha1.ClientServerSpec{ + Ca: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "client-server-ca-secret", + }, + Cert: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "client-server-cert-secret", + }, + }, + Peer: &etcdaenixiov1alpha1.PeerSpec{ + Ca: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "peer-ca-secret", + }, + Cert: etcdaenixiov1alpha1.SecretSpec{ + SecretName: "peer-cert-secret", + }, + }, + } + + containers := generateContainers(localCluster) + + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "ca-peer-cert", + MountPath: "/etc/etcd/pki/peer/ca", + ReadOnly: true, + })) + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "peer-cert", + MountPath: "/etc/etcd/pki/peer/cert", + ReadOnly: true, + })) + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "ca-server-cert", + MountPath: "/etc/etcd/pki/server/ca", + ReadOnly: true, + })) + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "server-cert", + MountPath: "/etc/etcd/pki/server/cert", + ReadOnly: true, + })) + + }) + }) }) From e00851ea07e90878a72d514bb013b0b68e591497 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Sat, 6 Apr 2024 22:22:43 +0000 Subject: [PATCH 03/13] fix --- internal/controller/factory/statefulset.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index 814cbb6c..a6721da0 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -112,7 +112,7 @@ func CreateOrUpdateStatefulSet( func generateVolumes(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Volume { volumes := []corev1.Volume{} - dataVolumeSource := corev1.VolumeSource{} + var dataVolumeSource corev1.VolumeSource if cluster.Spec.Storage.EmptyDir != nil { dataVolumeSource = corev1.VolumeSource{EmptyDir: cluster.Spec.Storage.EmptyDir} @@ -292,10 +292,9 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { "--key-file=/etc/etcd/pki/server/cert/tls.key", "--client-cert-auth=false", } + serverProtocol = "https" } - serverProtocol = "https" - args = append(args, []string{ "--name=$(POD_NAME)", "--listen-metrics-urls=http://0.0.0.0:2381", From 51a9967b14c0fff1c9c75efaf577e0f7aa0e98e9 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Sat, 6 Apr 2024 22:32:01 +0000 Subject: [PATCH 04/13] fix etcd to constant --- internal/controller/factory/statefulset.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index a6721da0..52cedcf7 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -32,6 +32,10 @@ import ( etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" ) +const ( + etcdContainerName = "etcd" +) + func CreateOrUpdateStatefulSet( ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster, @@ -194,7 +198,7 @@ func generateVolumeMounts(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Vol volumeMounts := []corev1.VolumeMount{} for _, c := range cluster.Spec.PodTemplate.Spec.Containers { - if c.Name == "etcd" { + if c.Name == etcdContainerName { volumeMounts = c.VolumeMounts @@ -333,7 +337,7 @@ func generateContainers(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Conta containers := make([]corev1.Container, 0, len(cluster.Spec.PodTemplate.Spec.Containers)) for _, c := range cluster.Spec.PodTemplate.Spec.Containers { - if c.Name == "etcd" { + if c.Name == etcdContainerName { c.Command = generateEtcdCommand() c.Args = generateEtcdArgs(cluster) c.Ports = mergePorts(c.Ports, []corev1.ContainerPort{ From 2353459d0b9baddf6d3ee51bd9ad8c27c55ba17f Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Mon, 8 Apr 2024 11:40:01 +0000 Subject: [PATCH 05/13] fix webhook validation messages --- api/v1alpha1/etcdcluster_webhook.go | 4 ++-- api/v1alpha1/etcdcluster_webhook_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index c12a0927..48861b43 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -281,7 +281,7 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { (security.Peer.Ca.SecretName == "" && security.Peer.Cert.SecretName != "") { allErrors = append(allErrors, field.Invalid( - field.NewPath("spec", "security", "peer.ca.secretName", "peer.cert.secretName"), + field.NewPath("spec", "security", "peer"), security.Peer, "both peer.ca.secretName and peer.cert.secretName must be filled or empty"), ) @@ -293,7 +293,7 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { (security.ClientServer.Ca.SecretName == "" && security.ClientServer.Cert.SecretName != "") { allErrors = append(allErrors, field.Invalid( - field.NewPath("spec", "security", "clientServer.ca.secretName", "clientServer.cert.secretName"), + field.NewPath("spec", "security", "clientServer"), security.ClientServer, "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty"), ) diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index c850530a..0603bbfd 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -153,7 +153,7 @@ var _ = Describe("EtcdCluster Webhook", func() { err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "peer.ca.secretName", "peer.cert.secretName"), + field.NewPath("spec", "security", "peer"), localCluster.Spec.Security.Peer, "both peer.ca.secretName and peer.cert.secretName must be filled or empty", ) @@ -173,7 +173,7 @@ var _ = Describe("EtcdCluster Webhook", func() { err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "peer.ca.secretName", "peer.cert.secretName"), + field.NewPath("spec", "security", "peer"), localCluster.Spec.Security.Peer, "both peer.ca.secretName and peer.cert.secretName must be filled or empty", ) @@ -183,7 +183,7 @@ var _ = Describe("EtcdCluster Webhook", func() { } }) - It("Should reject if only one secret in peer section is defined", func() { + It("Should reject if only one secret in clientServer section is defined", func() { localCluster := etcdCluster.DeepCopy() localCluster.Spec.Security.ClientServer = &ClientServerSpec{ Ca: SecretSpec{ @@ -193,7 +193,7 @@ var _ = Describe("EtcdCluster Webhook", func() { err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "clientServer.ca.secretName", "clientServer.cert.secretName"), + field.NewPath("spec", "security", "clientServer"), localCluster.Spec.Security.ClientServer, "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty", ) @@ -203,7 +203,7 @@ var _ = Describe("EtcdCluster Webhook", func() { } }) - It("Should reject if only one secret in peer section is defined", func() { + It("Should reject if only one secret in clientServer section is defined", func() { localCluster := etcdCluster.DeepCopy() localCluster.Spec.Security.ClientServer = &ClientServerSpec{ Cert: SecretSpec{ @@ -213,7 +213,7 @@ var _ = Describe("EtcdCluster Webhook", func() { err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "clientServer.ca.secretName", "clientServer.cert.secretName"), + field.NewPath("spec", "security", "clientServer"), localCluster.Spec.Security.ClientServer, "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty", ) From 4bfccfb21d6334ff6b2efe27affc562fefb44e41 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Mon, 8 Apr 2024 17:40:13 +0000 Subject: [PATCH 06/13] create etcdCLuster example with ext certs --- .../certificates/ca-certificate-peer.yaml | 25 --- .../certificates/ca-certificate-server.yaml | 25 --- examples/certificates/ca-issuer-peer.yaml | 8 - examples/certificates/ca-issuer-server.yaml | 8 - .../client-server-certificate-peer.yaml | 30 --- .../root-client-certificate-server.yaml | 14 -- examples/certificates/self-signed-issuer.yaml | 7 - .../server-certificate-server.yaml | 32 ---- ...tcdcluster-with-external-certificates.yaml | 176 ++++++++++++++++++ 9 files changed, 176 insertions(+), 149 deletions(-) delete mode 100644 examples/certificates/ca-certificate-peer.yaml delete mode 100644 examples/certificates/ca-certificate-server.yaml delete mode 100644 examples/certificates/ca-issuer-peer.yaml delete mode 100644 examples/certificates/ca-issuer-server.yaml delete mode 100644 examples/certificates/client-server-certificate-peer.yaml delete mode 100644 examples/certificates/root-client-certificate-server.yaml delete mode 100644 examples/certificates/self-signed-issuer.yaml delete mode 100644 examples/certificates/server-certificate-server.yaml create mode 100644 examples/manifests/etcdcluster-with-external-certificates.yaml diff --git a/examples/certificates/ca-certificate-peer.yaml b/examples/certificates/ca-certificate-peer.yaml deleted file mode 100644 index 856bad5e..00000000 --- a/examples/certificates/ca-certificate-peer.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: ca-certificate-crname-crnamespace-peer - namespace: etcd-operator-system -spec: - isCA: true - usages: - - "signing" - - "key encipherment" - - "cert sign" - commonName: ca-crname-crnamespace-peer - subject: - organizations: - - ACME Inc. - organizationalUnits: - - Widgets - secretName: ca-secret-crname-crnamespace-peer - privateKey: - algorithm: RSA - size: 4096 - issuerRef: - name: selfsigned-issuer-crname-crnamespace - kind: Issuer - group: cert-manager.io diff --git a/examples/certificates/ca-certificate-server.yaml b/examples/certificates/ca-certificate-server.yaml deleted file mode 100644 index 8958bb98..00000000 --- a/examples/certificates/ca-certificate-server.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: ca-certificate-crname-crnamespace-server - namespace: etcd-operator-system -spec: - isCA: true - usages: - - "signing" - - "key encipherment" - - "cert sign" - commonName: ca-crname-crnamespace-server - subject: - organizations: - - ACME Inc. - organizationalUnits: - - Widgets - secretName: ca-secret-crname-crnamespace-server - privateKey: - algorithm: RSA - size: 4096 - issuerRef: - name: selfsigned-issuer-crname-crnamespace - kind: Issuer - group: cert-manager.io diff --git a/examples/certificates/ca-issuer-peer.yaml b/examples/certificates/ca-issuer-peer.yaml deleted file mode 100644 index d6732c1e..00000000 --- a/examples/certificates/ca-issuer-peer.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: ca-issuer-crname-crnamespace-peer - namespace: etcd-operator-system -spec: - ca: - secretName: ca-secret-crname-crnamespace-peer diff --git a/examples/certificates/ca-issuer-server.yaml b/examples/certificates/ca-issuer-server.yaml deleted file mode 100644 index 7f40d1f7..00000000 --- a/examples/certificates/ca-issuer-server.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: ca-issuer-crname-crnamespace-server - namespace: etcd-operator-system -spec: - ca: - secretName: ca-secret-crname-crnamespace-server diff --git a/examples/certificates/client-server-certificate-peer.yaml b/examples/certificates/client-server-certificate-peer.yaml deleted file mode 100644 index 9a5e2b09..00000000 --- a/examples/certificates/client-server-certificate-peer.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: certificate-crname-crnamespace-peer - namespace: etcd-operator-system -spec: - secretName: crname-crnamespace-peer-tls - isCA: false - usages: - - server auth - - client auth - - signing - - key encipherment - dnsNames: - - etcdcluster-sample-0 - - etcdcluster-sample-0.etcdcluster-sample - - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc - - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc.cluster.local - - etcdcluster-sample-1 - - etcdcluster-sample-1.etcdcluster-sample - - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc - - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc.cluster.local - - etcdcluster-sample-2 - - etcdcluster-sample-2.etcdcluster-sample - - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc - - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc.cluster.local - - localhost - - "127.0.0.1" - issuerRef: - name: ca-issuer-crname-crnamespace-peer diff --git a/examples/certificates/root-client-certificate-server.yaml b/examples/certificates/root-client-certificate-server.yaml deleted file mode 100644 index 740c359b..00000000 --- a/examples/certificates/root-client-certificate-server.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: client-certificate-crname-crnamespace-peer -spec: - commonName: root - secretName: crname-crnamespace-client-peer-tls - usages: - - "signing" - - "key encipherment" - - "client auth" - issuerRef: - name: ca-issuer-crname-crnamespace-server - kind: Issuer diff --git a/examples/certificates/self-signed-issuer.yaml b/examples/certificates/self-signed-issuer.yaml deleted file mode 100644 index 028f4905..00000000 --- a/examples/certificates/self-signed-issuer.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer-crname-crnamespace - namespace: etcd-operator-system -spec: - selfSigned: {} diff --git a/examples/certificates/server-certificate-server.yaml b/examples/certificates/server-certificate-server.yaml deleted file mode 100644 index 4ca9deb8..00000000 --- a/examples/certificates/server-certificate-server.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: certificate-crname-crnamespace-server - namespace: etcd-operator-system -spec: - secretName: crname-crnamespace-server-tls - isCA: false - usages: - - server auth - - signing - - key encipherment - dnsNames: - - etcdcluster-sample-0 - - etcdcluster-sample-0.etcdcluster-sample - - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc - - etcdcluster-sample-0.etcdcluster-sample.etcd-operator-system.svc.cluster.local - - etcdcluster-sample-1 - - etcdcluster-sample-1.etcdcluster-sample - - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc - - etcdcluster-sample-1.etcdcluster-sample.etcd-operator-system.svc.cluster.local - - etcdcluster-sample-2 - - etcdcluster-sample-2.etcdcluster-sample - - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc - - etcdcluster-sample-2.etcdcluster-sample.etcd-operator-system.svc.cluster.local - - etcdcluster-sample-client - - etcdcluster-sample-client.etcd-operator-system.svc - - etcdcluster-sample-client.etcd-operator-system.svc.cluster.local - - localhost - - "127.0.0.1" - issuerRef: - name: ca-issuer-crname-crnamespace-server diff --git a/examples/manifests/etcdcluster-with-external-certificates.yaml b/examples/manifests/etcdcluster-with-external-certificates.yaml new file mode 100644 index 00000000..f092710e --- /dev/null +++ b/examples/manifests/etcdcluster-with-external-certificates.yaml @@ -0,0 +1,176 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test + namespace: default +spec: + security: + peer: + ca: + secretName: ca-peer-secret + cert: + secretName: peer-secret + clientServer: + ca: + secretName: ca-server-secret + cert: + secretName: server-secret +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: default +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-peer + namespace: default +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-peer + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-peer-secret + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-server + namespace: default +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-server + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-server-secret + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-peer + namespace: default +spec: + ca: + secretName: ca-peer-secret +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-server + namespace: default +spec: + ca: + secretName: ca-server-secret +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: certificate-server + namespace: default +spec: + secretName: server-secret + isCA: false + usages: + - server auth + - signing + - key encipherment + dnsNames: + - test-0 + - test-0.test + - test-0.test.default.svc + - test-0.test.default.svc.cluster.local + - test-1 + - test-1.test + - test-1.test.default.svc + - test-1.test.default.svc.cluster.local + - test-2 + - test-2.test + - test-2.test.default.svc + - test-2.test.default.svc.cluster.local + - test-client + - test-client.default.svc + - test-client.default.svc.cluster.local + - localhost + - "127.0.0.1" + issuerRef: + name: ca-issuer-server +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: certificate-peer + namespace: default +spec: + secretName: peer-secret + isCA: false + usages: + - server auth + - client auth + - signing + - key encipherment + dnsNames: + - test-0 + - test-0.test + - test-0.test.default.svc + - test-0.test.default.svc.cluster.local + - test-1 + - test-1.test + - test-1.test.default.svc + - test-1.test.default.svc.cluster.local + - test-2 + - test-2.test + - test-2.test.default.svc + - test-2.test.default.svc.cluster.local + - localhost + - "127.0.0.1" + issuerRef: + name: ca-issuer-peer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: client-certificate-server + namespace: default +spec: + commonName: root + secretName: client-secret + usages: + - "signing" + - "key encipherment" + - "client auth" + issuerRef: + name: ca-issuer-server + kind: Issuer From 4684515b43b1025b000d943fc27665e740adc2a1 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Mon, 8 Apr 2024 22:44:00 +0000 Subject: [PATCH 07/13] add private key size 4096 to certs --- .../etcdcluster-with-external-certificates.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/manifests/etcdcluster-with-external-certificates.yaml b/examples/manifests/etcdcluster-with-external-certificates.yaml index f092710e..068aa9f6 100644 --- a/examples/manifests/etcdcluster-with-external-certificates.yaml +++ b/examples/manifests/etcdcluster-with-external-certificates.yaml @@ -125,6 +125,10 @@ spec: - test-client.default.svc.cluster.local - localhost - "127.0.0.1" + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 4096 issuerRef: name: ca-issuer-server --- @@ -156,6 +160,10 @@ spec: - test-2.test.default.svc.cluster.local - localhost - "127.0.0.1" + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 4096 issuerRef: name: ca-issuer-peer --- @@ -171,6 +179,10 @@ spec: - "signing" - "key encipherment" - "client auth" + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 4096 issuerRef: name: ca-issuer-server kind: Issuer From a19e0d4111d396e9e699fcf831161f29263e74ca Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Mon, 8 Apr 2024 22:47:57 +0000 Subject: [PATCH 08/13] rename cert to serverCert --- api/v1alpha1/etcdcluster_types.go | 4 +--- api/v1alpha1/etcdcluster_webhook.go | 6 +++--- api/v1alpha1/etcdcluster_webhook_test.go | 6 +++--- api/v1alpha1/zz_generated.deepcopy.go | 3 +-- config/crd/bases/etcd.aenix.io_etcdclusters.yaml | 9 ++------- .../etcdcluster-with-external-certificates.yaml | 2 +- internal/controller/factory/statefulset.go | 2 +- internal/controller/factory/statefulset_test.go | 4 ++-- 8 files changed, 14 insertions(+), 22 deletions(-) diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index 27202b89..0878d28b 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -210,8 +210,6 @@ type SecuritySpec struct { Peer *PeerSpec `json:"peer,omitempty"` // +optional ClientServer *ClientServerSpec `json:"clientServer,omitempty"` - // +optional - Rbac RbacSpec `json:"rbac,omitempty"` } type PeerSpec struct { @@ -225,7 +223,7 @@ type ClientServerSpec struct { // +optional Ca SecretSpec `json:"ca,omitempty"` // +optional - Cert SecretSpec `json:"cert,omitempty"` + ServerCert SecretSpec `json:"serverCert,omitempty"` // +optional RootClientCert SecretSpec `json:"rootClientCert,omitempty"` } diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 48861b43..e682ddc4 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -289,13 +289,13 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { } if security.ClientServer != nil { - if (security.ClientServer.Ca.SecretName != "" && security.ClientServer.Cert.SecretName == "") || - (security.ClientServer.Ca.SecretName == "" && security.ClientServer.Cert.SecretName != "") { + if (security.ClientServer.Ca.SecretName != "" && security.ClientServer.ServerCert.SecretName == "") || + (security.ClientServer.Ca.SecretName == "" && security.ClientServer.ServerCert.SecretName != "") { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "security", "clientServer"), security.ClientServer, - "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty"), + "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty"), ) } } diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index 0603bbfd..4f899234 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -195,7 +195,7 @@ var _ = Describe("EtcdCluster Webhook", func() { expectedFieldErr := field.Invalid( field.NewPath("spec", "security", "clientServer"), localCluster.Spec.Security.ClientServer, - "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty", + "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", ) if Expect(err).To(HaveLen(1)) { Expect(*(err[0])).To(Equal(*expectedFieldErr)) @@ -206,7 +206,7 @@ var _ = Describe("EtcdCluster Webhook", func() { It("Should reject if only one secret in clientServer section is defined", func() { localCluster := etcdCluster.DeepCopy() localCluster.Spec.Security.ClientServer = &ClientServerSpec{ - Cert: SecretSpec{ + ServerCert: SecretSpec{ SecretName: "test-server-cert", }, } @@ -215,7 +215,7 @@ var _ = Describe("EtcdCluster Webhook", func() { expectedFieldErr := field.Invalid( field.NewPath("spec", "security", "clientServer"), localCluster.Spec.Security.ClientServer, - "both clientServer.ca.secretName and clientServer.cert.secretName must be filled or empty", + "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", ) if Expect(err).To(HaveLen(1)) { Expect(*(err[0])).To(Equal(*expectedFieldErr)) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a9cc47b6..130a09e9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -31,7 +31,7 @@ import ( func (in *ClientServerSpec) DeepCopyInto(out *ClientServerSpec) { *out = *in out.Ca = in.Ca - out.Cert = in.Cert + out.ServerCert = in.ServerCert out.RootClientCert = in.RootClientCert } @@ -412,7 +412,6 @@ func (in *SecuritySpec) DeepCopyInto(out *SecuritySpec) { *out = new(ClientServerSpec) **out = **in } - out.Rbac = in.Rbac } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecuritySpec. diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index 79f459a2..34436298 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -4582,12 +4582,12 @@ spec: secretName: type: string type: object - cert: + rootClientCert: properties: secretName: type: string type: object - rootClientCert: + serverCert: properties: secretName: type: string @@ -4606,11 +4606,6 @@ spec: type: string type: object type: object - rbac: - properties: - enabled: - type: boolean - type: object type: object storage: description: |- diff --git a/examples/manifests/etcdcluster-with-external-certificates.yaml b/examples/manifests/etcdcluster-with-external-certificates.yaml index 068aa9f6..6b75b464 100644 --- a/examples/manifests/etcdcluster-with-external-certificates.yaml +++ b/examples/manifests/etcdcluster-with-external-certificates.yaml @@ -14,7 +14,7 @@ spec: clientServer: ca: secretName: ca-server-secret - cert: + serverCert: secretName: server-secret --- apiVersion: cert-manager.io/v1 diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index 52cedcf7..209dca60 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -182,7 +182,7 @@ func generateVolumes(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Volume { Name: "server-cert", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Spec.Security.ClientServer.Cert.SecretName, + SecretName: cluster.Spec.Security.ClientServer.ServerCert.SecretName, }, }, }, diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index af3fc436..c5931b8b 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -123,7 +123,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { Ca: etcdaenixiov1alpha1.SecretSpec{ SecretName: "server-ca-secret", }, - Cert: etcdaenixiov1alpha1.SecretSpec{ + ServerCert: etcdaenixiov1alpha1.SecretSpec{ SecretName: "server-cert-secret", }, }, @@ -704,7 +704,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { Ca: etcdaenixiov1alpha1.SecretSpec{ SecretName: "client-server-ca-secret", }, - Cert: etcdaenixiov1alpha1.SecretSpec{ + ServerCert: etcdaenixiov1alpha1.SecretSpec{ SecretName: "client-server-cert-secret", }, }, From c80175a64365d377acd146144c1a00ee20d1c553 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Tue, 9 Apr 2024 15:27:44 +0000 Subject: [PATCH 09/13] update crd in helm chart --- charts/etcd-operator/crds/etcd-cluster.yaml | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/charts/etcd-operator/crds/etcd-cluster.yaml b/charts/etcd-operator/crds/etcd-cluster.yaml index 86e3ab01..01a1fc46 100644 --- a/charts/etcd-operator/crds/etcd-cluster.yaml +++ b/charts/etcd-operator/crds/etcd-cluster.yaml @@ -4318,6 +4318,41 @@ spec: format: int32 minimum: 0 type: integer + security: + description: Security describes security settings of etcd (authentication, certificates, rbac) + properties: + clientServer: + properties: + ca: + properties: + secretName: + type: string + type: object + rootClientCert: + properties: + secretName: + type: string + type: object + serverCert: + properties: + secretName: + type: string + type: object + type: object + peer: + properties: + ca: + properties: + secretName: + type: string + type: object + cert: + properties: + secretName: + type: string + type: object + type: object + type: object storage: description: |- StorageSpec defines the configured storage for a etcd members. From b823d9acd72945dd728599490d9154a3ce3f2737 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Tue, 9 Apr 2024 22:46:06 +0000 Subject: [PATCH 10/13] change mounting spec completely --- AUTH-DESIGN-2.md | 315 ++++++++++++++++++ api/v1alpha1/etcdcluster_types.go | 37 +- api/v1alpha1/etcdcluster_webhook.go | 32 +- api/v1alpha1/etcdcluster_webhook_test.go | 96 ++++-- api/v1alpha1/zz_generated.deepcopy.go | 93 +----- .../crd/bases/etcd.aenix.io_etcdclusters.yaml | 56 ++-- ...tcdcluster-with-external-certificates.yaml | 60 +++- internal/controller/factory/statefulset.go | 72 ++-- .../controller/factory/statefulset_test.go | 67 ++-- 9 files changed, 566 insertions(+), 262 deletions(-) create mode 100644 AUTH-DESIGN-2.md diff --git a/AUTH-DESIGN-2.md b/AUTH-DESIGN-2.md new file mode 100644 index 00000000..de86948b --- /dev/null +++ b/AUTH-DESIGN-2.md @@ -0,0 +1,315 @@ +### Peer.ca +| Option | Description | +| ------ | ----------- | +| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below | +| metadata | Metadata of generated secret. | +| duration | Expiration time of generated secret. | +| renewBefore | Time period before expiration time when certificate will be reissued. | +| privateKey | Private key configuration: algorithm and key size. | + +### Peer.cert +| Option | Description | +| ------ | ----------- | +| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below. If peer.ca.secretName is provided, then this certificate is generated from the CA that was provided by the user. You can't define the secret name in this section and do not define peer.ca.secretName. | +| metadata | Metadata of generated secret. | +| duration | Expiration time of generated secret. | +| renewBefore | Time period before expiration time when certificate will be reissued. | +| privateKey | Private key configuration: algorithm, key size and boolean parameter is it necessary to rotate private key when certificate is expired | + +### ClientServer section has the same fields as peer section. + +### Rbac +| Option | Description | +| ------ | ----------- | +| enabled | Enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. | + +```yaml +spec: + security: + peer: + enabled: true # optional + ca: + # if not defined, then operator generates CA by the spec below + secretName: ext-peer-ca-tls-secret + secretTemplate: + metadata: + name: peer-ca-tls-secret # optional + annotations: {} # optional + labels: {} # optional + duration: 86400h # optional + renewBefore: 720h # optional + privateKey: + algorithm: RSA # optional + size: 4096 # optional + cert: + secretName: ext-peer-tls-secret + secretTemplate: + metadata: + name: peer-tls-secret # optional + annotations: {} # optional + labels: {} # optional + duration: 720h + renewBefore: 180h + privateKey: + rotate: true # optional + algorithm: RSA + size: 4096 + clientServer: + enabled: true + ca: + secretName: ext-server-ca-tls-secret + secretTemplate: + metadata: + name: server-ca-tls-secret + annotations: {} # optional + labels: {} # optional + duration: 86400h + renewBefore: 720h + privateKey: + algorithm: RSA + size: 4096 + cert: + secretName: ext-server-tls-secret + secretTemplate: + metadata: + name: server-tls-secret + annotations: {} # optional + labels: {} # optional + extraSans: [] + duration: 720h + renewBefore: 180h + privateKey: + rotate: true + algorithm: RSA + size: 4096 + rootClientCert: + secretName: ext-client-tls-secret + secretTemplate: + metadata: + name: client-tls-secret + annotations: {} # optional + labels: {} # optional + duration: 720h + renewBefore: 180h + privateKey: + rotate: true + algorithm: RSA + size: 4096 + rbac: + enabled: true # optional +``` + +Important points: +* If field has a value and it is optional, then this value is a default. +* peer: + * If ca.secretName is not defined, operator generates its own CA. + * If ca.secretName is defined, then every field under secretName should not be defined. + * If cert.secretName id not defined, then certificate is generate by operator from the CA defined in the section above (user-managed or operator-managed). + * User must define ca.secretName if cert.secretName is defined. + * Algorithm is a list of the values. NOTE: look into the lib that generates certs what values exist (or to cert-manager). +* clientServer: + * See peer logic. + * RootClientCert uses server ca and has the same logic as server.cert. + * Rbac.enabled enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. + + + +security: + peerCertificate: {} + peerTrustedCACertficate: {} + clientCertificate: {} + serverCertificate: {} + trustedCACertificate: {} + +```yaml +spec: + security: + disableClientAuth: false + peerCertificate: + secretName: ext-peer-tls-secret + secretTemplate: + metadata: + name: peer-tls-secret + annotations: {} + labels: {} + duration: 720h + renewBefore: 180h + privateKey: + rotate: true # optional + algorithm: RSA + size: 4096 + peerTrustedCaCertficate: + # if not defined, then operator generates CA by the spec below + secretName: ext-peer-ca-tls-secret + secretTemplate: + metadata: + name: peer-ca-tls-secret + annotations: {} # optional + labels: {} # optional + duration: 86400h # optional + renewBefore: 720h # optional + privateKey: + algorithm: RSA # optional + size: 4096 # optional + serverCertificate: + secretName: ext-server-tls-secret + secretTemplate: + metadata: + name: server-tls-secret + annotations: {} + labels: {} + extraClientSans: [] + duration: 720h + renewBefore: 180h + privateKey: + rotate: true + algorithm: RSA + size: 4096 + trustedCaCertificate: + secretName: ext-server-ca-tls-secret + secretTemplate: + metadata: + name: server-ca-tls-secret + annotations: {} + labels: {} + duration: 86400h + renewBefore: 720h + privateKey: + algorithm: RSA + size: 4096 + clientCertificate: + secretName: ext-client-tls-secret + secretTemplate: + metadata: + name: client-tls-secret + annotations: {} + labels: {} + duration: 720h + renewBefore: 180h + privateKey: + rotate: true + algorithm: RSA + size: 4096 +``` + + + + + + +### Peer.ca +| Option | Description | +| ------ | ----------- | +| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below | +| metadata | Metadata of generated secret. | +| duration | Expiration time of generated secret. | +| renewBefore | Time period before expiration time when certificate will be reissued. | +| privateKey | Private key configuration: algorithm and key size. | + +### Peer.cert +| Option | Description | +| ------ | ----------- | +| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below. If peer.ca.secretName is provided, then this certificate is generated from the CA that was provided by the user. You can't define the secret name in this section and do not define peer.ca.secretName. | +| metadata | Metadata of generated secret. | +| duration | Expiration time of generated secret. | +| renewBefore | Time period before expiration time when certificate will be reissued. | +| privateKey | Private key configuration: algorithm, key size and boolean parameter is it necessary to rotate private key when certificate is expired | + +### ClientServer section has the same fields as peer section. + +### Rbac +| Option | Description | +| ------ | ----------- | +| enabled | Enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. | + +```yaml +spec: + security: + peer: + enabled: true # optional + ca: + # if not defined, then operator generates CA by the spec below + secretName: ext-peer-ca-tls-secret # oneof secretName or secretTemplate + secretTemplate: # oneof secretName or secretTemplate + annotations: {} # optional + labels: {} # optional + duration: 86400h # optional + renewBefore: 720h # optional + privateKey: + algorithm: RSA # optional + size: 4096 # optional + cert: + secretName: ext-peer-tls-secret + secretTemplate: + annotations: {} + labels: {} + duration: 720h + renewBefore: 180h + privateKey: + rotate: true # optional + algorithm: RSA + size: 4096 + server: + enabled: true + ca: + secretName: ext-server-ca-tls-secret + secretTemplate: + annotations: {} + labels: {} + duration: 86400h + renewBefore: 720h + privateKey: + algorithm: RSA + size: 4096 + cert: + secretName: ext-server-tls-secret + secretTemplate: + annotations: {} + labels: {} + extraSANs: [] + duration: 720h + renewBefore: 180h + privateKey: + rotate: true + algorithm: RSA + size: 4096 + client: + enabled: true + ca: + secretName: ext-server-ca-tls-secret + secretTemplate: + annotations: {} + labels: {} + duration: 86400h + renewBefore: 720h + privateKey: + algorithm: RSA + size: 4096 + cert: + secretName: ext-client-tls-secret + secretTemplate: + annotations: {} + labels: {} + duration: 720h + renewBefore: 180h + privateKey: + rotate: true + algorithm: RSA + size: 4096 + auth: + enabled: true # optional +``` + +Important points: +* If field has a value and it is optional, then this value is a default. +* peer: + * If ca.secretName is not defined, operator generates its own CA. + * If ca.secretName is defined, then every field under secretName should not be defined. + * If cert.secretName id not defined, then certificate is generate by operator from the CA defined in the section above (user-managed or operator-managed). + * User must define ca.secretName if cert.secretName is defined. + * Algorithm is a list of the values. NOTE: look into the lib that generates certs what values exist (or to cert-manager). +* clientServer: + * See peer logic. + * RootClientCert uses server ca and has the same logic as server.cert. + * Rbac.enabled enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. + diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index 0878d28b..b9a10c79 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -206,36 +206,29 @@ type StorageSpec struct { // SecuritySpec defines security settings for etcd. // +k8s:openapi-gen=true type SecuritySpec struct { + // Section for user-managed tls certificates // +optional - Peer *PeerSpec `json:"peer,omitempty"` - // +optional - ClientServer *ClientServerSpec `json:"clientServer,omitempty"` + TLS TLSSpec `json:"tls,omitempty"` } -type PeerSpec struct { +// TLSSpec defines user-managed certificates names. +type TLSSpec struct { + // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. // +optional - Ca SecretSpec `json:"ca,omitempty"` + PeerTrustedCASecret string `json:"peerTrustedCASecret,omitempty"` + // Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. // +optional - Cert SecretSpec `json:"cert,omitempty"` -} - -type ClientServerSpec struct { + PeerSecret string `json:"peerSecret,omitempty"` + // Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). + // It is expected to have tls.crt and tls.key fields in the secret. // +optional - Ca SecretSpec `json:"ca,omitempty"` + ServerSecret string `json:"serverSecret,omitempty"` + // Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. // +optional - ServerCert SecretSpec `json:"serverCert,omitempty"` - // +optional - RootClientCert SecretSpec `json:"rootClientCert,omitempty"` -} - -type SecretSpec struct { - // +optional - SecretName string `json:"secretName,omitempty"` -} - -type RbacSpec struct { + ClientTrustedCASecret string `json:"clientTrustedCASecret,omitempty"` + // Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. // +optional - Enabled bool `json:"enabled,omitempty"` + ClientSecret string `json:"clientSecret,omitempty"` } // EmbeddedPersistentVolumeClaim is an embedded version of k8s.io/api/core/v1.PersistentVolumeClaim. diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index e682ddc4..69253758 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -276,28 +276,24 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { security := r.Spec.Security - if security.Peer != nil { - if (security.Peer.Ca.SecretName != "" && security.Peer.Cert.SecretName == "") || - (security.Peer.Ca.SecretName == "" && security.Peer.Cert.SecretName != "") { + if (security.TLS.PeerSecret != "" && security.TLS.PeerTrustedCASecret == "") || + (security.TLS.PeerSecret == "" && security.TLS.PeerTrustedCASecret != "") { - allErrors = append(allErrors, field.Invalid( - field.NewPath("spec", "security", "peer"), - security.Peer, - "both peer.ca.secretName and peer.cert.secretName must be filled or empty"), - ) - } + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security", "tls"), + security.TLS, + "both spec.security.tls.peerSecret and spec.security.tls.peerTrustedCASecret must be filled or empty"), + ) } - if security.ClientServer != nil { - if (security.ClientServer.Ca.SecretName != "" && security.ClientServer.ServerCert.SecretName == "") || - (security.ClientServer.Ca.SecretName == "" && security.ClientServer.ServerCert.SecretName != "") { + if (security.TLS.ClientSecret != "" && security.TLS.ClientTrustedCASecret == "") || + (security.TLS.ClientSecret == "" && security.TLS.ClientTrustedCASecret != "") { - allErrors = append(allErrors, field.Invalid( - field.NewPath("spec", "security", "clientServer"), - security.ClientServer, - "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty"), - ) - } + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security", "tls"), + security.TLS, + "both spec.security.tls.clientSecret and spec.security.tls.clientTrustedCASecret must be filled or empty"), + ) } if len(allErrors) > 0 { diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index 4f899234..f30a45e8 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -143,19 +143,17 @@ var _ = Describe("EtcdCluster Webhook", func() { Expect(err).To(BeNil()) }) - It("Should reject if only one secret in peer section is defined", func() { + It("Should reject if only one peer secret is defined", func() { localCluster := etcdCluster.DeepCopy() - localCluster.Spec.Security.Peer = &PeerSpec{ - Ca: SecretSpec{ - SecretName: "test-peer-ca-cert", - }, + localCluster.Spec.Security.TLS = TLSSpec{ + PeerTrustedCASecret: "test-peer-ca-cert", } err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "peer"), - localCluster.Spec.Security.Peer, - "both peer.ca.secretName and peer.cert.secretName must be filled or empty", + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.peerSecret and spec.security.tls.peerTrustedCASecret must be filled or empty", ) if Expect(err).To(HaveLen(1)) { Expect(*(err[0])).To(Equal(*expectedFieldErr)) @@ -163,19 +161,17 @@ var _ = Describe("EtcdCluster Webhook", func() { } }) - It("Should reject if only one secret in peer section is defined", func() { + It("Should reject if only one peer secret is defined", func() { localCluster := etcdCluster.DeepCopy() - localCluster.Spec.Security.Peer = &PeerSpec{ - Cert: SecretSpec{ - SecretName: "test-peer-cert", - }, + localCluster.Spec.Security.TLS = TLSSpec{ + PeerSecret: "test-peer-cert", } err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "peer"), - localCluster.Spec.Security.Peer, - "both peer.ca.secretName and peer.cert.secretName must be filled or empty", + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.peerSecret and spec.security.tls.peerTrustedCASecret must be filled or empty", ) if Expect(err).To(HaveLen(1)) { Expect(*(err[0])).To(Equal(*expectedFieldErr)) @@ -183,19 +179,17 @@ var _ = Describe("EtcdCluster Webhook", func() { } }) - It("Should reject if only one secret in clientServer section is defined", func() { + It("Should reject if only one client secret is defined", func() { localCluster := etcdCluster.DeepCopy() - localCluster.Spec.Security.ClientServer = &ClientServerSpec{ - Ca: SecretSpec{ - SecretName: "test-ca-server-cert", - }, + localCluster.Spec.Security.TLS = TLSSpec{ + ClientTrustedCASecret: "test-client-ca-cert", } err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "clientServer"), - localCluster.Spec.Security.ClientServer, - "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.clientSecret and spec.security.tls.clientTrustedCASecret must be filled or empty", ) if Expect(err).To(HaveLen(1)) { Expect(*(err[0])).To(Equal(*expectedFieldErr)) @@ -203,19 +197,17 @@ var _ = Describe("EtcdCluster Webhook", func() { } }) - It("Should reject if only one secret in clientServer section is defined", func() { + It("Should reject if only one client secret is defined", func() { localCluster := etcdCluster.DeepCopy() - localCluster.Spec.Security.ClientServer = &ClientServerSpec{ - ServerCert: SecretSpec{ - SecretName: "test-server-cert", - }, + localCluster.Spec.Security.TLS = TLSSpec{ + ClientTrustedCASecret: "test-client-cert", } err := localCluster.validateSecurity() if Expect(err).NotTo(BeNil()) { expectedFieldErr := field.Invalid( - field.NewPath("spec", "security", "clientServer"), - localCluster.Spec.Security.ClientServer, - "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.clientSecret and spec.security.tls.clientTrustedCASecret must be filled or empty", ) if Expect(err).To(HaveLen(1)) { Expect(*(err[0])).To(Equal(*expectedFieldErr)) @@ -223,6 +215,46 @@ var _ = Describe("EtcdCluster Webhook", func() { } }) + // It("Should reject if only one secret in clientServer section is defined", func() { + // localCluster := etcdCluster.DeepCopy() + // localCluster.Spec.Security.ClientServer = &ClientServerSpec{ + // Ca: SecretSpec{ + // SecretName: "test-ca-server-cert", + // }, + // } + // err := localCluster.validateSecurity() + // if Expect(err).NotTo(BeNil()) { + // expectedFieldErr := field.Invalid( + // field.NewPath("spec", "security", "clientServer"), + // localCluster.Spec.Security.ClientServer, + // "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", + // ) + // if Expect(err).To(HaveLen(1)) { + // Expect(*(err[0])).To(Equal(*expectedFieldErr)) + // } + // } + // }) + + // It("Should reject if only one secret in clientServer section is defined", func() { + // localCluster := etcdCluster.DeepCopy() + // localCluster.Spec.Security.ClientServer = &ClientServerSpec{ + // ServerCert: SecretSpec{ + // SecretName: "test-server-cert", + // }, + // } + // err := localCluster.validateSecurity() + // if Expect(err).NotTo(BeNil()) { + // expectedFieldErr := field.Invalid( + // field.NewPath("spec", "security", "clientServer"), + // localCluster.Spec.Security.ClientServer, + // "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", + // ) + // if Expect(err).To(HaveLen(1)) { + // Expect(*(err[0])).To(Equal(*expectedFieldErr)) + // } + // } + // }) + }) Context("Validate PDB", func() { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 130a09e9..db42b721 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -27,24 +27,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClientServerSpec) DeepCopyInto(out *ClientServerSpec) { - *out = *in - out.Ca = in.Ca - out.ServerCert = in.ServerCert - out.RootClientCert = in.RootClientCert -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientServerSpec. -func (in *ClientServerSpec) DeepCopy() *ClientServerSpec { - if in == nil { - return nil - } - out := new(ClientServerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EmbeddedObjectMetadata) DeepCopyInto(out *EmbeddedObjectMetadata) { *out = *in @@ -194,7 +176,7 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) { if in.Security != nil { in, out := &in.Security, &out.Security *out = new(SecuritySpec) - (*in).DeepCopyInto(*out) + **out = **in } } @@ -230,23 +212,6 @@ func (in *EtcdClusterStatus) DeepCopy() *EtcdClusterStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PeerSpec) DeepCopyInto(out *PeerSpec) { - *out = *in - out.Ca = in.Ca - out.Cert = in.Cert -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerSpec. -func (in *PeerSpec) DeepCopy() *PeerSpec { - if in == nil { - return nil - } - out := new(PeerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodDisruptionBudgetSpec) DeepCopyInto(out *PodDisruptionBudgetSpec) { *out = *in @@ -369,49 +334,10 @@ func (in *PodTemplate) DeepCopy() *PodTemplate { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RbacSpec) DeepCopyInto(out *RbacSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RbacSpec. -func (in *RbacSpec) DeepCopy() *RbacSpec { - if in == nil { - return nil - } - out := new(RbacSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretSpec) DeepCopyInto(out *SecretSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSpec. -func (in *SecretSpec) DeepCopy() *SecretSpec { - if in == nil { - return nil - } - out := new(SecretSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecuritySpec) DeepCopyInto(out *SecuritySpec) { *out = *in - if in.Peer != nil { - in, out := &in.Peer, &out.Peer - *out = new(PeerSpec) - **out = **in - } - if in.ClientServer != nil { - in, out := &in.ClientServer, &out.ClientServer - *out = new(ClientServerSpec) - **out = **in - } + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecuritySpec. @@ -444,3 +370,18 @@ func (in *StorageSpec) DeepCopy() *StorageSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSSpec. +func (in *TLSSpec) DeepCopy() *TLSSpec { + if in == nil { + return nil + } + out := new(TLSSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index 34436298..cbfc3188 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -4575,36 +4575,34 @@ spec: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: - clientServer: + tls: + description: Section for user-managed tls certificates properties: - ca: - properties: - secretName: - type: string - type: object - rootClientCert: - properties: - secretName: - type: string - type: object - serverCert: - properties: - secretName: - type: string - type: object - type: object - peer: - properties: - ca: - properties: - secretName: - type: string - type: object - cert: - properties: - secretName: - type: string - type: object + clientSecret: + description: Client certificate for etcd-operator to do maintenance. + It is expected to have tls.crt and tls.key fields in the + secret. + type: string + clientTrustedCASecret: + description: Trusted CA for client certificates that are provided + by client to etcd. It is expected to have tls.crt field + in the secret. + type: string + peerSecret: + description: Certificate secret to secure peer-to-peer communication + between etcd nodes. It is expected to have tls.crt and tls.key + fields in the secret. + type: string + peerTrustedCASecret: + description: Trusted CA certificate secret to secure peer-to-peer + communication between etcd nodes. It is expected to have + tls.crt field in the secret. + type: string + serverSecret: + description: |- + Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). + It is expected to have tls.crt and tls.key fields in the secret. + type: string type: object type: object storage: diff --git a/examples/manifests/etcdcluster-with-external-certificates.yaml b/examples/manifests/etcdcluster-with-external-certificates.yaml index 6b75b464..59c3b43e 100644 --- a/examples/manifests/etcdcluster-with-external-certificates.yaml +++ b/examples/manifests/etcdcluster-with-external-certificates.yaml @@ -5,17 +5,14 @@ metadata: name: test namespace: default spec: + storage: {} security: - peer: - ca: - secretName: ca-peer-secret - cert: - secretName: peer-secret - clientServer: - ca: - secretName: ca-server-secret - serverCert: - secretName: server-secret + tls: + peerTrustedCASecret: ca-peer-secret + peerSecret: peer-secret + serverSecret: server-secret + clientTrustedCASecret: ca-client-secret + clientSecret: client-secret --- apiVersion: cert-manager.io/v1 kind: Issuer @@ -78,6 +75,32 @@ spec: group: cert-manager.io --- apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-client + namespace: default +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-certificate-client + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-client-secret + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: ca-issuer-peer @@ -96,9 +119,18 @@ spec: secretName: ca-server-secret --- apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-client + namespace: default +spec: + ca: + secretName: ca-client-secret +--- +apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: certificate-server + name: server-certificate namespace: default spec: secretName: server-secret @@ -135,7 +167,7 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: certificate-peer + name: peer-certificate namespace: default spec: secretName: peer-secret @@ -170,7 +202,7 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: client-certificate-server + name: client-certificate namespace: default spec: commonName: root @@ -184,5 +216,5 @@ spec: algorithm: RSA size: 4096 issuerRef: - name: ca-issuer-server + name: ca-issuer-client kind: Issuer diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index 209dca60..f9447901 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -144,45 +144,50 @@ func generateVolumes(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Volume { VolumeSource: dataVolumeSource, } - if cluster.Spec.Security != nil && cluster.Spec.Security.Peer != nil { + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.PeerSecret != "" { volumes = append(volumes, []corev1.Volume{ { - Name: "ca-peer-cert", + Name: "peer-trusted-ca-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Spec.Security.Peer.Ca.SecretName, + SecretName: cluster.Spec.Security.TLS.PeerTrustedCASecret, }, }, }, { - Name: "peer-cert", + Name: "peer-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Spec.Security.Peer.Cert.SecretName, + SecretName: cluster.Spec.Security.TLS.PeerSecret, }, }, }, }...) - } - if cluster.Spec.Security != nil && cluster.Spec.Security.ClientServer != nil { + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { volumes = append(volumes, []corev1.Volume{ { - Name: "ca-server-cert", + Name: "server-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Spec.Security.ClientServer.Ca.SecretName, + SecretName: cluster.Spec.Security.TLS.ServerSecret, }, }, }, + }...) + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + volumes = append(volumes, + []corev1.Volume{ { - Name: "server-cert", + Name: "client-trusted-ca-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Spec.Security.ClientServer.ServerCert.SecretName, + SecretName: cluster.Spec.Security.TLS.ClientTrustedCASecret, }, }, }, @@ -219,34 +224,38 @@ func generateVolumeMounts(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Vol } - if cluster.Spec.Security != nil && cluster.Spec.Security.Peer != nil { - + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.PeerSecret != "" { volumeMounts = append(volumeMounts, []corev1.VolumeMount{ { - Name: "ca-peer-cert", + Name: "peer-trusted-ca-certificate", ReadOnly: true, MountPath: "/etc/etcd/pki/peer/ca", }, { - Name: "peer-cert", + Name: "peer-certificate", ReadOnly: true, MountPath: "/etc/etcd/pki/peer/cert", }, }...) } - if cluster.Spec.Security != nil && cluster.Spec.Security.ClientServer != nil { - + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { volumeMounts = append(volumeMounts, []corev1.VolumeMount{ { - Name: "ca-server-cert", + Name: "server-certificate", ReadOnly: true, - MountPath: "/etc/etcd/pki/server/ca", + MountPath: "/etc/etcd/pki/server/cert", }, + }...) + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ { - Name: "server-cert", + Name: "client-trusted-ca-certificate", ReadOnly: true, - MountPath: "/etc/etcd/pki/server/cert", + MountPath: "/etc/etcd/pki/client/ca", }, }...) } @@ -276,7 +285,7 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { peerTlsSettings := []string{"--peer-auto-tls"} - if cluster.Spec.Security != nil && cluster.Spec.Security.Peer != nil { + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.PeerSecret != "" { peerTlsSettings = []string{ "--peer-trusted-ca-file=/etc/etcd/pki/peer/ca/ca.crt", "--peer-cert-file=/etc/etcd/pki/peer/cert/tls.crt", @@ -285,20 +294,26 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { } } + serverTlsSettings := []string{} serverProtocol := "http" - clientTlsSettings := []string{"--auto-tls"} - - if cluster.Spec.Security != nil && cluster.Spec.Security.ClientServer != nil { - clientTlsSettings = []string{ - "--trusted-ca-file=/etc/etcd/pki/server/ca/ca.crt", + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { + serverTlsSettings = []string{ "--cert-file=/etc/etcd/pki/server/cert/tls.crt", "--key-file=/etc/etcd/pki/server/cert/tls.key", - "--client-cert-auth=false", } serverProtocol = "https" } + clientTlsSettings := []string{} + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + clientTlsSettings = []string{ + "--trusted-ca-file=/etc/etcd/pki/client/ca/ca.crt", + "--client-cert-auth", + } + } + args = append(args, []string{ "--name=$(POD_NAME)", "--listen-metrics-urls=http://0.0.0.0:2381", @@ -310,6 +325,7 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { }...) args = append(args, peerTlsSettings...) + args = append(args, serverTlsSettings...) args = append(args, clientTlsSettings...) return args diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index c5931b8b..b57bdc5f 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -119,21 +119,12 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }, } etcdcluster.Spec.Security = &etcdaenixiov1alpha1.SecuritySpec{ - ClientServer: &etcdaenixiov1alpha1.ClientServerSpec{ - Ca: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "server-ca-secret", - }, - ServerCert: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "server-cert-secret", - }, - }, - Peer: &etcdaenixiov1alpha1.PeerSpec{ - Ca: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "peer-ca-secret", - }, - Cert: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "peer-cert-secret", - }, + TLS: etcdaenixiov1alpha1.TLSSpec{ + PeerTrustedCASecret: "peer-ca-secret", + PeerSecret: "peer-cert-secret", + ServerSecret: "server-cert-secret", + ClientTrustedCASecret: "client-ca-secret", + ClientSecret: "client-secret", }, } @@ -221,7 +212,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { By("Checking generated security volumes", func() { Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ - Name: "ca-peer-cert", + Name: "peer-trusted-ca-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "peer-ca-secret", @@ -230,7 +221,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }, })) Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ - Name: "peer-cert", + Name: "peer-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "peer-cert-secret", @@ -239,19 +230,19 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }, })) Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ - Name: "ca-server-cert", + Name: "server-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "server-ca-secret", + SecretName: "server-cert-secret", DefaultMode: ptr.To(int32(420)), }, }, })) Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ - Name: "server-cert", + Name: "client-trusted-ca-certificate", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "server-cert-secret", + SecretName: "client-ca-secret", DefaultMode: ptr.To(int32(420)), }, }, @@ -700,47 +691,37 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { It("should generate security volumes mounts", func() { localCluster := etcdCluster.DeepCopy() localCluster.Spec.Security = &etcdaenixiov1alpha1.SecuritySpec{ - ClientServer: &etcdaenixiov1alpha1.ClientServerSpec{ - Ca: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "client-server-ca-secret", - }, - ServerCert: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "client-server-cert-secret", - }, - }, - Peer: &etcdaenixiov1alpha1.PeerSpec{ - Ca: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "peer-ca-secret", - }, - Cert: etcdaenixiov1alpha1.SecretSpec{ - SecretName: "peer-cert-secret", - }, + TLS: etcdaenixiov1alpha1.TLSSpec{ + PeerTrustedCASecret: "peer-ca-secret", + PeerSecret: "peer-cert-secret", + ServerSecret: "server-cert-secret", + ClientTrustedCASecret: "client-ca-secret", + ClientSecret: "client-secret", }, } containers := generateContainers(localCluster) Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ - Name: "ca-peer-cert", + Name: "peer-trusted-ca-certificate", MountPath: "/etc/etcd/pki/peer/ca", ReadOnly: true, })) Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ - Name: "peer-cert", + Name: "peer-certificate", MountPath: "/etc/etcd/pki/peer/cert", ReadOnly: true, })) Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ - Name: "ca-server-cert", - MountPath: "/etc/etcd/pki/server/ca", + Name: "server-certificate", + MountPath: "/etc/etcd/pki/server/cert", ReadOnly: true, })) Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ - Name: "server-cert", - MountPath: "/etc/etcd/pki/server/cert", + Name: "client-trusted-ca-certificate", + MountPath: "/etc/etcd/pki/client/ca", ReadOnly: true, })) - }) }) From a6ecf51518f904af68016fd4561d2eaf95184e99 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Wed, 10 Apr 2024 21:24:00 +0000 Subject: [PATCH 11/13] remove commented lines --- api/v1alpha1/etcdcluster_webhook_test.go | 41 ------------------------ 1 file changed, 41 deletions(-) diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index f30a45e8..e9998d1c 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -214,47 +214,6 @@ var _ = Describe("EtcdCluster Webhook", func() { } } }) - - // It("Should reject if only one secret in clientServer section is defined", func() { - // localCluster := etcdCluster.DeepCopy() - // localCluster.Spec.Security.ClientServer = &ClientServerSpec{ - // Ca: SecretSpec{ - // SecretName: "test-ca-server-cert", - // }, - // } - // err := localCluster.validateSecurity() - // if Expect(err).NotTo(BeNil()) { - // expectedFieldErr := field.Invalid( - // field.NewPath("spec", "security", "clientServer"), - // localCluster.Spec.Security.ClientServer, - // "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", - // ) - // if Expect(err).To(HaveLen(1)) { - // Expect(*(err[0])).To(Equal(*expectedFieldErr)) - // } - // } - // }) - - // It("Should reject if only one secret in clientServer section is defined", func() { - // localCluster := etcdCluster.DeepCopy() - // localCluster.Spec.Security.ClientServer = &ClientServerSpec{ - // ServerCert: SecretSpec{ - // SecretName: "test-server-cert", - // }, - // } - // err := localCluster.validateSecurity() - // if Expect(err).NotTo(BeNil()) { - // expectedFieldErr := field.Invalid( - // field.NewPath("spec", "security", "clientServer"), - // localCluster.Spec.Security.ClientServer, - // "both clientServer.ca.secretName and ClientServer.ServerCert.secretName must be filled or empty", - // ) - // if Expect(err).To(HaveLen(1)) { - // Expect(*(err[0])).To(Equal(*expectedFieldErr)) - // } - // } - // }) - }) Context("Validate PDB", func() { From 864fe85978490a1f91319634c9bd4338cbaca914 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Wed, 10 Apr 2024 21:24:45 +0000 Subject: [PATCH 12/13] remove file --- AUTH-DESIGN-2.md | 315 ----------------------------------------------- 1 file changed, 315 deletions(-) delete mode 100644 AUTH-DESIGN-2.md diff --git a/AUTH-DESIGN-2.md b/AUTH-DESIGN-2.md deleted file mode 100644 index de86948b..00000000 --- a/AUTH-DESIGN-2.md +++ /dev/null @@ -1,315 +0,0 @@ -### Peer.ca -| Option | Description | -| ------ | ----------- | -| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below | -| metadata | Metadata of generated secret. | -| duration | Expiration time of generated secret. | -| renewBefore | Time period before expiration time when certificate will be reissued. | -| privateKey | Private key configuration: algorithm and key size. | - -### Peer.cert -| Option | Description | -| ------ | ----------- | -| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below. If peer.ca.secretName is provided, then this certificate is generated from the CA that was provided by the user. You can't define the secret name in this section and do not define peer.ca.secretName. | -| metadata | Metadata of generated secret. | -| duration | Expiration time of generated secret. | -| renewBefore | Time period before expiration time when certificate will be reissued. | -| privateKey | Private key configuration: algorithm, key size and boolean parameter is it necessary to rotate private key when certificate is expired | - -### ClientServer section has the same fields as peer section. - -### Rbac -| Option | Description | -| ------ | ----------- | -| enabled | Enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. | - -```yaml -spec: - security: - peer: - enabled: true # optional - ca: - # if not defined, then operator generates CA by the spec below - secretName: ext-peer-ca-tls-secret - secretTemplate: - metadata: - name: peer-ca-tls-secret # optional - annotations: {} # optional - labels: {} # optional - duration: 86400h # optional - renewBefore: 720h # optional - privateKey: - algorithm: RSA # optional - size: 4096 # optional - cert: - secretName: ext-peer-tls-secret - secretTemplate: - metadata: - name: peer-tls-secret # optional - annotations: {} # optional - labels: {} # optional - duration: 720h - renewBefore: 180h - privateKey: - rotate: true # optional - algorithm: RSA - size: 4096 - clientServer: - enabled: true - ca: - secretName: ext-server-ca-tls-secret - secretTemplate: - metadata: - name: server-ca-tls-secret - annotations: {} # optional - labels: {} # optional - duration: 86400h - renewBefore: 720h - privateKey: - algorithm: RSA - size: 4096 - cert: - secretName: ext-server-tls-secret - secretTemplate: - metadata: - name: server-tls-secret - annotations: {} # optional - labels: {} # optional - extraSans: [] - duration: 720h - renewBefore: 180h - privateKey: - rotate: true - algorithm: RSA - size: 4096 - rootClientCert: - secretName: ext-client-tls-secret - secretTemplate: - metadata: - name: client-tls-secret - annotations: {} # optional - labels: {} # optional - duration: 720h - renewBefore: 180h - privateKey: - rotate: true - algorithm: RSA - size: 4096 - rbac: - enabled: true # optional -``` - -Important points: -* If field has a value and it is optional, then this value is a default. -* peer: - * If ca.secretName is not defined, operator generates its own CA. - * If ca.secretName is defined, then every field under secretName should not be defined. - * If cert.secretName id not defined, then certificate is generate by operator from the CA defined in the section above (user-managed or operator-managed). - * User must define ca.secretName if cert.secretName is defined. - * Algorithm is a list of the values. NOTE: look into the lib that generates certs what values exist (or to cert-manager). -* clientServer: - * See peer logic. - * RootClientCert uses server ca and has the same logic as server.cert. - * Rbac.enabled enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. - - - -security: - peerCertificate: {} - peerTrustedCACertficate: {} - clientCertificate: {} - serverCertificate: {} - trustedCACertificate: {} - -```yaml -spec: - security: - disableClientAuth: false - peerCertificate: - secretName: ext-peer-tls-secret - secretTemplate: - metadata: - name: peer-tls-secret - annotations: {} - labels: {} - duration: 720h - renewBefore: 180h - privateKey: - rotate: true # optional - algorithm: RSA - size: 4096 - peerTrustedCaCertficate: - # if not defined, then operator generates CA by the spec below - secretName: ext-peer-ca-tls-secret - secretTemplate: - metadata: - name: peer-ca-tls-secret - annotations: {} # optional - labels: {} # optional - duration: 86400h # optional - renewBefore: 720h # optional - privateKey: - algorithm: RSA # optional - size: 4096 # optional - serverCertificate: - secretName: ext-server-tls-secret - secretTemplate: - metadata: - name: server-tls-secret - annotations: {} - labels: {} - extraClientSans: [] - duration: 720h - renewBefore: 180h - privateKey: - rotate: true - algorithm: RSA - size: 4096 - trustedCaCertificate: - secretName: ext-server-ca-tls-secret - secretTemplate: - metadata: - name: server-ca-tls-secret - annotations: {} - labels: {} - duration: 86400h - renewBefore: 720h - privateKey: - algorithm: RSA - size: 4096 - clientCertificate: - secretName: ext-client-tls-secret - secretTemplate: - metadata: - name: client-tls-secret - annotations: {} - labels: {} - duration: 720h - renewBefore: 180h - privateKey: - rotate: true - algorithm: RSA - size: 4096 -``` - - - - - - -### Peer.ca -| Option | Description | -| ------ | ----------- | -| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below | -| metadata | Metadata of generated secret. | -| duration | Expiration time of generated secret. | -| renewBefore | Time period before expiration time when certificate will be reissued. | -| privateKey | Private key configuration: algorithm and key size. | - -### Peer.cert -| Option | Description | -| ------ | ----------- | -| secretName | Secret name of user-provided secret. If not specified then operator generates certificate by the spec below. If peer.ca.secretName is provided, then this certificate is generated from the CA that was provided by the user. You can't define the secret name in this section and do not define peer.ca.secretName. | -| metadata | Metadata of generated secret. | -| duration | Expiration time of generated secret. | -| renewBefore | Time period before expiration time when certificate will be reissued. | -| privateKey | Private key configuration: algorithm, key size and boolean parameter is it necessary to rotate private key when certificate is expired | - -### ClientServer section has the same fields as peer section. - -### Rbac -| Option | Description | -| ------ | ----------- | -| enabled | Enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. | - -```yaml -spec: - security: - peer: - enabled: true # optional - ca: - # if not defined, then operator generates CA by the spec below - secretName: ext-peer-ca-tls-secret # oneof secretName or secretTemplate - secretTemplate: # oneof secretName or secretTemplate - annotations: {} # optional - labels: {} # optional - duration: 86400h # optional - renewBefore: 720h # optional - privateKey: - algorithm: RSA # optional - size: 4096 # optional - cert: - secretName: ext-peer-tls-secret - secretTemplate: - annotations: {} - labels: {} - duration: 720h - renewBefore: 180h - privateKey: - rotate: true # optional - algorithm: RSA - size: 4096 - server: - enabled: true - ca: - secretName: ext-server-ca-tls-secret - secretTemplate: - annotations: {} - labels: {} - duration: 86400h - renewBefore: 720h - privateKey: - algorithm: RSA - size: 4096 - cert: - secretName: ext-server-tls-secret - secretTemplate: - annotations: {} - labels: {} - extraSANs: [] - duration: 720h - renewBefore: 180h - privateKey: - rotate: true - algorithm: RSA - size: 4096 - client: - enabled: true - ca: - secretName: ext-server-ca-tls-secret - secretTemplate: - annotations: {} - labels: {} - duration: 86400h - renewBefore: 720h - privateKey: - algorithm: RSA - size: 4096 - cert: - secretName: ext-client-tls-secret - secretTemplate: - annotations: {} - labels: {} - duration: 720h - renewBefore: 180h - privateKey: - rotate: true - algorithm: RSA - size: 4096 - auth: - enabled: true # optional -``` - -Important points: -* If field has a value and it is optional, then this value is a default. -* peer: - * If ca.secretName is not defined, operator generates its own CA. - * If ca.secretName is defined, then every field under secretName should not be defined. - * If cert.secretName id not defined, then certificate is generate by operator from the CA defined in the section above (user-managed or operator-managed). - * User must define ca.secretName if cert.secretName is defined. - * Algorithm is a list of the values. NOTE: look into the lib that generates certs what values exist (or to cert-manager). -* clientServer: - * See peer logic. - * RootClientCert uses server ca and has the same logic as server.cert. - * Rbac.enabled enables role-based access control: creates root user in etcd, gives him root role and enables authentication in etcd. - From 60dcf8888ce5ee364f37be1a9108e0dab67f0388 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Wed, 10 Apr 2024 21:26:47 +0000 Subject: [PATCH 13/13] update helm crd --- charts/etcd-operator/crds/etcd-cluster.yaml | 48 ++++++++------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/charts/etcd-operator/crds/etcd-cluster.yaml b/charts/etcd-operator/crds/etcd-cluster.yaml index 01a1fc46..4b4ba363 100644 --- a/charts/etcd-operator/crds/etcd-cluster.yaml +++ b/charts/etcd-operator/crds/etcd-cluster.yaml @@ -4321,36 +4321,26 @@ spec: security: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: - clientServer: + tls: + description: Section for user-managed tls certificates properties: - ca: - properties: - secretName: - type: string - type: object - rootClientCert: - properties: - secretName: - type: string - type: object - serverCert: - properties: - secretName: - type: string - type: object - type: object - peer: - properties: - ca: - properties: - secretName: - type: string - type: object - cert: - properties: - secretName: - type: string - type: object + clientSecret: + description: Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + type: string + clientTrustedCASecret: + description: Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + type: string + peerSecret: + description: Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + type: string + peerTrustedCASecret: + description: Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + type: string + serverSecret: + description: |- + Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). + It is expected to have tls.crt and tls.key fields in the secret. + type: string type: object type: object storage: