feat: Use ip-address-manager to allocate IPs

smartxworks · Nov 24, 2022 · fcfed7d · fcfed7d
1 parent 718483f
commit fcfed7d
Show file tree

Hide file tree

Showing 19 changed files with 337 additions and 268 deletions.
diff --git a/Makefile b/Makefile
@@ -175,7 +175,8 @@ e2e-cluster-templates-v1alpha1: $(KUSTOMIZE) ## Generate cluster templates for v
 SKIP_RESOURCE_CLEANUP ?= false
 CERT_MANAGER_MANIFEST ?= https://github.com/cert-manager/cert-manager/releases/download/v1.8.2/cert-manager.yaml
 VIRTINK_MANIFEST ?= https://github.com/smartxworks/virtink/releases/download/v0.8.0/virtink.yaml	
-E2E_KIND_CLUSTER_NAME ?= capch-e2e-$(shell date "+%Y-%m-%d-%H-%M-%S")
+IP_ADDRESS_MANAGER_MANIFEST ?= https://github.com/metal3-io/ip-address-manager/releases/download/v1.2.1/ipam-components.yaml
+E2E_KIND_CLUSTER_NAME := capch-e2e-$(shell date "+%Y-%m-%d-%H-%M-%S")
 E2E_KIND_CLUSTER_KUBECONFIG := /tmp/$(E2E_KIND_CLUSTER_NAME).kubeconfig
 
 .PHONY: e2e
@@ -190,6 +191,9 @@ e2e: kind e2e-image kubectl cmctl kustomize ginkgo e2e-cluster-templates-v1alpha
 	KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(CMCTL) check api --wait=10m
 	KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(KUBECTL) apply -f $(VIRTINK_MANIFEST)
 	KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(KUBECTL) wait -n virtink-system deployment virt-controller --for condition=Available --timeout -1s
+	KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(KUBECTL) create namespace capm3-system
+	KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(KUBECTL) apply -f $(IP_ADDRESS_MANAGER_MANIFEST)
+	KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(KUBECTL) wait -n capm3-system deployment ipam-controller-manager --for condition=Available --timeout -1s
 
 	PATH=$(LOCALBIN):$(PATH) KUBECONFIG=$(E2E_KIND_CLUSTER_KUBECONFIG) $(GINKGO) -v -trace -tags=e2e ./test/e2e -- \
 		-e2e.artifacts-folder="$(REPO_ROOT)/_artifacts" \

diff --git a/README.md b/README.md
@@ -57,7 +57,7 @@ By default cluster-api-provider-virtink created Kubernetes Node is a Virtink Vir
 | ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- |
 | VIRTINK_CONTROL_PLANE_MACHINE_ROOTFS_CDI_IMAGE | The rootfs image for CDI of control plane machine (default `smartxworks/capch-rootfs-cdi-1.24.0`)                    |
 | VIRTINK_WORKER_MACHINE_ROOTFS_CDI_IMAGE        | The rootfs image for CDI of worker machine (default `smartxworks/capch-rootfs-cdi-1.24.0`)                           |
-| VIRTINK_NODE_ADDRESSES                         | The IP addresses for nodes, if provided a node will use one of the IP address in this list during whole life cycle, support IP address ("192.168.100.1"), IP range ("192.168.100.1-192.168.100.10") or CIDR ("192.168.100.1/24") form.                                                                                   |
+| VIRTINK_IP_POOL_NAME                           | The IPPool for machines to get IP address from                                                                       |
 
 This is an example to generate workload cluster configuration with persistent storage for an internal Virtink cluster that use Calico as CNI. You should be familiar with Calico [IP reservation](https://projectcalico.docs.tigera.io/reference/resources/ipreservation), Calico [use-specific-ip](https://projectcalico.docs.tigera.io/networking/use-specific-ip) for pod, and Calico [use-specific-mac-address](https://github.com/projectcalico/calico/blob/master/calico/networking/pod-mac-address.md) for pod.
 
@@ -72,10 +72,12 @@ spec:
         cni.projectcalico.org/ipAddrs: '["$IP_ADDRESS"]'
 ```
 
+We use [ip-address-manager](https://github.com/metal3-io/ip-address-manager) to manage the allocattion of IP addresses, so you need to create an [IPPool](https://github.com/metal3-io/ip-address-manager/blob/main/docs/api.md#ippool) resource.
+
 ```shell
-# replace to reserved IP addresses
-export VIRTINK_NODE_ADDRESSES='["172.22.161.241", "172.22.161.242"]'
-clusterctl generate cluster --from cluster-template-cdi-internal.yaml capi-quickstart
+# replace to created IPPool name.
+export VIRTINK_IP_POOL_NAME='capi-quickstart'
+clusterctl generate cluster --infrastructure virtink --flavor cdi-internal capi-quickstart
 ```
 
 ## License

diff --git a/api/v1beta1/virtinkcluster_types.go b/api/v1beta1/virtinkcluster_types.go
@@ -24,13 +24,6 @@ type VirtinkClusterSpec struct {
 
 	// InfraClusterSecretRef is a reference to a secret with a kubeconfig for external cluster used for infra.
 	InfraClusterSecretRef *corev1.ObjectReference `json:"infraClusterSecretRef,omitempty"`
-	NodeAddressConfig     *NodeAddressConfig      `json:"nodeAddressConfig,omitempty"`
-}
-
-type NodeAddressConfig struct {
-	// Addresses are list of IP addresses for allocating to nested cluster nodes,
-	// support IP address ("192.168.100.1"), IP range ("192.168.100.1-192.168.100.10") or CIDR ("192.168.100.1/24") form.
-	Addresses []string `json:"addresses,omitempty"`
 }
 
 // VirtinkClusterStatus defines the observed state of VirtinkCluster

diff --git a/api/v1beta1/virtinkmachine_types.go b/api/v1beta1/virtinkmachine_types.go
@@ -2,6 +2,7 @@ package v1beta1
 
 import (
 	virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
 	capierrors "sigs.k8s.io/cluster-api/errors"
@@ -17,8 +18,9 @@ type VirtinkMachineSpec struct {
 
 	ProviderID *string `json:"providerID,omitempty"`
 
-	VMSpec          virtv1alpha1.VirtualMachineSpec `json:"vmSpec"`
-	VolumeTemplates []VolumeTemplateSource          `json:"volumeTemplates,omitempty"`
+	VMSpec          virtv1alpha1.VirtualMachineSpec   `json:"vmSpec"`
+	VolumeTemplates []VolumeTemplateSource            `json:"volumeTemplates,omitempty"`
+	IPPoolRef       *corev1.TypedLocalObjectReference `json:"ipPoolRef,omitempty"`
 }
 
 type VolumeTemplateSource struct {

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclusters.yaml
@@ -104,17 +104,6 @@ spec:
                     description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
                     type: string
                 type: object
-              nodeAddressConfig:
-                properties:
-                  addresses:
-                    description: Addresses are list of IP addresses for allocating
-                      to nested cluster nodes, support IP address ("192.168.100.1"),
-                      IP range ("192.168.100.1-192.168.100.10") or CIDR ("192.168.100.1/24")
-                      form.
-                    items:
-                      type: string
-                    type: array
-                type: object
             type: object
           status:
             description: VirtinkClusterStatus defines the observed state of VirtinkCluster

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclustertemplates.yaml
@@ -102,17 +102,6 @@ spec:
                             description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
                             type: string
                         type: object
-                      nodeAddressConfig:
-                        properties:
-                          addresses:
-                            description: Addresses are list of IP addresses for allocating
-                              to nested cluster nodes, support IP address ("192.168.100.1"),
-                              IP range ("192.168.100.1-192.168.100.10") or CIDR ("192.168.100.1/24")
-                              form.
-                            items:
-                              type: string
-                            type: array
-                        type: object
                     type: object
                 required:
                 - spec

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachines.yaml
@@ -42,6 +42,26 @@ spec:
           spec:
             description: VirtinkMachineSpec defines the desired state of VirtinkMachine
             properties:
+              ipPoolRef:
+                description: TypedLocalObjectReference contains enough information
+                  to let you locate the typed referenced object inside the same namespace.
+                properties:
+                  apiGroup:
+                    description: APIGroup is the group for the resource being referenced.
+                      If APIGroup is not specified, the specified Kind must be in
+                      the core API group. For any other third-party types, APIGroup
+                      is required.
+                    type: string
+                  kind:
+                    description: Kind is the type of resource being referenced
+                    type: string
+                  name:
+                    description: Name is the name of resource being referenced
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               providerID:
                 type: string
               vmSpec:

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachinetemplates.yaml
@@ -44,6 +44,27 @@ spec:
                   spec:
                     description: VirtinkMachineSpec defines the desired state of VirtinkMachine
                     properties:
+                      ipPoolRef:
+                        description: TypedLocalObjectReference contains enough information
+                          to let you locate the typed referenced object inside the
+                          same namespace.
+                        properties:
+                          apiGroup:
+                            description: APIGroup is the group for the resource being
+                              referenced. If APIGroup is not specified, the specified
+                              Kind must be in the core API group. For any other third-party
+                              types, APIGroup is required.
+                            type: string
+                          kind:
+                            description: Kind is the type of resource being referenced
+                            type: string
+                          name:
+                            description: Name is the name of resource being referenced
+                            type: string
+                        required:
+                        - kind
+                        - name
+                        type: object
                       providerID:
                         type: string
                       vmSpec:

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -133,6 +133,34 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - ipam.metal3.io
+  resources:
+  - ipaddresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ipam.metal3.io
+  resources:
+  - ipclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ipam.metal3.io
+  resources:
+  - ipclaims/status
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - virt.virtink.smartx.com
   resources:

diff --git a/controllers/iprange/iprange.go b/controllers/iprange/iprange.go
diff --git a/controllers/iprange/iprange_test.go b/controllers/iprange/iprange_test.go