Merge pull request #2000 from pixiake/master

Support v1beta3 of kubeadm config
kubesphere · Sep 25, 2023 · 9cb8c71 · 9cb8c71
2 parents 2f9c1f3 + 5613458
commit 9cb8c71
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 27 deletions.
diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
@@ -173,9 +173,6 @@ func (cfg *ClusterSpec) GroupHosts() map[string][]*KubeHost {
 	if len(roleGroups[Etcd]) == 0 && cfg.Etcd.Type == KubeKey {
 		logger.Log.Fatal(errors.New("The number of etcd cannot be 0"))
 	}
-	if len(roleGroups[Registry]) <= 0 {
-		logger.Log.Fatal(errors.New("The number of registry must be big then 0."))
-	}
 
 	for _, host := range roleGroups[ControlPlane] {
 		host.SetRole(Master)

diff --git a/cmd/kk/pkg/etcd/templates/etcd_env.go b/cmd/kk/pkg/etcd/templates/etcd_env.go
@@ -95,7 +95,7 @@ ETCD_PEER_CLIENT_CERT_AUTH=True
 
 # CLI settings
 ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
-ETCDCTL_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
-ETCDCTL_KEY_FILE=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem
-ETCDCTL_CERT_FILE=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem
+ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
+ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem
+ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem
     `)))
diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go
@@ -45,7 +45,6 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates/v1beta2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
 )
 
@@ -249,11 +248,11 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 			}
 		}
 
-		_, ApiServerArgs := util.GetArgs(v1beta2.GetApiServerArgs(g.WithSecurityEnhancement, g.KubeConf.Cluster.Kubernetes.EnableAudit()), g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
-		_, ControllerManagerArgs := util.GetArgs(v1beta2.GetControllermanagerArgs(g.KubeConf.Cluster.Kubernetes.Version, g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
-		_, SchedulerArgs := util.GetArgs(v1beta2.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
+		_, ApiServerArgs := util.GetArgs(templates.GetApiServerArgs(g.WithSecurityEnhancement, g.KubeConf.Cluster.Kubernetes.EnableAudit()), g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
+		_, ControllerManagerArgs := util.GetArgs(templates.GetControllermanagerArgs(g.KubeConf.Cluster.Kubernetes.Version, g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
+		_, SchedulerArgs := util.GetArgs(templates.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
 
-		checkCgroupDriver, err := v1beta2.GetKubeletCgroupDriver(runtime, g.KubeConf)
+		checkCgroupDriver, err := templates.GetKubeletCgroupDriver(runtime, g.KubeConf)
 		if err != nil {
 			return err
 		}
@@ -273,8 +272,8 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 		}
 
 		templateAction := action.Template{
-			Template: v1beta2.KubeadmConfig,
-			Dst:      filepath.Join(common.KubeConfigDir, v1beta2.KubeadmConfig.Name()),
+			Template: templates.KubeadmConfig,
+			Dst:      filepath.Join(common.KubeConfigDir, templates.KubeadmConfig.Name()),
 			Data: util.Data{
 				"IsInitCluster":          g.IsInitConfiguration,
 				"ImageRepo":              strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "kube-apiserver").ImageRepo(), "/kube-apiserver"),
@@ -296,12 +295,13 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 				"ExternalEtcd":           externalEtcd,
 				"NodeCidrMaskSize":       g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
 				"CriSock":                g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint,
-				"ApiServerArgs":          v1beta2.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf),
+				"ApiServerArgs":          templates.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf),
 				"EnableAudit":            g.KubeConf.Cluster.Kubernetes.EnableAudit(),
-				"ControllerManagerArgs":  v1beta2.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
-				"SchedulerArgs":          v1beta2.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
-				"KubeletConfiguration":   v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
-				"KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf),
+				"ControllerManagerArgs":  templates.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
+				"SchedulerArgs":          templates.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
+				"KubeletConfiguration":   templates.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
+				"KubeProxyConfiguration": templates.GetKubeProxyConfiguration(g.KubeConf),
+				"IsV1beta3":              versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.22.0")),
 				"IsControlPlane":         host.IsRole(common.Master),
 				"CgroupDriver":           checkCgroupDriver,
 				"BootstrapToken":         bootstrapToken,

diff --git a/...netes/templates/v1beta2/kubeadm_config.go → ...kg/kubernetes/templates/kubeadm_config.go b/...netes/templates/v1beta2/kubeadm_config.go → ...kg/kubernetes/templates/kubeadm_config.go
@@ -14,7 +14,7 @@
  limitations under the License.
 */
 
-package v1beta2
+package templates
 
 import (
 	"fmt"
@@ -38,7 +38,7 @@ var (
 		dedent.Dedent(`
 {{- if .IsInitCluster -}}
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }}
 kind: ClusterConfiguration
 etcd:
 {{- if .EtcdTypeIsKubeadm }}
@@ -106,7 +106,7 @@ scheduler:
 {{ toYaml .SchedulerArgs | indent 4 }}
 
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }}
 kind: InitConfiguration
 localAPIEndpoint:
   advertiseAddress: {{ .AdvertiseAddress }}
@@ -128,7 +128,7 @@ kind: KubeletConfiguration
 
 {{- else -}}
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }}
 kind: JoinConfiguration
 discovery:
   bootstrapToken:
@@ -159,14 +159,10 @@ var (
 	FeatureGatesDefaultConfiguration = map[string]bool{
 		"RotateKubeletServerCertificate": true, //k8s 1.7+
 		"TTLAfterFinished":               true, //k8s 1.12+
-		"ExpandCSIVolumes":               true, //k8s 1.14+
-		"CSIStorageCapacity":             true, //k8s 1.19+
 	}
 	FeatureGatesSecurityDefaultConfiguration = map[string]bool{
 		"RotateKubeletServerCertificate": true, //k8s 1.7+
 		"TTLAfterFinished":               true, //k8s 1.12+
-		"ExpandCSIVolumes":               true, //k8s 1.14+
-		"CSIStorageCapacity":             true, //k8s 1.19+
 		"SeccompDefault":                 true, //kubelet
 	}
 

diff --git a/cmd/kk/pkg/pipelines/init_registry.go b/cmd/kk/pkg/pipelines/init_registry.go
@@ -18,16 +18,17 @@ package pipelines
 
 import (
 	"fmt"
-
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem"
+	"github.com/pkg/errors"
 )
 
 func NewInitRegistryPipeline(runtime *common.KubeRuntime) error {
@@ -74,6 +75,10 @@ func InitRegistry(args common.Argument, downloadCmd string) error {
 		return err
 	}
 
+	if len(runtime.GetHostsByRole("registry")) <= 0 {
+		logger.Log.Fatal(errors.New("The number of registry must be greater then 0."))
+	}
+
 	if err := NewInitRegistryPipeline(runtime); err != nil {
 		return err
 	}

diff --git a/cmd/kk/pkg/version/kubernetes/version_enum.go b/cmd/kk/pkg/version/kubernetes/version_enum.go
@@ -36,6 +36,8 @@ const (
 	V124
 	V125
 	V126
+	V127
+	V128
 )
 
 var VersionList = []Version{
@@ -47,6 +49,8 @@ var VersionList = []Version{
 	V124,
 	V125,
 	V126,
+	V127,
+	V128,
 }
 
 func (v Version) String() string {
@@ -67,6 +71,10 @@ func (v Version) String() string {
 		return "v1.25"
 	case V126:
 		return "v1.26"
+	case V127:
+		return "v1.27"
+	case V128:
+		return "v1.28"
 	default:
 		return "invalid option"
 	}