feat(platform): validate storage info (#2093)

pkg/platform/provider/baremetal/validation/cluster.go

@@ -22,10 +22,13 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	appsv1alpha1 "github.com/clusternet/apis/apps/v1alpha1"
 	"math"
 	"net"
+	"strconv"
 	"strings"
 	"time"
+	"tkestack.io/tke/pkg/mesh/util/json"
 
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
 	apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation"
@@ -125,6 +128,7 @@ func ValidatClusterSpec(platformClient platformv1client.PlatformV1Interface, clu
 	allErrs = append(allErrs, ValidateClusterSpecVersion(platformClient, clusterName, cls.Spec.Version, fldPath.Child("version"), phase)...)
 	allErrs = append(allErrs, ValidateCIDRs(cls, fldPath)...)
 	allErrs = append(allErrs, ValidateClusterProperty(&cls.Spec, fldPath.Child("properties"))...)
+	allErrs = append(allErrs, ValidateStorage(cls, fldPath)...)
 	if validateMachine {
 		allErrs = append(allErrs, ValidateClusterMachines(cls, fldPath.Child("machines"))...)
 	}
@@ -281,7 +285,7 @@ func ValidateClusterMachines(cls *platform.Cluster, fldPath *field.Path) field.E
 
 		selinuxResult.Name = AnywhereValidateItemSelinux
 		selinuxResult.Description = "Verify Selinux"
-		selinuxResult.ErrorList = firewallErrs
+		selinuxResult.ErrorList = selinuxErrs
 
 		allErrs = append(allErrs,
 			proxyResult.ToFieldError(),
@@ -350,14 +354,149 @@ func ValidateOSVersion(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList {
 func ValidateReservePorts(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList {
 	allErrs := field.ErrorList{}
 	for i, one := range sshs {
-		err := ssh.ReservePorts(one, reservePorts)
+		err := ssh.ReservePorts(one, "127.0.0.1", reservePorts)
 		if err != nil {
 			allErrs = append(allErrs, field.Invalid(fldPath.Index(i), one.Host, err.Error()))
 		}
 	}
 	return allErrs
 }
 
+func ValidateStorage(cls *platform.Cluster, fld *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	storageResult := TKEValidateResult{}
+	allErrs = append(allErrs, ValidateNFS(cls, fld)...)
+	allErrs = append(allErrs, ValidateCephFS(cls, fld)...)
+
+	storageResult.Checked = true
+	if _, ok := cls.Annotations[platform.AnywhereValidateAnno]; ok {
+		storageResult.Name = AnywhereValidateItemStorage
+		storageResult.Description = "Validate Storage Info"
+		storageResult.ErrorList = allErrs
+	}
+	return allErrs
+}
+
+func ValidateNFS(cls *platform.Cluster, fld *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if cls.Annotations[platformv1.AnywhereLocalizationsAnno] == "" {
+		return nil
+	}
+
+	localizationsJSON, err := base64.StdEncoding.DecodeString(cls.Annotations[platformv1.AnywhereLocalizationsAnno])
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fld, err, "decode error"))
+		return allErrs
+	}
+	var storageInfo *StorageInfo
+	storageInfo, err = GetStorageInfo(localizationsJSON)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fld, storageInfo, err.Error()))
+		return allErrs
+	}
+	if storageInfo == nil || !storageInfo.EnableNFS {
+		return nil
+	}
+
+	machine, err := cls.Spec.Machines[0].SSH()
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fld, err, "ssh machine failed"))
+	}
+	nfsServer := storageInfo.Nfs.Server
+	nfsPath := storageInfo.Nfs.Path
+	err = ssh.CheckNFS(machine, nfsServer, nfsPath)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fld, err, "check nfs failed"))
+	}
+
+	return allErrs
+}
+
+func ValidateCephFS(cls *platform.Cluster, fld *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if cls.Annotations[platformv1.AnywhereLocalizationsAnno] == "" {
+		return nil
+	}
+
+	localizationsJSON, err := base64.StdEncoding.DecodeString(cls.Annotations[platformv1.AnywhereLocalizationsAnno])
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fld, err, "decode error"))
+		return allErrs
+	}
+
+	var storageInfo *StorageInfo
+	storageInfo, err = GetStorageInfo(localizationsJSON)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fld, storageInfo, err.Error()))
+		return allErrs
+	}
+	if storageInfo == nil || !storageInfo.EnableCephfs {
+		return nil
+	}
+
+	for _, host := range storageInfo.CsiConfig.Monitors {
+		arr := strings.SplitN(host, ":", 2)
+		if len(arr) != 2 {
+			allErrs = append(allErrs, field.Invalid(fld, host, "invalid host format"))
+			continue
+		}
+
+		ip, port := arr[0], arr[1]
+		p, err := strconv.Atoi(port)
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(fld, err, "convert port error"))
+			continue
+		}
+
+		machine, err := cls.Spec.Machines[0].SSH()
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(fld, err, "ssh machine failed"))
+		}
+
+		err = ssh.ReservePorts(machine, ip, []int{p})
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(fld, err, "invalid port"))
+		}
+	}
+	return allErrs
+}
+
+func GetStorageInfo(annoData []byte) (*StorageInfo, error) {
+	localizations := new(appsv1alpha1.LocalizationList)
+	err := json.Unmarshal(annoData, localizations)
+	if err != nil {
+		return nil, err
+	}
+
+	res := &StorageInfo{}
+
+	for _, item := range localizations.Items {
+		if item.ObjectMeta.Name != "tke-storage" {
+			continue
+		}
+		n := len(item.Spec.Overrides)
+		if n == 0 {
+			return nil, nil
+		}
+		value := &StorageOverrideValue{}
+		err = json.Unmarshal([]byte(item.Spec.Overrides[n-1].Value), value)
+		if err != nil {
+			return nil, err
+		}
+		res.EnableNFS = value.Global.EnableNFS
+		res.Nfs.Server = value.NfsSubdirExternalProvisioner.Nfs.Server
+		res.Nfs.Path = value.NfsSubdirExternalProvisioner.Nfs.Path
+		res.EnableCephfs = value.Global.EnableCephFS
+		for _, cfg := range value.CephCsiCephfs.CsiConfig {
+			res.CsiConfig.Monitors = append(res.CsiConfig.Monitors, cfg.Monitors...)
+		}
+		return res, nil
+	}
+	return nil, nil
+}
+
 func ValidateFirewall(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList {
 	allErrs := field.ErrorList{}
 	for i, one := range sshs {

pkg/platform/provider/baremetal/validation/constants.go

@@ -29,6 +29,7 @@ const (
 	AnywhereValidateItemHostNetOverlapping = "HostNetOverlapping"
 	AnywhereValidateItemFirewall           = "Firewall"
 	AnywhereValidateItemSelinux            = "Selinux"
+	AnywhereValidateItemStorage            = "Storage"
 )
 
 const (

pkg/platform/provider/baremetal/validation/overridevalue.go

@@ -0,0 +1,61 @@
+/*
+ * Tencent is pleased to support the open source community by making TKEStack
+ * available.
+ *
+ * Copyright (C) 2012-2019 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * https://opensource.org/licenses/Apache-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package validation
+
+// StorageOverrideValue contains all the storage info used to validate from localization OverrideConfig
+type StorageOverrideValue struct {
+	Global                       StorageGlobalCfg             `json:"global"`
+	NfsSubdirExternalProvisioner NfsSubdirExternalProvisioner `json:"nfs-subdir-external-provisioner"`
+	CephCsiCephfs                CephCsiCephfs                `json:"ceph-csi-cephfs"`
+}
+type StorageGlobalCfg struct {
+	EnableNFS    bool `json:"enableNFS"`
+	EnableCephFS bool `json:"enableCephFS"`
+}
+type Nfs struct {
+	Server string `json:"server"`
+	Path   string `json:"path"`
+}
+type NfsSubdirExternalProvisioner struct {
+	Nfs Nfs `json:"nfs"`
+}
+type CsiConfig struct {
+	ClusterID string   `json:"clusterID"`
+	Monitors  []string `json:"monitors"`
+}
+type StorageClass struct {
+	ClusterID string `json:"clusterID"`
+	FsName    string `json:"fsName"`
+}
+type CephCsiSecret struct {
+	AdminID  string `json:"adminID"`
+	AdminKey string `json:"adminKey"`
+}
+type CephCsiCephfs struct {
+	CsiConfig    []CsiConfig   `json:"csiConfig"`
+	StorageClass StorageClass  `json:"storageClass"`
+	Secret       CephCsiSecret `json:"secret"`
+}
+
+type StorageInfo struct {
+	EnableNFS    bool
+	EnableCephfs bool
+	Nfs          Nfs
+	CsiConfig    CsiConfig
+}

pkg/util/ssh/os.go

@@ -156,10 +156,10 @@ func OSVersion(s Interface) (os string, err error) {
 	return id + version, nil
 }
 
-func ReservePorts(s Interface, ports []int) error {
+func ReservePorts(s Interface, ip string, ports []int) error {
 	var cmd, errMessage string
 	for _, port := range ports {
-		cmd += fmt.Sprintf(`bash -c "</dev/tcp/127.0.0.1/%d" &>/dev/null; echo $?; `, port)
+		cmd += fmt.Sprintf(`bash -c "</dev/tcp/%s/%d" &>/dev/null; echo $?; `, ip, port)
 	}
 	out, _, _, _ := s.Exec(cmd)
 	out = strings.TrimSuffix(out, "\n")
@@ -217,3 +217,11 @@ func SelinuxEnabled(s Interface) (enabled bool, err error) {
 	res := strings.TrimSpace(stdout)
 	return res == "0", nil
 }
+
+func CheckNFS(s Interface, server string, path string) (err error) {
+	_, stderr, exit, err := s.Execf("mkdir -p /tmp/nfs/&& mount -t nfs %s:%s /tmp/nfs/&& umount /tmp/nfs/&& rm -rf /tmp/nfs/", server, path)
+	if exit != 0 || err != nil {
+		return fmt.Errorf("check nfs failed:exit %d:stderr %s:error %s", exit, stderr, err)
+	}
+	return nil
+}