Feature: support direct disk mounting for service container

Disk UUID will be wrote into the config of service(chunkserver) container during curvebs deployment if disks `service_mount_device` was set `true`, then the service could directly mount disk device via `entrypoint.sh` and restart automatically after host power recover. Signed-off-by: Lijin Xiong <[email protected]>
opencurve · Mar 19, 2023 · 2c50545 · 2c50545
1 parent af342de
commit 2c50545
Show file tree

Hide file tree

Showing 18 changed files with 476 additions and 135 deletions.
diff --git a/cli/command/disks/commit.go b/cli/command/disks/commit.go
@@ -27,7 +27,6 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/opencurve/curveadm/cli/cli"
-	"github.com/opencurve/curveadm/internal/common"
 	comm "github.com/opencurve/curveadm/internal/common"
 	"github.com/opencurve/curveadm/internal/configure/disks"
 	"github.com/opencurve/curveadm/internal/configure/topology"
@@ -40,7 +39,8 @@ import (
 )
 
 const (
-	COMMIT_EXAMPLE = `Examples:
+	HOST_DEVICE_SEP = ":"
+	COMMIT_EXAMPLE  = `Examples:
   $ curveadm disks commit /path/to/disks.yaml  # Commit disks`
 )
 
@@ -89,35 +89,55 @@ func readAndCheckDisks(curveadm *cli.CurveAdm, options commitOptions) (string, [
 	}
 
 	// 3) check disks data
-	dcs, err = disks.ParseDisks(data, curveadm)
+	dcs, err = disks.ParseDisks(data)
 	return data, dcs, err
 }
 
 func assambleNewDiskRecords(dcs []*disks.DiskConfig,
 	oldDiskRecords []storage.Disk) ([]storage.Disk, []storage.Disk) {
-	keySep := ":"
 	newDiskMap := make(map[string]bool)
 
+	// "ServiceMountDevice=0" means write disk UUID into /etc/fstab for host mounting.
+	// "ServiceMountDevice=1" means not to update /etc/fstab, the disk UUID will be wrote
+	// into the config of service(chunkserver) container for disk automatic mounting.
+	serviceMountDevice := 0 // 0: false, 1: true
 	var newDiskRecords, diskRecordDeleteList []storage.Disk
 	for _, dc := range dcs {
 		for _, host := range dc.GetHost() {
-			key := strings.Join([]string{host, dc.GetDevice()}, keySep)
+			device := dc.GetDevice()
+			key := strings.Join([]string{host, device}, HOST_DEVICE_SEP)
 			newDiskMap[key] = true
+			if dc.GetServiceMount() {
+				serviceMountDevice = 1
+			}
+			diskSize := comm.DISK_DEFAULT_NULL_SIZE
+			diskUri := comm.DISK_DEFAULT_NULL_URI
+			diskChunkserverId := comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID
+			for _, dr := range oldDiskRecords {
+				if dr.Host == host && device == dr.Device {
+					diskSize = dr.Size
+					diskUri = dr.URI
+					diskChunkserverId = dr.ChunkServerID
+					break
+				}
+			}
 			newDiskRecords = append(
 				newDiskRecords, storage.Disk{
-					Host:          host,
-					Device:        dc.GetDevice(),
-					Size:          comm.DISK_DEFAULT_NULL_SIZE,
-					URI:           comm.DISK_DEFAULT_NULL_URI,
-					MountPoint:    dc.GetMountPoint(),
-					FormatPercent: dc.GetFormatPercent(),
-					ChunkServerID: comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+					Host:               host,
+					Device:             device,
+					Size:               diskSize,
+					URI:                diskUri,
+					MountPoint:         dc.GetMountPoint(),
+					ContainerImage:     dc.GetContainerImage(),
+					FormatPercent:      dc.GetFormatPercent(),
+					ChunkServerID:      diskChunkserverId,
+					ServiceMountDevice: serviceMountDevice,
 				})
 		}
 	}
 
 	for _, dr := range oldDiskRecords {
-		key := strings.Join([]string{dr.Host, dr.Device}, keySep)
+		key := strings.Join([]string{dr.Host, dr.Device}, HOST_DEVICE_SEP)
 		if _, ok := newDiskMap[key]; !ok {
 			diskRecordDeleteList = append(diskRecordDeleteList, dr)
 		}
@@ -127,19 +147,16 @@ func assambleNewDiskRecords(dcs []*disks.DiskConfig,
 }
 
 func writeDiskRecord(dr storage.Disk, curveadm *cli.CurveAdm) error {
-	if diskRecords, err := curveadm.Storage().GetDisk(
-		common.DISK_FILTER_DEVICE, dr.Host, dr.Device); err != nil {
+	if err := curveadm.Storage().SetDisk(
+		dr.Host,
+		dr.Device,
+		dr.MountPoint,
+		dr.ContainerImage,
+		dr.FormatPercent,
+		dr.ServiceMountDevice); err != nil {
 		return err
-	} else if len(diskRecords) == 0 {
-		if err := curveadm.Storage().SetDisk(
-			dr.Host,
-			dr.Device,
-			dr.MountPoint,
-			dr.ContainerImage,
-			dr.FormatPercent); err != nil {
-			return err
-		}
 	}
+
 	return nil
 }
 
@@ -153,15 +170,17 @@ func syncDiskRecords(data string, dcs []*disks.DiskConfig,
 	oldDiskRecordsString := tui.FormatDisks(oldDiskRecords)
 	newDiskRecordsString := tui.FormatDisks(newDiskRecords)
 
-	if !options.slient {
-		diff := utils.Diff(oldDiskRecordsString, newDiskRecordsString)
-		curveadm.WriteOutln(diff)
-	}
+	if len(newDiskRecords) != len(oldDiskRecords) {
+		if !options.slient {
+			diff := utils.Diff(oldDiskRecordsString, newDiskRecordsString)
+			curveadm.WriteOutln(diff)
+		}
 
-	pass := tuicomm.ConfirmYes("Disk changes are showing above. Do you want to continue?")
-	if !pass {
-		curveadm.WriteOut(tuicomm.PromptCancelOpetation("commit disk table"))
-		return errno.ERR_CANCEL_OPERATION
+		pass := tuicomm.ConfirmYes("Disk changes are showing above. Do you want to continue?")
+		if !pass {
+			curveadm.WriteOut(tuicomm.PromptCancelOpetation("commit disk table"))
+			return errno.ERR_CANCEL_OPERATION
+		}
 	}
 
 	// write new disk records

diff --git a/cli/command/format.go b/cli/command/format.go
@@ -155,9 +155,17 @@ func runFormat(curveadm *cli.CurveAdm, options formatOptions) error {
 			if err != nil {
 				return err
 			}
-			fc.UseDiskUri = true
-			chunkserverId := dr.ChunkServerID
-			if len(chunkserverId) > 1 {
+
+			fc.FromDiskRecord = true
+
+			// "ServiceMountDevice=0" means write disk UUID into /etc/fstab for host mounting.
+			// "ServiceMountDevice=1" means not to update /etc/fstab, the disk UUID will be wrote
+			// into the config of service(chunkserver) container for disk automatic mounting.
+			if dr.ServiceMountDevice != 0 {
+				fc.ServiceMountDevice = true
+			}
+
+			if dr.ChunkServerID != comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID {
 				// skip formatting the disk with nonempty chunkserver id
 				continue
 			}

diff --git a/configs/bs/cluster/disks.yaml b/configs/bs/cluster/disks.yaml
@@ -1,23 +1,24 @@
 global:
   format_percent: 95
   container_image: opencurvedocker/curvebs:v1.2
+  service_mount_device: false # disk device will be mounted by service(chunkserver) directly if set "true", default "false"
   host:
     - curve-1
     - curve-2
     - curve-3
 
 disk:
-  - device: /dev/sdb1
-    mount: /data/chunkserver0
+  - device: /dev/sdb1 # disk device path
+    mount: /data/chunkserver0 # mount point for disk formatting, consistent with the "data_dir" field of chunkserver service in topology
   - device: /dev/sdc1
     mount: /data/chunkserver1
-    format_percent: 90
+    format_percent: 90 # use a different value for disk format percent
   - device: /dev/sdd1
     mount: /data/chunkserver2
-    exclude:  # for the use case that some hosts have not certain disk device
+    exclude:  # for the use case that disk device does not exist in some hosts
       - curve-3
   - device: /dev/sde1
     mount: /data/chunkserver3
-    host:
+    host:    # override global host config, for the use case that disk device only exists in some hosts
       - curve-1
       - curve-2
diff --git a/internal/common/common.go b/internal/common/common.go
@@ -57,8 +57,8 @@ const (
 	DISK_FILTER_MOUNT                 = "mount"
 	DISK_FILTER_SERVICE               = "service"
 	DISK_EXCLUDE_HOST                 = "exclude"
-
-	DISK_FORMAT_PERCENT = "format_percent"
+	DISK_SERVICE_MOUNT_DEVICE         = "service_mount_device"
+	DISK_FORMAT_PERCENT               = "format_percent"
 
 	DISK_FORMAT_MOUNT_POINT     = "mount"
 	DISK_FORMAT_CONTAINER_IMAGE = "container_image"

diff --git a/internal/configure/common/item_set.go b/internal/configure/common/item_set.go
@@ -35,7 +35,7 @@ const (
 	REQUIRE_BOOL
 	REQUIRE_INT
 	REQUIRE_POSITIVE_INTEGER
-	REQUIRE_SLICE
+	REQUIRE_STRING_SLICE
 )
 
 type (
@@ -82,6 +82,25 @@ func (itemset *ItemSet) GetAll() []*Item {
 	return itemset.items
 }
 
+func convertSlice[T int | string | any](key, value any) ([]T, error) {
+	var slice []T
+	if !utils.IsAnySlice(value) || len(value.([]any)) == 0 {
+		return slice, errno.ERR_CONFIGURE_VALUE_REQUIRES_NONEMPTY_SLICE
+	}
+	anySlice := value.([]any)
+	switch anySlice[0].(type) {
+	case T:
+		for _, str := range anySlice {
+			slice = append(slice, str.(T))
+		}
+	default:
+		return slice, errno.ERR_UNSUPPORT_CONFIGURE_VALUE_TYPE.
+			F("%s: %v", key, value)
+	}
+
+	return slice, nil
+}
+
 func (itemset *ItemSet) Build(key string, value interface{}) (interface{}, error) {
 	item := itemset.Get(key)
 	if item == nil {
@@ -135,34 +154,12 @@ func (itemset *ItemSet) Build(key string, value interface{}) (interface{}, error
 			return v, nil
 		}
 
-	case REQUIRE_SLICE:
-		anySlice := value.([]any)
-		if len(anySlice) > 0 {
-			switch anySlice[0].(type) {
-			case string:
-				return convertSlice[string](value), nil
-			case int:
-				return convertSlice[int](value), nil
-			case bool:
-				return convertSlice[bool](value), nil
-			default:
-				return []any{}, errno.ERR_UNSUPPORT_CONFIGURE_VALUE_TYPE.
-					F("%s: %v", key, value)
-			}
-		}
-		return []any{}, nil
+	case REQUIRE_STRING_SLICE:
+		return convertSlice[string](key, value)
 
 	default:
 		// do nothing
 	}
 
 	return value, nil
 }
-
-func convertSlice[T int | string | any](value any) []T {
-	var slice []T
-	for _, str := range value.([]any) {
-		slice = append(slice, str.(T))
-	}
-	return slice
-}
diff --git a/internal/configure/disks/dc_get.go b/internal/configure/disks/dc_get.go
@@ -55,6 +55,14 @@ func (dc *DiskConfig) getInt(i *comm.Item) int {
 	return v.(int)
 }
 
+func (hc *DiskConfig) getBool(i *comm.Item) bool {
+	v := hc.get(i)
+	if v == nil {
+		return false
+	}
+	return v.(bool)
+}
+
 func (dc *DiskConfig) getStrSlice(i *comm.Item) []string {
 	v := dc.get(i)
 	if v == nil {
@@ -65,6 +73,7 @@ func (dc *DiskConfig) getStrSlice(i *comm.Item) []string {
 
 func (dc *DiskConfig) GetContainerImage() string { return dc.getString(CONFIG_GLOBAL_CONTAINER_IMAGE) }
 func (dc *DiskConfig) GetFormatPercent() int     { return dc.getInt(CONFIG_GLOBAL_FORMAT_PERCENT) }
+func (dc *DiskConfig) GetServiceMount() bool     { return dc.getBool(CONFIG_GLOBAL_SERVICE_MOUNT_DEVICE) }
 func (dc *DiskConfig) GetHost() []string         { return dc.getStrSlice(CONFIG_GLOBAL_HOST) }
 func (dc *DiskConfig) GetDevice() string         { return dc.getString(CONFIG_DISK_DEVICE) }
 func (dc *DiskConfig) GetMountPoint() string     { return dc.getString(CONFIG_DISK_MOUNT_POINT) }

diff --git a/internal/configure/disks/dc_item.go b/internal/configure/disks/dc_item.go
@@ -48,12 +48,20 @@ var (
 		DEFAULT_FORMAT_PERCENT,
 	)
 
+	CONFIG_GLOBAL_SERVICE_MOUNT_DEVICE = itemset.Insert(
+		common.DISK_SERVICE_MOUNT_DEVICE,
+		comm.REQUIRE_BOOL,
+		false,
+		false,
+	)
+
 	CONFIG_GLOBAL_HOST = itemset.Insert(
 		common.DISK_FILTER_HOST,
-		comm.REQUIRE_SLICE,
+		comm.REQUIRE_STRING_SLICE,
 		false,
 		nil,
 	)
+
 	CONFIG_DISK_DEVICE = itemset.Insert(
 		common.DISK_FILTER_DEVICE,
 		comm.REQUIRE_STRING,
@@ -70,7 +78,7 @@ var (
 
 	CONFIG_DISK_HOST_EXCLUDE = itemset.Insert(
 		common.DISK_EXCLUDE_HOST,
-		comm.REQUIRE_SLICE,
+		comm.REQUIRE_STRING_SLICE,
 		false,
 		nil,
 	)