Feature: support direct disk mounting for service container

Disk UUID will be wrote into the config of service(chunkserver) container during curvebs deployment if disks `service_mount_device` was set `true`, then the service cloud mount disk via `entrypoint.sh` and restart automatically after host power recover. Signed-off-by: Lijin Xiong <[email protected]>
opencurve · Mar 8, 2023 · 2ad43c8 · 2ad43c8
1 parent af342de
commit 2ad43c8
Show file tree

Hide file tree

Showing 15 changed files with 218 additions and 93 deletions.
diff --git a/cli/command/disks/commit.go b/cli/command/disks/commit.go
@@ -27,7 +27,6 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/opencurve/curveadm/cli/cli"
-	"github.com/opencurve/curveadm/internal/common"
 	comm "github.com/opencurve/curveadm/internal/common"
 	"github.com/opencurve/curveadm/internal/configure/disks"
 	"github.com/opencurve/curveadm/internal/configure/topology"
@@ -40,7 +39,8 @@ import (
 )
 
 const (
-	COMMIT_EXAMPLE = `Examples:
+	HOST_DEVICE_SEP = ":"
+	COMMIT_EXAMPLE  = `Examples:
   $ curveadm disks commit /path/to/disks.yaml  # Commit disks`
 )
 
@@ -95,29 +95,37 @@ func readAndCheckDisks(curveadm *cli.CurveAdm, options commitOptions) (string, [
 
 func assambleNewDiskRecords(dcs []*disks.DiskConfig,
 	oldDiskRecords []storage.Disk) ([]storage.Disk, []storage.Disk) {
-	keySep := ":"
 	newDiskMap := make(map[string]bool)
 
+	// "ServiceMountDevice=0" means write disk UUID into /etc/fstab for host mounting.
+	// "ServiceMountDevice=1" means not to update /etc/fstab, the disk UUID will be wrote
+	// into the config of service(chunkserver) container for disk automatic mounting.
+	serviceMountDevice := 0 // 0: false, 1: true
 	var newDiskRecords, diskRecordDeleteList []storage.Disk
 	for _, dc := range dcs {
 		for _, host := range dc.GetHost() {
-			key := strings.Join([]string{host, dc.GetDevice()}, keySep)
+			key := strings.Join([]string{host, dc.GetDevice()}, HOST_DEVICE_SEP)
 			newDiskMap[key] = true
+			if dc.GetServiceMount() {
+				serviceMountDevice = 1
+			}
 			newDiskRecords = append(
 				newDiskRecords, storage.Disk{
-					Host:          host,
-					Device:        dc.GetDevice(),
-					Size:          comm.DISK_DEFAULT_NULL_SIZE,
-					URI:           comm.DISK_DEFAULT_NULL_URI,
-					MountPoint:    dc.GetMountPoint(),
-					FormatPercent: dc.GetFormatPercent(),
-					ChunkServerID: comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+					Host:               host,
+					Device:             dc.GetDevice(),
+					Size:               comm.DISK_DEFAULT_NULL_SIZE,
+					URI:                comm.DISK_DEFAULT_NULL_URI,
+					MountPoint:         dc.GetMountPoint(),
+					ContainerImage:     dc.GetContainerImage(),
+					FormatPercent:      dc.GetFormatPercent(),
+					ChunkServerID:      comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+					ServiceMountDevice: serviceMountDevice,
 				})
 		}
 	}
 
 	for _, dr := range oldDiskRecords {
-		key := strings.Join([]string{dr.Host, dr.Device}, keySep)
+		key := strings.Join([]string{dr.Host, dr.Device}, HOST_DEVICE_SEP)
 		if _, ok := newDiskMap[key]; !ok {
 			diskRecordDeleteList = append(diskRecordDeleteList, dr)
 		}
@@ -127,19 +135,16 @@ func assambleNewDiskRecords(dcs []*disks.DiskConfig,
 }
 
 func writeDiskRecord(dr storage.Disk, curveadm *cli.CurveAdm) error {
-	if diskRecords, err := curveadm.Storage().GetDisk(
-		common.DISK_FILTER_DEVICE, dr.Host, dr.Device); err != nil {
+	if err := curveadm.Storage().SetDisk(
+		dr.Host,
+		dr.Device,
+		dr.MountPoint,
+		dr.ContainerImage,
+		dr.FormatPercent,
+		dr.ServiceMountDevice); err != nil {
 		return err
-	} else if len(diskRecords) == 0 {
-		if err := curveadm.Storage().SetDisk(
-			dr.Host,
-			dr.Device,
-			dr.MountPoint,
-			dr.ContainerImage,
-			dr.FormatPercent); err != nil {
-			return err
-		}
 	}
+
 	return nil
 }
 
@@ -153,15 +158,17 @@ func syncDiskRecords(data string, dcs []*disks.DiskConfig,
 	oldDiskRecordsString := tui.FormatDisks(oldDiskRecords)
 	newDiskRecordsString := tui.FormatDisks(newDiskRecords)
 
-	if !options.slient {
-		diff := utils.Diff(oldDiskRecordsString, newDiskRecordsString)
-		curveadm.WriteOutln(diff)
-	}
+	if len(newDiskRecords) != len(oldDiskRecords) {
+		if !options.slient {
+			diff := utils.Diff(oldDiskRecordsString, newDiskRecordsString)
+			curveadm.WriteOutln(diff)
+		}
 
-	pass := tuicomm.ConfirmYes("Disk changes are showing above. Do you want to continue?")
-	if !pass {
-		curveadm.WriteOut(tuicomm.PromptCancelOpetation("commit disk table"))
-		return errno.ERR_CANCEL_OPERATION
+		pass := tuicomm.ConfirmYes("Disk changes are showing above. Do you want to continue?")
+		if !pass {
+			curveadm.WriteOut(tuicomm.PromptCancelOpetation("commit disk table"))
+			return errno.ERR_CANCEL_OPERATION
+		}
 	}
 
 	// write new disk records

diff --git a/cli/command/format.go b/cli/command/format.go
@@ -155,9 +155,17 @@ func runFormat(curveadm *cli.CurveAdm, options formatOptions) error {
 			if err != nil {
 				return err
 			}
-			fc.UseDiskUri = true
-			chunkserverId := dr.ChunkServerID
-			if len(chunkserverId) > 1 {
+
+			fc.FromDiskRecord = true
+
+			// "ServiceMountDevice=0" means write disk UUID into /etc/fstab for host mounting.
+			// "ServiceMountDevice=1" means not to update /etc/fstab, the disk UUID will be wrote
+			// into the config of service(chunkserver) container for disk automatic mounting.
+			if dr.ServiceMountDevice != 0 {
+				fc.ServiceMountDevice = true
+			}
+
+			if dr.ChunkServerID != comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID {
 				// skip formatting the disk with nonempty chunkserver id
 				continue
 			}

diff --git a/configs/bs/cluster/disks.yaml b/configs/bs/cluster/disks.yaml
@@ -1,23 +1,24 @@
 global:
   format_percent: 95
   container_image: opencurvedocker/curvebs:v1.2
+  service_mount_device: false # disk device will be mounted by service(chunkserver) container of curvebs if set "true", default "false"
   host:
     - curve-1
     - curve-2
     - curve-3
 
 disk:
-  - device: /dev/sdb1
-    mount: /data/chunkserver0
+  - device: /dev/sdb1 # disk device path
+    mount: /data/chunkserver0 # mount point for disk formatting, consistent with the "data_dir" field fo topology chunkserver service
   - device: /dev/sdc1
     mount: /data/chunkserver1
-    format_percent: 90
+    format_percent: 90 # use a different value of disk format percent
   - device: /dev/sdd1
     mount: /data/chunkserver2
-    exclude:  # for the use case that some hosts have not certain disk device
+    exclude:  # for the use case that disk device does not exist in some hosts
       - curve-3
   - device: /dev/sde1
     mount: /data/chunkserver3
-    host:
+    host:    # override global host config, for the use case tht disk device only exists in some hosts
       - curve-1
       - curve-2
diff --git a/internal/common/common.go b/internal/common/common.go
@@ -57,8 +57,8 @@ const (
 	DISK_FILTER_MOUNT                 = "mount"
 	DISK_FILTER_SERVICE               = "service"
 	DISK_EXCLUDE_HOST                 = "exclude"
-
-	DISK_FORMAT_PERCENT = "format_percent"
+	DISK_SERVICE_MOUNT_DEVICE         = "service_mount_device"
+	DISK_FORMAT_PERCENT               = "format_percent"
 
 	DISK_FORMAT_MOUNT_POINT     = "mount"
 	DISK_FORMAT_CONTAINER_IMAGE = "container_image"

diff --git a/internal/configure/disks/dc_get.go b/internal/configure/disks/dc_get.go
@@ -55,6 +55,14 @@ func (dc *DiskConfig) getInt(i *comm.Item) int {
 	return v.(int)
 }
 
+func (hc *DiskConfig) getBool(i *comm.Item) bool {
+	v := hc.get(i)
+	if v == nil {
+		return false
+	}
+	return v.(bool)
+}
+
 func (dc *DiskConfig) getStrSlice(i *comm.Item) []string {
 	v := dc.get(i)
 	if v == nil {
@@ -65,6 +73,7 @@ func (dc *DiskConfig) getStrSlice(i *comm.Item) []string {
 
 func (dc *DiskConfig) GetContainerImage() string { return dc.getString(CONFIG_GLOBAL_CONTAINER_IMAGE) }
 func (dc *DiskConfig) GetFormatPercent() int     { return dc.getInt(CONFIG_GLOBAL_FORMAT_PERCENT) }
+func (dc *DiskConfig) GetServiceMount() bool     { return dc.getBool(CONFIG_GLOBAL_SERVICE_MOUNT_DEVICE) }
 func (dc *DiskConfig) GetHost() []string         { return dc.getStrSlice(CONFIG_GLOBAL_HOST) }
 func (dc *DiskConfig) GetDevice() string         { return dc.getString(CONFIG_DISK_DEVICE) }
 func (dc *DiskConfig) GetMountPoint() string     { return dc.getString(CONFIG_DISK_MOUNT_POINT) }

diff --git a/internal/configure/disks/dc_item.go b/internal/configure/disks/dc_item.go
@@ -48,12 +48,20 @@ var (
 		DEFAULT_FORMAT_PERCENT,
 	)
 
+	CONFIG_GLOBAL_SERVICE_MOUNT_DEVICE = itemset.Insert(
+		common.DISK_SERVICE_MOUNT_DEVICE,
+		comm.REQUIRE_BOOL,
+		false,
+		false,
+	)
+
 	CONFIG_GLOBAL_HOST = itemset.Insert(
 		common.DISK_FILTER_HOST,
 		comm.REQUIRE_SLICE,
 		false,
 		nil,
 	)
+
 	CONFIG_DISK_DEVICE = itemset.Insert(
 		common.DISK_FILTER_DEVICE,
 		comm.REQUIRE_STRING,

diff --git a/internal/configure/disks/disks.go b/internal/configure/disks/disks.go
@@ -110,17 +110,27 @@ func GenDiskURI(proto, uri string) string {
 	return strings.Join([]string{proto, uri}, DISK_URI_SEP)
 }
 
-func GetDiskId(disk storage.Disk) (string, error) {
-	uriSlice := strings.Split(disk.URI, DISK_URI_SEP)
-	if len(uriSlice) == 0 {
-		return "", errno.ERR_INVALID_DISK_URI.
-			F("The disk[%s:%s] URI[%s] is invalid", disk.Host, disk.Device, disk.URI)
+func returnInvalidDiskUriError(disk storage.Disk) error {
+	return errno.ERR_INVALID_DISK_URI.
+		F("The URI[%s] of disk[%s:%s] is invalid", disk.Host, disk.Device, disk.URI)
+}
+
+func GetDiskId(disk storage.Disk) (diskId, diskUriProto string, err error) {
+	// valide disk uri:
+	// 1. fs:uuid//8035a617-72ec-4c06-8719-8aca79234ef9
+	// 2. (not implemented) maybe "nvme:pci//00:00.1"
+	diskUriComponants := strings.Split(disk.URI, DISK_URI_SEP)
+	if len(diskUriComponants) < 2 {
+		return "", diskUriProto, returnInvalidDiskUriError(disk)
 	}
 
-	if uriSlice[0] == DISK_URI_PROTO_FS_UUID {
-		return uriSlice[1], nil
+	diskUriProto = diskUriComponants[0]
+	switch diskUriProto {
+	case DISK_URI_PROTO_FS_UUID:
+		return diskUriComponants[1], diskUriProto, nil
+	default:
+		return "", diskUriProto, returnInvalidDiskUriError(disk)
 	}
-	return "", nil
 }
 
 func (dc *DiskConfig) Build() error {

diff --git a/internal/configure/format.go b/internal/configure/format.go
@@ -46,12 +46,13 @@ const (
  */
 type (
 	FormatConfig struct {
-		ContainerIamge string
-		Host           string
-		Device         string
-		MountPoint     string
-		FormtPercent   int
-		UseDiskUri     bool
+		ContainerIamge     string
+		Host               string
+		Device             string
+		MountPoint         string
+		FormtPercent       int
+		FromDiskRecord     bool
+		ServiceMountDevice bool
 	}
 
 	Format struct {

diff --git a/internal/errno/errno.go b/internal/errno/errno.go
@@ -543,7 +543,7 @@ var (
 	ERR_START_CRONTAB_IN_CONTAINER_FAILED = EC(690000, "start crontab in container failed")
 
 	// 800: deploy
-	ERR_DISK_DEVICE_NOT_FORMATTED = EC(800000, "disk device not formatted")
+	ERR_DISK_DEVICE_NOT_FORMATTED = EC(800000, "disk device is unformatted")
 
 	// 900: others
 	ERR_CANCEL_OPERATION = EC(CODE_CANCEL_OPERATION, "cancel operation")

diff --git a/internal/storage/sql.go b/internal/storage/sql.go
@@ -71,8 +71,9 @@ var (
 			uri TEXT NOT NULL,
 			disk_format_mount_point TEXT NOT NULL,
 			format_percent TEXT NOT NULL,
-			container_image_location TEXT NOT NULL,
+			container_image TEXT NOT NULL,
 			chunkserver_id TEXT NOT NULL,
+			service_mount_device INTEGER DEFAULT 0,
 			lastmodified_time DATE NOT NULL
 		)
 	`
@@ -175,13 +176,15 @@ var (
 		uri,
 		disk_format_mount_point,
 		format_percent,
-		container_image_location,
+		container_image,
 		chunkserver_id,
+		service_mount_device,
 		lastmodified_time
-		) VALUES(?, ?, ?, ?, ?, ?, ?, ?, datetime('now','localtime'))`
+		) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now','localtime'))`
 
 	SET_DISK = `UPDATE disk SET disk_format_mount_point = ?, format_percent = ?,
-	container_image_location = ?,lastmodified_time = datetime('now','localtime') WHERE id = ?`
+				container_image = ?, service_mount_device = ?,
+				lastmodified_time = datetime('now','localtime') WHERE id = ?`
 
 	SET_DISK_URI = `UPDATE disk SET uri = ?,
 		lastmodified_time = datetime('now','localtime') WHERE host = ? AND device = ?`

diff --git a/internal/storage/storage.go b/internal/storage/storage.go
@@ -52,16 +52,17 @@ type Disks struct {
 }
 
 type Disk struct {
-	Id               int
-	Host             string
-	Device           string
-	Size             string
-	URI              string
-	MountPoint       string
-	FormatPercent    int
-	ContainerImage   string
-	ChunkServerID    string
-	LastmodifiedTime time.Time
+	Id                 int
+	Host               string
+	Device             string
+	Size               string
+	URI                string
+	MountPoint         string
+	FormatPercent      int
+	ContainerImage     string
+	ChunkServerID      string
+	ServiceMountDevice int
+	LastmodifiedTime   time.Time
 }
 
 type Cluster struct {
@@ -301,11 +302,13 @@ func (s *Storage) GetDisks() ([]Disks, error) {
 }
 
 // disk
-func (s *Storage) SetDisk(host, device, mount, containerImage string, formatPercent int) error {
+func (s *Storage) SetDisk(host, device, mount, containerImage string,
+	formatPercent, serviceMountDevice int) error {
 	diskRecords, err := s.GetDisk(SELECT_DISK_BY_DEVICE_PATH, host, device)
 	if err != nil {
 		return err
-	} else if len(diskRecords) == 0 {
+	}
+	if len(diskRecords) == 0 {
 		return s.execSQL(
 			INSERT_DISK,
 			host,
@@ -316,9 +319,16 @@ func (s *Storage) SetDisk(host, device, mount, containerImage string, formatPerc
 			formatPercent,
 			containerImage,
 			comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+			serviceMountDevice,
 		)
 	}
-	return s.execSQL(SET_DISK, mount, formatPercent, containerImage, diskRecords[0].Id)
+	return s.execSQL(
+		SET_DISK,
+		mount,
+		formatPercent,
+		containerImage,
+		serviceMountDevice,
+		diskRecords[0].Id)
 }
 
 func (s *Storage) UpdateDiskURI(host, device, devUri string) error {
@@ -413,6 +423,7 @@ func (s *Storage) GetDisk(filter string, args ...interface{}) ([]Disk, error) {
 			&disk.FormatPercent,
 			&disk.ContainerImage,
 			&disk.ChunkServerID,
+			&disk.ServiceMountDevice,
 			&disk.LastmodifiedTime)
 		diskRecords = append(diskRecords, disk)
 	}