Feature: support direct disk mounting for service container

Disk UUID will be passed to container `Cmd` config during curvebs deployment if disk `service_mount_device` was enabled, then service(chunkserver) cloud mount disk via `entrypoint.sh` and restart automatically after host power recover. Signed-off-by: Lijin Xiong <[email protected]>
opencurve · Mar 3, 2023 · 101f5a7 · 101f5a7
1 parent af342de
commit 101f5a7
Show file tree

Hide file tree

Showing 16 changed files with 183 additions and 82 deletions.
diff --git a/cli/command/disks/commit.go b/cli/command/disks/commit.go
@@ -27,7 +27,6 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/opencurve/curveadm/cli/cli"
-	"github.com/opencurve/curveadm/internal/common"
 	comm "github.com/opencurve/curveadm/internal/common"
 	"github.com/opencurve/curveadm/internal/configure/disks"
 	"github.com/opencurve/curveadm/internal/configure/topology"
@@ -98,20 +97,26 @@ func assambleNewDiskRecords(dcs []*disks.DiskConfig,
 	keySep := ":"
 	newDiskMap := make(map[string]bool)
 
+	var serviceMountDevice int
 	var newDiskRecords, diskRecordDeleteList []storage.Disk
 	for _, dc := range dcs {
 		for _, host := range dc.GetHost() {
 			key := strings.Join([]string{host, dc.GetDevice()}, keySep)
 			newDiskMap[key] = true
+			if dc.GetServiceMount() {
+				serviceMountDevice = 1
+			}
 			newDiskRecords = append(
 				newDiskRecords, storage.Disk{
-					Host:          host,
-					Device:        dc.GetDevice(),
-					Size:          comm.DISK_DEFAULT_NULL_SIZE,
-					URI:           comm.DISK_DEFAULT_NULL_URI,
-					MountPoint:    dc.GetMountPoint(),
-					FormatPercent: dc.GetFormatPercent(),
-					ChunkServerID: comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+					Host:               host,
+					Device:             dc.GetDevice(),
+					Size:               comm.DISK_DEFAULT_NULL_SIZE,
+					URI:                comm.DISK_DEFAULT_NULL_URI,
+					MountPoint:         dc.GetMountPoint(),
+					ContainerImage:     dc.GetContainerImage(),
+					FormatPercent:      dc.GetFormatPercent(),
+					ChunkServerID:      comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+					ServiceMountDevice: serviceMountDevice,
 				})
 		}
 	}
@@ -127,19 +132,16 @@ func assambleNewDiskRecords(dcs []*disks.DiskConfig,
 }
 
 func writeDiskRecord(dr storage.Disk, curveadm *cli.CurveAdm) error {
-	if diskRecords, err := curveadm.Storage().GetDisk(
-		common.DISK_FILTER_DEVICE, dr.Host, dr.Device); err != nil {
+	if err := curveadm.Storage().SetDisk(
+		dr.Host,
+		dr.Device,
+		dr.MountPoint,
+		dr.ContainerImage,
+		dr.FormatPercent,
+		dr.ServiceMountDevice); err != nil {
 		return err
-	} else if len(diskRecords) == 0 {
-		if err := curveadm.Storage().SetDisk(
-			dr.Host,
-			dr.Device,
-			dr.MountPoint,
-			dr.ContainerImage,
-			dr.FormatPercent); err != nil {
-			return err
-		}
 	}
+
 	return nil
 }
 
@@ -153,15 +155,17 @@ func syncDiskRecords(data string, dcs []*disks.DiskConfig,
 	oldDiskRecordsString := tui.FormatDisks(oldDiskRecords)
 	newDiskRecordsString := tui.FormatDisks(newDiskRecords)
 
-	if !options.slient {
-		diff := utils.Diff(oldDiskRecordsString, newDiskRecordsString)
-		curveadm.WriteOutln(diff)
-	}
+	if len(newDiskRecords) != len(oldDiskRecords) {
+		if !options.slient {
+			diff := utils.Diff(oldDiskRecordsString, newDiskRecordsString)
+			curveadm.WriteOutln(diff)
+		}
 
-	pass := tuicomm.ConfirmYes("Disk changes are showing above. Do you want to continue?")
-	if !pass {
-		curveadm.WriteOut(tuicomm.PromptCancelOpetation("commit disk table"))
-		return errno.ERR_CANCEL_OPERATION
+		pass := tuicomm.ConfirmYes("Disk changes are showing above. Do you want to continue?")
+		if !pass {
+			curveadm.WriteOut(tuicomm.PromptCancelOpetation("commit disk table"))
+			return errno.ERR_CANCEL_OPERATION
+		}
 	}
 
 	// write new disk records

diff --git a/cli/command/format.go b/cli/command/format.go
@@ -156,6 +156,9 @@ func runFormat(curveadm *cli.CurveAdm, options formatOptions) error {
 				return err
 			}
 			fc.UseDiskUri = true
+			if dr.ServiceMountDevice != 0 {
+				fc.ServiceMountDevice = true
+			}
 			chunkserverId := dr.ChunkServerID
 			if len(chunkserverId) > 1 {
 				// skip formatting the disk with nonempty chunkserver id

diff --git a/configs/bs/cluster/disks.yaml b/configs/bs/cluster/disks.yaml
@@ -1,6 +1,7 @@
 global:
   format_percent: 95
   container_image: opencurvedocker/curvebs:v1.2
+  service_mount_device: false # disk device will be mounted by service containers of curvebs if set "true", default "false"
   host:
     - curve-1
     - curve-2

diff --git a/internal/common/common.go b/internal/common/common.go
@@ -57,8 +57,8 @@ const (
 	DISK_FILTER_MOUNT                 = "mount"
 	DISK_FILTER_SERVICE               = "service"
 	DISK_EXCLUDE_HOST                 = "exclude"
-
-	DISK_FORMAT_PERCENT = "format_percent"
+	DISK_SERVICE_MOUNT_DEVICE         = "service_mount_device"
+	DISK_FORMAT_PERCENT               = "format_percent"
 
 	DISK_FORMAT_MOUNT_POINT     = "mount"
 	DISK_FORMAT_CONTAINER_IMAGE = "container_image"

diff --git a/internal/configure/disks/dc_get.go b/internal/configure/disks/dc_get.go
@@ -55,6 +55,14 @@ func (dc *DiskConfig) getInt(i *comm.Item) int {
 	return v.(int)
 }
 
+func (hc *DiskConfig) getBool(i *comm.Item) bool {
+	v := hc.get(i)
+	if v == nil {
+		return false
+	}
+	return v.(bool)
+}
+
 func (dc *DiskConfig) getStrSlice(i *comm.Item) []string {
 	v := dc.get(i)
 	if v == nil {
@@ -65,6 +73,7 @@ func (dc *DiskConfig) getStrSlice(i *comm.Item) []string {
 
 func (dc *DiskConfig) GetContainerImage() string { return dc.getString(CONFIG_GLOBAL_CONTAINER_IMAGE) }
 func (dc *DiskConfig) GetFormatPercent() int     { return dc.getInt(CONFIG_GLOBAL_FORMAT_PERCENT) }
+func (dc *DiskConfig) GetServiceMount() bool     { return dc.getBool(CONFIG_GLOBAL_SERVICE_MOUNT) }
 func (dc *DiskConfig) GetHost() []string         { return dc.getStrSlice(CONFIG_GLOBAL_HOST) }
 func (dc *DiskConfig) GetDevice() string         { return dc.getString(CONFIG_DISK_DEVICE) }
 func (dc *DiskConfig) GetMountPoint() string     { return dc.getString(CONFIG_DISK_MOUNT_POINT) }

diff --git a/internal/configure/disks/dc_item.go b/internal/configure/disks/dc_item.go
@@ -48,12 +48,20 @@ var (
 		DEFAULT_FORMAT_PERCENT,
 	)
 
+	CONFIG_GLOBAL_SERVICE_MOUNT = itemset.Insert(
+		common.DISK_SERVICE_MOUNT_DEVICE,
+		comm.REQUIRE_BOOL,
+		false,
+		false,
+	)
+
 	CONFIG_GLOBAL_HOST = itemset.Insert(
 		common.DISK_FILTER_HOST,
 		comm.REQUIRE_SLICE,
 		false,
 		nil,
 	)
+
 	CONFIG_DISK_DEVICE = itemset.Insert(
 		common.DISK_FILTER_DEVICE,
 		comm.REQUIRE_STRING,

diff --git a/internal/configure/disks/disks.go b/internal/configure/disks/disks.go
@@ -110,17 +110,17 @@ func GenDiskURI(proto, uri string) string {
 	return strings.Join([]string{proto, uri}, DISK_URI_SEP)
 }
 
-func GetDiskId(disk storage.Disk) (string, error) {
+func GetDiskId(disk storage.Disk) (string, string, error) {
 	uriSlice := strings.Split(disk.URI, DISK_URI_SEP)
 	if len(uriSlice) == 0 {
-		return "", errno.ERR_INVALID_DISK_URI.
+		return "", "", errno.ERR_INVALID_DISK_URI.
 			F("The disk[%s:%s] URI[%s] is invalid", disk.Host, disk.Device, disk.URI)
 	}
 
 	if uriSlice[0] == DISK_URI_PROTO_FS_UUID {
-		return uriSlice[1], nil
+		return uriSlice[1], DISK_URI_PROTO_FS_UUID, nil
 	}
-	return "", nil
+	return "", "", nil
 }
 
 func (dc *DiskConfig) Build() error {

diff --git a/internal/configure/format.go b/internal/configure/format.go
@@ -46,12 +46,13 @@ const (
  */
 type (
 	FormatConfig struct {
-		ContainerIamge string
-		Host           string
-		Device         string
-		MountPoint     string
-		FormtPercent   int
-		UseDiskUri     bool
+		ContainerIamge     string
+		Host               string
+		Device             string
+		MountPoint         string
+		FormtPercent       int
+		UseDiskUri         bool
+		ServiceMountDevice bool
 	}
 
 	Format struct {

diff --git a/internal/errno/errno.go b/internal/errno/errno.go
@@ -543,7 +543,7 @@ var (
 	ERR_START_CRONTAB_IN_CONTAINER_FAILED = EC(690000, "start crontab in container failed")
 
 	// 800: deploy
-	ERR_DISK_DEVICE_NOT_FORMATTED = EC(800000, "disk device not formatted")
+	ERR_DISK_DEVICE_NOT_FORMATTED = EC(800000, "disk device is unformatted")
 
 	// 900: others
 	ERR_CANCEL_OPERATION = EC(CODE_CANCEL_OPERATION, "cancel operation")

diff --git a/internal/storage/sql.go b/internal/storage/sql.go
@@ -71,8 +71,9 @@ var (
 			uri TEXT NOT NULL,
 			disk_format_mount_point TEXT NOT NULL,
 			format_percent TEXT NOT NULL,
-			container_image_location TEXT NOT NULL,
+			container_image TEXT NOT NULL,
 			chunkserver_id TEXT NOT NULL,
+			service_mount_device INTEGER DEFAULT 0,
 			lastmodified_time DATE NOT NULL
 		)
 	`
@@ -175,13 +176,15 @@ var (
 		uri,
 		disk_format_mount_point,
 		format_percent,
-		container_image_location,
+		container_image,
 		chunkserver_id,
+		service_mount_device,
 		lastmodified_time
-		) VALUES(?, ?, ?, ?, ?, ?, ?, ?, datetime('now','localtime'))`
+		) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now','localtime'))`
 
-	SET_DISK = `UPDATE disk SET disk_format_mount_point = ?, format_percent = ?,
-	container_image_location = ?,lastmodified_time = datetime('now','localtime') WHERE id = ?`
+	SET_DISK = `UPDATE disk SET disk_format_mount_point = ?, 
+				format_percent = ?,	container_image = ?, service_mount_device = ?,
+				lastmodified_time = datetime('now','localtime') WHERE id = ?`
 
 	SET_DISK_URI = `UPDATE disk SET uri = ?,
 		lastmodified_time = datetime('now','localtime') WHERE host = ? AND device = ?`

diff --git a/internal/storage/storage.go b/internal/storage/storage.go
@@ -52,16 +52,17 @@ type Disks struct {
 }
 
 type Disk struct {
-	Id               int
-	Host             string
-	Device           string
-	Size             string
-	URI              string
-	MountPoint       string
-	FormatPercent    int
-	ContainerImage   string
-	ChunkServerID    string
-	LastmodifiedTime time.Time
+	Id                 int
+	Host               string
+	Device             string
+	Size               string
+	URI                string
+	MountPoint         string
+	FormatPercent      int
+	ContainerImage     string
+	ChunkServerID      string
+	ServiceMountDevice int
+	LastmodifiedTime   time.Time
 }
 
 type Cluster struct {
@@ -301,7 +302,8 @@ func (s *Storage) GetDisks() ([]Disks, error) {
 }
 
 // disk
-func (s *Storage) SetDisk(host, device, mount, containerImage string, formatPercent int) error {
+func (s *Storage) SetDisk(host, device, mount, containerImage string,
+	formatPercent, service_mount_device int) error {
 	diskRecords, err := s.GetDisk(SELECT_DISK_BY_DEVICE_PATH, host, device)
 	if err != nil {
 		return err
@@ -316,9 +318,16 @@ func (s *Storage) SetDisk(host, device, mount, containerImage string, formatPerc
 			formatPercent,
 			containerImage,
 			comm.DISK_DEFAULT_NULL_CHUNKSERVER_ID,
+			service_mount_device,
 		)
 	}
-	return s.execSQL(SET_DISK, mount, formatPercent, containerImage, diskRecords[0].Id)
+	return s.execSQL(
+		SET_DISK,
+		mount,
+		formatPercent,
+		containerImage,
+		service_mount_device,
+		diskRecords[0].Id)
 }
 
 func (s *Storage) UpdateDiskURI(host, device, devUri string) error {
@@ -413,6 +422,7 @@ func (s *Storage) GetDisk(filter string, args ...interface{}) ([]Disk, error) {
 			&disk.FormatPercent,
 			&disk.ContainerImage,
 			&disk.ChunkServerID,
+			&disk.ServiceMountDevice,
 			&disk.LastmodifiedTime)
 		diskRecords = append(diskRecords, disk)
 	}

diff --git a/internal/task/task/bs/format.go b/internal/task/task/bs/format.go
@@ -287,13 +287,17 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
 		Directory:   mountPoint,
 		ExecOptions: curveadm.ExecOptions(),
 	})
-	t.AddStep(&step2EditFSTab{
-		host:       host,
-		device:     device,
-		oldUUID:    &oldUUID,
-		mountPoint: mountPoint,
-		curveadm:   curveadm,
-	})
+
+	if !fc.ServiceMountDevice {
+		t.AddStep(&step2EditFSTab{
+			host:       host,
+			device:     device,
+			oldUUID:    &oldUUID,
+			mountPoint: mountPoint,
+			curveadm:   curveadm,
+		})
+	}
+
 	if fc.UseDiskUri {
 		t.AddStep(&step2UpdateDiskSizeUri{
 			host:     host,

diff --git a/internal/task/task/bs/format_stop.go b/internal/task/task/bs/format_stop.go
@@ -92,14 +92,16 @@ func NewStopFormatTask(curveadm *cli.CurveAdm, fc *configure.FormatConfig) (*tas
 		Out:         &oldUUID,
 		ExecOptions: curveadm.ExecOptions(),
 	})
-	t.AddStep(&step2EditFSTab{
-		host:       host,
-		device:     device,
-		oldUUID:    &oldUUID,
-		mountPoint: mountPoint,
-		curveadm:   curveadm,
-		skipAdd:    true,
-	})
+	if !fc.ServiceMountDevice {
+		t.AddStep(&step2EditFSTab{
+			host:       host,
+			device:     device,
+			oldUUID:    &oldUUID,
+			mountPoint: mountPoint,
+			curveadm:   curveadm,
+			skipAdd:    true,
+		})
+	}
 
 	// 2: list container id and add step to task
 	t.AddStep(&step.ListContainers{