Service: Remove already used disks from selection

api/services_cluster.go

@@ -116,7 +116,7 @@ func removeClusterMember(state state.State, r *http.Request) response.Response {
 		if s.Type() == types.MicroCeph {
 			cephService := ceph.(*service.CephService)
 
-			disks, err := cephService.GetDisks(r.Context(), "")
+			disks, err := cephService.GetDisks(r.Context(), "", nil)
 			if err != nil {
 				return err
 			}

api/status.go

@@ -160,7 +160,7 @@ func cephStatus(ctx context.Context, s service.Service) (clusterMembers []microT
 		return nil, nil, nil, err
 	}
 
-	disks, err := cephService.GetDisks(ctx, "")
+	disks, err := cephService.GetDisks(ctx, "", nil)
 	if err != nil {
 		return nil, nil, nil, err
 	}

cmd/microcloud/ask.go

@@ -282,17 +282,6 @@ func (c *initConfig) askDisks(sh *service.Handler) error {
 	return nil
 }
 
-func parseDiskPath(disk api.ResourcesStorageDisk) string {
-	devicePath := "/dev/" + disk.ID
-	if disk.DeviceID != "" {
-		devicePath = "/dev/disk/by-id/" + disk.DeviceID
-	} else if disk.DevicePath != "" {
-		devicePath = "/dev/disk/by-path/" + disk.DevicePath
-	}
-
-	return devicePath
-}
-
 func (c *initConfig) askLocalPool(sh *service.Handler) error {
 	useJoinConfig := false
 	askSystems := map[string]bool{}
@@ -313,15 +302,21 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error {
 
 	availableDisks := map[string]map[string]api.ResourcesStorageDisk{}
 	for name, state := range c.state {
+		// Skip this system if it already has the local storage pool configured
+		// and isn't marked for disk selection.
+		// This ensures that when adding new systems to the cluster the existing ones
+		// don't have to show any disk because they are probably already used for either local or remote storage.
+		if !askSystems[name] {
+			continue
+		}
+
 		if len(state.AvailableDisks) == 0 {
 			logger.Infof("Skipping local storage pool creation, peer %q has too few disks", name)
 
 			return nil
 		}
 
-		if askSystems[name] {
-			availableDisks[name] = state.AvailableDisks
-		}
+		availableDisks[name] = state.AvailableDisks
 	}
 
 	// Local storage is already set up on every system, or if not every system has a disk.
@@ -340,11 +335,11 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error {
 		}
 
 		sort.Slice(sortedDisks, func(i, j int) bool {
-			return parseDiskPath(sortedDisks[i]) < parseDiskPath(sortedDisks[j])
+			return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j])
 		})
 
 		for _, disk := range sortedDisks {
-			devicePath := parseDiskPath(disk)
+			devicePath := service.FormatDiskPath(disk)
 			data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath})
 		}
 	}
@@ -469,7 +464,7 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error {
 	for target, path := range selectedDisks {
 		newAvailableDisks[target] = map[string]api.ResourcesStorageDisk{}
 		for id, disk := range availableDisks[target] {
-			if parseDiskPath(disk) != path {
+			if service.FormatDiskPath(disk) != path {
 				newAvailableDisks[target][id] = disk
 			}
 		}
@@ -683,12 +678,12 @@ func (c *initConfig) askRemotePool(sh *service.Handler) error {
 
 				// Ensure the list of disks is sorted by name.
 				sort.Slice(sortedDisks, func(i, j int) bool {
-					return parseDiskPath(sortedDisks[i]) < parseDiskPath(sortedDisks[j])
+					return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j])
 				})
 
 				for _, disk := range sortedDisks {
 					// Skip any disks that have been reserved for the local storage pool.
-					devicePath := parseDiskPath(disk)
+					devicePath := service.FormatDiskPath(disk)
 					data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath})
 				}
 			}

cmd/microcloud/main_init.go

@@ -794,7 +794,7 @@ func (c *initConfig) setupCluster(s *service.Handler) error {
 
 		cephService := s.Services[types.MicroCeph].(*service.CephService)
 
-		allDisks, err := cephService.GetDisks(context.Background(), s.Name)
+		allDisks, err := cephService.GetDisks(context.Background(), "", nil)
 		if err != nil {
 			return err
 		}

cmd/microcloud/preseed.go

@@ -965,7 +965,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map
 				system.MicroCephDisks = append(
 					system.MicroCephDisks,
 					cephTypes.DisksPost{
-						Path:    []string{parseDiskPath(disk)},
+						Path:    []string{service.FormatDiskPath(disk)},
 						Wipe:    filter.Wipe,
 						Encrypt: filter.Encrypt,
 					},
@@ -1040,12 +1040,12 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map
 			if len(matched) > 0 {
 				zfsMachines[peer] = true
 				if c.bootstrap {
-					system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, parseDiskPath(matched[0])))
+					system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, service.FormatDiskPath(matched[0])))
 					if s.Name == peer {
 						system.StoragePools = append(system.StoragePools, lxd.DefaultZFSStoragePool())
 					}
 				} else {
-					system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, parseDiskPath(matched[0]))...)
+					system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, service.FormatDiskPath(matched[0]))...)
 				}
 
 				zfsMatches[filter.Find] = zfsMatches[filter.Find] + 1

service/microceph.go

@@ -165,10 +165,25 @@ func (s CephService) GetServices(ctx context.Context, target string) (cephTypes.
 }
 
 // GetDisks returns the list of configured disks.
-func (s CephService) GetDisks(ctx context.Context, target string) (cephTypes.Disks, error) {
-	c, err := s.Client(target)
-	if err != nil {
-		return nil, err
+func (s CephService) GetDisks(ctx context.Context, target string, cert *x509.Certificate) (cephTypes.Disks, error) {
+	var c *client.Client
+	var err error
+
+	if target == "" {
+		c, err = s.Client("")
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		c, err = s.remoteClient(cert, target)
+		if err != nil {
+			return nil, err
+		}
+
+		c, err = cloudClient.UseAuthProxy(c, types.MicroCeph, cloudClient.AuthConfig{})
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	disks := cephTypes.Disks{}

service/system_information.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 
 	"github.com/canonical/lxd/shared/api"
+	cephTypes "github.com/canonical/microceph/microceph/api/types"
 
 	"github.com/canonical/microcloud/microcloud/api/types"
 	"github.com/canonical/microcloud/microcloud/multicast"
@@ -100,9 +101,28 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul
 		return nil, fmt.Errorf("Failed to get system resources of peer %q: %w", s.ClusterName, err)
 	}
 
+	var microceph *CephService
+
+	// Fetch disks which are already used for remote storage.
+	var usedCephDisks cephTypes.Disks
+	if len(s.ExistingServices[types.MicroCeph]) > 0 {
+		microceph = sh.Services[types.MicroCeph].(*CephService)
+
+		if localSystem {
+			usedCephDisks, err = microceph.GetDisks(ctx, "", nil)
+		} else {
+			usedCephDisks, err = microceph.GetDisks(ctx, s.ClusterAddress, connectInfo.Certificate)
+		}
+
+		if err != nil && !api.StatusErrorCheck(err, http.StatusServiceUnavailable) {
+			return nil, fmt.Errorf("Failed to get Ceph disks on %q: %w", s.ClusterName, err)
+		}
+	}
+
 	if allResources != nil {
 		for _, disk := range allResources.Storage.Disks {
 			// Exclude non-pristine disks with partitions.
+			// Disks already used for local storage (zfs) contain a partition and are therefore excluded by this check.
 			if len(disk.Partitions) != 0 {
 				continue
 			}
@@ -112,6 +132,18 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul
 				continue
 			}
 
+			// Exclude disks which are already used for remote storage.
+			diskUsed := false
+			for _, usedCephDisk := range usedCephDisks {
+				if usedCephDisk.Path == FormatDiskPath(disk) && usedCephDisk.Location == connectInfo.Name {
+					diskUsed = true
+				}
+			}
+
+			if diskUsed {
+				continue
+			}
+
 			s.AvailableDisks[disk.ID] = disk
 		}
 	}
@@ -167,8 +199,6 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul
 	}
 
 	if len(s.ExistingServices[types.MicroCeph]) > 0 {
-		microceph := sh.Services[types.MicroCeph].(*CephService)
-
 		if localSystem {
 			s.CephConfig, err = microceph.ClusterConfig(ctx, "", nil)
 		} else {
@@ -351,3 +381,15 @@ func ClustersConflict(systems map[string]SystemInformation, services map[types.S
 
 	return false, ""
 }
+
+// FormatDiskPath returns a disk's path representation.
+func FormatDiskPath(disk api.ResourcesStorageDisk) string {
+	devicePath := "/dev/" + disk.ID
+	if disk.DeviceID != "" {
+		devicePath = "/dev/disk/by-id/" + disk.DeviceID
+	} else if disk.DevicePath != "" {
+		devicePath = "/dev/disk/by-path/" + disk.DevicePath
+	}
+
+	return devicePath
+}