diff --git a/api/services_cluster.go b/api/services_cluster.go index 63a847c6c..4cc3eedc5 100644 --- a/api/services_cluster.go +++ b/api/services_cluster.go @@ -116,7 +116,7 @@ func removeClusterMember(state state.State, r *http.Request) response.Response { if s.Type() == types.MicroCeph { cephService := ceph.(*service.CephService) - disks, err := cephService.GetDisks(r.Context(), "") + disks, err := cephService.GetDisks(r.Context(), "", nil) if err != nil { return err } diff --git a/api/status.go b/api/status.go index 7196dbc2b..d486ad2da 100644 --- a/api/status.go +++ b/api/status.go @@ -160,7 +160,7 @@ func cephStatus(ctx context.Context, s service.Service) (clusterMembers []microT return nil, nil, nil, err } - disks, err := cephService.GetDisks(ctx, "") + disks, err := cephService.GetDisks(ctx, "", nil) if err != nil { return nil, nil, nil, err } diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 196678d54..2344cba7d 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -282,17 +282,6 @@ func (c *initConfig) askDisks(sh *service.Handler) error { return nil } -func parseDiskPath(disk api.ResourcesStorageDisk) string { - devicePath := "/dev/" + disk.ID - if disk.DeviceID != "" { - devicePath = "/dev/disk/by-id/" + disk.DeviceID - } else if disk.DevicePath != "" { - devicePath = "/dev/disk/by-path/" + disk.DevicePath - } - - return devicePath -} - func (c *initConfig) askLocalPool(sh *service.Handler) error { useJoinConfig := false askSystems := map[string]bool{} @@ -313,15 +302,21 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { availableDisks := map[string]map[string]api.ResourcesStorageDisk{} for name, state := range c.state { + // Skip this system if it already has the local storage pool configured + // and isn't marked for disk selection. + // This ensures that when adding new systems to the cluster the existing ones + // don't have to show any disk because they are probably already used for either local or remote storage. + if !askSystems[name] { + continue + } + if len(state.AvailableDisks) == 0 { logger.Infof("Skipping local storage pool creation, peer %q has too few disks", name) return nil } - if askSystems[name] { - availableDisks[name] = state.AvailableDisks - } + availableDisks[name] = state.AvailableDisks } // Local storage is already set up on every system, or if not every system has a disk. @@ -340,11 +335,11 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { } sort.Slice(sortedDisks, func(i, j int) bool { - return parseDiskPath(sortedDisks[i]) < parseDiskPath(sortedDisks[j]) + return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j]) }) for _, disk := range sortedDisks { - devicePath := parseDiskPath(disk) + devicePath := service.FormatDiskPath(disk) data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath}) } } @@ -469,7 +464,7 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { for target, path := range selectedDisks { newAvailableDisks[target] = map[string]api.ResourcesStorageDisk{} for id, disk := range availableDisks[target] { - if parseDiskPath(disk) != path { + if service.FormatDiskPath(disk) != path { newAvailableDisks[target][id] = disk } } @@ -683,12 +678,12 @@ func (c *initConfig) askRemotePool(sh *service.Handler) error { // Ensure the list of disks is sorted by name. sort.Slice(sortedDisks, func(i, j int) bool { - return parseDiskPath(sortedDisks[i]) < parseDiskPath(sortedDisks[j]) + return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j]) }) for _, disk := range sortedDisks { // Skip any disks that have been reserved for the local storage pool. - devicePath := parseDiskPath(disk) + devicePath := service.FormatDiskPath(disk) data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath}) } } diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index f610b1847..4298eaf0a 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -794,7 +794,7 @@ func (c *initConfig) setupCluster(s *service.Handler) error { cephService := s.Services[types.MicroCeph].(*service.CephService) - allDisks, err := cephService.GetDisks(context.Background(), s.Name) + allDisks, err := cephService.GetDisks(context.Background(), "", nil) if err != nil { return err } diff --git a/cmd/microcloud/preseed.go b/cmd/microcloud/preseed.go index 7cadbf204..7f4a0be32 100644 --- a/cmd/microcloud/preseed.go +++ b/cmd/microcloud/preseed.go @@ -965,7 +965,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map system.MicroCephDisks = append( system.MicroCephDisks, cephTypes.DisksPost{ - Path: []string{parseDiskPath(disk)}, + Path: []string{service.FormatDiskPath(disk)}, Wipe: filter.Wipe, Encrypt: filter.Encrypt, }, @@ -1040,12 +1040,12 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map if len(matched) > 0 { zfsMachines[peer] = true if c.bootstrap { - system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, parseDiskPath(matched[0]))) + system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, service.FormatDiskPath(matched[0]))) if s.Name == peer { system.StoragePools = append(system.StoragePools, lxd.DefaultZFSStoragePool()) } } else { - system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, parseDiskPath(matched[0]))...) + system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, service.FormatDiskPath(matched[0]))...) } zfsMatches[filter.Find] = zfsMatches[filter.Find] + 1 diff --git a/service/microceph.go b/service/microceph.go index 097aace66..05dfb02af 100644 --- a/service/microceph.go +++ b/service/microceph.go @@ -165,10 +165,25 @@ func (s CephService) GetServices(ctx context.Context, target string) (cephTypes. } // GetDisks returns the list of configured disks. -func (s CephService) GetDisks(ctx context.Context, target string) (cephTypes.Disks, error) { - c, err := s.Client(target) - if err != nil { - return nil, err +func (s CephService) GetDisks(ctx context.Context, target string, cert *x509.Certificate) (cephTypes.Disks, error) { + var c *client.Client + var err error + + if target == "" { + c, err = s.Client("") + if err != nil { + return nil, err + } + } else { + c, err = s.remoteClient(cert, target) + if err != nil { + return nil, err + } + + c, err = cloudClient.UseAuthProxy(c, types.MicroCeph, cloudClient.AuthConfig{}) + if err != nil { + return nil, err + } } disks := cephTypes.Disks{} diff --git a/service/system_information.go b/service/system_information.go index 4778fd5e1..42920889c 100644 --- a/service/system_information.go +++ b/service/system_information.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/canonical/lxd/shared/api" + cephTypes "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microcloud/microcloud/api/types" "github.com/canonical/microcloud/microcloud/multicast" @@ -100,9 +101,28 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul return nil, fmt.Errorf("Failed to get system resources of peer %q: %w", s.ClusterName, err) } + var microceph *CephService + + // Fetch disks which are already used for remote storage. + var usedCephDisks cephTypes.Disks + if len(s.ExistingServices[types.MicroCeph]) > 0 { + microceph = sh.Services[types.MicroCeph].(*CephService) + + if localSystem { + usedCephDisks, err = microceph.GetDisks(ctx, "", nil) + } else { + usedCephDisks, err = microceph.GetDisks(ctx, s.ClusterAddress, connectInfo.Certificate) + } + + if err != nil && !api.StatusErrorCheck(err, http.StatusServiceUnavailable) { + return nil, fmt.Errorf("Failed to get Ceph disks on %q: %w", s.ClusterName, err) + } + } + if allResources != nil { for _, disk := range allResources.Storage.Disks { // Exclude non-pristine disks with partitions. + // Disks already used for local storage (zfs) contain a partition and are therefore excluded by this check. if len(disk.Partitions) != 0 { continue } @@ -112,6 +132,18 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul continue } + // Exclude disks which are already used for remote storage. + diskUsed := false + for _, usedCephDisk := range usedCephDisks { + if usedCephDisk.Path == FormatDiskPath(disk) && usedCephDisk.Location == connectInfo.Name { + diskUsed = true + } + } + + if diskUsed { + continue + } + s.AvailableDisks[disk.ID] = disk } } @@ -167,8 +199,6 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul } if len(s.ExistingServices[types.MicroCeph]) > 0 { - microceph := sh.Services[types.MicroCeph].(*CephService) - if localSystem { s.CephConfig, err = microceph.ClusterConfig(ctx, "", nil) } else { @@ -351,3 +381,15 @@ func ClustersConflict(systems map[string]SystemInformation, services map[types.S return false, "" } + +// FormatDiskPath returns a disk's path representation. +func FormatDiskPath(disk api.ResourcesStorageDisk) string { + devicePath := "/dev/" + disk.ID + if disk.DeviceID != "" { + devicePath = "/dev/disk/by-id/" + disk.DeviceID + } else if disk.DevicePath != "" { + devicePath = "/dev/disk/by-path/" + disk.DevicePath + } + + return devicePath +}