From 34543a32cc927314994e32601a04cd50563c5ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Mon, 7 Jul 2025 16:31:25 +0200 Subject: [PATCH 1/7] service/microceph: Allow fetching disks using temporary trust store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow passing a certificate which allows reaching out to the remote MicroCeph (through MicroCloud) using the temporary trust store to fetch the list of already used disks. Signed-off-by: Julian Pelizäus --- service/microceph.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/service/microceph.go b/service/microceph.go index 097aace66..05dfb02af 100644 --- a/service/microceph.go +++ b/service/microceph.go @@ -165,10 +165,25 @@ func (s CephService) GetServices(ctx context.Context, target string) (cephTypes. } // GetDisks returns the list of configured disks. -func (s CephService) GetDisks(ctx context.Context, target string) (cephTypes.Disks, error) { - c, err := s.Client(target) - if err != nil { - return nil, err +func (s CephService) GetDisks(ctx context.Context, target string, cert *x509.Certificate) (cephTypes.Disks, error) { + var c *client.Client + var err error + + if target == "" { + c, err = s.Client("") + if err != nil { + return nil, err + } + } else { + c, err = s.remoteClient(cert, target) + if err != nil { + return nil, err + } + + c, err = cloudClient.UseAuthProxy(c, types.MicroCeph, cloudClient.AuthConfig{}) + if err != nil { + return nil, err + } } disks := cephTypes.Disks{} From 46ac6833d0a275033d0b0e6039b1b922e8e6a9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Mon, 7 Jul 2025 16:32:54 +0200 Subject: [PATCH 2/7] api: Update invocations of MicroCeph's GetDisk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass nil as the certificate as the cluster is already formed at this stage and we don't anymore need to rely on the temporary trust store but can instead communicate with the local MicroCeph directly. Signed-off-by: Julian Pelizäus --- api/services_cluster.go | 2 +- api/status.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/services_cluster.go b/api/services_cluster.go index 63a847c6c..4cc3eedc5 100644 --- a/api/services_cluster.go +++ b/api/services_cluster.go @@ -116,7 +116,7 @@ func removeClusterMember(state state.State, r *http.Request) response.Response { if s.Type() == types.MicroCeph { cephService := ceph.(*service.CephService) - disks, err := cephService.GetDisks(r.Context(), "") + disks, err := cephService.GetDisks(r.Context(), "", nil) if err != nil { return err } diff --git a/api/status.go b/api/status.go index 7196dbc2b..d486ad2da 100644 --- a/api/status.go +++ b/api/status.go @@ -160,7 +160,7 @@ func cephStatus(ctx context.Context, s service.Service) (clusterMembers []microT return nil, nil, nil, err } - disks, err := cephService.GetDisks(ctx, "") + disks, err := cephService.GetDisks(ctx, "", nil) if err != nil { return nil, nil, nil, err } From 9866d7b59aeb5e7d8e40f654fe07987f7989fc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Mon, 7 Jul 2025 16:33:16 +0200 Subject: [PATCH 3/7] cmd/microcloud/init: Update invocations of MicroCeph's GetDisk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass nil as the certificate as the cluster is already formed at this stage and we don't anymore need to rely on the temporary trust store but can instead communicate with the local MicroCeph directly. Signed-off-by: Julian Pelizäus --- cmd/microcloud/main_init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index f610b1847..4298eaf0a 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -794,7 +794,7 @@ func (c *initConfig) setupCluster(s *service.Handler) error { cephService := s.Services[types.MicroCeph].(*service.CephService) - allDisks, err := cephService.GetDisks(context.Background(), s.Name) + allDisks, err := cephService.GetDisks(context.Background(), "", nil) if err != nil { return err } From 7a4eebb3aff9bca7cee59281b50ce03d492865b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Mon, 7 Jul 2025 18:39:15 +0200 Subject: [PATCH 4/7] service: Move general disk path utility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows a more wider use of the disk path formatting across the code base. Signed-off-by: Julian Pelizäus --- cmd/microcloud/ask.go | 11 ----------- service/system_information.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 196678d54..dfda63fb4 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -282,17 +282,6 @@ func (c *initConfig) askDisks(sh *service.Handler) error { return nil } -func parseDiskPath(disk api.ResourcesStorageDisk) string { - devicePath := "/dev/" + disk.ID - if disk.DeviceID != "" { - devicePath = "/dev/disk/by-id/" + disk.DeviceID - } else if disk.DevicePath != "" { - devicePath = "/dev/disk/by-path/" + disk.DevicePath - } - - return devicePath -} - func (c *initConfig) askLocalPool(sh *service.Handler) error { useJoinConfig := false askSystems := map[string]bool{} diff --git a/service/system_information.go b/service/system_information.go index 4778fd5e1..30ad5f355 100644 --- a/service/system_information.go +++ b/service/system_information.go @@ -351,3 +351,15 @@ func ClustersConflict(systems map[string]SystemInformation, services map[types.S return false, "" } + +// FormatDiskPath returns a disk's path representation. +func FormatDiskPath(disk api.ResourcesStorageDisk) string { + devicePath := "/dev/" + disk.ID + if disk.DeviceID != "" { + devicePath = "/dev/disk/by-id/" + disk.DeviceID + } else if disk.DevicePath != "" { + devicePath = "/dev/disk/by-path/" + disk.DevicePath + } + + return devicePath +} From d449c9e6f43945e9c7917e78dd5d811c874c4276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Mon, 7 Jul 2025 18:40:42 +0200 Subject: [PATCH 5/7] cmd/microcloud: Update usage of FormatDiskPath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julian Pelizäus --- cmd/microcloud/ask.go | 10 +++++----- cmd/microcloud/preseed.go | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index dfda63fb4..7012ab204 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -329,11 +329,11 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { } sort.Slice(sortedDisks, func(i, j int) bool { - return parseDiskPath(sortedDisks[i]) < parseDiskPath(sortedDisks[j]) + return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j]) }) for _, disk := range sortedDisks { - devicePath := parseDiskPath(disk) + devicePath := service.FormatDiskPath(disk) data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath}) } } @@ -458,7 +458,7 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { for target, path := range selectedDisks { newAvailableDisks[target] = map[string]api.ResourcesStorageDisk{} for id, disk := range availableDisks[target] { - if parseDiskPath(disk) != path { + if service.FormatDiskPath(disk) != path { newAvailableDisks[target][id] = disk } } @@ -672,12 +672,12 @@ func (c *initConfig) askRemotePool(sh *service.Handler) error { // Ensure the list of disks is sorted by name. sort.Slice(sortedDisks, func(i, j int) bool { - return parseDiskPath(sortedDisks[i]) < parseDiskPath(sortedDisks[j]) + return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j]) }) for _, disk := range sortedDisks { // Skip any disks that have been reserved for the local storage pool. - devicePath := parseDiskPath(disk) + devicePath := service.FormatDiskPath(disk) data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath}) } } diff --git a/cmd/microcloud/preseed.go b/cmd/microcloud/preseed.go index 7cadbf204..7f4a0be32 100644 --- a/cmd/microcloud/preseed.go +++ b/cmd/microcloud/preseed.go @@ -965,7 +965,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map system.MicroCephDisks = append( system.MicroCephDisks, cephTypes.DisksPost{ - Path: []string{parseDiskPath(disk)}, + Path: []string{service.FormatDiskPath(disk)}, Wipe: filter.Wipe, Encrypt: filter.Encrypt, }, @@ -1040,12 +1040,12 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map if len(matched) > 0 { zfsMachines[peer] = true if c.bootstrap { - system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, parseDiskPath(matched[0]))) + system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, service.FormatDiskPath(matched[0]))) if s.Name == peer { system.StoragePools = append(system.StoragePools, lxd.DefaultZFSStoragePool()) } } else { - system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, parseDiskPath(matched[0]))...) + system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, service.FormatDiskPath(matched[0]))...) } zfsMatches[filter.Find] = zfsMatches[filter.Find] + 1 From 6997df98d3d7143fb5f758c3c77d328fdd76f4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Mon, 7 Jul 2025 18:41:40 +0200 Subject: [PATCH 6/7] service: Filter out disks already used for storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure that when collecting a system's information we don't return any disks which are already used by MicroCloud for either local or remote storage. Signed-off-by: Julian Pelizäus --- service/system_information.go | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/service/system_information.go b/service/system_information.go index 30ad5f355..42920889c 100644 --- a/service/system_information.go +++ b/service/system_information.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/canonical/lxd/shared/api" + cephTypes "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microcloud/microcloud/api/types" "github.com/canonical/microcloud/microcloud/multicast" @@ -100,9 +101,28 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul return nil, fmt.Errorf("Failed to get system resources of peer %q: %w", s.ClusterName, err) } + var microceph *CephService + + // Fetch disks which are already used for remote storage. + var usedCephDisks cephTypes.Disks + if len(s.ExistingServices[types.MicroCeph]) > 0 { + microceph = sh.Services[types.MicroCeph].(*CephService) + + if localSystem { + usedCephDisks, err = microceph.GetDisks(ctx, "", nil) + } else { + usedCephDisks, err = microceph.GetDisks(ctx, s.ClusterAddress, connectInfo.Certificate) + } + + if err != nil && !api.StatusErrorCheck(err, http.StatusServiceUnavailable) { + return nil, fmt.Errorf("Failed to get Ceph disks on %q: %w", s.ClusterName, err) + } + } + if allResources != nil { for _, disk := range allResources.Storage.Disks { // Exclude non-pristine disks with partitions. + // Disks already used for local storage (zfs) contain a partition and are therefore excluded by this check. if len(disk.Partitions) != 0 { continue } @@ -112,6 +132,18 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul continue } + // Exclude disks which are already used for remote storage. + diskUsed := false + for _, usedCephDisk := range usedCephDisks { + if usedCephDisk.Path == FormatDiskPath(disk) && usedCephDisk.Location == connectInfo.Name { + diskUsed = true + } + } + + if diskUsed { + continue + } + s.AvailableDisks[disk.ID] = disk } } @@ -167,8 +199,6 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul } if len(s.ExistingServices[types.MicroCeph]) > 0 { - microceph := sh.Services[types.MicroCeph].(*CephService) - if localSystem { s.CephConfig, err = microceph.ClusterConfig(ctx, "", nil) } else { From bd47863f45947638d77050d8b7f97ee2ff4bc524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Peliz=C3=A4us?= Date: Tue, 8 Jul 2025 12:33:34 +0200 Subject: [PATCH 7/7] cmd/microcloud/ask: Skip systems which aren't marked for local storage contribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When growing a cluster (microcloud add), it might be that the already existing members don't anymore have any left over disks. In this case they aren't marked to contribute disks to the local storage when adding new cluster members. Therefore we can skip those and only ever exit in case a member which should contribute local storage doesn't have any disks. That is the case when creating a new cluster or when adding a cluster member which cannot fulfill the requirement of the already existing cluster. Signed-off-by: Julian Pelizäus --- cmd/microcloud/ask.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 7012ab204..2344cba7d 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -302,15 +302,21 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { availableDisks := map[string]map[string]api.ResourcesStorageDisk{} for name, state := range c.state { + // Skip this system if it already has the local storage pool configured + // and isn't marked for disk selection. + // This ensures that when adding new systems to the cluster the existing ones + // don't have to show any disk because they are probably already used for either local or remote storage. + if !askSystems[name] { + continue + } + if len(state.AvailableDisks) == 0 { logger.Infof("Skipping local storage pool creation, peer %q has too few disks", name) return nil } - if askSystems[name] { - availableDisks[name] = state.AvailableDisks - } + availableDisks[name] = state.AvailableDisks } // Local storage is already set up on every system, or if not every system has a disk.