diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 4ac65288d..7dd7872ea 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -582,7 +582,7 @@ func getTargetCephNetworks(sh *service.Handler, s *InitSystem) (publicCephNetwor } func (c *initConfig) askRemotePool(sh *service.Handler) error { - // If MicroCeph is not installed, skip this block entirely. + // If MicroCeph is not installed or an existing Ceph cluster should not be added, skip this block entirely. if sh.Services[types.MicroCeph] == nil { return nil } @@ -624,10 +624,68 @@ func (c *initConfig) askRemotePool(sh *service.Handler) error { } var selectedDisks map[string][]string var wipeDisks map[string]map[string]bool - availableDiskCount := 0 + if len(askSystemsRemote) != 0 { + // existingClusterDisks contains a slice of disks configured on each of the MicroCeph cluster members. + // That allows checking whether or not the configured disks meet the recommendations. + existingClusterDisks := map[string][]string{} + + // availableDiskCount is the total amount of unconfigured disks across all of the remote systems. + availableDiskCount := 0 + + // availableDisks contains a map of unconfigured disks on each of the remote systems. + // Those disks aren't yet used for neither local nor remote storage. availableDisks := map[string]map[string]api.ResourcesStorageDisk{} + + // Set to true after checking one of the existing MicroCeph cluster members for existing disks. + existingClusterDisksChecked := false + for name, state := range c.state { + // Collect list of already existing remote storage disks. + // This allows understanding which disks on which cluster members are already configured for remote storage. + // The information can then be yielded to the user and allows skipping the selection of additional disks + // in case the user only wants to configure distributed storage without adding additional disks to MicroCeph. + // That scenario is important when adding an existing MicroCeph cluster to MicroCloud. + if state.ServiceClustered(types.MicroCeph) && !existingClusterDisksChecked { + cephService := sh.Services[types.MicroCeph].(*service.CephService) + system, ok := c.systems[name] + if !ok { + return fmt.Errorf("Failed to find system %q", name) + } + + var cert *x509.Certificate + var address string + + // When asking for remote pool configuration the MicroCloud cluster isn't yet formed. + // But we have already established temporary trust with all of the remote systems. + // Use the temporary trust store certificate only in case we are not trying to request + // the disks from the local system itself. + if name != sh.Name { + cert = system.ServerInfo.Certificate + address = state.ClusterAddress + } + + disks, err := cephService.GetDisks(context.TODO(), address, cert) + if err != nil { + return fmt.Errorf("Failed to get disks of existing %s cluster on %q: %w", types.MicroCeph, name, err) + } + + // Only initialize the slice if there are existing disks on this MicroCeph cluster member. + // We consolidate the length of the map later to indicate whether or not there are already existing disks. + if len(disks) > 0 { + existingClusterDisks[name] = []string{} + } + + // Fetching the disks from one of the existing MicroCeph cluster members is sufficient. + // As the disks are known cluster wide, each member should respond with the same number. + for _, disk := range disks { + existingClusterDisks[disk.Location] = append(existingClusterDisks[disk.Location], disk.Path) + } + + // Skip checking every other MicroCeph cluster member as we have already collected the existing disks. + existingClusterDisksChecked = true + } + if askSystemsRemote[name] { availableDisks[name] = state.AvailableDisks @@ -637,7 +695,7 @@ func (c *initConfig) askRemotePool(sh *service.Handler) error { } } - if availableDiskCount == 0 { + if availableDiskCount == 0 && len(existingClusterDisks) == 0 { tui.PrintWarning("No disks available for distributed storage. Skipping configuration") return nil @@ -648,116 +706,146 @@ func (c *initConfig) askRemotePool(sh *service.Handler) error { return err } - // Ask if the user is okay with fully remote ceph on some systems. - if len(askSystemsRemote) != availableDiskCount && wantsDisks { - warning := "Unable to find disks on some systems" - question := "Continue anyway?" - wantsDisks, err = c.asker.AskBoolWarn(warning, question, true) - if err != nil { - return err + if len(existingClusterDisks) > 0 && wantsDisks { + fmt.Println() + + for target, disks := range existingClusterDisks { + if len(disks) > 0 { + fmt.Println(tui.SummarizeResult("Using %d disk(s) already setup on %s for remote storage pool", len(disks), target)) + } } - } - if !wantsDisks { - return nil + fmt.Println() } var insufficientDisks bool - err = c.askRetry("Change disk selection?", func() error { - selectedDisks = map[string][]string{} - wipeDisks = map[string]map[string]bool{} - header := []string{"LOCATION", "MODEL", "CAPACITY", "TYPE", "PATH"} - data := [][]string{} - for peer, disks := range availableDisks { - sortedDisks := []api.ResourcesStorageDisk{} - for _, disk := range disks { - sortedDisks = append(sortedDisks, disk) + + if availableDiskCount > 0 && wantsDisks { + err = c.askRetry("Change disk selection?", func() error { + selectedDisks = map[string][]string{} + wipeDisks = map[string]map[string]bool{} + header := []string{"LOCATION", "MODEL", "CAPACITY", "TYPE", "PATH"} + data := [][]string{} + for peer, disks := range availableDisks { + sortedDisks := []api.ResourcesStorageDisk{} + for _, disk := range disks { + sortedDisks = append(sortedDisks, disk) + } + + // Ensure the list of disks is sorted by name. + sort.Slice(sortedDisks, func(i, j int) bool { + return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j]) + }) + + for _, disk := range sortedDisks { + // Skip any disks that have been reserved for the local storage pool. + devicePath := service.FormatDiskPath(disk) + data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath}) + } } - // Ensure the list of disks is sorted by name. - sort.Slice(sortedDisks, func(i, j int) bool { - return service.FormatDiskPath(sortedDisks[i]) < service.FormatDiskPath(sortedDisks[j]) - }) + if len(data) == 0 { + return errors.New("Invalid disk configuration. Found no available disks") + } - for _, disk := range sortedDisks { - // Skip any disks that have been reserved for the local storage pool. - devicePath := service.FormatDiskPath(disk) - data = append(data, []string{peer, disk.Model, units.GetByteSizeStringIEC(int64(disk.Size), 2), disk.Type, devicePath}) + sort.Sort(cli.SortColumnsNaturally(data)) + var toWipe []map[string]string + table := tui.NewSelectableTable(header, data) + selected, err := table.Render(context.Background(), c.asker, "Select from the available unpartitioned disks:") + if err != nil { + return err } - } - if len(data) == 0 { - return errors.New("Invalid disk configuration. Found no available disks") - } + if len(selected) > 0 { + newRows := make([][]string, len(selected)) + for row := range selected { + newRows[row] = make([]string, len(header)) + for j, h := range header { + newRows[row][j] = selected[row][h] + } + } - sort.Sort(cli.SortColumnsNaturally(data)) - var toWipe []map[string]string - table := tui.NewSelectableTable(header, data) - selected, err := table.Render(context.Background(), c.asker, "Select from the available unpartitioned disks:") - if err != nil { - return err - } + toWipe, err = table.Render(context.Background(), c.asker, "Select which disks to wipe:", newRows...) + if err != nil { + return err + } + } - if len(selected) > 0 { - newRows := make([][]string, len(selected)) - for row := range selected { - newRows[row] = make([]string, len(header)) - for j, h := range header { - newRows[row][j] = selected[row][h] + targetDisks := map[string][]string{} + for _, entry := range selected { + target := entry["LOCATION"] + path := entry["PATH"] + if targetDisks[target] == nil { + targetDisks[target] = []string{} } + + targetDisks[target] = append(targetDisks[target], path) } - toWipe, err = table.Render(context.Background(), c.asker, "Select which disks to wipe:", newRows...) - if err != nil { - return err + wipeDisks = map[string]map[string]bool{} + for _, entry := range toWipe { + target := entry["LOCATION"] + path := entry["PATH"] + if wipeDisks[target] == nil { + wipeDisks[target] = map[string]bool{} + } + + wipeDisks[target][path] = true } - } - targetDisks := map[string][]string{} - for _, entry := range selected { - target := entry["LOCATION"] - path := entry["PATH"] - if targetDisks[target] == nil { - targetDisks[target] = []string{} + selectedDisks = targetDisks + + // Error in case no disks were selected or there isn't an existing Ceph cluster with disks configured. + if len(targetDisks) == 0 && len(existingClusterDisks) == 0 { + return errors.New("No disks were selected") } - targetDisks[target] = append(targetDisks[target], path) - } + mergedDisks := map[string][]string{} - wipeDisks = map[string]map[string]bool{} - for _, entry := range toWipe { - target := entry["LOCATION"] - path := entry["PATH"] - if wipeDisks[target] == nil { - wipeDisks[target] = map[string]bool{} + // Merge both the already existing disks and the new selected ones. + // This allows identifying if the selection follows the recommendations. + for clusterMember, disks := range existingClusterDisks { + _, ok := mergedDisks[clusterMember] + if !ok { + mergedDisks[clusterMember] = []string{} + } + + mergedDisks[clusterMember] = append(mergedDisks[clusterMember], disks...) } - wipeDisks[target][path] = true - } + for clusterMember, disks := range selectedDisks { + _, ok := mergedDisks[clusterMember] + if !ok { + mergedDisks[clusterMember] = []string{} + } - selectedDisks = targetDisks + mergedDisks[clusterMember] = append(mergedDisks[clusterMember], disks...) + } - if len(targetDisks) == 0 { - return errors.New("No disks were selected") - } + insufficientDisks = !useJoinConfigRemote && len(mergedDisks) < RecommendedOSDHosts - insufficientDisks = !useJoinConfigRemote && len(targetDisks) < RecommendedOSDHosts + if insufficientDisks { + return fmt.Errorf("Disk configuration does not meet recommendations for fault tolerance. At least %d systems must supply disks. Continuing with this configuration will inhibit MicroCloud's ability to retain data on system failure", RecommendedOSDHosts) + } - if insufficientDisks { - // This error will be printed to STDOUT as a normal message, so it includes a new-line for readability. - return fmt.Errorf("Disk configuration does not meet recommendations for fault tolerance. At least %d systems must supply disks. Continuing with this configuration will inhibit MicroCloud's ability to retain data on system failure", RecommendedOSDHosts) + return nil + }) + if err != nil { + return err } - - return nil - }) - if err != nil { - return err } - if len(selectedDisks) == 0 { + if len(selectedDisks) == 0 && len(existingClusterDisks) == 0 { + // Skip distributed storage if there are neither disks selected nor is there an existing cluster with disks configured. return nil - } else { - fmt.Println() + } else if len(selectedDisks) > 0 { + // Print the newline only in case we haven't printed the notification about + // already existing disks for the remote storage pool. + // If we are reusing disks and also adding new ones, the two sections + // should only be separated by a single new line. + if len(existingClusterDisks) == 0 { + fmt.Println() + } for target, disks := range selectedDisks { if len(disks) > 0 { diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index e099db5e8..ec25609fe 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -4,7 +4,7 @@ unset_interactive_vars() { unset SKIP_LOOKUP LOOKUP_IFACE SKIP_SERVICE EXPECT_PEERS PEERS_FILTER REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ - SETUP_CEPH CEPH_MISSING_DISKS CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK CEPH_PUBLIC_NETWORK \ + SETUP_CEPH CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK CEPH_PUBLIC_NETWORK \ PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_UNDERLAY_NETWORK OVN_UNDERLAY_FILTER OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ REPLACE_PROFILE CEPH_RETRY_HA MULTI_NODE } @@ -34,9 +34,9 @@ microcloud_interactive() { SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. - SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. + SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating Ceph storage pool setup. + SKIP_CEPH_DISKS=${SKIP_CEPH_DISKS:-} # (yes/no) input to skip adding additional Ceph disks and only reuse the existing cluster and its disks. SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. - CEPH_MISSING_DISKS=${CEPH_MISSING_DISKS:-} # (yes/no) input for warning about eligible disk detection. CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. CEPH_RETRY_HA=${CEPH_RETRY_HA:-} # (yes/no) input for warning setup is not HA. @@ -104,19 +104,26 @@ fi if [ -n "${SETUP_CEPH}" ]; then setup="${setup} ${SETUP_CEPH} # add remote disks (yes/no) -${CEPH_MISSING_DISKS} # continue with some peers missing disks? (yes/no) -$([ "${SETUP_CEPH}" = "yes" ] && printf "table:wait 300ms") # wait for the table to populate +" + if [ "${SKIP_CEPH_DISKS}" != "yes" ]; then + setup="${setup} +$([ "${SETUP_CEPH}" = "yes" ] && printf "table:wait 300ms") # wait for the table to populate $([ -n "${CEPH_FILTER}" ] && printf "table:filter %s" "${CEPH_FILTER}") # filter ceph disks -$([ "${SETUP_CEPH}" = "yes" ] && printf "table:select-all") # select all disk matching the filter +$([ "${SETUP_CEPH}" = "yes" ] && printf "table:select-all") # select all disk matching the filter $([ "${SETUP_CEPH}" = "yes" ] && printf -- "table:done") -$([ "${CEPH_WIPE}" = "yes" ] && printf "table:select-all") # wipe all disks +$([ "${CEPH_WIPE}" = "yes" ] && printf "table:select-all") # wipe all disks $([ "${SETUP_CEPH}" = "yes" ] && printf -- "table:done") $([ "${SETUP_CEPH}" = "yes" ] && printf "%s" "${CEPH_RETRY_HA}" ) # allow ceph setup without 3 systems supplying disks. -${CEPH_ENCRYPT} # encrypt disks? (yes/no) +$(true) # workaround for set -e +" + fi + + setup="${setup} +${CEPH_ENCRYPT} # encrypt disks? (yes/no) ${SETUP_CEPHFS} $([ "${SETUP_CEPH}" = "yes" ] && printf "%s" "${CEPH_CLUSTER_NETWORK}" ) # set ceph cluster network $([ "${SETUP_CEPH}" = "yes" ] && printf "%s" "${CEPH_PUBLIC_NETWORK}" ) # set ceph public network -$(true) # workaround for set -e +$(true) # workaround for set -e " fi diff --git a/test/suites/basic.sh b/test/suites/basic.sh index ca9a88412..f9437197e 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -740,7 +740,6 @@ test_disk_mismatch() { export ZFS_WIPE="yes" export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" - export CEPH_MISSING_DISKS="yes" export CEPH_WIPE="yes" export CEPH_ENCRYPT="no" export SETUP_OVN="no" @@ -886,6 +885,43 @@ test_reuse_cluster() { bootstrap_microceph micro03 ! join_session init micro01 micro02 micro03 || false lxc exec micro01 -- tail -1 out | grep "Some systems are already part of different MicroCeph clusters. Aborting initialization" -q + + reset_systems 3 2 3 + echo "Create a MicroCloud that re-uses an existing MicroCeph with disks already setup to configure distributed storage" + unset_interactive_vars + + export MULTI_NODE="yes" + export LOOKUP_IFACE="enp5s0" + export EXPECT_PEERS=2 + export REUSE_EXISTING_COUNT=1 + export REUSE_EXISTING="yes" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="yes" + export SKIP_CEPH_DISKS="yes" + export SETUP_CEPHFS="yes" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export OVN_UNDERLAY_NETWORK="no" + + bootstrap_microceph micro01 + for m in micro02 micro03; do + token="$(lxc exec micro01 -- microceph cluster add "${m}")" + lxc exec "${m}" -- microceph cluster join "${token}" + done + + for m in micro01 micro02 micro03; do + lxc exec "${m}" -- microceph disk add /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_disk2 + done + + join_session init micro01 micro02 micro03 + services_validator } test_remove_cluster() {