From 0037f87169032d79ab5bc2aa22533f101e0393ff Mon Sep 17 00:00:00 2001 From: Jitendra Yejare <11752425+jyejare@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:08:09 +0530 Subject: [PATCH] Implement OCP cluster cleanup using openshift-install Replace the dry-run-only OCP stub with a full cleanup implementation that uses the openshift-install CLI to destroy leftover OCP clusters on AWS, addressing review feedback from PR #166. - Add LeftoverAWSOcp class to model cluster resources and metadata - Add destroy_ocp_cluster/write_metadata_file/check_installer_exists utils - Support OCP exceptions in config for cluster exclusion - Move OCP constants to constants.py, remove dead code from utils.py - Install openshift-install CLI in Dockerfile.dev Co-authored-by: Cursor Signed-off-by: Jitendra Yejare <11752425+jyejare@users.noreply.github.com> --- Dockerfile.dev | 8 ++ cloudwash/constants.py | 11 ++ cloudwash/entities/resources/ocps.py | 200 ++++++++++++++++++++++----- cloudwash/providers/aws.py | 2 + cloudwash/utils.py | 144 +++++++++++-------- conf/aws.yaml.template | 2 + 6 files changed, 274 insertions(+), 93 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 84874a7f2..ea32a2898 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -21,6 +21,12 @@ RUN git clone --depth=1 https://github.com/RedHatQE/cloudwash.git && \ RUN /bin/bash -c 'cd ${CLOUDWASH_DIR}; for conffile in conf/*.yaml.template; do cp -- "$conffile" "${conffile%.yaml.template}.yaml"; done' +# Install openshift-install CLI for OCP cluster cleanup +RUN curl -kf https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-install-linux.tar.gz \ + -o openshift-install-linux.tar.gz && \ + tar zxf openshift-install-linux.tar.gz && \ + chmod +x openshift-install + # adding .profile to environment variables, so it will be kept between shell sessions RUN echo "source ${APP_ROOT}/.profile" >> ${APP_ROOT}/bin/activate && touch ${APP_ROOT}/.profile @@ -28,6 +34,8 @@ RUN echo "source ${APP_ROOT}/.profile" >> ${APP_ROOT}/bin/activate && touch ${AP USER 0 RUN fix-permissions ${APP_ROOT} -P && \ git config --global --add safe.directory ${CLOUDWASH_DIR} +RUN cp openshift-install /usr/local/bin/ && \ + rm -f openshift-install openshift-install-linux.tar.gz USER 1001 WORKDIR "${CLOUDWASH_DIR}" diff --git a/cloudwash/constants.py b/cloudwash/constants.py index 677f581f5..4faa46284 100644 --- a/cloudwash/constants.py +++ b/cloudwash/constants.py @@ -3,3 +3,14 @@ gce_data = ['VMS', 'NICS', 'DISCS'] vmware_data = ['VMS', 'NICS', 'DISCS'] container_data = ['CONTAINERS'] + +OCP_TAG_SUBSTR = "kubernetes.io/cluster/" +CLUSTER_NAME_TAGS = [ + "clusterName", + "api.openshift.com/name", +] +CLUSTER_ID_TAGS = [ + "openshiftClusterID", + "api.openshift.com/id", +] +CLUSTER_EXP_DATE_TAG = "expirationDate" diff --git a/cloudwash/entities/resources/ocps.py b/cloudwash/entities/resources/ocps.py index b47785199..a51f38b40 100644 --- a/cloudwash/entities/resources/ocps.py +++ b/cloudwash/entities/resources/ocps.py @@ -1,20 +1,92 @@ +import tempfile + from cloudwash.config import settings +from cloudwash.constants import CLUSTER_EXP_DATE_TAG +from cloudwash.constants import CLUSTER_ID_TAGS +from cloudwash.constants import CLUSTER_NAME_TAGS +from cloudwash.constants import OCP_TAG_SUBSTR from cloudwash.entities.resources.base import OCPsCleanup -from cloudwash.utils import calculate_time_threshold +from cloudwash.logger import logger +from cloudwash.utils import are_resources_older_than +from cloudwash.utils import check_installer_exists +from cloudwash.utils import destroy_ocp_cluster from cloudwash.utils import dry_data -from cloudwash.utils import filter_resources_by_time_modified -from cloudwash.utils import group_ocps_by_cluster -from cloudwash.utils import OCP_TAG_SUBSTR +from cloudwash.utils import write_metadata_file + + +class LeftoverAWSOcp: + """Represents an OCP cluster with its collected metadata and filtered AWS resources.""" + + def __init__(self, infra_id: str, region: str): + self.infra_id = infra_id + self.region = region + self.associated_resources = {"Resources": [], "Instances": []} + self._cluster_name = "" + self._cluster_id = "" + self._expiration_date = "" + + def __repr__(self): + instances = len(self.associated_resources.get("Instances", [])) + resources = len(self.associated_resources.get("Resources", [])) + return ( + f"LeftoverAWSOcp({self.infra_id}, region={self.region}, " + f"instances={instances}, resources={resources})" + ) + + def _collect_all_tags(self) -> dict: + """Build a merged tag dict from all associated resources (first value wins).""" + tags = {} + for resources_list in self.associated_resources.values(): + for resource in resources_list: + for tag in resource.get_tags(): + key = tag.get("Key", "") + if key and key not in tags: + tags[key] = tag.get("Value", "") + return tags + + def get_cluster_info(self): + """Extract cluster name, ID, and expiration date from resource tags.""" + tags = self._collect_all_tags() + if not self._expiration_date: + self._expiration_date = tags.get(CLUSTER_EXP_DATE_TAG, "") + if not self._cluster_name: + for name_tag in CLUSTER_NAME_TAGS: + if name_tag in tags: + self._cluster_name = tags[name_tag] + break + if not self._cluster_id: + for id_tag in CLUSTER_ID_TAGS: + if id_tag in tags: + self._cluster_id = tags[id_tag] + break + + def get_cluster_metadata(self) -> dict: + """Build the metadata dict required by openshift-install for cluster teardown.""" + infra_id = self.infra_id + cluster_name = self._cluster_name or infra_id + cluster_id = self._cluster_id or infra_id + + logger.info(f"\nPreparing metadata for cluster: {infra_id}") + return { + "clusterName": cluster_name, + "clusterID": cluster_id, + "infraID": infra_id, + "aws": { + "region": self.region, + "identifier": [{f"{OCP_TAG_SUBSTR}{infra_id}": "owned"}], + }, + } class CleanOCPs(OCPsCleanup): - def __init__(self, client): - self.client = client - self._delete = [] + def __init__(self): + self._delete = {"resources": {}, "clusters": []} + self._cluster_map = {} self.list() def _set_dry(self): - dry_data['OCPS']['delete'] = self._delete + dry_data['OCPS']['delete'] = self._delete["resources"] + dry_data['OCPS']['clusters'] = self._delete["clusters"] def list(self): pass @@ -24,40 +96,100 @@ def remove(self): def cleanup(self): if not settings.dry_run: - self.remove() + check_installer_exists() + with tempfile.TemporaryDirectory() as tmpdir: + for cluster_name in self._delete["clusters"]: + cluster = self._cluster_map[cluster_name] + cluster.get_cluster_info() + metadata = cluster.get_cluster_metadata() + metadata_path = write_metadata_file( + cluster_metadata=metadata, cleanup_dir=tmpdir + ) + destroy_ocp_cluster( + metadata_path=metadata_path, + cluster_name=cluster_name, + ) class CleanAWSOcps(CleanOCPs): + def __init__(self, client): + self.client = client + self.cleaning_region = self.client.cleaning_region + super().__init__() + + def group_ocps_by_cluster(self, resources: list = None) -> dict: + """Group AWS resources under their originating OCP clusters by infra ID. + + :param list resources: AWS resources collected by region and SLA + :return: Dict mapping cluster infra IDs to LeftoverAWSOcp instances + """ + if resources is None: + resources = [] + clusters_map = {} + + for resource in resources: + for key in resource.get_tags(regex=OCP_TAG_SUBSTR): + cluster_infra_id = key.get("Key") + if OCP_TAG_SUBSTR in cluster_infra_id: + parts = cluster_infra_id.split(OCP_TAG_SUBSTR) + if len(parts) < 2 or not parts[1]: + continue + cluster_infra_id = parts[1] + if cluster_infra_id not in clusters_map: + clusters_map[cluster_infra_id] = LeftoverAWSOcp( + infra_id=cluster_infra_id, region=self.cleaning_region + ) + + if hasattr(resource, 'ec2_instance'): + clusters_map[cluster_infra_id].associated_resources["Instances"].append( + resource + ) + else: + clusters_map[cluster_infra_id].associated_resources["Resources"].append( + resource + ) + return clusters_map + + def _filter_deletable(self): + """Apply SLA and exception filters to determine which clusters are deletable.""" + sla = settings.aws.criteria.ocps.get("SLA") + exceptions = list(settings.aws.exceptions.get("OCPS", [])) + + for cluster_name, cluster in self._cluster_map.items(): + if cluster_name in exceptions: + continue + + instances = cluster.associated_resources.get("Instances", []) + resources = cluster.associated_resources.get("Resources", []) + + if instances: + if are_resources_older_than(time_ref=sla, resources=instances): + self._delete["clusters"].append(cluster_name) + self._delete["resources"].update(self._resources_by_type(resources + instances)) + elif resources: + if are_resources_older_than(time_ref=sla, resources=resources): + self._delete["clusters"].append(cluster_name) + self._delete["resources"].update(self._resources_by_type(resources)) + + @staticmethod + def _resources_by_type(resources: list) -> dict: + """Group resource names by their resource_type.""" + result = {} + for r in resources: + result.setdefault(r.resource_type, []).append(r.name) + return result + def list(self): resources = [] - time_threshold = calculate_time_threshold(time_ref=settings.aws.criteria.ocps.get("SLA")) - - ocp_prefix = list(settings.aws.criteria.ocps.get("OCP_PREFIXES") or [""]) - for prefix in ocp_prefix: + ocp_prefixes = list(settings.aws.criteria.ocps.get("OCP_PREFIXES") or [""]) + for prefix in ocp_prefixes: query = " ".join( - [f"tag.key:{OCP_TAG_SUBSTR}{prefix}*", f"region:{self.client.cleaning_region}"] + [f"tag.key:{OCP_TAG_SUBSTR}{prefix}*", f"region:{self.cleaning_region}"] ) resources.extend(self.client.list_resources(query=query)) - # Prepare resources to be filtered before deletion - cluster_map = group_ocps_by_cluster(resources=resources) - for cluster_name in cluster_map.keys(): - cluster_resources = cluster_map[cluster_name].get("Resources") - instances = cluster_map[cluster_name].get("Instances") + self._cluster_map = self.group_ocps_by_cluster(resources=resources) + self._filter_deletable() - if instances: - # For resources with associated EC2 Instances, filter by Instances SLA - if not filter_resources_by_time_modified( - time_threshold, - resources=instances, - ): - self._delete.extend(cluster_resources) - else: - # For resources with no associated EC2 Instances, identify as leftovers - self._delete.extend( - filter_resources_by_time_modified(time_threshold, resources=cluster_resources) - ) - - # Sort resources by type - self._delete = sorted(self._delete, key=lambda x: x.resource_type) + self._delete["clusters"] = sorted(self._delete["clusters"]) self._set_dry() diff --git a/cloudwash/providers/aws.py b/cloudwash/providers/aws.py index f9959827e..6aeb1cebc 100644 --- a/cloudwash/providers/aws.py +++ b/cloudwash/providers/aws.py @@ -29,6 +29,8 @@ def cleanup(**kwargs): # Emptying the dry data for previous region everytime for items in data: dry_data[items]['delete'] = [] + if items == 'OCPS': + dry_data[items]['clusters'] = [] logger.info(f"\nResources from the region: {region}") awscleanup.ocps.cleanup() if is_dry_run: diff --git a/cloudwash/utils.py b/cloudwash/utils.py index 60e3256d5..e27a1ee6b 100644 --- a/cloudwash/utils.py +++ b/cloudwash/utils.py @@ -1,7 +1,10 @@ """Common utils for cleanup activities of all CRs""" import importlib.resources +import json +import subprocess from collections import namedtuple from datetime import datetime +from pathlib import Path import dateparser import dominate @@ -20,8 +23,6 @@ from cloudwash.assets import css from cloudwash.logger import logger -OCP_TAG_SUBSTR = "kubernetes.io/cluster/" - _vms_dict = {"VMS": {"delete": [], "stop": [], "skip": []}} _containers_dict = {"CONTAINERS": {"delete": [], "stop": [], "skip": []}} @@ -29,7 +30,7 @@ "NICS": {"delete": []}, "DISCS": {"delete": []}, "PIPS": {"delete": []}, - "OCPS": {"delete": []}, + "OCPS": {"delete": [], "clusters": []}, "RESOURCES": {"delete": []}, "STACKS": {"delete": []}, "IMAGES": {"delete": []}, @@ -62,12 +63,7 @@ def resourcewise_data(dry_data=None) -> dict: "deletable_pips": dry_data["PIPS"]["delete"] if "PIPS" in dry_data else None, "deletable_resources": dry_data["RESOURCES"]["delete"], "deletable_stacks": dry_data["STACKS"]["delete"] if "STACKS" in dry_data else None, - "deletable_ocps": { - ocp.resource_type: [ - r.name for r in dry_data["OCPS"]["delete"] if r.resource_type == ocp.resource_type - ] - for ocp in dry_data["OCPS"]["delete"] - }, + "deletable_ocps": dry_data["OCPS"]["delete"], } return resource_data @@ -231,33 +227,6 @@ def gce_zones() -> list: return zones -def group_ocps_by_cluster(resources: list = None) -> dict: - """Group different types of AWS resources under their original OCP clusters - - :param list resources: AWS resources collected by defined region and sla - :return: A dictionary with the clusters as keys and the associated resources as values - """ - if resources is None: - resources = [] - clusters_map = {} - - for resource in resources: - for key in resource.get_tags(regex=OCP_TAG_SUBSTR): - cluster_name = key.get("Key") - if OCP_TAG_SUBSTR in cluster_name: - cluster_name = cluster_name.split(OCP_TAG_SUBSTR)[1] - if cluster_name not in clusters_map.keys(): - clusters_map[cluster_name] = {"Resources": [], "Instances": []} - - # Set cluster's EC2 instances - if hasattr(resource, 'ec2_instance'): - clusters_map[cluster_name]["Instances"].append(resource) - # Set resource under cluster - else: - clusters_map[cluster_name]["Resources"].append(resource) - return clusters_map - - def calculate_time_threshold(time_ref=""): """Parses a time reference for data filtering @@ -280,29 +249,86 @@ def calculate_time_threshold(time_ref=""): return time_threshold -def filter_resources_by_time_modified( - time_threshold, +def are_resources_older_than( + time_ref: str, resources: list[ResourceExplorerResource] = None, -) -> list: - """ - Filter list of AWS resources by checking modification date ("LastReportedAt") - :param datetime time_threshold: Time filtering criteria - :param list resources: List of resources to be filtered out - :return: list of resources that last modified before time threshold +) -> bool: + """Check if all resources in the list were last modified before the SLA threshold. - :Example: - Use the time_ref "1h" to collect resources that exist for more than an hour + :param str time_ref: Relative time reference in {value}{unit} format (e.g. "7d", "1h") + :param list resources: AWS resources to check against the time threshold + :return: True if every resource was last modified before the threshold """ - filtered_resources = [] - - for resource in resources: - # Will not collect resources recorded during the SLA time - if resource.date_modified > time_threshold: - continue - filtered_resources.append(resource) - return filtered_resources - - -def delete_ocp(ocp): - # WIP: add support for deletion - pass + time_threshold = calculate_time_threshold(time_ref=time_ref) + return all(r.date_modified <= time_threshold for r in resources) + + +def check_installer_exists(): + """Verify the openshift-install CLI is available on PATH, exit if not found.""" + try: + subprocess.run( + ['openshift-install', '--help'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + logger.info("Found openshift-install CLI") + except FileNotFoundError: + logger.exception( + "openshift-install CLI not found. " + "Use the Docker container environment or install locally from: " + "https://mirror.openshift.com/pub/openshift-v4/x86_64/" + "clients/ocp/stable/openshift-install-linux.tar.gz" + "\nFor more information: https://github.com/openshift/installer" + ) + exit(1) + + +def destroy_ocp_cluster(metadata_path: str, cluster_name: str): + """Run openshift-install destroy cluster using the provided metadata file. + + :param str metadata_path: Path to the metadata.json file for the cluster + :param str cluster_name: Human-readable cluster name for logging + """ + metadata = Path(metadata_path) + if not metadata.exists(): + logger.error(f"Failed to load cluster info from metadata path: {metadata_path}") + return + + cleanup_dir = str(metadata.parent) + try: + logger.info(f"Starting to destroy OCP cluster: {cluster_name}") + result = subprocess.run( + [ + 'openshift-install', + 'destroy', + 'cluster', + '--dir', + cleanup_dir, + '--log-level=debug', + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if result.returncode != 0: + logger.error(f"Failed to cleanup OCP cluster {cluster_name}:\n{result.stdout}") + else: + logger.debug(result.stdout) + logger.info(f"Successfully destroyed OCP cluster: {cluster_name}") + except subprocess.SubprocessError as ex: + logger.error(f"Failed to cleanup OCP cluster {cluster_name}:\n{ex}") + + +def write_metadata_file(cluster_metadata: dict, cleanup_dir: str) -> str: + """Write cluster metadata JSON required by openshift-install. + + :param dict cluster_metadata: Metadata dict with clusterName, clusterID, infraID, aws info + :param str cleanup_dir: Directory to write the metadata.json into + :return: Path to the written metadata file + """ + metadata_file = Path(cleanup_dir) / "metadata.json" + metadata_file.write_text(json.dumps(cluster_metadata)) + logger.debug(f"Metadata written to {metadata_file}") + return str(metadata_file) diff --git a/conf/aws.yaml.template b/conf/aws.yaml.template index d648a0c6f..63e789bec 100644 --- a/conf/aws.yaml.template +++ b/conf/aws.yaml.template @@ -45,3 +45,5 @@ AWS: # CloudFormations names that would be skipped from cleanup STACK_LIST: [] IMAGES: [] + # OCP cluster infra IDs that should be skipped from cleanup + OCPS: []