diff --git a/xWorkflows/xWorkflow4/daemonSetCpu.py b/xWorkflows/xWorkflow4/daemonSetCpu.py new file mode 100644 index 0000000..4d1cecd --- /dev/null +++ b/xWorkflows/xWorkflow4/daemonSetCpu.py @@ -0,0 +1,84 @@ +import logging +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + DaemonSetEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateCpuRequest: str + +@action +def daemonSetCpu(event: DaemonSetEvent, params: resizeParams): + """ + Change the CPU requests of a DaemonSet + """ + if not event.get_daemonset(): + # Log an error message if the DaemonSet is not found + logging.error("Failed to get the DaemonSet for CPU adjustment") + return + + daemonSet = event.get_daemonset() + daemonSetName = daemonSet.metadata.name + daemonSetNamespace = daemonSet.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + apps_v1 = client.AppsV1Api() + + try: + # Read the existing DaemonSet specification + daemonSetSpec = apps_v1.read_namespaced_daemon_set(name=daemonSetName, namespace=daemonSetNamespace) + + # Update the CPU requests in the DaemonSet spec + for container in daemonSetSpec.spec.template.spec.containers: + container.resources.requests['cpu'] = params.updateCpuRequest + + # Update the DaemonSet with the modified spec + apps_v1.patch_namespaced_daemon_set( + name=daemonSetName, + namespace=daemonSetNamespace, + body={"spec": daemonSetSpec.spec} + ) + + logging.info(f"DaemonSet {daemonSetName} in namespace {daemonSetNamespace} has been updated with new CPU requests.") + + functionName = "Change CPU requests of DaemonSet" + finding = Finding( + title="DaemonSet CPU upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{daemonSetName} CPU is Upgraded."), + ] + ) + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying DaemonSet: {e}") + functionName = "Change CPU requests of DaemonSet Failed" + finding = Finding( + title="DaemonSet CPU upgradation Failed", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{daemonSetName} CPU Upgradation Failed."), + ] + ) + + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/daemonSetMemory.py b/xWorkflows/xWorkflow4/daemonSetMemory.py new file mode 100644 index 0000000..6ab4a3f --- /dev/null +++ b/xWorkflows/xWorkflow4/daemonSetMemory.py @@ -0,0 +1,85 @@ +import logging + +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + DaemonSetEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateMemoryRequest: str + +@action +def daemonSetMemory(event: DaemonSetEvent, params: resizeParams): + """ + Change the Memory requests of a DaemonSet + """ + if not event.get_daemonset(): + # Log an error message if the DaemonSet is not found + logging.error("Failed to get the DaemonSet for Memory adjustment") + return + + daemonSet = event.get_daemonset() + daemonSetName = daemonSet.metadata.name + daemonSetNamespace = daemonSet.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + apps_v1 = client.AppsV1Api() + + try: + # Read the existing DaemonSet specification + daemonSetSpec = apps_v1.read_namespaced_daemon_set(name=daemonSetName, namespace=daemonSetNamespace) + + # Update the Memory requests in the DaemonSet spec + for container in daemonSetSpec.spec.template.spec.containers: + container.resources.requests['memory'] = params.updateMemoryRequest + + # Update the DaemonSet with the modified spec + apps_v1.patch_namespaced_daemon_set( + name=daemonSetName, + namespace=daemonSetNamespace, + body={"spec": daemonSetSpec.spec} + ) + + logging.info(f"DaemonSet {daemonSetName} in namespace {daemonSetNamespace} has been updated with new Memory requests.") + + functionName = "Change Memory requests of DaemonSet" + finding = Finding( + title="DaemonSet Memory upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{daemonSetName} Memory is Upgraded."), + ] + ) + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying DaemonSet: {e}") + functionName = "Change Memory requests of DaemonSet Failed" + finding = Finding( + title="DaemonSet Memory upgradation Failed", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=True, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{daemonSetName} Memory Upgradation Failed."), + ] + ) + + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/deploymentCpu.py b/xWorkflows/xWorkflow4/deploymentCpu.py new file mode 100644 index 0000000..263a180 --- /dev/null +++ b/xWorkflows/xWorkflow4/deploymentCpu.py @@ -0,0 +1,70 @@ +import logging +import time +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + DeploymentEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateCpuRequest: str + +@action +def deploymentCpu(event: DeploymentEvent, params: resizeParams): + """ + Change the CPU requests of a Deployment + """ + if not event.get_deployment(): + # Log an error message if the deployment is not found + logging.error("Failed to get the deployment for CPU adjustment") + return + + deployment = event.get_deployment() + deploymentName = deployment.metadata.name + deploymentNamespace = deployment.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + apps_v1 = client.AppsV1Api() + + try: + # Read the existing Deployment specification + deploymentSpec = apps_v1.read_namespaced_deployment(name=deploymentName, namespace=deploymentNamespace) + + # Update the CPU requests in the Deployment spec + for container in deploymentSpec.spec.template.spec.containers: + container.resources.requests['cpu'] = params.updateCpuRequest + + # Update the Deployment with the modified spec + apps_v1.patch_namespaced_deployment( + name=deploymentName, + namespace=deploymentNamespace, + body={"spec": deploymentSpec.spec} + ) + + logging.info(f"Deployment {deploymentName} in namespace {deploymentNamespace} has been updated with new CPU requests.") + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying deployment: {e}") + + functionName = "Change CPU requests of Deployment" + finding = Finding( + title="Deployment CPU upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{deploymentName} CPU is Upgraded."), + ] + ) + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/deploymentMemory.py b/xWorkflows/xWorkflow4/deploymentMemory.py new file mode 100644 index 0000000..f2a86ee --- /dev/null +++ b/xWorkflows/xWorkflow4/deploymentMemory.py @@ -0,0 +1,70 @@ +import logging +import time +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + DeploymentEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateMemoryRequest: str + +@action +def deploymentMemory(event: DeploymentEvent, params: resizeParams): + """ + Change the Memory requests of a Deployment + """ + if not event.get_deployment(): + # Log an error message if the deployment is not found + logging.error("Failed to get the deployment for Memory adjustment") + return + + deployment = event.get_deployment() + deploymentName = deployment.metadata.name + deploymentNamespace = deployment.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + apps_v1 = client.AppsV1Api() + + try: + # Read the existing Deployment specification + deploymentSpec = apps_v1.read_namespaced_deployment(name=deploymentName, namespace=deploymentNamespace) + + # Update the Memory requests in the Deployment spec + for container in deploymentSpec.spec.template.spec.containers: + container.resources.requests['memory'] = params.updateMemoryRequest + + # Update the Deployment with the modified spec + apps_v1.patch_namespaced_deployment( + name=deploymentName, + namespace=deploymentNamespace, + body={"spec": deploymentSpec.spec} + ) + + logging.info(f"Deployment {deploymentName} in namespace {deploymentNamespace} has been updated with new Memory requests.") + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying deployment: {e}") + + functionName = "Change Memory requests of Deployment" + finding = Finding( + title="Deployment Memory upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{deploymentName} Memory is Upgraded."), + ] + ) + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/podCpu.py b/xWorkflows/xWorkflow4/podCpu.py new file mode 100644 index 0000000..14c48ba --- /dev/null +++ b/xWorkflows/xWorkflow4/podCpu.py @@ -0,0 +1,92 @@ +import logging +import time +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + PodEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateCpuRequest: str + +@action +def podCpu(event: PodEvent, params: resizeParams): + """ + Change the CPU requests of a Pod + """ + if not event.get_pod(): + # Log an error message if the pod is not found + logging.error("Failed to get the pod for CPU adjustment") + return + + pod = event.get_pod() + podName = pod.metadata.name + podNamespace = pod.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + core_v1 = client.CoreV1Api() + + try: + # Read the existing Pod specification + podSpec = core_v1.read_namespaced_pod(name=podName, namespace=podNamespace) + + # Create a new Pod spec without resource_version + new_pod_spec = client.V1Pod( + metadata=client.V1ObjectMeta( + name=pod.metadata.name, + namespace=pod.metadata.namespace, + labels=pod.metadata.labels, + annotations=pod.metadata.annotations, + ), + spec=podSpec.spec, + ) + + # Update the CPU requests in the new Pod spec + for container in new_pod_spec.spec.containers: + container.resources.requests['cpu'] = params.updateCpuRequest + + # Delete the existing Pod + core_v1.delete_namespaced_pod(name=podName, namespace=podNamespace) + + # Wait for the Pod to be fully deleted + for _ in range(30): # Retry up to 30 times + try: + # Check if the pod is deleted + core_v1.read_namespaced_pod(name=podName, namespace=podNamespace) + time.sleep(1) # Wait for a short time before retrying + except client.exceptions.ApiException as e: + if e.status == 404: # Pod not found, which means it’s deleted + break + raise # Reraise other exceptions + + # Recreate the Pod with the updated specification + core_v1.create_namespaced_pod(namespace=podNamespace, body=new_pod_spec) + + logging.info(f"Pod {podName} in namespace {podNamespace} has been updated with new CPU requests.") + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying pod: {e}") + + + functionName = "Change CPU requests of Pod" + finding = Finding( + title="Pod CPU upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{podName} CPU is Upgraded."), + ] + ) + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/podMemory.py b/xWorkflows/xWorkflow4/podMemory.py new file mode 100644 index 0000000..ab908ae --- /dev/null +++ b/xWorkflows/xWorkflow4/podMemory.py @@ -0,0 +1,92 @@ +import logging +import time +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + PodEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateMemoryRequest: str + +@action +def podMemory(event: PodEvent, params: resizeParams): + """ + Change the Memory requests of a Pod + """ + if not event.get_pod(): + # Log an error message if the pod is not found + logging.error("Failed to get the pod for Memory adjustment") + return + + pod = event.get_pod() + podName = pod.metadata.name + podNamespace = pod.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + core_v1 = client.CoreV1Api() + + try: + # Read the existing Pod specification + podSpec = core_v1.read_namespaced_pod(name=podName, namespace=podNamespace) + + # Create a new Pod spec without resource_version + new_pod_spec = client.V1Pod( + metadata=client.V1ObjectMeta( + name=pod.metadata.name, + namespace=pod.metadata.namespace, + labels=pod.metadata.labels, + annotations=pod.metadata.annotations, + ), + spec=podSpec.spec, + ) + + # Update the CPU requests in the new Pod spec + for container in new_pod_spec.spec.containers: + container.resources.requests['memory'] = params.updateMemoryRequest + + # Delete the existing Pod + core_v1.delete_namespaced_pod(name=podName, namespace=podNamespace) + + # Wait for the Pod to be fully deleted + for _ in range(30): # Retry up to 30 times + try: + # Check if the pod is deleted + core_v1.read_namespaced_pod(name=podName, namespace=podNamespace) + time.sleep(1) # Wait for a short time before retrying + except client.exceptions.ApiException as e: + if e.status == 404: # Pod not found, which means it’s deleted + break + raise # Reraise other exceptions + + # Recreate the Pod with the updated specification + core_v1.create_namespaced_pod(namespace=podNamespace, body=new_pod_spec) + + logging.info(f"Pod {podName} in namespace {podNamespace} has been updated with new Memory requests.") + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying pod: {e}") + + + functionName = "Change Memory requests of Pod" + finding = Finding( + title="Pod Memory upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{podName} Memory is Upgraded."), + ] + ) + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/readme.md b/xWorkflows/xWorkflow4/readme.md new file mode 100644 index 0000000..89e482e --- /dev/null +++ b/xWorkflows/xWorkflow4/readme.md @@ -0,0 +1,133 @@ + +# X-Workflow 4: Container Requests Right Sizing in a Kubernetes Cluster + +## Overview + +X-Workflow 4 aims to identify the right size of container requests in a Kubernetes cluster, and display them on a dashboard using API for user information. The workflow involves querying Kubecost for cost-based workloads utilization data, storing this information in MongoDB, and providing a user interface for cluster monitoring. + + + +## How it Works: + +1. **Kubecost Query Pod**: + - The query pod continuously queries Kubecost to retrieve pods data based on container requests. + - The query pod dumps this data into a MongoDB database deployed as a StatefulSet in the cluster. +2. **Flask Backend**: + - The backend pod queries the MongoDB database to display results to users about resize the container requests. +3. **Robusta Action** + - The robusta actions will reize the container requests. + +## Robusta Action: +We have to create the robusta action for pods, deployments, deamonsets and statefulset etc. +In case of pods, flask API will call robusta action by passing the name, namespace and updated values(CPU or memeory). +```bash +# Pod Schema +name: +namespace: +updateCpuRequest: +"OR" +updateMemoryRequest: +``` +Note incase of Deployment "name" field requires the name of deployment instead of podname rest remains same. This rule should be same for Daemonset and Statefulset meaning instead of pod name we need the name of the resource. + +### Schema for Pods +The name of the actions in case of pods are: +```bash +podCpu and podMemory +``` +```bash +# Pod Schema +name: +namespace: +updateCpuRequest: +"OR" +updateMemoryRequest: +``` +So the final schema would be like this: +```bash +# Incase of cpu + payload = { + "action_name": "podCpu", + "action_params": {"name": podname, "namespace": podnamespace,"updateCpuRequest":cpuRequests} + }, +# Incase of memory + payload = { + "action_name": "podMemory", + "action_params": {"name": podname, "namespace": podnamespace,"updateMemoryRequest":cpuRequests} + } +``` +### Schema for Deployment +The name of the actions in case of deployments are: +```bash +deploymentCpu and deploymentMemory +``` +```bash +name: +namespace: +updateCpuRequest: +"OR" +updateMemoryRequest: +``` +So the final schema would be like this: +```bash +# Incase of cpu + payload = { + "action_name": "deploymentCpu", + "action_params": {"name": deploymentname, "namespace": deploymentnamespace,"updateCpuRequest":cpuRequests} + }, +# Incase of memory + payload = { + "action_name": "deploymentMemory", + "action_params": {"name": deploymentname, "namespace": deploymentnamespace, "updateMemoryRequest":cpuRequests} + } +``` +### Schema for DaemonSet +The name of the actions in case of daemonsets are: +```bash +daemonSetCpu and daemonSetMemory +``` +```bash +name: +namespace: +updateCpuRequest: +"OR" +updateMemoryRequest: +``` +So the final schema would be like this: +```bash +# Incase of cpu + payload = { + "action_name": "daemonSetCpu", + "action_params": {"name": daemonSetname, "namespace": daemonSetnamespace,"updateCpuRequest":cpuRequests} + }, +# Incase of memory + payload = { + "action_name": "daemonSetMemory", + "action_params": {"name": daemonSetname, "namespace": daemonSetnamespace, "updateMemoryRequest":cpuRequests} + } +``` +### Schema for StatefulSet +The name of the actions in case of statefulSet are: +```bash +statefulSetCpu and statefulSetMemory +``` +```bash +name: +namespace: +updateCpuRequest: +"OR" +updateMemoryRequest: +``` +So the final schema would be like this: +```bash +# Incase of cpu + payload = { + "action_name": "statefulSetCpu", + "action_params": {"name": statefulSetname, "namespace": statefulSetnamespace,"updateCpuRequest":cpuRequests} + }, +# Incase of memory + payload = { + "action_name": "statefulSetMemory", + "action_params": {"name": statefulSetname, "namespace": statefulSetnamespace, "updateMemoryRequest":cpuRequests} + } +``` \ No newline at end of file diff --git a/xWorkflows/xWorkflow4/statefulSetCPU.py b/xWorkflows/xWorkflow4/statefulSetCPU.py new file mode 100644 index 0000000..55a6b43 --- /dev/null +++ b/xWorkflows/xWorkflow4/statefulSetCPU.py @@ -0,0 +1,86 @@ +import logging +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + StatefulSetEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class resizeParams(ActionParams): + updateCpuRequest: str + +@action +def statefulSetCpu(event: StatefulSetEvent, params: resizeParams): + """ + Change the CPU requests of a StatefulSet + """ + if not event.get_statefulset(): + # Log an error message if the StatefulSet is not found + logging.error("Failed to get the StatefulSet for CPU adjustment") + return + + statefulSet = event.get_statefulset() + statefulSetName = statefulSet.metadata.name + statefulSetNamespace = statefulSet.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + apps_v1 = client.AppsV1Api() + + try: + # Read the existing StatefulSet specification + statefulSetSpec = apps_v1.read_namespaced_stateful_set(name=statefulSetName, namespace=statefulSetNamespace) + + # Update the CPU requests in the StatefulSet spec + for container in statefulSetSpec.spec.template.spec.containers: + if container.resources.requests is None: + container.resources.requests = {} + container.resources.requests['cpu'] = params.updateCpuRequest + + # Update the StatefulSet with the modified spec + apps_v1.patch_namespaced_stateful_set( + name=statefulSetName, + namespace=statefulSetNamespace, + body={"spec": statefulSetSpec.spec} + ) + + logging.info(f"StatefulSet {statefulSetName} in namespace {statefulSetNamespace} has been updated with new CPU requests.") + + functionName = "Change CPU requests of StatefulSet" + finding = Finding( + title="StatefulSet CPU upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{statefulSetName} CPU is Upgraded."), + ] + ) + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying StatefulSet: {e}") + functionName = "Change CPU requests of StatefulSet Failed" + finding = Finding( + title="StatefulSet CPU upgradation Failed", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{statefulSetName} CPU Upgradation Failed."), + ] + ) + + event.add_finding(finding) diff --git a/xWorkflows/xWorkflow4/statefulSetMemory.py b/xWorkflows/xWorkflow4/statefulSetMemory.py new file mode 100644 index 0000000..f7e2877 --- /dev/null +++ b/xWorkflows/xWorkflow4/statefulSetMemory.py @@ -0,0 +1,86 @@ +import logging +from robusta.api import ( + Finding, + FindingSource, + FindingType, + MarkdownBlock, + StatefulSetEvent, + action, + ActionParams, +) +from kubernetes import client, config + +class ResizeParams(ActionParams): + updateMemoryRequest: str + +@action +def statefulSetMemory(event: StatefulSetEvent, params: ResizeParams): + """ + Change the Memory requests of a StatefulSet + """ + if not event.get_statefulset(): + # Log an error message if the StatefulSet is not found + logging.error("Failed to get the StatefulSet for Memory adjustment") + return + + statefulSet = event.get_statefulset() + statefulSetName = statefulSet.metadata.name + statefulSetNamespace = statefulSet.metadata.namespace + + # Load the kube config + config.load_incluster_config() + + # Create a Kubernetes API client + apps_v1 = client.AppsV1Api() + + try: + # Read the existing StatefulSet specification + statefulSetSpec = apps_v1.read_namespaced_stateful_set(name=statefulSetName, namespace=statefulSetNamespace) + + # Update the Memory requests in the StatefulSet spec + for container in statefulSetSpec.spec.template.spec.containers: + if container.resources.requests is None: + container.resources.requests = {} + container.resources.requests['memory'] = params.updateMemoryRequest + + # Update the StatefulSet with the modified spec + apps_v1.patch_namespaced_stateful_set( + name=statefulSetName, + namespace=statefulSetNamespace, + body={"spec": statefulSetSpec.spec} + ) + + logging.info(f"StatefulSet {statefulSetName} in namespace {statefulSetNamespace} has been updated with new Memory requests.") + + functionName = "Change Memory requests of StatefulSet" + finding = Finding( + title="StatefulSet Memory Upgradation", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{statefulSetName} Memory is Upgraded."), + ] + ) + except client.exceptions.ApiException as e: + logging.error(f"Exception when modifying StatefulSet: {e}") + functionName = "Change Memory requests of StatefulSet Failed" + finding = Finding( + title="StatefulSet Memory Upgradation Failed", + source=FindingSource.MANUAL, + aggregation_key=functionName, + finding_type=FindingType.REPORT, + failure=False, + ) + print(params) + finding.add_enrichment( + [ + MarkdownBlock(f"{statefulSetName} Memory Upgradation Failed."), + ] + ) + + event.add_finding(finding)