split backup map & backup jobs

Author: ggtrd

README.md

@@ -5,15 +5,13 @@
 > Azure Blob Storage), and pushes structured telemetry to Grafana Loki and
 > Prometheus - all from a single, provider-agnostic codebase.
 
-
 - 🗜️ **Single ZIP archive** per backup run using Python's built-in `zipfile`
 - ☁️ **Provider-agnostic** storage layer - swap S3 for Azure with one env var
 - 📋 **JSON-structured logs** parsed natively by Grafana Loki
 - 🛡️ **ZIP path-traversal protection** during restore
 - 🔒 **Dual Azure auth** - shared-key or service-principal
 - 📊 **Prometheus metrics** pushed to Pushgateway after every run
 
-
 ## Installation
 ### Install from sources
 * create venv
@@ -31,73 +29,20 @@
 * create `.env` file
   * AWS/S3
     ```
-    # Core
-    MODE=export
-    NAMESPACE=tenant-sphinx
-    PROVIDER=s3
-    BUCKET=velero
-    BACKUP_DIR=/tmp/clustersnap
-    REMOTE_PREFIX=clustersnap/backups
-    LOG_LEVEL=INFO
-
-    # AWS S3 / MinIO / SeaweedFS
-    AWS_ACCESS_KEY_ID=minioadmin
-    AWS_SECRET_ACCESS_KEY="minio&é&é&"
-    AWS_REGION=us-east-1
-    # Remove or leave blank for real AWS S3; set to MinIO / SeaweedFS address otherwise:
-    S3_ENDPOINT_URL=http://localhost:9000  minio-api.local
-
-    # Observability
-    PUSHGATEWAY_URL=http://localhost:9091  
+    tofill
     ```
 
   * Azure Blob Storage (shared-key)
     ```
-    # Core
-    MODE=export
-    NAMESPACE=tenant-alpha
-    PROVIDER=azure
-    BUCKET=clustersnap-backups
-    BACKUP_DIR=/tmp/clustersnap
-    REMOTE_PREFIX=clustersnap/backups
-    LOG_LEVEL=INFO
-
-    # Azure Blob (shared-key)
-    AZURE_STORAGE_ACCOUNT=mystorageaccount
-    AZURE_STORAGE_KEY=<your-base64-account-key>
-
-    # Observability
-    PUSHGATEWAY_URL=http://localhost:9091
-    ```
-
-  * Azure Blob Storage (service-principal)
-    ```
-    # Core
-    MODE=export
-    NAMESPACE=tenant-alpha
-    PROVIDER=azure
-    BUCKET=clustersnap-backups
-    BACKUP_DIR=/tmp/clustersnap
-    REMOTE_PREFIX=clustersnap/backups
-    LOG_LEVEL=INFO
-
-    # Azure Blob (service-principal)
-    AZURE_STORAGE_ACCOUNT=mystorageaccount
-    AZURE_CLIENT_ID=<app-registration-client-id>
-    AZURE_CLIENT_SECRET=<app-registration-client-secret>
-    AZURE_TENANT_ID=<azure-tenant-id>
-
-    # Observability
-    PUSHGATEWAY_URL=http://localhost:9091
+    tofill
     ```
 
-
 * run
   ```
   python -m src.main
   ```
 
-## Install locally with Docker
+<!-- ## Install locally with Docker
 * build
   ```
   docker build . --tag clustersnap:local
@@ -106,7 +51,7 @@
 * run
   ```
   docker run clustersnap:local
-  ```
+  ``` -->
 
 ## Install on Kubernetes (pod)
 * create secret to pull image
@@ -117,14 +62,18 @@
     --docker-password=GITHUB_PAT \
     --docker-email=GITHUB_ACCOUNT
   ```
+* create secret for remote storage access
+  * edit `src/templates/backup.secret.yaml` with your own config
+  * apply
+    ```
+    NAMESPACE=tenant-test0 envsubst < src/templates/backup.secret.yaml | kubectl apply -f -
+    ```
 * create pod
   ```
   NAMESPACE=tenant-test0 envsubst < tests/resources.yaml | kubectl apply -f -
   ```
 
-
 ## Usage
-
 * local
 ```
 python -m src.main --c CONTEXT -n NAMESPACE
@@ -135,9 +84,7 @@ just run the image
 
 
 ## Developpers
-
 ### Architecture
-
 ```
 clustersnap/
 ├── src/
@@ -170,19 +117,6 @@ clustersnap/
 ```
 Clustersnap -> Look for existing Deployments/Statefuleset -> List attached PVC -> List volumes in each PVC
 ```
-```
-CronJob trigger
-     │
-     ▼
-main.py --mode export
-     │
-     ├─► config.py          - validate all env vars (fail fast)
-     ├─► storage/<provider> - health check (HeadBucket / GetContainerProperties)
-     ├─► exporter.py        - zip /data/src-disks → /tmp/clustersnap/<ns>/<ts>.zip
-     ├─► storage.upload()   - stream ZIP to bucket
-     ├─► metrics.py         - record success / duration / size
-     └─► metrics.push()     - push to Prometheus Pushgateway
-```
 
 ### Workflow of import
 tofill

src/helpers/kubernetes.py

@@ -5,8 +5,9 @@
 
 class Kubernetes:
 
-    def __init__(self, context: str = None, given_namespace: str = None):
-        self.given_namespace = given_namespace
+    def __init__(self, context: str, namespace: str):
+        self.context = context
+        self.namespace = namespace
         try:
             # Connect from CLI (based on kubeconfig)
             if context:
@@ -22,21 +23,18 @@ def __init__(self, context: str = None, given_namespace: str = None):
             print(f"error: {e}")
 
 
-    def get_current_namespace(self) -> str:
-        # Use namespace if given from args
-        if self.given_namespace:
-            return self.given_namespace
+    def get_context(self) -> str:
+        if self.context:
+            return self.context
 
-        # Or fallback on current namespace if running in a pod
-        ns_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
-        if os.path.exists(ns_path):
-            with open(ns_path, "r") as f:
-                return f.read().strip()
-        return "default"
+
+    def get_namespace(self) -> str:
+        if self.namespace:
+            return self.namespace
 
 
     def list_pvc(self) -> list:
-        current_ns = self.get_current_namespace()
+        current_ns = self.get_namespace()
         pvc_list = []
         try:
             response = self.client_corev1.list_namespaced_persistent_volume_claim(namespace=current_ns)
@@ -54,7 +52,7 @@ def list_pvc(self) -> list:
 
     # Look for PVC from Deployments and StatefulSets
     def get_pvc(self, pvc: str) -> list:
-        current_ns = self.get_current_namespace()
+        current_ns = self.get_namespace()
         targets = []
 
         try:
@@ -110,15 +108,32 @@ def _list_pvc_mount_points(self, pvc: str, pod_spec) -> list:
         return results
 
 
+    # Check if a secret exists in a namespace
+    def check_secret_exists(self, secret: str) -> bool:
+        current_ns = self.get_namespace()
+        try:
+            self.client_corev1.read_namespaced_secret(name=secret, namespace=current_ns)
+            return True
+        except Exception as e:
+            if e.status == 404:
+                return False
+            print(f"error: {e}")
+            return False
 
 
     def create_backup_job(self, pvc: str, mounted_path: str) -> bool:
-        current_ns = self.get_current_namespace()
-        
+        current_ns = self.get_namespace()
+
+        # Ensure secret clustersnap-config exists. It must contains the remote storage access
+        secret_required = "clustersnap-config"
+        if not self.check_secret_exists(secret_required):
+            print(f"error: Secret '{secret_required}' is missing in namespace '{current_ns}'. Cancelling backup job.")
+            return False
+
         job_name = f"backup-{pvc}"[-63:].lower().strip("-")
         print(f"starting backup job: {job_name}")
 
-        template_path = os.path.join(os.path.dirname(__file__), "../..", "templates", "backup-job.yaml")
+        template_path = os.path.join(os.path.dirname(__file__), "../..", "templates", "backup.job.yaml")
 
         try:
             with open(template_path, "r") as f:

src/main.py

@@ -3,25 +3,11 @@
 from src.helpers.kubernetes import Kubernetes
 
 
-def main():
-    # # If an argument is provided = we're using from CLI
-    # try:
-    parser = argparse.ArgumentParser(description="Backup Kubernetes volumes")
-    parser.add_index = False
-    parser.add_argument("-c", "--context", type=str, default=None, help="Local kubectl context")
-    parser.add_argument("-n", "--namespace", type=str, default=None, help="Kubernetes namespace")
-    args = parser.parse_args()
-
-    kubernetes = Kubernetes(context=args.context, given_namespace=args.namespace)
-    print(f"context: {args.context}")
-
-    # # If no argument is provided = we're in Kubernetes
-    # except:
-    #     kubernetes = Kubernetes()
-    #     print(f"context: local")
-
-    current_ns = kubernetes.get_current_namespace()
+# Get a YAML map of what will be backuped
+def get_backup_map(kubernetes: Kubernetes):
+    current_ns = kubernetes.get_namespace()
 
+    print(f"context: {kubernetes.get_context()}")
     print(f"namespaces:")
     print(f"- name: {current_ns}")
 
@@ -30,6 +16,7 @@ def main():
         print("error: no PVC found or unauthorized access.")
         return 0
 
+    # Loop to display the backup map
     print(f"  claims:")
     for pvc in kubernetes.list_pvc():
         print(f"    - name: {pvc['name']}")
@@ -44,13 +31,42 @@ def main():
                 print(f"        path: {m['mount_path']}")
                 print(f"        read_only: {m['read_only']}")
                 print(f"        resource: {m['resource']}")
-
-                # volume_name_clean = print(f"{m['resource'].split("/")[1]}-{m['container_name']}-{m['mount_path'].replace('/','')}")
-                # kubernetes.create_backup_job(pvc=pvc['name'], mounted_path=m['mount_path'], volume_name=m['volume_name_clean'])
-
-                # kubernetes.create_backup_job(pvc=pvc['name'], mounted_path=m['mount_path'])
         else:
             print(f"      mounts: {mounts}")
 
+
+# Run a backup job
+def run_backup_job(kubernetes: Kubernetes):
+    # Display the map before running the job
+    get_backup_map(kubernetes)
+
+    for pvc in kubernetes.list_pvc():
+        mounts = kubernetes.get_pvc(pvc['name'])
+        if mounts:
+            for m in mounts:
+                kubernetes.create_backup_job(pvc=pvc['name'], mounted_path=m['mount_path'])
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Backup Kubernetes volumes")
+    parser.add_index = False
+    parser.add_argument("-c", "--context", type=str, default=None, help="Local kubectl context")
+    parser.add_argument("-n", "--namespace", type=str, default=None, help="Kubernetes namespace")
+    parser.add_argument("-j", "--job", action="store_true", help="Run a backup job")
+    parser.add_argument("-m", "--map", action="store_true", help="Get the backup map")
+    args = parser.parse_args()
+
+    kubernetes = Kubernetes(context=args.context, namespace=args.namespace)
+    
+    if args.job:
+        run_backup_job(kubernetes)
+
+    elif args.map:
+        get_backup_map(kubernetes)
+
+    else:
+        print('please run with --job or --map (--help for more info)')
+
+
 if __name__ == "__main__":
     sys.exit(main())

src/templates/backup.job.yaml

@@ -22,16 +22,19 @@ spec:
                 secretKeyRef:
                   name: clustersnap-config
                   key: storage-address
+                  optional: false
             - name: STORAGE_USERNAME
               valueFrom:
                 secretKeyRef:
                   name: clustersnap-config
                   key: storage-username
+                  optional: false
             - name: STORAGE_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: clustersnap-config
                   key: storage-password
+                  optional: false
       volumes:
         - name: volume-to-backup
           persistentVolumeClaim: