make benchmark scripts gh actions ready.

Author: ParagEkbote

.github/workflows/run_benchmark.yml

@@ -7,6 +7,10 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
 
+    env:
+      # Make token available to scripts
+      REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
@@ -23,20 +27,31 @@ jobs:
 
       - name: Run all benchmark scripts
         run: |
-          BENCH_DIR="/workspaces/quantized-containerized-models/benchmark"
+          BENCH_DIR="./benchmark"
+          RESULTS_DIR="./benchmark/results"
+
+          mkdir -p "$RESULTS_DIR"
+
           echo "Running benchmarks in: $BENCH_DIR"
 
+          # Run each benchmark script and ensure results get saved
           find "$BENCH_DIR" -type f -name "benchmark.py" | while read script; do
             echo "=============================================="
             echo "Executing: $script"
             echo "=============================================="
+
+            # Ensure each script can detect output directory
+            RESULTS_ENV="BENCHMARK_OUTPUT_DIR=$RESULTS_DIR"
+
+            # Run the benchmark with env vars applied
+            REPLICATE_API_TOKEN="$REPLICATE_API_TOKEN" \
+            BENCHMARK_OUTPUT_DIR="$RESULTS_DIR" \
             python "$script"
           done
 
       - name: Upload all benchmark results
         uses: actions/upload-artifact@v4
         with:
           name: benchmark-results
-          path: |
-            /quantized-containerized-models/benchmark
+          path: benchmark/results
           if-no-files-found: warn

Makefile

@@ -112,7 +112,7 @@ deployment: ## Run deployment tests
 # CI → Unit+Linting tests only
 # ----------------------------------------
 .PHONY: ci
-ci: lint unit 
+ci: lint unit
 
 
 # ----------------------------------------

benchmark/flux_fast_lora_hotswap/benchmark.py

@@ -1,44 +1,56 @@
-import replicate
+import os
 import time
 import json
 import logging
 from datetime import datetime
+from pathlib import Path
+
+import replicate
+
+
+# ============================================================
+# Output directory (for GitHub Actions artifacts)
+# ============================================================
+OUTPUT_DIR = Path(os.getenv("BENCHMARK_OUTPUT_DIR", "benchmark_results_flux_text2img"))
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+LOG_FILE = OUTPUT_DIR / "benchmark_flux_lora_text2img.log"
 
-# ---------------------------------------------------------
-# Logging Setup
-# ---------------------------------------------------------
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s - %(levelname)s - %(message)s",
-    handlers=[logging.FileHandler("benchmark_flux_lora_text2img.log"), logging.StreamHandler()],
+    handlers=[logging.FileHandler(LOG_FILE), logging.StreamHandler()],
 )
 logger = logging.getLogger(__name__)
 
 
-# ---------------------------------------------------------
+# ============================================================
 # Benchmark Function
-# ---------------------------------------------------------
+# ============================================================
 def benchmark_flux_lora_text2img(
     num_runs=3,
     prompt="A majestic dragon soaring above a futuristic city.",
     trigger_word="Painting",
 ):
     """
-    Benchmark for Flux Fast LoRA (text-to-image) variant.
-    Only uses prompt + trigger_word as required by schema.
+    Benchmark Flux Fast LoRA (text → image).
+    Only uses: prompt + trigger_word (schema compliant).
     """
 
     deployment_id = (
         "paragekbote/flux-fast-lora-hotswap:"
         "a958687317369721e1ce66e5436fa989bcff2e40a13537d9b4aa4c6af4a34539"
     )
 
-    # Schema-compliant input
+    # Replicate client using secure API token (required)
+    client = replicate.Client(api_token=os.environ["REPLICATE_API_TOKEN"])
+
     input_params = {
         "prompt": prompt,
         "trigger_word": trigger_word,
     }
 
+    # Header logging
     logger.info("=" * 90)
     logger.info("FLUX FAST LORA (TEXT → IMAGE) BENCHMARK")
     logger.info("=" * 90)
@@ -50,44 +62,45 @@ def benchmark_flux_lora_text2img(
 
     results = {
         "deployment_id": deployment_id,
-        "deployment_name": "Flux-Fast-LoRA-Text2Img",
+        "deployment_name": "flux-fast-lora-text2img",
         "timestamp": datetime.now().isoformat(),
         "input_params": input_params,
         "runs": [],
     }
 
     run_times = []
 
-    # ---------------------------------------------------------
+    # ============================================================
     # Execute Runs
-    # ---------------------------------------------------------
+    # ============================================================
     for run_num in range(1, num_runs + 1):
         logger.info(f"--- Run {run_num}/{num_runs} ---")
 
         try:
             start = time.time()
 
-            # Replicate returns a File object for images
-            output_file = replicate.run(deployment_id, input=input_params)
+            # Replicate returns a single File object for image output
+            output_file = client.run(deployment_id, input=input_params)
+
+            # Read image bytes
             img_bytes = output_file.read()
 
             elapsed = time.time() - start
             run_times.append(elapsed)
 
-            # Save to disk
-            out_path = f"flux_lora_text2img_run_{run_num}.png"
-            with open(out_path, "wb") as f:
-                f.write(img_bytes)
+            # Save image output
+            img_path = OUTPUT_DIR / f"flux_lora_text2img_run_{run_num}.png"
+            img_path.write_bytes(img_bytes)
 
             logger.info(f"✓ Completed in {elapsed:.2f}s")
-            logger.info(f"  Generated: {out_path}")
+            logger.info(f"  Generated: {img_path}")
             logger.info(f"  URL: {output_file.url()}")
 
             results["runs"].append(
                 {
                     "run_number": run_num,
                     "elapsed_time": elapsed,
-                    "output_file": out_path,
+                    "output_file": str(img_path),
                     "file_url": output_file.url(),
                     "status": "success",
                 }
@@ -97,85 +110,91 @@ def benchmark_flux_lora_text2img(
             err = str(e)
             logger.error(f"✗ Run failed: {err}")
 
-            results["runs"].append({
-                "run_number": run_num,
-                "status": "failed",
-                "error": err
-            })
+            results["runs"].append(
+                {
+                    "run_number": run_num,
+                    "status": "failed",
+                    "error": err,
+                }
+            )
 
-    # ---------------------------------------------------------
+    # ============================================================
     # Statistics
-    # ---------------------------------------------------------
+    # ============================================================
     if run_times:
-        avg_t = sum(run_times) / len(run_times)
-        min_t = min(run_times)
-        max_t = max(run_times)
+        avg_time = sum(run_times) / len(run_times)
+        min_time = min(run_times)
+        max_time = max(run_times)
 
-        std = (sum((t - avg_t) ** 2 for t in run_times) / len(run_times)) ** 0.5
-        cv = (std / avg_t * 100) if avg_t else 0
+        std_dev = (sum((t - avg_time) ** 2 for t in run_times) / len(run_times)) ** 0.5
+        cv = (std_dev / avg_time * 100) if avg_time else 0
 
-        cold = run_times[0]
-        warm = run_times[1:]
-        warm_avg = sum(warm) / len(warm) if warm else None
+        cold_start = run_times[0]
+        warm_times = run_times[1:]
+        warm_avg = sum(warm_times) / len(warm_times) if warm_times else None
 
         results["statistics"] = {
             "successful_runs": len(run_times),
             "failed_runs": num_runs - len(run_times),
-            "avg_latency": avg_t,
-            "min_latency": min_t,
-            "max_latency": max_t,
-            "std_dev_latency": std,
+            "avg_latency": avg_time,
+            "min_latency": min_time,
+            "max_latency": max_time,
+            "std_dev_latency": std_dev,
             "latency_cv_percent": cv,
-            "cold_start_latency": cold,
+            "cold_start_latency": cold_start,
             "warm_avg_latency": warm_avg,
-            "cold_vs_warm_ratio": cold / warm_avg if warm_avg else None,
+            "cold_vs_warm_ratio": cold_start / warm_avg if warm_avg else None,
         }
 
-        # Insights
         insights = []
 
-        if warm_avg and cold > warm_avg * 1.5:
-            insights.append(f"Cold start significantly slower ({cold:.2f}s vs warm {warm_avg:.2f}s).")
+        if warm_avg and cold_start > warm_avg * 1.5:
+            insights.append(
+                f"Cold start significantly slower ({cold_start:.2f}s vs warm {warm_avg:.2f}s)."
+            )
 
         if cv < 20:
             insights.append(f"Stable performance (CV {cv:.1f}%).")
         elif cv > 50:
             insights.append(f"High latency variability (CV {cv:.1f}%).")
 
-        if avg_t < 4:
-            insights.append("Extremely fast text-to-image generation.")
-        elif avg_t < 8:
+        if avg_time < 4:
+            insights.append("Extremely fast image generation.")
+        elif avg_time < 8:
             insights.append("Good generation speed.")
         else:
-            insights.append("Slow generation — possible cold backend or high load.")
+            insights.append("Slow generation — backend cold or overloaded.")
 
         results["insights"] = insights
 
-        # Log summary
+        # Summary log
         logger.info("\n" + "=" * 90)
         logger.info("BENCHMARK SUMMARY")
         logger.info("=" * 90)
-        for k, v in results["statistics"].items():
-            logger.info(f"{k}: {v}")
+
+        for key, val in results["statistics"].items():
+            logger.info(f"{key}: {val}")
+
         logger.info("\nInsights:")
-        for i in insights:
-            logger.info(f" • {i}")
+        for insight in insights:
+            logger.info(f" • {insight}")
+
         logger.info("=" * 90)
 
-    # ---------------------------------------------------------
-    # Save JSON
-    # ---------------------------------------------------------
-    with open("flux_lora_text2img_benchmark.json", "w") as f:
-        json.dump(results, f, indent=2)
+    # ============================================================
+    # Save JSON Results
+    # ============================================================
+    json_path = OUTPUT_DIR / "flux_lora_text2img_benchmark.json"
+    json_path.write_text(json.dumps(results, indent=2))
 
-    logger.info("\n✓ Saved: flux_lora_text2img_benchmark.json")
-    logger.info("✓ Logs: benchmark_flux_lora_text2img.log\n")
+    logger.info(f"\n✓ JSON saved to {json_path}")
+    logger.info(f"✓ Logs saved to {LOG_FILE}\n")
 
     return results
 
 
-# ---------------------------------------------------------
-# Run the benchmark
-# ---------------------------------------------------------
+# ============================================================
+# Execute
+# ============================================================
 if __name__ == "__main__":
     benchmark_flux_lora_text2img(num_runs=3)