From cb7959c5b11f055707c3d5ce2ee256d3b3070f6c Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 13:34:13 +0200 Subject: [PATCH 01/48] automate notebook examples end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/get_or_create_api_key.py: create/retrieve AIGatewayKey (importable + CLI) - scripts/setup_mlflow_credentials.py: one-time MLflow PAT → k8s secret (importable + CLI, optional --uri/--username/--password args) - notebooks/dask/dask_example.ipynb: auto-detect namespace from SA token file - mlflow/{quickstart,image}: read MLflow creds from mlflow-credentials k8s secret instead of hardcoded placeholders - mlflow/{kfp,mobile-price}: update markdown to reference setup_mlflow_credentials.py - serving/minimal-s3-model: auto-create API key via get_or_create_api_key - serving/mlflow-kserve-inference-protocols: add setup cell that applies YAML templates with substituted namespace/username, waits for ISVC readiness, and auto-fetches URIs and API key - serving/mlflow-kserve-minimal/apply.py: deploy InferenceService with substituted values, wait for ready, print URI - pipelines/lightweight-python-package/submit-cluster.py: print KFP_RUN_ID for CI polling - cleanup.py per example: idempotent kubectl delete for k8s resources only - ci/run_all.py: papermill-based orchestrator with parallel phases, KFP run polling, and cleanup in finally block --- ci/run_all.py | 454 ++++++++++++++++++ hparam-tuning/minimal-mnist/cleanup.py | 54 +++ mlflow/mlflow-image-example.ipynb | 25 +- mlflow/mlflow-kfp-example.ipynb | 11 +- mlflow/mlflow-quickstart-example.ipynb | 72 +-- .../mlflow-mobile-price-classification.ipynb | 8 +- notebooks/dask/cleanup.py | 53 ++ notebooks/dask/dask_example.ipynb | 6 +- .../submit-cluster.py | 5 +- scripts/get_or_create_api_key.py | 116 +++++ scripts/setup_mlflow_credentials.py | 125 +++++ serving/minimal-s3-model/cleanup.py | 63 +++ .../minimal-s3-model/minimal-s3-model.ipynb | 9 +- .../cleanup.py | 56 +++ .../inference_protocol_version_example.ipynb | 107 ++++- serving/mlflow-kserve-minimal/apply.py | 150 ++++++ serving/mlflow-kserve-minimal/cleanup.py | 49 ++ 17 files changed, 1271 insertions(+), 92 deletions(-) create mode 100644 ci/run_all.py create mode 100644 hparam-tuning/minimal-mnist/cleanup.py create mode 100644 notebooks/dask/cleanup.py create mode 100644 scripts/get_or_create_api_key.py create mode 100644 scripts/setup_mlflow_credentials.py create mode 100644 serving/minimal-s3-model/cleanup.py create mode 100644 serving/mlflow-kserve-inference-protocols/cleanup.py create mode 100644 serving/mlflow-kserve-minimal/apply.py create mode 100644 serving/mlflow-kserve-minimal/cleanup.py diff --git a/ci/run_all.py b/ci/run_all.py new file mode 100644 index 0000000..17d61ea --- /dev/null +++ b/ci/run_all.py @@ -0,0 +1,454 @@ +""" +CI orchestrator — runs all automatable examples and reports results. + +Execution model +--------------- +Phase 1 (parallel, self-contained): + notebooks/dask/dask_example.ipynb + mlflow/mlflow-quickstart-example.ipynb + mlflow/mlflow-image-example.ipynb + mlflow/mlflow-kfp-example.ipynb + serving/minimal-s3-model/minimal-s3-model.ipynb + +Phase 2 (parallel, pipeline submissions — return fast): + mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb + pipelines/lightweight-components/mobile-price-classifications.ipynb + pipelines/lightweight-python-package/submit-cluster.py (Python script) + +Phase 3 (depends on Phase 2 mlflow-mobile-price notebook completing): + serving/mlflow-kserve-minimal/apply.py + test_inference_service.py + serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb + +Phase 4 (KFP run polling — runs alongside Phase 3): + Poll all KFP run IDs extracted from Phase 2 output notebooks until + Succeeded or timeout. + +Phase 5 — cleanup (always, in finally block): + All cleanup.py scripts in parallel. + +Usage from a Jupyter notebook +------------------------------ + import subprocess + subprocess.run(["python", "ci/run_all.py"], check=True) + +CLI usage +--------- + python ci/run_all.py [--timeout-notebook 1800] [--timeout-pipeline 3600] + [--include-keda] [--dry-run] + +Prerequisites +------------- + pip install papermill kfp + python scripts/setup_mlflow_credentials.py +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import subprocess +import sys +import time +from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# ── Repo root ───────────────────────────────────────────────────────────────── +_REPO_ROOT = Path( + subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() +) + +# ── KFP run ID pattern (UUID v4) ───────────────────────────────────────────── +_RUN_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +# ── Result dataclass ────────────────────────────────────────────────────────── + + +@dataclass +class Result: + name: str + status: str = "PENDING" # PASS | FAIL | SKIP + duration: float = 0.0 + error: str = "" + kfp_run_ids: list[str] = field(default_factory=list) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> tuple[Path, str]: + """Execute a notebook with papermill. Returns (output_path, stderr).""" + try: + import papermill as pm + except ImportError: + raise RuntimeError("papermill is not installed. Run: pip install papermill") + + output_path = output_dir / nb_path.name + output_dir.mkdir(parents=True, exist_ok=True) + pm.execute_notebook( + str(nb_path), + str(output_path), + kernel_name="python3", + execution_timeout=timeout, + cwd=str(nb_path.parent), + ) + return output_path, "" + + +def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: + """Run a plain Python script. Returns (stdout, stderr).""" + result = subprocess.run( + [sys.executable, str(script_path)], + capture_output=True, + text=True, + timeout=timeout, + cwd=str(script_path.parent), + ) + if result.returncode != 0: + raise RuntimeError(result.stderr) + return result.stdout, result.stderr + + +def _extract_run_ids_from_notebook(output_nb: Path) -> list[str]: + """Parse a papermill output notebook for KFP run IDs.""" + with open(output_nb) as fh: + nb = json.load(fh) + ids: list[str] = [] + for cell in nb.get("cells", []): + for output in cell.get("outputs", []): + text = "".join( + output.get("text", []) + output.get("data", {}).get("text/plain", []) + ) + ids.extend(_RUN_ID_RE.findall(text)) + return list(dict.fromkeys(ids)) # deduplicate, preserve order + + +def _extract_run_ids_from_stdout(stdout: str) -> list[str]: + """Parse stdout from a script for KFP_RUN_ID= lines.""" + ids = [] + for line in stdout.splitlines(): + if line.startswith("KFP_RUN_ID="): + ids.append(line.split("=", 1)[1].strip()) + return ids + + +def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: + """Poll a KFP run until terminal state. Returns final status string.""" + try: + from kfp.client import Client + except ImportError: + return "SKIP (kfp not installed)" + client = Client() + deadline = time.time() + timeout + while time.time() < deadline: + run = client.get_run(run_id) + state = run.run.status + if state in ("Succeeded", "Failed", "Error", "Skipped"): + return state + time.sleep(interval) + return f"TIMEOUT after {timeout}s" + + +def _run_cleanup(cleanup_path: Path) -> None: + """Run a cleanup.py; log but don't raise on failure.""" + try: + subprocess.run( + [sys.executable, str(cleanup_path)], + capture_output=True, + text=True, + cwd=str(cleanup_path.parent), + timeout=120, + ) + except Exception as exc: # noqa: BLE001 + print(f" WARNING: cleanup {cleanup_path.name} failed: {exc}", file=sys.stderr) + + +# ── Executor ────────────────────────────────────────────────────────────────── + + +def _submit_notebook( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + nb_path: Path, + output_dir: Path, + timeout: int, +) -> Future: + result = Result(name=name) + results[name] = result + + def _run(): + t0 = time.time() + try: + out, _ = _run_notebook(nb_path, output_dir, timeout) + result.status = "PASS" + result.kfp_run_ids = _extract_run_ids_from_notebook(out) + except Exception as exc: # noqa: BLE001 + result.status = "FAIL" + result.error = str(exc)[:300] + finally: + result.duration = time.time() - t0 + + return executor.submit(_run) + + +def _submit_script( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + script_path: Path, + timeout: int, +) -> Future: + result = Result(name=name) + results[name] = result + + def _run(): + t0 = time.time() + try: + stdout, _ = _run_script(script_path, timeout) + result.status = "PASS" + result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) + except Exception as exc: # noqa: BLE001 + result.status = "FAIL" + result.error = str(exc)[:300] + finally: + result.duration = time.time() - t0 + + return executor.submit(_run) + + +# ── Report ──────────────────────────────────────────────────────────────────── + + +def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: + col = 50 + print("\n" + "=" * 70) + print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") + print("-" * 70) + passed = failed = 0 + for r in results.values(): + dur = f"{r.duration:.0f}s" if r.duration else "-" + print(f"{r.name:<{col}} {r.status:<10} {dur}") + if r.status == "FAIL": + print(f" {'':>{col}} {r.error}") + failed += 1 + else: + passed += 1 + if poll_results: + print() + print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") + print("-" * 70) + for run_id, status in poll_results.items(): + short = run_id[:8] + "..." + ok = status == "Succeeded" + if not ok: + failed += 1 + else: + passed += 1 + print(f"{short:<{col}} {status:<10}") + print("=" * 70) + print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") + print() + + +# ── Main ────────────────────────────────────────────────────────────────────── + + +def run_all( + timeout_notebook: int = 1800, + timeout_pipeline: int = 3600, + include_keda: bool = False, + dry_run: bool = False, +) -> dict[str, Result]: + root = _REPO_ROOT + output_dir = root / "ci" / "output" + results: dict[str, Result] = {} + poll_results: dict[str, str] = {} + + if dry_run: + print("[dry-run] Would execute the following phases:") + for label in [ + "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model", + "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", + "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", + "Phase 4: KFP run polling", + "Phase 5: cleanup", + ]: + print(f" {label}") + return results + + cleanup_scripts = [ + root / "notebooks" / "dask" / "cleanup.py", + root / "serving" / "minimal-s3-model" / "cleanup.py", + root / "serving" / "mlflow-kserve-minimal" / "cleanup.py", + root / "serving" / "mlflow-kserve-inference-protocols" / "cleanup.py", + root / "hparam-tuning" / "minimal-mnist" / "cleanup.py", + ] + + try: + with ThreadPoolExecutor(max_workers=8) as executor: + # ── Phase 1: independent notebooks ─────────────────────────────── + print("Phase 1: running independent notebooks in parallel...") + phase1_futures = {} + for name, rel in [ + ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), + ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), + ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), + ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), + ( + "serving/minimal-s3-model", + "serving/minimal-s3-model/minimal-s3-model.ipynb", + ), + ]: + f = _submit_notebook( + executor, results, name, root / rel, output_dir, timeout_notebook + ) + phase1_futures[name] = f + + # wait for all phase 1 + for name, f in phase1_futures.items(): + f.result() + print(f" [{results[name].status}] {name}") + + # ── Phase 2: pipeline submissions (return fast) ─────────────────── + print("\nPhase 2: submitting pipelines...") + phase2_futures = {} + for name, rel in [ + ( + "mlflow/mobile-price-classification", + "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", + ), + ( + "pipelines/lightweight-components", + "pipelines/lightweight-components/mobile-price-classifications.ipynb", + ), + ]: + f = _submit_notebook( + executor, results, name, root / rel, output_dir, timeout_notebook + ) + phase2_futures[name] = f + + lpp_future = _submit_script( + executor, + results, + "pipelines/lightweight-python-package", + root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", + timeout_notebook, + ) + phase2_futures["pipelines/lightweight-python-package"] = lpp_future + + # Wait only for mlflow-mobile-price before launching phase 3 + mlflow_mobile_future = phase2_futures["mlflow/mobile-price-classification"] + mlflow_mobile_future.result() + print( + f" [{results['mlflow/mobile-price-classification'].status}] mlflow/mobile-price-classification" + ) + + # ── Phase 3: MLflow KServe examples (need registered model) ─────── + print("\nPhase 3: deploying MLflow KServe InferenceServices...") + phase3_futures = {} + for name, rel in [ + ( + "serving/mlflow-kserve-minimal", + "serving/mlflow-kserve-minimal/apply.py", + ), + ( + "serving/mlflow-kserve-inference-protocols", + "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + ), + ]: + path = root / rel + if path.suffix == ".ipynb": + f = _submit_notebook( + executor, results, name, path, output_dir, timeout_notebook + ) + else: + f = _submit_script(executor, results, name, path, timeout_notebook) + phase3_futures[name] = f + + # Wait for remaining phase 2 futures while phase 3 runs + for name, f in phase2_futures.items(): + if name != "mlflow/mobile-price-classification": + f.result() + print(f" [{results[name].status}] {name}") + + # Wait for phase 3 + for name, f in phase3_futures.items(): + f.result() + print(f" [{results[name].status}] {name}") + + # ── Phase 4: KFP run polling ────────────────────────────────────── + all_run_ids: list[str] = [] + for r in results.values(): + all_run_ids.extend(r.kfp_run_ids) + + if all_run_ids: + print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") + poll_futures = { + run_id: executor.submit(_poll_kfp_run, run_id, timeout_pipeline) + for run_id in all_run_ids + } + for run_id, f in poll_futures.items(): + poll_results[run_id] = f.result() + print(f" [{poll_results[run_id]}] {run_id[:8]}...") + else: + print("\nPhase 4: no KFP run IDs found, skipping poll.") + + finally: + # ── Phase 5: cleanup ───────────────────────────────────────────────── + print("\nPhase 5: running cleanup scripts...") + with ThreadPoolExecutor(max_workers=4) as ex: + futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] + for f in as_completed(futs): + f.result() # already swallows exceptions inside _run_cleanup + + _print_report(results, poll_results) + return results + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--timeout-notebook", + type=int, + default=1800, + help="Per-notebook execution timeout in seconds (default: 1800)", + ) + parser.add_argument( + "--timeout-pipeline", + type=int, + default=3600, + help="KFP run poll timeout in seconds (default: 3600)", + ) + parser.add_argument( + "--include-keda", + action="store_true", + help="Include GPU-dependent KEDA autoscaling example (opt-in)", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print plan without executing anything" + ) + args = parser.parse_args() + + results = run_all( + timeout_notebook=args.timeout_notebook, + timeout_pipeline=args.timeout_pipeline, + include_keda=args.include_keda, + dry_run=args.dry_run, + ) + + failed = sum(1 for r in results.values() if r.status == "FAIL") + sys.exit(1 if failed else 0) diff --git a/hparam-tuning/minimal-mnist/cleanup.py b/hparam-tuning/minimal-mnist/cleanup.py new file mode 100644 index 0000000..32d6475 --- /dev/null +++ b/hparam-tuning/minimal-mnist/cleanup.py @@ -0,0 +1,54 @@ +""" +Cleanup script for the hparam-tuning/minimal-mnist example. + +Deletes (idempotently): + - Katib Experiment ``minimal-mnist`` + - Any Trial pods created by the Experiment (Katib garbage-collects these + automatically, but this speeds things up if you need to clean up quickly) + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_EXPERIMENT_NAME = "minimal-mnist" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + # Deleting the Experiment cascades to Trials via owner references + _kubectl_delete("experiment", _EXPERIMENT_NAME, "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 186cdf6..0a5b36d 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -42,12 +42,27 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", + "import base64, json, os, subprocess\n", "\n", - "# Set your MLflow credentials\n", - "os.environ['MLFLOW_TRACKING_URI'] = 'https:///mlflow'\n", - "os.environ['MLFLOW_TRACKING_USERNAME'] = '' \n", - "os.environ['MLFLOW_TRACKING_PASSWORD'] = ''" + "# Load MLflow credentials from the mlflow-credentials Kubernetes secret.\n", + "# Run scripts/setup_mlflow_credentials.py once to create this secret.\n", + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " _ns = _f.read().strip()\n", + "\n", + "_result = subprocess.run(\n", + " [\"kubectl\", \"get\", \"secret\", \"mlflow-credentials\", \"-n\", _ns, \"-o\", \"json\"],\n", + " capture_output=True, text=True,\n", + ")\n", + "if _result.returncode != 0:\n", + " raise RuntimeError(\n", + " \"mlflow-credentials secret not found.\\n\"\n", + " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", + " )\n", + "_data = json.loads(_result.stdout)[\"data\"]\n", + "os.environ[\"MLFLOW_TRACKING_URI\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_URI\"]).decode()\n", + "os.environ[\"MLFLOW_TRACKING_USERNAME\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_USERNAME\"]).decode()\n", + "os.environ[\"MLFLOW_TRACKING_PASSWORD\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_PASSWORD\"]).decode()\n", + "os.environ[\"MLFLOW_ENABLE_PROXY_MULTIPART_UPLOAD\"] = \"true\"\n" ] }, { diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 7ef127f..4477f41 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -23,15 +23,12 @@ " - Copy the generated token and store it securely\n", " - Note: You won't be able to see the token again!\n", "\n", - "Now change copy below command and change it to your values:\n", + "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", "```bash\n", - "kubectl create secret generic mlflow-credentials \\\n", - "--from-literal=MLFLOW_TRACKING_URI='https:///mlflow' \\\n", - "--from-literal=MLFLOW_TRACKING_USERNAME='your-email@example.com' \\\n", - "--from-literal=MLFLOW_TRACKING_PASSWORD='your-personal-access-token'\n", + "# From a JupyterLab terminal:\n", + "python examples/scripts/setup_mlflow_credentials.py\n", "```\n", - "\n", - "You can then paste the above command in the terminal built into jupyter-lab and press enter to run it.\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", "## How it works\n", "\n", diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index f1927bc..06c796a 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -52,61 +52,27 @@ }, "outputs": [], "source": [ - "import os\n", + "import base64, json, os, subprocess\n", "\n", - "# Set your MLflow credentials\n", - "os.environ['MLFLOW_TRACKING_URI'] = 'https:///mlflow'\n", - "os.environ['MLFLOW_TRACKING_USERNAME'] = '' \n", - "os.environ['MLFLOW_TRACKING_PASSWORD'] = ''\n", + "# Load MLflow credentials from the mlflow-credentials Kubernetes secret.\n", + "# Run scripts/setup_mlflow_credentials.py once to create this secret.\n", + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " _ns = _f.read().strip()\n", "\n", - "# Enable presigned URL uploads to avoid RAM buffering in MLflow server\n", - "os.environ['MLFLOW_ENABLE_PROXY_MULTIPART_UPLOAD'] = 'true'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "02a920aa-df99-4902-b9ca-b534542882c7", - "metadata": {}, - "outputs": [], - "source": [ - "# Validate MLflow configuration\n", - "import os\n", - "\n", - "REQUIRED_ENV_VARS = [\"MLFLOW_TRACKING_URI\", \"MLFLOW_TRACKING_USERNAME\", \"MLFLOW_TRACKING_PASSWORD\"]\n", - "missing_vars = [var for var in REQUIRED_ENV_VARS if not os.environ.get(var)]\n", - "placeholder_vars = [\n", - " var for var in REQUIRED_ENV_VARS \n", - " if os.environ.get(var, \"\").startswith((\"https://<\", \"/mlflow' \\\n", - " --from-literal=MLFLOW_TRACKING_USERNAME='your-email@example.com' \\\n", - " --from-literal=MLFLOW_TRACKING_PASSWORD='your-personal-access-token'\n", + "# From a JupyterLab terminal:\n", + "python examples/scripts/setup_mlflow_credentials.py\n", "```\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." ] diff --git a/notebooks/dask/cleanup.py b/notebooks/dask/cleanup.py new file mode 100644 index 0000000..0269852 --- /dev/null +++ b/notebooks/dask/cleanup.py @@ -0,0 +1,53 @@ +""" +Cleanup script for the dask example. + +Deletes (idempotently): + - Any DaskCluster CRs left running in the current namespace + (dask-kubernetes names them after the KubeCluster object; the example + uses the default name ``dask-cluster``) + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_CLUSTER_NAME = "dask-cluster" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete("daskcluster", _CLUSTER_NAME, "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index a0686ce..77fc563 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -17,7 +17,8 @@ "metadata": {}, "outputs": [], "source": [ - "namespace = 'your-namespace-name' # put the name of your namespace here" + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " namespace = _f.read().strip()\n" ] }, { @@ -27,8 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "if namespace == 'your-namespace-name':\n", - " raise ValueError('You need to set the `namespace` variable to your personal namespace')" + "print(f\"Using namespace: {namespace}\")\n" ] }, { diff --git a/pipelines/lightweight-python-package/submit-cluster.py b/pipelines/lightweight-python-package/submit-cluster.py index d9499ae..fc4485b 100644 --- a/pipelines/lightweight-python-package/submit-cluster.py +++ b/pipelines/lightweight-python-package/submit-cluster.py @@ -4,7 +4,9 @@ if __name__ == "__main__": - with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r", encoding="utf-8") as namespace_file: + with open( + "/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r", encoding="utf-8" + ) as namespace_file: namespace = namespace_file.read().strip() s3_bucket = f"{namespace}-data" @@ -30,3 +32,4 @@ experiment_name="mobile-price-classification-containerized", run_name=f"Containerized pipeline {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", ) + print(f"KFP_RUN_ID={run.run_id}") diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py new file mode 100644 index 0000000..534aa12 --- /dev/null +++ b/scripts/get_or_create_api_key.py @@ -0,0 +1,116 @@ +""" +Utility to create or retrieve a prokube AIGatewayKey for use in examples. + +Usage from a notebook +--------------------- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from get_or_create_api_key import get_or_create_api_key + + API_KEY = get_or_create_api_key() + +CLI usage +--------- + python get_or_create_api_key.py + # prints the key to stdout + +The key is stored as an AIGatewayKey CR named ``examples-key`` backed by a +Secret named ``examples-key-secret`` in the notebook's namespace. Both +resources are created if they do not already exist. Re-running is safe. +""" + +from __future__ import annotations + +import base64 +import json +import secrets +import subprocess +import sys + +_KEY_NAME = "examples-key" +_SECRET_NAME = "examples-key-secret" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["kubectl", *args], + input=input, + capture_output=True, + text=True, + ) + + +def _key_exists(namespace: str) -> bool: + result = _kubectl( + "get", "aigatewaykey", _KEY_NAME, "-n", namespace, "--ignore-not-found" + ) + return bool(result.stdout.strip()) + + +def _read_key_from_secret(namespace: str) -> str: + result = _kubectl("get", "secret", _SECRET_NAME, "-n", namespace, "-o", "json") + if result.returncode != 0: + raise RuntimeError(f"Could not read secret {_SECRET_NAME}:\n{result.stderr}") + data = json.loads(result.stdout)["data"] + return base64.b64decode(data["token"]).decode() + + +def _apply(manifest: str, namespace: str) -> None: + result = _kubectl("apply", "-n", namespace, "-f", "-", input=manifest) + if result.returncode != 0: + raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") + + +def _create_key(namespace: str) -> str: + key_value = f"pk_live_{secrets.token_hex(20)}" + + _apply( + f"apiVersion: v1\n" + f"kind: Secret\n" + f"metadata:\n" + f" name: {_SECRET_NAME}\n" + f" namespace: {namespace}\n" + f"stringData:\n" + f" token: {key_value}\n", + namespace, + ) + + _apply( + f"apiVersion: prokube.ai/v1alpha1\n" + f"kind: AIGatewayKey\n" + f"metadata:\n" + f" name: {_KEY_NAME}\n" + f" namespace: {namespace}\n" + f"spec:\n" + f" displayName: Examples API key\n" + f" secretRef:\n" + f" name: {_SECRET_NAME}\n" + f" key: token\n" + f" scopes:\n" + f" - type: workspace\n", + namespace, + ) + + return key_value + + +def get_or_create_api_key() -> str: + """Return the API key for the current namespace, creating it if needed.""" + ns = _namespace() + if _key_exists(ns): + return _read_key_from_secret(ns) + return _create_key(ns) + + +if __name__ == "__main__": + try: + print(get_or_create_api_key()) + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/setup_mlflow_credentials.py b/scripts/setup_mlflow_credentials.py new file mode 100644 index 0000000..b1b3011 --- /dev/null +++ b/scripts/setup_mlflow_credentials.py @@ -0,0 +1,125 @@ +""" +One-time setup: store MLflow credentials in a Kubernetes secret so that +example notebooks can read them without hardcoded values. + +This is the only step in the automation plan that requires human input — +the MLflow Personal Access Token (PAT) must be created via the MLflow UI +(Permissions → Create access key) and cannot be obtained programmatically. + +Usage from a notebook +--------------------- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from setup_mlflow_credentials import setup_mlflow_credentials + + setup_mlflow_credentials( + uri="https://my-cluster.example.com/mlflow/", + username="alice@example.com", + password="", + ) + +CLI usage (any argument can be omitted; you will be prompted for it) +--------------------------------------------------------------------- + python setup_mlflow_credentials.py \\ + --uri https://my-cluster.example.com/mlflow/ \\ + --username alice@example.com \\ + --password + +The credentials are stored in a Secret named ``mlflow-credentials`` in the +notebook's namespace. Re-running updates the secret in place. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_SECRET_NAME = "mlflow-credentials" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _prompt(label: str, secret: bool = False) -> str: + if secret: + import getpass + + return getpass.getpass(f"{label}: ") + return input(f"{label}: ").strip() + + +def setup_mlflow_credentials( + uri: str | None = None, + username: str | None = None, + password: str | None = None, +) -> None: + """Create or update the ``mlflow-credentials`` secret. + + Any parameter left as ``None`` will be requested interactively. + """ + if uri is None: + print("MLflow tracking URI — typically https:///mlflow/") + uri = _prompt("MLFLOW_TRACKING_URI") + if username is None: + username = _prompt("MLFLOW_TRACKING_USERNAME (your login e-mail)") + if password is None: + password = _prompt( + "MLFLOW_TRACKING_PASSWORD (Personal Access Token)", secret=True + ) + + ns = _namespace() + + result = subprocess.run( + [ + "kubectl", + "create", + "secret", + "generic", + _SECRET_NAME, + "-n", + ns, + f"--from-literal=MLFLOW_TRACKING_URI={uri}", + f"--from-literal=MLFLOW_TRACKING_USERNAME={username}", + f"--from-literal=MLFLOW_TRACKING_PASSWORD={password}", + "--save-config", + "--dry-run=client", + "-o", + "yaml", + ], + capture_output=True, + text=True, + check=True, + ) + + apply = subprocess.run( + ["kubectl", "apply", "-n", ns, "-f", "-"], + input=result.stdout, + capture_output=True, + text=True, + ) + if apply.returncode != 0: + raise RuntimeError(f"kubectl apply failed:\n{apply.stderr}") + + print(f"Secret '{_SECRET_NAME}' created/updated in namespace '{ns}'.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--uri", default=None, help="MLFLOW_TRACKING_URI") + parser.add_argument("--username", default=None, help="MLFLOW_TRACKING_USERNAME") + parser.add_argument( + "--password", default=None, help="MLFLOW_TRACKING_PASSWORD (PAT)" + ) + args = parser.parse_args() + + try: + setup_mlflow_credentials(args.uri, args.username, args.password) + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/serving/minimal-s3-model/cleanup.py b/serving/minimal-s3-model/cleanup.py new file mode 100644 index 0000000..9059dd3 --- /dev/null +++ b/serving/minimal-s3-model/cleanup.py @@ -0,0 +1,63 @@ +""" +Cleanup script for the minimal-s3-model example. + +Deletes (idempotently): + - InferenceService ``kserve-object-storage-test`` + - AIGatewayKey ``examples-key`` and its backing Secret ``examples-key-secret`` + (only when --include-api-key is passed, since the key is shared across examples) + +Usage +----- + python cleanup.py [--include-api-key] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(include_api_key: bool = False, dry_run: bool = False) -> None: + ns = _namespace() + + _kubectl_delete( + "inferenceservice", "kserve-object-storage-test", "-n", ns, dry_run=dry_run + ) + + if include_api_key: + _kubectl_delete("aigatewaykey", "examples-key", "-n", ns, dry_run=dry_run) + _kubectl_delete("secret", "examples-key-secret", "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--include-api-key", + action="store_true", + help="Also delete the shared AIGatewayKey", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(include_api_key=args.include_api_key, dry_run=args.dry_run) diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index 6d1b063..f1b5e91 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -307,9 +307,12 @@ "metadata": {}, "outputs": [], "source": [ - "INFERENCE_SERVICE_API_KEY = \"\"\n", - "if not INFERENCE_SERVICE_API_KEY:\n", - " raise RuntimeError(\"Please provide the API Key that will be used to test the deployed InferenceService\")" + "import sys, subprocess, os\n", + "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", + "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", + "from get_or_create_api_key import get_or_create_api_key\n", + "\n", + "INFERENCE_SERVICE_API_KEY = get_or_create_api_key()\n" ] }, { diff --git a/serving/mlflow-kserve-inference-protocols/cleanup.py b/serving/mlflow-kserve-inference-protocols/cleanup.py new file mode 100644 index 0000000..f4fb07d --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/cleanup.py @@ -0,0 +1,56 @@ +""" +Cleanup script for the mlflow-kserve-inference-protocols example. + +Deletes (idempotently): + - InferenceService ``v1-mobile-price-classification-inference`` + - InferenceService ``v2-mobile-price-classification-inference`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_ISVC_NAMES = [ + "v1-mobile-price-classification-inference", + "v2-mobile-price-classification-inference", +] + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + for name in _ISVC_NAMES: + _kubectl_delete("inferenceservice", name, "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index d6971c5..46e1d2c 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -340,10 +340,88 @@ "source": [ "---\n", "\n", - "## 6. Live Testing (Optional)\n", + "## 6. Live Testing\n", "\n", - "If you have deployed both InferenceServices, you can test them here. \n", - "Fill in your connection details and run the cells below." + "The cells below deploy both InferenceServices, wait for them to become ready,\n", + "and run inference requests automatically.\n", + "\n", + "**Prerequisite:** the `mlflow-credentials` secret must exist in your namespace\n", + "(run `scripts/setup_mlflow_credentials.py` once if you have not done so).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "auto-setup-isvc", + "metadata": {}, + "outputs": [], + "source": [ + "import base64, json, os, subprocess, sys\n", + "\n", + "# ── 1. Namespace & MLflow username ───────────────────────────────────────────\n", + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " _ns = _f.read().strip()\n", + "\n", + "_secret = subprocess.run(\n", + " [\"kubectl\", \"get\", \"secret\", \"mlflow-credentials\", \"-n\", _ns, \"-o\", \"json\"],\n", + " capture_output=True, text=True,\n", + ")\n", + "if _secret.returncode != 0:\n", + " raise RuntimeError(\n", + " \"mlflow-credentials secret not found.\\n\"\n", + " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", + " )\n", + "_creds = json.loads(_secret.stdout)[\"data\"]\n", + "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode()\n", + "\n", + "# ── 2. Apply InferenceService YAMLs with substituted namespace & username ────\n", + "_isvc_names = [\n", + " (\"v1-InferenceService.yaml\", \"v1-mobile-price-classification-inference\"),\n", + " (\"v2-InferenceService.yaml\", \"v2-mobile-price-classification-inference\"),\n", + "]\n", + "for _yaml_file, _ in _isvc_names:\n", + " _manifest = open(_yaml_file).read()\n", + " _manifest = _manifest.replace(\"\", _ns).replace(\"\", _username)\n", + " _r = subprocess.run(\n", + " [\"kubectl\", \"apply\", \"-n\", _ns, \"-f\", \"-\"],\n", + " input=_manifest, capture_output=True, text=True,\n", + " )\n", + " if _r.returncode != 0:\n", + " raise RuntimeError(f\"Failed to apply {_yaml_file}:\\n{_r.stderr}\")\n", + " print(f\"Applied {_yaml_file}\")\n", + "\n", + "# ── 3. Wait for both ISVCs to be ready (up to 10 min each) ───────────────────\n", + "print(\"Waiting for InferenceServices to become ready...\")\n", + "for _, _name in _isvc_names:\n", + " _r = subprocess.run(\n", + " [\"kubectl\", \"wait\", \"inferenceservice\", _name, \"-n\", _ns,\n", + " \"--for=condition=Ready\", \"--timeout=600s\"],\n", + " capture_output=True, text=True,\n", + " )\n", + " if _r.returncode != 0:\n", + " raise RuntimeError(f\"InferenceService {_name} did not become ready:\\n{_r.stderr}\")\n", + " print(f\" {_name}: Ready\")\n", + "\n", + "# ── 4. Fetch external URLs from ISVC status ───────────────────────────────────\n", + "def _isvc_url(name):\n", + " return subprocess.check_output(\n", + " [\"kubectl\", \"get\", \"inferenceservice\", name, \"-n\", _ns,\n", + " \"-o\", \"jsonpath={.status.url}\"],\n", + " text=True,\n", + " ).strip()\n", + "\n", + "_uri_v1 = _isvc_url(\"v1-mobile-price-classification-inference\")\n", + "_uri_v2 = _isvc_url(\"v2-mobile-price-classification-inference\")\n", + "\n", + "# ── 5. Get / create inference API key ────────────────────────────────────────\n", + "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", + "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", + "from get_or_create_api_key import get_or_create_api_key\n", + "_api_key = get_or_create_api_key()\n", + "\n", + "print(f\"\\nv1 URI : {_uri_v1}\")\n", + "print(f\"v2 URI : {_uri_v2}\")\n", + "print(\"API key: [set]\")\n" ] }, { @@ -356,27 +434,24 @@ "import requests\n", "\n", "# ---------------------------------------------------------------------------\n", - "# Connection settings (fill these in)\n", + "# Connection settings (auto-populated by the setup cell above)\n", "# ---------------------------------------------------------------------------\n", - "INFERENCE_SERVICE_URI_V1 = \"\" # external URL, no trailing slash (copied from KServe InferenceService status.url)\n", - "INFERENCE_SERVICE_URI_V2 = \"\" # external URL, no trailing slash (copied from KServe InferenceService status.url)\n", - "API_KEY = \"\" # API key, provided by your administrator \n", + "INFERENCE_SERVICE_URI_V1 = _uri_v1\n", + "INFERENCE_SERVICE_URI_V2 = _uri_v2\n", + "API_KEY = _api_key\n", "\n", "# ---------------------------------------------------------------------------\n", "# Model names (must match metadata.name in the InferenceService YAMLs)\n", "# ---------------------------------------------------------------------------\n", - "MODEL_NAME_V1 = \"v1-mobile-price-classification-inference\" # from v1_InferenceService.yaml\n", - "MODEL_NAME_V2 = \"v2-mobile-price-classification-inference\" # from v2_InferenceService.yaml\n", + "MODEL_NAME_V1 = \"v1-mobile-price-classification-inference\"\n", + "MODEL_NAME_V2 = \"v2-mobile-price-classification-inference\"\n", "\n", "HEADERS = {\"X-Api-Key\": API_KEY, \"Content-Type\": \"application/json\"}\n", "\n", - "if not INFERENCE_SERVICE_URI_V1 or not INFERENCE_SERVICE_URI_V2 or not API_KEY:\n", - " raise ValueError(\"Please fill in the connection settings (URIs and API key) before running the tests.\")\n", - "\n", - "print(f\"v1 URI : {INFERENCE_SERVICE_URI_V1}\")\n", - "print(f\"v2 URI : {INFERENCE_SERVICE_URI_V2}\")\n", - "print(f\"v1 model : {MODEL_NAME_V1}\")\n", - "print(f\"v2 model : {MODEL_NAME_V2}\")" + "print(f\"v1 URI : {INFERENCE_SERVICE_URI_V1}\")\n", + "print(f\"v2 URI : {INFERENCE_SERVICE_URI_V2}\")\n", + "print(f\"v1 model: {MODEL_NAME_V1}\")\n", + "print(f\"v2 model: {MODEL_NAME_V2}\")\n" ] }, { diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py new file mode 100644 index 0000000..b3896bd --- /dev/null +++ b/serving/mlflow-kserve-minimal/apply.py @@ -0,0 +1,150 @@ +""" +Deploy the mlflow-kserve-minimal InferenceService. + +Substitutes ```` and ```` in InferenceService.yaml +from the cluster environment, applies it, waits for readiness, and prints the +external URL so it can be used by ``test_inference_service.py``. + +Usage from a notebook +--------------------- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "serving", "mlflow-kserve-minimal")) + from apply import deploy + + isvc_uri = deploy() + +CLI usage +--------- + python apply.py + # prints ISVC_URI= to stdout + +Prerequisites +------------- +- ``mlflow-credentials`` secret must exist (run scripts/setup_mlflow_credentials.py). +- The MLflow model ``mobile-price-svm-`` version 1 must be registered + (run mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb first). +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys + +_ISVC_NAME = "mobile-price-svm" +_YAML_TEMPLATE = os.path.join(os.path.dirname(__file__), "InferenceService.yaml") + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _mlflow_username(namespace: str) -> str: + result = subprocess.run( + [ + "kubectl", + "get", + "secret", + "mlflow-credentials", + "-n", + namespace, + "-o", + "json", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + "mlflow-credentials secret not found.\n" + "Run scripts/setup_mlflow_credentials.py to create it." + ) + data = json.loads(result.stdout)["data"] + return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-n", namespace, "-f", "-"], + input=manifest, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") + + +def _wait_ready(name: str, namespace: str, timeout: int = 600) -> None: + print( + f"Waiting for InferenceService '{name}' to become ready (timeout {timeout}s)..." + ) + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "-n", + namespace, + "--for=condition=Ready", + f"--timeout={timeout}s", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"InferenceService '{name}' did not become ready:\n{result.stderr}" + ) + print(f" '{name}' is Ready.") + + +def _isvc_url(name: str, namespace: str) -> str: + return subprocess.check_output( + [ + "kubectl", + "get", + "inferenceservice", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.url}", + ], + text=True, + ).strip() + + +def deploy(timeout: int = 600) -> str: + """Apply InferenceService.yaml, wait for readiness, return the external URL.""" + ns = _namespace() + username = _mlflow_username(ns) + + with open(_YAML_TEMPLATE) as fh: + manifest = fh.read() + + manifest = ( + manifest.replace("", _ISVC_NAME) + .replace("", ns) + .replace("", username) + ) + + _kubectl_apply(manifest, ns) + print(f"Applied InferenceService '{_ISVC_NAME}' in namespace '{ns}'.") + + _wait_ready(_ISVC_NAME, ns, timeout=timeout) + url = _isvc_url(_ISVC_NAME, ns) + print(f"ISVC_URI={url}") + return url + + +if __name__ == "__main__": + try: + deploy() + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/serving/mlflow-kserve-minimal/cleanup.py b/serving/mlflow-kserve-minimal/cleanup.py new file mode 100644 index 0000000..96c341c --- /dev/null +++ b/serving/mlflow-kserve-minimal/cleanup.py @@ -0,0 +1,49 @@ +""" +Cleanup script for the mlflow-kserve-minimal example. + +Deletes (idempotently): + - InferenceService ``mobile-price-svm`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete("inferenceservice", "mobile-price-svm", "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) From ca88b4e828892eaddd5ee5de4dc4b3a194fb4ec4 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 14:26:08 +0200 Subject: [PATCH 02/48] Better error handling --- ci/run_all.py | 108 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 17d61ea..1cf3229 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -45,9 +45,7 @@ from __future__ import annotations import argparse -import importlib.util import json -import os import re import subprocess import sys @@ -55,13 +53,30 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path -from typing import Optional + # ── Repo root ───────────────────────────────────────────────────────────────── _REPO_ROOT = Path( subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() ) + +# ── Prerequisite: papermill ─────────────────────────────────────────────────── + + +def _ensure_papermill() -> None: + """Install papermill if it is not already available.""" + try: + import papermill # noqa: F401 + except ImportError: + print("papermill not found — installing...") + subprocess.run( + [sys.executable, "-m", "pip", "install", "papermill"], + check=True, + ) + print("papermill installed.") + + # ── KFP run ID pattern (UUID v4) ───────────────────────────────────────────── _RUN_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") @@ -86,23 +101,53 @@ def _namespace() -> str: return fh.read().strip() -def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> tuple[Path, str]: - """Execute a notebook with papermill. Returns (output_path, stderr).""" +def _format_papermill_error(exc: Exception) -> str: + """Extract a human-readable summary from a PapermillExecutionError.""" try: - import papermill as pm + from papermill.exceptions import PapermillExecutionError + + if not isinstance(exc, PapermillExecutionError): + return str(exc) except ImportError: - raise RuntimeError("papermill is not installed. Run: pip install papermill") + return str(exc) + + lines = [ + f"Cell {exc.exec_count} raised {exc.ename}: {exc.evalue}", + ] + # Show the failing cell source (first 5 lines) + if exc.source: + src_lines = exc.source.strip().splitlines()[:5] + lines.append(" Cell source:") + for src_line in src_lines: + lines.append(f" {src_line}") + if len(exc.source.strip().splitlines()) > 5: + lines.append(" ...") + # Show the innermost traceback frame (last non-empty line) + if exc.traceback: + tb_lines = [l for l in exc.traceback if l.strip()] + if tb_lines: + lines.append(f" Traceback (last): {tb_lines[-1].strip()}") + return "\n".join(lines) + + +def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: + """Execute a notebook with papermill. Returns the output notebook path.""" + import papermill as pm + from papermill.exceptions import PapermillExecutionError output_path = output_dir / nb_path.name output_dir.mkdir(parents=True, exist_ok=True) - pm.execute_notebook( - str(nb_path), - str(output_path), - kernel_name="python3", - execution_timeout=timeout, - cwd=str(nb_path.parent), - ) - return output_path, "" + try: + pm.execute_notebook( + str(nb_path), + str(output_path), + kernel_name="python3", + execution_timeout=timeout, + cwd=str(nb_path.parent), + ) + except PapermillExecutionError as exc: + raise RuntimeError(_format_papermill_error(exc)) from exc + return output_path def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: @@ -115,7 +160,9 @@ def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: cwd=str(script_path.parent), ) if result.returncode != 0: - raise RuntimeError(result.stderr) + # Include both stderr and stdout for full context + detail = (result.stderr or result.stdout or "(no output)").strip() + raise RuntimeError(detail) return result.stdout, result.stderr @@ -190,12 +237,12 @@ def _submit_notebook( def _run(): t0 = time.time() try: - out, _ = _run_notebook(nb_path, output_dir, timeout) + out = _run_notebook(nb_path, output_dir, timeout) result.status = "PASS" result.kfp_run_ids = _extract_run_ids_from_notebook(out) except Exception as exc: # noqa: BLE001 result.status = "FAIL" - result.error = str(exc)[:300] + result.error = str(exc) finally: result.duration = time.time() - t0 @@ -220,7 +267,7 @@ def _run(): result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) except Exception as exc: # noqa: BLE001 result.status = "FAIL" - result.error = str(exc)[:300] + result.error = str(exc) finally: result.duration = time.time() - t0 @@ -230,6 +277,15 @@ def _run(): # ── Report ──────────────────────────────────────────────────────────────────── +def _print_result(r: Result) -> None: + """Print a single result line, with error detail if failed.""" + dur = f"{r.duration:.0f}s" if r.duration else "-" + print(f" [{r.status}] {r.name} ({dur})") + if r.status == "FAIL" and r.error: + for line in r.error.splitlines(): + print(f" {line}") + + def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: col = 50 print("\n" + "=" * 70) @@ -240,7 +296,6 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N dur = f"{r.duration:.0f}s" if r.duration else "-" print(f"{r.name:<{col}} {r.status:<10} {dur}") if r.status == "FAIL": - print(f" {'':>{col}} {r.error}") failed += 1 else: passed += 1 @@ -250,14 +305,21 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print("-" * 70) for run_id, status in poll_results.items(): short = run_id[:8] + "..." - ok = status == "Succeeded" - if not ok: + if status != "Succeeded": failed += 1 else: passed += 1 print(f"{short:<{col}} {status:<10}") print("=" * 70) print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") + if failed: + print("\nFailed details:") + print("-" * 70) + for r in results.values(): + if r.status == "FAIL" and r.error: + print(f"\n{r.name}:") + for line in r.error.splitlines(): + print(f" {line}") print() @@ -275,6 +337,8 @@ def run_all( results: dict[str, Result] = {} poll_results: dict[str, str] = {} + _ensure_papermill() + if dry_run: print("[dry-run] Would execute the following phases:") for label in [ From ca176040ab1308179e5965d507be2533e270e4be Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 14:28:45 +0200 Subject: [PATCH 03/48] ci: fix output readability and error reporting - Auto-install papermill if missing (pip install at startup) - Disable papermill progress bars (progress_bar=False) to prevent interleaved tqdm output from parallel notebook executions - Print [START] when each notebook/script is submitted so names are visible immediately, not just after completion - Switch from sequential f.result() to as_completed() so results print as each finishes rather than in submission order - Extract structured PapermillExecutionError details: cell number, cell source (first 5 lines), innermost traceback frame - Include full stderr/stdout in script failure messages (was truncated) - Print error details inline after each [FAIL] line during execution - Print full 'Failed details' section at the bottom of the report - Fix phase 2 remaining results printing under Phase 3 header --- ci/run_all.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 1cf3229..4c38585 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -144,6 +144,7 @@ def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: kernel_name="python3", execution_timeout=timeout, cwd=str(nb_path.parent), + progress_bar=False, # avoid interleaved tqdm bars from parallel threads ) except PapermillExecutionError as exc: raise RuntimeError(_format_papermill_error(exc)) from exc @@ -233,6 +234,7 @@ def _submit_notebook( ) -> Future: result = Result(name=name) results[name] = result + print(f" [START ] {name}") def _run(): t0 = time.time() @@ -258,6 +260,7 @@ def _submit_script( ) -> Future: result = Result(name=name) results[name] = result + print(f" [START ] {name}") def _run(): t0 = time.time() @@ -379,10 +382,10 @@ def run_all( ) phase1_futures[name] = f - # wait for all phase 1 - for name, f in phase1_futures.items(): - f.result() - print(f" [{results[name].status}] {name}") + # wait for all phase 1, printing results as each finishes + _future_to_name = {v: k for k, v in phase1_futures.items()} + for f in as_completed(phase1_futures.values()): + _print_result(results[_future_to_name[f]]) # ── Phase 2: pipeline submissions (return fast) ─────────────────── print("\nPhase 2: submitting pipelines...") @@ -412,11 +415,8 @@ def run_all( phase2_futures["pipelines/lightweight-python-package"] = lpp_future # Wait only for mlflow-mobile-price before launching phase 3 - mlflow_mobile_future = phase2_futures["mlflow/mobile-price-classification"] - mlflow_mobile_future.result() - print( - f" [{results['mlflow/mobile-price-classification'].status}] mlflow/mobile-price-classification" - ) + phase2_futures["mlflow/mobile-price-classification"].result() + _print_result(results["mlflow/mobile-price-classification"]) # ── Phase 3: MLflow KServe examples (need registered model) ─────── print("\nPhase 3: deploying MLflow KServe InferenceServices...") @@ -440,16 +440,19 @@ def run_all( f = _submit_script(executor, results, name, path, timeout_notebook) phase3_futures[name] = f - # Wait for remaining phase 2 futures while phase 3 runs - for name, f in phase2_futures.items(): - if name != "mlflow/mobile-price-classification": - f.result() - print(f" [{results[name].status}] {name}") - - # Wait for phase 3 - for name, f in phase3_futures.items(): - f.result() - print(f" [{results[name].status}] {name}") + # Phase 2 remaining + phase 3 run concurrently; print as each finishes + print("\nPhase 2 (remaining) + Phase 3 running concurrently...") + remaining = { + **{ + k: v + for k, v in phase2_futures.items() + if k != "mlflow/mobile-price-classification" + }, + **phase3_futures, + } + _future_to_name2 = {v: k for k, v in remaining.items()} + for f in as_completed(remaining.values()): + _print_result(results[_future_to_name2[f]]) # ── Phase 4: KFP run polling ────────────────────────────────────── all_run_ids: list[str] = [] From caf78e114c16a6f41f099baf0665999f5d4ddf08 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 16:05:22 +0200 Subject: [PATCH 04/48] ci: fix KFP v2 polling and prevent Phase 4 crash from aborting run - _poll_kfp_run: use run.state (KFP v2 V2beta1Run) with fallback to run.run.status (KFP v1) for compatibility; normalise to uppercase - Terminal states updated to KFP v2 set: SUCCEEDED/FAILED/ERROR/CANCELED/SKIPPED - Phase 4: wrap f.result() in try/except so a polling error records POLL_ERROR in the report instead of crashing the whole CI run - _print_report: compare status.upper() == 'SUCCEEDED' (case-insensitive) --- ci/run_all.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 4c38585..932ffaa 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -190,6 +190,9 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: return ids +_KFP_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "ERROR", "CANCELED", "SKIPPED"} + + def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: """Poll a KFP run until terminal state. Returns final status string.""" try: @@ -200,8 +203,14 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: deadline = time.time() + timeout while time.time() < deadline: run = client.get_run(run_id) - state = run.run.status - if state in ("Succeeded", "Failed", "Error", "Skipped"): + # KFP v2 SDK: state is on the run object directly as a string + # KFP v1 SDK: state is on run.run.status + state = str( + getattr(run, "state", None) + or getattr(getattr(run, "run", None), "status", None) + or "UNKNOWN" + ).upper() + if state in _KFP_TERMINAL_STATES: return state time.sleep(interval) return f"TIMEOUT after {timeout}s" @@ -308,10 +317,10 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print("-" * 70) for run_id, status in poll_results.items(): short = run_id[:8] + "..." - if status != "Succeeded": - failed += 1 - else: + if status.upper() == "SUCCEEDED": passed += 1 + else: + failed += 1 print(f"{short:<{col}} {status:<10}") print("=" * 70) print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") @@ -466,8 +475,12 @@ def run_all( for run_id in all_run_ids } for run_id, f in poll_futures.items(): - poll_results[run_id] = f.result() - print(f" [{poll_results[run_id]}] {run_id[:8]}...") + try: + poll_results[run_id] = f.result() + except Exception as exc: # noqa: BLE001 + poll_results[run_id] = f"POLL_ERROR: {exc}" + state = poll_results[run_id] + print(f" [{state}] {run_id[:8]}...") else: print("\nPhase 4: no KFP run IDs found, skipping poll.") From d629a2bc6acd2b4fb2f26600ca8895d71b268535 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 16:41:18 +0200 Subject: [PATCH 05/48] fix: YAML filename typo and CI report/skip improvements - inference_protocol_version_example.ipynb: fix pre-existing bug where cell read v1_InferenceService.yaml (underscore) but file is v1-InferenceService.yaml (dash); same for v2 - ci/run_all.py: move summary table to the very end (after Failed details) so final verdict is the last thing visible in the terminal - ci/run_all.py: skip Phase 3 ISVC tests when mlflow/mobile-price-classification fails, avoiding a 10-minute ISVC timeout on a model that was never registered --- ci/run_all.py | 94 ++++++++++++------- .../inference_protocol_version_example.ipynb | 4 +- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 932ffaa..afb0895 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -300,30 +300,19 @@ def _print_result(r: Result) -> None: def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: col = 50 - print("\n" + "=" * 70) - print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") - print("-" * 70) passed = failed = 0 for r in results.values(): - dur = f"{r.duration:.0f}s" if r.duration else "-" - print(f"{r.name:<{col}} {r.status:<10} {dur}") if r.status == "FAIL": failed += 1 else: passed += 1 - if poll_results: - print() - print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") - print("-" * 70) - for run_id, status in poll_results.items(): - short = run_id[:8] + "..." - if status.upper() == "SUCCEEDED": - passed += 1 - else: - failed += 1 - print(f"{short:<{col}} {status:<10}") - print("=" * 70) - print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") + for run_id, status in poll_results.items(): + if status.upper() == "SUCCEEDED": + passed += 1 + else: + failed += 1 + + # Failed details first so they're easy to scroll back to if failed: print("\nFailed details:") print("-" * 70) @@ -332,6 +321,25 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print(f"\n{r.name}:") for line in r.error.splitlines(): print(f" {line}") + for run_id, status in poll_results.items(): + if status.upper() != "SUCCEEDED": + print(f"\nKFP run {run_id[:8]}...: {status}") + + # Summary table last — easy to see final verdict at a glance + print("\n" + "=" * 70) + print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") + print("-" * 70) + for r in results.values(): + dur = f"{r.duration:.0f}s" if r.duration else "-" + print(f"{r.name:<{col}} {r.status:<10} {dur}") + if poll_results: + print() + print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") + print("-" * 70) + for run_id, status in poll_results.items(): + print(f"{run_id[:8] + '...':<{col}} {status:<10}") + print("=" * 70) + print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") print() @@ -429,25 +437,43 @@ def run_all( # ── Phase 3: MLflow KServe examples (need registered model) ─────── print("\nPhase 3: deploying MLflow KServe InferenceServices...") - phase3_futures = {} - for name, rel in [ - ( + if results["mlflow/mobile-price-classification"].status == "FAIL": + print( + " [SKIP] mlflow-mobile-price-classification failed — " + "skipping ISVC tests that depend on the registered model" + ) + for name in ( "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-minimal/apply.py", - ), - ( "serving/mlflow-kserve-inference-protocols", - "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - ), - ]: - path = root / rel - if path.suffix == ".ipynb": - f = _submit_notebook( - executor, results, name, path, output_dir, timeout_notebook + ): + results[name] = Result( + name=name, + status="SKIP", + error="prerequisite mlflow/mobile-price-classification failed", ) - else: - f = _submit_script(executor, results, name, path, timeout_notebook) - phase3_futures[name] = f + phase3_futures = {} + else: + phase3_futures = {} + for name, rel in [ + ( + "serving/mlflow-kserve-minimal", + "serving/mlflow-kserve-minimal/apply.py", + ), + ( + "serving/mlflow-kserve-inference-protocols", + "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + ), + ]: + path = root / rel + if path.suffix == ".ipynb": + f = _submit_notebook( + executor, results, name, path, output_dir, timeout_notebook + ) + else: + f = _submit_script( + executor, results, name, path, timeout_notebook + ) + phase3_futures[name] = f # Phase 2 remaining + phase 3 run concurrently; print as each finishes print("\nPhase 2 (remaining) + Phase 3 running concurrently...") diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 46e1d2c..83f7898 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -53,8 +53,8 @@ "metadata": {}, "outputs": [], "source": [ - "v1_yaml = Path(\"v1_InferenceService.yaml\").read_text()\n", - "v2_yaml = Path(\"v2_InferenceService.yaml\").read_text()\n", + "v1_yaml = Path(\"v1-InferenceService.yaml\").read_text()\n", + "v2_yaml = Path(\"v2-InferenceService.yaml\").read_text()\n", "\n", "display(Markdown(\n", " \"| v1 InferenceService | v2 InferenceService |\\n\"\n", From d2bee85416d2cc4601fc80fa1e9489d17cc0986e Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 17:40:39 +0200 Subject: [PATCH 06/48] ci: label KFP poll rows with source notebook name --- ci/run_all.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index afb0895..d2b477d 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -298,7 +298,11 @@ def _print_result(r: Result) -> None: print(f" {line}") -def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: +def _print_report( + results: dict[str, Result], + poll_results: dict[str, str], + run_id_to_name: dict[str, str], +) -> None: col = 50 passed = failed = 0 for r in results.values(): @@ -323,7 +327,8 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print(f" {line}") for run_id, status in poll_results.items(): if status.upper() != "SUCCEEDED": - print(f"\nKFP run {run_id[:8]}...: {status}") + source = run_id_to_name.get(run_id, "unknown") + print(f"\nKFP run {run_id[:8]}... (from {source}): {status}") # Summary table last — easy to see final verdict at a glance print("\n" + "=" * 70) @@ -334,10 +339,12 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print(f"{r.name:<{col}} {r.status:<10} {dur}") if poll_results: print() - print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") + print(f"{'KFP RUN (source notebook)':<{col}} {'STATUS':<10}") print("-" * 70) for run_id, status in poll_results.items(): - print(f"{run_id[:8] + '...':<{col}} {status:<10}") + source = run_id_to_name.get(run_id, "unknown") + label = f"{run_id[:8]}... ({source})" + print(f"{label:<{col}} {status:<10}") print("=" * 70) print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") print() @@ -518,7 +525,13 @@ def run_all( for f in as_completed(futs): f.result() # already swallows exceptions inside _run_cleanup - _print_report(results, poll_results) + # Build run_id → source notebook name map from all results + run_id_to_name: dict[str, str] = {} + for r in results.values(): + for run_id in r.kfp_run_ids: + run_id_to_name[run_id] = r.name + + _print_report(results, poll_results, run_id_to_name) return results From 58f575b7e5cf51c34df51d85bfd4f8e70cf9a4a2 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 15:43:34 +0200 Subject: [PATCH 07/48] fix: AIGatewayKey owner field, MLflow pre-flight check, ISVC timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/get_or_create_api_key.py: - Add required 'owner' field to AIGatewayKey manifest (CRD validation was rejecting the resource without it); reads MLFLOW_TRACKING_USERNAME from the mlflow-credentials secret when available, falls back to notebook-examples@ ci/run_all.py: - Add _check_mlflow_credentials() pre-flight check: validates secret exists AND credentials work (quick MLflow REST API call); skips all MLflow-dependent tests with a clear message if either check fails - Phase 1 and Phase 2 skip notebooks already resolved by pre-flight - Fix Phase 3 timing: after mlflow-mobile-price notebook finishes, poll its KFP run inline (blocking) before deploying ISVCs — previously Phase 3 deployed immediately while the pipeline (and model registration) was still running, causing ISVC timeouts - Phase 4 skips run IDs already polled inline before Phase 3 --- ci/run_all.py | 141 +++++++++++++++++++++++++++---- scripts/get_or_create_api_key.py | 23 +++++ 2 files changed, 149 insertions(+), 15 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index d2b477d..265508a 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -350,6 +350,72 @@ def _print_report( print() +# ── MLflow pre-flight check ─────────────────────────────────────────────────── + +# Notebooks that require working MLflow credentials +_MLFLOW_DEPENDENT = frozenset( + { + "mlflow/mlflow-quickstart", + "mlflow/mlflow-image-example", + "mlflow/mlflow-kfp-example", + "mlflow/mobile-price-classification", + } +) + + +def _check_mlflow_credentials() -> tuple[bool, str]: + """Return (ok, reason). Checks secret existence then validates credentials + with a quick call to the MLflow REST API.""" + import base64 as _b64 + import json as _json + import urllib.error + import urllib.request + + ns = _namespace() + r = subprocess.run( + ["kubectl", "get", "secret", "mlflow-credentials", "-n", ns, "-o", "json"], + capture_output=True, + text=True, + ) + if r.returncode != 0: + return False, ( + "mlflow-credentials secret not found — " + "run scripts/setup_mlflow_credentials.py first" + ) + + try: + data = _json.loads(r.stdout)["data"] + uri = _b64.b64decode(data["MLFLOW_TRACKING_URI"]).decode().rstrip("/") + username = _b64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + password = _b64.b64decode(data["MLFLOW_TRACKING_PASSWORD"]).decode() + except KeyError as exc: + return ( + False, + f"mlflow-credentials secret is missing key {exc} — re-run setup_mlflow_credentials.py", + ) + + creds = _b64.b64encode(f"{username}:{password}".encode()).decode() + req = urllib.request.Request( + f"{uri}/api/2.0/mlflow/experiments/search?max_results=1", + headers={"Authorization": f"Basic {creds}"}, + ) + try: + urllib.request.urlopen(req, timeout=8) + return True, "OK" + except urllib.error.HTTPError as exc: + if exc.code in (401, 403): + return False, ( + "MLflow credentials are invalid or the PAT has expired — " + "re-run scripts/setup_mlflow_credentials.py" + ) + return ( + False, + f"MLflow API returned HTTP {exc.code} — check MLFLOW_TRACKING_URI in the secret", + ) + except Exception as exc: # noqa: BLE001 + return False, f"Could not reach MLflow at {uri}: {exc}" + + # ── Main ────────────────────────────────────────────────────────────────────── @@ -366,6 +432,16 @@ def run_all( _ensure_papermill() + # Pre-flight: validate MLflow credentials; skip dependent tests if unavailable + print("Pre-flight: checking MLflow credentials...") + mlflow_ok, mlflow_reason = _check_mlflow_credentials() + if mlflow_ok: + print(" [OK] MLflow credentials valid") + else: + print(f" [SKIP] {mlflow_reason}") + for name in _MLFLOW_DEPENDENT: + results[name] = Result(name=name, status="SKIP", error=mlflow_reason) + if dry_run: print("[dry-run] Would execute the following phases:") for label in [ @@ -401,6 +477,9 @@ def run_all( "serving/minimal-s3-model/minimal-s3-model.ipynb", ), ]: + if name in results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue f = _submit_notebook( executor, results, name, root / rel, output_dir, timeout_notebook ) @@ -424,6 +503,9 @@ def run_all( "pipelines/lightweight-components/mobile-price-classifications.ipynb", ), ]: + if name in results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue f = _submit_notebook( executor, results, name, root / rel, output_dir, timeout_notebook ) @@ -438,26 +520,51 @@ def run_all( ) phase2_futures["pipelines/lightweight-python-package"] = lpp_future - # Wait only for mlflow-mobile-price before launching phase 3 - phase2_futures["mlflow/mobile-price-classification"].result() - _print_result(results["mlflow/mobile-price-classification"]) + # Wait for mlflow-mobile-price notebook; then poll its KFP run + # inline before Phase 3 — the ISVC needs the registered model. + _mobile_price_skipped = ( + "mlflow/mobile-price-classification" not in phase2_futures + ) + if not _mobile_price_skipped: + phase2_futures["mlflow/mobile-price-classification"].result() + _print_result(results["mlflow/mobile-price-classification"]) + + _mobile_price_ok = ( + not _mobile_price_skipped + and results["mlflow/mobile-price-classification"].status == "PASS" + ) + + if _mobile_price_ok: + _mobile_run_ids = results[ + "mlflow/mobile-price-classification" + ].kfp_run_ids + if _mobile_run_ids: + print( + " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." + ) + for _run_id in _mobile_run_ids: + _state = _poll_kfp_run(_run_id, timeout_pipeline) + poll_results[_run_id] = ( + _state # recorded here; Phase 4 will skip it + ) + print(f" [{_state}] {_run_id[:8]}...") + if _state.upper() != "SUCCEEDED": + _mobile_price_ok = False # ── Phase 3: MLflow KServe examples (need registered model) ─────── print("\nPhase 3: deploying MLflow KServe InferenceServices...") - if results["mlflow/mobile-price-classification"].status == "FAIL": - print( - " [SKIP] mlflow-mobile-price-classification failed — " - "skipping ISVC tests that depend on the registered model" + if not _mobile_price_ok: + _reason = ( + "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" + if _mobile_price_skipped + else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" ) + print(f" [SKIP] {_reason}") for name in ( "serving/mlflow-kserve-minimal", "serving/mlflow-kserve-inference-protocols", ): - results[name] = Result( - name=name, - status="SKIP", - error="prerequisite mlflow/mobile-price-classification failed", - ) + results[name] = Result(name=name, status="SKIP", error=_reason) phase3_futures = {} else: phase3_futures = {} @@ -497,9 +604,13 @@ def run_all( _print_result(results[_future_to_name2[f]]) # ── Phase 4: KFP run polling ────────────────────────────────────── - all_run_ids: list[str] = [] - for r in results.values(): - all_run_ids.extend(r.kfp_run_ids) + # Exclude run IDs already polled inline before Phase 3 + all_run_ids: list[str] = [ + rid + for r in results.values() + for rid in r.kfp_run_ids + if rid not in poll_results + ] if all_run_ids: print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index 534aa12..a9a9528 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -67,8 +67,30 @@ def _apply(manifest: str, namespace: str) -> None: raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") +def _owner(namespace: str) -> str: + """Determine the key owner. + + Tries to read MLFLOW_TRACKING_USERNAME from the mlflow-credentials secret + (the canonical identity in the prokube platform). Falls back to a + namespace-scoped placeholder if the secret is absent. + """ + result = _kubectl( + "get", + "secret", + "mlflow-credentials", + "-n", + namespace, + "-o", + "jsonpath={.data.MLFLOW_TRACKING_USERNAME}", + ) + if result.returncode == 0 and result.stdout.strip(): + return base64.b64decode(result.stdout.strip()).decode() + return f"notebook-examples@{namespace}" + + def _create_key(namespace: str) -> str: key_value = f"pk_live_{secrets.token_hex(20)}" + owner = _owner(namespace) _apply( f"apiVersion: v1\n" @@ -89,6 +111,7 @@ def _create_key(namespace: str) -> str: f" namespace: {namespace}\n" f"spec:\n" f" displayName: Examples API key\n" + f" owner: {owner}\n" f" secretRef:\n" f" name: {_SECRET_NAME}\n" f" key: token\n" From 9c04ad47218dcfb2e83aa2ac52dbf387be5289a8 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 16:24:26 +0200 Subject: [PATCH 08/48] ci: pipeline error details, ci-skip cell tag mechanism, tag debug sections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci/run_all.py: - _poll_kfp_run now returns (status, error_detail); on FAILED state fetches run.error.message from the KFP v2 API so the user can see what went wrong without opening the KFP UI - Add poll_errors dict; Failed details section in the report shows the KFP error message under each failed run - Add _strip_ci_skip_cells(): preprocess notebooks before papermill runs by replacing cells tagged 'ci-skip' with a comment placeholder - _run_notebook() calls _strip_ci_skip_cells() transparently (cwd stays the original notebook directory so relative imports still work) pipelines/lightweight-components/mobile-price-classifications.ipynb: - Tag cells 26-30 (the 'Debugging with Lightweight Components' section) with 'ci-skip' — these cells hardcode specific S3 artifact UUIDs from past manual runs and cannot execute in CI --- ci/run_all.py | 56 +++++++++++++++---- .../mobile-price-classifications.ipynb | 50 +++++++++++++---- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 265508a..8228cdc 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -130,20 +130,41 @@ def _format_papermill_error(exc: Exception) -> str: return "\n".join(lines) +def _strip_ci_skip_cells(nb_path: Path, output_dir: Path) -> Path: + """Return a copy of the notebook with 'ci-skip' tagged cells replaced by a comment. + Returns the original path unchanged if no cells are tagged.""" + with open(nb_path) as fh: + nb = json.load(fh) + skipped = 0 + for cell in nb["cells"]: + if "ci-skip" in cell.get("metadata", {}).get("tags", []): + cell["source"] = ["# [CI] cell skipped (ci-skip tag)\n"] + skipped += 1 + if skipped == 0: + return nb_path + stripped = output_dir / f"_stripped_{nb_path.name}" + stripped.parent.mkdir(parents=True, exist_ok=True) + with open(stripped, "w") as fh: + json.dump(nb, fh, indent=1) + print(f" ({skipped} ci-skip cell(s) stripped from {nb_path.name})") + return stripped + + def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: """Execute a notebook with papermill. Returns the output notebook path.""" import papermill as pm from papermill.exceptions import PapermillExecutionError - output_path = output_dir / nb_path.name output_dir.mkdir(parents=True, exist_ok=True) + nb_to_run = _strip_ci_skip_cells(nb_path, output_dir) + output_path = output_dir / nb_path.name try: pm.execute_notebook( - str(nb_path), + str(nb_to_run), str(output_path), kernel_name="python3", execution_timeout=timeout, - cwd=str(nb_path.parent), + cwd=str(nb_path.parent), # always cwd to original notebook directory progress_bar=False, # avoid interleaved tqdm bars from parallel threads ) except PapermillExecutionError as exc: @@ -193,12 +214,12 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: _KFP_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "ERROR", "CANCELED", "SKIPPED"} -def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: - """Poll a KFP run until terminal state. Returns final status string.""" +def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, str]: + """Poll a KFP run until terminal state. Returns (status, error_detail).""" try: from kfp.client import Client except ImportError: - return "SKIP (kfp not installed)" + return "SKIP (kfp not installed)", "" client = Client() deadline = time.time() + timeout while time.time() < deadline: @@ -211,9 +232,14 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: or "UNKNOWN" ).upper() if state in _KFP_TERMINAL_STATES: - return state + error_detail = "" + if state not in ("SUCCEEDED", "SKIPPED"): + err = getattr(run, "error", None) + if err: + error_detail = getattr(err, "message", "") or "" + return state, error_detail time.sleep(interval) - return f"TIMEOUT after {timeout}s" + return f"TIMEOUT after {timeout}s", "" def _run_cleanup(cleanup_path: Path) -> None: @@ -301,6 +327,7 @@ def _print_result(r: Result) -> None: def _print_report( results: dict[str, Result], poll_results: dict[str, str], + poll_errors: dict[str, str], run_id_to_name: dict[str, str], ) -> None: col = 50 @@ -329,6 +356,10 @@ def _print_report( if status.upper() != "SUCCEEDED": source = run_id_to_name.get(run_id, "unknown") print(f"\nKFP run {run_id[:8]}... (from {source}): {status}") + err = poll_errors.get(run_id, "") + if err: + for line in err.splitlines(): + print(f" {line}") # Summary table last — easy to see final verdict at a glance print("\n" + "=" * 70) @@ -429,6 +460,7 @@ def run_all( output_dir = root / "ci" / "output" results: dict[str, Result] = {} poll_results: dict[str, str] = {} + poll_errors: dict[str, str] = {} _ensure_papermill() @@ -543,10 +575,11 @@ def run_all( " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." ) for _run_id in _mobile_run_ids: - _state = _poll_kfp_run(_run_id, timeout_pipeline) + _state, _err = _poll_kfp_run(_run_id, timeout_pipeline) poll_results[_run_id] = ( _state # recorded here; Phase 4 will skip it ) + poll_errors[_run_id] = _err print(f" [{_state}] {_run_id[:8]}...") if _state.upper() != "SUCCEEDED": _mobile_price_ok = False @@ -620,9 +653,10 @@ def run_all( } for run_id, f in poll_futures.items(): try: - poll_results[run_id] = f.result() + poll_results[run_id], poll_errors[run_id] = f.result() except Exception as exc: # noqa: BLE001 poll_results[run_id] = f"POLL_ERROR: {exc}" + poll_errors[run_id] = "" state = poll_results[run_id] print(f" [{state}] {run_id[:8]}...") else: @@ -642,7 +676,7 @@ def run_all( for run_id in r.kfp_run_ids: run_id_to_name[run_id] = r.name - _print_report(results, poll_results, run_id_to_name) + _print_report(results, poll_results, poll_errors, run_id_to_name) return results diff --git a/pipelines/lightweight-components/mobile-price-classifications.ipynb b/pipelines/lightweight-components/mobile-price-classifications.ipynb index 5179316..fd6af05 100644 --- a/pipelines/lightweight-components/mobile-price-classifications.ipynb +++ b/pipelines/lightweight-components/mobile-price-classifications.ipynb @@ -357,7 +357,9 @@ { "cell_type": "markdown", "id": "40b9ef30-3352-44ec-a0e4-9e52f032194e", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "### Evaluate model on validation dataset" ] @@ -366,7 +368,9 @@ "cell_type": "code", "execution_count": null, "id": "3f9453f2-0c56-4c25-af05-7dfe42eaf3b1", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "@dsl.component(\n", @@ -416,7 +420,9 @@ { "cell_type": "markdown", "id": "836412b8-fe89-48e9-8383-53a5db9761bf", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "### Run predictions on test dataset" ] @@ -425,7 +431,9 @@ "cell_type": "code", "execution_count": null, "id": "12e40c2d-889e-4cc6-bb9d-a722653acb30", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "@dsl.component(\n", @@ -478,7 +486,9 @@ { "cell_type": "markdown", "id": "604fab6f-5dc8-47e0-b15f-868032f076b2", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "## Build pipeline" ] @@ -622,7 +632,11 @@ { "cell_type": "markdown", "id": "2e73f8fc", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "source": [ "# Debugging with Lightweight Components\n", "Debugging can be challenging when using lightweight components in Kubeflow Pipelines. A practical approach is to download the artifacts from the steps preceding the failing one from object storage, and then run the functions used in the components locally. You can easily copy the paths to these artifacts from the Kubeflow Pipelines UI. Once you have these paths, you can use them as shown in the example below to download and read in Pandas DataFrames or perform similar operations. Make sure to adjust the paths according to your specific setup requirements." @@ -632,7 +646,11 @@ "cell_type": "code", "execution_count": null, "id": "315f6c39-4341-45a8-8c24-a81e0a92225f", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "storage_options={\n", @@ -646,7 +664,11 @@ "cell_type": "code", "execution_count": null, "id": "09f1655a-2e79-49c7-aae3-d698169cb7ee", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "# reading pandas dataframes (Modify paths!)\n", @@ -664,7 +686,11 @@ "cell_type": "code", "execution_count": null, "id": "f6fc60c0-714e-4623-bc2c-b3366de6f2b1", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "# use the mc tool to download model artifact\n", @@ -675,7 +701,11 @@ "cell_type": "code", "execution_count": null, "id": "e14a9baa-e110-45f1-90e9-c6eb2df71d9e", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "from joblib import load\n", From 1e5b8ada7083c9bf6510771da2ffabf6f13f8077 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 16:55:16 +0200 Subject: [PATCH 09/48] fix: dask version pins + KFP task-level error extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit notebooks/dask/dask_example.ipynb: - Drop pinned dask-kubernetes==2024.9.0 and dask[complete]==2024.9.1 which require anyio<4. Unpinned install resolves versions compatible with the cluster's anyio 4.12.1 and the updated dask operator. ci/run_all.py: - Add _get_failed_task_logs(): when a KFP run is FAILED and run.error is None (common in KFP v2), walks task_details → child_tasks → pod_name and tails the last 10 lines of the failed task pod's logs via kubectl. Falls back silently if pods are already gone or logs are unavailable. - _poll_kfp_run: reads namespace from SA token file and passes it to _get_failed_task_logs so CI now surfaces the actual pipeline failure message rather than showing nothing. --- ci/run_all.py | 61 ++++++++++++++++++++++++++++++- notebooks/dask/dask_example.ipynb | 2 +- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 8228cdc..a88bebb 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -214,6 +214,55 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: _KFP_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "ERROR", "CANCELED", "SKIPPED"} +def _get_failed_task_logs(run: object, namespace: str) -> str: + """Best-effort: tail logs from the pod(s) of the first failed KFP task. + + Walks run.run_details.task_details → child_tasks → pod_name and tries + kubectl logs on those pods. Silently returns "" on any failure. + """ + try: + task_details = ( + getattr(getattr(run, "run_details", None), "task_details", None) or [] + ) + for task in task_details: + if "FAIL" not in str(getattr(task, "state", "")).upper(): + continue + task_name = getattr(task, "display_name", "unknown-task") + for ct in getattr(task, "child_tasks", None) or []: + pod = ( + ct.get("pod_name") + if isinstance(ct, dict) + else getattr(ct, "pod_name", None) + ) + if not pod: + continue + for container in ("main", "user-main"): + result = subprocess.run( + [ + "kubectl", + "logs", + pod, + "-n", + namespace, + "-c", + container, + "--tail=15", + ], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0 and result.stdout.strip(): + header = f"[Task '{task_name}' / pod {pod[:30]}...]" + tail = "\n".join( + f" {l}" for l in result.stdout.strip().splitlines()[-10:] + ) + return f"{header}\n{tail}" + except Exception: # noqa: BLE001 + pass + return "" + + def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, str]: """Poll a KFP run until terminal state. Returns (status, error_detail).""" try: @@ -221,6 +270,11 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s except ImportError: return "SKIP (kfp not installed)", "" client = Client() + try: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + namespace = fh.read().strip() + except OSError: + namespace = "" deadline = time.time() + timeout while time.time() < deadline: run = client.get_run(run_id) @@ -234,9 +288,12 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s if state in _KFP_TERMINAL_STATES: error_detail = "" if state not in ("SUCCEEDED", "SKIPPED"): + # run.error is often None even for failed runs in KFP v2 — + # the real failure is in the task pod logs err = getattr(run, "error", None) - if err: - error_detail = getattr(err, "message", "") or "" + error_detail = (getattr(err, "message", "") or "") if err else "" + if not error_detail and namespace: + error_detail = _get_failed_task_logs(run, namespace) return state, error_detail time.sleep(interval) return f"TIMEOUT after {timeout}s", "" diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index 77fc563..21f4f86 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets dask-kubernetes==2024.9.0 pandas \"dask[complete]\"==2024.9.1" + "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\"\n" ] }, { From 857dbeeb9440acd7254adcd934363342c1ff9518 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:07:49 +0200 Subject: [PATCH 10/48] =?UTF-8?q?fix:=20mlflow=3D=3D3.10.0=20(non-existent?= =?UTF-8?q?)=20=E2=86=92=20mlflow>=3D3.0.0,<4=20in=20pipeline=20components?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The train_model and evaluate_model KFP components had mlflow==3.10.0 pinned in packages_to_install. That version does not exist (latest 3.x is 3.1.4), causing every pipeline run to fail at the package install step. Changed to mlflow>=3.0.0,<4 which installs the latest available 3.x release and stays compatible with the 3.x MLflow tracking server. --- .../mlflow-mobile-price-classification.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 13036b0..b2a3023 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -364,7 +364,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def train_model(\n", @@ -444,7 +444,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\", \"matplotlib\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\", \"matplotlib\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def evaluate_model(\n", From aa00d4452fce323c0fa6dd76cbb73ec9bfd145de Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:11:30 +0200 Subject: [PATCH 11/48] revert: restore mlflow==3.10.0 in pipeline components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin was correct — 3.10.0 exists on public PyPI and matches the server. The failure is that KFP component pods (python:3.9 base) have restricted network access and can only see an older PyPI snapshot. Root cause: network policy issue for component pods, not a version mismatch. Needs a platform-level fix (open network access or use a base image that already has mlflow pre-installed). --- .../mlflow-mobile-price-classification.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index b2a3023..13036b0 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -364,7 +364,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def train_model(\n", @@ -444,7 +444,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\", \"matplotlib\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\", \"matplotlib\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def evaluate_model(\n", From 81d0c5caeea5a148373373157fbd1a36a2c42373 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:13:28 +0200 Subject: [PATCH 12/48] fix: upgrade KFP component base images from python:3.9 to python:3.11 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mlflow>=3.2.0 requires Python >=3.10. With python:3.9 as the base image, pip silently excludes all mlflow versions that have Requires-Python>=3.10, leaving only versions up to 3.1.4 visible — which is why mlflow==3.10.0 appeared 'not found'. Upgrading to python:3.11 makes the full PyPI version range visible and satisfies the requirement. Affects all @dsl.component decorators in: - mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb (7 components) - pipelines/lightweight-components/mobile-price-classifications.ipynb (7 components) --- .../mlflow-mobile-price-classification.ipynb | 14 +++++++------- .../mobile-price-classifications.ipynb | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 13036b0..ef18fd4 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -171,7 +171,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"pyarrow\", \"s3fs\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def read_data(\n", " train_data_path: str,\n", @@ -206,7 +206,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def split_data(\n", " train_df: Input[Dataset],\n", @@ -254,7 +254,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def fit_scaler(\n", " train_x: Input[Dataset],\n", @@ -295,7 +295,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def tune_hyperparams(\n", " train_x: Input[Dataset],\n", @@ -365,7 +365,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def train_model(\n", " train_x: Input[Dataset],\n", @@ -445,7 +445,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\", \"matplotlib\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def evaluate_model(\n", " val_x: Input[Dataset],\n", @@ -530,7 +530,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"plotly\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def test_model(\n", " test_x: Input[Dataset],\n", diff --git a/pipelines/lightweight-components/mobile-price-classifications.ipynb b/pipelines/lightweight-components/mobile-price-classifications.ipynb index fd6af05..d4954c4 100644 --- a/pipelines/lightweight-components/mobile-price-classifications.ipynb +++ b/pipelines/lightweight-components/mobile-price-classifications.ipynb @@ -125,7 +125,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"pyarrow\", \"s3fs\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def read_data(\n", " train_data_path: str,\n", @@ -160,7 +160,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def split_data(\n", " train_df: Input[Dataset],\n", @@ -208,7 +208,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def fit_scaler(\n", " train_x: Input[Dataset],\n", @@ -249,7 +249,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def tune_hyperparams(\n", " train_x: Input[Dataset],\n", @@ -319,7 +319,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def train_model(\n", " train_x: Input[Dataset],\n", @@ -375,7 +375,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def evaluate_model(\n", " val_x: Input[Dataset],\n", @@ -438,7 +438,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"plotly\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def test_model(\n", " test_x: Input[Dataset],\n", From a66758f6e168ad74523f44b341c6e38a14ac0765 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:17:56 +0200 Subject: [PATCH 13/48] build: add mobile-price-classification components image v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockerfile: python:3.9-slim → python:3.11-slim (same Python version issue as the KFP lightweight components — newer packages require Python >=3.10) pipeline.py: COMPONENTS_IMAGE default bumped to :v2; falls back to COMPONENTS_IMAGE env var so the image path can be overridden without a code change. .github/workflows/mobile-price-classification-components.yaml: new workflow_dispatch workflow to build and push the image. Accepts a tag input (default: v2) and pushes :v2, :latest, and :commit-. --- ...obile-price-classification-components.yaml | 53 +++++++++++++++++++ .../lightweight-python-package/Dockerfile | 2 +- .../lightweight-python-package/pipeline.py | 30 ++++++++--- 3 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/mobile-price-classification-components.yaml diff --git a/.github/workflows/mobile-price-classification-components.yaml b/.github/workflows/mobile-price-classification-components.yaml new file mode 100644 index 0000000..612d5fe --- /dev/null +++ b/.github/workflows/mobile-price-classification-components.yaml @@ -0,0 +1,53 @@ +name: Build Mobile Price Classification Components Image + +on: + workflow_dispatch: + inputs: + tag: + description: "Image tag (e.g. v2, v3)" + required: true + default: "v2" + +env: + DOCKERFILE_PATH: "pipelines/lightweight-python-package/Dockerfile" + BUILD_CONTEXT_DIR: "pipelines/lightweight-python-package/" + IMAGE_NAME: "mobile-price-classification" + IMAGE_PUSH_PATH: "${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }}" + IMAGE_PUSH_SECRET: "${{ secrets.GCP_SA_KEY }}" + +jobs: + build-image: + runs-on: ubuntu-latest + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Auth GCloud CLI + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ env.IMAGE_PUSH_SECRET }} + + - name: Docker Login to Google Artifact Registry + uses: docker/login-action@v3 + with: + registry: europe-west3-docker.pkg.dev + username: _json_key + password: ${{ env.IMAGE_PUSH_SECRET }} + + - name: Build and push image + run: | + TAG=${{ github.event.inputs.tag }} + VERSIONED=${{ env.IMAGE_PUSH_PATH }}/${{ env.IMAGE_NAME }}:${TAG} + LATEST=${{ env.IMAGE_PUSH_PATH }}/${{ env.IMAGE_NAME }}:latest + COMMIT=${{ env.IMAGE_PUSH_PATH }}/${{ env.IMAGE_NAME }}:commit-${{ github.sha }} + BUILD_CONTEXT="${GITHUB_WORKSPACE}/${{ env.BUILD_CONTEXT_DIR }}" + docker build -t $VERSIONED -f ${{ env.DOCKERFILE_PATH }} "${BUILD_CONTEXT}" + docker tag $VERSIONED $LATEST + docker tag $VERSIONED $COMMIT + docker push $VERSIONED + docker push $LATEST + docker push $COMMIT + echo "Pushed: $VERSIONED" diff --git a/pipelines/lightweight-python-package/Dockerfile b/pipelines/lightweight-python-package/Dockerfile index 071487b..7a50c53 100644 --- a/pipelines/lightweight-python-package/Dockerfile +++ b/pipelines/lightweight-python-package/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim WORKDIR /app diff --git a/pipelines/lightweight-python-package/pipeline.py b/pipelines/lightweight-python-package/pipeline.py index f1d89b2..f770aaf 100644 --- a/pipelines/lightweight-python-package/pipeline.py +++ b/pipelines/lightweight-python-package/pipeline.py @@ -2,10 +2,22 @@ from typing import Dict, List from kfp import dsl -from kfp.dsl import HTML, Input, Output, Dataset, Artifact, Model, ClassificationMetrics, Markdown +from kfp.dsl import ( + HTML, + Input, + Output, + Dataset, + Artifact, + Model, + ClassificationMetrics, + Markdown, +) -COMPONENTS_IMAGE = "europe-west3-docker.pkg.dev/prokube-internal/prokube-customer/mobile-price-classification:v1" +COMPONENTS_IMAGE = os.environ.get( + "COMPONENTS_IMAGE", + "europe-west3-docker.pkg.dev/prokube-internal/prokube-customer/mobile-price-classification:v2", +) @dsl.component(base_image=COMPONENTS_IMAGE) @@ -187,16 +199,18 @@ def mobile_price_classification_pipeline( test_data_path=test_data_path, ) # Use the cluster internal object storage endpoint - read_data_task.set_env_variable('AWS_ENDPOINT_URL', f'http://{os.environ["S3_ENDPOINT"]}') + read_data_task.set_env_variable( + "AWS_ENDPOINT_URL", f"http://{os.environ['S3_ENDPOINT']}" + ) # Use Kubernetes secrets to provide AWS credentials to the read_data component kubernetes.use_secret_as_env( read_data_task, - secret_name='s3creds', + secret_name="s3creds", secret_key_to_env={ - 'AWS_ACCESS_KEY_ID': 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY': 'AWS_SECRET_ACCESS_KEY', - } - ) + "AWS_ACCESS_KEY_ID": "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY": "AWS_SECRET_ACCESS_KEY", + }, + ) # Step 2: Split the data split_data_task = split_data( From 407b5f335668cf6fd4b84f50d17e62aaad85f7bb Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 18:22:41 +0200 Subject: [PATCH 14/48] fix: upgrade anyio>=4.12 in dask notebook pip install httpx_ws (a dask-kubernetes dependency via kr8s) uses anyio.AsyncContextManagerMixin which was removed in anyio 4.0 and re-added as a compat alias in anyio 4.12+. The etst-0 CI notebook has anyio 4.6.0 (no alias) while test-0 has 4.12.1 (alias present). Adding anyio>=4.12 to the pip install ensures the compat alias is available regardless of which notebook image version is running. --- notebooks/dask/dask_example.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index 21f4f86..4733128 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\"\n" + "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\" \"anyio>=4.12\"\n" ] }, { From 16322db492e793cdf1140e2b6fc968d79dd57a4a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 18:30:08 +0200 Subject: [PATCH 15/48] fix: pin dask to 2026.3.0 and update worker image tag Pin dask[complete] and dask-kubernetes to 2026.3.0 to match the latest available daskdev/dask Docker image (no 2026.6.0 image exists yet on Docker Hub). Mismatched scheduler versions between the client (2026.6.0 from an unpinned install) and the worker image caused: TypeError: Scheduler.identity() got an unexpected keyword argument 'n_workers' Also updates the KubeCluster image from daskdev/dask:2024.9.1-py3.11 to daskdev/dask:2026.3.0-py3.11. --- notebooks/dask/dask_example.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index 4733128..b8ac14b 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\" \"anyio>=4.12\"\n" + "!pip install --upgrade ipywidgets \"dask-kubernetes==2026.3.0\" pandas \"dask[complete]==2026.3.0\" \"anyio>=4.12\"\n" ] }, { @@ -120,7 +120,7 @@ }, "outputs": [], "source": [ - "cluster = KubeCluster(name=\"test-cluster\", namespace=namespace, image=\"daskdev/dask:2024.9.1-py3.11\")\n", + "cluster = KubeCluster(name=\"test-cluster\", namespace=namespace, image=\"daskdev/dask:2026.3.0-py3.11\")\n", "cluster.scale(3)\n", "client = Client(cluster)" ] From bfc6024b96f8e5f92c7d1fa9c2e9b2783d838975 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 10:35:48 +0200 Subject: [PATCH 16/48] ci: add mobile-price-classification, mnist-vae, minimal-container-components, mlflow-kserve test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipelines/minimal-container-components/submit-cluster.py: - Print KFP_RUN_ID so CI can poll the pipeline run status notebooks/mnist-vae/visualizations.ipynb: - Tag the ipywidgets.interact cell with ci-skip (GUI-only, blocks headless CI) serving/mlflow-kserve-minimal/apply.py: - Add test() function: gets API key, runs test_inference_service.py with API_KEY and INFERENCE_SERVICE_URI env vars set, asserts success - Add deploy_and_test() which chains deploy() then test() - __main__ now calls deploy_and_test() so CI gets a full smoke test ci/run_all.py: - _run_script: accept extra_args for passing CLI flags (e.g. --max_epochs) - _submit_script: propagate extra_args - _submit_chain: new helper to run a sequential list of notebook/script steps as a single named CI result (used for mnist-vae train→visualize) - Phase 1: add notebooks/mobile-price-classification (standalone notebook) and notebooks/mnist-vae (training 3 epochs → visualizations chain) - Phase 2: add pipelines/minimal-container-components - Phase 3: mlflow-kserve-minimal now calls apply.py which does deploy_and_test; inference-protocols unchanged --- ci/run_all.py | 127 ++++++++++++++---- notebooks/mnist-vae/visualizations.ipynb | 6 +- .../submit-cluster.py | 14 +- serving/mlflow-kserve-minimal/apply.py | 46 ++++++- 4 files changed, 156 insertions(+), 37 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index a88bebb..6750e6c 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -172,17 +172,21 @@ def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: return output_path -def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: +def _run_script( + script_path: Path, + timeout: int, + extra_args: list[str] | None = None, +) -> tuple[str, str]: """Run a plain Python script. Returns (stdout, stderr).""" + cmd = [sys.executable, str(script_path)] + (extra_args or []) result = subprocess.run( - [sys.executable, str(script_path)], + cmd, capture_output=True, text=True, timeout=timeout, cwd=str(script_path.parent), ) if result.returncode != 0: - # Include both stderr and stdout for full context detail = (result.stderr or result.stdout or "(no output)").strip() raise RuntimeError(detail) return result.stdout, result.stderr @@ -349,6 +353,7 @@ def _submit_script( name: str, script_path: Path, timeout: int, + extra_args: list[str] | None = None, ) -> Future: result = Result(name=name) results[name] = result @@ -357,7 +362,7 @@ def _submit_script( def _run(): t0 = time.time() try: - stdout, _ = _run_script(script_path, timeout) + stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) result.status = "PASS" result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) except Exception as exc: # noqa: BLE001 @@ -369,6 +374,45 @@ def _run(): return executor.submit(_run) +def _submit_chain( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + steps: list[tuple[str, Path, dict]], + output_dir: Path, + timeout: int, +) -> Future: + """Submit a sequential chain of (kind, path, kwargs) steps as a single named result. + + kind is 'notebook' or 'script'. kwargs may include 'extra_args' for scripts. + All steps run in one thread; the first failure stops the chain. + """ + result = Result(name=name) + results[name] = result + print(f" [START ] {name}") + + def _run(): + t0 = time.time() + try: + for kind, path, kwargs in steps: + if kind == "notebook": + out = _run_notebook(path, output_dir, timeout) + result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) + elif kind == "script": + stdout, _ = _run_script( + path, timeout, extra_args=kwargs.get("extra_args") + ) + result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + result.status = "PASS" + except Exception as exc: # noqa: BLE001 + result.status = "FAIL" + result.error = str(exc) + finally: + result.duration = time.time() - t0 + + return executor.submit(_run) + + # ── Report ──────────────────────────────────────────────────────────────────── @@ -558,6 +602,10 @@ def run_all( phase1_futures = {} for name, rel in [ ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), + ( + "notebooks/mobile-price-classification", + "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", + ), ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), @@ -574,6 +622,24 @@ def run_all( ) phase1_futures[name] = f + # mnist-vae: training script (reduced epochs) → visualization notebook + # run as a sequential chain in a single thread, parallel to other Phase 1 + phase1_futures["notebooks/mnist-vae"] = _submit_chain( + executor, + results, + "notebooks/mnist-vae", + steps=[ + ( + "script", + root / "notebooks/mnist-vae/run_training.py", + {"extra_args": ["--max_epochs", "3"]}, + ), + ("notebook", root / "notebooks/mnist-vae/visualizations.ipynb", {}), + ], + output_dir=output_dir, + timeout=timeout_notebook, + ) + # wait for all phase 1, printing results as each finishes _future_to_name = {v: k for k, v in phase1_futures.items()} for f in as_completed(phase1_futures.values()): @@ -609,6 +675,18 @@ def run_all( ) phase2_futures["pipelines/lightweight-python-package"] = lpp_future + mcc_future = _submit_script( + executor, + results, + "pipelines/minimal-container-components", + root + / "pipelines" + / "minimal-container-components" + / "submit-cluster.py", + timeout_notebook, + ) + phase2_futures["pipelines/minimal-container-components"] = mcc_future + # Wait for mlflow-mobile-price notebook; then poll its KFP run # inline before Phase 3 — the ISVC needs the registered model. _mobile_price_skipped = ( @@ -658,26 +736,29 @@ def run_all( phase3_futures = {} else: phase3_futures = {} - for name, rel in [ - ( - "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-minimal/apply.py", - ), - ( + + # mlflow-kserve-minimal: deploy then immediately smoke-test + # deploy_and_test() in apply.py handles both steps + phase3_futures["serving/mlflow-kserve-minimal"] = _submit_script( + executor, + results, + "serving/mlflow-kserve-minimal", + root / "serving" / "mlflow-kserve-minimal" / "apply.py", + timeout_notebook, + ) + + # inference-protocols notebook handles its own deploy + test + phase3_futures["serving/mlflow-kserve-inference-protocols"] = ( + _submit_notebook( + executor, + results, "serving/mlflow-kserve-inference-protocols", - "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - ), - ]: - path = root / rel - if path.suffix == ".ipynb": - f = _submit_notebook( - executor, results, name, path, output_dir, timeout_notebook - ) - else: - f = _submit_script( - executor, results, name, path, timeout_notebook - ) - phase3_futures[name] = f + root + / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + output_dir, + timeout_notebook, + ) + ) # Phase 2 remaining + phase 3 run concurrently; print as each finishes print("\nPhase 2 (remaining) + Phase 3 running concurrently...") diff --git a/notebooks/mnist-vae/visualizations.ipynb b/notebooks/mnist-vae/visualizations.ipynb index 95a78da..354fa49 100644 --- a/notebooks/mnist-vae/visualizations.ipynb +++ b/notebooks/mnist-vae/visualizations.ipynb @@ -101,7 +101,11 @@ "cell_type": "code", "execution_count": null, "id": "d032c5d3-1476-40b6-b32f-68b462c27890", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "def generate_image(z1=0, z2=0):\n", diff --git a/pipelines/minimal-container-components/submit-cluster.py b/pipelines/minimal-container-components/submit-cluster.py index 334d1f8..6b95a77 100644 --- a/pipelines/minimal-container-components/submit-cluster.py +++ b/pipelines/minimal-container-components/submit-cluster.py @@ -7,15 +7,13 @@ if __name__ == "__main__": client = Client() - compiler.Compiler().compile(container_components_pipeline, 'pipeline.yaml') + compiler.Compiler().compile(container_components_pipeline, "pipeline.yaml") run = client.create_run_from_pipeline_package( - 'pipeline.yaml', + "pipeline.yaml", enable_caching=False, - arguments={ - "input1": "Hello", - "input2": "world!" - }, - experiment_name='minimal-container-components-experiment', - run_name=f'Simple pipeline {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}' + arguments={"input1": "Hello", "input2": "world!"}, + experiment_name="minimal-container-components-experiment", + run_name=f"Simple pipeline {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", ) + print(f"KFP_RUN_ID={run.run_id}") diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index b3896bd..8268ef8 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -1,9 +1,9 @@ """ -Deploy the mlflow-kserve-minimal InferenceService. +Deploy the mlflow-kserve-minimal InferenceService and run an inference test. Substitutes ```` and ```` in InferenceService.yaml -from the cluster environment, applies it, waits for readiness, and prints the -external URL so it can be used by ``test_inference_service.py``. +from the cluster environment, applies it, waits for readiness, runs a smoke +test via ``test_inference_service.py``, and prints the external URL. Usage from a notebook --------------------- @@ -17,7 +17,7 @@ CLI usage --------- python apply.py - # prints ISVC_URI= to stdout + # prints ISVC_URI= and runs inference smoke test Prerequisites ------------- @@ -142,9 +142,45 @@ def deploy(timeout: int = 600) -> str: return url +_HERE = os.path.dirname(os.path.abspath(__file__)) +_TEST_SCRIPT = os.path.join(_HERE, "test_inference_service.py") +_TEST_JSON = os.path.join(_HERE, "v2-mlflow-inference-body.json") + + +def test(uri: str) -> None: + """Run the inference smoke test against the deployed ISVC.""" + # Get / create the API key + _root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from get_or_create_api_key import get_or_create_api_key # noqa: PLC0415 + + api_key = get_or_create_api_key() + + print(f"Running inference smoke test against {uri} ...") + result = subprocess.run( + [sys.executable, _TEST_SCRIPT, "--json", _TEST_JSON, "--model", _ISVC_NAME], + capture_output=True, + text=True, + env={**os.environ, "API_KEY": api_key, "INFERENCE_SERVICE_URI": uri}, + ) + if result.returncode != 0: + raise RuntimeError(f"Inference test failed:\n{result.stdout}\n{result.stderr}") + print("Inference test passed.") + print(result.stdout.strip()) + + +def deploy_and_test(timeout: int = 600) -> str: + """Deploy the ISVC, wait for readiness, run smoke test, return the URL.""" + uri = deploy(timeout=timeout) + test(uri) + return uri + + if __name__ == "__main__": try: - deploy() + deploy_and_test() except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) From ebd9da27656d0764db0823d603f27cc7ada18bf9 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 13:45:33 +0200 Subject: [PATCH 17/48] ci: make notebooks/mnist-vae opt-in via --include-pytorch pytorch_lightning is not installed in all notebook images. Running the example without it fails immediately with ModuleNotFoundError. Add --include-pytorch flag (default off); mnist-vae is skipped with a clear message unless the flag is passed. Same pattern as --include-keda. --- ci/run_all.py | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 6750e6c..84d269b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -555,6 +555,7 @@ def run_all( timeout_notebook: int = 1800, timeout_pipeline: int = 3600, include_keda: bool = False, + include_pytorch: bool = False, dry_run: bool = False, ) -> dict[str, Result]: root = _REPO_ROOT @@ -622,23 +623,31 @@ def run_all( ) phase1_futures[name] = f - # mnist-vae: training script (reduced epochs) → visualization notebook - # run as a sequential chain in a single thread, parallel to other Phase 1 - phase1_futures["notebooks/mnist-vae"] = _submit_chain( - executor, - results, - "notebooks/mnist-vae", - steps=[ - ( - "script", - root / "notebooks/mnist-vae/run_training.py", - {"extra_args": ["--max_epochs", "3"]}, - ), - ("notebook", root / "notebooks/mnist-vae/visualizations.ipynb", {}), - ], - output_dir=output_dir, - timeout=timeout_notebook, - ) + # mnist-vae: opt-in (requires pytorch_lightning, not in all images) + if include_pytorch: + phase1_futures["notebooks/mnist-vae"] = _submit_chain( + executor, + results, + "notebooks/mnist-vae", + steps=[ + ( + "script", + root / "notebooks/mnist-vae/run_training.py", + {"extra_args": ["--max_epochs", "3"]}, + ), + ( + "notebook", + root / "notebooks/mnist-vae/visualizations.ipynb", + {}, + ), + ], + output_dir=output_dir, + timeout=timeout_notebook, + ) + else: + print( + " [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)" + ) # wait for all phase 1, printing results as each finishes _future_to_name = {v: k for k, v in phase1_futures.items()} @@ -840,6 +849,11 @@ def run_all( action="store_true", help="Include GPU-dependent KEDA autoscaling example (opt-in)", ) + parser.add_argument( + "--include-pytorch", + action="store_true", + help="Include pytorch_lightning examples (opt-in; requires pytorch in the notebook image)", + ) parser.add_argument( "--dry-run", action="store_true", help="Print plan without executing anything" ) @@ -849,6 +863,7 @@ def run_all( timeout_notebook=args.timeout_notebook, timeout_pipeline=args.timeout_pipeline, include_keda=args.include_keda, + include_pytorch=args.include_pytorch, dry_run=args.dry_run, ) From e07aa7f5b0c8b14512b9b4f7e9264ffbd442dff1 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 15:13:21 +0200 Subject: [PATCH 18/48] ci: install pytorch-lightning before mnist-vae training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pytorch-full image ships PyTorch but not pytorch-lightning. Add a 'pip' step kind to _submit_chain and use it to install pytorch-lightning before running run_training.py when --include-pytorch is set. Also documents the numpy/sklearn issue: the pytorch-full image pins numpy to 1.26.4 but scikit-learn 1.6.1 requires numpy >=2.0 for the _signature_descriptor ABI. Fix is in the image (upgrade numpy to 2.x — PyTorch 2.5.1 supports it) not in the examples. --- ci/run_all.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/run_all.py b/ci/run_all.py index 84d269b..70e41b1 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -378,7 +378,7 @@ def _submit_chain( executor: ThreadPoolExecutor, results: dict[str, Result], name: str, - steps: list[tuple[str, Path, dict]], + steps: list[tuple[str, Path | None, dict]], output_dir: Path, timeout: int, ) -> Future: @@ -403,6 +403,14 @@ def _run(): path, timeout, extra_args=kwargs.get("extra_args") ) result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + elif kind == "pip": + r = subprocess.run( + [sys.executable, "-m", "pip", "install", "-q"] + + kwargs.get("packages", []), + capture_output=True, + text=True, + check=True, + ) result.status = "PASS" except Exception as exc: # noqa: BLE001 result.status = "FAIL" @@ -630,6 +638,8 @@ def run_all( results, "notebooks/mnist-vae", steps=[ + # install pytorch-lightning if not already present in the image + ("pip", None, {"packages": ["pytorch-lightning"]}), ( "script", root / "notebooks/mnist-vae/run_training.py", From c03700dc30d18ecb64d31866706aaf8921cce502 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 15:16:05 +0200 Subject: [PATCH 19/48] fix: self-install pytorch-lightning in run_training.py pytorch-lightning is not bundled in all notebook images. The script now installs it silently on first run if absent, so users and CI don't need to pre-install it. The explicit pip step in the CI chain is removed since the script handles its own dependency. --- ci/run_all.py | 3 +-- notebooks/mnist-vae/run_training.py | 30 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 70e41b1..f5cef9b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -638,8 +638,7 @@ def run_all( results, "notebooks/mnist-vae", steps=[ - # install pytorch-lightning if not already present in the image - ("pip", None, {"packages": ["pytorch-lightning"]}), + # run_training.py self-installs pytorch-lightning if absent ( "script", root / "notebooks/mnist-vae/run_training.py", diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index e52458f..1967d93 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -1,3 +1,15 @@ +import subprocess +import sys + +# pytorch-lightning is not bundled in all notebook images — install if absent +try: + import pytorch_lightning # noqa: F401 +except ImportError: + subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", "pytorch-lightning"], + check=True, + ) + import click import pytorch_lightning as pl from pytorch_lightning.loggers import TensorBoardLogger @@ -5,9 +17,14 @@ from model.datamodule import MNISTDataModule import torch + @click.command() -@click.option('--hidden_dim', default=400, type=int, help='Dimension of the hidden layer.') -@click.option('--latent_dim', default=2, type=int, help='Dimension of the latent space.') +@click.option( + "--hidden_dim", default=400, type=int, help="Dimension of the hidden layer." +) +@click.option( + "--latent_dim", default=2, type=int, help="Dimension of the latent space." +) def run(hidden_dim: int, latent_dim: int) -> None: """ Train a VAE model on the MNIST dataset using PyTorch Lightning. @@ -23,8 +40,8 @@ def run(hidden_dim: int, latent_dim: int) -> None: # Initialize model model = VAE( - input_dim=784, # 28x28 pixels - hidden_dim=hidden_dim, # Dimension of the hidden layer + input_dim=784, # 28x28 pixels + hidden_dim=hidden_dim, # Dimension of the hidden layer latent_dim=latent_dim, # Dimension of the latent space ) @@ -36,11 +53,12 @@ def run(hidden_dim: int, latent_dim: int) -> None: max_epochs=50, accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=1, - logger=logger + logger=logger, ) # Start training trainer.fit(model=model, datamodule=dm) -if __name__ == '__main__': + +if __name__ == "__main__": run() From d710ea4e4ac2a5401d968a18f27bd100504f95b8 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 16:53:43 +0200 Subject: [PATCH 20/48] fix: remove multi_class param from LogisticRegression (removed in sklearn 1.6) --- mlflow/mlflow-quickstart-example.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index 06c796a..b60514d 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -112,7 +112,6 @@ "params = {\n", " \"solver\": \"lbfgs\",\n", " \"max_iter\": 1000,\n", - " \"multi_class\": \"auto\",\n", " \"random_state\": 8888,\n", "}\n", "\n", From 63727634d518ac48e88588fd0481924116f7f0e1 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 17:38:06 +0200 Subject: [PATCH 21/48] fix: strip @domain from MLflow username when constructing ISVC model name The KFP pipeline registers models as 'mobile-price-svm-{local_user}' using MLFLOW_TRACKING_USERNAME.split('@')[0] (cell 19 of the notebook). Both apply.py and the inference-protocols setup cell were passing the full email (e.g. 'developer1@prokube.cloud') as the model name suffix, causing the ISVC storageUri to reference a non-existent model and timing out waiting for readiness. --- .../inference_protocol_version_example.ipynb | 2 +- serving/mlflow-kserve-minimal/apply.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 83f7898..0d07609 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -372,7 +372,7 @@ " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", " )\n", "_creds = json.loads(_secret.stdout)[\"data\"]\n", - "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode()\n", + "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode().split(\"@\")[0]\n", "\n", "# ── 2. Apply InferenceService YAMLs with substituted namespace & username ────\n", "_isvc_names = [\n", diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index 8268ef8..ba24be9 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -64,7 +64,11 @@ def _mlflow_username(namespace: str) -> str: "Run scripts/setup_mlflow_credentials.py to create it." ) data = json.loads(result.stdout)["data"] - return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + full = base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + # Pipeline registers models as "mobile-price-svm-{local_user}" where + # local_user = MLFLOW_TRACKING_USERNAME.split('@')[0], e.g. + # "developer1@prokube.cloud" → "developer1" + return full.split("@")[0] def _kubectl_apply(manifest: str, namespace: str) -> None: From 023c31f198369f17d3e7c38dff8c0b2f98d7900a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 18:43:00 +0200 Subject: [PATCH 22/48] fix: add mlflow-isvc-sa SA to avoid KServe S3 credential injection conflict KServe v0.18.0 injects S3 env vars (S3_ENDPOINT, AWS_ENDPOINT_URL, S3_USE_HTTPS) into the storage-initializer for every ISVC whose SA references s3creds. The MLflow storage-initializer sets the same vars via valueFrom, producing a value+valueFrom conflict Kubernetes rejects at admission. Pods are never created and the ISVC times out. - Add ServiceAccount.yaml: mlflow-isvc-sa with regcred-prokube and regcred-dev imagePullSecrets but no s3creds reference - Set serviceAccountName: mlflow-isvc-sa in InferenceService.yaml - Update apply.py to apply the SA manifest before the ISVC - Document the root cause and fix in README.md --- .../InferenceService.yaml | 1 + serving/mlflow-kserve-minimal/README.md | 57 ++++++++++++++++++- .../mlflow-kserve-minimal/ServiceAccount.yaml | 20 +++++++ serving/mlflow-kserve-minimal/apply.py | 9 ++- 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 serving/mlflow-kserve-minimal/ServiceAccount.yaml diff --git a/serving/mlflow-kserve-minimal/InferenceService.yaml b/serving/mlflow-kserve-minimal/InferenceService.yaml index ecbdc2d..6000116 100644 --- a/serving/mlflow-kserve-minimal/InferenceService.yaml +++ b/serving/mlflow-kserve-minimal/InferenceService.yaml @@ -5,6 +5,7 @@ metadata: namespace: "" spec: predictor: + serviceAccountName: mlflow-isvc-sa model: modelFormat: name: mlflow diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index b3c7461..cdd0e24 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -12,9 +12,58 @@ InferenceService using the v2 inference protocol. It uses the built-in - `kubectl` access to your prokube namespace. (already installed in a pk-notebook) - Python with the `requests` package installed (for testing, already installed in a pk-notebook) +## Why a dedicated ServiceAccount is required + +> [!IMPORTANT] +> MLflow ISVCs **must** use the dedicated `mlflow-isvc-sa` ServiceAccount +> defined in `ServiceAccount.yaml`. Do not use the namespace `default` SA. + +### Root cause: KServe S3 credential injection conflict + +KServe v0.18.0 automatically injects S3 credentials into the +`storage-initializer` init container for every ISVC whose ServiceAccount +references a secret annotated as an S3 credential source (e.g. the +`s3creds` secret present in every prokube namespace). + +The injection produces two kinds of env entries for the same variable names +(`S3_ENDPOINT`, `AWS_ENDPOINT_URL`, `S3_USE_HTTPS`, etc.): + +| Source | Env form | +|---|---| +| Secret annotations (`serving.kserve.io/s3-endpoint`, `s3-usehttps`, …) | `value: ` | +| Secret keys (mounted via `valueFrom.secretKeyRef`) | `valueFrom: {secretKeyRef: …}` | + +Kubernetes rejects a container spec that sets **both** `value` and +`valueFrom` on the same env var. The admission webhook blocks the pod, the +storage-initializer never runs, and the ISVC times out with +`timed out waiting for condition`. + +This only affects `mlflow://` ISVCs: S3-backed ISVCs (`s3://`) rely on +exactly that injection and work correctly with the `default` SA. + +### The fix + +`ServiceAccount.yaml` defines `mlflow-isvc-sa`, a dedicated SA that: + +- Holds the `regcred-prokube` and `regcred-dev` imagePullSecrets so that + the private prokube storage-initializer image can be pulled. +- Does **not** reference `s3creds`, so KServe skips S3 credential injection + entirely for any ISVC that uses it. + +Both `InferenceService.yaml` and `apply.py` already reference this SA. + ## Deploy the InferenceService -1. Open `InferenceService.yaml` and replace the placeholder values: +1. Open `ServiceAccount.yaml` and replace `` with your + Kubeflow namespace, then apply it: + + ```sh + kubectl apply -f ServiceAccount.yaml -n + ``` + + You only need to do this once per namespace. + +2. Open `InferenceService.yaml` and replace the placeholder values: | Placeholder | Description | |---|---| @@ -32,17 +81,19 @@ InferenceService using the v2 inference protocol. It uses the built-in > You need to the a MLFlow ClusterStorageContainer in order to use the > `mlflow://` scheme (prokube platform versions >= 1.7.0) -2. Apply the manifest: +3. Apply the manifest: ```sh kubectl apply -f InferenceService.yaml -n ``` -3. Wait for the InferenceService to become ready. You can check the status in +4. Wait for the InferenceService to become ready. You can check the status in the Kubeflow Endpoints UI or via: ```sh kubectl get inferenceservice -n ``` +Alternatively, `apply.py` handles steps 1–4 automatically (see below). + ## Test the Deployment A test script and sample request body are provided to verify the deployment. diff --git a/serving/mlflow-kserve-minimal/ServiceAccount.yaml b/serving/mlflow-kserve-minimal/ServiceAccount.yaml new file mode 100644 index 0000000..ae26f66 --- /dev/null +++ b/serving/mlflow-kserve-minimal/ServiceAccount.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mlflow-isvc-sa + namespace: "" +# imagePullSecrets grant access to the private prokube container registry +# where the custom MLflow storage-initializer image lives. +# These secrets are provisioned automatically by the prokube platform. +imagePullSecrets: + - name: regcred-prokube + - name: regcred-dev +# NOTE: intentionally does not reference the 's3creds' Secret. +# KServe v0.18.0 injects S3 env vars into the storage-initializer init +# container for every ISVC whose ServiceAccount references 's3creds'. When +# the storageUri uses the mlflow:// scheme the injected env contains both a +# plain 'value' field (from the secret's annotations) and a 'valueFrom' +# field on the same variable names (S3_ENDPOINT, AWS_ENDPOINT_URL, +# S3_USE_HTTPS). Kubernetes rejects that combination at admission, so pods +# are never created and the ISVC times out. By using a dedicated SA without +# 's3creds' the injection is skipped entirely. diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index ba24be9..fd259b8 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -36,6 +36,7 @@ _ISVC_NAME = "mobile-price-svm" _YAML_TEMPLATE = os.path.join(os.path.dirname(__file__), "InferenceService.yaml") +_SA_YAML_TEMPLATE = os.path.join(os.path.dirname(__file__), "ServiceAccount.yaml") def _namespace() -> str: @@ -124,10 +125,16 @@ def _isvc_url(name: str, namespace: str) -> str: def deploy(timeout: int = 600) -> str: - """Apply InferenceService.yaml, wait for readiness, return the external URL.""" + """Apply ServiceAccount.yaml and InferenceService.yaml, wait for readiness, return the external URL.""" ns = _namespace() username = _mlflow_username(ns) + with open(_SA_YAML_TEMPLATE) as fh: + sa_manifest = fh.read().replace("", ns) + + _kubectl_apply(sa_manifest, ns) + print(f"Applied ServiceAccount 'mlflow-isvc-sa' in namespace '{ns}'.") + with open(_YAML_TEMPLATE) as fh: manifest = fh.read() From f2a20159195de209bb99bc977cd86b8830c33b4a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 18:43:06 +0200 Subject: [PATCH 23/48] fix: add mlflow-isvc-sa SA to inference-protocols example Same KServe S3 credential injection conflict fix as mlflow-kserve-minimal: the default SA references s3creds, causing a value+valueFrom env conflict for mlflow:// ISVCs that Kubernetes rejects at admission. - Add ServiceAccount.yaml: mlflow-isvc-sa with imagePullSecrets and no s3creds reference - Set serviceAccountName: mlflow-isvc-sa in v1 and v2 InferenceService manifests - Add README.md documenting the root cause, fix, deploy steps, and cleanup --- .../README.md | 110 ++++++++++++++++++ .../ServiceAccount.yaml | 20 ++++ .../v1-InferenceService.yaml | 1 + .../v2-InferenceService.yaml | 1 + 4 files changed, 132 insertions(+) create mode 100644 serving/mlflow-kserve-inference-protocols/README.md create mode 100644 serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md new file mode 100644 index 0000000..04feceb --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -0,0 +1,110 @@ +# MLflow KServe Inference Protocols + +This directory demonstrates both the v1 and v2 inference protocols for +MLflow-tracked models deployed as KServe InferenceServices on prokube. + +It uses two ISVCs side-by-side: + +| ISVC | Model format | Protocol | YAML | +|---|---|---|---| +| `v1-mobile-price-classification-inference` | `sklearn` | v1 | `v1-InferenceService.yaml` | +| `v2-mobile-price-classification-inference` | `mlflow` | v2 | `v2-InferenceService.yaml` | + +The notebook `inference_protocol_version_example.ipynb` walks through deploying +both and comparing request/response shapes for each protocol. + +## Prerequisites + +- A model trained and registered in MLflow. See the + [mobile price classification MLflow example](../../mlflow/mobile-price-classification/) + for how to train and register the SVM model used here. +- `kubectl` access to your prokube namespace. (already installed in a pk-notebook) +- Python with the `requests` package installed (for testing, already installed in a pk-notebook) + +## Why a dedicated ServiceAccount is required + +> [!IMPORTANT] +> MLflow ISVCs **must** use the dedicated `mlflow-isvc-sa` ServiceAccount +> defined in `ServiceAccount.yaml`. Do not use the namespace `default` SA. + +### Root cause: KServe S3 credential injection conflict + +KServe v0.18.0 automatically injects S3 credentials into the +`storage-initializer` init container for every ISVC whose ServiceAccount +references a secret annotated as an S3 credential source (e.g. the +`s3creds` secret present in every prokube namespace). + +The injection produces two kinds of env entries for the same variable names +(`S3_ENDPOINT`, `AWS_ENDPOINT_URL`, `S3_USE_HTTPS`, etc.): + +| Source | Env form | +|---|---| +| Secret annotations (`serving.kserve.io/s3-endpoint`, `s3-usehttps`, …) | `value: ` | +| Secret keys (mounted via `valueFrom.secretKeyRef`) | `valueFrom: {secretKeyRef: …}` | + +Kubernetes rejects a container spec that sets **both** `value` and +`valueFrom` on the same env var. The admission webhook blocks the pod, the +storage-initializer never runs, and the ISVC times out with +`timed out waiting for condition`. + +This only affects `mlflow://` ISVCs: S3-backed ISVCs (`s3://`) rely on +exactly that injection and work correctly with the `default` SA. + +### The fix + +`ServiceAccount.yaml` defines `mlflow-isvc-sa`, a dedicated SA that: + +- Holds the `regcred-prokube` and `regcred-dev` imagePullSecrets so that + the private prokube storage-initializer image can be pulled. +- Does **not** reference `s3creds`, so KServe skips S3 credential injection + entirely for any ISVC that uses it. + +Both `v1-InferenceService.yaml` and `v2-InferenceService.yaml` already +reference this SA. + +## Deploy + +1. Replace `` in `ServiceAccount.yaml` with your namespace + and apply it (once per namespace): + + ```sh + kubectl apply -f ServiceAccount.yaml -n + ``` + +2. Replace the placeholder values in both ISVC YAMLs: + + | Placeholder | Description | + |---|---| + | `` | Your Kubeflow namespace / workspace | + | `` | Your username, matching the model registered in MLflow | + +3. Apply both ISVCs: + + ```sh + kubectl apply -f v1-InferenceService.yaml -n + kubectl apply -f v2-InferenceService.yaml -n + ``` + +4. Wait for both to become ready: + + ```sh + kubectl get inferenceservice -n + ``` + + > [!WARNING] + > You need a MLFlow ClusterStorageContainer in order to use the + > `mlflow://` scheme (prokube platform versions >= 1.7.0) + +## Cleanup + +To delete both ISVCs: + +```sh +python cleanup.py +``` + +Or with `--dry-run` to preview the commands: + +```sh +python cleanup.py --dry-run +``` diff --git a/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml new file mode 100644 index 0000000..ae26f66 --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mlflow-isvc-sa + namespace: "" +# imagePullSecrets grant access to the private prokube container registry +# where the custom MLflow storage-initializer image lives. +# These secrets are provisioned automatically by the prokube platform. +imagePullSecrets: + - name: regcred-prokube + - name: regcred-dev +# NOTE: intentionally does not reference the 's3creds' Secret. +# KServe v0.18.0 injects S3 env vars into the storage-initializer init +# container for every ISVC whose ServiceAccount references 's3creds'. When +# the storageUri uses the mlflow:// scheme the injected env contains both a +# plain 'value' field (from the secret's annotations) and a 'valueFrom' +# field on the same variable names (S3_ENDPOINT, AWS_ENDPOINT_URL, +# S3_USE_HTTPS). Kubernetes rejects that combination at admission, so pods +# are never created and the ISVC times out. By using a dedicated SA without +# 's3creds' the injection is skipped entirely. diff --git a/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml b/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml index c2b2f1f..bb11b3e 100644 --- a/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml +++ b/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml @@ -5,6 +5,7 @@ metadata: namespace: "" spec: predictor: + serviceAccountName: mlflow-isvc-sa model: modelFormat: name: sklearn diff --git a/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml b/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml index 1f4db07..c933783 100644 --- a/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml +++ b/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml @@ -5,6 +5,7 @@ metadata: namespace: "" spec: predictor: + serviceAccountName: mlflow-isvc-sa model: modelFormat: name: mlflow From 411b023e03678c59378d6bf5d420484aee05d832 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:18:45 +0200 Subject: [PATCH 24/48] fix: skip KFP run ID extraction for ISVC-only notebooks in CI The inference-protocols notebook deploys InferenceServices and makes inference calls. Its output contains UUIDs (ISVC UIDs, request IDs) that the broad UUID regex in _extract_run_ids_from_notebook picks up. Phase 4 then polls these as KFP run IDs, which returns 404. Add extract_run_ids flag to _submit_notebook (default True) and pass False for serving/mlflow-kserve-inference-protocols, which never submits KFP pipelines. --- ci/run_all.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index f5cef9b..a170a6b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -327,6 +327,7 @@ def _submit_notebook( nb_path: Path, output_dir: Path, timeout: int, + extract_run_ids: bool = True, ) -> Future: result = Result(name=name) results[name] = result @@ -337,7 +338,8 @@ def _run(): try: out = _run_notebook(nb_path, output_dir, timeout) result.status = "PASS" - result.kfp_run_ids = _extract_run_ids_from_notebook(out) + if extract_run_ids: + result.kfp_run_ids = _extract_run_ids_from_notebook(out) except Exception as exc: # noqa: BLE001 result.status = "FAIL" result.error = str(exc) @@ -765,7 +767,10 @@ def run_all( timeout_notebook, ) - # inference-protocols notebook handles its own deploy + test + # inference-protocols notebook handles its own deploy + test. + # extract_run_ids=False: this notebook is not a pipeline submission; + # its output contains ISVC/request UUIDs that must not be polled + # as KFP run IDs (they return 404 from the KFP API). phase3_futures["serving/mlflow-kserve-inference-protocols"] = ( _submit_notebook( executor, @@ -775,6 +780,7 @@ def run_all( / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", output_dir, timeout_notebook, + extract_run_ids=False, ) ) From 074f62e7e6a79ccec022686633da356127eb5f67 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:40:10 +0200 Subject: [PATCH 25/48] fix: resolve libstdc++ CXXABI_1.3.15 error and wire --max_epochs in mnist-vae scipy 1.17+ is compiled against CXXABI_1.3.15 which the system libstdc++ lacks (tops out at 1.3.13). The conda-bundled libstdc++ provides it but is not on LD_LIBRARY_PATH by default. Re-exec the process with /opt/conda/lib prepended to LD_LIBRARY_PATH before any imports so the dynamic linker picks up the right library. Also add --max_epochs click option (default 50) so the CI can pass --max_epochs 3 for a fast smoke-test run; previously the option was silently ignored and the trainer always ran 50 epochs. --- notebooks/mnist-vae/run_training.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index 1967d93..4064976 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -1,6 +1,23 @@ +import os import subprocess import sys +# Re-exec with conda's libstdc++ prepended to LD_LIBRARY_PATH when needed. +# scipy 1.17+ requires CXXABI_1.3.15 which the system libstdc++ may lack; the +# conda-bundled libstdc++ provides it. LD_LIBRARY_PATH must be set before the +# process starts (the dynamic linker reads it once), so we re-exec if the +# conda lib dir is not already in the path. +_conda_lib = os.path.normpath( + os.path.join(os.path.dirname(sys.executable), "..", "lib") +) +_ld = os.environ.get("LD_LIBRARY_PATH", "") +if os.path.isdir(_conda_lib) and _conda_lib not in _ld.split(":"): + os.execvpe( + sys.executable, + [sys.executable] + sys.argv, + {**os.environ, "LD_LIBRARY_PATH": f"{_conda_lib}:{_ld}"}, + ) + # pytorch-lightning is not bundled in all notebook images — install if absent try: import pytorch_lightning # noqa: F401 @@ -25,13 +42,15 @@ @click.option( "--latent_dim", default=2, type=int, help="Dimension of the latent space." ) -def run(hidden_dim: int, latent_dim: int) -> None: +@click.option("--max_epochs", default=50, type=int, help="Number of training epochs.") +def run(hidden_dim: int, latent_dim: int, max_epochs: int) -> None: """ Train a VAE model on the MNIST dataset using PyTorch Lightning. Args: hidden_dim (int): Dimension of the hidden layer. latent_dim (int): Dimension of the latent space. + max_epochs (int): Number of training epochs. """ # Initialize data module @@ -50,7 +69,7 @@ def run(hidden_dim: int, latent_dim: int) -> None: # Initialize trainer trainer = pl.Trainer( - max_epochs=50, + max_epochs=max_epochs, accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=1, logger=logger, From 0e91774759dfc2554e9baa3cc9d37a53b2d035e8 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:55:39 +0200 Subject: [PATCH 26/48] fix: also self-install torchvision when missing in mnist-vae torchvision is absent from the base notebook image independently of pytorch-lightning. Extend the self-install guard to check both and install whichever packages are missing in a single pip call. --- notebooks/mnist-vae/run_training.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index 4064976..290ac12 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -18,12 +18,19 @@ {**os.environ, "LD_LIBRARY_PATH": f"{_conda_lib}:{_ld}"}, ) -# pytorch-lightning is not bundled in all notebook images — install if absent +# pytorch-lightning and torchvision are not bundled in all notebook images +_missing = [] try: import pytorch_lightning # noqa: F401 except ImportError: + _missing.append("pytorch-lightning") +try: + import torchvision # noqa: F401 +except ImportError: + _missing.append("torchvision") +if _missing: subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", "pytorch-lightning"], + [sys.executable, "-m", "pip", "install", "-q"] + _missing, check=True, ) From 9adf9a898c4d25fe52ad67aae95ac2060f70b49c Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:58:15 +0200 Subject: [PATCH 27/48] fix: self-install tensorboard when missing in mnist-vae --- notebooks/mnist-vae/run_training.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index 290ac12..afa0cdd 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -18,7 +18,7 @@ {**os.environ, "LD_LIBRARY_PATH": f"{_conda_lib}:{_ld}"}, ) -# pytorch-lightning and torchvision are not bundled in all notebook images +# pytorch-lightning, torchvision, and tensorboard are not bundled in all notebook images _missing = [] try: import pytorch_lightning # noqa: F401 @@ -28,6 +28,10 @@ import torchvision # noqa: F401 except ImportError: _missing.append("torchvision") +try: + import tensorboard # noqa: F401 +except ImportError: + _missing.append("tensorboard") if _missing: subprocess.run( [sys.executable, "-m", "pip", "install", "-q"] + _missing, From d65e2ccdd1c483e84f489db5057161e247e38f6f Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 20:11:16 +0200 Subject: [PATCH 28/48] feat: add CI coverage for hf-vllm-completion, kserve-keda, shadow-deployment Three previously untested examples now have apply.py + cleanup.py and are wired into ci/run_all.py: serving/hf-vllm-completion (always-on in Phase 1) - Deploys distilbert-inf-serv (HuggingFace CPU backend) - Smoke-tests via cluster-internal URL; no API key required serving/kserve-keda-autoscaling (opt-in: --include-keda) - Checks scaledobjects.keda.sh CRD; exits 0 with SKIP if absent - Deploys OPT-125M ISVC + ScaledObject, waits for Active=True - Smoke-tests via cluster-internal predictor Service serving/minimal-example-shadow-deployment (opt-in: --include-shadow) - Checks postgresclusters CRD; exits 0 with SKIP if absent - Deploys PostgresCluster, waits for primary, creates schema via one-shot psql pod, deploys doubler + tripler ISVCs - Smoke-tests primary via cluster-internal URL (factor=2 check) - Istio VirtualService mirroring is not tested in CI Also fixes the dead --include-keda flag (was parsed but never passed into run_all()) and adds --include-shadow on the same pattern. --- ci/run_all.py | 93 +++++- serving/hf-vllm-completion/apply.py | 126 ++++++++ serving/hf-vllm-completion/cleanup.py | 51 +++ serving/kserve-keda-autoscaling/apply.py | 185 +++++++++++ serving/kserve-keda-autoscaling/cleanup.py | 53 +++ .../apply.py | 303 ++++++++++++++++++ .../cleanup.py | 59 ++++ 7 files changed, 858 insertions(+), 12 deletions(-) create mode 100644 serving/hf-vllm-completion/apply.py create mode 100644 serving/hf-vllm-completion/cleanup.py create mode 100644 serving/kserve-keda-autoscaling/apply.py create mode 100644 serving/kserve-keda-autoscaling/cleanup.py create mode 100644 serving/minimal-example-shadow-deployment/apply.py create mode 100644 serving/minimal-example-shadow-deployment/cleanup.py diff --git a/ci/run_all.py b/ci/run_all.py index a170a6b..204a141 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -566,6 +566,7 @@ def run_all( timeout_pipeline: int = 3600, include_keda: bool = False, include_pytorch: bool = False, + include_shadow: bool = False, dry_run: bool = False, ) -> dict[str, Result]: root = _REPO_ROOT @@ -586,23 +587,27 @@ def run_all( for name in _MLFLOW_DEPENDENT: results[name] = Result(name=name, status="SKIP", error=mlflow_reason) - if dry_run: - print("[dry-run] Would execute the following phases:") - for label in [ - "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model", - "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", - "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", - "Phase 4: KFP run polling", - "Phase 5: cleanup", - ]: - print(f" {label}") - return results + if dry_run: + print("[dry-run] Would execute the following phases:") + for label in [ + "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", + "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", + "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", + "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", + "Phase 4: KFP run polling", + "Phase 5: cleanup", + ]: + print(f" {label}") + return results cleanup_scripts = [ root / "notebooks" / "dask" / "cleanup.py", root / "serving" / "minimal-s3-model" / "cleanup.py", root / "serving" / "mlflow-kserve-minimal" / "cleanup.py", root / "serving" / "mlflow-kserve-inference-protocols" / "cleanup.py", + root / "serving" / "hf-vllm-completion" / "cleanup.py", + root / "serving" / "kserve-keda-autoscaling" / "cleanup.py", + root / "serving" / "minimal-example-shadow-deployment" / "cleanup.py", root / "hparam-tuning" / "minimal-mnist" / "cleanup.py", ] @@ -633,6 +638,61 @@ def run_all( ) phase1_futures[name] = f + # hf-vllm-completion (CPU): self-contained, no KFP pipelines + phase1_futures["serving/hf-vllm-completion"] = _submit_script( + executor, + results, + "serving/hf-vllm-completion", + root / "serving" / "hf-vllm-completion" / "apply.py", + timeout_notebook, + ) + + # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs + # and exits 0 with a skip message if KEDA is not installed. + if include_keda: + phase1_futures["serving/kserve-keda-autoscaling"] = _submit_script( + executor, + results, + "serving/kserve-keda-autoscaling", + root / "serving" / "kserve-keda-autoscaling" / "apply.py", + timeout_notebook, + ) + else: + print( + " [SKIP ] serving/kserve-keda-autoscaling (pass --include-keda to enable)" + ) + results["serving/kserve-keda-autoscaling"] = Result( + name="serving/kserve-keda-autoscaling", + status="SKIP", + error="opt-in: pass --include-keda to run this example", + ) + + # minimal-example-shadow-deployment: opt-in; apply.py self-checks for + # the postgres-operator CRD and exits 0 if not found. Istio + # VirtualService mirroring is not verified in CI (requires domain config). + if include_shadow: + phase1_futures["serving/minimal-example-shadow-deployment"] = ( + _submit_script( + executor, + results, + "serving/minimal-example-shadow-deployment", + root + / "serving" + / "minimal-example-shadow-deployment" + / "apply.py", + timeout_notebook, + ) + ) + else: + print( + " [SKIP ] serving/minimal-example-shadow-deployment (pass --include-shadow to enable)" + ) + results["serving/minimal-example-shadow-deployment"] = Result( + name="serving/minimal-example-shadow-deployment", + status="SKIP", + error="opt-in: pass --include-shadow to run this example", + ) + # mnist-vae: opt-in (requires pytorch_lightning, not in all images) if include_pytorch: phase1_futures["notebooks/mnist-vae"] = _submit_chain( @@ -862,7 +922,15 @@ def run_all( parser.add_argument( "--include-keda", action="store_true", - help="Include GPU-dependent KEDA autoscaling example (opt-in)", + help="Include KEDA autoscaling example (opt-in; requires KEDA installed in the cluster)", + ) + parser.add_argument( + "--include-shadow", + action="store_true", + help=( + "Include minimal-example-shadow-deployment (opt-in; " + "requires CrunchyData postgres-operator; Istio VirtualService not tested)" + ), ) parser.add_argument( "--include-pytorch", @@ -878,6 +946,7 @@ def run_all( timeout_notebook=args.timeout_notebook, timeout_pipeline=args.timeout_pipeline, include_keda=args.include_keda, + include_shadow=args.include_shadow, include_pytorch=args.include_pytorch, dry_run=args.dry_run, ) diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py new file mode 100644 index 0000000..19b7c7c --- /dev/null +++ b/serving/hf-vllm-completion/apply.py @@ -0,0 +1,126 @@ +""" +Deploy the DistilBERT CPU InferenceService (HuggingFace backend) and smoke-test it. + +The model (distilbert-base-uncased-finetuned-sst-2-english, ~250 MB) is +downloaded from the HuggingFace Hub on first start, so the ISVC may take a +few minutes to become ready. + +Usage +----- + python apply.py +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + +_ISVC_NAME = "distilbert-inf-serv" +_MODEL_NAME = "distilbert" +_YAML = os.path.join(os.path.dirname(__file__), "inference-service-cpu.yaml") + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + print(result.stdout.strip()) + + +def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: + print( + f"Waiting for InferenceService '{name}' to become ready (timeout {timeout}s)..." + ) + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + # Dump ISVC status to help diagnose + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace], + check=False, + ) + raise RuntimeError( + f"InferenceService '{name}' did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + + +def _smoke_test(namespace: str, timeout: int = 120) -> None: + """POST to the internal cluster URL; retries until the model responds.""" + url = ( + f"http://{_ISVC_NAME}.{namespace}.svc.cluster.local" + f"/v1/models/{_MODEL_NAME}:predict" + ) + payload = json.dumps({"instances": ["KServe is wonderful!"]}).encode() + deadline = time.time() + timeout + last_err: Exception | None = None + while time.time() < deadline: + try: + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=15) as resp: + body = json.loads(resp.read()) + if "predictions" not in body: + raise RuntimeError(f"unexpected response body: {body}") + print(f"Smoke test passed: {body}") + return + except Exception as exc: + last_err = exc + time.sleep(5) + raise RuntimeError(f"Smoke test failed after {timeout}s: {last_err}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 900) -> None: + ns = _namespace() + + with open(_YAML) as fh: + manifest = fh.read() + + _kubectl_apply(manifest, ns) + print(f"Applied InferenceService '{_ISVC_NAME}' in namespace '{ns}'.") + + _wait_isvc_ready(_ISVC_NAME, ns, timeout) + print(f"InferenceService '{_ISVC_NAME}' is ready.") + + _smoke_test(ns) + + +if __name__ == "__main__": + deploy() diff --git a/serving/hf-vllm-completion/cleanup.py b/serving/hf-vllm-completion/cleanup.py new file mode 100644 index 0000000..a874d37 --- /dev/null +++ b/serving/hf-vllm-completion/cleanup.py @@ -0,0 +1,51 @@ +""" +Cleanup script for the hf-vllm-completion example. + +Deletes (idempotently): + - InferenceService ``distilbert-inf-serv`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete( + "inferenceservice", "distilbert-inf-serv", "-n", ns, dry_run=dry_run + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py new file mode 100644 index 0000000..8cca44c --- /dev/null +++ b/serving/kserve-keda-autoscaling/apply.py @@ -0,0 +1,185 @@ +""" +Deploy the KEDA autoscaling example (OPT-125M, vLLM backend, CPU) and verify +that both the InferenceService and the ScaledObject become active. + +KEDA CRD check +-------------- +If the ``scaledobjects.keda.sh`` CRD is absent the script exits with 0 and a +SKIP message rather than failing — the example is opt-in because KEDA is not +installed in every prokube cluster. + +Usage +----- + python apply.py +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + +_ISVC_NAME = "opt-125m" +_SCALED_OBJECT_NAME = "opt-125m-scaledobject" +_ISVC_YAML = os.path.join(os.path.dirname(__file__), "inference-service.yaml") +_SO_YAML = os.path.join(os.path.dirname(__file__), "scaled-object.yaml") + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + print(result.stdout.strip()) + + +def _keda_available() -> bool: + """Return True if KEDA CRDs are installed in this cluster.""" + result = subprocess.run( + ["kubectl", "get", "crd", "scaledobjects.keda.sh"], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: + print( + f"Waiting for InferenceService '{name}' to become ready (timeout {timeout}s)..." + ) + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace], + check=False, + ) + raise RuntimeError( + f"InferenceService '{name}' did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + + +def _wait_scaledobject_active(name: str, namespace: str, timeout: int = 120) -> None: + """Poll until KEDA reports the ScaledObject as Active=True.""" + print(f"Waiting for ScaledObject '{name}' to become active...") + deadline = time.time() + timeout + while time.time() < deadline: + r = subprocess.run( + [ + "kubectl", + "get", + "scaledobject", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Active')].status}", + ], + capture_output=True, + text=True, + ) + if r.stdout.strip() == "True": + print(f"ScaledObject '{name}' is active.") + return + time.sleep(10) + raise RuntimeError( + f"ScaledObject '{name}' did not become active within {timeout}s. " + "Check 'kubectl describe scaledobject' for trigger errors." + ) + + +def _smoke_test(namespace: str, timeout: int = 60) -> None: + """Send a single completion request via the cluster-internal predictor Service.""" + # RawDeployment mode exposes the predictor as a plain Kubernetes Service named + # -predictor — no external gateway or API key required. + url = ( + f"http://opt-125m-predictor.{namespace}.svc.cluster.local/openai/v1/completions" + ) + payload = json.dumps( + {"model": "opt-125m", "prompt": "KServe is", "max_tokens": 8} + ).encode() + deadline = time.time() + timeout + last_err: Exception | None = None + while time.time() < deadline: + try: + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=20) as resp: + body = json.loads(resp.read()) + if "choices" not in body: + raise RuntimeError(f"unexpected response body: {body}") + text = body["choices"][0].get("text", "") + print(f"Smoke test passed: {text!r}") + return + except Exception as exc: + last_err = exc + time.sleep(5) + raise RuntimeError(f"Smoke test failed after {timeout}s: {last_err}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 900) -> None: + if not _keda_available(): + print( + "SKIP: KEDA CRDs (scaledobjects.keda.sh) not found in this cluster.\n" + "Ask your cluster admin to install KEDA, or pass --include-keda only\n" + "when targeting a cluster with KEDA installed." + ) + sys.exit(0) + + ns = _namespace() + + with open(_ISVC_YAML) as fh: + isvc_manifest = fh.read() + _kubectl_apply(isvc_manifest, ns) + print(f"Applied InferenceService '{_ISVC_NAME}' in namespace '{ns}'.") + + _wait_isvc_ready(_ISVC_NAME, ns, timeout) + print(f"InferenceService '{_ISVC_NAME}' is ready.") + + with open(_SO_YAML) as fh: + so_manifest = fh.read() + _kubectl_apply(so_manifest, ns) + print(f"Applied ScaledObject '{_SCALED_OBJECT_NAME}' in namespace '{ns}'.") + + _wait_scaledobject_active(_SCALED_OBJECT_NAME, ns) + _smoke_test(ns) + + +if __name__ == "__main__": + deploy() diff --git a/serving/kserve-keda-autoscaling/cleanup.py b/serving/kserve-keda-autoscaling/cleanup.py new file mode 100644 index 0000000..d0c5946 --- /dev/null +++ b/serving/kserve-keda-autoscaling/cleanup.py @@ -0,0 +1,53 @@ +""" +Cleanup script for the kserve-keda-autoscaling example. + +Deletes (idempotently, ScaledObject first to stop KEDA activity): + - ScaledObject ``opt-125m-scaledobject`` + - InferenceService ``opt-125m`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + # Delete the ScaledObject before the ISVC so KEDA stops managing the HPA + # before the underlying Deployment is gone. + _kubectl_delete("scaledobject", "opt-125m-scaledobject", "-n", ns, dry_run=dry_run) + _kubectl_delete("inferenceservice", "opt-125m", "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py new file mode 100644 index 0000000..272b932 --- /dev/null +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -0,0 +1,303 @@ +""" +Deploy the shadow deployment example and smoke-test the primary InferenceService. + +What this script does +--------------------- +1. Checks for the CrunchyData postgres-operator CRD — exits 0 with a SKIP + message if not found (it is installed by default in prokube clusters). +2. Deploys the PostgresCluster (postgres-cluster.yaml) and waits for the + primary pod to become ready. +3. Creates the two tables the transformer needs (inference_requests and + inference_response) via a short-lived psql pod. +4. Deploys the primary (doubler) and shadow (tripler) InferenceServices and + waits for both to become ready. +5. Sends a smoke-test request to the primary via its cluster-internal URL + and verifies the response contains predictions. + +What this script does NOT do +----------------------------- +- Apply the Istio VirtualService manifests — those embed namespace and domain + name and must be configured manually for each environment. +- Verify the shadow mirroring behaviour (that requires traffic via the + VirtualService, which is out of scope for automated CI). + +Usage +----- + python apply.py +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + +_ROOT = os.path.dirname(__file__) + +_PG_CLUSTER_NAME = "inferencing-postgres" +_PG_SECRET_NAME = "inferencing-postgres-pguser-transformer-admin" +_PG_HOST_TEMPLATE = "inferencing-postgres-primary.{ns}.svc" +_PG_USER = "transformer-admin" +_PG_DB = "scale-inference" + +_DOUBLER_ISVC = "double-minimal-custom-inference" +_TRIPLER_ISVC = "triple-minimal-custom-inference" + +_SCHEMA_SQL = """\ +CREATE TABLE IF NOT EXISTS public.inference_requests ( + request_id uuid NOT NULL, + request_time timestamp with time zone NULL, + request_data json NULL, + predict_url text NULL, + created_at timestamp NULL, + PRIMARY KEY (request_id) +); +CREATE TABLE IF NOT EXISTS public.inference_response ( + request_id uuid NOT NULL, + request_data json NULL, + created_at timestamp NULL, + PRIMARY KEY (request_id) +); +""" + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + print(result.stdout.strip()) + + +def _crd_available(crd_name: str) -> bool: + result = subprocess.run( + ["kubectl", "get", "crd", crd_name], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def _wait_for_secret(name: str, namespace: str, timeout: int = 300) -> None: + """Block until the CrunchyData operator creates the user secret.""" + print( + f"Waiting for secret '{name}' to appear (postgres-operator creates it after cluster init)..." + ) + deadline = time.time() + timeout + while time.time() < deadline: + r = subprocess.run( + ["kubectl", "get", "secret", name, "-n", namespace], + capture_output=True, + text=True, + ) + if r.returncode == 0: + return + time.sleep(10) + raise RuntimeError( + f"Secret '{name}' did not appear within {timeout}s. " + "Check the PostgresCluster status." + ) + + +def _wait_pg_primary_ready(namespace: str, timeout: int = 300) -> None: + """Wait for the postgres primary pod to be ready.""" + print("Waiting for PostgresCluster primary pod to be ready...") + result = subprocess.run( + [ + "kubectl", + "wait", + "pod", + "-l", + f"postgres-operator.crunchydata.com/cluster={_PG_CLUSTER_NAME}," + "postgres-operator.crunchydata.com/role=master", + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"PostgresCluster primary did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + print("PostgresCluster primary is ready.") + + +def _get_secret_value(name: str, key: str, namespace: str) -> str: + result = subprocess.run( + [ + "kubectl", + "get", + "secret", + name, + "-n", + namespace, + "-o", + f"jsonpath={{.data.{key}}}", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise RuntimeError( + f"Could not read key '{key}' from secret '{name}': {result.stderr}" + ) + return base64.b64decode(result.stdout.strip()).decode() + + +def _create_schema(namespace: str, password: str) -> None: + """Run a one-shot psql pod to create the required tables.""" + print("Creating database schema via temporary psql pod...") + host = _PG_HOST_TEMPLATE.format(ns=namespace) + result = subprocess.run( + [ + "kubectl", + "run", + "pg-schema-init", + "--rm", + "-i", + "--restart=Never", + f"-n={namespace}", + "--image=postgres:17", + f"--env=PGPASSWORD={password}", + "--", + "psql", + f"-h={host}", + f"-U={_PG_USER}", + f"-d={_PG_DB}", + ], + input=_SCHEMA_SQL, + text=True, + capture_output=True, + timeout=120, + ) + if result.returncode != 0: + raise RuntimeError(f"Schema creation failed:\n{result.stderr or result.stdout}") + print("Schema created (or already existed).") + + +def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: + print(f"Waiting for InferenceService '{name}' (timeout {timeout}s)...") + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace], + check=False, + ) + raise RuntimeError( + f"InferenceService '{name}' did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + + +def _smoke_test(namespace: str, timeout: int = 120) -> None: + """POST numeric values to the primary (doubler) ISVC and verify predictions.""" + url = ( + f"http://{_DOUBLER_ISVC}.{namespace}.svc.cluster.local/v1/models/model:predict" + ) + payload = json.dumps({"values": [1.0, 2.0, 3.0]}).encode() + deadline = time.time() + timeout + last_err: Exception | None = None + while time.time() < deadline: + try: + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=15) as resp: + body = json.loads(resp.read()) + if "predictions" not in body: + raise RuntimeError(f"unexpected response body: {body}") + print(f"Smoke test passed: {body}") + return + except Exception as exc: + last_err = exc + time.sleep(5) + raise RuntimeError(f"Smoke test failed after {timeout}s: {last_err}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 600) -> None: + if not _crd_available("postgresclusters.postgres-operator.crunchydata.com"): + print( + "SKIP: CrunchyData postgres-operator CRD not found in this cluster.\n" + "The shadow deployment example requires the postgres-operator to be installed." + ) + sys.exit(0) + + ns = _namespace() + + # 1. Postgres cluster + with open(os.path.join(_ROOT, "postgres-cluster.yaml")) as fh: + pg_manifest = fh.read() + _kubectl_apply(pg_manifest, ns) + print(f"Applied PostgresCluster '{_PG_CLUSTER_NAME}' in namespace '{ns}'.") + + _wait_pg_primary_ready(ns, timeout=300) + _wait_for_secret(_PG_SECRET_NAME, ns, timeout=120) + + password = _get_secret_value(_PG_SECRET_NAME, "password", ns) + _create_schema(ns, password) + + # 2. Both InferenceServices + for yaml_file in ( + "doubler-inference-service.yaml", + "tripler-inference-service.yaml", + ): + with open(os.path.join(_ROOT, yaml_file)) as fh: + manifest = fh.read() + _kubectl_apply(manifest, ns) + + print("Applied doubler and tripler InferenceServices.") + + for isvc_name in (_DOUBLER_ISVC, _TRIPLER_ISVC): + _wait_isvc_ready(isvc_name, ns, timeout) + + print("Both InferenceServices are ready.") + print( + "NOTE: Istio VirtualService manifests (istio/) are not applied in CI — " + "they require namespace and domain substitution and must be configured manually." + ) + + _smoke_test(ns) + + +if __name__ == "__main__": + deploy() diff --git a/serving/minimal-example-shadow-deployment/cleanup.py b/serving/minimal-example-shadow-deployment/cleanup.py new file mode 100644 index 0000000..eafd42b --- /dev/null +++ b/serving/minimal-example-shadow-deployment/cleanup.py @@ -0,0 +1,59 @@ +""" +Cleanup script for the minimal-example-shadow-deployment example. + +Deletes (idempotently): + - InferenceService ``double-minimal-custom-inference`` + - InferenceService ``triple-minimal-custom-inference`` + - PostgresCluster ``inferencing-postgres`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete( + "inferenceservice", "double-minimal-custom-inference", "-n", ns, dry_run=dry_run + ) + _kubectl_delete( + "inferenceservice", "triple-minimal-custom-inference", "-n", ns, dry_run=dry_run + ) + _kubectl_delete( + "postgrescluster", "inferencing-postgres", "-n", ns, dry_run=dry_run + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) From d9a27fe832f3c742fc2e4b6c4938f8c80562d98c Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 10:24:46 +0200 Subject: [PATCH 29/48] fix: detect postgres-operator via apply instead of CRD get The previous approach checked 'kubectl get crd postgresclusters...' which requires cluster-level RBAC that notebook SAs typically don't have, causing the check to fail and the example to silently skip regardless of whether the operator is installed. Instead, attempt to apply postgres-cluster.yaml and treat a 'no matches for kind' / 'server doesn't have a resource type' error as a graceful skip. Any other error (permissions, connectivity) is raised so CI marks it as FAIL rather than a false PASS. Also strengthen the smoke test to verify the doubler's actual output (FACTOR=2: [1,2,3] must produce [2,4,6]) rather than just checking that a predictions key exists. --- .../apply.py | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index 272b932..d7089ea 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -86,13 +86,30 @@ def _kubectl_apply(manifest: str, namespace: str) -> None: print(result.stdout.strip()) -def _crd_available(crd_name: str) -> bool: +def _try_apply(manifest: str, namespace: str) -> str | None: + """Apply a manifest; return the error string if the resource type is unknown. + + Returns None on success, or a non-empty string if kubectl rejected the + manifest because the CRD does not exist in this cluster. Any other error + (permissions, connectivity, …) is raised as RuntimeError so CI marks the + example as FAIL rather than silently skipping it. + """ result = subprocess.run( - ["kubectl", "get", "crd", crd_name], - capture_output=True, + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, text=True, + capture_output=True, ) - return result.returncode == 0 + if result.returncode == 0: + print(result.stdout.strip()) + return None + stderr = result.stderr or result.stdout + if ( + "no matches for kind" in stderr + or "the server doesn't have a resource type" in stderr + ): + return stderr.strip() + raise RuntimeError(stderr) def _wait_for_secret(name: str, namespace: str, timeout: int = 300) -> None: @@ -224,11 +241,17 @@ def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: def _smoke_test(namespace: str, timeout: int = 120) -> None: - """POST numeric values to the primary (doubler) ISVC and verify predictions.""" + """POST numeric values to the primary (doubler) ISVC and verify predictions. + + The doubler predictor multiplies each input value by FACTOR=2, so + [1.0, 2.0, 3.0] must produce predictions [2.0, 4.0, 6.0]. + """ url = ( f"http://{_DOUBLER_ISVC}.{namespace}.svc.cluster.local/v1/models/model:predict" ) - payload = json.dumps({"values": [1.0, 2.0, 3.0]}).encode() + inputs = [1.0, 2.0, 3.0] + expected = [2.0, 4.0, 6.0] + payload = json.dumps({"values": inputs}).encode() deadline = time.time() + timeout last_err: Exception | None = None while time.time() < deadline: @@ -241,9 +264,15 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: ) with urllib.request.urlopen(req, timeout=15) as resp: body = json.loads(resp.read()) - if "predictions" not in body: - raise RuntimeError(f"unexpected response body: {body}") - print(f"Smoke test passed: {body}") + predictions = body.get("predictions") + if predictions is None: + raise RuntimeError(f"no 'predictions' key in response: {body}") + if predictions != expected: + raise RuntimeError( + f"doubler (FACTOR=2) returned wrong predictions: " + f"got {predictions}, expected {expected}" + ) + print(f"Smoke test passed: {inputs} → {predictions} (FACTOR=2 verified)") return except Exception as exc: last_err = exc @@ -255,19 +284,19 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: def deploy(timeout: int = 600) -> None: - if not _crd_available("postgresclusters.postgres-operator.crunchydata.com"): - print( - "SKIP: CrunchyData postgres-operator CRD not found in this cluster.\n" - "The shadow deployment example requires the postgres-operator to be installed." - ) - sys.exit(0) - ns = _namespace() - # 1. Postgres cluster + # 1. Postgres cluster — detect operator availability via the apply itself with open(os.path.join(_ROOT, "postgres-cluster.yaml")) as fh: pg_manifest = fh.read() - _kubectl_apply(pg_manifest, ns) + skip_reason = _try_apply(pg_manifest, ns) + if skip_reason: + print( + f"SKIP: postgres-operator is not installed in this cluster " + f"(kubectl apply returned: {skip_reason}).\n" + "The shadow deployment example requires the CrunchyData postgres-operator." + ) + sys.exit(0) print(f"Applied PostgresCluster '{_PG_CLUSTER_NAME}' in namespace '{ns}'.") _wait_pg_primary_ready(ns, timeout=300) From 6fcfac666aa4bd3d9a4e84f3c743c2b5c5033662 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 10:25:14 +0200 Subject: [PATCH 30/48] fix: detect KEDA via ScaledObject apply instead of CRD get Same RBAC issue as the shadow-deployment fix: notebook SAs lack cluster-level permission to 'kubectl get crd', so the KEDA check always returned False and the example silently skipped. Deploy the ISVC first (always), then attempt to apply the ScaledObject and treat 'no matches for kind' as a graceful skip. This also gives more useful CI signal: the ISVC smoke-test still runs even when KEDA is absent. --- serving/kserve-keda-autoscaling/apply.py | 44 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py index 8cca44c..ee0518a 100644 --- a/serving/kserve-keda-autoscaling/apply.py +++ b/serving/kserve-keda-autoscaling/apply.py @@ -49,14 +49,29 @@ def _kubectl_apply(manifest: str, namespace: str) -> None: print(result.stdout.strip()) -def _keda_available() -> bool: - """Return True if KEDA CRDs are installed in this cluster.""" +def _try_apply_scaledobject(manifest: str, namespace: str) -> str | None: + """Apply the ScaledObject manifest; return an error string if KEDA is absent. + + Returns None on success, or a non-empty string if kubectl rejected the + manifest because the ScaledObject CRD is not installed. Any other error + is raised so CI marks the example as FAIL instead of silently skipping. + """ result = subprocess.run( - ["kubectl", "get", "crd", "scaledobjects.keda.sh"], - capture_output=True, + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, text=True, + capture_output=True, ) - return result.returncode == 0 + if result.returncode == 0: + print(result.stdout.strip()) + return None + stderr = result.stderr or result.stdout + if ( + "no matches for kind" in stderr + or "the server doesn't have a resource type" in stderr + ): + return stderr.strip() + raise RuntimeError(stderr) def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: @@ -154,14 +169,6 @@ def _smoke_test(namespace: str, timeout: int = 60) -> None: def deploy(timeout: int = 900) -> None: - if not _keda_available(): - print( - "SKIP: KEDA CRDs (scaledobjects.keda.sh) not found in this cluster.\n" - "Ask your cluster admin to install KEDA, or pass --include-keda only\n" - "when targeting a cluster with KEDA installed." - ) - sys.exit(0) - ns = _namespace() with open(_ISVC_YAML) as fh: @@ -172,9 +179,18 @@ def deploy(timeout: int = 900) -> None: _wait_isvc_ready(_ISVC_NAME, ns, timeout) print(f"InferenceService '{_ISVC_NAME}' is ready.") + # Apply ScaledObject — detect KEDA availability via the apply itself rather + # than 'kubectl get crd' which requires cluster-level RBAC notebook SAs lack. with open(_SO_YAML) as fh: so_manifest = fh.read() - _kubectl_apply(so_manifest, ns) + skip_reason = _try_apply_scaledobject(so_manifest, ns) + if skip_reason: + print( + f"SKIP: KEDA is not installed in this cluster " + f"(kubectl apply returned: {skip_reason}).\n" + "The ScaledObject was not created; the ISVC is running without autoscaling." + ) + sys.exit(0) print(f"Applied ScaledObject '{_SCALED_OBJECT_NAME}' in namespace '{ns}'.") _wait_scaledobject_active(_SCALED_OBJECT_NAME, ns) From e7665fec17bcb0e90ac2316b7f12089329fac00b Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 12:34:53 +0200 Subject: [PATCH 31/48] fix: surface actionable error messages for postgres-operator failures Replace the raw kubectl stderr blob with targeted messages: - 'no matches for kind': operator not installed - Forbidden on postgres-operator resources: SA lacks RBAC, with the exact grant needed (get/create/update/patch on postgresclusters) The example still FAILs in CI (correct) but the error line now tells the operator admin exactly what to fix. --- .../apply.py | 47 +++++++------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index d7089ea..ac70806 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -32,7 +32,6 @@ import json import os import subprocess -import sys import time import urllib.error import urllib.request @@ -75,25 +74,6 @@ def _namespace() -> str: def _kubectl_apply(manifest: str, namespace: str) -> None: - result = subprocess.run( - ["kubectl", "apply", "-f", "-", "-n", namespace], - input=manifest, - text=True, - capture_output=True, - ) - if result.returncode != 0: - raise RuntimeError(result.stderr or result.stdout) - print(result.stdout.strip()) - - -def _try_apply(manifest: str, namespace: str) -> str | None: - """Apply a manifest; return the error string if the resource type is unknown. - - Returns None on success, or a non-empty string if kubectl rejected the - manifest because the CRD does not exist in this cluster. Any other error - (permissions, connectivity, …) is raised as RuntimeError so CI marks the - example as FAIL rather than silently skipping it. - """ result = subprocess.run( ["kubectl", "apply", "-f", "-", "-n", namespace], input=manifest, @@ -102,13 +82,24 @@ def _try_apply(manifest: str, namespace: str) -> str | None: ) if result.returncode == 0: print(result.stdout.strip()) - return None + return stderr = result.stderr or result.stdout + # Translate common failure modes into actionable messages if ( "no matches for kind" in stderr or "the server doesn't have a resource type" in stderr ): - return stderr.strip() + raise RuntimeError( + "postgres-operator is not installed in this cluster. " + "The shadow deployment example requires the CrunchyData postgres-operator." + ) + if "Forbidden" in stderr and "postgres-operator.crunchydata.com" in stderr: + raise RuntimeError( + "The notebook ServiceAccount lacks RBAC permission to manage PostgresCluster " + "resources.\n" + "Ask your cluster admin to grant get/create/update/patch on " + "postgresclusters.postgres-operator.crunchydata.com in this namespace." + ) raise RuntimeError(stderr) @@ -286,17 +277,11 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: def deploy(timeout: int = 600) -> None: ns = _namespace() - # 1. Postgres cluster — detect operator availability via the apply itself + # 1. Postgres cluster — _kubectl_apply raises with an actionable message on + # "no matches for kind" (operator not installed) or Forbidden (RBAC missing). with open(os.path.join(_ROOT, "postgres-cluster.yaml")) as fh: pg_manifest = fh.read() - skip_reason = _try_apply(pg_manifest, ns) - if skip_reason: - print( - f"SKIP: postgres-operator is not installed in this cluster " - f"(kubectl apply returned: {skip_reason}).\n" - "The shadow deployment example requires the CrunchyData postgres-operator." - ) - sys.exit(0) + _kubectl_apply(pg_manifest, ns) print(f"Applied PostgresCluster '{_PG_CLUSTER_NAME}' in namespace '{ns}'.") _wait_pg_primary_ready(ns, timeout=300) From dcdfb3ee52abddd845ecc26dc5c7d8409fb29819 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:03:20 +0200 Subject: [PATCH 32/48] fix: suppress traceback in apply.py scripts, print clean error message Unhandled exceptions print a full Python traceback to stderr which the CI captures verbatim. Catch at __main__ level and print just str(exc) so CI shows the actionable message without the stack noise. --- serving/hf-vllm-completion/apply.py | 6 +++++- serving/kserve-keda-autoscaling/apply.py | 6 +++++- serving/minimal-example-shadow-deployment/apply.py | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py index 19b7c7c..8b13032 100644 --- a/serving/hf-vllm-completion/apply.py +++ b/serving/hf-vllm-completion/apply.py @@ -123,4 +123,8 @@ def deploy(timeout: int = 900) -> None: if __name__ == "__main__": - deploy() + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py index ee0518a..5f928f0 100644 --- a/serving/kserve-keda-autoscaling/apply.py +++ b/serving/kserve-keda-autoscaling/apply.py @@ -198,4 +198,8 @@ def deploy(timeout: int = 900) -> None: if __name__ == "__main__": - deploy() + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index ac70806..1bbe915 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -32,6 +32,7 @@ import json import os import subprocess +import sys import time import urllib.error import urllib.request @@ -314,4 +315,8 @@ def deploy(timeout: int = 600) -> None: if __name__ == "__main__": - deploy() + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) From 69ef93b80eb72c5cb1ab21660ec788c4b9cbbd6a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:10:02 +0200 Subject: [PATCH 33/48] refactor: break run_all() into phase helpers and add _timed_run wrapper --- ci/run_all.py | 767 +++++++++++++++++++++++++++----------------------- 1 file changed, 413 insertions(+), 354 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 204a141..35cd151 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -53,6 +53,7 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path +from typing import Callable # ── Repo root ───────────────────────────────────────────────────────────────── @@ -320,15 +321,17 @@ def _run_cleanup(cleanup_path: Path) -> None: # ── Executor ────────────────────────────────────────────────────────────────── -def _submit_notebook( +def _timed_run( executor: ThreadPoolExecutor, results: dict[str, Result], name: str, - nb_path: Path, - output_dir: Path, - timeout: int, - extract_run_ids: bool = True, + work: Callable[[Result], None], ) -> Future: + """Submit `work(result)` to the executor with shared timing + error handling. + + `work` mutates the Result (e.g. sets kfp_run_ids); status/duration are set + here. The first exception is recorded as FAIL. + """ result = Result(name=name) results[name] = result print(f" [START ] {name}") @@ -336,10 +339,8 @@ def _submit_notebook( def _run(): t0 = time.time() try: - out = _run_notebook(nb_path, output_dir, timeout) + work(result) result.status = "PASS" - if extract_run_ids: - result.kfp_run_ids = _extract_run_ids_from_notebook(out) except Exception as exc: # noqa: BLE001 result.status = "FAIL" result.error = str(exc) @@ -349,6 +350,23 @@ def _run(): return executor.submit(_run) +def _submit_notebook( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + nb_path: Path, + output_dir: Path, + timeout: int, + extract_run_ids: bool = True, +) -> Future: + def work(result: Result) -> None: + out = _run_notebook(nb_path, output_dir, timeout) + if extract_run_ids: + result.kfp_run_ids = _extract_run_ids_from_notebook(out) + + return _timed_run(executor, results, name, work) + + def _submit_script( executor: ThreadPoolExecutor, results: dict[str, Result], @@ -357,23 +375,11 @@ def _submit_script( timeout: int, extra_args: list[str] | None = None, ) -> Future: - result = Result(name=name) - results[name] = result - print(f" [START ] {name}") + def work(result: Result) -> None: + stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) + result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) - def _run(): - t0 = time.time() - try: - stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) - result.status = "PASS" - result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) - except Exception as exc: # noqa: BLE001 - result.status = "FAIL" - result.error = str(exc) - finally: - result.duration = time.time() - t0 - - return executor.submit(_run) + return _timed_run(executor, results, name, work) def _submit_chain( @@ -389,38 +395,27 @@ def _submit_chain( kind is 'notebook' or 'script'. kwargs may include 'extra_args' for scripts. All steps run in one thread; the first failure stops the chain. """ - result = Result(name=name) - results[name] = result - print(f" [START ] {name}") - def _run(): - t0 = time.time() - try: - for kind, path, kwargs in steps: - if kind == "notebook": - out = _run_notebook(path, output_dir, timeout) - result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) - elif kind == "script": - stdout, _ = _run_script( - path, timeout, extra_args=kwargs.get("extra_args") - ) - result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) - elif kind == "pip": - r = subprocess.run( - [sys.executable, "-m", "pip", "install", "-q"] - + kwargs.get("packages", []), - capture_output=True, - text=True, - check=True, - ) - result.status = "PASS" - except Exception as exc: # noqa: BLE001 - result.status = "FAIL" - result.error = str(exc) - finally: - result.duration = time.time() - t0 + def work(result: Result) -> None: + for kind, path, kwargs in steps: + if kind == "notebook": + out = _run_notebook(path, output_dir, timeout) + result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) + elif kind == "script": + stdout, _ = _run_script( + path, timeout, extra_args=kwargs.get("extra_args") + ) + result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + elif kind == "pip": + subprocess.run( + [sys.executable, "-m", "pip", "install", "-q"] + + kwargs.get("packages", []), + capture_output=True, + text=True, + check=True, + ) - return executor.submit(_run) + return _timed_run(executor, results, name, work) # ── Report ──────────────────────────────────────────────────────────────────── @@ -558,26 +553,67 @@ def _check_mlflow_credentials() -> tuple[bool, str]: return False, f"Could not reach MLflow at {uri}: {exc}" -# ── Main ────────────────────────────────────────────────────────────────────── +# ── Orchestration context ───────────────────────────────────────────────────── -def run_all( - timeout_notebook: int = 1800, - timeout_pipeline: int = 3600, - include_keda: bool = False, - include_pytorch: bool = False, - include_shadow: bool = False, - dry_run: bool = False, -) -> dict[str, Result]: - root = _REPO_ROOT - output_dir = root / "ci" / "output" - results: dict[str, Result] = {} - poll_results: dict[str, str] = {} - poll_errors: dict[str, str] = {} +@dataclass +class _Context: + """Shared mutable state threaded through every phase.""" + + executor: ThreadPoolExecutor + root: Path + output_dir: Path + timeout_notebook: int + timeout_pipeline: int + results: dict[str, Result] + poll_results: dict[str, str] + poll_errors: dict[str, str] + + +_CLEANUP_RELPATHS = [ + "notebooks/dask/cleanup.py", + "serving/minimal-s3-model/cleanup.py", + "serving/mlflow-kserve-minimal/cleanup.py", + "serving/mlflow-kserve-inference-protocols/cleanup.py", + "serving/hf-vllm-completion/cleanup.py", + "serving/kserve-keda-autoscaling/cleanup.py", + "serving/minimal-example-shadow-deployment/cleanup.py", + "hparam-tuning/minimal-mnist/cleanup.py", +] + +_DRY_RUN_PHASES = [ + "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", + "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", + "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", + "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", + "Phase 4: KFP run polling", + "Phase 5: cleanup", +] + + +def _drain(ctx: _Context, futures: dict[str, Future]) -> None: + """Wait on a {name: Future} map, printing each result as it completes.""" + by_future = {v: k for k, v in futures.items()} + for f in as_completed(futures.values()): + _print_result(ctx.results[by_future[f]]) + + +def _skip(ctx: _Context, name: str, reason: str, label: str | None = None) -> None: + """Record a SKIP result and print a one-line notice.""" + print(f" [SKIP ] {label or name}") + ctx.results[name] = Result(name=name, status="SKIP", error=reason) + + +# ── Pre-flight ──────────────────────────────────────────────────────────────── - _ensure_papermill() - # Pre-flight: validate MLflow credentials; skip dependent tests if unavailable +def _preflight(results: dict[str, Result]) -> tuple[bool, str]: + """Ensure papermill, then validate MLflow credentials. + + Returns (mlflow_ok, reason). On failure, MLflow-dependent examples are + recorded as SKIP in `results`. + """ + _ensure_papermill() print("Pre-flight: checking MLflow credentials...") mlflow_ok, mlflow_reason = _check_mlflow_credentials() if mlflow_ok: @@ -586,311 +622,334 @@ def run_all( print(f" [SKIP] {mlflow_reason}") for name in _MLFLOW_DEPENDENT: results[name] = Result(name=name, status="SKIP", error=mlflow_reason) + return mlflow_ok, mlflow_reason - if dry_run: - print("[dry-run] Would execute the following phases:") - for label in [ - "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", - "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", - "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", - "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", - "Phase 4: KFP run polling", - "Phase 5: cleanup", - ]: - print(f" {label}") - return results - - cleanup_scripts = [ - root / "notebooks" / "dask" / "cleanup.py", - root / "serving" / "minimal-s3-model" / "cleanup.py", - root / "serving" / "mlflow-kserve-minimal" / "cleanup.py", - root / "serving" / "mlflow-kserve-inference-protocols" / "cleanup.py", - root / "serving" / "hf-vllm-completion" / "cleanup.py", - root / "serving" / "kserve-keda-autoscaling" / "cleanup.py", - root / "serving" / "minimal-example-shadow-deployment" / "cleanup.py", - root / "hparam-tuning" / "minimal-mnist" / "cleanup.py", - ] - try: - with ThreadPoolExecutor(max_workers=8) as executor: - # ── Phase 1: independent notebooks ─────────────────────────────── - print("Phase 1: running independent notebooks in parallel...") - phase1_futures = {} - for name, rel in [ - ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), - ( - "notebooks/mobile-price-classification", - "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", - ), - ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), - ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), - ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), - ( - "serving/minimal-s3-model", - "serving/minimal-s3-model/minimal-s3-model.ipynb", - ), - ]: - if name in results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") - continue - f = _submit_notebook( - executor, results, name, root / rel, output_dir, timeout_notebook - ) - phase1_futures[name] = f - - # hf-vllm-completion (CPU): self-contained, no KFP pipelines - phase1_futures["serving/hf-vllm-completion"] = _submit_script( - executor, - results, - "serving/hf-vllm-completion", - root / "serving" / "hf-vllm-completion" / "apply.py", - timeout_notebook, - ) +# ── Phases ────────────────────────────────────────────────────────────────── - # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs - # and exits 0 with a skip message if KEDA is not installed. - if include_keda: - phase1_futures["serving/kserve-keda-autoscaling"] = _submit_script( - executor, - results, - "serving/kserve-keda-autoscaling", - root / "serving" / "kserve-keda-autoscaling" / "apply.py", - timeout_notebook, - ) - else: - print( - " [SKIP ] serving/kserve-keda-autoscaling (pass --include-keda to enable)" - ) - results["serving/kserve-keda-autoscaling"] = Result( - name="serving/kserve-keda-autoscaling", - status="SKIP", - error="opt-in: pass --include-keda to run this example", - ) - # minimal-example-shadow-deployment: opt-in; apply.py self-checks for - # the postgres-operator CRD and exits 0 if not found. Istio - # VirtualService mirroring is not verified in CI (requires domain config). - if include_shadow: - phase1_futures["serving/minimal-example-shadow-deployment"] = ( - _submit_script( - executor, - results, - "serving/minimal-example-shadow-deployment", - root - / "serving" - / "minimal-example-shadow-deployment" - / "apply.py", - timeout_notebook, - ) - ) - else: - print( - " [SKIP ] serving/minimal-example-shadow-deployment (pass --include-shadow to enable)" - ) - results["serving/minimal-example-shadow-deployment"] = Result( - name="serving/minimal-example-shadow-deployment", - status="SKIP", - error="opt-in: pass --include-shadow to run this example", - ) +def _phase1( + ctx: _Context, + include_keda: bool, + include_shadow: bool, + include_pytorch: bool, +) -> None: + """Phase 1: independent notebooks + self-contained serving examples.""" + print("Phase 1: running independent notebooks in parallel...") + futures: dict[str, Future] = {} + for name, rel in [ + ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), + ( + "notebooks/mobile-price-classification", + "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", + ), + ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), + ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), + ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), + ("serving/minimal-s3-model", "serving/minimal-s3-model/minimal-s3-model.ipynb"), + ]: + if name in ctx.results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue + futures[name] = _submit_notebook( + ctx.executor, + ctx.results, + name, + ctx.root / rel, + ctx.output_dir, + ctx.timeout_notebook, + ) - # mnist-vae: opt-in (requires pytorch_lightning, not in all images) - if include_pytorch: - phase1_futures["notebooks/mnist-vae"] = _submit_chain( - executor, - results, - "notebooks/mnist-vae", - steps=[ - # run_training.py self-installs pytorch-lightning if absent - ( - "script", - root / "notebooks/mnist-vae/run_training.py", - {"extra_args": ["--max_epochs", "3"]}, - ), - ( - "notebook", - root / "notebooks/mnist-vae/visualizations.ipynb", - {}, - ), - ], - output_dir=output_dir, - timeout=timeout_notebook, - ) - else: - print( - " [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)" - ) + # hf-vllm-completion (CPU): self-contained, no KFP pipelines + futures["serving/hf-vllm-completion"] = _submit_script( + ctx.executor, + ctx.results, + "serving/hf-vllm-completion", + ctx.root / "serving" / "hf-vllm-completion" / "apply.py", + ctx.timeout_notebook, + ) - # wait for all phase 1, printing results as each finishes - _future_to_name = {v: k for k, v in phase1_futures.items()} - for f in as_completed(phase1_futures.values()): - _print_result(results[_future_to_name[f]]) + # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs + # and exits 0 with a skip message if KEDA is not installed. + if include_keda: + futures["serving/kserve-keda-autoscaling"] = _submit_script( + ctx.executor, + ctx.results, + "serving/kserve-keda-autoscaling", + ctx.root / "serving" / "kserve-keda-autoscaling" / "apply.py", + ctx.timeout_notebook, + ) + else: + _skip( + ctx, + "serving/kserve-keda-autoscaling", + "opt-in: pass --include-keda to run this example", + label="serving/kserve-keda-autoscaling (pass --include-keda to enable)", + ) - # ── Phase 2: pipeline submissions (return fast) ─────────────────── - print("\nPhase 2: submitting pipelines...") - phase2_futures = {} - for name, rel in [ - ( - "mlflow/mobile-price-classification", - "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", - ), + # minimal-example-shadow-deployment: opt-in; apply.py self-checks for + # the postgres-operator CRD and exits 0 if not found. Istio + # VirtualService mirroring is not verified in CI (requires domain config). + if include_shadow: + futures["serving/minimal-example-shadow-deployment"] = _submit_script( + ctx.executor, + ctx.results, + "serving/minimal-example-shadow-deployment", + ctx.root / "serving" / "minimal-example-shadow-deployment" / "apply.py", + ctx.timeout_notebook, + ) + else: + _skip( + ctx, + "serving/minimal-example-shadow-deployment", + "opt-in: pass --include-shadow to run this example", + label="serving/minimal-example-shadow-deployment (pass --include-shadow to enable)", + ) + + # mnist-vae: opt-in (requires pytorch_lightning, not in all images) + if include_pytorch: + futures["notebooks/mnist-vae"] = _submit_chain( + ctx.executor, + ctx.results, + "notebooks/mnist-vae", + steps=[ + # run_training.py self-installs pytorch-lightning if absent ( - "pipelines/lightweight-components", - "pipelines/lightweight-components/mobile-price-classifications.ipynb", + "script", + ctx.root / "notebooks/mnist-vae/run_training.py", + {"extra_args": ["--max_epochs", "3"]}, ), - ]: - if name in results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") - continue - f = _submit_notebook( - executor, results, name, root / rel, output_dir, timeout_notebook - ) - phase2_futures[name] = f - - lpp_future = _submit_script( - executor, - results, - "pipelines/lightweight-python-package", - root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", - timeout_notebook, - ) - phase2_futures["pipelines/lightweight-python-package"] = lpp_future - - mcc_future = _submit_script( - executor, - results, - "pipelines/minimal-container-components", - root - / "pipelines" - / "minimal-container-components" - / "submit-cluster.py", - timeout_notebook, - ) - phase2_futures["pipelines/minimal-container-components"] = mcc_future + ("notebook", ctx.root / "notebooks/mnist-vae/visualizations.ipynb", {}), + ], + output_dir=ctx.output_dir, + timeout=ctx.timeout_notebook, + ) + else: + print(" [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)") + + _drain(ctx, futures) - # Wait for mlflow-mobile-price notebook; then poll its KFP run - # inline before Phase 3 — the ISVC needs the registered model. - _mobile_price_skipped = ( - "mlflow/mobile-price-classification" not in phase2_futures - ) - if not _mobile_price_skipped: - phase2_futures["mlflow/mobile-price-classification"].result() - _print_result(results["mlflow/mobile-price-classification"]) - _mobile_price_ok = ( - not _mobile_price_skipped - and results["mlflow/mobile-price-classification"].status == "PASS" +def _phase2_submit(ctx: _Context) -> dict[str, Future]: + """Phase 2: submit pipeline notebooks + scripts (they return fast).""" + print("\nPhase 2: submitting pipelines...") + futures: dict[str, Future] = {} + for name, rel in [ + ( + "mlflow/mobile-price-classification", + "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", + ), + ( + "pipelines/lightweight-components", + "pipelines/lightweight-components/mobile-price-classifications.ipynb", + ), + ]: + if name in ctx.results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue + futures[name] = _submit_notebook( + ctx.executor, + ctx.results, + name, + ctx.root / rel, + ctx.output_dir, + ctx.timeout_notebook, + ) + + futures["pipelines/lightweight-python-package"] = _submit_script( + ctx.executor, + ctx.results, + "pipelines/lightweight-python-package", + ctx.root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", + ctx.timeout_notebook, + ) + futures["pipelines/minimal-container-components"] = _submit_script( + ctx.executor, + ctx.results, + "pipelines/minimal-container-components", + ctx.root / "pipelines" / "minimal-container-components" / "submit-cluster.py", + ctx.timeout_notebook, + ) + return futures + + +def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> bool: + """Wait for the mlflow-mobile-price notebook and poll its KFP run inline. + + Phase 3 ISVCs need the registered model, so this must complete before + Phase 3 starts. Returns True if the model is registered and ready. + """ + name = "mlflow/mobile-price-classification" + skipped = name not in phase2_futures + if skipped: + return False + + phase2_futures[name].result() + _print_result(ctx.results[name]) + if ctx.results[name].status != "PASS": + return False + + run_ids = ctx.results[name].kfp_run_ids + if not run_ids: + return True + + print( + " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." + ) + ok = True + for run_id in run_ids: + state, err = _poll_kfp_run(run_id, ctx.timeout_pipeline) + ctx.poll_results[run_id] = state # recorded here; Phase 4 will skip it + ctx.poll_errors[run_id] = err + print(f" [{state}] {run_id[:8]}...") + if state.upper() != "SUCCEEDED": + ok = False + return ok + + +def _phase3_submit( + ctx: _Context, mobile_price_ok: bool, mobile_price_skipped: bool +) -> dict[str, Future]: + """Phase 3: deploy MLflow KServe InferenceServices (need registered model).""" + print("\nPhase 3: deploying MLflow KServe InferenceServices...") + isvc_names = ( + "serving/mlflow-kserve-minimal", + "serving/mlflow-kserve-inference-protocols", + ) + if not mobile_price_ok: + reason = ( + "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" + if mobile_price_skipped + else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" + ) + print(f" [SKIP] {reason}") + for name in isvc_names: + ctx.results[name] = Result(name=name, status="SKIP", error=reason) + return {} + + futures: dict[str, Future] = {} + # mlflow-kserve-minimal: deploy then immediately smoke-test + # deploy_and_test() in apply.py handles both steps + futures["serving/mlflow-kserve-minimal"] = _submit_script( + ctx.executor, + ctx.results, + "serving/mlflow-kserve-minimal", + ctx.root / "serving" / "mlflow-kserve-minimal" / "apply.py", + ctx.timeout_notebook, + ) + # inference-protocols notebook handles its own deploy + test. + # extract_run_ids=False: this notebook is not a pipeline submission; + # its output contains ISVC/request UUIDs that must not be polled + # as KFP run IDs (they return 404 from the KFP API). + futures["serving/mlflow-kserve-inference-protocols"] = _submit_notebook( + ctx.executor, + ctx.results, + "serving/mlflow-kserve-inference-protocols", + ctx.root + / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + ctx.output_dir, + ctx.timeout_notebook, + extract_run_ids=False, + ) + return futures + + +def _phase4_poll(ctx: _Context) -> None: + """Phase 4: poll all KFP run IDs not already polled inline before Phase 3.""" + all_run_ids = [ + rid + for r in ctx.results.values() + for rid in r.kfp_run_ids + if rid not in ctx.poll_results + ] + if not all_run_ids: + print("\nPhase 4: no KFP run IDs found, skipping poll.") + return + + print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") + poll_futures = { + run_id: ctx.executor.submit(_poll_kfp_run, run_id, ctx.timeout_pipeline) + for run_id in all_run_ids + } + for run_id, f in poll_futures.items(): + try: + ctx.poll_results[run_id], ctx.poll_errors[run_id] = f.result() + except Exception as exc: # noqa: BLE001 + ctx.poll_results[run_id] = f"POLL_ERROR: {exc}" + ctx.poll_errors[run_id] = "" + print(f" [{ctx.poll_results[run_id]}] {run_id[:8]}...") + + +def _phase5_cleanup(cleanup_scripts: list[Path]) -> None: + """Phase 5: run all cleanup scripts in parallel (always, even on failure).""" + print("\nPhase 5: running cleanup scripts...") + with ThreadPoolExecutor(max_workers=4) as ex: + futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] + for f in as_completed(futs): + f.result() # already swallows exceptions inside _run_cleanup + + +# ── Main ────────────────────────────────────────────────────────────────────── + + +def run_all( + timeout_notebook: int = 1800, + timeout_pipeline: int = 3600, + include_keda: bool = False, + include_pytorch: bool = False, + include_shadow: bool = False, + dry_run: bool = False, +) -> dict[str, Result]: + root = _REPO_ROOT + results: dict[str, Result] = {} + + mlflow_ok, _ = _preflight(results) + if not mlflow_ok and dry_run: + print("[dry-run] Would execute the following phases:") + for label in _DRY_RUN_PHASES: + print(f" {label}") + return results + + cleanup_scripts = [root / rel for rel in _CLEANUP_RELPATHS] + poll_results: dict[str, str] = {} + poll_errors: dict[str, str] = {} + + try: + with ThreadPoolExecutor(max_workers=8) as executor: + ctx = _Context( + executor=executor, + root=root, + output_dir=root / "ci" / "output", + timeout_notebook=timeout_notebook, + timeout_pipeline=timeout_pipeline, + results=results, + poll_results=poll_results, + poll_errors=poll_errors, ) - if _mobile_price_ok: - _mobile_run_ids = results[ - "mlflow/mobile-price-classification" - ].kfp_run_ids - if _mobile_run_ids: - print( - " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." - ) - for _run_id in _mobile_run_ids: - _state, _err = _poll_kfp_run(_run_id, timeout_pipeline) - poll_results[_run_id] = ( - _state # recorded here; Phase 4 will skip it - ) - poll_errors[_run_id] = _err - print(f" [{_state}] {_run_id[:8]}...") - if _state.upper() != "SUCCEEDED": - _mobile_price_ok = False - - # ── Phase 3: MLflow KServe examples (need registered model) ─────── - print("\nPhase 3: deploying MLflow KServe InferenceServices...") - if not _mobile_price_ok: - _reason = ( - "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" - if _mobile_price_skipped - else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" - ) - print(f" [SKIP] {_reason}") - for name in ( - "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-inference-protocols", - ): - results[name] = Result(name=name, status="SKIP", error=_reason) - phase3_futures = {} - else: - phase3_futures = {} - - # mlflow-kserve-minimal: deploy then immediately smoke-test - # deploy_and_test() in apply.py handles both steps - phase3_futures["serving/mlflow-kserve-minimal"] = _submit_script( - executor, - results, - "serving/mlflow-kserve-minimal", - root / "serving" / "mlflow-kserve-minimal" / "apply.py", - timeout_notebook, - ) + _phase1(ctx, include_keda, include_shadow, include_pytorch) + phase2_futures = _phase2_submit(ctx) - # inference-protocols notebook handles its own deploy + test. - # extract_run_ids=False: this notebook is not a pipeline submission; - # its output contains ISVC/request UUIDs that must not be polled - # as KFP run IDs (they return 404 from the KFP API). - phase3_futures["serving/mlflow-kserve-inference-protocols"] = ( - _submit_notebook( - executor, - results, - "serving/mlflow-kserve-inference-protocols", - root - / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - output_dir, - timeout_notebook, - extract_run_ids=False, - ) - ) + mobile_price_skipped = ( + "mlflow/mobile-price-classification" not in phase2_futures + ) + mobile_price_ok = _await_mobile_price(ctx, phase2_futures) + phase3_futures = _phase3_submit(ctx, mobile_price_ok, mobile_price_skipped) # Phase 2 remaining + phase 3 run concurrently; print as each finishes print("\nPhase 2 (remaining) + Phase 3 running concurrently...") remaining = { - **{ - k: v - for k, v in phase2_futures.items() - if k != "mlflow/mobile-price-classification" - }, - **phase3_futures, + k: v + for k, v in phase2_futures.items() + if k != "mlflow/mobile-price-classification" } - _future_to_name2 = {v: k for k, v in remaining.items()} - for f in as_completed(remaining.values()): - _print_result(results[_future_to_name2[f]]) - - # ── Phase 4: KFP run polling ────────────────────────────────────── - # Exclude run IDs already polled inline before Phase 3 - all_run_ids: list[str] = [ - rid - for r in results.values() - for rid in r.kfp_run_ids - if rid not in poll_results - ] - - if all_run_ids: - print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") - poll_futures = { - run_id: executor.submit(_poll_kfp_run, run_id, timeout_pipeline) - for run_id in all_run_ids - } - for run_id, f in poll_futures.items(): - try: - poll_results[run_id], poll_errors[run_id] = f.result() - except Exception as exc: # noqa: BLE001 - poll_results[run_id] = f"POLL_ERROR: {exc}" - poll_errors[run_id] = "" - state = poll_results[run_id] - print(f" [{state}] {run_id[:8]}...") - else: - print("\nPhase 4: no KFP run IDs found, skipping poll.") + remaining.update(phase3_futures) + _drain(ctx, remaining) + + _phase4_poll(ctx) finally: - # ── Phase 5: cleanup ───────────────────────────────────────────────── - print("\nPhase 5: running cleanup scripts...") - with ThreadPoolExecutor(max_workers=4) as ex: - futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] - for f in as_completed(futs): - f.result() # already swallows exceptions inside _run_cleanup + _phase5_cleanup(cleanup_scripts) # Build run_id → source notebook name map from all results run_id_to_name: dict[str, str] = {} From 91d33087f27558dd0bd7a100870215f1c47651b2 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:16:20 +0200 Subject: [PATCH 34/48] refactor: declarative Example/Step registration table in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace ad-hoc per-phase logic with a single _EXAMPLES list. Adding a new example is now one entry in the table — no phase function to find, no cleanup list to remember separately, opt-in flag and mlflow-dependency declared inline. Key changes: - Step / Example dataclasses describe what to run - _EXAMPLES table is the single source of truth for all examples, their phase, cleanup scripts, opt-in gates and mlflow dependency - _make_work() generates a work closure from a list of Steps, replacing _submit_notebook / _submit_script / _submit_chain - _run_phase() filters _EXAMPLES by phase and submits matching work - _preflight() and cleanup list are derived from _EXAMPLES - _print_dry_run() is also derived from _EXAMPLES (stays current) - Fix --dry-run bug: preflight no longer runs in dry-run mode --- ci/run_all.py | 760 ++++++++++++++++++++++++-------------------------- 1 file changed, 366 insertions(+), 394 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 35cd151..591b0e6 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -4,27 +4,25 @@ Execution model --------------- Phase 1 (parallel, self-contained): - notebooks/dask/dask_example.ipynb - mlflow/mlflow-quickstart-example.ipynb - mlflow/mlflow-image-example.ipynb - mlflow/mlflow-kfp-example.ipynb - serving/minimal-s3-model/minimal-s3-model.ipynb + All examples whose phase=1 in _EXAMPLES, including opt-in ones. Phase 2 (parallel, pipeline submissions — return fast): - mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb - pipelines/lightweight-components/mobile-price-classifications.ipynb - pipelines/lightweight-python-package/submit-cluster.py (Python script) + All examples whose phase=2 in _EXAMPLES. Phase 3 (depends on Phase 2 mlflow-mobile-price notebook completing): - serving/mlflow-kserve-minimal/apply.py + test_inference_service.py - serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb + All examples whose phase=3 in _EXAMPLES. Phase 4 (KFP run polling — runs alongside Phase 3): Poll all KFP run IDs extracted from Phase 2 output notebooks until Succeeded or timeout. Phase 5 — cleanup (always, in finally block): - All cleanup.py scripts in parallel. + All cleanup.py scripts derived from _EXAMPLES, in parallel. + +Adding a new example +-------------------- +Add one entry to _EXAMPLES below. Everything else (cleanup, phase +scheduling, opt-in gating, dry-run output) is derived from the table. Usage from a Jupyter notebook ------------------------------ @@ -34,7 +32,8 @@ CLI usage --------- python ci/run_all.py [--timeout-notebook 1800] [--timeout-pipeline 3600] - [--include-keda] [--dry-run] + [--include-keda] [--include-shadow] [--include-pytorch] + [--dry-run] Prerequisites ------------- @@ -53,7 +52,7 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path -from typing import Callable +from typing import Callable, Literal # ── Repo root ───────────────────────────────────────────────────────────────── @@ -94,6 +93,206 @@ class Result: kfp_run_ids: list[str] = field(default_factory=list) +# ── Example registry ────────────────────────────────────────────────────────── + + +@dataclass +class Step: + """One execution unit within an Example — a notebook or a Python script.""" + + kind: Literal["notebook", "script"] + path: str # relative to repo root + extra_args: list[str] = field(default_factory=list) + # True → extract KFP run IDs from output (notebooks: broad UUID regex; + # scripts: KFP_RUN_ID= lines). Set False for ISVC examples + # whose output contains UUIDs that are not KFP run IDs. + extract_run_ids: bool = True + + +@dataclass +class Example: + """One CI example. Add a new entry to _EXAMPLES to include it in CI.""" + + name: str + steps: list[Step] + phase: int # 1 = independent 2 = pipeline submit 3 = needs mlflow model + cleanup: str | None = None # relative path to cleanup.py, or None + opt_in: str | None = ( + None # argparse dest that gates this example, e.g. "include_keda" + ) + mlflow_dependent: bool = ( + False # skip automatically when MLflow credentials are unavailable + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Registration table — add new examples here. +# ───────────────────────────────────────────────────────────────────────────── +_EXAMPLES: list[Example] = [ + # ── Phase 1: independent, self-contained ───────────────────────────────── + Example( + name="notebooks/dask", + steps=[Step("notebook", "notebooks/dask/dask_example.ipynb")], + phase=1, + cleanup="notebooks/dask/cleanup.py", + ), + Example( + name="notebooks/mobile-price-classification", + steps=[ + Step( + "notebook", + "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", + ) + ], + phase=1, + ), + Example( + name="mlflow/mlflow-quickstart", + steps=[Step("notebook", "mlflow/mlflow-quickstart-example.ipynb")], + phase=1, + mlflow_dependent=True, + ), + Example( + name="mlflow/mlflow-image-example", + steps=[Step("notebook", "mlflow/mlflow-image-example.ipynb")], + phase=1, + mlflow_dependent=True, + ), + Example( + name="mlflow/mlflow-kfp-example", + steps=[Step("notebook", "mlflow/mlflow-kfp-example.ipynb")], + phase=1, + mlflow_dependent=True, + ), + Example( + name="serving/minimal-s3-model", + steps=[Step("notebook", "serving/minimal-s3-model/minimal-s3-model.ipynb")], + phase=1, + cleanup="serving/minimal-s3-model/cleanup.py", + ), + Example( + name="serving/hf-vllm-completion", + steps=[ + Step("script", "serving/hf-vllm-completion/apply.py", extract_run_ids=False) + ], + phase=1, + cleanup="serving/hf-vllm-completion/cleanup.py", + ), + Example( + name="serving/kserve-keda-autoscaling", + steps=[ + Step( + "script", + "serving/kserve-keda-autoscaling/apply.py", + extract_run_ids=False, + ) + ], + phase=1, + opt_in="include_keda", + cleanup="serving/kserve-keda-autoscaling/cleanup.py", + ), + Example( + name="serving/minimal-example-shadow-deployment", + steps=[ + Step( + "script", + "serving/minimal-example-shadow-deployment/apply.py", + extract_run_ids=False, + ) + ], + phase=1, + opt_in="include_shadow", + cleanup="serving/minimal-example-shadow-deployment/cleanup.py", + ), + Example( + name="notebooks/mnist-vae", + steps=[ + Step( + "script", + "notebooks/mnist-vae/run_training.py", + extra_args=["--max_epochs", "3"], + extract_run_ids=False, + ), + Step( + "notebook", + "notebooks/mnist-vae/visualizations.ipynb", + extract_run_ids=False, + ), + ], + phase=1, + opt_in="include_pytorch", + ), + # ── Phase 2: pipeline submissions (return fast; KFP runs polled in Phase 4) + Example( + name="mlflow/mobile-price-classification", + steps=[ + Step( + "notebook", + "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", + ) + ], + phase=2, + mlflow_dependent=True, + ), + Example( + name="pipelines/lightweight-components", + steps=[ + Step( + "notebook", + "pipelines/lightweight-components/mobile-price-classifications.ipynb", + ) + ], + phase=2, + ), + Example( + name="pipelines/lightweight-python-package", + steps=[ + Step("script", "pipelines/lightweight-python-package/submit-cluster.py") + ], + phase=2, + ), + Example( + name="pipelines/minimal-container-components", + steps=[ + Step("script", "pipelines/minimal-container-components/submit-cluster.py") + ], + phase=2, + ), + # ── Phase 3: MLflow KServe ISVCs (need model registered by Phase 2) ────── + Example( + name="serving/mlflow-kserve-minimal", + steps=[ + Step( + "script", + "serving/mlflow-kserve-minimal/apply.py", + extract_run_ids=False, + ) + ], + phase=3, + mlflow_dependent=True, + cleanup="serving/mlflow-kserve-minimal/cleanup.py", + ), + Example( + name="serving/mlflow-kserve-inference-protocols", + steps=[ + Step( + "notebook", + "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + extract_run_ids=False, + ) + ], + phase=3, + mlflow_dependent=True, + cleanup="serving/mlflow-kserve-inference-protocols/cleanup.py", + ), +] + +# Cleanup scripts for resources not covered by an Example entry above. +_EXTRA_CLEANUP_PATHS: list[str] = [ + "hparam-tuning/minimal-mnist/cleanup.py", +] + + # ── Helpers ─────────────────────────────────────────────────────────────────── @@ -115,7 +314,6 @@ def _format_papermill_error(exc: Exception) -> str: lines = [ f"Cell {exc.exec_count} raised {exc.ename}: {exc.evalue}", ] - # Show the failing cell source (first 5 lines) if exc.source: src_lines = exc.source.strip().splitlines()[:5] lines.append(" Cell source:") @@ -123,7 +321,6 @@ def _format_papermill_error(exc: Exception) -> str: lines.append(f" {src_line}") if len(exc.source.strip().splitlines()) > 5: lines.append(" ...") - # Show the innermost traceback frame (last non-empty line) if exc.traceback: tb_lines = [l for l in exc.traceback if l.strip()] if tb_lines: @@ -132,8 +329,7 @@ def _format_papermill_error(exc: Exception) -> str: def _strip_ci_skip_cells(nb_path: Path, output_dir: Path) -> Path: - """Return a copy of the notebook with 'ci-skip' tagged cells replaced by a comment. - Returns the original path unchanged if no cells are tagged.""" + """Return a copy of the notebook with 'ci-skip' tagged cells replaced by a comment.""" with open(nb_path) as fh: nb = json.load(fh) skipped = 0 @@ -165,8 +361,8 @@ def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: str(output_path), kernel_name="python3", execution_timeout=timeout, - cwd=str(nb_path.parent), # always cwd to original notebook directory - progress_bar=False, # avoid interleaved tqdm bars from parallel threads + cwd=str(nb_path.parent), + progress_bar=False, ) except PapermillExecutionError as exc: raise RuntimeError(_format_papermill_error(exc)) from exc @@ -204,7 +400,7 @@ def _extract_run_ids_from_notebook(output_nb: Path) -> list[str]: output.get("text", []) + output.get("data", {}).get("text/plain", []) ) ids.extend(_RUN_ID_RE.findall(text)) - return list(dict.fromkeys(ids)) # deduplicate, preserve order + return list(dict.fromkeys(ids)) def _extract_run_ids_from_stdout(stdout: str) -> list[str]: @@ -220,11 +416,7 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: def _get_failed_task_logs(run: object, namespace: str) -> str: - """Best-effort: tail logs from the pod(s) of the first failed KFP task. - - Walks run.run_details.task_details → child_tasks → pod_name and tries - kubectl logs on those pods. Silently returns "" on any failure. - """ + """Best-effort: tail logs from the pod(s) of the first failed KFP task.""" try: task_details = ( getattr(getattr(run, "run_details", None), "task_details", None) or [] @@ -283,8 +475,6 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s deadline = time.time() + timeout while time.time() < deadline: run = client.get_run(run_id) - # KFP v2 SDK: state is on the run object directly as a string - # KFP v1 SDK: state is on run.run.status state = str( getattr(run, "state", None) or getattr(getattr(run, "run", None), "status", None) @@ -293,8 +483,6 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s if state in _KFP_TERMINAL_STATES: error_detail = "" if state not in ("SUCCEEDED", "SKIPPED"): - # run.error is often None even for failed runs in KFP v2 — - # the real failure is in the task pod logs err = getattr(run, "error", None) error_detail = (getattr(err, "message", "") or "") if err else "" if not error_detail and namespace: @@ -318,7 +506,47 @@ def _run_cleanup(cleanup_path: Path) -> None: print(f" WARNING: cleanup {cleanup_path.name} failed: {exc}", file=sys.stderr) -# ── Executor ────────────────────────────────────────────────────────────────── +# ── Orchestration context ───────────────────────────────────────────────────── + + +@dataclass +class _Context: + """Shared mutable state threaded through every phase.""" + + executor: ThreadPoolExecutor + root: Path + output_dir: Path + timeout_notebook: int + timeout_pipeline: int + results: dict[str, Result] + poll_results: dict[str, str] + poll_errors: dict[str, str] + + +# ── Execution primitives ────────────────────────────────────────────────────── + + +def _make_work( + steps: list[Step], root: Path, output_dir: Path, timeout: int +) -> Callable[[Result], None]: + """Build a work(result) closure from a list of Steps.""" + + def work(result: Result) -> None: + for step in steps: + if step.kind == "notebook": + out = _run_notebook(root / step.path, output_dir, timeout) + if step.extract_run_ids: + result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) + elif step.kind == "script": + stdout, _ = _run_script( + root / step.path, + timeout, + extra_args=step.extra_args or None, + ) + if step.extract_run_ids: + result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + + return work def _timed_run( @@ -327,16 +555,12 @@ def _timed_run( name: str, work: Callable[[Result], None], ) -> Future: - """Submit `work(result)` to the executor with shared timing + error handling. - - `work` mutates the Result (e.g. sets kfp_run_ids); status/duration are set - here. The first exception is recorded as FAIL. - """ + """Submit work(result) to the executor with shared timing + error handling.""" result = Result(name=name) results[name] = result print(f" [START ] {name}") - def _run(): + def _run() -> None: t0 = time.time() try: work(result) @@ -350,79 +574,23 @@ def _run(): return executor.submit(_run) -def _submit_notebook( - executor: ThreadPoolExecutor, - results: dict[str, Result], - name: str, - nb_path: Path, - output_dir: Path, - timeout: int, - extract_run_ids: bool = True, -) -> Future: - def work(result: Result) -> None: - out = _run_notebook(nb_path, output_dir, timeout) - if extract_run_ids: - result.kfp_run_ids = _extract_run_ids_from_notebook(out) - - return _timed_run(executor, results, name, work) - - -def _submit_script( - executor: ThreadPoolExecutor, - results: dict[str, Result], - name: str, - script_path: Path, - timeout: int, - extra_args: list[str] | None = None, -) -> Future: - def work(result: Result) -> None: - stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) - result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) - - return _timed_run(executor, results, name, work) - - -def _submit_chain( - executor: ThreadPoolExecutor, - results: dict[str, Result], - name: str, - steps: list[tuple[str, Path | None, dict]], - output_dir: Path, - timeout: int, -) -> Future: - """Submit a sequential chain of (kind, path, kwargs) steps as a single named result. - - kind is 'notebook' or 'script'. kwargs may include 'extra_args' for scripts. - All steps run in one thread; the first failure stops the chain. - """ +def _drain(ctx: _Context, futures: dict[str, Future]) -> None: + """Wait on a {name: Future} map, printing each result as it completes.""" + by_future = {v: k for k, v in futures.items()} + for f in as_completed(futures.values()): + _print_result(ctx.results[by_future[f]]) - def work(result: Result) -> None: - for kind, path, kwargs in steps: - if kind == "notebook": - out = _run_notebook(path, output_dir, timeout) - result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) - elif kind == "script": - stdout, _ = _run_script( - path, timeout, extra_args=kwargs.get("extra_args") - ) - result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) - elif kind == "pip": - subprocess.run( - [sys.executable, "-m", "pip", "install", "-q"] - + kwargs.get("packages", []), - capture_output=True, - text=True, - check=True, - ) - return _timed_run(executor, results, name, work) +def _skip(ctx: _Context, name: str, reason: str) -> None: + """Record a SKIP result and print a one-line notice.""" + print(f" [SKIP ] {name}") + ctx.results[name] = Result(name=name, status="SKIP", error=reason) # ── Report ──────────────────────────────────────────────────────────────────── def _print_result(r: Result) -> None: - """Print a single result line, with error detail if failed.""" dur = f"{r.duration:.0f}s" if r.duration else "-" print(f" [{r.status}] {r.name} ({dur})") if r.status == "FAIL" and r.error: @@ -443,13 +611,12 @@ def _print_report( failed += 1 else: passed += 1 - for run_id, status in poll_results.items(): + for status in poll_results.values(): if status.upper() == "SUCCEEDED": passed += 1 else: failed += 1 - # Failed details first so they're easy to scroll back to if failed: print("\nFailed details:") print("-" * 70) @@ -467,7 +634,6 @@ def _print_report( for line in err.splitlines(): print(f" {line}") - # Summary table last — easy to see final verdict at a glance print("\n" + "=" * 70) print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") print("-" * 70) @@ -487,22 +653,34 @@ def _print_report( print() -# ── MLflow pre-flight check ─────────────────────────────────────────────────── - -# Notebooks that require working MLflow credentials -_MLFLOW_DEPENDENT = frozenset( - { - "mlflow/mlflow-quickstart", - "mlflow/mlflow-image-example", - "mlflow/mlflow-kfp-example", - "mlflow/mobile-price-classification", +def _print_dry_run() -> None: + by_phase: dict[int, list[str]] = {} + for ex in _EXAMPLES: + label = ex.name + if ex.opt_in: + label += f" (--{ex.opt_in.replace('_', '-')})" + if ex.mlflow_dependent: + label += " (mlflow)" + by_phase.setdefault(ex.phase, []).append(label) + phase_names = { + 1: "independent", + 2: "pipeline submissions", + 3: "MLflow KServe ISVCs", } -) + print("[dry-run] Would execute the following phases:") + for phase, labels in sorted(by_phase.items()): + print(f" Phase {phase} ({phase_names.get(phase, f'phase {phase}')}):") + for label in labels: + print(f" {label}") + print(" Phase 4: KFP run polling") + print(" Phase 5: cleanup") + + +# ── Pre-flight ──────────────────────────────────────────────────────────────── def _check_mlflow_credentials() -> tuple[bool, str]: - """Return (ok, reason). Checks secret existence then validates credentials - with a quick call to the MLflow REST API.""" + """Return (ok, reason). Checks secret existence then validates with MLflow API.""" import base64 as _b64 import json as _json import urllib.error @@ -519,7 +697,6 @@ def _check_mlflow_credentials() -> tuple[bool, str]: "mlflow-credentials secret not found — " "run scripts/setup_mlflow_credentials.py first" ) - try: data = _json.loads(r.stdout)["data"] uri = _b64.b64decode(data["MLFLOW_TRACKING_URI"]).decode().rstrip("/") @@ -553,235 +730,59 @@ def _check_mlflow_credentials() -> tuple[bool, str]: return False, f"Could not reach MLflow at {uri}: {exc}" -# ── Orchestration context ───────────────────────────────────────────────────── - - -@dataclass -class _Context: - """Shared mutable state threaded through every phase.""" - - executor: ThreadPoolExecutor - root: Path - output_dir: Path - timeout_notebook: int - timeout_pipeline: int - results: dict[str, Result] - poll_results: dict[str, str] - poll_errors: dict[str, str] - - -_CLEANUP_RELPATHS = [ - "notebooks/dask/cleanup.py", - "serving/minimal-s3-model/cleanup.py", - "serving/mlflow-kserve-minimal/cleanup.py", - "serving/mlflow-kserve-inference-protocols/cleanup.py", - "serving/hf-vllm-completion/cleanup.py", - "serving/kserve-keda-autoscaling/cleanup.py", - "serving/minimal-example-shadow-deployment/cleanup.py", - "hparam-tuning/minimal-mnist/cleanup.py", -] +def _preflight(results: dict[str, Result]) -> bool: + """Install papermill and validate MLflow credentials. -_DRY_RUN_PHASES = [ - "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", - "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", - "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", - "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", - "Phase 4: KFP run polling", - "Phase 5: cleanup", -] - - -def _drain(ctx: _Context, futures: dict[str, Future]) -> None: - """Wait on a {name: Future} map, printing each result as it completes.""" - by_future = {v: k for k, v in futures.items()} - for f in as_completed(futures.values()): - _print_result(ctx.results[by_future[f]]) - - -def _skip(ctx: _Context, name: str, reason: str, label: str | None = None) -> None: - """Record a SKIP result and print a one-line notice.""" - print(f" [SKIP ] {label or name}") - ctx.results[name] = Result(name=name, status="SKIP", error=reason) - - -# ── Pre-flight ──────────────────────────────────────────────────────────────── - - -def _preflight(results: dict[str, Result]) -> tuple[bool, str]: - """Ensure papermill, then validate MLflow credentials. - - Returns (mlflow_ok, reason). On failure, MLflow-dependent examples are - recorded as SKIP in `results`. + MLflow-dependent examples are recorded as SKIP in `results` on failure. + Returns True if MLflow credentials are valid. """ _ensure_papermill() print("Pre-flight: checking MLflow credentials...") - mlflow_ok, mlflow_reason = _check_mlflow_credentials() + mlflow_ok, reason = _check_mlflow_credentials() if mlflow_ok: print(" [OK] MLflow credentials valid") else: - print(f" [SKIP] {mlflow_reason}") - for name in _MLFLOW_DEPENDENT: - results[name] = Result(name=name, status="SKIP", error=mlflow_reason) - return mlflow_ok, mlflow_reason + print(f" [SKIP] {reason}") + for ex in _EXAMPLES: + if ex.mlflow_dependent: + results[ex.name] = Result(name=ex.name, status="SKIP", error=reason) + return mlflow_ok -# ── Phases ────────────────────────────────────────────────────────────────── +# ── Phases ──────────────────────────────────────────────────────────────────── -def _phase1( +def _run_phase( ctx: _Context, - include_keda: bool, - include_shadow: bool, - include_pytorch: bool, -) -> None: - """Phase 1: independent notebooks + self-contained serving examples.""" - print("Phase 1: running independent notebooks in parallel...") + phase: int, + opts: dict[str, bool], +) -> dict[str, Future]: + """Submit all examples for a given phase; return {name: Future}.""" futures: dict[str, Future] = {} - for name, rel in [ - ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), - ( - "notebooks/mobile-price-classification", - "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", - ), - ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), - ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), - ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), - ("serving/minimal-s3-model", "serving/minimal-s3-model/minimal-s3-model.ipynb"), - ]: - if name in ctx.results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") + for ex in _EXAMPLES: + if ex.phase != phase: continue - futures[name] = _submit_notebook( - ctx.executor, - ctx.results, - name, - ctx.root / rel, - ctx.output_dir, - ctx.timeout_notebook, - ) - - # hf-vllm-completion (CPU): self-contained, no KFP pipelines - futures["serving/hf-vllm-completion"] = _submit_script( - ctx.executor, - ctx.results, - "serving/hf-vllm-completion", - ctx.root / "serving" / "hf-vllm-completion" / "apply.py", - ctx.timeout_notebook, - ) - - # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs - # and exits 0 with a skip message if KEDA is not installed. - if include_keda: - futures["serving/kserve-keda-autoscaling"] = _submit_script( - ctx.executor, - ctx.results, - "serving/kserve-keda-autoscaling", - ctx.root / "serving" / "kserve-keda-autoscaling" / "apply.py", - ctx.timeout_notebook, - ) - else: - _skip( - ctx, - "serving/kserve-keda-autoscaling", - "opt-in: pass --include-keda to run this example", - label="serving/kserve-keda-autoscaling (pass --include-keda to enable)", - ) - - # minimal-example-shadow-deployment: opt-in; apply.py self-checks for - # the postgres-operator CRD and exits 0 if not found. Istio - # VirtualService mirroring is not verified in CI (requires domain config). - if include_shadow: - futures["serving/minimal-example-shadow-deployment"] = _submit_script( - ctx.executor, - ctx.results, - "serving/minimal-example-shadow-deployment", - ctx.root / "serving" / "minimal-example-shadow-deployment" / "apply.py", - ctx.timeout_notebook, - ) - else: - _skip( - ctx, - "serving/minimal-example-shadow-deployment", - "opt-in: pass --include-shadow to run this example", - label="serving/minimal-example-shadow-deployment (pass --include-shadow to enable)", - ) - - # mnist-vae: opt-in (requires pytorch_lightning, not in all images) - if include_pytorch: - futures["notebooks/mnist-vae"] = _submit_chain( - ctx.executor, - ctx.results, - "notebooks/mnist-vae", - steps=[ - # run_training.py self-installs pytorch-lightning if absent - ( - "script", - ctx.root / "notebooks/mnist-vae/run_training.py", - {"extra_args": ["--max_epochs", "3"]}, - ), - ("notebook", ctx.root / "notebooks/mnist-vae/visualizations.ipynb", {}), - ], - output_dir=ctx.output_dir, - timeout=ctx.timeout_notebook, - ) - else: - print(" [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)") - - _drain(ctx, futures) - - -def _phase2_submit(ctx: _Context) -> dict[str, Future]: - """Phase 2: submit pipeline notebooks + scripts (they return fast).""" - print("\nPhase 2: submitting pipelines...") - futures: dict[str, Future] = {} - for name, rel in [ - ( - "mlflow/mobile-price-classification", - "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", - ), - ( - "pipelines/lightweight-components", - "pipelines/lightweight-components/mobile-price-classifications.ipynb", - ), - ]: - if name in ctx.results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") + if ex.name in ctx.results: # already SKIP from pre-flight + print(f" [SKIP ] {ex.name}") continue - futures[name] = _submit_notebook( - ctx.executor, - ctx.results, - name, - ctx.root / rel, - ctx.output_dir, - ctx.timeout_notebook, - ) - - futures["pipelines/lightweight-python-package"] = _submit_script( - ctx.executor, - ctx.results, - "pipelines/lightweight-python-package", - ctx.root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", - ctx.timeout_notebook, - ) - futures["pipelines/minimal-container-components"] = _submit_script( - ctx.executor, - ctx.results, - "pipelines/minimal-container-components", - ctx.root / "pipelines" / "minimal-container-components" / "submit-cluster.py", - ctx.timeout_notebook, - ) + if ex.opt_in and not opts.get(ex.opt_in, False): + _skip( + ctx, ex.name, f"opt-in: pass --{ex.opt_in.replace('_', '-')} to enable" + ) + continue + work = _make_work(ex.steps, ctx.root, ctx.output_dir, ctx.timeout_notebook) + futures[ex.name] = _timed_run(ctx.executor, ctx.results, ex.name, work) return futures def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> bool: - """Wait for the mlflow-mobile-price notebook and poll its KFP run inline. + """Wait for mlflow-mobile-price and poll its KFP run inline. Phase 3 ISVCs need the registered model, so this must complete before Phase 3 starts. Returns True if the model is registered and ready. """ name = "mlflow/mobile-price-classification" - skipped = name not in phase2_futures - if skipped: + if name not in phase2_futures: return False phase2_futures[name].result() @@ -799,7 +800,7 @@ def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> boo ok = True for run_id in run_ids: state, err = _poll_kfp_run(run_id, ctx.timeout_pipeline) - ctx.poll_results[run_id] = state # recorded here; Phase 4 will skip it + ctx.poll_results[run_id] = state ctx.poll_errors[run_id] = err print(f" [{state}] {run_id[:8]}...") if state.upper() != "SUCCEEDED": @@ -807,55 +808,7 @@ def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> boo return ok -def _phase3_submit( - ctx: _Context, mobile_price_ok: bool, mobile_price_skipped: bool -) -> dict[str, Future]: - """Phase 3: deploy MLflow KServe InferenceServices (need registered model).""" - print("\nPhase 3: deploying MLflow KServe InferenceServices...") - isvc_names = ( - "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-inference-protocols", - ) - if not mobile_price_ok: - reason = ( - "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" - if mobile_price_skipped - else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" - ) - print(f" [SKIP] {reason}") - for name in isvc_names: - ctx.results[name] = Result(name=name, status="SKIP", error=reason) - return {} - - futures: dict[str, Future] = {} - # mlflow-kserve-minimal: deploy then immediately smoke-test - # deploy_and_test() in apply.py handles both steps - futures["serving/mlflow-kserve-minimal"] = _submit_script( - ctx.executor, - ctx.results, - "serving/mlflow-kserve-minimal", - ctx.root / "serving" / "mlflow-kserve-minimal" / "apply.py", - ctx.timeout_notebook, - ) - # inference-protocols notebook handles its own deploy + test. - # extract_run_ids=False: this notebook is not a pipeline submission; - # its output contains ISVC/request UUIDs that must not be polled - # as KFP run IDs (they return 404 from the KFP API). - futures["serving/mlflow-kserve-inference-protocols"] = _submit_notebook( - ctx.executor, - ctx.results, - "serving/mlflow-kserve-inference-protocols", - ctx.root - / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - ctx.output_dir, - ctx.timeout_notebook, - extract_run_ids=False, - ) - return futures - - def _phase4_poll(ctx: _Context) -> None: - """Phase 4: poll all KFP run IDs not already polled inline before Phase 3.""" all_run_ids = [ rid for r in ctx.results.values() @@ -881,12 +834,11 @@ def _phase4_poll(ctx: _Context) -> None: def _phase5_cleanup(cleanup_scripts: list[Path]) -> None: - """Phase 5: run all cleanup scripts in parallel (always, even on failure).""" print("\nPhase 5: running cleanup scripts...") with ThreadPoolExecutor(max_workers=4) as ex: futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] for f in as_completed(futs): - f.result() # already swallows exceptions inside _run_cleanup + f.result() # ── Main ────────────────────────────────────────────────────────────────────── @@ -903,14 +855,20 @@ def run_all( root = _REPO_ROOT results: dict[str, Result] = {} - mlflow_ok, _ = _preflight(results) - if not mlflow_ok and dry_run: - print("[dry-run] Would execute the following phases:") - for label in _DRY_RUN_PHASES: - print(f" {label}") + if dry_run: + _print_dry_run() return results - cleanup_scripts = [root / rel for rel in _CLEANUP_RELPATHS] + _preflight(results) + + opts = { + "include_keda": include_keda, + "include_pytorch": include_pytorch, + "include_shadow": include_shadow, + } + cleanup_scripts = [root / ex.cleanup for ex in _EXAMPLES if ex.cleanup] + [ + root / p for p in _EXTRA_CLEANUP_PATHS + ] poll_results: dict[str, str] = {} poll_errors: dict[str, str] = {} @@ -927,16 +885,31 @@ def run_all( poll_errors=poll_errors, ) - _phase1(ctx, include_keda, include_shadow, include_pytorch) - phase2_futures = _phase2_submit(ctx) + print("Phase 1: running independent examples in parallel...") + _drain(ctx, _run_phase(ctx, 1, opts)) - mobile_price_skipped = ( - "mlflow/mobile-price-classification" not in phase2_futures - ) + print("\nPhase 2: submitting pipelines...") + phase2_futures = _run_phase(ctx, 2, opts) mobile_price_ok = _await_mobile_price(ctx, phase2_futures) - phase3_futures = _phase3_submit(ctx, mobile_price_ok, mobile_price_skipped) - # Phase 2 remaining + phase 3 run concurrently; print as each finishes + print("\nPhase 3: deploying MLflow KServe InferenceServices...") + if mobile_price_ok: + phase3_futures = _run_phase(ctx, 3, opts) + else: + skipped = "mlflow/mobile-price-classification" not in phase2_futures + reason = ( + "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" + if skipped + else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" + ) + print(f" [SKIP] {reason}") + for ex in _EXAMPLES: + if ex.phase == 3: + ctx.results[ex.name] = Result( + name=ex.name, status="SKIP", error=reason + ) + phase3_futures = {} + print("\nPhase 2 (remaining) + Phase 3 running concurrently...") remaining = { k: v @@ -951,12 +924,9 @@ def run_all( finally: _phase5_cleanup(cleanup_scripts) - # Build run_id → source notebook name map from all results - run_id_to_name: dict[str, str] = {} - for r in results.values(): - for run_id in r.kfp_run_ids: - run_id_to_name[run_id] = r.name - + run_id_to_name = { + run_id: r.name for r in results.values() for run_id in r.kfp_run_ids + } _print_report(results, poll_results, poll_errors, run_id_to_name) return results @@ -981,7 +951,7 @@ def run_all( parser.add_argument( "--include-keda", action="store_true", - help="Include KEDA autoscaling example (opt-in; requires KEDA installed in the cluster)", + help="Include KEDA autoscaling example (opt-in; requires KEDA in the cluster; runs on CPU)", ) parser.add_argument( "--include-shadow", @@ -997,7 +967,9 @@ def run_all( help="Include pytorch_lightning examples (opt-in; requires pytorch in the notebook image)", ) parser.add_argument( - "--dry-run", action="store_true", help="Print plan without executing anything" + "--dry-run", + action="store_true", + help="Print plan without executing anything", ) args = parser.parse_args() From 74a043ccf0df96eefa83f2429e299e1d93420574 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:31:36 +0200 Subject: [PATCH 35/48] docs: add ci/README.md with conventions for adding examples, scripts, and ci-skip tags --- ci/README.md | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 ci/README.md diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..73c697d --- /dev/null +++ b/ci/README.md @@ -0,0 +1,156 @@ +# CI — contributor guide + +> This document is written primarily for AI agents making changes to this +> repository. It describes the conventions that must be followed for a new +> example to be picked up by `ci/run_all.py` correctly. + +--- + +## Adding a new example + +Register it in the `_EXAMPLES` list in `run_all.py`. Everything else — +phase scheduling, opt-in gating, cleanup, dry-run output, MLflow-credential +skipping — is derived from that entry automatically. + +```python +Example( + name="serving/my-new-example", # display name and result key + steps=[ + Step("script", "serving/my-new-example/apply.py"), + # or: Step("notebook", "serving/my-new-example/example.ipynb") + # chain multiple steps if needed (executed sequentially) + ], + phase=1, # see Phase rules below + cleanup="serving/my-new-example/cleanup.py", # omit if no K8s resources + opt_in="include_foo", # omit if always enabled + mlflow_dependent=False, # True = skip when MLflow creds absent +) +``` + +### Phase rules + +| Phase | When to use | +|-------|-------------| +| 1 | Self-contained: does not depend on anything else in CI | +| 2 | Submits a KFP pipeline and returns fast; actual run is polled in Phase 4 | +| 3 | Requires a model already registered by the Phase 2 mlflow-mobile-price pipeline | + +### Opt-in flag + +Add `opt_in="include_foo"` and a corresponding `--include-foo` argument in +the `__main__` block of `run_all.py`. Use opt-in when the example requires +cluster add-ons (KEDA, postgres-operator, GPU nodes) that are not guaranteed +to be present. + +--- + +## apply.py and cleanup.py + +### cleanup.py — always add when K8s resources are created + +Any example that creates Kubernetes resources (InferenceService, Deployment, +Service, CRD instance, …) must have a `cleanup.py` in its directory. +CI runs all cleanup scripts in parallel in a `finally` block so they execute +even on failure. + +A cleanup script must: +- Delete resources idempotently (`--ignore-not-found`) +- Never raise on failure (print a warning at most) +- Read the namespace from the pod filesystem: + +```python +with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: + ns = f.read().strip() +``` + +**Do not** add a `cleanup.py` for examples that only create in-cluster +transient objects managed by KFP (pipeline runs, artifacts) — those are +cleaned up by KFP's own retention policy. + +### apply.py — only for serving examples driven by a script + +Add an `apply.py` only when the CI execution of a serving example is driven +by a Python script rather than a notebook. This is the case when: + +- The notebook's deploy/test section requires manual inputs that cannot be + automated, so a separate script owns the full deploy-wait-smoketest cycle. +- The example has no notebook at all. + +An `apply.py` must exit non-zero on failure (CI relies on the return code). +It must print a clean error message to stderr and suppress the full traceback: + +```python +if __name__ == "__main__": + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) +``` + +If you add an `apply.py`, always add the matching `cleanup.py` as well. + +--- + +## scripts/ utilities + +`scripts/` contains two cluster utilities importable from notebooks and apply +scripts. Both work identically whether called from a notebook cell or from an +`apply.py`. + +### setup_mlflow_credentials.py + +One-time, interactive. Stores MLflow credentials in the +`mlflow-credentials` K8s secret. **Requires human input** (the MLflow +Personal Access Token cannot be obtained programmatically). Run it once from +a JupyterLab terminal: + +```bash +python examples/scripts/setup_mlflow_credentials.py +``` + +Do not call this from CI. CI validates the secret exists in the preflight +check and skips MLflow-dependent examples if it does not. + +### get_or_create_api_key.py + +Idempotent, no human input. Creates or retrieves the `examples-key` +AIGatewayKey CR. Call it from any notebook cell or script that needs an +inference API key: + +```python +import sys, subprocess, os +_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() +sys.path.insert(0, os.path.join(_root, "scripts")) +from get_or_create_api_key import get_or_create_api_key + +API_KEY = get_or_create_api_key() +``` + +--- + +## ci-skip cell tag + +Tag a notebook cell with `ci-skip` to have CI replace it with a no-op comment +before execution. Use this **only** for cells that genuinely cannot run +headlessly: + +- Cells that require manual input (hardcoded paths, paste-your-token + placeholders, `input()` calls). +- Interactive widget cells (`ipywidgets`, `ipykernel` display-only code). + +**Do not** use `ci-skip` to work around a fixable automation problem. If a +cell fails because it needs credentials or a namespace, fix it to load those +automatically (see the MLflow credential pattern above). `ci-skip` is a last +resort, not a convenience. + +To tag a cell in JupyterLab: select the cell → Property Inspector (right +panel) → add `ci-skip` under Cell Tags. + +In raw notebook JSON the tag appears as: + +```json +"metadata": { + "tags": ["ci-skip"] +} +``` From 62a72ace1df507515ae8cdd0b4b428def9f8721b Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:52:52 +0200 Subject: [PATCH 36/48] docs: add ci/ and scripts/ to repo overview, link to ci/README.md --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 05c3307..4e315ef 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,21 @@ For full platform documentation, see [docs.prokube.ai](https://docs.prokube.ai/) ```py . ├── .github # workflows to build images +├── ci # CI orchestrator (run_all.py) and contributor conventions ├── hparam-tuning # hyperparameter tuning examples (Katib) ├── images # custom container images used by examples ├── mlflow # MLflow experiment tracking examples ├── notebooks # Jupyter notebook examples (Dask, MNIST VAE, etc.) ├── pipelines # Kubeflow Pipelines examples ├── rstudio # RStudio examples +├── scripts # shared utilities (credential setup, API key management) ├── serving # model serving examples (KServe, vLLM, shadow deployments) ``` +Many serving examples include an `apply.py` (deploy + smoke-test) and a `cleanup.py` +(teardown of Kubernetes resources) alongside the notebook. These are used by the CI +orchestrator and can also be run manually. + ## Note about storage Storage on Kubernetes is a complex topic and a deep dive is outside the scope of this repository. There are two types of storage you might encounter here — block and object storage. @@ -43,6 +49,12 @@ Some examples require a minimum prokube platform version. If an example is not l |---|---|---| | `serving/minimal-s3-model` | v1.7.0 | Requires s3creds secret with KServe support | +## CI + +Examples are tested end-to-end by `ci/run_all.py`, which runs inside a Kubeflow +notebook pod. See [`ci/README.md`](ci/README.md) for how the orchestrator works and +how to add a new example to CI. + ## Contributing All code contributions should go via pull requests. Make sure your code is clearly documented and that it adheres to established standards (e.g. PEP). From 5556e658e3639f5b6b54055be218ae4c64d9cd0f Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:06:46 +0200 Subject: [PATCH 37/48] Cosmetics --- .../mlflow-mobile-price-classification.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index ef18fd4..1c14910 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -55,12 +55,12 @@ "source": [ "## MLflow Setup\n", "\n", - "To use MLflow tracking in this pipeline, create a Kubernetes secret with your MLflow credentials:\n", + "To use MLflow tracking in this pipeline, create a Kubernetes secret with your MLflow credentials.\n", "\n", "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "python examples/scripts/setup_mlflow_credentials.py\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", From cd9f21f00b24630fe22dfc41f1a351ae134986e4 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:09:06 +0200 Subject: [PATCH 38/48] docs: align MLflow setup instructions across all MLflow notebooks --- mlflow/mlflow-image-example.ipynb | 20 ++++++++------------ mlflow/mlflow-kfp-example.ipynb | 26 +++++++------------------- mlflow/mlflow-quickstart-example.ipynb | 20 ++++++++------------ 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 0a5b36d..041a21a 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -9,20 +9,16 @@ "\n", "This notebook demonstrates how to use MLflow in the prokube platform with Personal Access Token (PAT) authentication on a slightly more complex example then the Quick Start.\n", "\n", - "## Prerequisites\n", + "## MLflow Setup\n", "\n", - "- Generate your Personal Access Token:\n", - " - Open [MLflow UI](/mlflow) in your browser\n", - " - Navigate to the [Permissions](/mlflow/oidc/ui/#) page\n", - " - Click on \"Create access key\" button\n", - " - Copy the generated token and store it securely\n", - " - Note: You won't be able to see the token again!\n", + "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "```bash\n", + "# From a JupyterLab terminal:\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "```\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "- Configure the credentials below with your values\n", - "\n", - "## Authentication\n", - "\n", - "The prokube MLflow setup uses OIDC authentication with PAT support for programmatic access.\n" + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." ] }, { diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 4477f41..23732ed 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -7,36 +7,24 @@ "source": [ "# Simple pipeline with MLflow model tracking\n", "\n", - "An experiment with iris dataset. In general, to use MLflow in a Kubeflow Pipeline, the necessary environment variables should be passed to the containers using the MLflow logic. This is implemented in the `add_env_vars_to_tasks` function. \n", + "An experiment with iris dataset. In general, to use MLflow in a Kubeflow Pipeline, the necessary environment variables should be passed to the containers using the MLflow logic. This is implemented in the `add_env_vars_to_tasks` function.\n", "\n", - "\n", - "## Prerequisites\n", - "\n", - "To use MLflow in Kubeflow Pipelines, you need to create a Kubernetes secret with your MLflow credentials. \n", - "\n", - "To generate \n", - "\n", - "- Generate your Personal Access Token (re-use your old one, if you have already generated it):\n", - " - Open [MLflow UI](/mlflow) in your browser\n", - " - Navigate to the [Permissions](/mlflow/oidc/ui/#) page\n", - " - Click on \"Create access key\" button\n", - " - Copy the generated token and store it securely\n", - " - Note: You won't be able to see the token again!\n", + "## MLflow Setup\n", "\n", "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "python examples/scripts/setup_mlflow_credentials.py\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#).\n", + "\n", "## How it works\n", "\n", - "In this pipeline, the add_mlflow_env_vars_to_tasks function injects the MLflow credentials from the Kubernetes secret into each pipeline component. This allows the components to authenticate with\n", - "MLflow and track experiments.\n", + "In this pipeline, the add_mlflow_env_vars_to_tasks function injects the MLflow credentials from the Kubernetes secret into each pipeline component. This allows the components to authenticate with MLflow and track experiments.\n", "\n", - "This notebook defines a simple pipeline for preprocessing data, training and logging model, and prediction on test data. It also shows one way to handle MLflow experiment info inside the pipeline - by\n", - "passing run information between components." + "This notebook defines a simple pipeline for preprocessing data, training and logging model, and prediction on test data. It also shows one way to handle MLflow experiment info inside the pipeline - by passing run information between components." ] }, { diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index b60514d..7a73868 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -27,20 +27,16 @@ "\n", "This notebook demonstrates how to use MLflow in the prokube platform with Personal Access Token (PAT) authentication.\n", "\n", - "## Prerequisites\n", + "## MLflow Setup\n", "\n", - "- Generate your Personal Access Token:\n", - " - Open [MLflow UI](/mlflow) in your browser\n", - " - Navigate to the [Permissions](/mlflow/oidc/ui/#) page\n", - " - Click on \"Create access key\" button\n", - " - Copy the generated token and store it securely\n", - " - Note: You won't be able to see the token again!\n", + "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "```bash\n", + "# From a JupyterLab terminal:\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "```\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "- Configure the credentials below with your values\n", - "\n", - "## Authentication\n", - "\n", - "The prokube MLflow setup uses OIDC authentication with PAT support for programmatic access.\n" + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." ] }, { From bd0f9d549a856c7306e1af46206654f74d79a549 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:16:47 +0200 Subject: [PATCH 39/48] fix: update MLflow permissions link and add Create Access Token hint --- mlflow/mlflow-image-example.ipynb | 2 +- mlflow/mlflow-kfp-example.ipynb | 2 +- mlflow/mlflow-quickstart-example.ipynb | 2 +- .../mlflow-mobile-price-classification.ipynb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 041a21a..19a05ab 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -18,7 +18,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\"." ] }, { diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 23732ed..4fc26bf 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -18,7 +18,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#).\n", + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\".\n", "\n", "## How it works\n", "\n", diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index 7a73868..7e3e289 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -36,7 +36,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\"." ] }, { diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 1c14910..770471b 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -64,7 +64,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\"." ] }, { From c3a3b05345724783565cf0763833fce550a00a21 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:30:02 +0200 Subject: [PATCH 40/48] refactor: extract inference-protocols deploy into apply.py, notebook calls deploy() --- ci/run_all.py | 4 +- .../apply.py | 219 ++++++++++++++++++ .../inference_protocol_version_example.ipynb | 71 +----- 3 files changed, 226 insertions(+), 68 deletions(-) create mode 100644 serving/mlflow-kserve-inference-protocols/apply.py diff --git a/ci/run_all.py b/ci/run_all.py index 591b0e6..63fdce6 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -276,8 +276,8 @@ class Example: name="serving/mlflow-kserve-inference-protocols", steps=[ Step( - "notebook", - "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + "script", + "serving/mlflow-kserve-inference-protocols/apply.py", extract_run_ids=False, ) ], diff --git a/serving/mlflow-kserve-inference-protocols/apply.py b/serving/mlflow-kserve-inference-protocols/apply.py new file mode 100644 index 0000000..6455bf8 --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/apply.py @@ -0,0 +1,219 @@ +""" +Deploy the v1 and v2 MLflow KServe InferenceServices and smoke-test both. + +Substitutes ```` and ```` in the YAML manifests from +the cluster environment, applies them, waits for readiness, and verifies that +both endpoints return identical predictions for the same input. + +Usage from a notebook +--------------------- + import sys + sys.path.insert(0, ".") # run from the serving/mlflow-kserve-inference-protocols/ dir + from apply import deploy + + v1_uri, v2_uri, api_key = deploy() + +CLI usage +--------- + python apply.py + +Prerequisites +------------- +- ``mlflow-credentials`` secret must exist (run scripts/setup_mlflow_credentials.py). +- The MLflow model ``mobile-price-svm-`` version 1 must be registered + (run mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb first). +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +import urllib.error +import urllib.request + +_HERE = os.path.dirname(__file__) + +_ISVC_NAMES = { + "v1": "v1-mobile-price-classification-inference", + "v2": "v2-mobile-price-classification-inference", +} +_YAML_FILES = { + "v1": os.path.join(_HERE, "v1-InferenceService.yaml"), + "v2": os.path.join(_HERE, "v2-InferenceService.yaml"), +} +_BODY_FILES = { + "v1": os.path.join(_HERE, "v1-mlflow-inference-body.json"), + "v2": os.path.join(_HERE, "v2-mlflow-inference-body.json"), +} + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _mlflow_username(namespace: str) -> str: + result = subprocess.run( + [ + "kubectl", + "get", + "secret", + "mlflow-credentials", + "-n", + namespace, + "-o", + "json", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + "mlflow-credentials secret not found.\n" + "Run scripts/setup_mlflow_credentials.py to create it." + ) + data = json.loads(result.stdout)["data"] + return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode().split("@")[0] + + +def _apply_yaml(yaml_file: str, namespace: str, username: str) -> None: + manifest = open(yaml_file).read() + manifest = manifest.replace("", namespace).replace( + "", username + ) + result = subprocess.run( + ["kubectl", "apply", "-n", namespace, "-f", "-"], + input=manifest, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"kubectl apply failed for {os.path.basename(yaml_file)}:\n{result.stderr}" + ) + print(result.stdout.strip()) + + +def _wait_ready(name: str, namespace: str, timeout: int = 600) -> None: + print(f" Waiting for {name} (timeout {timeout}s)...") + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "-n", + namespace, + "--for=condition=Ready", + f"--timeout={timeout}s", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace] + ) + raise RuntimeError(f"{name} did not become ready:\n{result.stderr}") + + +def _isvc_url(name: str, namespace: str) -> str: + return subprocess.check_output( + [ + "kubectl", + "get", + "inferenceservice", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.url}", + ], + text=True, + ).strip() + + +def _get_api_key() -> str: + root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + sys.path.insert(0, os.path.join(root, "scripts")) + from get_or_create_api_key import get_or_create_api_key + + return get_or_create_api_key() + + +def _smoke_test(uri: str, name: str, protocol: str, api_key: str) -> None: + """POST one inference request; raise on non-2xx or missing predictions.""" + body = json.load(open(_BODY_FILES[protocol])) + if protocol == "v1": + url = f"{uri}/v1/models/{name}:predict" + pred_key = "predictions" + else: + url = f"{uri}/v2/models/{name}/infer" + pred_key = "outputs" + + data = json.dumps(body).encode() + req = urllib.request.Request( + url, + data=data, + headers={"Content-Type": "application/json", "X-Api-Key": api_key}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + result = json.loads(resp.read()) + except urllib.error.HTTPError as exc: + raise RuntimeError( + f"Smoke test {protocol} returned HTTP {exc.code}: {exc.read().decode()}" + ) from exc + + if pred_key not in result: + raise RuntimeError(f"Smoke test {protocol}: unexpected response: {result}") + print(f" [{protocol}] smoke test passed — {len(result[pred_key])} prediction(s)") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 600) -> tuple[str, str, str]: + """Deploy both ISVCs, wait for readiness, and return (v1_uri, v2_uri, api_key).""" + ns = _namespace() + username = _mlflow_username(ns) + + print("Applying InferenceService manifests...") + for proto, yaml_file in _YAML_FILES.items(): + _apply_yaml(yaml_file, ns, username) + + print("Waiting for InferenceServices to become ready...") + for proto, name in _ISVC_NAMES.items(): + _wait_ready(name, ns, timeout) + + v1_uri = _isvc_url(_ISVC_NAMES["v1"], ns) + v2_uri = _isvc_url(_ISVC_NAMES["v2"], ns) + api_key = _get_api_key() + + print(f"v1 URI: {v1_uri}") + print(f"v2 URI: {v2_uri}") + + print("Running smoke tests...") + _smoke_test(v1_uri, _ISVC_NAMES["v1"], "v1", api_key) + _smoke_test(v2_uri, _ISVC_NAMES["v2"], "v2", api_key) + + return v1_uri, v2_uri, api_key + + +if __name__ == "__main__": + try: + v1_uri, v2_uri, api_key = deploy() + print(f"ISVC_URI_V1={v1_uri}") + print(f"ISVC_URI_V2={v2_uri}") + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 0d07609..710c07b 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -356,72 +356,11 @@ "metadata": {}, "outputs": [], "source": [ - "import base64, json, os, subprocess, sys\n", - "\n", - "# ── 1. Namespace & MLflow username ───────────────────────────────────────────\n", - "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", - " _ns = _f.read().strip()\n", - "\n", - "_secret = subprocess.run(\n", - " [\"kubectl\", \"get\", \"secret\", \"mlflow-credentials\", \"-n\", _ns, \"-o\", \"json\"],\n", - " capture_output=True, text=True,\n", - ")\n", - "if _secret.returncode != 0:\n", - " raise RuntimeError(\n", - " \"mlflow-credentials secret not found.\\n\"\n", - " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", - " )\n", - "_creds = json.loads(_secret.stdout)[\"data\"]\n", - "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode().split(\"@\")[0]\n", - "\n", - "# ── 2. Apply InferenceService YAMLs with substituted namespace & username ────\n", - "_isvc_names = [\n", - " (\"v1-InferenceService.yaml\", \"v1-mobile-price-classification-inference\"),\n", - " (\"v2-InferenceService.yaml\", \"v2-mobile-price-classification-inference\"),\n", - "]\n", - "for _yaml_file, _ in _isvc_names:\n", - " _manifest = open(_yaml_file).read()\n", - " _manifest = _manifest.replace(\"\", _ns).replace(\"\", _username)\n", - " _r = subprocess.run(\n", - " [\"kubectl\", \"apply\", \"-n\", _ns, \"-f\", \"-\"],\n", - " input=_manifest, capture_output=True, text=True,\n", - " )\n", - " if _r.returncode != 0:\n", - " raise RuntimeError(f\"Failed to apply {_yaml_file}:\\n{_r.stderr}\")\n", - " print(f\"Applied {_yaml_file}\")\n", - "\n", - "# ── 3. Wait for both ISVCs to be ready (up to 10 min each) ───────────────────\n", - "print(\"Waiting for InferenceServices to become ready...\")\n", - "for _, _name in _isvc_names:\n", - " _r = subprocess.run(\n", - " [\"kubectl\", \"wait\", \"inferenceservice\", _name, \"-n\", _ns,\n", - " \"--for=condition=Ready\", \"--timeout=600s\"],\n", - " capture_output=True, text=True,\n", - " )\n", - " if _r.returncode != 0:\n", - " raise RuntimeError(f\"InferenceService {_name} did not become ready:\\n{_r.stderr}\")\n", - " print(f\" {_name}: Ready\")\n", - "\n", - "# ── 4. Fetch external URLs from ISVC status ───────────────────────────────────\n", - "def _isvc_url(name):\n", - " return subprocess.check_output(\n", - " [\"kubectl\", \"get\", \"inferenceservice\", name, \"-n\", _ns,\n", - " \"-o\", \"jsonpath={.status.url}\"],\n", - " text=True,\n", - " ).strip()\n", - "\n", - "_uri_v1 = _isvc_url(\"v1-mobile-price-classification-inference\")\n", - "_uri_v2 = _isvc_url(\"v2-mobile-price-classification-inference\")\n", - "\n", - "# ── 5. Get / create inference API key ────────────────────────────────────────\n", - "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", - "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", - "from get_or_create_api_key import get_or_create_api_key\n", - "_api_key = get_or_create_api_key()\n", - "\n", - "print(f\"\\nv1 URI : {_uri_v1}\")\n", - "print(f\"v2 URI : {_uri_v2}\")\n", - "print(\"API key: [set]\")\n" + "import sys\n", + "sys.path.insert(0, \".\")\n", + "from apply import deploy\n", + "\n", + "_uri_v1, _uri_v2, _api_key = deploy()\n" ] }, { From 5c81be1eac6a5b5161b052dde2a82ccd5ede3bd6 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:33:47 +0200 Subject: [PATCH 41/48] fix: remove hardcoded namespace from ServiceAccount YAMLs --- serving/mlflow-kserve-inference-protocols/README.md | 5 ++--- .../mlflow-kserve-inference-protocols/ServiceAccount.yaml | 1 - serving/mlflow-kserve-minimal/README.md | 7 ++----- serving/mlflow-kserve-minimal/ServiceAccount.yaml | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md index 04feceb..7c14762 100644 --- a/serving/mlflow-kserve-inference-protocols/README.md +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -64,11 +64,10 @@ reference this SA. ## Deploy -1. Replace `` in `ServiceAccount.yaml` with your namespace - and apply it (once per namespace): +1. Apply the ServiceAccount (once per namespace): ```sh - kubectl apply -f ServiceAccount.yaml -n + kubectl apply -f ServiceAccount.yaml ``` 2. Replace the placeholder values in both ISVC YAMLs: diff --git a/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml index ae26f66..1f2b812 100644 --- a/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml +++ b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: mlflow-isvc-sa - namespace: "" # imagePullSecrets grant access to the private prokube container registry # where the custom MLflow storage-initializer image lives. # These secrets are provisioned automatically by the prokube platform. diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index cdd0e24..32465cb 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -54,15 +54,12 @@ Both `InferenceService.yaml` and `apply.py` already reference this SA. ## Deploy the InferenceService -1. Open `ServiceAccount.yaml` and replace `` with your - Kubeflow namespace, then apply it: +1. Apply the ServiceAccount (once per namespace): ```sh - kubectl apply -f ServiceAccount.yaml -n + kubectl apply -f ServiceAccount.yaml ``` - You only need to do this once per namespace. - 2. Open `InferenceService.yaml` and replace the placeholder values: | Placeholder | Description | diff --git a/serving/mlflow-kserve-minimal/ServiceAccount.yaml b/serving/mlflow-kserve-minimal/ServiceAccount.yaml index ae26f66..1f2b812 100644 --- a/serving/mlflow-kserve-minimal/ServiceAccount.yaml +++ b/serving/mlflow-kserve-minimal/ServiceAccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: mlflow-isvc-sa - namespace: "" # imagePullSecrets grant access to the private prokube container registry # where the custom MLflow storage-initializer image lives. # These secrets are provisioned automatically by the prokube platform. From efd4a9632adcffc616951d8eaeb1247c6988019e Mon Sep 17 00:00:00 2001 From: Korel Date: Thu, 18 Jun 2026 17:42:40 +0200 Subject: [PATCH 42/48] Update GitHub Actions versions in mobile-price-classification workflow * Bump `actions/checkout` from v4 to v7 * Bump `google-github-actions/auth` from v2 to v3 * Bump `docker/login-action` from v3 to v4 --- .../workflows/mobile-price-classification-components.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mobile-price-classification-components.yaml b/.github/workflows/mobile-price-classification-components.yaml index 612d5fe..6c17a48 100644 --- a/.github/workflows/mobile-price-classification-components.yaml +++ b/.github/workflows/mobile-price-classification-components.yaml @@ -23,15 +23,15 @@ jobs: run: rm -rf /opt/hostedtoolcache - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Auth GCloud CLI - uses: google-github-actions/auth@v2 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ env.IMAGE_PUSH_SECRET }} - name: Docker Login to Google Artifact Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: europe-west3-docker.pkg.dev username: _json_key From 4229ae6de66e7382bda40292301242068bf68535 Mon Sep 17 00:00:00 2001 From: Korel Date: Thu, 18 Jun 2026 18:41:15 +0200 Subject: [PATCH 43/48] Add optional --namespace flag to get_or_create_api_key script * Accept an optional `ns` parameter in `get_or_create_api_key()`, falling back to the auto-detected namespace when not provided * Add `argparse`-based CLI with `--namespace`/`-n` flag when the script is run directly * Allows callers to override the target Kubernetes namespace without changing the default behavior --- scripts/get_or_create_api_key.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index a9a9528..a0085a2 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -22,6 +22,7 @@ from __future__ import annotations +import argparse import base64 import json import secrets @@ -123,9 +124,9 @@ def _create_key(namespace: str) -> str: return key_value -def get_or_create_api_key() -> str: +def get_or_create_api_key(ns=None) -> str: """Return the API key for the current namespace, creating it if needed.""" - ns = _namespace() + ns = ns or _namespace() if _key_exists(ns): return _read_key_from_secret(ns) return _create_key(ns) @@ -133,7 +134,14 @@ def get_or_create_api_key() -> str: if __name__ == "__main__": try: - print(get_or_create_api_key()) + argparser = argparse.ArgumentParser(description="Get or create a prokube API key") + argparser.add_argument( + "--namespace", + "-n", + help="The Kubernetes namespace to use (defaults to the pods namespace)", + ) + args = argparser.parse_args() + print(get_or_create_api_key(args.namespace)) except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) From e89ef2ceb0c22cec1dfd6b21744fb0d2e43db955 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 17:43:04 +0200 Subject: [PATCH 44/48] fix: apply ServiceAccount in inference-protocols apply.py, drop dead SA namespace replace in minimal --- serving/mlflow-kserve-inference-protocols/apply.py | 4 ++++ serving/mlflow-kserve-minimal/apply.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/serving/mlflow-kserve-inference-protocols/apply.py b/serving/mlflow-kserve-inference-protocols/apply.py index 6455bf8..eb83970 100644 --- a/serving/mlflow-kserve-inference-protocols/apply.py +++ b/serving/mlflow-kserve-inference-protocols/apply.py @@ -36,6 +36,7 @@ _HERE = os.path.dirname(__file__) +_SA_YAML = os.path.join(_HERE, "ServiceAccount.yaml") _ISVC_NAMES = { "v1": "v1-mobile-price-classification-inference", "v2": "v2-mobile-price-classification-inference", @@ -187,6 +188,9 @@ def deploy(timeout: int = 600) -> tuple[str, str, str]: ns = _namespace() username = _mlflow_username(ns) + print("Applying ServiceAccount...") + _apply_yaml(_SA_YAML, ns, username) + print("Applying InferenceService manifests...") for proto, yaml_file in _YAML_FILES.items(): _apply_yaml(yaml_file, ns, username) diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index fd259b8..339185f 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -130,7 +130,7 @@ def deploy(timeout: int = 600) -> str: username = _mlflow_username(ns) with open(_SA_YAML_TEMPLATE) as fh: - sa_manifest = fh.read().replace("", ns) + sa_manifest = fh.read() _kubectl_apply(sa_manifest, ns) print(f"Applied ServiceAccount 'mlflow-isvc-sa' in namespace '{ns}'.") From e130b8bbe7a2b35b515c97f9b68ea2c26445c010 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:24:42 +0200 Subject: [PATCH 45/48] feat: support older deployments in get_or_create_api_key Auto-detect the AIGatewayKey CRD; if absent (older clusters using a hardcoded EnvoyFilter), fall back to the INFERENCE_SERVICE_API_KEY env var injected by the admin, and finally to an interactive getpass prompt so notebooks degrade gracefully instead of crashing. --- scripts/get_or_create_api_key.py | 90 ++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index a0085a2..2869b91 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -1,5 +1,5 @@ """ -Utility to create or retrieve a prokube AIGatewayKey for use in examples. +Utility to create or retrieve a prokube API key for use in examples. Usage from a notebook --------------------- @@ -15,9 +15,23 @@ python get_or_create_api_key.py # prints the key to stdout -The key is stored as an AIGatewayKey CR named ``examples-key`` backed by a -Secret named ``examples-key-secret`` in the notebook's namespace. Both -resources are created if they do not already exist. Re-running is safe. +Resolution order +---------------- +The function returns a usable model-serving API key, choosing the mechanism +that fits the deployment it runs on: + +1. **New deployments (AI Gateway).** If the ``AIGatewayKey`` CRD is present, + the key is stored as an AIGatewayKey CR named ``examples-key`` backed by a + Secret named ``examples-key-secret`` in the notebook's namespace. Both + resources are created if they do not already exist. Re-running is safe. + +2. **Older deployments (hardcoded EnvoyFilter).** Older clusters do not have + the AIGatewayKey CRD; instead an admin pre-provisions a single API key and + injects it into the notebook pod via the ``INFERENCE_SERVICE_API_KEY`` + environment variable. When the CRD is absent, that env var is used. + +3. **Fallback.** If neither is available, the caller is prompted to paste the + key interactively, so the example degrades gracefully instead of crashing. """ from __future__ import annotations @@ -25,13 +39,19 @@ import argparse import base64 import json +import os import secrets import subprocess import sys +from getpass import getpass _KEY_NAME = "examples-key" _SECRET_NAME = "examples-key-secret" +# Env var an admin injects into the notebook pod on older (pre-AI-Gateway) +# deployments that rely on a hardcoded EnvoyFilter for auth. +_API_KEY_ENV_VAR = "INFERENCE_SERVICE_API_KEY" + def _namespace() -> str: with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: @@ -47,6 +67,24 @@ def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProces ) +def _aigatewaykey_crd_available() -> bool: + """Return True if the AIGatewayKey CRD is registered in the cluster. + + Used to distinguish new AI-Gateway deployments (CRD present) from older + deployments that rely on a hardcoded EnvoyFilter and an admin-provisioned + API key. + """ + result = _kubectl( + "get", + "crd", + "aigatewaykeys.prokube.ai", + "--ignore-not-found", + "-o", + "name", + ) + return result.returncode == 0 and bool(result.stdout.strip()) + + def _key_exists(namespace: str) -> bool: result = _kubectl( "get", "aigatewaykey", _KEY_NAME, "-n", namespace, "--ignore-not-found" @@ -124,17 +162,49 @@ def _create_key(namespace: str) -> str: return key_value +def _key_from_env() -> str | None: + """Return the admin-provisioned API key from the environment, if set.""" + value = os.environ.get(_API_KEY_ENV_VAR, "").strip() + return value or None + + def get_or_create_api_key(ns=None) -> str: - """Return the API key for the current namespace, creating it if needed.""" - ns = ns or _namespace() - if _key_exists(ns): - return _read_key_from_secret(ns) - return _create_key(ns) + """Return a model-serving API key for the current namespace. + + See the module docstring for the resolution order: AIGatewayKey CR on new + deployments, the ``INFERENCE_SERVICE_API_KEY`` env var on older ones, and an + interactive prompt as a last resort. + """ + # New deployments: manage an AIGatewayKey CR (fully automated). + if _aigatewaykey_crd_available(): + ns = ns or _namespace() + if _key_exists(ns): + return _read_key_from_secret(ns) + return _create_key(ns) + + # Older deployments: admin injects the key via an env var. + env_key = _key_from_env() + if env_key: + return env_key + + # Fallback: prompt instead of crashing. + key = getpass( + f"AIGatewayKey CRD not found and ${_API_KEY_ENV_VAR} is unset. " + "Please enter the model-serving API key (ask your cluster admin): " + ).strip() + if not key: + raise RuntimeError( + "No API key available: the AIGatewayKey CRD is absent, " + f"${_API_KEY_ENV_VAR} is unset, and no key was entered." + ) + return key if __name__ == "__main__": try: - argparser = argparse.ArgumentParser(description="Get or create a prokube API key") + argparser = argparse.ArgumentParser( + description="Get or create a prokube API key" + ) argparser.add_argument( "--namespace", "-n", From b0907e85cd382445319e44f54ed59672c6badb69 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:29:39 +0200 Subject: [PATCH 46/48] fix: detect AIGatewayKey CRD via api-resources, not get crd Notebook pod service accounts lack RBAC permission to read cluster-scoped CRD objects, so the previous 'kubectl get crd' check always returned False in-cluster. API discovery (kubectl api-resources) is allowed for all authenticated users and reliably detects whether the CRD is registered. --- scripts/get_or_create_api_key.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index 2869b91..e2eb076 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -70,19 +70,19 @@ def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProces def _aigatewaykey_crd_available() -> bool: """Return True if the AIGatewayKey CRD is registered in the cluster. - Used to distinguish new AI-Gateway deployments (CRD present) from older - deployments that rely on a hardcoded EnvoyFilter and an admin-provisioned - API key. + Uses the API discovery endpoint (kubectl api-resources) rather than + directly reading the CRD object, because notebook pod service accounts + typically lack RBAC permission to get cluster-scoped CRD resources. + API discovery is allowed for all authenticated Kubernetes users. """ result = _kubectl( - "get", - "crd", - "aigatewaykeys.prokube.ai", - "--ignore-not-found", + "api-resources", + "--api-group=prokube.ai", "-o", "name", + "--no-headers", ) - return result.returncode == 0 and bool(result.stdout.strip()) + return result.returncode == 0 and "aigatewaykeys" in result.stdout def _key_exists(namespace: str) -> bool: From 463b2ab1165097d7f041559b8ddb6f175c9f2cfb Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:38:34 +0200 Subject: [PATCH 47/48] docs: add tip about auto-created examples key and manual override in s3 notebook --- serving/minimal-s3-model/minimal-s3-model.ipynb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index f1b5e91..df52e86 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -297,7 +297,21 @@ "id": "2df5cbae-f406-4c33-be90-dbb0b61cf81d", "metadata": {}, "source": [ - "Enter the model serving API key in the cell below. If not known, ask the cluster administrator for it." + "
\n", + " 💡 Tip:\n", + "
\n", + " The cell below automatically creates a dedicated examples-key AIGatewayKey and its backing Secret in your namespace — no manual steps required.\n", + "

\n", + " If you'd rather use your own key, replace the cell contents with:\n", + " 
INFERENCE_SERVICE_API_KEY = \"your-key-here\"
\n", + "
\n", + "
" ] }, { From 86a2f4b8f711199fb9940748388c1d1556dc67ac Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:39:51 +0200 Subject: [PATCH 48/48] docs: document auto-created examples-key and manual override in serving READMEs --- .../mlflow-kserve-inference-protocols/README.md | 9 +++++++++ serving/mlflow-kserve-minimal/README.md | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md index 7c14762..ddea2de 100644 --- a/serving/mlflow-kserve-inference-protocols/README.md +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -62,6 +62,15 @@ exactly that injection and work correctly with the `default` SA. Both `v1-InferenceService.yaml` and `v2-InferenceService.yaml` already reference this SA. +## API key + +> [!TIP] +> `apply.py` (and the notebook's `deploy()` call) automatically creates a +> dedicated `examples-key` AIGatewayKey and its backing Secret +> (`examples-key-secret`) in your namespace. If you'd rather use your own key, +> pass it to the smoke-test step directly or set `API_KEY` in the environment +> before calling `apply.py`. + ## Deploy 1. Apply the ServiceAccount (once per namespace): diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index 32465cb..2a8ac2a 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -95,14 +95,22 @@ Alternatively, `apply.py` handles steps 1–4 automatically (see below). A test script and sample request body are provided to verify the deployment. +> [!TIP] +> `apply.py` automatically creates a dedicated `examples-key` AIGatewayKey and +> its backing Secret (`examples-key-secret`) in your namespace — no manual key +> management required. If you'd rather supply your own key, set `API_KEY` +> before running the test script: +> ```sh +> export API_KEY=your-key-here +> ``` + 1. Set the required environment variables (optional): ```sh - export API_KEY= + export API_KEY= # auto-created by apply.py if omitted export INFERENCE_SERVICE_URI= export PROTOCOL_VERSION=v2 ``` - You can find the inference service URL in the Kubeflow Endpoints UI. If you - don't know your API key, reach out to your prokube admin. + You can find the inference service URL in the Kubeflow Endpoints UI. 2. Run the test script: ```sh