From a8d79133b6225af8392baa9a7ef255b52b46a989 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 12:20:01 -0400 Subject: [PATCH 001/141] K0: exercise the kubeai backend against a real cluster A GPU-less check on k3s: KubeAI's cpu profile runs real vLLM on CPU, and kubeai_k3s.sh drives doctor, `run` (acquire, one chat completion, release) and checks the released Model is pruned. First real run of the backend since it landed: it passed unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/e2e_tests/kubeai-cpu-values.yaml | 9 +++ dev/e2e_tests/kubeai_k3s.sh | 64 +++++++++++++++++++ .../plan-backend-unification-2026-09-24.md | 13 +++- 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 dev/e2e_tests/kubeai-cpu-values.yaml create mode 100755 dev/e2e_tests/kubeai_k3s.sh diff --git a/dev/e2e_tests/kubeai-cpu-values.yaml b/dev/e2e_tests/kubeai-cpu-values.yaml new file mode 100644 index 00000000..20300543 --- /dev/null +++ b/dev/e2e_tests/kubeai-cpu-values.yaml @@ -0,0 +1,9 @@ +# KubeAI on a GPU-less machine, for exercising the kubeai backend for real. +# The chart's `cpu` profile runs vLLM's CPU image (needs AVX-512). Sized for +# a sub-1B model; raise the requests for anything larger. +resourceProfiles: + cpu: + imageName: "cpu" + requests: + cpu: 8 + memory: "16Gi" diff --git a/dev/e2e_tests/kubeai_k3s.sh b/dev/e2e_tests/kubeai_k3s.sh new file mode 100755 index 00000000..e80d9c5f --- /dev/null +++ b/dev/e2e_tests/kubeai_k3s.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# End-to-end check of the kubeai backend against a real (GPU-less) cluster. +# +# Needs: k3s (or any cluster) with KubeAI installed from +# dev/e2e_tests/kubeai-cpu-values.yaml, KUBECONFIG pointing at it, and the +# KubeAI gateway reachable at KUBEAI_BASE_URL (default: a port-forward, +# `kubectl -n kubeai port-forward svc/kubeai 8000:80`). The CPU vLLM image +# needs AVX-512. +# +# Runs in a throwaway config/data dir, so it never touches a real setup: +# bash dev/e2e_tests/kubeai_k3s.sh +set -euo pipefail + +WORK=${WORK:-$(mktemp -d -t infer-stack-kubeai-XXXX)} +export INFER_STACK_CONFIG_DIR=$WORK/cfg INFER_STACK_DATA_DIR=$WORK/data +MODEL=${MODEL:-Qwen/Qwen2.5-0.5B-Instruct} +ENDPOINT=${ENDPOINT:-qwen-tiny} +KUBEAI_BASE_URL=${KUBEAI_BASE_URL:-http://127.0.0.1:8000/openai/v1} +echo "[kubeai-e2e] work dir: $WORK" + +infer-stack config init --backend kubeai --yes >/dev/null +infer-stack config set kubeai_resource_profile cpu >/dev/null +infer-stack config set kubeai_base_url "$KUBEAI_BASE_URL" >/dev/null +cat > "$INFER_STACK_CONFIG_DIR/catalog.yaml" <&2 + exit 1 +fi +echo "[kubeai-e2e] PASS" diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 0c8b4e8f..24121cce 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -80,8 +80,17 @@ No GPU needed. Add `dev/e2e_tests/kubeai_kind.sh`: acquire, `run` a request, release, and verify the Model is pruned. Without it every step below is fake-verified only, which is how KubeAI drifted. -Open: installing `kind`/`kubectl`/`helm` on the guest needs network access to -their release downloads. +**Done 2026-09-24**, with k3s instead of kind (k3s runs as a systemd service +on the guest; `--disable traefik --disable servicelb` keeps ports 80/443 +free). KubeAI 0.23.4's `cpu` profile runs real vLLM 0.11.2 on CPU (needs +AVX-512), which is better than the simulator: KubeAI builds vLLM's own +command line, which the simulator's CLI would reject. Setup: +`dev/e2e_tests/kubeai-cpu-values.yaml`; check: +`dev/e2e_tests/kubeai_k3s.sh`, which passed. `doctor` passed its four checks; +acquire → ready took 143 s with an image pull, the full `run` took 86 s with +the image cached, a real completion came back, and release pruned the Model. +A Docker container on the node reaches KubeAI's gateway at its cluster IP, +which K1 relies on. ### K1. One client contract: the LiteLLM gateway fronts both backends From 711048b28fb46e2f7cb13b27b2efd6a7282e04cf Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 12:39:14 -0400 Subject: [PATCH 002/141] K1: front the kubeai backend with the LiteLLM gateway On KubeAI a client had to name a model by its Model name, a DNS slug of the served name; cards send the endpoint alias and got HTTP 404 (verified on k3s). The kubeai backend now owns a gateway-only ComposeBackend (own state dir, project infer-stack-gateway) and feeds it a new generic `upstream` route row per alias: the KubeAI URL plus the Model name. Clients get the same base_url, managed key and alias as on compose; readiness is judged through the gateway by alias; `secrets rotate` works unchanged. The gateway reaches KubeAI at the Service's cluster IP by default (kubeai_gateway_upstream overrides). --no-litellm keeps direct access. dev/kubeai_e2e.sh (the existing script; a duplicate I had started is folded in) sends the alias as a card does, gains GATEWAY=0 and a CPU profile, and now fails on a failed generation: the old check sat in a `&&` list where `set -e` does not apply, so a 404 printed PASS. Verified on k3s + KubeAI 0.23.4 (real vLLM on CPU): GATEWAY=1 passes, GATEWAY=0 fails with the 404, rotation accepts the new key and rejects the old, and no Model is left behind either way. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 16 +++ dev/e2e_tests/kubeai_k3s.sh | 64 ----------- dev/kubeai_e2e.sh | 31 ++++-- .../plan-backend-unification-2026-09-24.md | 9 ++ docs/kubeai-backend.md | 37 +++++-- infer_stack/backends/kubeai.py | 83 +++++++++++++- infer_stack/cli/commands_leasing.py | 16 +++ infer_stack/cli/commands_meta.py | 4 + infer_stack/leasing/compose.py | 14 +++ tests/test_leasing_kubeai.py | 102 ++++++++++++++++++ 10 files changed, 291 insertions(+), 85 deletions(-) delete mode 100755 dev/e2e_tests/kubeai_k3s.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d9819b8..95c6bdd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,22 @@ scheduler with its own GPU accounting: a lease held outside it, manual or keep-warm, occupies a GPU the scheduler believes is free, and the job it places there cannot start. +### The kubeai backend puts the LiteLLM gateway in front of the cluster + +On the kubeai backend a client had to name a model by its KubeAI Model name +(a DNS slug of the served name), not the endpoint alias. Cards send the +alias, so a card that ran on the compose backend got HTTP 404 on a cluster +(verified on k3s). The kubeai backend now runs the same LiteLLM gateway, +routing each alias to its Model: one `OPENAI_BASE_URL`, the managed key, and +the alias as the model name on both backends. `secrets rotate` works on it. +`--no-litellm` keeps the old direct access. New setting: +`kubeai_gateway_upstream`, for a gateway that cannot reach the cluster +Service's IP. + +`dev/kubeai_e2e.sh` now sends the alias as a card does, and fails when the +request fails. Before, a failed generation fell through to PASS: the check +sat in a `&&` list, where `set -e` does not apply. + ### Custom container launches are catalog data, not recipes `runtime.serve_recipe` is gone. An endpoint whose image has its own launcher diff --git a/dev/e2e_tests/kubeai_k3s.sh b/dev/e2e_tests/kubeai_k3s.sh deleted file mode 100755 index e80d9c5f..00000000 --- a/dev/e2e_tests/kubeai_k3s.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# End-to-end check of the kubeai backend against a real (GPU-less) cluster. -# -# Needs: k3s (or any cluster) with KubeAI installed from -# dev/e2e_tests/kubeai-cpu-values.yaml, KUBECONFIG pointing at it, and the -# KubeAI gateway reachable at KUBEAI_BASE_URL (default: a port-forward, -# `kubectl -n kubeai port-forward svc/kubeai 8000:80`). The CPU vLLM image -# needs AVX-512. -# -# Runs in a throwaway config/data dir, so it never touches a real setup: -# bash dev/e2e_tests/kubeai_k3s.sh -set -euo pipefail - -WORK=${WORK:-$(mktemp -d -t infer-stack-kubeai-XXXX)} -export INFER_STACK_CONFIG_DIR=$WORK/cfg INFER_STACK_DATA_DIR=$WORK/data -MODEL=${MODEL:-Qwen/Qwen2.5-0.5B-Instruct} -ENDPOINT=${ENDPOINT:-qwen-tiny} -KUBEAI_BASE_URL=${KUBEAI_BASE_URL:-http://127.0.0.1:8000/openai/v1} -echo "[kubeai-e2e] work dir: $WORK" - -infer-stack config init --backend kubeai --yes >/dev/null -infer-stack config set kubeai_resource_profile cpu >/dev/null -infer-stack config set kubeai_base_url "$KUBEAI_BASE_URL" >/dev/null -cat > "$INFER_STACK_CONFIG_DIR/catalog.yaml" <&2 - exit 1 -fi -echo "[kubeai-e2e] PASS" diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 8d21d1f7..cec829dd 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -8,6 +8,10 @@ # ./scripts/install_kubeai.sh /tmp/kubeai-values.yaml kubeai # kubectl -n kubeai port-forward svc/kubeai 8000:80 & # +# Without a GPU (verified on k3s): install the chart with +# dev/e2e_tests/kubeai-cpu-values.yaml, whose `cpu` profile runs real vLLM on +# CPU (needs AVX-512), and run with E2E_RESOURCE_PROFILE=cpu. +# # Then: ./dev/kubeai_e2e.sh # # Knobs (env): @@ -17,6 +21,9 @@ # E2E_BASE_URL gateway url (default http://127.0.0.1:8000/openai/v1) # E2E_TIMEOUT acquire readiness budget seconds (default 900 — # first run pulls model weights into the cluster) +# GATEWAY 1 (default): clients go through infer-stack's LiteLLM +# gateway, as on the compose backend. 0: straight to +# KubeAI, where the alias below does NOT route (404). set -euo pipefail MODEL="${E2E_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" @@ -24,6 +31,7 @@ PROFILE="${E2E_RESOURCE_PROFILE:-nvidia-gpu}" NAMESPACE="${E2E_NAMESPACE:-kubeai}" BASE_URL="${E2E_BASE_URL:-http://127.0.0.1:8000/openai/v1}" TIMEOUT="${E2E_TIMEOUT:-900}" +ALIAS="$MODEL" # Isolated config/data roots, baked INLINE on every command (never rely on # exported env reaching subprocesses — see dev/audit notes on tmux env loss). @@ -56,7 +64,8 @@ models: e2e-tiny: source: hf://$MODEL endpoints: - e2e-tiny: + # Named like real catalogs, unlike its KubeAI Model name: a card sends this. + $ALIAS: engine: vllm model: e2e-tiny reclaim: {policy: stop} @@ -68,23 +77,31 @@ EOF run_is config set backend kubeai run_is config set kubeai_namespace "$NAMESPACE" run_is config set kubeai_base_url "$BASE_URL" +if [ "${GATEWAY:-1}" = 0 ]; then run_is config set litellm false; fi echo '== doctor (preflight)' run_is doctor echo '== acquire (readiness = a real generation through the gateway)' -run_is acquire e2e-tiny --yes --ttl 30m --timeout "$TIMEOUT" \ +run_is acquire "$ALIAS" --yes --ttl 30m --timeout "$TIMEOUT" \ --env-file "$WORK/lease.env" -echo '== the descriptor works for a plain OpenAI client' +echo '== a client that knows only the env file and the alias (like a card)' # shellcheck disable=SC1090 source "$WORK/lease.env" -REQUEST_NAME="$INFER_STACK_ENDPOINT_E2E_TINY" -curl -sf "$OPENAI_BASE_URL/chat/completions" \ +# An explicit `if`: in a `curl | grep && echo` list `set -e` does not fire, so +# a 404 used to fall through to PASS. +if curl -sS --fail-with-body "$OPENAI_BASE_URL/chat/completions" \ + -H "Authorization: Bearer ${OPENAI_API_KEY:-EMPTY}" \ -H 'Content-Type: application/json' \ - -d "{\"model\": \"$REQUEST_NAME\", \"max_tokens\": 8, + -d "{\"model\": \"$ALIAS\", \"max_tokens\": 8, \"messages\": [{\"role\": \"user\", \"content\": \"say ok\"}]}" \ - | grep -q 'choices' && echo ' generation ok' + | grep -q 'choices'; then + echo " generation ok via $OPENAI_BASE_URL" +else + echo "!! no generation for model=$ALIAS via $OPENAI_BASE_URL" >&2 + exit 1 +fi echo '== release prunes the Model (reclaim: stop)' run_is release --env-file "$WORK/lease.env" --yes diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 24121cce..85b63c4b 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -115,6 +115,15 @@ Recommendation: (1) first, since it reuses everything and unblocks cards now; (2) when a real multi-node run needs it. Record (1)'s single host as a known limitation. +**Done 2026-09-24 as (1).** The kubeai backend owns a gateway-only +`ComposeBackend` (its own state dir and project, `infer-stack-gateway`) and +feeds it one generic `upstream` route row per alias (`api_base` + the Model +name). The same row type is what external endpoints need. Verified on k3s +with `dev/kubeai_e2e.sh`: without the gateway, the alias returned 404; with +it, the same request answered, and `secrets rotate` accepted the new key and +rejected the old one. Static routes only; dynamic routing stays +compose-only. + ### K2. One liveness authority Generalise `residency()` into a backend-neutral snapshot: per deployment, diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 374f1aa5..9b068555 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -10,10 +10,19 @@ only the realization layer changes: |---|---|---| | unit of serving | docker compose service | KubeAI `Model` CR | | GPU placement | planned locally (`plan_placement`) | cluster-scheduled via `resourceProfile` | -| front door | LiteLLM gateway | KubeAI's own OpenAI-compatible gateway | -| request name | endpoint alias (LiteLLM route) | Model CR name (dns-slug of the served name) | -| auth | managed `LITELLM_MASTER_KEY` | none (`api_key: EMPTY`) | -| state dir | `/leasing/compose/` | `/leasing/kubeai/` (`models.yaml` + sidecar) | +| front door | LiteLLM gateway | the same LiteLLM gateway, on this host, routing to KubeAI | +| request name | endpoint alias | endpoint alias | +| auth | managed `LITELLM_MASTER_KEY` | the same | +| state dir | `/leasing/compose/` | `/leasing/kubeai/` (`models.yaml` + sidecar), `/leasing/kubeai-gateway/` | + +Clients see the same contract on both backends: one `OPENAI_BASE_URL`, the +managed key, and the endpoint alias as the model name, so a card runs +unchanged. The gateway is a one-service Compose project +(`infer-stack-gateway`) that routes each alias to KubeAI under the Model's +name. `secrets rotate` works as on compose. With `--no-litellm` (or `config +set litellm false`) there is no gateway: clients talk to KubeAI directly and +must use the Model name from the env file (`INFER_STACK_ENDPOINT_*`); the +alias gets HTTP 404. A deployment in the desired set renders as one `Model` CR labeled `infer-stack/managed=true` + `infer-stack/deployment=`; `apply` is @@ -54,6 +63,9 @@ infer-stack config set kubeai_namespace kubeai infer-stack config set kubeai_base_url http://127.0.0.1:8000/openai/v1 # fallback profile for endpoints whose runtime omits resource_profile: infer-stack config set kubeai_resource_profile nvidia-gpu-rtx-4090 +# how the gateway reaches KubeAI (default: the kubeai Service's cluster IP, +# reachable from a cluster node; set an ingress URL when this host is not one): +infer-stack config set kubeai_gateway_upstream http://kubeai.example/openai/v1 ``` Catalog endpoints opt into a specific profile per endpoint; the GPU count is @@ -82,9 +94,8 @@ Then the normal verbs just work: ```bash infer-stack acquire qwen-coder --ttl 2h --env-file lease.env --yes -source lease.env # OPENAI_BASE_URL -> the KubeAI gateway -# note: request the model by its CR name (the env-file's endpoint mapping -# carries it), not the endpoint alias — KubeAI has no alias layer. +source lease.env # OPENAI_BASE_URL + OPENAI_API_KEY -> the gateway +# request model=qwen-coder, the endpoint alias, as on the compose backend infer-stack release --env-file lease.env ``` @@ -105,6 +116,12 @@ infer-stack release --env-file lease.env (default 1/1 — the lease lifecycle, not the autoscaler, decides residency). - The legacy profile-era renderer (`infer_stack/backends/kubeai_renderer.py`, `kubeai_ops.py`) is superseded by this backend and kept only for reference. -- `dev/kubeai_e2e.sh` runs the full lifecycle (doctor → acquire → generation - through the gateway → release → prune-verified) against a real cluster with - an isolated config/data root; use it as the first smoke test on new setups. +- `dev/kubeai_e2e.sh` runs the full lifecycle (doctor → acquire → a request + by alias, as a card makes it → release → prune verified) against a real + cluster with an isolated config/data root; use it as the first smoke test + on new setups. Without a GPU, install the chart with + `dev/e2e_tests/kubeai-cpu-values.yaml` (real vLLM on CPU; needs AVX-512) + and run it with `E2E_RESOURCE_PROFILE=cpu`. Verified on k3s 2026-09-24. +- The gateway runs on the host running infer-stack, so that host is in every + request's path. Dynamic routing (`dynamic_routing`) is compose-only; the + KubeAI gateway uses static routes. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 88e5c348..c407d9b5 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -268,6 +268,8 @@ def __init__( run: Callable[[list[str]], str] | None = None, http: Any = None, assume_yes: bool = True, + gateway: Any = None, + gateway_upstream: str | None = None, ): self.state_dir = Path(state_dir) self.state_dir.mkdir(parents=True, exist_ok=True) @@ -285,6 +287,15 @@ def __init__( self.last_unplaced: set[str] = set() # KubeAI/k8s schedules; there are no host GPU indices to report. self.last_assignments: dict[str, list[int]] = {} + # The LiteLLM gateway in front of the cluster: a gateway-only + # ComposeBackend. With it, clients use the same front door, key and + # request names (the endpoint aliases) as on the compose backend. + # Without it, clients talk to KubeAI directly under its Model names. + self.gateway = gateway + # How that gateway reaches KubeAI. Unset: the KubeAI Service's cluster + # IP, which containers on a cluster node can reach; a gateway off the + # cluster needs an ingress URL here. + self.gateway_upstream = gateway_upstream # -- state-dir plumbing -------------------------------------------------- @@ -327,6 +338,51 @@ def _cluster_models(self) -> dict[str, str | None]: models[name] = (meta.get('labels') or {}).get(DEPLOYMENT_LABEL) return models + # -- the gateway ----------------------------------------------------------- + + @property + def litellm(self) -> bool: + return self.gateway is not None + + @property + def litellm_port(self) -> int | None: + return self.gateway.litellm_port if self.gateway is not None else None + + def master_key(self) -> str: + return self.gateway.master_key() + + def rotate_master_key(self) -> dict[str, str]: + return self.gateway.rotate_master_key() + + def restore_env(self, values) -> None: + self.gateway.restore_env(values) + + def gateway_accepts(self, key: str, *, wait: float = 0.0): + return self.gateway.gateway_accepts(key, wait=wait) + + def _upstream_url(self) -> str: + """Where the gateway sends requests for this cluster's Models.""" + if not self.gateway_upstream: + ip = self._kubectl(['get', 'service', 'kubeai', '-o', + 'jsonpath={.spec.clusterIP}']).strip() + if not ip: + raise RuntimeError('the kubeai Service has no cluster IP; set ' + '`config set kubeai_gateway_upstream `') + self.gateway_upstream = f'http://{ip}/openai/v1' + return self.gateway_upstream.rstrip('/') + + def _render_gateway(self, rendered: RenderedModels) -> None: + """Route each endpoint alias through the gateway to its Model.""" + from ..leasing.compose import UPSTREAM_ROUTE + + base = self._upstream_url() + self.gateway.upstream_routes = { + alias: {'engine': UPSTREAM_ROUTE, 'served': name, 'api_base': base} + for alias, name in rendered.request_names.items() + } + # Gateway only: no engines on this host, so nothing to place. + self.gateway.converge([], apply=False) + # -- converge-style surface ------------------------------------------------ def converge(self, desired: list[Deployment], *, apply: bool = True): @@ -371,6 +427,8 @@ def converge(self, desired: list[Deployment], *, apply: bool = True): 'request_names': rendered.request_names, } ) + if self.gateway is not None: + self._render_gateway(rendered) if not apply: logger.info( 'rendered {} Model(s) to {} (not applied; ' @@ -395,6 +453,8 @@ def render_profile(self) -> dict: 'namespace': self.namespace, 'base_url': self.base_url, 'resource_profile': self.default_resource_profile, + 'gateway': self.gateway is not None, + 'gateway_upstream': self.gateway_upstream, 'catalogs': catalog_sources(self.catalog), } @@ -416,6 +476,7 @@ def use_profile(self, profile: dict) -> None: self.namespace = profile['namespace'] self.base_url = profile['base_url'] self.default_resource_profile = profile['resource_profile'] + self.gateway_upstream = profile.get('gateway_upstream') or self.gateway_upstream sources = profile.get('catalogs') or [] self.catalog = CatalogUnion.from_sources(sources) if sources else None @@ -457,6 +518,8 @@ def apply(self) -> None: self._kubectl( ['delete', 'models.kubeai.org', name, '--ignore-not-found'] ) + if self.gateway is not None: + self.gateway.apply() def observe(self) -> set[str]: """Deployment ids with a managed Model CR on the cluster (best-effort).""" @@ -475,12 +538,20 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: """ if deployment.id not in self.observe(): return Readiness(False, 'Model CR not on the cluster') - name = model_name_for(_served_name(deployment)) served = deployment.served.get(endpoint) or {} protocol = served.get('protocol') or 'chat' + if self.gateway is not None: + # Ready means ready the way a client sees it: the alias, through + # the gateway, with its key. + base, model = f'{self.gateway._gateway_base()}/v1', endpoint + headers = self.gateway._auth_headers() + else: + base, model = self.base_url, model_name_for(_served_name(deployment)) + headers = None ok, reason = openai_ready( - base_url=self.base_url, - model=name, + base_url=base, + headers=headers, + model=model, protocol=protocol, require_listed=True, require_generation=True, @@ -496,6 +567,8 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: no alias layer. The gateway is unauthenticated, so the api key is the literal ``EMPTY`` placeholder and no key env var is advertised. """ + if self.gateway is not None: + return self.gateway.access(endpoints) request_names = self._load_sidecar().get('request_names') or {} return { 'base_url': self.base_url, @@ -523,11 +596,13 @@ def teardown(self, deployment: Deployment) -> None: ) def down(self) -> None: - """Delete every infer-stack-managed Model (explicit stop).""" + """Delete every infer-stack-managed Model (explicit stop), and the gateway.""" for name in sorted(self._cluster_models()): self._kubectl( ['delete', 'models.kubeai.org', name, '--ignore-not-found'] ) + if self.gateway is not None: + self.gateway.down() # -- preflight ------------------------------------------------------------- diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index e3f66355..8167ad50 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -276,6 +276,20 @@ def _make_backend(config, *, interactive: bool = False): if name == 'kubeai': from ..backends.kubeai import KubeaiBackend + gateway = None + if _resolve_litellm(config): + # The same LiteLLM front door as the compose backend, fronting the + # cluster: one base_url, the managed key, and endpoint aliases as + # request names. Its own state dir and compose project, so it can + # never touch a compose stack's containers on the same host. + gateway = ComposeBackend( + state_dir=data_root() / 'leasing' / 'kubeai-gateway', + inventory={'gpu_count': 0, 'gpus': []}, + project='infer-stack-gateway', + litellm=True, + ui=False, + assume_yes=_resolve_assume_yes(config, interactive=interactive), + ) backend = KubeaiBackend( state_dir=data_root() / 'leasing' / 'kubeai', namespace=get_setting('kubeai_namespace') or 'kubeai', @@ -283,6 +297,8 @@ def _make_backend(config, *, interactive: bool = False): default_resource_profile=get_setting('kubeai_resource_profile') or None, assume_yes=_resolve_assume_yes(config, interactive=interactive), + gateway=gateway, + gateway_upstream=get_setting('kubeai_gateway_upstream') or None, ) try: backend.catalog = _load_catalog(config) # frozen into the profile diff --git a/infer_stack/cli/commands_meta.py b/infer_stack/cli/commands_meta.py index fa43990b..8d382139 100644 --- a/infer_stack/cli/commands_meta.py +++ b/infer_stack/cli/commands_meta.py @@ -362,6 +362,10 @@ class _Setting: 'kubeai_resource_profile': 'Fallback KubeAI resourceProfiles name for catalog endpoints whose ' 'runtime omits resource_profile.', + 'kubeai_gateway_upstream': + 'URL the LiteLLM gateway uses to reach KubeAI (default: the kubeai ' + "Service's cluster IP, reachable from a cluster node; set an ingress " + 'URL when the gateway runs off the cluster).', } diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 3fafe135..5a7b346f 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -97,6 +97,9 @@ # ComposeBackend._update_route_registry. LITELLM_REGISTRY_FILENAME = 'litellm_registry.json' LITELLM_REGISTRY_VERSION = 1 +# A route-registry row for a server this project does not run: +# ``{'engine': UPSTREAM_ROUTE, 'served': , 'api_base': }``. +UPSTREAM_ROUTE = 'upstream' POSTGRES_SERVICE = 'postgres-litellm' POSTGRES_CONTAINER_PORT = 5432 POSTGRES_DB_NAME = 'litellm' @@ -928,6 +931,12 @@ def _litellm_model_list_from_registry( f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' ) entries.append(_ollama_route_entry(name, tag, api_base)) + elif engine == UPSTREAM_ROUTE and row.get('api_base'): + # An OpenAI-compatible server this project does not run (a KubeAI + # cluster's gateway): the row carries its address and the name it + # serves the model under. + entries.append(_vllm_route_entry(name, row.get('served') or name, + str(row['api_base']))) return entries @@ -1914,6 +1923,9 @@ def __init__( # Called with a one-line message during long steps (image pulls); the # TUI sets it. Always also logged. self.progress: Callable[[str], None] | None = None + # alias -> UPSTREAM_ROUTE row, set by a backend that uses this one only + # as its gateway (the kubeai backend). Merged into the route registry. + self.upstream_routes: dict[str, dict[str, Any]] = {} self.images = {**PINNED_IMAGES, **(images or {})} self.ports = {**DEFAULT_PORTS, **(ports or {})} # Merge over the defaults (not replace) so a caller-supplied partial @@ -2608,6 +2620,8 @@ def _merged_route_registry( # live cross-runbook deployment routable (and, via persistence, routable # past release). incoming.update(_registry_incoming_from_deployments(desired, assignments)) + # Routes to servers another backend runs (see UPSTREAM_ROUTE). + incoming.update(self.upstream_routes) merged, warnings = _merge_route_registry(existing, incoming) for w in warnings: logger.warning(' route registry: {}', w) diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 899c7875..f5414eb4 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -532,3 +532,105 @@ def test_doctor_cli_backends_without_preflight(monkeypatch, capsys): ) assert DoctorCLI.main(argv=[]) == 0 assert 'nothing to verify' in capsys.readouterr().out + + +# -- the LiteLLM gateway in front of the cluster (plan step K1) --------------- +# +# Real-cluster evidence: dev/e2e_tests/kubeai_k3s.sh. Without the gateway a +# card's alias got HTTP 404 from KubeAI; with it the same request answered. + + +UPSTREAM = 'http://10.43.0.9/openai/v1' + + +class GatewayHttp(FakeHttp): + """The LiteLLM gateway on 127.0.0.1, routing by its rendered model_list.""" + + def __init__(self, kubectl, gateway_dir): + super().__init__(kubectl) + self.gateway_dir = Path(gateway_dir) + + def _routes(self): + cfg = self.gateway_dir / 'litellm_config.yaml' + doc = yaml.safe_load(cfg.read_text()) if cfg.exists() else {} + return {e['model_name']: e['litellm_params'] for e in doc.get('model_list') or []} + + def get(self, url, **kw): + if url.startswith('http://127.0.0.1:') and url.endswith('/models'): + return self._Resp(200, {'data': [{'id': a} for a in self._routes()]}) + return super().get(url, **kw) + + def post(self, url, **kw): + if url.startswith('http://127.0.0.1:'): + route = self._routes().get((kw.get('json') or {}).get('model')) + if route is None or route['api_base'] != UPSTREAM: + return self._Resp(404, {'detail': 'no such alias'}) + kw = {**kw, 'json': {**kw['json'], 'model': route['model'].split('/', 1)[1]}} + return super().post(url, **kw) + + +def make_gateway_backend(tmp_path): + from infer_stack.leasing.compose import ComposeBackend + from test_leasing_compose import FakeDocker + + kubectl = FakeKubectl() + gateway = ComposeBackend( + state_dir=tmp_path / 'gateway', inventory={'gpu_count': 0, 'gpus': []}, + run=FakeDocker(), http=GatewayHttp(kubectl, tmp_path / 'gateway'), + project='infer-stack-gateway', litellm=True, ui=False, + images={'litellm': 'litellm:test'}, + ) + be = KubeaiBackend(state_dir=tmp_path / 'kubeai', run=kubectl, + http=gateway.http, gateway=gateway, gateway_upstream=UPSTREAM) + return be, kubectl + + +def test_gateway_routes_each_alias_to_its_model(tmp_path): + be, _ = make_gateway_backend(tmp_path) + dep = vllm('grp-a', served='Qwen/Qwen2.5-0.5B') + dep.served = {'Qwen/Qwen2.5-0.5B': {'served_model_name': 'Qwen/Qwen2.5-0.5B', + 'protocol': 'chat'}} + be.converge([dep]) + route = GatewayHttp._routes(be.gateway.http)['Qwen/Qwen2.5-0.5B'] + assert route == {'model': 'openai/qwen-qwen2-5-0-5b', 'api_base': UPSTREAM, + 'api_key': 'EMPTY'} + + +def test_with_a_gateway_clients_use_the_alias_and_the_managed_key(tmp_path): + be, _ = make_gateway_backend(tmp_path) + dep = vllm('grp-a', served='Qwen/Qwen2.5-0.5B') + dep.served = {'tiny': {'served_model_name': 'Qwen/Qwen2.5-0.5B', 'protocol': 'chat'}} + be.converge([dep]) + info = be.access(['tiny']) + assert info['request_names'] == {'tiny': 'tiny'} # the alias, as on compose + assert info['base_url'].startswith('http://127.0.0.1:') + assert info['api_key'] == be.gateway.master_key() + # Ready is judged the way a client sees it: the alias, through the gateway. + assert be.probe_ready(dep, 'tiny').ready + + +def test_without_a_gateway_clients_still_see_kubeai_directly(tmp_path): + be, _ = make_backend(tmp_path) + be.converge([vllm('grp-a', served='qwen-32b')]) + assert be.litellm is False + assert be.access(['grp-a'])['request_names'] == {'grp-a': 'qwen-32b'} + + +def test_rotating_the_key_recreates_the_gateway_in_front_of_the_cluster(tmp_path): + from infer_stack.leasing.residency import FINGERPRINT_LABEL + + be, _ = make_gateway_backend(tmp_path) + ledger = Ledger(SqliteStore(str(tmp_path / 'ledger.db'))) + ctl = Controller(ledger, be) + out = ctl.acquire('alice', [_req('qwen', profile='cpu')], wait=False) + ctl.release(out.lease.id) + old = be.master_key() + + def fingerprint(): + doc = yaml.safe_load(be.gateway.compose_file.read_text()) + return doc['services']['litellm']['labels'][FINGERPRINT_LABEL] + + before = fingerprint() + ctl.rotate_gateway_key() + assert be.master_key() != old + assert fingerprint() != before From b7bc3e8ca78ef3ac4bdae0960f4d713ebdcd9d36 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 12:40:50 -0400 Subject: [PATCH 003/141] Remove the unused profile-era KubeAI renderer render_kubeai_artifacts had no callers once the profile path was excised; it was a second KubeAI renderer beside the backend. The doc claimed it and a kubeai_ops.py (already gone) were kept for reference. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 4 + docs/kubeai-backend.md | 2 - infer_stack/backends/__init__.py | 3 +- infer_stack/backends/kubeai_renderer.py | 204 ------------------------ 4 files changed, 5 insertions(+), 208 deletions(-) delete mode 100644 infer_stack/backends/kubeai_renderer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c6bdd0..bac4ff49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ the alias as the model name on both backends. `secrets rotate` works on it. `kubeai_gateway_upstream`, for a gateway that cannot reach the cluster Service's IP. +The profile-era KubeAI renderer (`infer_stack.backends.render_kubeai_artifacts`) +is removed. Nothing called it since the profile path was excised; the kubeai +backend is the one renderer for KubeAI. + `dev/kubeai_e2e.sh` now sends the alias as a card does, and fails when the request fails. Before, a failed generation fell through to PASS: the check sat in a `&&` list, where `set -e` does not apply. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 9b068555..b6255345 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -114,8 +114,6 @@ infer-stack release --env-file lease.env `--backend compose` for those. - `min_replicas` / `max_replicas` runtime keys pass through to the CR (default 1/1 — the lease lifecycle, not the autoscaler, decides residency). -- The legacy profile-era renderer (`infer_stack/backends/kubeai_renderer.py`, - `kubeai_ops.py`) is superseded by this backend and kept only for reference. - `dev/kubeai_e2e.sh` runs the full lifecycle (doctor → acquire → a request by alias, as a card makes it → release → prune verified) against a real cluster with an isolated config/data root; use it as the first smoke test diff --git a/infer_stack/backends/__init__.py b/infer_stack/backends/__init__.py index 065b27bc..aabed088 100644 --- a/infer_stack/backends/__init__.py +++ b/infer_stack/backends/__init__.py @@ -1,4 +1,3 @@ from .kubeai import KubeaiBackend -from .kubeai_renderer import render_kubeai_artifacts # legacy profile-era path -__all__ = ['KubeaiBackend', 'render_kubeai_artifacts'] +__all__ = ['KubeaiBackend'] diff --git a/infer_stack/backends/kubeai_renderer.py b/infer_stack/backends/kubeai_renderer.py deleted file mode 100644 index 21512310..00000000 --- a/infer_stack/backends/kubeai_renderer.py +++ /dev/null @@ -1,204 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import yaml - -from ..config import KUBEAI_GENERATED_SUBDIR, normalized_output -from ..diff_prompt import confirm_writes -from ..profile_runtime import vllm_args - - -def _resource_profile_values(plan: dict[str, Any]) -> dict[str, Any]: - values_doc = plan.get('deployment', {}).get('resource_profiles_values', {}) - return values_doc or {'resourceProfiles': {}} - - -def _kubeai_resource_profile(service: dict[str, Any]) -> str: - profile = str(service.get('resource_profile', '')) - if not profile or ':' in profile: - return profile - gpu_count = max( - 1, - len(service.get('gpu_indices', [])), - int(service.get('tensor_parallel_size', 1) or 1) - * int(service.get('pipeline_parallel_size', 1) or 1) - * int(service.get('data_parallel_size', 1) or 1), - ) - return f'{profile}:{gpu_count}' - - -def _kubeai_args(service: dict[str, Any]) -> list[str]: - kubeai_service = dict(service) - kubeai_service['served_model_name'] = service['profile_public_name'] - return vllm_args(kubeai_service) - - -def _model_doc(service: dict[str, Any]) -> dict[str, Any]: - doc = { - 'apiVersion': 'kubeai.org/v1', - 'kind': 'Model', - 'metadata': { - 'name': service['kubernetes_name'], - 'annotations': { - 'infer-stack/profile-name': service['profile_name'], - 'infer-stack/public-name': service['profile_public_name'], - 'infer-stack/logical-model-name': service['logical_model_name'], - 'infer-stack/protocol-mode': service['protocol_mode'], - }, - }, - 'spec': { - 'features': service.get('features', ['TextGeneration']), - 'url': service['model_url'], - 'engine': service.get('engine', 'VLLM'), - 'resourceProfile': _kubeai_resource_profile(service), - 'minReplicas': int(service.get('min_replicas', 0)), - 'maxReplicas': int(service.get('max_replicas', 1)), - 'args': _kubeai_args(service), - }, - } - if service.get('priority_class_name'): - doc['spec']['priorityClassName'] = service['priority_class_name'] - return doc - - -def render_kubeai_artifacts( - lock_data: dict, *, assume_yes: bool = True -) -> None: - """Render the KubeAI backend artifacts. - - When ``assume_yes`` is False, all rendered YAML files are diffed against - their on-disk versions and the user is prompted via Rich before any file - is written. - """ - deployment = lock_data.get('deployment', {}) - cluster = deployment.get('cluster', {}) - namespace = cluster.get('namespace', 'kubeai') - output_root = Path( - normalized_output(deployment.get('output'))['generated_dir'] - ) - generated = output_root / KUBEAI_GENERATED_SUBDIR - generated.mkdir(parents=True, exist_ok=True) - - namespace_doc = { - 'apiVersion': 'v1', - 'kind': 'Namespace', - 'metadata': {'name': namespace}, - } - namespace_text = yaml.safe_dump(namespace_doc, sort_keys=False) - - values_doc = _resource_profile_values(lock_data) - values_text = yaml.safe_dump(values_doc, sort_keys=False) - - model_docs = [ - _model_doc(service) - for service in ( - deployment.get('providers', {}).get('vllm', {}).get('runtimes', {}) - or {} - ).values() - ] - model_text = '---\n'.join( - yaml.safe_dump(doc, sort_keys=False) for doc in model_docs - ) - - ingress = cluster.get('ingress', {}) or {} - ingress_path = generated / 'ingress.yaml' - ingress_text: str | None = None - if ingress.get('enabled'): - path_prefix = ingress.get('path_prefix', '/') or '/' - ingress_doc: dict[str, Any] = { - 'apiVersion': 'networking.k8s.io/v1', - 'kind': 'Ingress', - 'metadata': { - 'name': cluster.get('service_name', 'kubeai'), - 'namespace': namespace, - }, - 'spec': { - 'ingressClassName': ingress.get('class_name', 'traefik'), - 'rules': [ - { - 'http': { - 'paths': [ - { - 'path': path_prefix, - 'pathType': 'Prefix', - 'backend': { - 'service': { - 'name': cluster.get( - 'service_name', 'kubeai' - ), - 'port': {'number': 80}, - } - }, - } - ] - } - } - ], - }, - } - if ingress.get('host'): - ingress_doc['spec']['rules'][0]['host'] = ingress['host'] - if ingress.get('tls_secret_name') and ingress.get('host'): - ingress_doc['spec']['tls'] = [ - { - 'hosts': [ingress['host']], - 'secretName': ingress['tls_secret_name'], - } - ] - ingress_text = yaml.safe_dump(ingress_doc, sort_keys=False) - - readme = f"""# Generated KubeAI artifacts - -Namespace: `{namespace}` -Release: `{cluster.get('kubeai_release_name', 'kubeai')}` -Chart: `{cluster.get('kubeai_chart', 'kubeai/kubeai')}` - -Files: -- `namespace.yaml`: namespace to apply before the chart and models -- `kubeai-values.yaml`: custom resource profiles for the KubeAI chart -- `models.yaml`: KubeAI `Model` objects derived intentionally from the selected serving profile(s) -- `ingress.yaml`: optional ingress for one stable hostname - -Typical flow: - -```bash -kubectl apply -f {generated}/namespace.yaml -helm repo add kubeai https://www.kubeai.org --force-update -helm repo update -helm upgrade --install {cluster.get('kubeai_release_name', 'kubeai')} {cluster.get('kubeai_chart', 'kubeai/kubeai')} \ - -n {namespace} --create-namespace \ - -f {generated}/kubeai-values.yaml \ - --wait -kubectl apply -f {generated}/models.yaml -``` -""" - - namespace_path = generated / 'namespace.yaml' - values_path = generated / 'kubeai-values.yaml' - models_path = generated / 'models.yaml' - readme_path = generated / 'README.md' - - planned: dict[Path, str] = { - namespace_path: namespace_text, - values_path: values_text, - models_path: model_text, - readme_path: readme, - } - if ingress_text is not None: - planned[ingress_path] = ingress_text - - if not confirm_writes( - planned, assume_yes=assume_yes, title='Pending KubeAI render' - ): - raise SystemExit('Aborted by user; no files were written.') - - namespace_path.write_text(namespace_text, encoding='utf-8') - values_path.write_text(values_text, encoding='utf-8') - models_path.write_text(model_text, encoding='utf-8') - readme_path.write_text(readme, encoding='utf-8') - if ingress_text is not None: - ingress_path.write_text(ingress_text, encoding='utf-8') - elif ingress_path.exists(): - ingress_path.unlink() From e84c131e8f04c65730625105a4e3cfbca90197fb Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 12:51:40 -0400 Subject: [PATCH 004/141] K2+K4: strict pod residency and shared crash diagnosis for kubeai K2: KubeaiBackend.residency() builds the same Residency as compose from `kubectl get pods` (KubeAI copies the Model's infer-stack labels onto its pods), mapping pod states onto the Docker vocabulary and keeping the Kubernetes reason; a kubectl failure raises ResidencyUnknown. `gc --orphans` and `network migrate` stop using "has residency" to mean compose. K4: the crash diagnosis moves to leasing/diagnosis.py (diagnose_startup(instances, read_logs)); compose re-exports the old names. The kubeai backend feeds it pods and pod logs (the previous run last, where the crash is), probes fail fast with the engine's words, and a not-ready wait names the pod reason (Unschedulable, ImagePullBackOff). `error: unrecognized arguments` is now a fatal signature on both backends. Verified on k3s + KubeAI: a Model whose vLLM rejected a flag failed its acquire in 52 s (timeout 800 s), quoting the error and its likely cause; live residency read running, starting and crashed pods correctly. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 16 ++ .../plan-backend-unification-2026-09-24.md | 30 +++- docs/kubeai-backend.md | 6 + infer_stack/backends/kubeai.py | 75 ++++++++- infer_stack/cli/commands_leasing.py | 8 +- infer_stack/leasing/compose.py | 129 ++------------- infer_stack/leasing/diagnosis.py | 150 ++++++++++++++++++ infer_stack/leasing/residency.py | 97 +++++++++++ tests/test_leasing_kubeai.py | 108 +++++++++++++ tests/test_leasing_startup_failure.py | 2 + 10 files changed, 494 insertions(+), 127 deletions(-) create mode 100644 infer_stack/leasing/diagnosis.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bac4ff49..5781eaed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,22 @@ scheduler with its own GPU accounting: a lease held outside it, manual or keep-warm, occupies a GPU the scheduler believes is free, and the job it places there cannot start. +### The kubeai backend fails fast on an engine that cannot start + +The crash diagnosis (restart count, exit code, and the engine log classified +as fatal, transient or unknown) was Docker-only, so on a cluster a model that +could never start held its lease for the whole timeout. It now lives in +`infer_stack/leasing/diagnosis.py` and both backends use it. The kubeai +backend reads pods through a new strict `residency()` (a kubectl failure +raises instead of reading as "nothing running") and quotes the crashed run's +log (`kubectl logs --previous`). A not-ready wait names the pod's reason, such +as `Unschedulable` or `ImagePullBackOff`. vLLM rejecting a flag (`error: +unrecognized arguments`) is now recognised as fatal on both backends; before, +it waited for two restarts. + +`gc --orphans` and `network migrate` now refuse a non-compose backend +explicitly. They had used "the backend has `residency`" to mean compose. + ### The kubeai backend puts the LiteLLM gateway in front of the cluster On the kubeai backend a client had to name a model by its KubeAI Model name diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 85b63c4b..350eb231 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -38,12 +38,18 @@ path carries the keep-warm fix, atomic acquire and serialised publication review; the other path is the pre-September one. In production only KubeAI takes it, and nothing has run it at scale. -### C. Two liveness authorities +### C. No strict liveness view on KubeAI -Compose answers "what is running" with `residency()`: strict, and it raises -`ResidencyUnknown` rather than guess. KubeAI answers with `observe()`, which -returns the **empty set** when `kubectl` fails. Compose had that exact bug: -an unreadable Docker looked like "nothing running" and led to wrong decisions. +Both backends have a lenient `observe()` that returns an empty set when the +runtime cannot be read. That is deliberate: it is for reporting. Compose also +has `residency()`, a strict view that raises `ResidencyUnknown` rather than +guess, and every decision that stops, evicts or hands over a GPU uses it. +KubeAI had only the lenient view. (Corrected 2026-09-24: the first draft +called KubeAI's `observe()` a bug; it matches Compose's contract.) + +Three CLI commands (`gc --orphans`, its orphan listing, `network migrate`) +used "has `residency`" to mean "is the Compose backend", which is a duplicate +authority of its own. ### D. Failure diagnosis only exists on Compose @@ -132,6 +138,14 @@ reason, waiting reason, and GPUs when the backend knows them. Build KubeAI's from `kubectl get pods -l model= -o json`, raising `ResidencyUnknown` when kubectl fails. `observe()` becomes a thin view of it on both backends. +**Done 2026-09-24.** `residency_from_pods` builds the same `Residency` +from `kubectl get pods`, selecting on the `infer-stack/managed` label KubeAI +copies onto pods. Pod states map onto the Docker vocabulary, with the +Kubernetes reason kept in `Container.reason`, and a kubectl failure raises. +With it, `_never_ran`, the `config publish` quiescence check and the status +view use strict residency on KubeAI too. The three CLI commands now check for +the Compose backend explicitly. + ### K3. One acquire path Give placement to the backend: @@ -155,6 +169,12 @@ The fail-fast wait, the "likely cause" text and the TUI error path then work on both. Pull progress stays Compose-only; Kubernetes pulls by itself, and `ImagePullBackOff` is reported as a failure. +**Done 2026-09-24.** The diagnosis moved to `leasing/diagnosis.py` +(`diagnose_startup(instances, read_logs)`), and both backends call it. On k3s, +a Model whose vLLM rejected a flag failed its acquire in 52 s with the +engine's error quoted, instead of waiting out the 800 s timeout. Adding +`error: unrecognized arguments` as a fatal signature helps compose as well. + ### K5. Small cleanups - one served-name function diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index b6255345..df1f4e8f 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -107,6 +107,12 @@ infer-stack release --env-file lease.env - **Placement can only fail at admission.** The render never rejects for capacity (the cluster schedules); a Model the cluster cannot place sits not-ready until the acquire's `--timeout`, which then rolls the lease back. + The wait says why (`pod: Unschedulable`, `pod: ImagePullBackOff`). +- **An engine that cannot start fails the acquire at once**, as on compose: + the crash diagnosis reads the pods (`kubectl get pods`, restart count, last + exit) and the previous run's log, and quotes the engine's error with a + likely cause. Verified on k3s: a rejected vLLM flag failed in 52 s instead + of the 800 s timeout. Loud render failures do exist for: a missing `resource_profile` (no invalid CR is ever written), served-name collisions between simultaneously live deployments, and **ollama endpoints** — KubeAI serves models, not daemons, diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index c407d9b5..aa0e0a0e 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -521,6 +521,71 @@ def apply(self) -> None: if self.gateway is not None: self.gateway.apply() + def residency(self): + """The managed Models' pods, strictly: raises rather than guess. + + Unlike :meth:`observe`, a kubectl failure is never "nothing running"; + see :mod:`infer_stack.leasing.residency`. + """ + from ..leasing.residency import ( + POD_MANAGED_LABEL, + ResidencyUnknown, + residency_from_pods, + ) + + try: + raw = self._kubectl(['get', 'pods', '-l', f'{POD_MANAGED_LABEL}=true', + '-o', 'json']) + except Exception as ex: # noqa: BLE001 - any failure is "unknown" + raise ResidencyUnknown(f'kubectl get pods failed: {ex}') from ex + return residency_from_pods(raw) + + def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: + """Recent engine logs: each pod's current run, then its previous one. + + After a crash the kubelet has already restarted the container, so the + cause is in the previous run's log; it goes last, where the diagnosis + quotes from. Fail-open to ``''``. + """ + from ..leasing.residency import ResidencyUnknown + + try: + pods = self.residency().containers(deployment.id) + except ResidencyUnknown: + return '' + parts: list[str] = [] + for pod in pods: + runs = [[]] + ([['--previous']] if pod.restart_count else []) + for extra in runs: + try: + parts.append(self._kubectl(['logs', pod.container_id, '--tail', + str(tail), *extra])) + except Exception: # noqa: BLE001 - gone, or no previous run + pass + return '\n'.join(p for p in parts if p) + + def startup_failure(self, deployment: Deployment) -> str | None: + """Why this Model's engine cannot start, or ``None`` if it may still load.""" + from ..leasing.diagnosis import diagnose_startup + from ..leasing.residency import ResidencyUnknown + + try: + pods = self.residency().containers(deployment.id) + except ResidencyUnknown: + return None # cannot read the cluster: say nothing + return diagnose_startup(pods, lambda: self.deployment_logs(deployment, tail=200)) + + def _waiting_reason(self, deployment: Deployment) -> str: + """The pod's own reason for not running yet (``ImagePullBackOff``...).""" + from ..leasing.residency import ResidencyUnknown + + try: + pods = self.residency().containers(deployment.id) + except ResidencyUnknown: + return '' + reasons = sorted({p.reason for p in pods if p.reason and p.state != 'running'}) + return ', '.join(reasons) + def observe(self) -> set[str]: """Deployment ids with a managed Model CR on the cluster (best-effort).""" try: @@ -557,7 +622,15 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: require_generation=True, http=self.http, ) - return Readiness(ok, reason) + if ok: + return Readiness(True, reason) + # A crash-looping engine keeps its pod "Running" between restarts, so + # check every not-ready probe, not only when the Model is missing. + failure = self.startup_failure(deployment) + if failure is not None: + return Readiness(False, failure, fatal=True) + waiting = self._waiting_reason(deployment) + return Readiness(False, f'{reason} (pod: {waiting})' if waiting else reason) def access(self, endpoints: list[str]) -> dict[str, Any] | None: """Where a client reaches these endpoints (env-file descriptor). diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 8167ad50..56ad31d5 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1270,7 +1270,7 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config, interactive=True) if config.orphans: - if not callable(getattr(controller.backend, 'residency', None)): + if not isinstance(controller.backend, ComposeBackend): raise SystemExit('gc --orphans needs the compose backend') def confirm(found): @@ -1359,8 +1359,8 @@ def main(cls, argv=True, **kwargs): held = [g for g in deployments if g.state in (DeploymentState.LIVE, DeploymentState.IDLE)] observed, assignments = _placement_view(controller) - can_orphan = bool(config.orphans) and callable( - getattr(controller.backend, 'residency', None)) + can_orphan = bool(config.orphans) and isinstance( + controller.backend, ComposeBackend) found: list = [] @@ -2819,7 +2819,7 @@ def main(cls, argv=True, **kwargs): if not config.subnet: raise SystemExit('network migrate: --subnet is required') controller = _open_controller(config, interactive=True) - if not callable(getattr(controller.backend, 'residency', None)): + if not isinstance(controller.backend, ComposeBackend): raise SystemExit('network migrate needs the compose backend') try: rec = controller.network_migrate(config.subnet, force=bool(config.force)) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 5a7b346f..27306411 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -260,93 +260,17 @@ def profile_images(profile: dict[str, Any]) -> list[str]: return sorted(wanted) -#: Restarts after which Docker's own bookkeeping says an engine is looping, -#: not loading. Two is already conclusive: a model that loads does not exit. -CRASH_LOOP_RESTARTS = 2 - -#: Engine log lines worth quoting verbatim, and what they mean. Each is -#: UNRECOVERABLE: the same container, restarted, fails the same way, so one -#: crash is already conclusive and there is nothing to wait for. -_ENGINE_ERROR_HINTS = ( - ('trust_remote_code', 'the model needs trust_remote_code=True ' - '(set `runtime.trust_remote_code: true` on the endpoint)'), - ('does not recognize this architecture', 'this engine cannot read the ' - "model's architecture (it may need trust_remote_code=True)"), - ('are not supported for now', 'this vLLM build does not implement the ' - "model's architecture"), - ('is not supported', 'this vLLM build does not implement the ' - "model's architecture"), - ('No supported config format', 'the model repository has no config this ' - 'engine can read'), - ('ValidationError', 'the engine rejected its own configuration'), - ('401 Client Error', 'the model is gated: set HF_TOKEN with ' - '`infer-stack env HF_TOKEN=...`'), - ('403 Client Error', 'the model is gated: set HF_TOKEN with ' - '`infer-stack env HF_TOKEN=...`'), +# Crash diagnosis is backend-neutral (the kubeai backend uses it too); these +# names stay importable from here. +from .diagnosis import ( # noqa: E402,F401 + CRASH_LOOP_RESTARTS, + _ENGINE_ERROR_HINTS, + _TRANSIENT_ENGINE_SIGNATURES, + _engine_error_summary, + classify_engine_log, + diagnose_startup, ) -#: Failures a RESTART CAN FIX: the hub was unreachable, a download was cut off, -#: a port was still held by the container we just replaced. `restart: -#: unless-stopped` exists for exactly these, so a restart count alone must not -#: condemn an engine -- the whole point of the policy is that the next attempt -#: succeeds. Weigh a new entry by one question: would running the same container -#: again plausibly work? If yes it belongs here; if no it belongs above. -_TRANSIENT_ENGINE_SIGNATURES = ( - 'Max retries exceeded', - 'Connection reset by peer', - 'Connection refused', - 'Temporary failure in name resolution', - 'Failed to resolve', - 'Read timed out', - 'ReadTimeoutError', - 'ConnectionError', - 'IncompleteRead', - 'Consistency check failed', # a truncated HF download - '429 Client Error', # hub rate limit - '500 Server Error', - '502 Server Error', - '503 Server Error', - '504 Server Error', - 'Address already in use', -) - - -def classify_engine_log(logs: str) -> str | None: - """``'fatal'``, ``'transient'``, or ``None`` when the log says neither. - - Order matters: an unrecoverable signature wins over a transient one, because - a hub timeout earlier in the same log does not make a rejected config - loadable. CUDA OOM counts as fatal — the allocation is deterministic, so the - restart repeats it — and it is the one class with a documented remedy - (:mod:`infer_stack.leasing.vram`). - """ - from .vram import looks_like_cuda_oom - - if not logs: - return None - if any(needle in logs for needle, _ in _ENGINE_ERROR_HINTS): - return 'fatal' - if looks_like_cuda_oom(logs): - return 'fatal' - if any(needle in logs for needle in _TRANSIENT_ENGINE_SIGNATURES): - return 'transient' - return None - - -def _engine_error_summary(logs: str) -> str: - """The engine's own error, quoted, with a hint when we recognise it.""" - from .vram import looks_like_cuda_oom - - if not logs.strip(): - return '; no engine log available (`infer-stack logs` for more)' - hint = next((note for needle, note in _ENGINE_ERROR_HINTS if needle in logs), None) - if hint is None and looks_like_cuda_oom(logs): - hint = 'the GPU ran out of memory for this configuration' - lines = [line.strip() for line in logs.splitlines() if line.strip()] - quoted = ' | '.join(lines[-3:])[:400] - summary = f'; last log: {quoted}' - return f'{summary}; likely cause: {hint}' if hint else summary - def _network_name() -> str: from .network import NETWORK_NAME @@ -2493,39 +2417,10 @@ def startup_failure(self, deployment: Deployment) -> str | None: residency = self.residency() except ResidencyUnknown: return None # cannot read Docker: say nothing - containers = residency.containers(deployment.id) - if len(containers) != 1: - return None # absent (not created yet) or ambiguous - container = containers[0] - exited_for_good = ( - container.state in {'exited', 'dead'} - and (container.exit_code or 0) != 0 + return diagnose_startup( + residency.containers(deployment.id), + lambda: self.deployment_logs(deployment, tail=200), ) - crashed = exited_for_good or container.restart_count >= 1 - if not crashed: - return None # created, starting, or healthy - # The log decides, not the restart count alone. `restart: - # unless-stopped` is there so a hub timeout or a port still held by the - # container we replaced resolves itself; condemning those would make the - # policy pointless. An unrecoverable error, by contrast, repeats - # identically, so waiting for a second restart only wastes the GPU. - logs = self.deployment_logs(deployment, tail=200) - verdict = classify_engine_log(logs) - if verdict == 'transient' and container.will_be_restarted: - return None # a retry is coming, and may work - if verdict == 'transient': - # Transient, but nothing will run it again: waiting is as pointless - # as for an unrecoverable error, and the cause still belongs in the - # message. - return (f'engine is not starting (exited with code ' - f'{container.exit_code} and will not be restarted)' - f'{_engine_error_summary(logs)}') - looping = container.restart_count >= CRASH_LOOP_RESTARTS - if verdict != 'fatal' and not (exited_for_good or looping): - return None # unrecognised: keep today's budget - why = (f'restarted {container.restart_count} time(s)' if container.restart_count - else f'exited with code {container.exit_code}') - return f'engine is not starting ({why}){_engine_error_summary(logs)}' def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: """Recent engine logs for a deployment's compose service. diff --git a/infer_stack/leasing/diagnosis.py b/infer_stack/leasing/diagnosis.py new file mode 100644 index 00000000..7a7b8ac7 --- /dev/null +++ b/infer_stack/leasing/diagnosis.py @@ -0,0 +1,150 @@ +"""Why an engine is not starting: shared by every backend. + +A backend supplies the instances of one deployment (containers, or pods) as +:class:`~infer_stack.leasing.residency.Container` records, and a way to read +their recent logs. :func:`diagnose_startup` decides whether waiting could +still help, so the fail-fast wait, the "likely cause" text and the TUI's +error path behave the same on every backend. +""" + +from __future__ import annotations + +from typing import Callable, Sequence + +#: Restarts after which the runtime's own bookkeeping says an engine is +#: looping, not loading. Two is already conclusive: a model that loads does not exit. +CRASH_LOOP_RESTARTS = 2 + +#: Engine log lines worth quoting verbatim, and what they mean. Each is +#: UNRECOVERABLE: the same container, restarted, fails the same way, so one +#: crash is already conclusive and there is nothing to wait for. +_ENGINE_ERROR_HINTS = ( + ('trust_remote_code', 'the model needs trust_remote_code=True ' + '(set `runtime.trust_remote_code: true` on the endpoint)'), + ('does not recognize this architecture', 'this engine cannot read the ' + "model's architecture (it may need trust_remote_code=True)"), + ('are not supported for now', 'this vLLM build does not implement the ' + "model's architecture"), + ('is not supported', 'this vLLM build does not implement the ' + "model's architecture"), + ('No supported config format', 'the model repository has no config this ' + 'engine can read'), + ('ValidationError', 'the engine rejected its own configuration'), + ('error: unrecognized arguments', 'the engine rejected a command-line flag ' + '(check `runtime.extra_args` against this vLLM version)'), + ('401 Client Error', 'the model is gated: set HF_TOKEN with ' + '`infer-stack env HF_TOKEN=...`'), + ('403 Client Error', 'the model is gated: set HF_TOKEN with ' + '`infer-stack env HF_TOKEN=...`'), +) + +#: Failures a RESTART CAN FIX: the hub was unreachable, a download was cut off, +#: a port was still held by the container we just replaced. `restart: +#: unless-stopped` exists for exactly these, so a restart count alone must not +#: condemn an engine -- the whole point of the policy is that the next attempt +#: succeeds. Weigh a new entry by one question: would running the same container +#: again plausibly work? If yes it belongs here; if no it belongs above. +_TRANSIENT_ENGINE_SIGNATURES = ( + 'Max retries exceeded', + 'Connection reset by peer', + 'Connection refused', + 'Temporary failure in name resolution', + 'Failed to resolve', + 'Read timed out', + 'ReadTimeoutError', + 'ConnectionError', + 'IncompleteRead', + 'Consistency check failed', # a truncated HF download + '429 Client Error', # hub rate limit + '500 Server Error', + '502 Server Error', + '503 Server Error', + '504 Server Error', + 'Address already in use', +) + + +def classify_engine_log(logs: str) -> str | None: + """``'fatal'``, ``'transient'``, or ``None`` when the log says neither. + + Order matters: an unrecoverable signature wins over a transient one, because + a hub timeout earlier in the same log does not make a rejected config + loadable. CUDA OOM counts as fatal — the allocation is deterministic, so the + restart repeats it — and it is the one class with a documented remedy + (:mod:`infer_stack.leasing.vram`). + """ + from .vram import looks_like_cuda_oom + + if not logs: + return None + if any(needle in logs for needle, _ in _ENGINE_ERROR_HINTS): + return 'fatal' + if looks_like_cuda_oom(logs): + return 'fatal' + if any(needle in logs for needle in _TRANSIENT_ENGINE_SIGNATURES): + return 'transient' + return None + + +def _engine_error_summary(logs: str) -> str: + """The engine's own error, quoted, with a hint when we recognise it.""" + from .vram import looks_like_cuda_oom + + if not logs.strip(): + return '; no engine log available (`infer-stack logs` for more)' + hint = next((note for needle, note in _ENGINE_ERROR_HINTS if needle in logs), None) + if hint is None and looks_like_cuda_oom(logs): + hint = 'the GPU ran out of memory for this configuration' + lines = [line.strip() for line in logs.splitlines() if line.strip()] + quoted = ' | '.join(lines[-3:])[:400] + summary = f'; last log: {quoted}' + return f'{summary}; likely cause: {hint}' if hint else summary + + + + +def diagnose_startup(instances: Sequence, read_logs: Callable[[], str]) -> str | None: + """Diagnose an engine that cannot start, or ``None`` if it may still load. + + A restart policy makes an engine that exits immediately restart forever, + so a model that can never start looks exactly like one that is still + loading, and an acquire waits out its whole timeout. The runtime's + bookkeeping says an instance crashed; its LOG says whether another attempt + could ever work (:func:`classify_engine_log`): + + * an unrecoverable error -- a rejected config or flag, an architecture + this build does not implement, a gated repo, a CUDA OOM -- is fatal on + the FIRST crash, because the restart reproduces it exactly; + * a transient one -- an unreachable hub, a truncated download, a port + still held -- is never fatal while something will retry it; + * an unrecognised crash keeps the blunt budget of + :data:`CRASH_LOOP_RESTARTS` restarts. + + ``instances`` must be exactly one (absent or ambiguous: ``None``). The + returned string carries the engine's last words, because the cause is in + its log and nowhere else. + """ + if len(instances) != 1: + return None # absent (not created yet) or ambiguous + instance = instances[0] + exited_for_good = ( + instance.state in {'exited', 'dead'} + and (instance.exit_code or 0) != 0 + ) + crashed = exited_for_good or instance.restart_count >= 1 + if not crashed: + return None # created, starting, or healthy + logs = read_logs() + verdict = classify_engine_log(logs) + if verdict == 'transient' and instance.will_be_restarted: + return None # a retry is coming, and may work + if verdict == 'transient': + return (f'engine is not starting (exited with code ' + f'{instance.exit_code} and will not be restarted)' + f'{_engine_error_summary(logs)}') + looping = instance.restart_count >= CRASH_LOOP_RESTARTS + if verdict != 'fatal' and not (exited_for_good or looping): + return None # unrecognised: keep the restart budget + why = (f'restarted {instance.restart_count} time(s)' if instance.restart_count + else f'exited with code {instance.exit_code}') + return f'engine is not starting ({why}){_engine_error_summary(logs)}' diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index e768d8a3..1dadb691 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -106,6 +106,10 @@ class Container: #: will try to start the container again. restart_policy: str = '' restart_max: int = 0 + #: Why the instance is not running, when its runtime says: + #: ``CrashLoopBackOff``, ``ImagePullBackOff``, ``OOMKilled`` (Kubernetes). + #: Empty when there is nothing to say, or the runtime does not say. + reason: str = '' @property def warm(self) -> bool: @@ -272,3 +276,96 @@ def residency_from_inspect(raw: str, *, project: str) -> Residency: }, tuple(sorted(others, key=lambda c: c.container_id)), ) + + +#: Label a managed Kubernetes pod carries (copied by KubeAI from its Model). +POD_DEPLOYMENT_LABEL = 'infer-stack/deployment' +POD_MANAGED_LABEL = 'infer-stack/managed' + + +def residency_from_pods(raw: str) -> Residency: + """Build a :class:`Residency` from ``kubectl get pods -o json`` output. + + One pod is one instance (a :class:`Container`), keyed by the + ``infer-stack/deployment`` label KubeAI copies from the Model. The state + is mapped to the Docker vocabulary the rest of infer-stack reads: + + * running container -> ``running`` + * ``CrashLoopBackOff`` -> ``restarting`` (still warm: the kubelet retries) + * any other waiting reason, or a pod not started yet -> ``created`` + * terminated -> ``exited``; a pod being deleted -> ``removing`` + + The waiting or last-termination reason is kept in :attr:`Container.reason`. + GPUs are left empty: the cluster, not this host, owns placement. Raises + :class:`ResidencyUnknown` on output that is not a pod list. + + Example: + >>> raw = json.dumps({'items': [{ + ... 'metadata': {'name': 'model-q-1', 'labels': { + ... 'infer-stack/deployment': 'grp-a', 'infer-stack/managed': 'true', + ... 'model': 'q'}}, + ... 'status': {'phase': 'Running', 'containerStatuses': [{ + ... 'state': {'waiting': {'reason': 'CrashLoopBackOff'}}, + ... 'lastState': {'terminated': {'exitCode': 1, 'reason': 'Error'}}, + ... 'restartCount': 3}]}}]}) + >>> c = residency_from_pods(raw).containers('grp-a')[0] + >>> (c.state, c.reason, c.restart_count, c.exit_code, c.warm) + ('restarting', 'CrashLoopBackOff', 3, 1, True) + """ + try: + data = json.loads(raw or '{}') + except json.JSONDecodeError as ex: + raise ResidencyUnknown(f'kubectl get pods output is not JSON: {ex}') from ex + items = data.get('items') if isinstance(data, dict) else None + if not isinstance(items, list): + raise ResidencyUnknown('kubectl get pods output has no items list') + grouped: dict[str, list[Container]] = {} + others: list[Container] = [] + for pod in items: + meta = pod.get('metadata') or {} + labels = meta.get('labels') or {} + status = pod.get('status') or {} + statuses = status.get('containerStatuses') or [] + first = statuses[0] if statuses else {} + current = first.get('state') or {} + last = (first.get('lastState') or {}).get('terminated') or {} + waiting = (current.get('waiting') or {}).get('reason') or '' + if meta.get('deletionTimestamp'): + state = 'removing' + elif 'running' in current: + state = 'running' + elif waiting == 'CrashLoopBackOff': + state = 'restarting' + elif 'terminated' in current: + state = 'exited' + else: + state = 'created' + ended = current.get('terminated') or last + if not waiting and not statuses: + # Not started at all: the pod's own condition says why, e.g. the + # scheduler found no node with the resources ("Unschedulable"). + waiting = next((str(c.get('reason') or '') for c in status.get('conditions') or [] + if c.get('status') == 'False' and c.get('reason')), '') + ready = any(c.get('type') == 'Ready' and c.get('status') == 'True' + for c in status.get('conditions') or []) + container = Container( + container_id=str(meta.get('name') or ''), + deployment_id=str(labels.get(POD_DEPLOYMENT_LABEL) or ''), + state=state, + service=str(labels.get('model') or ''), + labelled=labels.get(POD_MANAGED_LABEL) == 'true', + health='healthy' if ready else ('starting' if state == 'running' else ''), + restart_count=int(first.get('restartCount') or 0), + exit_code=(None if ended.get('exitCode') is None else int(ended['exitCode'])), + # A Deployment's pods are always restarted by the kubelet. + restart_policy='always', + reason=waiting or str(ended.get('reason') or ''), + ) + if container.deployment_id: + grouped.setdefault(container.deployment_id, []).append(container) + else: + others.append(container) + return Residency( + by_deployment={gid: tuple(found) for gid, found in grouped.items()}, + others=tuple(others), + ) diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index f5414eb4..03b32cd7 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -634,3 +634,111 @@ def fingerprint(): ctl.rotate_gateway_key() assert be.master_key() != old assert fingerprint() != before + + +# -- strict residency from pods, and crash diagnosis (plan steps K2, K4) ------ +# +# Pod shapes follow real `kubectl get pods -o json` output captured on k3s + +# KubeAI 0.23.4: KubeAI copies the Model's labels onto its pods, and a pod +# whose engine rejected a flag reads Running, restartCount 1, lastState +# terminated {exitCode: 2, reason: Error}. + + +def _pod(name, gid, *, state=None, restarts=0, last=None, ready=False, + conditions=None, statuses=True): + status = {'phase': 'Running', + 'conditions': conditions or [{'type': 'Ready', + 'status': 'True' if ready else 'False'}]} + if statuses: + status['containerStatuses'] = [{ + 'name': 'server', 'restartCount': restarts, + 'state': state or {'running': {'startedAt': 't'}}, + 'lastState': {'terminated': last} if last else {}, + }] + return {'metadata': {'name': name, 'labels': { + 'infer-stack/deployment': gid, 'infer-stack/managed': 'true', + 'model': name.split('-')[1], 'app': 'model'}}, + 'status': status} + + +class PodKubectl(FakeKubectl): + """FakeKubectl plus pods and per-pod logs (current, and --previous).""" + + def __init__(self): + super().__init__() + self.pods: list[dict] = [] + self.logs: dict[tuple[str, bool], str] = {} + self.fail_pods = False + + def __call__(self, args): + if len(args) > 4 and args[3] == 'get' and args[4] == 'pods': + self.calls.append(args) + if self.fail_pods: + raise RuntimeError('connection refused') + return json.dumps({'items': self.pods}) + if len(args) > 3 and args[3] == 'logs': + self.calls.append(args) + return self.logs.get((args[4], '--previous' in args), '') + return super().__call__(args) + + +def make_pod_backend(tmp_path): + kubectl = PodKubectl() + be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=FakeHttp(kubectl)) + return be, kubectl + + +def test_residency_reads_pods_and_never_guesses(tmp_path): + from infer_stack.leasing.residency import ResidencyUnknown + + be, kubectl = make_pod_backend(tmp_path) + kubectl.pods = [_pod('model-qwen-1', 'grp-a', ready=True)] + pod = be.residency().resident('grp-a') + assert (pod.state, pod.health, pod.labelled) == ('running', 'healthy', True) + kubectl.fail_pods = True + with pytest.raises(ResidencyUnknown): + be.residency() # a failed look is never "nothing running" + + +def test_an_unschedulable_pod_says_why(tmp_path): + be, kubectl = make_pod_backend(tmp_path) + kubectl.pods = [_pod('model-big-1', 'grp-b', statuses=False, conditions=[ + {'type': 'PodScheduled', 'status': 'False', 'reason': 'Unschedulable'}])] + (pod,) = be.residency().containers('grp-b') + assert (pod.state, pod.reason, pod.warm) == ('created', 'Unschedulable', False) + + +def test_a_rejected_flag_fails_the_wait_with_the_engines_words(tmp_path): + be, kubectl = make_pod_backend(tmp_path) + dep = vllm('grp-x', served='broken') + be.converge([dep]) + kubectl.pods = [_pod('model-broken-1', 'grp-x', restarts=1, + last={'exitCode': 2, 'reason': 'Error'})] + kubectl.logs[('model-broken-1', True)] = ( + 'usage: ...\napi_server.py: error: unrecognized arguments: --bogus\n') + be.http.post = lambda url, **kw: FakeHttp._Resp(503, {'detail': 'not ready'}) + probe = be.probe_ready(dep, 'grp-x') + assert probe.fatal and not probe.ready + assert 'unrecognized arguments: --bogus' in probe.detail # the previous run's log + assert 'rejected a command-line flag' in probe.detail + + +def test_a_slow_start_is_not_a_failure(tmp_path): + be, kubectl = make_pod_backend(tmp_path) + dep = vllm('grp-y', served='slow') + be.converge([dep]) + kubectl.pods = [_pod('model-slow-1', 'grp-y')] # running, not ready + be.http.post = lambda url, **kw: FakeHttp._Resp(503, {'detail': 'not ready'}) + probe = be.probe_ready(dep, 'grp-y') + assert not probe.ready and not probe.fatal + + +def test_compose_only_commands_refuse_kubeai_explicitly(tmp_path, monkeypatch): + """`gc --orphans` used "has residency" to mean compose; kubeai has it now.""" + from infer_stack.cli import commands_leasing + + be, _ = make_pod_backend(tmp_path) + ctl = Controller(Ledger(SqliteStore(str(tmp_path / 'l.db'))), be) + monkeypatch.setattr(commands_leasing, '_open_controller', lambda *a, **k: ctl) + with pytest.raises(SystemExit, match='needs the compose backend'): + commands_leasing.GcCLI.main(argv=['--orphans', '--yes']) diff --git a/tests/test_leasing_startup_failure.py b/tests/test_leasing_startup_failure.py index 88fbc6b8..9f0068e4 100644 --- a/tests/test_leasing_startup_failure.py +++ b/tests/test_leasing_startup_failure.py @@ -150,6 +150,8 @@ def test_an_unrecoverable_error_is_fatal_on_the_very_first_crash(tmp_path): (CRASH_LOG, 'fatal'), ('ValueError: quantization fp8 is not supported on this device', 'fatal'), ('torch.OutOfMemoryError: CUDA out of memory', 'fatal'), + # vLLM rejecting a flag (e.g. from runtime.extra_args) repeats every restart. + ('api_server.py: error: unrecognized arguments: --bogus', 'fatal'), (HUB_TIMEOUT_LOG, 'transient'), ('OSError: [Errno 98] Address already in use', 'transient'), ('INFO starting\nSegmentation fault', None), From 791f4c8ea73c0af960ef1c7e2d411ad59c754c10 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 12:53:59 -0400 Subject: [PATCH 005/141] K5: one served-name rule, shared GPU count, env on kubeai leasing.models.served_name replaces five copies of the fallback chain. Two LiteLLM route builders had used `or deployment.id` where the engine used the first served alias, so without served_model_name the route named a model the engine did not serve. KubeAI's resource-profile units come from placement.required_gpu_count, and runtime.env reaches the Model's spec.env (templates filled, reserved names enforced at catalog load); command and mounts stay compose-only. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 7 ++++ README.md | 4 ++- .../plan-backend-unification-2026-09-24.md | 5 +++ infer_stack/backends/kubeai.py | 35 +++++++++++-------- infer_stack/leasing/compose.py | 18 ++++------ infer_stack/leasing/models.py | 22 ++++++++++++ tests/test_leasing_kubeai.py | 13 ++++++- 7 files changed, 75 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5781eaed..218fa5f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,13 @@ as `Unschedulable` or `ImagePullBackOff`. vLLM rejecting a flag (`error: unrecognized arguments`) is now recognised as fatal on both backends; before, it waited for two restarts. +`runtime.env` now works on the kubeai backend (the Model's `spec.env`, with +the same templates and reserved names). The served-name rule lives in one +place, `leasing.models.served_name`. Two gateway route builders fell back to +the deployment id where the engine used the first served alias, so a +deployment without `served_model_name` routed to a name its engine did not +serve. + `gc --orphans` and `network migrate` now refuse a non-compose backend explicitly. They had used "the backend has `residency`" to mean compose. diff --git a/README.md b/README.md index 41a2877a..642c5ee3 100644 --- a/README.md +++ b/README.md @@ -650,7 +650,9 @@ never silently retunes an endpoint after the fact. launcher instead. - All of these are deployment identity: endpoints that launch differently never share a process. -- Compose only; KubeAI refuses an endpoint with `command`, `env` or `mounts`. +- `env` works on both backends (on KubeAI it becomes the Model's `spec.env`). + `command` and `mounts` are Compose only; KubeAI refuses an endpoint with + either rather than serve stock vLLM in its place. ### Reasoning / thinking models diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 350eb231..4cdac436 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -183,6 +183,11 @@ engine's error quoted, instead of waiting out the 800 s timeout. Adding - TUI engine logs through a backend `deployment_logs` stream, with `kubectl logs -f` on KubeAI +**Done 2026-09-24** except TUI logs: there is one `served_name` (it fixed a +real disagreement, noted in the changelog), KubeAI's GPU units come from +`placement.required_gpu_count`, and KubeAI takes `runtime.env`. The unused +profile-era KubeAI renderer is deleted. TUI engine logs on KubeAI remain open. + ## Order and size K0, then K1, is the minimum for a card to run unchanged on a cluster. K2 and diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index aa0e0a0e..fdf03484 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -64,17 +64,16 @@ def model_name_for(served: str) -> str: def _served_name(deployment: Deployment) -> str: - return deployment.spec.get('served_model_name') or ( - sorted(deployment.served)[0] if deployment.served else deployment.id - ) + from ..leasing.models import served_name + + return served_name(deployment) def _gpu_count(deployment: Deployment) -> int: - runtime = deployment.spec.get('runtime', {}) or {} - tp = int(runtime.get('tensor_parallel_size', 1) or 1) - pp = int(runtime.get('pipeline_parallel_size', 1) or 1) - dp = int(runtime.get('data_parallel_size', 1) or 1) - return max(1, tp * pp * dp) + """Resource-profile units for a Model: the planner's GPU count, at least 1.""" + from ..leasing.placement import required_gpu_count + + return max(1, required_gpu_count(deployment)) def _model_doc( @@ -125,10 +124,16 @@ def _model_doc( }, 'spec': spec, } - # Attention backend is a vLLM env var, not a CLI arg (see compose._vllm_service); - # forward it through the KubeAI Model's env map for parity across backends. + # Container environment through the Model's env map, as compose renders + # it: runtime.env (templates filled, values as strings), plus the attention + # backend, which is a vLLM env var rather than a flag. + from ..leasing.launch import env_string, fill + + env = {str(k): fill(env_string(v), svc) for k, v in svc['env'].items()} if svc.get('attention_backend'): - spec['env'] = {'VLLM_ATTENTION_BACKEND': str(svc['attention_backend'])} + env['VLLM_ATTENTION_BACKEND'] = str(svc['attention_backend']) + if env: + spec['env'] = env return doc @@ -178,11 +183,11 @@ def render_models( from ..leasing.launch import translate_legacy runtime = translate_legacy(deployment.spec.get('runtime', {}) or {}) - custom = [k for k in ('command', 'env', 'mounts') if runtime.get(k)] + custom = [k for k in ('command', 'mounts') if runtime.get(k)] if custom: - # A KubeAI Model runs stock vLLM; it has no place for a container - # command, environment or host mounts. Fail closed, never silently - # serve the stock engine instead. + # A KubeAI Model runs stock vLLM: it has no place for a container + # command or host mounts (env maps onto spec.env). Fail closed, + # never silently serve the stock engine instead. out.unrenderable.add(deployment.id) out.errors.append( f"{deployment.id}: runtime.{custom[0]} describes a custom container " diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 27306411..a9c1cb71 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -54,7 +54,7 @@ from ..profile_runtime import simulator_args, vllm_args from .backend import ConvergeScaffold, Readiness from .launch import env_string, fill, translate_legacy -from .models import Deployment, is_reservation +from .models import Deployment, is_reservation, served_name from .placement import plan_placement from .residency import ( # labels live beside the code that reads them back FINGERPRINT_LABEL, @@ -180,9 +180,7 @@ def vllm_service_name(deployment: Deployment, *, unique: bool = False) -> str: :meth:`ComposeBackend.observe` still maps service names through the render sidecar; it is for reporting, not for decisions that touch a GPU.) """ - served = deployment.spec.get('served_model_name') or ( - sorted(deployment.served)[0] if deployment.served else deployment.id - ) + served = served_name(deployment) if unique: return _unique_vllm_service_name(served, deployment.id) return vllm_service_name_for(served) @@ -429,9 +427,7 @@ def vllm_service_dict(deployment: Deployment) -> dict[str, Any]: # A deployment recorded before the generic launch fields may still carry # `serve_recipe`; read it as the fields it meant. runtime = translate_legacy(deployment.spec.get('runtime', {}) or {}) - served = deployment.spec.get('served_model_name') or ( - sorted(deployment.served)[0] if deployment.served else deployment.id - ) + served = served_name(deployment) return { 'served_model_name': served, 'tensor_parallel_size': int(runtime.get('tensor_parallel_size', 1) or 1), @@ -699,7 +695,7 @@ def _litellm_model_list( if deployment.id not in assignments: continue if deployment.engine == 'vllm': - served = deployment.spec.get('served_model_name') or deployment.id + served = served_name(deployment) api_base = f'http://{vllm_service_name(deployment)}:8000/v1' for endpoint in sorted(deployment.served): entries.append(_vllm_route_entry(endpoint, served, api_base)) @@ -810,9 +806,7 @@ def _registry_incoming_from_deployments( if deployment.id not in assignments: continue if deployment.engine == 'vllm': - served = deployment.spec.get('served_model_name') or ( - sorted(deployment.served)[0] if deployment.served else deployment.id - ) + served = served_name(deployment) for endpoint in sorted(deployment.served): incoming[endpoint] = {'engine': 'vllm', 'served': served} elif deployment.engine == 'ollama': @@ -970,7 +964,7 @@ def _litellm_routes( if deployment.id not in assignments: continue if deployment.engine == 'vllm': - served = deployment.spec.get('served_model_name') or deployment.id + served = served_name(deployment) api_base = ( f'http://{vllm_service_name(deployment, unique=True)}' f':{VLLM_CONTAINER_PORT}/v1' diff --git a/infer_stack/leasing/models.py b/infer_stack/leasing/models.py index c6fc922d..3677c5fd 100644 --- a/infer_stack/leasing/models.py +++ b/infer_stack/leasing/models.py @@ -73,6 +73,28 @@ class Sharing: # Synthetic endpoint/claim name for a reservation (it serves nothing). RESERVED_ENDPOINT = 'reserved-gpu' +def served_name(deployment: Any) -> str: + """The model name a deployment's engine serves under: the one rule. + + The spec's ``served_model_name``, else its first served alias, else its + id. The engine's ``--served-model-name``, the gateway route's upstream + model, the Compose service name and the KubeAI Model name all derive + from this, so they cannot disagree. + + Example: + >>> from types import SimpleNamespace as NS + >>> served_name(NS(spec={'served_model_name': 'q'}, served={'a': {}}, id='g')) + 'q' + >>> served_name(NS(spec={}, served={'b': {}, 'a': {}}, id='g')) + 'a' + >>> served_name(NS(spec={}, served={}, id='g')) + 'g' + """ + return deployment.spec.get('served_model_name') or ( + sorted(deployment.served)[0] if deployment.served else deployment.id + ) + + def is_reservation(obj: Any) -> bool: """True if a :class:`Deployment` / :class:`EndpointRequest` is a GPU reservation.""" return getattr(obj, 'engine', None) == RESERVED_ENGINE diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 03b32cd7..de85cbc9 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -185,7 +185,7 @@ def test_render_attention_backend_reaches_cr_env(): @pytest.mark.parametrize('launch', [ {'serve_recipe': 'hyperqwen-3090-single'}, # a legacy entry, translated {'command': ['single']}, - {'env': {'SPEC': 'mtp'}}, + {'mounts': {'/cache': 'x/cache'}}, ]) def test_render_refuses_a_custom_container_launch(launch): rendered = render_models( @@ -742,3 +742,14 @@ def test_compose_only_commands_refuse_kubeai_explicitly(tmp_path, monkeypatch): monkeypatch.setattr(commands_leasing, '_open_controller', lambda *a, **k: ctl) with pytest.raises(SystemExit, match='needs the compose backend'): commands_leasing.GcCLI.main(argv=['--orphans', '--yes']) + + +def test_runtime_env_reaches_the_model_like_it_reaches_a_container(): + rendered = render_models( + [vllm('grp-e', served='q', env={'MODE': 'fast', 'CTX': '{max_model_len}', 'ON': True}, + attention_backend='TORCH_SDPA')], + namespace='kubeai', default_resource_profile='cpu', + ) + (doc,) = rendered.docs + assert doc['spec']['env'] == {'MODE': 'fast', 'CTX': '4096', 'ON': 'true', + 'VLLM_ATTENTION_BACKEND': 'TORCH_SDPA'} From bbbe61067ba4a00d906217b90388fd49ac377ad3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 12:54:30 -0400 Subject: [PATCH 006/141] Journal: backend unification K0-K2, K4, K5 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/journals/claude.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dev/journals/claude.md b/dev/journals/claude.md index caadd793..9bb84781 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3180,3 +3180,40 @@ labels are stable enough to key residency on. **Takeaway.** Before unifying internals, check what callers see. Two backends that share 90% of their code but answer to different names are two products to the people using them. + +## 2026-09-24 12:54:30 -0400 + +**Intent.** The user approved the backend-unification plan, asked for the +work on a branch (`dev/backend-unification`), and authorised installing k3s +on the guest to test KubeAI for real. Model: Claude Opus 5.5 (1M context). + +**What happened.** K0: k3s plus KubeAI 0.23.4, using the chart's `cpu` +profile. It runs real vLLM on CPU, which beats the simulator: KubeAI builds +vLLM's own command line, and the simulator's CLI would have refused it. The +backend passed unchanged on its first real run since July. K1: the kubeai +backend now owns a gateway-only ComposeBackend and feeds it generic +`upstream` route rows. Before, a card's alias got 404 from KubeAI; after, +the same request answered. K2 and K4: strict pod residency, and the crash +diagnosis moved into a backend-neutral module. A Model whose vLLM rejected a +flag failed in 52 s instead of the 800 s timeout. K5: one served-name rule. +It fixed a real disagreement between the engine and two route builders. +Removed the dead profile-era KubeAI renderer. + +**What I got wrong along the way.** I wrote a second e2e script before +finding `dev/kubeai_e2e.sh`, a duplicate authority created in the middle of +a de-duplication task, and folded it back in. That script's generation check +could never fail (a `curl | grep && echo` list, where `set -e` does not +fire), so it "passed" on a 404. The plan's claim that KubeAI's lenient +`observe()` was a bug was wrong: Compose's is lenient by the same contract. +I corrected the plan. + +**Open: K3 (one acquire path).** It is feasible: only 12 controller tests and +the null backend depend on the old path. It carries a semantic decision the +user should make. On a cluster nothing displaces an idle keep-warm Model, so +the September starvation incident can recur on KubeAI unless admission +evicts idle keep-warm Models when a new one cannot be scheduled. + +**Takeaways.** (1) A test that has never failed is not evidence. Make it +fail on purpose first; GATEWAY=0 is that switch here. (2) "Has method X" is +a poor stand-in for "is backend Y": when a second backend gains X, every +such check silently changes meaning. From 2c513760add95ab4d594397bd1e87ead092a2de3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 14:10:22 -0400 Subject: [PATCH 007/141] G-A/B: move naming rules and the gateway's module code out of compose.py compose.py mixed engine containers with the front door. Two new modules: - leasing/naming.py: dns_slug, the vLLM/Ollama service-name rules and the engine container ports. Engines, gateway routes and the kubeai backend all derive names and upstream addresses from these. - leasing/gateway.py: route rows and the LiteLLM model_list renderers, the route registry functions, the LiteLLM/Postgres/Open WebUI/nginx service renderers, the dynamic-routing constants, and set_master_key. Moved unchanged. ENGINE_LABEL joins the other labels in residency.py. compose keeps importing what it uses (so `from infer_stack.leasing.compose import dns_slug` still works); importers of gateway names now import them from gateway. No behaviour change: suite 898 passed, ty and flake8 clean. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- .../plan-backend-unification-2026-09-24.md | 28 + infer_stack/backends/kubeai.py | 2 +- infer_stack/leasing/compose.py | 826 +----------------- infer_stack/leasing/gateway.py | 732 ++++++++++++++++ infer_stack/leasing/naming.py | 99 +++ infer_stack/leasing/residency.py | 2 + tests/test_leasing_compose.py | 2 +- tests/test_leasing_dynamic_routing.py | 6 +- tests/test_leasing_route_registry.py | 2 +- tests/test_leasing_secrets.py | 7 +- 10 files changed, 917 insertions(+), 789 deletions(-) create mode 100644 infer_stack/leasing/gateway.py create mode 100644 infer_stack/leasing/naming.py diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 4cdac436..ac31001b 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -202,3 +202,31 @@ to the kind script. - multi-cluster, or scheduling across a mix of Compose hosts and a cluster - KubeAI autoscaling policy beyond today's `min_replicas` / `max_replicas` - custom container launches on KubeAI + +## G. Pull the gateway out of compose.py (added 2026-09-24) + +`compose.py` mixes two jobs: engine containers (placement, render, +selective apply, pulls, diagnosis) and the front door (LiteLLM config and +service, route registry, dynamic-route reconciliation, keys, Open WebUI, +nginx). K1 reused the whole `ComposeBackend` as a gateway-only instance for +KubeAI. That added a mode rather than naming the seam. The user asked for +the tool to be more elegant, architecture first and the TUI after. + +The gateway methods depend only on the state dir, ports, the HTTP client, +the clock, the catalog and the flags (measured by AST), not on engine state. +So the split is: + +- **A. `leasing/naming.py`**: `dns_slug` and the vLLM/Ollama service-name + rules. Engines and gateway routes both use them, and so does KubeAI. +- **B. `leasing/gateway.py`**: the route functions, front-door service + renderers and gateway constants, moved as-is. +- **C. `Gateway`**: keys, route registry and dynamic-route reconciliation. + `ComposeBackend` owns one and keeps thin public delegations (`master_key`, + `access`, ...). +- **D. `render_front_door`**: `render_compose`'s second half. + +`compose` re-exports nothing private; importers (mostly tests) move to the +new modules. Each step leaves the full suite, `ty` and flake8 green. The +gateway still runs as a Compose project (it is a container), so KubeAI keeps +a `ComposeBackend` with no engines: that is now simply "a Compose project +with only the front door", not a special mode. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index fdf03484..f57e2559 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -378,7 +378,7 @@ def _upstream_url(self) -> str: def _render_gateway(self, rendered: RenderedModels) -> None: """Route each endpoint alias through the gateway to its Model.""" - from ..leasing.compose import UPSTREAM_ROUTE + from ..leasing.gateway import UPSTREAM_ROUTE base = self._upstream_url() self.gateway.upstream_routes = { diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index ce980fc2..b012a327 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -40,7 +40,6 @@ import hashlib import json -import re import time from dataclasses import dataclass, field from pathlib import Path @@ -53,10 +52,55 @@ from ..probe import openai_ready from ..profile_runtime import simulator_args, vllm_args from .backend import ConvergeScaffold, Readiness +from .gateway import ( + API_KEY_ENV, + DB_PASSWORD_ENV, + _dump_route_registry, + LITELLM_CONFIG_FILENAME, + LITELLM_CONTAINER_PORT, + _litellm_model_list, + _litellm_model_list_from_catalog, + _litellm_model_list_from_registry, + LITELLM_REGISTRY_FILENAME, + LITELLM_REGISTRY_VERSION, + _litellm_routes, + LITELLM_ROUTES_FILENAME, + _litellm_service, + LITELLM_SERVICE, + _merge_route_registry, + _nginx_conf, + NGINX_CONFIG_FILENAME, + _nginx_service, + NGINX_SERVICE, + _open_webui_service, + OPEN_WEBUI_SERVICE, + _postgres_service, + POSTGRES_SERVICE, + _registry_incoming_from_catalog, + _registry_incoming_from_deployments, + ROUTE_ID_PREFIX, + ROUTE_RECONCILE_BOOTSTRAP_S, + ROUTE_RECONCILE_STEADY_S, + SALT_KEY_ENV, + _seed_registry_from_litellm_config, + set_master_key, +) from .launch import env_string, fill, translate_legacy from .models import Deployment, is_reservation, served_name +from .naming import ( # noqa: F401 (public names re-exported for callers) + OLLAMA_CONTAINER_PORT, + VLLM_CONTAINER_PORT, + _dns_slug, + _unique_vllm_service_name, + dns_slug, + ollama_service_name, + ollama_service_name_for, + vllm_service_name, + vllm_service_name_for, +) from .placement import plan_placement from .residency import ( # labels live beside the code that reads them back + ENGINE_LABEL, FINGERPRINT_LABEL, SERVICE_LABEL, COMPOSE_PROJECT_LABEL, @@ -68,46 +112,8 @@ LEASING_PROJECT = 'infer-stack' # docker compose project name for leased stacks VLLM_HOST_PORT_BASE = 18000 -VLLM_CONTAINER_PORT = 8000 -OLLAMA_CONTAINER_PORT = 11434 -LITELLM_CONTAINER_PORT = 4000 STATE_FILENAME = 'leasing-compose-state.json' COMPOSE_FILENAME = 'docker-compose.yml' -LITELLM_CONFIG_FILENAME = 'litellm_config.yaml' -LITELLM_SERVICE = 'litellm' -API_KEY_ENV = 'LITELLM_MASTER_KEY' -# The key LiteLLM encrypts credentials stored in its database with. Unset, it -# uses the master key -- so rotating the master key would make every -# DB-stored route undecryptable. `set_master_key` pins it to the pre-rotation -# master key the first time the key changes; it must never change after that. -SALT_KEY_ENV = 'LITELLM_SALT_KEY' - -# Dynamic-routing (admin-API) extras. When dynamic routing is on, the gateway's -# route table is managed live via LiteLLM's admin API against a Postgres-backed -# model store, instead of a static config file. See render_compose + -# ComposeBackend._reconcile_routes and docs/litellm-gateway-routing.md. -LITELLM_ROUTES_FILENAME = 'litellm_routes.json' # rendered desired route set -# Append-only route registry for static-superset mode: accumulates the semantic -# route inputs (served name / engine / host) of every catalog *and* every live -# deployment ever merged, across all runbooks sharing this state dir. The gateway -# `model_list` is rendered from the whole registry, so a converge under one -# runbook's catalog can no longer strip another's still-live routes, and once -# every catalog has been merged once the rendered config is byte-stable (the -# gateway is never recreated). See docs/litellm-gateway-routing.md and -# ComposeBackend._update_route_registry. -LITELLM_REGISTRY_FILENAME = 'litellm_registry.json' -LITELLM_REGISTRY_VERSION = 1 -# A route-registry row for a server this project does not run: -# ``{'engine': UPSTREAM_ROUTE, 'served': , 'api_base': }``. -UPSTREAM_ROUTE = 'upstream' -POSTGRES_SERVICE = 'postgres-litellm' -POSTGRES_CONTAINER_PORT = 5432 -POSTGRES_DB_NAME = 'litellm' -POSTGRES_DB_USER = 'litellm' -DB_PASSWORD_ENV = 'LITELLM_DB_PASSWORD' # managed secret in the sidecar .env -# Marks a LiteLLM route as infer-stack-managed, so reconcile only ever deletes -# routes it created (never a model added by hand through the UI/admin API). -ROUTE_ID_PREFIX = 'isr-' VLLM_DEFAULTS = { 'gpu_memory_utilization': 0.9, @@ -116,90 +122,8 @@ 'max_num_seqs': 256, } -ENGINE_LABEL = 'infer-stack.engine' - - -def dns_slug(text: str) -> str: - """A lowercase ``[a-z0-9-]`` label safe as a compose service / DNS / - Kubernetes object name (shared by the compose and kubeai backends).""" - out = re.sub(r'[^a-z0-9]+', '-', str(text).lower()).strip('-') - return out or 'model' - - -_dns_slug = dns_slug # historical internal name - - -def vllm_service_name_for(served: str) -> str: - """Deterministic compose/DNS service name for a vLLM upstream: ``vllm-``. - - Derived purely from the served model name (vLLM's ``--served-model-name``, - the Open WebUI label, the alias the user chose), so it is identical whether - computed from a live :class:`Deployment` or from a catalog endpoint. That - stability is what lets the LiteLLM gateway carry a *static* route table (one - per catalog endpoint) whose upstream hosts match the containers when they - come up — so adding/removing models does not rewrite the gateway's config and - the gateway is never recreated (no "blip"); see :func:`_litellm_model_list`. - """ - return f'vllm-{_dns_slug(served)}' - - -def _unique_vllm_service_name(served: str, deployment_id: str) -> str: - """Per-deployment vLLM service/DNS name: ``vllm--``. - - The static-superset gateway needs a name derivable from the served model - *alone* (so a catalog route can address it without knowing the live - deployment) — but that deliberately drops the deployment id, which - **collapses every** ``--dedicated`` **deployment of one model onto a single - container** (hence one GPU). Dynamic routing manages the gateway's routes - live via the admin API, so the upstream host no longer has to be predictable - from the catalog. That frees us to give each deployment its **own** service, - so N dedicated deployments of one model become N containers on N GPUs. The - suffix is the deployment id's hex tail, keeping the name short and DNS-safe. - """ - tail = deployment_id.rsplit('-', 1)[-1][:8] or 'x' - return f'{vllm_service_name_for(served)}-{_dns_slug(tail)}' - - -def vllm_service_name(deployment: Deployment, *, unique: bool = False) -> str: - """Compose service name for a vLLM deployment (see :func:`vllm_service_name_for`). - - Default (``unique=False``, static-superset mode): deterministic from the - served model name only — *no* deployment-id suffix — so it matches the - gateway's pre-rendered route for that endpoint. Trade-off: two - *simultaneously desired* deployments that share a served name collide on this - name; under the static gateway the catalog endpoint is the unit, so that case - (including same-model ``--dedicated``) is unsupported. - - ``unique=True`` (dynamic-routing mode): append the deployment-id tail - (:func:`_unique_vllm_service_name`) so same-model dedicated deployments get - distinct containers/GPUs; the admin-API route table addresses each by name. - - Either way the container carries the ``infer-stack.deployment`` label. - :meth:`ComposeBackend.residency` correlates containers to deployments by that - label, so the choice of suffix does not affect it. (The lenient - :meth:`ComposeBackend.observe` still maps service names through the render - sidecar; it is for reporting, not for decisions that touch a GPU.) - """ - served = served_name(deployment) - if unique: - return _unique_vllm_service_name(served, deployment.id) - return vllm_service_name_for(served) -def ollama_service_name_for(host: str) -> str: - """Deterministic service name for an Ollama daemon: ``ollama-``. - - One daemon per host (Ollama coalesces tags onto it), so the host is the - stable key — matching :func:`vllm_service_name_for`'s role for vLLM so the - gateway's static route table addresses it regardless of which tags are live. - """ - return f'ollama-{_dns_slug(host)}' - - -def ollama_service_name(deployment: Deployment) -> str: - host = deployment.spec.get('host') or deployment.id - return ollama_service_name_for(host) - class ApplyAborted(RuntimeError): """Selective apply refused to act; the change stays pending.""" @@ -654,639 +578,6 @@ def _ollama_service( return service -def _vllm_route_entry( - model_name: str, served: str, api_base: str -) -> dict[str, Any]: - """One LiteLLM ``model_list`` entry routing ``model_name`` to a vLLM upstream. - - Shared by every render path (legacy per-deployment, catalog-superset, and - the route registry) so a registry-rendered entry can never drift from what - the catalog/deployment paths produce for the same endpoint.""" - return { - 'model_name': model_name, - 'litellm_params': { - 'model': f'openai/{served}', - 'api_base': api_base, - 'api_key': 'EMPTY', - }, - } - - -def _ollama_route_entry( - model_name: str, tag: str, api_base: str -) -> dict[str, Any]: - """One LiteLLM ``model_list`` entry routing ``model_name`` to an Ollama tag - (see :func:`_vllm_route_entry` for why this is factored out).""" - return { - 'model_name': model_name, - 'litellm_params': { - 'model': f'ollama/{tag}', - 'api_base': api_base, - }, - } - - -def _litellm_model_list( - deployments: list[Deployment], assignments: dict[str, list[int]] -) -> list[dict[str, Any]]: - """One LiteLLM ``model_list`` entry per served endpoint alias.""" - entries: list[dict[str, Any]] = [] - for deployment in sorted(deployments, key=lambda g: (g.created_at, g.id)): - if deployment.id not in assignments: - continue - if deployment.engine == 'vllm': - served = served_name(deployment) - api_base = f'http://{vllm_service_name(deployment)}:8000/v1' - for endpoint in sorted(deployment.served): - entries.append(_vllm_route_entry(endpoint, served, api_base)) - elif deployment.engine == 'ollama': - api_base = f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' - for endpoint, payload in sorted(deployment.served.items()): - tag = payload.get('model', endpoint) - entries.append(_ollama_route_entry(endpoint, tag, api_base)) - return entries - - -def _litellm_model_list_from_catalog(catalog: Any) -> list[dict[str, Any]]: - """A *static superset* ``model_list``: one route per catalog endpoint. - - Unlike :func:`_litellm_model_list` (which routes only the currently-placed - deployments), this routes *every* catalog endpoint to its deterministic - upstream host (:func:`vllm_service_name_for` / :func:`ollama_service_name_for`). - The resulting config therefore depends only on the catalog, not on which - models happen to be up — so acquiring/releasing a model leaves the gateway's - config (and its container) untouched (no blip). A route whose upstream is not - currently running simply errors/cools-down until it comes up; the - ``router_settings`` below make that warmup self-healing. ``/v1/models`` lists - the whole catalog (some upstreams down) rather than only the live set. - - This static-superset path is the default. Its one limitation — it cannot give - same-model ``--dedicated`` deployments distinct upstreams, and cannot route - non-catalog acquires without a config change — is addressed by the opt-in - *dynamic routing* mode (``dynamic_routing=True``), which manages routes live - via LiteLLM's admin API against a Postgres model store (see - :func:`_litellm_routes`, :meth:`ComposeBackend._reconcile_routes`, and - ``docs/litellm-gateway-routing.md``). The two are mutually exclusive per - converge; this function is used only when dynamic routing is off. - """ - entries: list[dict[str, Any]] = [] - for name in sorted(getattr(catalog, 'endpoints', {})): - try: - req = catalog.resolve_endpoint(name) - except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway - continue - if req.engine == 'vllm': - served = req.served.get('served_model_name') or name - api_base = ( - f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' - ) - entries.append(_vllm_route_entry(name, served, api_base)) - elif req.engine == 'ollama': - host = req.spec.get('host') or req.host - tag = req.served.get('model') or name - api_base = ( - f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' - ) - entries.append(_ollama_route_entry(name, tag, api_base)) - return entries - - -# -- Route registry (static-superset persistence) -------------------------- -# -# The registry stores *semantic* route inputs (served name / engine / host), -# never rendered LiteLLM entries — render derives entries through the same -# helpers the catalog/deployment paths use (:func:`_litellm_model_list_from_registry`), -# so a future renderer change propagates to old registry rows automatically. -# All functions here are pure; the backend owns the file I/O and locking. - - -def _registry_incoming_from_catalog(catalog: Any) -> dict[str, dict[str, Any]]: - """Semantic route rows for every resolvable endpoint of ``catalog``. - - Mirrors :func:`_litellm_model_list_from_catalog`'s iteration (unresolvable - endpoints skipped) but emits registry rows keyed by endpoint name. A vLLM - row carries only ``served`` (the upstream host is re-derived at render via - :func:`vllm_service_name_for`); an Ollama row carries ``model`` (tag) + - ``host``.""" - incoming: dict[str, dict[str, Any]] = {} - for name in sorted(getattr(catalog, 'endpoints', {})): - try: - req = catalog.resolve_endpoint(name) - except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway - continue - if req.engine == 'vllm': - served = req.served.get('served_model_name') or name - incoming[name] = {'engine': 'vllm', 'served': served} - elif req.engine == 'ollama': - host = req.spec.get('host') or req.host - tag = req.served.get('model') or name - incoming[name] = {'engine': 'ollama', 'model': tag, 'host': host} - return incoming - - -def _registry_incoming_from_deployments( - deployments: list[Deployment], assignments: dict[str, list[int]] -) -> dict[str, dict[str, Any]]: - """Semantic route rows for every *placed* deployment in ``assignments``. - - ``deployments`` is the full ``desired`` set (which spans all runbooks via - the shared ledger), so this keeps non-catalog / dedicated acquires routable - and — because the registry persists — routable past release. One row per key - of ``deployment.served`` (a coalesced deployment can back several endpoint - aliases). Only ``vllm``/``ollama`` engines contribute; ``RESERVED_ENGINE`` - and unknown engines render no service, so they contribute no row — exactly - as :func:`render_compose`'s service loop skips them. - - The vLLM ``served`` uses the same fallback chain as :func:`vllm_service_name` - (``spec['served_model_name'] or sorted(served)[0] or id``), so a - catalog-listed endpoint acquired live reduces to the identical row a catalog - merge produces — live-vs-released status never moves the rendered bytes.""" - incoming: dict[str, dict[str, Any]] = {} - for deployment in deployments: - if deployment.id not in assignments: - continue - if deployment.engine == 'vllm': - served = served_name(deployment) - for endpoint in sorted(deployment.served): - incoming[endpoint] = {'engine': 'vllm', 'served': served} - elif deployment.engine == 'ollama': - host = deployment.spec.get('host') or deployment.id - for endpoint, payload in sorted(deployment.served.items()): - tag = payload.get('model', endpoint) - incoming[endpoint] = { - 'engine': 'ollama', - 'model': tag, - 'host': host, - } - return incoming - - -def _litellm_model_list_from_registry( - registry: dict[str, Any] -) -> list[dict[str, Any]]: - """Render the gateway ``model_list`` from the whole accumulated registry. - - Iterates ``sorted(entries)`` (determinism, §8) and derives each upstream - ``api_base`` through the live naming helpers, so the registry never becomes - a rendered-config parse surface.""" - entries: list[dict[str, Any]] = [] - rows = registry.get('entries', {}) if isinstance(registry, dict) else {} - for name in sorted(rows): - row = rows[name] - if not isinstance(row, dict): - continue - engine = row.get('engine') - if engine == 'vllm': - served = row.get('served') or name - api_base = ( - f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' - ) - entries.append(_vllm_route_entry(name, served, api_base)) - elif engine == 'ollama': - tag = row.get('model') or name - host = row.get('host') or name - api_base = ( - f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' - ) - entries.append(_ollama_route_entry(name, tag, api_base)) - elif engine == UPSTREAM_ROUTE and row.get('api_base'): - # An OpenAI-compatible server this project does not run (a KubeAI - # cluster's gateway): the row carries its address and the name it - # serves the model under. - entries.append(_vllm_route_entry(name, row.get('served') or name, - str(row['api_base']))) - return entries - - -def _merge_route_registry( - existing: dict[str, Any], incoming: dict[str, dict[str, Any]] -) -> tuple[dict[str, Any], list[str]]: - """Merge ``incoming`` semantic rows into ``existing`` (append-only). - - Idempotent (merging identical rows is a no-op) and additive (never removes a - row). On a conflict — same key, different row — *incoming wins* and a warning - naming both definitions is emitted; the changed definition changes the - rendered bytes, which is the one justified recreate. The existing ``version`` - is preserved (an unknown version merged under is not silently rewritten to - the current schema; see :meth:`ComposeBackend._load_route_registry`).""" - version = LITELLM_REGISTRY_VERSION - entries: dict[str, dict[str, Any]] = {} - if isinstance(existing, dict): - version = existing.get('version', LITELLM_REGISTRY_VERSION) - prior = existing.get('entries') - if isinstance(prior, dict): - entries = {k: v for k, v in prior.items()} - warnings: list[str] = [] - for name in sorted(incoming): - row = incoming[name] - if name in entries and entries[name] != row: - warnings.append( - f"route {name!r} redefined: {entries[name]} -> {row} " - '(incoming wins; gateway will be recreated once)' - ) - entries[name] = row - return {'version': version, 'entries': entries}, warnings - - -def _seed_registry_from_litellm_config( - config_text: str, -) -> tuple[dict[str, Any], list[str]]: - """One-shot upgrade seed: recover registry rows from a rendered - ``litellm_config.yaml`` so the first post-upgrade converge does not strip - the other runbooks' routes. - - Single-format, migration-time only (no cross-version promise): ``openai/`` - inverts *exactly* to ``{engine: vllm, served}``. Ollama rows are skipped with - a warning — the host survives only as a non-invertible ``dns_slug`` inside - ``api_base`` — and re-enter the registry at the next converge that has them - in its catalog or live set. Anything else unparseable is likewise skipped.""" - entries: dict[str, dict[str, Any]] = {} - warnings: list[str] = [] - try: - data = yaml.safe_load(config_text) or {} - except Exception: # noqa: BLE001 - a torn file must not brick seeding - return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}}, [ - 'seed: litellm_config.yaml is unparseable; starting an empty registry' - ] - for entry in data.get('model_list', []) or []: - name = entry.get('model_name') - model = (entry.get('litellm_params') or {}).get('model', '') - if not name: - continue - if isinstance(model, str) and model.startswith('openai/'): - entries[name] = {'engine': 'vllm', 'served': model[len('openai/'):]} - elif isinstance(model, str) and model.startswith('ollama/'): - warnings.append( - f"seed: skipping Ollama route {name!r} (host not recoverable " - 'from the rendered api_base; it re-enters at its next converge)' - ) - else: - warnings.append(f'seed: skipping unparseable route {name!r}') - return {'version': LITELLM_REGISTRY_VERSION, 'entries': entries}, warnings - - -def _dump_route_registry(registry: dict[str, Any]) -> str: - """Canonical, byte-stable serialization (§3): sorted keys + trailing - newline. A nondeterministic dump would manufacture phantom hash changes.""" - return json.dumps(registry, sort_keys=True, indent=2) + '\n' - - -def _route_id(deployment_id: str, endpoint: str) -> str: - """Deterministic LiteLLM model id for one (deployment, endpoint) route. - - Stable across converges, so route reconcile (:meth:`ComposeBackend. - _reconcile_routes`) can identify one logical route across renders. A route - whose id disappears is deleted by exactly this id; a route whose id remains - but whose observable routing semantics drifted is replaced under the same id. - The ``isr-`` prefix marks it - infer-stack-managed so reconcile never deletes a model someone added by hand. - """ - digest = hashlib.sha256(f'{deployment_id}|{endpoint}'.encode()).hexdigest() - return f'{ROUTE_ID_PREFIX}{digest[:32]}' - - -def _litellm_routes( - deployments: list[Deployment], assignments: dict[str, list[int]] -) -> list[dict[str, Any]]: - """Desired LiteLLM route set for the *live* deployments (dynamic routing). - - One entry per (placed deployment, served endpoint), addressing the - deployment's **own** unique upstream service (:func:`vllm_service_name` with - ``unique=True``). Several dedicated deployments of the same model therefore - yield several entries that share one public ``model_name`` but point at - distinct upstreams — LiteLLM load-balances the alias across them, so each - runs on its own GPU while clients still ask for the single name. Each entry - carries a deterministic ``model_info.id`` (:func:`_route_id`) so applying the - set via the admin API is an idempotent diff, not fire-and-forget calls. - """ - entries: list[dict[str, Any]] = [] - for deployment in sorted(deployments, key=lambda g: (g.created_at, g.id)): - if deployment.id not in assignments: - continue - if deployment.engine == 'vllm': - served = served_name(deployment) - api_base = ( - f'http://{vllm_service_name(deployment, unique=True)}' - f':{VLLM_CONTAINER_PORT}/v1' - ) - for endpoint in sorted(deployment.served): - entries.append( - { - 'model_name': endpoint, - 'litellm_params': { - 'model': f'openai/{served}', - 'api_base': api_base, - 'api_key': 'EMPTY', - }, - 'model_info': {'id': _route_id(deployment.id, endpoint)}, - } - ) - elif deployment.engine == 'ollama': - api_base = ( - f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' - ) - for endpoint, payload in sorted(deployment.served.items()): - tag = payload.get('model', endpoint) - entries.append( - { - 'model_name': endpoint, - 'litellm_params': { - 'model': f'ollama/{tag}', - 'api_base': api_base, - }, - 'model_info': {'id': _route_id(deployment.id, endpoint)}, - } - ) - return entries - - -CONFIG_HASH_LABEL = 'infer-stack.config-hash' - - -def _postgres_service( - images: dict[str, str], state: dict[str, str] -) -> dict[str, Any]: - """Postgres backing LiteLLM's runtime model store (dynamic routing only). - - LiteLLM's admin API (``/model/new`` / ``/model/delete``) only functions with - ``STORE_MODEL_IN_DB=true`` + a database, so dynamic routing needs a DB. This - is an **internal** service (no published host port); LiteLLM reaches it on - the compose network at ``postgres-litellm:5432``. The password is the managed - :data:`DB_PASSWORD_ENV` secret in the sidecar ``.env`` (interpolated by - ``docker compose --env-file``), so it never appears literally in the YAML. - The healthcheck lets the litellm service ``depends_on`` it (condition: - service_healthy) so the gateway only starts once the DB can accept queries. - """ - data_path = state.get('postgres_litellm') or str( - Path(next(iter(state.values()), '.')).parent / 'postgres-litellm' - ) - return { - 'image': images.get('postgres', PINNED_IMAGES['postgres']), - 'environment': { - 'POSTGRES_USER': POSTGRES_DB_USER, - 'POSTGRES_PASSWORD': '${' + DB_PASSWORD_ENV + '}', - 'POSTGRES_DB': POSTGRES_DB_NAME, - }, - 'volumes': [f'{data_path}:/var/lib/postgresql/data'], - 'restart': 'unless-stopped', - 'labels': {ENGINE_LABEL: 'postgres'}, - 'healthcheck': { - 'test': [ - 'CMD-SHELL', - f'pg_isready -U {POSTGRES_DB_USER} -d {POSTGRES_DB_NAME}', - ], - 'interval': '5s', - 'timeout': '5s', - 'retries': 30, - 'start_period': '30s', - }, - } - - -def _litellm_service( - service_names: list[str], - host_port: int, - images: dict[str, str], - aux_dir: str, - master_key: str | None = None, - config_hash: str | None = None, - *, - dynamic_routing: bool = False, - salt_key: bool = False, -) -> dict[str, Any]: - # Reference the managed key via ${...} rather than baking the literal secret - # into the compose YAML. Its value lives in the sidecar .env next to the - # compose file (written by master_key()), which `docker compose --env-file` - # loads for interpolation — so the container and the readiness probe (which - # reads the same .env) still agree regardless of the caller's shell env. - key_value = ( - '${' + API_KEY_ENV + '}' - if master_key is not None - else '${' + API_KEY_ENV + ':-sk-local}' - ) - environment = {API_KEY_ENV: key_value} - if salt_key: - # Only when the .env has one: LiteLLM treats an EMPTY salt as a key, - # so a `${...:-}` default would silently change the encryption key. - environment[SALT_KEY_ENV] = '${' + SALT_KEY_ENV + '}' - if dynamic_routing: - # DB-backed runtime model store so the admin API (/model/new, - # /model/delete) works; the gateway then never needs recreating to learn - # a route. Both are read from the env by LiteLLM. The password is - # interpolated from the sidecar .env, so no secret lands in the YAML. - environment['DATABASE_URL'] = ( - f'postgresql://{POSTGRES_DB_USER}:${{{DB_PASSWORD_ENV}}}' - f'@{POSTGRES_SERVICE}:{POSTGRES_CONTAINER_PORT}/{POSTGRES_DB_NAME}' - ) - environment['STORE_MODEL_IN_DB'] = 'True' - labels = {ENGINE_LABEL: 'litellm'} - if config_hash is not None: - # LiteLLM reads its routing config once at startup; the file is bind- - # mounted, so a config change alone does NOT change this service's spec - # and `docker compose up -d` would leave the old container (and old - # routes) running. Stamping the config hash onto a label makes the spec - # change exactly when the config does, so converge recreates LiteLLM and - # it picks up new/removed aliases. Without this, coalescing a second - # alias onto a live deployment never becomes routable (readiness times out). - labels[CONFIG_HASH_LABEL] = config_hash - service: dict[str, Any] = { - 'image': images['litellm'], - 'command': [ - '--config', - '/etc/litellm/config.yaml', - '--port', - str(LITELLM_CONTAINER_PORT), - ], - 'ports': [f'{host_port}:{LITELLM_CONTAINER_PORT}'], - 'volumes': [f'{aux_dir}/{LITELLM_CONFIG_FILENAME}:/etc/litellm/config.yaml:ro'], - 'environment': environment, - 'restart': 'unless-stopped', - 'labels': labels, - } - if dynamic_routing: - # Wait for the DB to accept queries before the gateway boots; do NOT add - # per-model depends_on (that would churn the spec, i.e. blip, on every - # model change). The route table is filled in afterward via the API. - service['depends_on'] = { - POSTGRES_SERVICE: {'condition': 'service_healthy'} - } - elif service_names: - # Only wait on upstreams when there are any (zero models -> empty gateway). - service['depends_on'] = sorted(service_names) - return service - - -OPEN_WEBUI_SERVICE = 'open-webui' -OPEN_WEBUI_CONTAINER_PORT = 8080 - -NGINX_SERVICE = 'reverse-proxy' -NGINX_CONTAINER_PORT = 80 -NGINX_CONFIG_FILENAME = 'nginx.conf' - - -def _open_webui_service( - host_port: int, - images: dict[str, str], - state: dict[str, str], - master_key: str | None, - *, - openai_urls: list[str] | None = None, - ollama_urls: list[str] | None = None, - depends_on: list[str] | None = None, -) -> dict[str, Any]: - """A managed Open WebUI pointed at whatever front door is available. - - Open WebUI holds two independent kinds of connection, wired here from the - rendered services: - - * **OpenAI** (``openai_urls``) — the chat/completions front door. This is the - LiteLLM gateway when it is enabled (so every declared endpoint alias is - reachable at one URL); with LiteLLM off it falls back to the rendered - upstreams' own ``/v1`` (a single vLLM/Ollama service, or several joined as - ``OPENAI_API_BASE_URLS``). With nothing to point at, the OpenAI API is - disabled rather than left dangling. - * **Ollama** (``ollama_urls``) — the *native* Ollama API of any rendered - Ollama daemon. This is what lets you pull/run/delete models from the UI - and have the daemon load them on demand, independent of LiteLLM — i.e. a - true drop-in for a hand-run ``ollama`` + Open WebUI stack. - - The spec is kept as independent of which models are live as it can be: the - LiteLLM URL is fixed, and the Ollama daemon's service name is its stable - structural id, so adding/removing other models does not rewrite this service - and ``docker compose up -d`` leaves the UI running (the legacy "the UI never - blinks" behavior). Chat history persists under the data dir. - """ - # Reference the managed key via ${...} (resolved from the sidecar .env, see - # _litellm_service) instead of inlining the secret into the compose YAML. - key_value = ( - '${' + API_KEY_ENV + '}' - if master_key is not None - else '${' + API_KEY_ENV + ':-sk-local}' - ) - data_path = state.get('open_webui') or str( - Path(next(iter(state.values()), '.')).parent / 'open-webui' - ) - openai_urls = list(openai_urls or []) - ollama_urls = list(ollama_urls or []) - env: dict[str, str] = { - # Single-user workstation default; the port shouldn't be exposed - # publicly. Tracked as a knob in dev/leasing-followups.md. - 'WEBUI_AUTH': 'False', - } - if openai_urls: - env['ENABLE_OPENAI_API'] = 'True' - if len(openai_urls) == 1: - env['OPENAI_API_BASE_URL'] = openai_urls[0] - else: - env['OPENAI_API_BASE_URLS'] = ';'.join(openai_urls) - env['OPENAI_API_KEY'] = key_value - else: - env['ENABLE_OPENAI_API'] = 'False' - if ollama_urls: - env['ENABLE_OLLAMA_API'] = 'True' - if len(ollama_urls) == 1: - env['OLLAMA_BASE_URL'] = ollama_urls[0] - else: - env['OLLAMA_BASE_URLS'] = ';'.join(ollama_urls) - else: - env['ENABLE_OLLAMA_API'] = 'False' - service: dict[str, Any] = { - 'image': images['open_webui'], - 'ports': [f'{host_port}:{OPEN_WEBUI_CONTAINER_PORT}'], - 'environment': env, - 'volumes': [f'{data_path}:/app/backend/data'], - 'restart': 'unless-stopped', - 'labels': {ENGINE_LABEL: 'open-webui'}, - } - if depends_on: - service['depends_on'] = sorted(depends_on) - return service - - -def _nginx_conf(*, litellm: bool, ui: bool) -> str: - """A minimal HTTP reverse-proxy conf: one origin, path-routed. - - ``/v1/`` -> the LiteLLM gateway (the OpenAI API), ``/`` -> Open WebUI (or the - gateway when there's no UI). Plain HTTP — no TLS, no auth — so the value is - "one port, nothing to remember", not security. The ``map`` is valid here - because a ``conf.d/*.conf`` file is included in nginx's ``http`` context. - """ - api = f'http://{LITELLM_SERVICE}:{LITELLM_CONTAINER_PORT}' - locations = '' - if litellm: - locations += ( - ' location /v1/ {\n' - f' proxy_pass {api}/v1/;\n' - ' proxy_set_header Host $host;\n' - ' proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n' - ' proxy_set_header X-Forwarded-Proto $scheme;\n' - ' proxy_read_timeout 600s;\n' - ' }\n' - ) - # `/` serves the UI when present, else the gateway (so hitting the host root - # still lands somewhere useful). Upgrade headers keep Open WebUI's websockets - # working; client_max_body_size 0 allows large uploads. - if ui: - root = f'http://{OPEN_WEBUI_SERVICE}:{OPEN_WEBUI_CONTAINER_PORT}' - elif litellm: - root = api - else: - root = '' - if root: - locations += ( - ' location / {\n' - f' proxy_pass {root};\n' - ' proxy_http_version 1.1;\n' - ' proxy_set_header Upgrade $http_upgrade;\n' - ' proxy_set_header Connection $connection_upgrade;\n' - ' proxy_set_header Host $host;\n' - ' proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n' - ' proxy_set_header X-Forwarded-Proto $scheme;\n' - ' }\n' - ) - return ( - 'map $http_upgrade $connection_upgrade {\n' - ' default upgrade;\n' - " '' close;\n" - '}\n\n' - 'server {\n' - f' listen {NGINX_CONTAINER_PORT};\n' - ' server_name _;\n' - ' client_max_body_size 0;\n' - f'{locations}' - '}\n' - ) - - -def _nginx_service( - host_port: int, - images: dict[str, str], - *, - aux_dir: str, - depends_on: list[str], - config_path: str | None = None, - config_hash: str | None = None, -) -> dict[str, Any]: - # BYO config (config_path) is mounted verbatim; otherwise the generated - # nginx.conf in the state dir is used. - mount = config_path or f'{aux_dir}/{NGINX_CONFIG_FILENAME}' - labels = {ENGINE_LABEL: 'nginx'} - if config_hash is not None: - # Same trick as LiteLLM: the conf is bind-mounted, so stamp its hash on a - # label to force a recreate when the routing changes. - labels[CONFIG_HASH_LABEL] = config_hash - service: dict[str, Any] = { - 'image': images['nginx'], - 'ports': [f'{host_port}:{NGINX_CONTAINER_PORT}'], - 'volumes': [f'{mount}:/etc/nginx/conf.d/default.conf:ro'], - 'restart': 'unless-stopped', - 'labels': labels, - } - if depends_on: - service['depends_on'] = sorted(depends_on) - return service - - def render_compose( deployments: list[Deployment], assignments: dict[str, list[int]], @@ -1534,15 +825,6 @@ def render_compose( DOCKER_TIMEOUT_CONVERGE = 1800.0 # compose up / down DOCKER_TIMEOUT_PULL = 3600.0 # pull, manifest inspect, in-container model pulls -#: Budget for reconciling dynamic routes against the gateway: listing, POSTs -#: and verification together. This default is the bootstrap budget (a fresh -#: gateway waits on Postgres health and runs DB migrations); it preserves the -#: previous 90 x 2 s listing retry. -ROUTE_RECONCILE_BOOTSTRAP_S = 180.0 -#: Budget when the gateway was already running before this apply (steady state). -#: Short, because the controller holds its host-wide lock while applying; on -#: expiry the change stays pending and the next applying operation retries. -ROUTE_RECONCILE_STEADY_S = 20.0 def _docker_timeout(args: list[str]) -> float: @@ -1761,24 +1043,6 @@ def _parse_ps(out: str) -> set[str]: return running -def set_master_key(env_path: Path, key: str) -> None: - """Replace the LiteLLM master key in ``env_path`` without losing DB routes. - - The first time the key changes, the old one is pinned as - ``LITELLM_SALT_KEY`` -- the value LiteLLM has been encrypting stored - credentials with. After that the salt stays put and only the key moves. - """ - if not key.startswith('sk-'): - # master_key() would silently replace it on the next render. - raise ValueError(f'{API_KEY_ENV} must start with "sk-" (LiteLLM rejects others)') - existing = parse_env_file(env_path) - values = {API_KEY_ENV: key} - old = existing.get(API_KEY_ENV, '').strip() - if old and old != key and not existing.get(SALT_KEY_ENV, '').strip(): - values[SALT_KEY_ENV] = old - write_env_file(env_path, values) - - class ComposeBackend(ConvergeScaffold): """Single-host docker compose backend (converge-style). diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py new file mode 100644 index 00000000..a15d174c --- /dev/null +++ b/infer_stack/leasing/gateway.py @@ -0,0 +1,732 @@ +"""The front door: the LiteLLM gateway, its routes and keys, and the UI and +proxy beside it. + +Engines (vLLM, Ollama) are the backend's business; everything a client +talks to is here. A backend hands the gateway where its models are (route +rows), and the gateway renders its Compose services and config, keeps the +route registry, reconciles dynamic routes through LiteLLM's admin API, and +manages the master key. The compose backend runs it beside its engines; the +kubeai backend runs it alone, in front of a cluster. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import yaml + +from ..config import PINNED_IMAGES +from ..env_utils import parse_env_file, write_env_file +from .models import Deployment, served_name +from .naming import ( + OLLAMA_CONTAINER_PORT, + VLLM_CONTAINER_PORT, + ollama_service_name, + ollama_service_name_for, + vllm_service_name, + vllm_service_name_for, +) +from .residency import ENGINE_LABEL + +LITELLM_CONTAINER_PORT = 4000 +LITELLM_CONFIG_FILENAME = 'litellm_config.yaml' +LITELLM_SERVICE = 'litellm' +API_KEY_ENV = 'LITELLM_MASTER_KEY' +# The key LiteLLM encrypts credentials stored in its database with. Unset, it +# uses the master key -- so rotating the master key would make every +# DB-stored route undecryptable. `set_master_key` pins it to the pre-rotation +# master key the first time the key changes; it must never change after that. +SALT_KEY_ENV = 'LITELLM_SALT_KEY' + +# Dynamic-routing (admin-API) extras. When dynamic routing is on, the gateway's +# route table is managed live via LiteLLM's admin API against a Postgres-backed +# model store, instead of a static config file. See render_compose + +# ComposeBackend._reconcile_routes and docs/litellm-gateway-routing.md. +LITELLM_ROUTES_FILENAME = 'litellm_routes.json' # rendered desired route set +# Append-only route registry for static-superset mode: accumulates the semantic +# route inputs (served name / engine / host) of every catalog *and* every live +# deployment ever merged, across all runbooks sharing this state dir. The gateway +# `model_list` is rendered from the whole registry, so a converge under one +# runbook's catalog can no longer strip another's still-live routes, and once +# every catalog has been merged once the rendered config is byte-stable (the +# gateway is never recreated). See docs/litellm-gateway-routing.md and +# ComposeBackend._update_route_registry. +LITELLM_REGISTRY_FILENAME = 'litellm_registry.json' +LITELLM_REGISTRY_VERSION = 1 +# A route-registry row for a server this project does not run: +# ``{'engine': UPSTREAM_ROUTE, 'served': , 'api_base': }``. +UPSTREAM_ROUTE = 'upstream' +POSTGRES_SERVICE = 'postgres-litellm' +POSTGRES_CONTAINER_PORT = 5432 +POSTGRES_DB_NAME = 'litellm' +POSTGRES_DB_USER = 'litellm' +DB_PASSWORD_ENV = 'LITELLM_DB_PASSWORD' # managed secret in the sidecar .env +# Marks a LiteLLM route as infer-stack-managed, so reconcile only ever deletes +# routes it created (never a model added by hand through the UI/admin API). +ROUTE_ID_PREFIX = 'isr-' + +#: Budget for reconciling dynamic routes against the gateway: listing, POSTs +#: and verification together. This default is the bootstrap budget (a fresh +#: gateway waits on Postgres health and runs DB migrations); it preserves the +#: previous 90 x 2 s listing retry. +ROUTE_RECONCILE_BOOTSTRAP_S = 180.0 +#: Budget when the gateway was already running before this apply (steady state). +#: Short, because the controller holds its host-wide lock while applying; on +#: expiry the change stays pending and the next applying operation retries. +ROUTE_RECONCILE_STEADY_S = 20.0 + + +def _vllm_route_entry( + model_name: str, served: str, api_base: str +) -> dict[str, Any]: + """One LiteLLM ``model_list`` entry routing ``model_name`` to a vLLM upstream. + + Shared by every render path (legacy per-deployment, catalog-superset, and + the route registry) so a registry-rendered entry can never drift from what + the catalog/deployment paths produce for the same endpoint.""" + return { + 'model_name': model_name, + 'litellm_params': { + 'model': f'openai/{served}', + 'api_base': api_base, + 'api_key': 'EMPTY', + }, + } + + +def _ollama_route_entry( + model_name: str, tag: str, api_base: str +) -> dict[str, Any]: + """One LiteLLM ``model_list`` entry routing ``model_name`` to an Ollama tag + (see :func:`_vllm_route_entry` for why this is factored out).""" + return { + 'model_name': model_name, + 'litellm_params': { + 'model': f'ollama/{tag}', + 'api_base': api_base, + }, + } + + +def _litellm_model_list( + deployments: list[Deployment], assignments: dict[str, list[int]] +) -> list[dict[str, Any]]: + """One LiteLLM ``model_list`` entry per served endpoint alias.""" + entries: list[dict[str, Any]] = [] + for deployment in sorted(deployments, key=lambda g: (g.created_at, g.id)): + if deployment.id not in assignments: + continue + if deployment.engine == 'vllm': + served = served_name(deployment) + api_base = f'http://{vllm_service_name(deployment)}:8000/v1' + for endpoint in sorted(deployment.served): + entries.append(_vllm_route_entry(endpoint, served, api_base)) + elif deployment.engine == 'ollama': + api_base = f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' + for endpoint, payload in sorted(deployment.served.items()): + tag = payload.get('model', endpoint) + entries.append(_ollama_route_entry(endpoint, tag, api_base)) + return entries + + +def _litellm_model_list_from_catalog(catalog: Any) -> list[dict[str, Any]]: + """A *static superset* ``model_list``: one route per catalog endpoint. + + Unlike :func:`_litellm_model_list` (which routes only the currently-placed + deployments), this routes *every* catalog endpoint to its deterministic + upstream host (:func:`vllm_service_name_for` / :func:`ollama_service_name_for`). + The resulting config therefore depends only on the catalog, not on which + models happen to be up — so acquiring/releasing a model leaves the gateway's + config (and its container) untouched (no blip). A route whose upstream is not + currently running simply errors/cools-down until it comes up; the + ``router_settings`` below make that warmup self-healing. ``/v1/models`` lists + the whole catalog (some upstreams down) rather than only the live set. + + This static-superset path is the default. Its one limitation — it cannot give + same-model ``--dedicated`` deployments distinct upstreams, and cannot route + non-catalog acquires without a config change — is addressed by the opt-in + *dynamic routing* mode (``dynamic_routing=True``), which manages routes live + via LiteLLM's admin API against a Postgres model store (see + :func:`_litellm_routes`, :meth:`ComposeBackend._reconcile_routes`, and + ``docs/litellm-gateway-routing.md``). The two are mutually exclusive per + converge; this function is used only when dynamic routing is off. + """ + entries: list[dict[str, Any]] = [] + for name in sorted(getattr(catalog, 'endpoints', {})): + try: + req = catalog.resolve_endpoint(name) + except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway + continue + if req.engine == 'vllm': + served = req.served.get('served_model_name') or name + api_base = ( + f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' + ) + entries.append(_vllm_route_entry(name, served, api_base)) + elif req.engine == 'ollama': + host = req.spec.get('host') or req.host + tag = req.served.get('model') or name + api_base = ( + f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' + ) + entries.append(_ollama_route_entry(name, tag, api_base)) + return entries + + +# -- Route registry (static-superset persistence) -------------------------- +# +# The registry stores *semantic* route inputs (served name / engine / host), +# never rendered LiteLLM entries — render derives entries through the same +# helpers the catalog/deployment paths use (:func:`_litellm_model_list_from_registry`), +# so a future renderer change propagates to old registry rows automatically. +# All functions here are pure; the backend owns the file I/O and locking. + + +def _registry_incoming_from_catalog(catalog: Any) -> dict[str, dict[str, Any]]: + """Semantic route rows for every resolvable endpoint of ``catalog``. + + Mirrors :func:`_litellm_model_list_from_catalog`'s iteration (unresolvable + endpoints skipped) but emits registry rows keyed by endpoint name. A vLLM + row carries only ``served`` (the upstream host is re-derived at render via + :func:`vllm_service_name_for`); an Ollama row carries ``model`` (tag) + + ``host``.""" + incoming: dict[str, dict[str, Any]] = {} + for name in sorted(getattr(catalog, 'endpoints', {})): + try: + req = catalog.resolve_endpoint(name) + except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway + continue + if req.engine == 'vllm': + served = req.served.get('served_model_name') or name + incoming[name] = {'engine': 'vllm', 'served': served} + elif req.engine == 'ollama': + host = req.spec.get('host') or req.host + tag = req.served.get('model') or name + incoming[name] = {'engine': 'ollama', 'model': tag, 'host': host} + return incoming + + +def _registry_incoming_from_deployments( + deployments: list[Deployment], assignments: dict[str, list[int]] +) -> dict[str, dict[str, Any]]: + """Semantic route rows for every *placed* deployment in ``assignments``. + + ``deployments`` is the full ``desired`` set (which spans all runbooks via + the shared ledger), so this keeps non-catalog / dedicated acquires routable + and — because the registry persists — routable past release. One row per key + of ``deployment.served`` (a coalesced deployment can back several endpoint + aliases). Only ``vllm``/``ollama`` engines contribute; ``RESERVED_ENGINE`` + and unknown engines render no service, so they contribute no row — exactly + as :func:`render_compose`'s service loop skips them. + + The vLLM ``served`` uses the same fallback chain as :func:`vllm_service_name` + (``spec['served_model_name'] or sorted(served)[0] or id``), so a + catalog-listed endpoint acquired live reduces to the identical row a catalog + merge produces — live-vs-released status never moves the rendered bytes.""" + incoming: dict[str, dict[str, Any]] = {} + for deployment in deployments: + if deployment.id not in assignments: + continue + if deployment.engine == 'vllm': + served = served_name(deployment) + for endpoint in sorted(deployment.served): + incoming[endpoint] = {'engine': 'vllm', 'served': served} + elif deployment.engine == 'ollama': + host = deployment.spec.get('host') or deployment.id + for endpoint, payload in sorted(deployment.served.items()): + tag = payload.get('model', endpoint) + incoming[endpoint] = { + 'engine': 'ollama', + 'model': tag, + 'host': host, + } + return incoming + + +def _litellm_model_list_from_registry( + registry: dict[str, Any] +) -> list[dict[str, Any]]: + """Render the gateway ``model_list`` from the whole accumulated registry. + + Iterates ``sorted(entries)`` (determinism, §8) and derives each upstream + ``api_base`` through the live naming helpers, so the registry never becomes + a rendered-config parse surface.""" + entries: list[dict[str, Any]] = [] + rows = registry.get('entries', {}) if isinstance(registry, dict) else {} + for name in sorted(rows): + row = rows[name] + if not isinstance(row, dict): + continue + engine = row.get('engine') + if engine == 'vllm': + served = row.get('served') or name + api_base = ( + f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' + ) + entries.append(_vllm_route_entry(name, served, api_base)) + elif engine == 'ollama': + tag = row.get('model') or name + host = row.get('host') or name + api_base = ( + f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' + ) + entries.append(_ollama_route_entry(name, tag, api_base)) + elif engine == UPSTREAM_ROUTE and row.get('api_base'): + # An OpenAI-compatible server this project does not run (a KubeAI + # cluster's gateway): the row carries its address and the name it + # serves the model under. + entries.append(_vllm_route_entry(name, row.get('served') or name, + str(row['api_base']))) + return entries + + +def _merge_route_registry( + existing: dict[str, Any], incoming: dict[str, dict[str, Any]] +) -> tuple[dict[str, Any], list[str]]: + """Merge ``incoming`` semantic rows into ``existing`` (append-only). + + Idempotent (merging identical rows is a no-op) and additive (never removes a + row). On a conflict — same key, different row — *incoming wins* and a warning + naming both definitions is emitted; the changed definition changes the + rendered bytes, which is the one justified recreate. The existing ``version`` + is preserved (an unknown version merged under is not silently rewritten to + the current schema; see :meth:`ComposeBackend._load_route_registry`).""" + version = LITELLM_REGISTRY_VERSION + entries: dict[str, dict[str, Any]] = {} + if isinstance(existing, dict): + version = existing.get('version', LITELLM_REGISTRY_VERSION) + prior = existing.get('entries') + if isinstance(prior, dict): + entries = {k: v for k, v in prior.items()} + warnings: list[str] = [] + for name in sorted(incoming): + row = incoming[name] + if name in entries and entries[name] != row: + warnings.append( + f"route {name!r} redefined: {entries[name]} -> {row} " + '(incoming wins; gateway will be recreated once)' + ) + entries[name] = row + return {'version': version, 'entries': entries}, warnings + + +def _seed_registry_from_litellm_config( + config_text: str, +) -> tuple[dict[str, Any], list[str]]: + """One-shot upgrade seed: recover registry rows from a rendered + ``litellm_config.yaml`` so the first post-upgrade converge does not strip + the other runbooks' routes. + + Single-format, migration-time only (no cross-version promise): ``openai/`` + inverts *exactly* to ``{engine: vllm, served}``. Ollama rows are skipped with + a warning — the host survives only as a non-invertible ``dns_slug`` inside + ``api_base`` — and re-enter the registry at the next converge that has them + in its catalog or live set. Anything else unparseable is likewise skipped.""" + entries: dict[str, dict[str, Any]] = {} + warnings: list[str] = [] + try: + data = yaml.safe_load(config_text) or {} + except Exception: # noqa: BLE001 - a torn file must not brick seeding + return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}}, [ + 'seed: litellm_config.yaml is unparseable; starting an empty registry' + ] + for entry in data.get('model_list', []) or []: + name = entry.get('model_name') + model = (entry.get('litellm_params') or {}).get('model', '') + if not name: + continue + if isinstance(model, str) and model.startswith('openai/'): + entries[name] = {'engine': 'vllm', 'served': model[len('openai/'):]} + elif isinstance(model, str) and model.startswith('ollama/'): + warnings.append( + f"seed: skipping Ollama route {name!r} (host not recoverable " + 'from the rendered api_base; it re-enters at its next converge)' + ) + else: + warnings.append(f'seed: skipping unparseable route {name!r}') + return {'version': LITELLM_REGISTRY_VERSION, 'entries': entries}, warnings + + +def _dump_route_registry(registry: dict[str, Any]) -> str: + """Canonical, byte-stable serialization (§3): sorted keys + trailing + newline. A nondeterministic dump would manufacture phantom hash changes.""" + return json.dumps(registry, sort_keys=True, indent=2) + '\n' + + +def _route_id(deployment_id: str, endpoint: str) -> str: + """Deterministic LiteLLM model id for one (deployment, endpoint) route. + + Stable across converges, so route reconcile (:meth:`ComposeBackend. + _reconcile_routes`) can identify one logical route across renders. A route + whose id disappears is deleted by exactly this id; a route whose id remains + but whose observable routing semantics drifted is replaced under the same id. + The ``isr-`` prefix marks it + infer-stack-managed so reconcile never deletes a model someone added by hand. + """ + digest = hashlib.sha256(f'{deployment_id}|{endpoint}'.encode()).hexdigest() + return f'{ROUTE_ID_PREFIX}{digest[:32]}' + + +def _litellm_routes( + deployments: list[Deployment], assignments: dict[str, list[int]] +) -> list[dict[str, Any]]: + """Desired LiteLLM route set for the *live* deployments (dynamic routing). + + One entry per (placed deployment, served endpoint), addressing the + deployment's **own** unique upstream service (:func:`vllm_service_name` with + ``unique=True``). Several dedicated deployments of the same model therefore + yield several entries that share one public ``model_name`` but point at + distinct upstreams — LiteLLM load-balances the alias across them, so each + runs on its own GPU while clients still ask for the single name. Each entry + carries a deterministic ``model_info.id`` (:func:`_route_id`) so applying the + set via the admin API is an idempotent diff, not fire-and-forget calls. + """ + entries: list[dict[str, Any]] = [] + for deployment in sorted(deployments, key=lambda g: (g.created_at, g.id)): + if deployment.id not in assignments: + continue + if deployment.engine == 'vllm': + served = served_name(deployment) + api_base = ( + f'http://{vllm_service_name(deployment, unique=True)}' + f':{VLLM_CONTAINER_PORT}/v1' + ) + for endpoint in sorted(deployment.served): + entries.append( + { + 'model_name': endpoint, + 'litellm_params': { + 'model': f'openai/{served}', + 'api_base': api_base, + 'api_key': 'EMPTY', + }, + 'model_info': {'id': _route_id(deployment.id, endpoint)}, + } + ) + elif deployment.engine == 'ollama': + api_base = ( + f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' + ) + for endpoint, payload in sorted(deployment.served.items()): + tag = payload.get('model', endpoint) + entries.append( + { + 'model_name': endpoint, + 'litellm_params': { + 'model': f'ollama/{tag}', + 'api_base': api_base, + }, + 'model_info': {'id': _route_id(deployment.id, endpoint)}, + } + ) + return entries + + +CONFIG_HASH_LABEL = 'infer-stack.config-hash' + + +def _postgres_service( + images: dict[str, str], state: dict[str, str] +) -> dict[str, Any]: + """Postgres backing LiteLLM's runtime model store (dynamic routing only). + + LiteLLM's admin API (``/model/new`` / ``/model/delete``) only functions with + ``STORE_MODEL_IN_DB=true`` + a database, so dynamic routing needs a DB. This + is an **internal** service (no published host port); LiteLLM reaches it on + the compose network at ``postgres-litellm:5432``. The password is the managed + :data:`DB_PASSWORD_ENV` secret in the sidecar ``.env`` (interpolated by + ``docker compose --env-file``), so it never appears literally in the YAML. + The healthcheck lets the litellm service ``depends_on`` it (condition: + service_healthy) so the gateway only starts once the DB can accept queries. + """ + data_path = state.get('postgres_litellm') or str( + Path(next(iter(state.values()), '.')).parent / 'postgres-litellm' + ) + return { + 'image': images.get('postgres', PINNED_IMAGES['postgres']), + 'environment': { + 'POSTGRES_USER': POSTGRES_DB_USER, + 'POSTGRES_PASSWORD': '${' + DB_PASSWORD_ENV + '}', + 'POSTGRES_DB': POSTGRES_DB_NAME, + }, + 'volumes': [f'{data_path}:/var/lib/postgresql/data'], + 'restart': 'unless-stopped', + 'labels': {ENGINE_LABEL: 'postgres'}, + 'healthcheck': { + 'test': [ + 'CMD-SHELL', + f'pg_isready -U {POSTGRES_DB_USER} -d {POSTGRES_DB_NAME}', + ], + 'interval': '5s', + 'timeout': '5s', + 'retries': 30, + 'start_period': '30s', + }, + } + + +def _litellm_service( + service_names: list[str], + host_port: int, + images: dict[str, str], + aux_dir: str, + master_key: str | None = None, + config_hash: str | None = None, + *, + dynamic_routing: bool = False, + salt_key: bool = False, +) -> dict[str, Any]: + # Reference the managed key via ${...} rather than baking the literal secret + # into the compose YAML. Its value lives in the sidecar .env next to the + # compose file (written by master_key()), which `docker compose --env-file` + # loads for interpolation — so the container and the readiness probe (which + # reads the same .env) still agree regardless of the caller's shell env. + key_value = ( + '${' + API_KEY_ENV + '}' + if master_key is not None + else '${' + API_KEY_ENV + ':-sk-local}' + ) + environment = {API_KEY_ENV: key_value} + if salt_key: + # Only when the .env has one: LiteLLM treats an EMPTY salt as a key, + # so a `${...:-}` default would silently change the encryption key. + environment[SALT_KEY_ENV] = '${' + SALT_KEY_ENV + '}' + if dynamic_routing: + # DB-backed runtime model store so the admin API (/model/new, + # /model/delete) works; the gateway then never needs recreating to learn + # a route. Both are read from the env by LiteLLM. The password is + # interpolated from the sidecar .env, so no secret lands in the YAML. + environment['DATABASE_URL'] = ( + f'postgresql://{POSTGRES_DB_USER}:${{{DB_PASSWORD_ENV}}}' + f'@{POSTGRES_SERVICE}:{POSTGRES_CONTAINER_PORT}/{POSTGRES_DB_NAME}' + ) + environment['STORE_MODEL_IN_DB'] = 'True' + labels = {ENGINE_LABEL: 'litellm'} + if config_hash is not None: + # LiteLLM reads its routing config once at startup; the file is bind- + # mounted, so a config change alone does NOT change this service's spec + # and `docker compose up -d` would leave the old container (and old + # routes) running. Stamping the config hash onto a label makes the spec + # change exactly when the config does, so converge recreates LiteLLM and + # it picks up new/removed aliases. Without this, coalescing a second + # alias onto a live deployment never becomes routable (readiness times out). + labels[CONFIG_HASH_LABEL] = config_hash + service: dict[str, Any] = { + 'image': images['litellm'], + 'command': [ + '--config', + '/etc/litellm/config.yaml', + '--port', + str(LITELLM_CONTAINER_PORT), + ], + 'ports': [f'{host_port}:{LITELLM_CONTAINER_PORT}'], + 'volumes': [f'{aux_dir}/{LITELLM_CONFIG_FILENAME}:/etc/litellm/config.yaml:ro'], + 'environment': environment, + 'restart': 'unless-stopped', + 'labels': labels, + } + if dynamic_routing: + # Wait for the DB to accept queries before the gateway boots; do NOT add + # per-model depends_on (that would churn the spec, i.e. blip, on every + # model change). The route table is filled in afterward via the API. + service['depends_on'] = { + POSTGRES_SERVICE: {'condition': 'service_healthy'} + } + elif service_names: + # Only wait on upstreams when there are any (zero models -> empty gateway). + service['depends_on'] = sorted(service_names) + return service + + +OPEN_WEBUI_SERVICE = 'open-webui' +OPEN_WEBUI_CONTAINER_PORT = 8080 + +NGINX_SERVICE = 'reverse-proxy' +NGINX_CONTAINER_PORT = 80 +NGINX_CONFIG_FILENAME = 'nginx.conf' + + +def _open_webui_service( + host_port: int, + images: dict[str, str], + state: dict[str, str], + master_key: str | None, + *, + openai_urls: list[str] | None = None, + ollama_urls: list[str] | None = None, + depends_on: list[str] | None = None, +) -> dict[str, Any]: + """A managed Open WebUI pointed at whatever front door is available. + + Open WebUI holds two independent kinds of connection, wired here from the + rendered services: + + * **OpenAI** (``openai_urls``) — the chat/completions front door. This is the + LiteLLM gateway when it is enabled (so every declared endpoint alias is + reachable at one URL); with LiteLLM off it falls back to the rendered + upstreams' own ``/v1`` (a single vLLM/Ollama service, or several joined as + ``OPENAI_API_BASE_URLS``). With nothing to point at, the OpenAI API is + disabled rather than left dangling. + * **Ollama** (``ollama_urls``) — the *native* Ollama API of any rendered + Ollama daemon. This is what lets you pull/run/delete models from the UI + and have the daemon load them on demand, independent of LiteLLM — i.e. a + true drop-in for a hand-run ``ollama`` + Open WebUI stack. + + The spec is kept as independent of which models are live as it can be: the + LiteLLM URL is fixed, and the Ollama daemon's service name is its stable + structural id, so adding/removing other models does not rewrite this service + and ``docker compose up -d`` leaves the UI running (the legacy "the UI never + blinks" behavior). Chat history persists under the data dir. + """ + # Reference the managed key via ${...} (resolved from the sidecar .env, see + # _litellm_service) instead of inlining the secret into the compose YAML. + key_value = ( + '${' + API_KEY_ENV + '}' + if master_key is not None + else '${' + API_KEY_ENV + ':-sk-local}' + ) + data_path = state.get('open_webui') or str( + Path(next(iter(state.values()), '.')).parent / 'open-webui' + ) + openai_urls = list(openai_urls or []) + ollama_urls = list(ollama_urls or []) + env: dict[str, str] = { + # Single-user workstation default; the port shouldn't be exposed + # publicly. Tracked as a knob in dev/leasing-followups.md. + 'WEBUI_AUTH': 'False', + } + if openai_urls: + env['ENABLE_OPENAI_API'] = 'True' + if len(openai_urls) == 1: + env['OPENAI_API_BASE_URL'] = openai_urls[0] + else: + env['OPENAI_API_BASE_URLS'] = ';'.join(openai_urls) + env['OPENAI_API_KEY'] = key_value + else: + env['ENABLE_OPENAI_API'] = 'False' + if ollama_urls: + env['ENABLE_OLLAMA_API'] = 'True' + if len(ollama_urls) == 1: + env['OLLAMA_BASE_URL'] = ollama_urls[0] + else: + env['OLLAMA_BASE_URLS'] = ';'.join(ollama_urls) + else: + env['ENABLE_OLLAMA_API'] = 'False' + service: dict[str, Any] = { + 'image': images['open_webui'], + 'ports': [f'{host_port}:{OPEN_WEBUI_CONTAINER_PORT}'], + 'environment': env, + 'volumes': [f'{data_path}:/app/backend/data'], + 'restart': 'unless-stopped', + 'labels': {ENGINE_LABEL: 'open-webui'}, + } + if depends_on: + service['depends_on'] = sorted(depends_on) + return service + + +def _nginx_conf(*, litellm: bool, ui: bool) -> str: + """A minimal HTTP reverse-proxy conf: one origin, path-routed. + + ``/v1/`` -> the LiteLLM gateway (the OpenAI API), ``/`` -> Open WebUI (or the + gateway when there's no UI). Plain HTTP — no TLS, no auth — so the value is + "one port, nothing to remember", not security. The ``map`` is valid here + because a ``conf.d/*.conf`` file is included in nginx's ``http`` context. + """ + api = f'http://{LITELLM_SERVICE}:{LITELLM_CONTAINER_PORT}' + locations = '' + if litellm: + locations += ( + ' location /v1/ {\n' + f' proxy_pass {api}/v1/;\n' + ' proxy_set_header Host $host;\n' + ' proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n' + ' proxy_set_header X-Forwarded-Proto $scheme;\n' + ' proxy_read_timeout 600s;\n' + ' }\n' + ) + # `/` serves the UI when present, else the gateway (so hitting the host root + # still lands somewhere useful). Upgrade headers keep Open WebUI's websockets + # working; client_max_body_size 0 allows large uploads. + if ui: + root = f'http://{OPEN_WEBUI_SERVICE}:{OPEN_WEBUI_CONTAINER_PORT}' + elif litellm: + root = api + else: + root = '' + if root: + locations += ( + ' location / {\n' + f' proxy_pass {root};\n' + ' proxy_http_version 1.1;\n' + ' proxy_set_header Upgrade $http_upgrade;\n' + ' proxy_set_header Connection $connection_upgrade;\n' + ' proxy_set_header Host $host;\n' + ' proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n' + ' proxy_set_header X-Forwarded-Proto $scheme;\n' + ' }\n' + ) + return ( + 'map $http_upgrade $connection_upgrade {\n' + ' default upgrade;\n' + " '' close;\n" + '}\n\n' + 'server {\n' + f' listen {NGINX_CONTAINER_PORT};\n' + ' server_name _;\n' + ' client_max_body_size 0;\n' + f'{locations}' + '}\n' + ) + + +def _nginx_service( + host_port: int, + images: dict[str, str], + *, + aux_dir: str, + depends_on: list[str], + config_path: str | None = None, + config_hash: str | None = None, +) -> dict[str, Any]: + # BYO config (config_path) is mounted verbatim; otherwise the generated + # nginx.conf in the state dir is used. + mount = config_path or f'{aux_dir}/{NGINX_CONFIG_FILENAME}' + labels = {ENGINE_LABEL: 'nginx'} + if config_hash is not None: + # Same trick as LiteLLM: the conf is bind-mounted, so stamp its hash on a + # label to force a recreate when the routing changes. + labels[CONFIG_HASH_LABEL] = config_hash + service: dict[str, Any] = { + 'image': images['nginx'], + 'ports': [f'{host_port}:{NGINX_CONTAINER_PORT}'], + 'volumes': [f'{mount}:/etc/nginx/conf.d/default.conf:ro'], + 'restart': 'unless-stopped', + 'labels': labels, + } + if depends_on: + service['depends_on'] = sorted(depends_on) + return service + + +def set_master_key(env_path: Path, key: str) -> None: + """Replace the LiteLLM master key in ``env_path`` without losing DB routes. + + The first time the key changes, the old one is pinned as + ``LITELLM_SALT_KEY`` -- the value LiteLLM has been encrypting stored + credentials with. After that the salt stays put and only the key moves. + """ + if not key.startswith('sk-'): + # master_key() would silently replace it on the next render. + raise ValueError(f'{API_KEY_ENV} must start with "sk-" (LiteLLM rejects others)') + existing = parse_env_file(env_path) + values = {API_KEY_ENV: key} + old = existing.get(API_KEY_ENV, '').strip() + if old and old != key and not existing.get(SALT_KEY_ENV, '').strip(): + values[SALT_KEY_ENV] = old + write_env_file(env_path, values) + + diff --git a/infer_stack/leasing/naming.py b/infer_stack/leasing/naming.py new file mode 100644 index 00000000..c2d58c82 --- /dev/null +++ b/infer_stack/leasing/naming.py @@ -0,0 +1,99 @@ +"""The names infer-stack gives the things it runs, in one place. + +A compose service, a Kubernetes object, and the upstream host a gateway +route points at are all derived from the served model name (or an Ollama +host). The engine side and the gateway side must agree on these exactly, so +both import them from here. +""" + +from __future__ import annotations + +import re + +from .models import Deployment, served_name + +#: The port every engine container serves on inside the Compose network. +VLLM_CONTAINER_PORT = 8000 +OLLAMA_CONTAINER_PORT = 11434 + + +def dns_slug(text: str) -> str: + """A lowercase ``[a-z0-9-]`` label safe as a compose service / DNS / + Kubernetes object name (shared by the compose and kubeai backends).""" + out = re.sub(r'[^a-z0-9]+', '-', str(text).lower()).strip('-') + return out or 'model' + + +_dns_slug = dns_slug # historical internal name + + +def vllm_service_name_for(served: str) -> str: + """Deterministic compose/DNS service name for a vLLM upstream: ``vllm-``. + + Derived purely from the served model name (vLLM's ``--served-model-name``, + the Open WebUI label, the alias the user chose), so it is identical whether + computed from a live :class:`Deployment` or from a catalog endpoint. That + stability is what lets the LiteLLM gateway carry a *static* route table (one + per catalog endpoint) whose upstream hosts match the containers when they + come up — so adding/removing models does not rewrite the gateway's config and + the gateway is never recreated (no "blip"); see :func:`_litellm_model_list`. + """ + return f'vllm-{_dns_slug(served)}' + + +def _unique_vllm_service_name(served: str, deployment_id: str) -> str: + """Per-deployment vLLM service/DNS name: ``vllm--``. + + The static-superset gateway needs a name derivable from the served model + *alone* (so a catalog route can address it without knowing the live + deployment) — but that deliberately drops the deployment id, which + **collapses every** ``--dedicated`` **deployment of one model onto a single + container** (hence one GPU). Dynamic routing manages the gateway's routes + live via the admin API, so the upstream host no longer has to be predictable + from the catalog. That frees us to give each deployment its **own** service, + so N dedicated deployments of one model become N containers on N GPUs. The + suffix is the deployment id's hex tail, keeping the name short and DNS-safe. + """ + tail = deployment_id.rsplit('-', 1)[-1][:8] or 'x' + return f'{vllm_service_name_for(served)}-{_dns_slug(tail)}' + + +def vllm_service_name(deployment: Deployment, *, unique: bool = False) -> str: + """Compose service name for a vLLM deployment (see :func:`vllm_service_name_for`). + + Default (``unique=False``, static-superset mode): deterministic from the + served model name only — *no* deployment-id suffix — so it matches the + gateway's pre-rendered route for that endpoint. Trade-off: two + *simultaneously desired* deployments that share a served name collide on this + name; under the static gateway the catalog endpoint is the unit, so that case + (including same-model ``--dedicated``) is unsupported. + + ``unique=True`` (dynamic-routing mode): append the deployment-id tail + (:func:`_unique_vllm_service_name`) so same-model dedicated deployments get + distinct containers/GPUs; the admin-API route table addresses each by name. + + Either way the container carries the ``infer-stack.deployment`` label. + :meth:`ComposeBackend.residency` correlates containers to deployments by that + label, so the choice of suffix does not affect it. (The lenient + :meth:`ComposeBackend.observe` still maps service names through the render + sidecar; it is for reporting, not for decisions that touch a GPU.) + """ + served = served_name(deployment) + if unique: + return _unique_vllm_service_name(served, deployment.id) + return vllm_service_name_for(served) + + +def ollama_service_name_for(host: str) -> str: + """Deterministic service name for an Ollama daemon: ``ollama-``. + + One daemon per host (Ollama coalesces tags onto it), so the host is the + stable key — matching :func:`vllm_service_name_for`'s role for vLLM so the + gateway's static route table addresses it regardless of which tags are live. + """ + return f'ollama-{_dns_slug(host)}' + + +def ollama_service_name(deployment: Deployment) -> str: + host = deployment.spec.get('host') or deployment.id + return ollama_service_name_for(host) diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index 1dadb691..bc5e9446 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -51,6 +51,8 @@ #: Labels every rendered service carries: its service name, and a behavioural #: fingerprint that changes only when the service's behaviour does. SERVICE_LABEL = 'infer-stack.service' +#: Which engine a service runs (``vllm``, ``ollama``, ``litellm``...). +ENGINE_LABEL = 'infer-stack.engine' FINGERPRINT_LABEL = 'infer-stack.fingerprint' #: Labels Docker Compose puts on every container of a project. COMPOSE_PROJECT_LABEL = 'com.docker.compose.project' diff --git a/tests/test_leasing_compose.py b/tests/test_leasing_compose.py index b7426f59..232c7537 100644 --- a/tests/test_leasing_compose.py +++ b/tests/test_leasing_compose.py @@ -929,7 +929,7 @@ def test_litellm_config_hash_label_tracks_model_list(tmp_path): (spec unchanged), so the new alias never became routable. Stamping the config hash onto a label makes converge recreate litellm on a config change. """ - from infer_stack.leasing.compose import CONFIG_HASH_LABEL + from infer_stack.leasing.gateway import CONFIG_HASH_LABEL def label(deployment): rc = render_compose( diff --git a/tests/test_leasing_dynamic_routing.py b/tests/test_leasing_dynamic_routing.py index 4235143c..036aa610 100644 --- a/tests/test_leasing_dynamic_routing.py +++ b/tests/test_leasing_dynamic_routing.py @@ -23,12 +23,12 @@ from infer_stack.hardware import simulate_inventory from infer_stack.leasing import ComposeBackend, render_compose -from infer_stack.leasing.compose import ( +from infer_stack.leasing.compose import vllm_service_name +from infer_stack.leasing.gateway import ( POSTGRES_SERVICE, ROUTE_ID_PREFIX, _litellm_routes, _route_id, - vllm_service_name, ) from infer_stack.leasing.models import Deployment, DeploymentState @@ -517,7 +517,7 @@ def test_apply_returns_true_once_routes_verify(tmp_path): def test_apply_uses_the_short_deadline_when_the_gateway_was_already_up(tmp_path): - from infer_stack.leasing.compose import ( + from infer_stack.leasing.gateway import ( ROUTE_RECONCILE_BOOTSTRAP_S, ROUTE_RECONCILE_STEADY_S, ) diff --git a/tests/test_leasing_route_registry.py b/tests/test_leasing_route_registry.py index a911243f..b29d8574 100644 --- a/tests/test_leasing_route_registry.py +++ b/tests/test_leasing_route_registry.py @@ -21,7 +21,7 @@ from infer_stack.hardware import simulate_inventory from infer_stack.leasing import Catalog, ComposeBackend, render_compose -from infer_stack.leasing.compose import ( +from infer_stack.leasing.gateway import ( CONFIG_HASH_LABEL, LITELLM_CONFIG_FILENAME, LITELLM_REGISTRY_FILENAME, diff --git a/tests/test_leasing_secrets.py b/tests/test_leasing_secrets.py index e8d1dc3b..08a2f2d8 100644 --- a/tests/test_leasing_secrets.py +++ b/tests/test_leasing_secrets.py @@ -14,8 +14,11 @@ from infer_stack.env_utils import parse_env_file from infer_stack.hardware import simulate_inventory from infer_stack.leasing import Controller, Ledger, SqliteStore -from infer_stack.leasing.compose import ( - API_KEY_ENV, SALT_KEY_ENV, ComposeBackend, set_master_key, +from infer_stack.leasing.compose import ComposeBackend +from infer_stack.leasing.gateway import ( + API_KEY_ENV, + SALT_KEY_ENV, + set_master_key, ) from infer_stack.leasing.profile import ProfileMismatch from infer_stack.leasing.residency import FINGERPRINT_LABEL From ba93ad952c22eb711a083ac6fc3ab45bbcfdfe50 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 14:22:33 -0400 Subject: [PATCH 008/141] G-C: a Gateway class owns the front door's state The gateway's methods move off ComposeBackend into leasing.gateway.Gateway: keys (master_key, rotate, restore_env, gateway_accepts, db_password), the route registry (load, merge, save), dynamic-route reconciliation (_reconcile_routes, _list_managed_routes, _post_route), and access(). ComposeBackend owns one (`backend.gateway`), keeps thin public delegations (master_key, access, litellm_port, ...), and stores the settings the gateway reads (litellm, ui, dynamic_routing, reverse proxy, ports, http, clock, sleep) on it through properties, so there is one copy of each. The registry merge no longer reads backend state: each backend supplies its rows (compose from its catalog and deployments; kubeai persists its upstream rows with merge_route_registry, the `routes seed` path), and the gateway merges, persists and renders them. The upstream_routes side-channel is gone. compose.py 2818 -> 2483 lines. Suite 898 passed; ty and flake8 clean. Live on the guest: a gateway-only compose apply answered 200 with its key, and `secrets rotate` accepted the new key and rejected the old. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/backends/kubeai.py | 11 +- infer_stack/cli/commands_leasing.py | 14 +- infer_stack/leasing/compose.py | 559 ++++++-------------------- infer_stack/leasing/gateway.py | 476 +++++++++++++++++++++- tests/test_leasing_admission.py | 4 +- tests/test_leasing_dynamic_routing.py | 24 +- tests/test_leasing_profile.py | 2 +- tests/test_leasing_secrets.py | 2 +- 8 files changed, 616 insertions(+), 476 deletions(-) diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index f57e2559..2385bd10 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -381,10 +381,12 @@ def _render_gateway(self, rendered: RenderedModels) -> None: from ..leasing.gateway import UPSTREAM_ROUTE base = self._upstream_url() - self.gateway.upstream_routes = { + # This backend's route rows, persisted like any other (the registry is + # append-only, so a released Model stays routable, as on compose). + self.gateway.merge_route_registry({ alias: {'engine': UPSTREAM_ROUTE, 'served': name, 'api_base': base} for alias, name in rendered.request_names.items() - } + }) # Gateway only: no engines on this host, so nothing to place. self.gateway.converge([], apply=False) @@ -613,8 +615,9 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: if self.gateway is not None: # Ready means ready the way a client sees it: the alias, through # the gateway, with its key. - base, model = f'{self.gateway._gateway_base()}/v1', endpoint - headers = self.gateway._auth_headers() + front = self.gateway.gateway # the runner's leasing.gateway.Gateway + base, model = f'{front._gateway_base()}/v1', endpoint + headers = front._auth_headers() else: base, model = self.base_url, model_name_for(_served_name(deployment)) headers = None diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index a97f64c2..e5bbc64c 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2384,7 +2384,7 @@ def main(cls, argv=True, **kwargs): 'the gateway out of its database' ) if key == 'LITELLM_MASTER_KEY': - from ..leasing.compose import set_master_key + from ..leasing.gateway import set_master_key try: # Pins the salt first, so DB-stored routes stay readable. set_master_key(env_path, value) @@ -2485,7 +2485,7 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) backend = _require_compose_backend(controller) - registry = backend._load_route_registry() + registry = backend.gateway._load_route_registry() entries = registry.get('entries', {}) live = _live_endpoints(controller) @@ -2558,7 +2558,7 @@ class RoutesPruneCLI(_ApprovalMixin): def main(cls, argv=True, **kwargs): from ..diff_prompt import confirm_writes from ..leasing.backend import ConvergeAborted - from ..leasing.compose import ( + from ..leasing.gateway import ( LITELLM_REGISTRY_VERSION, _dump_route_registry, _registry_incoming_from_catalog, @@ -2578,7 +2578,7 @@ def prune_plan() -> tuple[dict, dict, list[str]]: if backend.catalog is not None: keep.update(_registry_incoming_from_catalog(backend.catalog)) keep.update(_registry_incoming_from_deployments(desired, plan.assignments)) - current = backend._load_route_registry().get('entries', {}) + current = backend.gateway._load_route_registry().get('entries', {}) return current, keep, sorted(set(current) - set(keep)) # Preview outside the lock (the prompt must not hold it); the change @@ -2595,7 +2595,7 @@ def prune_plan() -> tuple[dict, dict, list[str]]: for name in dropped: print(f' - {name}') ok = confirm_writes( - {backend._registry_file: _dump_route_registry(pruned)}, + {backend.gateway._registry_file: _dump_route_registry(pruned)}, assume_yes=False, title='infer-stack routes prune', ) @@ -2613,7 +2613,7 @@ def change(): entries = {k: v for k, v in current.items() if k not in drop} with backend._converge_lock(): backend._atomic_write( - backend._registry_file, + backend.gateway._registry_file, _dump_route_registry( {'version': LITELLM_REGISTRY_VERSION, 'entries': entries}), ) @@ -2686,7 +2686,7 @@ def main(cls, argv=True, **kwargs): ) def change(): - before = set(backend._load_route_registry().get('entries', {})) + before = set(backend.gateway._load_route_registry().get('entries', {})) backend.merge_route_registry(incoming) return sorted(set(incoming) - before) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index b012a327..c3fdac03 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -48,26 +48,21 @@ import yaml from ..config import DEFAULT_PORTS, PINNED_IMAGES, default_state_paths -from ..env_utils import ensure_secret, parse_env_file, write_env_file +from ..env_utils import parse_env_file from ..probe import openai_ready from ..profile_runtime import simulator_args, vllm_args from .backend import ConvergeScaffold, Readiness from .gateway import ( + Gateway, API_KEY_ENV, - DB_PASSWORD_ENV, - _dump_route_registry, LITELLM_CONFIG_FILENAME, LITELLM_CONTAINER_PORT, _litellm_model_list, _litellm_model_list_from_catalog, _litellm_model_list_from_registry, - LITELLM_REGISTRY_FILENAME, - LITELLM_REGISTRY_VERSION, _litellm_routes, - LITELLM_ROUTES_FILENAME, _litellm_service, LITELLM_SERVICE, - _merge_route_registry, _nginx_conf, NGINX_CONFIG_FILENAME, _nginx_service, @@ -78,12 +73,9 @@ POSTGRES_SERVICE, _registry_incoming_from_catalog, _registry_incoming_from_deployments, - ROUTE_ID_PREFIX, ROUTE_RECONCILE_BOOTSTRAP_S, ROUTE_RECONCILE_STEADY_S, SALT_KEY_ENV, - _seed_registry_from_litellm_config, - set_master_key, ) from .launch import env_string, fill, translate_legacy from .models import Deployment, is_reservation, served_name @@ -1101,15 +1093,19 @@ def __init__( if http is None: import requests http = requests - self.http = http + # The front door: gateway settings, keys, routes. Created before the + # settings below, which are stored on it. + self.gateway = Gateway( + self.state_dir, + ports={**DEFAULT_PORTS, **(ports or {})}, + litellm=litellm, ui=ui, reverse_proxy=reverse_proxy, + reverse_proxy_port=reverse_proxy_port, dynamic_routing=dynamic_routing, + http=http, sleep=sleep, clock=clock, + ) # Called with a one-line message during long steps (image pulls); the # TUI sets it. Always also logged. self.progress: Callable[[str], None] | None = None - # alias -> UPSTREAM_ROUTE row, set by a backend that uses this one only - # as its gateway (the kubeai backend). Merged into the route registry. - self.upstream_routes: dict[str, dict[str, Any]] = {} self.images = {**PINNED_IMAGES, **(images or {})} - self.ports = {**DEFAULT_PORTS, **(ports or {})} # Merge over the defaults (not replace) so a caller-supplied partial # state dict — tests, embedders — still resolves every cache-mount key. self.state = {**default_state_paths(), **(state or {})} @@ -1117,10 +1113,6 @@ def __init__( self.reserved = tuple(reserved) self.project = project self.skip_display = skip_display - self.litellm = litellm - self.ui = ui - self.reverse_proxy = reverse_proxy - self.reverse_proxy_port = reverse_proxy_port self.reverse_proxy_config = reverse_proxy_config # Set by use_profile(): the published proxy config content. self._profile_proxy_text: str | None = None @@ -1136,9 +1128,6 @@ def __init__( # against a Postgres-backed model store, instead of a static config file. # Gives each deployment its own upstream (so same-model --dedicated # deployments land on distinct GPUs) with no gateway recreation/blip. - self.dynamic_routing = dynamic_routing - self._sleep = sleep - self._clock = clock self.last_errors: list[str] = [] self.last_unplaced: set[str] = set() # desired deployment ids placement skipped self.last_assignments: dict[str, list[int]] = {} # deployment id -> GPU ids @@ -1178,104 +1167,112 @@ def inventory(self, value: dict[str, Any] | None) -> None: def compose_file(self) -> Path: return self.state_dir / COMPOSE_FILENAME + # -- the front door (leasing.gateway) ------------------------------------ + # Settings the gateway owns. Kept as attributes of the backend because + # profiles and callers set them here; they are stored in one place. + @property - def _state_file(self) -> Path: - return self.state_dir / STATE_FILENAME + def http(self) -> Any: + return self.gateway.http + + @http.setter + def http(self, value: Any) -> None: + self.gateway.http = value @property - def litellm_port(self) -> int: - return self.ports.get('litellm', DEFAULT_PORTS['litellm']) + def _clock(self) -> Callable[[], float]: + return self.gateway._clock + + @_clock.setter + def _clock(self, value: Callable[[], float]) -> None: + self.gateway._clock = value @property - def ui_port(self) -> int: - return self.ports.get('open_webui', DEFAULT_PORTS['open_webui']) + def _sleep(self) -> Callable[[float], None]: + return self.gateway._sleep + + @_sleep.setter + def _sleep(self, value: Callable[[float], None]) -> None: + self.gateway._sleep = value @property - def _env_path(self) -> Path: - return self.state_dir / '.env' + def litellm(self) -> bool: + return self.gateway.litellm + + @litellm.setter + def litellm(self, value: bool) -> None: + self.gateway.litellm = value @property - def _routes_file(self) -> Path: - return self.state_dir / LITELLM_ROUTES_FILENAME + def ui(self) -> bool: + return self.gateway.ui + + @ui.setter + def ui(self, value: bool) -> None: + self.gateway.ui = value @property - def _registry_file(self) -> Path: - return self.state_dir / LITELLM_REGISTRY_FILENAME + def dynamic_routing(self) -> bool: + return self.gateway.dynamic_routing - def master_key(self) -> str: - """The managed LiteLLM master key. + @dynamic_routing.setter + def dynamic_routing(self, value: bool) -> None: + self.gateway.dynamic_routing = value - infer-stack manages this secret in the state dir's ``.env``: reused if - already present (you may pin your own ``sk-`` key there), otherwise - generated and persisted. The caller doesn't need to invent or export it - — it is baked into the LiteLLM service, used by the readiness probe, and - shipped in the env-file descriptor (``infer-stack env KEY`` prints it). - """ - existing = parse_env_file(self._env_path) - key = ensure_secret(existing, API_KEY_ENV, prefix='sk-') - if key != existing.get(API_KEY_ENV): - write_env_file(self._env_path, {API_KEY_ENV: key}) - return key + @property + def reverse_proxy(self) -> bool: + return self.gateway.reverse_proxy - def rotate_master_key(self) -> dict[str, str | None]: - """Write a fresh master key to the ``.env``; return the values it replaced. + @reverse_proxy.setter + def reverse_proxy(self, value: bool) -> None: + self.gateway.reverse_proxy = value - Only the file: the gateway and Open WebUI pick it up when the next apply - recreates them (their fingerprints hash the key). The caller holds the - publication lock and passes the return value to - :meth:`restore_env` if that apply does not happen. - """ - before = parse_env_file(self._env_path) - set_master_key(self._env_path, ensure_secret({}, API_KEY_ENV, prefix='sk-')) - return {k: before.get(k) for k in (API_KEY_ENV, SALT_KEY_ENV)} + @property + def reverse_proxy_port(self) -> int: + return self.gateway.reverse_proxy_port - def restore_env(self, values: dict[str, str | None]) -> None: - """Put back what :meth:`rotate_master_key` replaced.""" - from ..env_utils import remove_env_keys + @reverse_proxy_port.setter + def reverse_proxy_port(self, value: int) -> None: + self.gateway.reverse_proxy_port = value + + @property + def ports(self) -> dict[str, int]: + return self.gateway.ports + + @ports.setter + def ports(self, value: dict[str, int]) -> None: + self.gateway.ports = value - write_env_file(self._env_path, {k: v for k, v in values.items() if v is not None}) - remove_env_keys(self._env_path, [k for k, v in values.items() if v is None]) + @property + def litellm_port(self) -> int: + return self.gateway.litellm_port + + @property + def ui_port(self) -> int: + return self.gateway.ui_port + + def master_key(self) -> str: + return self.gateway.master_key() + + def rotate_master_key(self) -> dict[str, str | None]: + return self.gateway.rotate_master_key() + + def restore_env(self, values: dict[str, str | None]) -> None: + self.gateway.restore_env(values) def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: - """Does the gateway accept ``key``? ``None`` if it never answered. + return self.gateway.gateway_accepts(key, wait=wait) - Polls ``/v1/models`` for up to ``wait`` seconds while the gateway is - unreachable or still starting. A definite no is 401/403, or the 400 - LiteLLM actually answers a wrong key with (measured on the pinned image). - """ - deadline = self._clock() + wait - while True: - try: - resp = self.http.get( - f'{self._gateway_base()}/v1/models', - headers={'Authorization': f'Bearer {key}'}, timeout=10.0, - ) - status = getattr(resp, 'status_code', 0) - except Exception: # noqa: BLE001 - not up yet - status = 0 - if status == 200: - return True - if status in (400, 401, 403): - return False - if self._clock() >= deadline: - return None - self._sleep(2.0) - - def db_password(self) -> str: - """The managed Postgres password for LiteLLM's model store. - - Same managed-secret pattern as :meth:`master_key`: reused if already - present in the state-dir ``.env`` (you may pin your own), else generated - and persisted. ``docker compose --env-file`` interpolates it into the - postgres + litellm services, so it never appears literally in the YAML. - Only used when ``dynamic_routing`` is on. ``token_urlsafe`` output is safe - inside the ``postgresql://`` URL (no ``@ : /`` characters). - """ - existing = parse_env_file(self._env_path) - pw = ensure_secret(existing, DB_PASSWORD_ENV) - if pw != existing.get(DB_PASSWORD_ENV): - write_env_file(self._env_path, {DB_PASSWORD_ENV: pw}) - return pw + def merge_route_registry(self, incoming: dict[str, dict[str, Any]]) -> dict[str, Any]: + return self.gateway.merge_route_registry(incoming) + + def access(self, endpoints: list[str]) -> dict[str, Any] | None: + """Where a client reaches these endpoints: the front door.""" + return self.gateway.access(endpoints) + + @property + def _state_file(self) -> Path: + return self.state_dir / STATE_FILENAME def _ensure_state_dir(self) -> None: """Raise a legible error if the state dir could not be created. @@ -1421,8 +1418,8 @@ def _compose(self, args: list[str]) -> str: # directory* (wherever infer-stack was invoked), not the state dir, so we # point it explicitly. Only when present: a litellm-less stack never # writes one, and a missing --env-file path is a hard error. - if self._env_path.exists(): - cmd += ['--env-file', str(self._env_path)] + if self.gateway._env_path.exists(): + cmd += ['--env-file', str(self.gateway._env_path)] cmd += ['-p', self.project, '-f', str(self.compose_file)] return self.run([*cmd, *args]) @@ -1478,7 +1475,7 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A if self.litellm and self.dynamic_routing: # The DB secret must exist before rendering, so docker compose # --env-file can interpolate ${LITELLM_DB_PASSWORD} at apply time. - self.db_password() + self.gateway.db_password() route_registry = None if self.litellm and not self.dynamic_routing: # Unconditional in static-superset mode: `self.catalog` may be None; @@ -1490,7 +1487,7 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A desired, plan.assignments, images=self.images, ports=self.ports, state=self.state, litellm=self.litellm, litellm_port=self.litellm_port, litellm_master_key=self.master_key() if self.litellm else None, - litellm_salt_key=SALT_KEY_ENV in parse_env_file(self._env_path), + litellm_salt_key=SALT_KEY_ENV in parse_env_file(self.gateway._env_path), ui=self.ui, ui_port=self.ui_port, reverse_proxy=self.reverse_proxy, reverse_proxy_port=self.reverse_proxy_port, reverse_proxy_config=self.reverse_proxy_config, aux_dir=self.state_dir, @@ -1516,9 +1513,9 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A if rendered.nginx_config is not None: planned[self.state_dir / NGINX_CONFIG_FILENAME] = rendered.nginx_config if rendered.litellm_routes is not None: - planned[self._routes_file] = json.dumps(rendered.litellm_routes, indent=2) + planned[self.gateway._routes_file] = json.dumps(rendered.litellm_routes, indent=2) fingerprints = stamp_fingerprints( - rendered.compose, files=planned, env_file=self._env_path, + rendered.compose, files=planned, env_file=self.gateway._env_path, ) planned[self.compose_file] = yaml.safe_dump(rendered.compose, sort_keys=False) return {'plan': plan, 'rendered': rendered, 'planned': planned, @@ -1702,132 +1699,27 @@ def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: except Exception: return '' - def _load_route_registry(self) -> dict[str, Any]: - """Read the route registry, tolerantly (fail-open — a broken registry - must never block a converge). - - Missing file → seed from the live ``litellm_config.yaml`` if present - (upgrade migration, §6), else an empty registry. An *unknown* schema - version whose ``entries`` still parses as a name→row map is preserved - as-is (render what's understood, warn, do NOT rewrite) rather than - reseeded, so a binary rollback doesn't discard the accumulated union. - Only a structurally unusable file (not a map / garbage JSON) falls back - to seeding.""" - from .._log import logger - - if not self._registry_file.exists(): - config = self.state_dir / LITELLM_CONFIG_FILENAME - if config.exists(): - seeded, warnings = _seed_registry_from_litellm_config( - config.read_text() - ) - for w in warnings: - logger.warning(' route registry: {}', w) - logger.info( - ' route registry: seeded {} vLLM route(s) from {}', - len(seeded['entries']), config.name, - ) - return seeded - return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} - try: - data = json.loads(self._registry_file.read_text()) - except (OSError, json.JSONDecodeError) as exc: - logger.warning( - ' route registry: {} is unreadable ({}); rebuilding from seed', - self._registry_file.name, exc, - ) - data = None - if not isinstance(data, dict) or not isinstance( - data.get('entries'), dict - ): - config = self.state_dir / LITELLM_CONFIG_FILENAME - if config.exists(): - seeded, warnings = _seed_registry_from_litellm_config( - config.read_text() - ) - for w in warnings: - logger.warning(' route registry: {}', w) - return seeded - return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} - version = data.get('version') - if version != LITELLM_REGISTRY_VERSION: - logger.warning( - ' route registry: unknown schema version {!r} in {} — ' - 'rendering as-is without rewrite (fields this renderer does ' - 'not understand are ignored)', - version, self._registry_file.name, - ) - return data - def _merged_route_registry( self, desired: list[Deployment], assignments: dict[str, list[int]] ) -> dict[str, Any]: - """The route registry merged with the catalog and ``desired``, in memory.""" - from .._log import logger + """The route registry merged with this backend's rows, in memory. - existing = self._load_route_registry() + The rows are this backend's to supply: every catalog endpoint, and every + placed deployment (``desired`` spans all runbooks via the shared ledger, + so a live cross-runbook deployment stays routable, and past release). + """ incoming: dict[str, dict[str, Any]] = {} if self.catalog is not None: incoming.update(_registry_incoming_from_catalog(self.catalog)) - # `desired` spans all runbooks via the shared ledger, so this keeps every - # live cross-runbook deployment routable (and, via persistence, routable - # past release). incoming.update(_registry_incoming_from_deployments(desired, assignments)) - # Routes to servers another backend runs (see UPSTREAM_ROUTE). - incoming.update(self.upstream_routes) - merged, warnings = _merge_route_registry(existing, incoming) - for w in warnings: - logger.warning(' route registry: {}', w) - return merged - - def _save_route_registry(self, merged: dict[str, Any]) -> None: - """Persist a merged registry if it changed (under the converge flock).""" - from .._log import logger - - existing = self._load_route_registry() - if merged == existing: - return - prior = existing.get('entries', {}) if isinstance(existing, dict) else {} - added = sorted(set(merged['entries']) - set(prior)) - updated = sorted( - k for k in merged['entries'] if k in prior and merged['entries'][k] != prior[k] - ) - if added: - logger.info(' route registry: +{} route(s): {}', len(added), ', '.join(added)) - if updated: - logger.info(' route registry: updated route(s): {}', ', '.join(updated)) - self._atomic_write(self._registry_file, _dump_route_registry(merged)) + return self.gateway.merged_route_registry(incoming) def _update_route_registry( self, desired: list[Deployment], assignments: dict[str, list[int]] ) -> dict[str, Any]: """Merge and persist the route registry; return it (kept for callers).""" merged = self._merged_route_registry(desired, assignments) - self._save_route_registry(merged) - return merged - - def merge_route_registry( - self, incoming: dict[str, dict[str, Any]] - ) -> dict[str, Any]: - """Public write path for out-of-converge registry seeds (``routes seed``). - - Takes the converge flock, read-merge-writes the registry, and returns the - merged dict. ``converge`` only ever merges the invoking process's own - catalog, so a standalone caller (seeding a *sibling* runbook's catalog) - needs this to fold extra rows in before the follow-up ``reconcile`` - renders+applies. The flock here and the one the subsequent converge takes - are sequential acquisitions, not nested — no reentrancy concern.""" - from .._log import logger - - with self._converge_lock(): - existing = self._load_route_registry() - merged, warnings = _merge_route_registry(existing, incoming) - for w in warnings: - logger.warning(' route registry: {}', w) - if merged != existing: - self._atomic_write( - self._registry_file, _dump_route_registry(merged) - ) + self.gateway._save_route_registry(merged) return merged def converge(self, desired: list[Deployment], *, apply: bool = True, placement=None): @@ -1883,7 +1775,7 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N if docs['addresses'] is not None and self.on_addresses is not None: self.on_addresses(docs['addresses']) if docs['route_registry'] is not None: - self._save_route_registry(docs['route_registry']) + self.gateway._save_route_registry(docs['route_registry']) for path, text in planned.items(): if path != self.compose_file: self._atomic_write(path, text) @@ -1960,7 +1852,7 @@ def apply(self) -> bool: networks=doc.get('networks') or {}, ) if dynamic and 'litellm' in services: - return self._reconcile_routes( + return self.gateway._reconcile_routes( deadline_s=ROUTE_RECONCILE_STEADY_S if 'litellm' in outcome.kept_services else ROUTE_RECONCILE_BOOTSTRAP_S, ) @@ -2264,208 +2156,6 @@ def _service_running(self, service: str) -> bool: # -- dynamic routing (admin API) -------------------------------------- - def _gateway_base(self) -> str: - return f'http://127.0.0.1:{self.litellm_port}' - - def _auth_headers(self) -> dict[str, str]: - return {'Authorization': f'Bearer {self.master_key()}'} - - def _desired_routes(self) -> list[dict[str, Any]]: - """The rendered desired route set (litellm_routes.json), or empty.""" - try: - data = json.loads(self._routes_file.read_text()) - except (FileNotFoundError, json.JSONDecodeError): - return [] - return data if isinstance(data, list) else [] - - def _reconcile_routes( - self, *, deadline_s: float = ROUTE_RECONCILE_BOOTSTRAP_S, delay: float = 2.0, - ) -> bool: - """Make the live gateway's managed routes match the rendered route set. - - The render half wrote the desired routes (one per live deployment× - endpoint) to ``litellm_routes.json``; this is the apply half. List the - gateway's current models, add the missing routes and delete the ones no - longer desired -- through the admin API, with **no** container restart -- - then list again to verify both ids and routing semantics, re-diffing and - retrying failed calls until the table matches or the budget runs out. - - Properties this relies on: - - * **Idempotent.** A redundant apply re-diffs to the same set and does - nothing. - * **Drift-healing.** Routes lost to a gateway/DB restart reappear in the - diff and are re-added; stale routes from a prior run (still in the DB) - are deleted because they're no longer desired. - * **Co-existence.** Only routes infer-stack created (id prefix ``isr-``) - are ever deleted, so a model added by hand through the UI/API is left - alone. - - **Bounded and reported.** Everything -- listing retries while the gateway - starts, every POST, and the final verification -- shares one wall-clock - budget, ``deadline_s``. A retry count alone would not bound it: a listing - can take 10 s and a POST 30 s. Returns ``True`` only when the verified - managed route set equals the desired set; any failure is logged and - returns ``False`` rather than raising, so the caller decides whether an - unverified route set blocks anything. - """ - from .._log import logger - - deadline = self._clock() + max(0.0, deadline_s) - desired = { - r['model_info']['id']: r - for r in self._desired_routes() - if isinstance(r.get('model_info'), dict) and r['model_info'].get('id') - } - desired_semantics = {rid: self._route_semantics(route) - for rid, route in desired.items()} - rounds = 0 - while True: - current = self._list_managed_routes(deadline=deadline, delay=delay) - if current is None: - logger.warning( - 'dynamic routing: route set not reconciled and verified within ' - '{:g}s; leaving it for the next apply', deadline_s, - ) - return False - mismatched = sorted( - rid for rid in desired.keys() & current.keys() - if desired_semantics[rid] != current[rid] - ) - to_add_ids = sorted((desired.keys() - current.keys()) | set(mismatched)) - to_delete = sorted((current.keys() - desired.keys()) | set(mismatched)) - to_add = [desired[rid] for rid in to_add_ids] - if not (to_add or to_delete): - return True # this listing is the verification - if rounds: - logger.info('dynamic routing: route set still differs; retrying') - rounds += 1 - ok = True - # A same-id semantic drift must be removed before it can be re-added; - # model/new is not an update API on every LiteLLM release. - for rid in to_delete: - # ok_if_missing: with a shared gateway, another converge may have - # deleted this route already; "not found in db" means the desired - # end-state (route gone) is reached, so don't treat it as an error. - ok &= self._post_route( - '/model/delete', {'id': rid}, rid, ok_if_missing=True, - deadline=deadline, - ) - for route in to_add: - ok &= self._post_route( - '/model/new', route, route.get('model_name'), deadline=deadline, - ) - logger.info( - 'dynamic routing: +{} route(s), -{} route(s), ~{} replacement(s) ' - '(now {} desired)', - len(to_add), len(to_delete), len(mismatched), len(desired), - ) - if not ok: - # A transient admin-API failure: spend the rest of the budget - # re-diffing rather than giving up with most of it unused. - if deadline - self._clock() <= delay: - return False - self._sleep(delay) - - @staticmethod - def _route_semantics(route: dict[str, Any]) -> dict[str, Any]: - """Observable route fields infer-stack owns and must verify. - - LiteLLM's model-info response contains additional database/runtime fields - and may redact credentials. The public alias, upstream model, and - upstream base URL are the routing semantics infer-stack can both set and - reliably observe. A matching managed id with different values here is - drift and is replaced, not accepted as healthy. - """ - params = route.get('litellm_params') or {} - return { - 'model_name': route.get('model_name'), - 'model': params.get('model'), - 'api_base': params.get('api_base'), - } - - def _list_managed_routes( - self, *, deadline: float, delay: float - ) -> dict[str, dict[str, Any]] | None: - """Observable semantics of infer-stack-managed gateway routes. - - Retries while the gateway is unreachable, until ``deadline`` (a value of - ``self._clock``). Each request's own timeout is capped by the time left. - Returns ``None`` if no listing succeeded in time. - """ - while True: - remaining = deadline - self._clock() - if remaining <= 0: - return None - resp = None - try: - resp = self.http.get( - f'{self._gateway_base()}/v1/model/info', - headers=self._auth_headers(), - timeout=min(10.0, remaining), - ) - except Exception: # noqa: BLE001 - the gateway may still be starting - resp = None - if resp is not None and getattr(resp, 'status_code', 0) == 200: - routes: dict[str, dict[str, Any]] = {} - for m in (resp.json().get('data') or []): - rid = (m.get('model_info') or {}).get('id') - if isinstance(rid, str) and rid.startswith(ROUTE_ID_PREFIX): - routes[rid] = self._route_semantics(m) - return routes - if deadline - self._clock() <= delay: - return None - self._sleep(delay) - - def _post_route( - self, - path: str, - payload: dict[str, Any], - label: Any, - *, - ok_if_missing: bool = False, - deadline: float | None = None, - ) -> bool: - """POST one admin-API call (``/model/new`` or ``/model/delete``). - - Returns whether it reached its desired end state. A failure is logged, - not raised, so the remaining calls still run. ``ok_if_missing`` accepts a - "model not found" response (a delete whose target is already gone). - The request timeout is capped by the time left before ``deadline``. - """ - from .._log import logger - - timeout = 30.0 - if deadline is not None: - remaining = deadline - self._clock() - if remaining <= 0: - logger.warning( - 'dynamic routing: POST {} {} skipped: route deadline passed', - path, label, - ) - return False - timeout = min(timeout, remaining) - try: - resp = self.http.post( - f'{self._gateway_base()}{path}', - headers=self._auth_headers(), - json=payload, - timeout=timeout, - ) - except Exception as ex: # noqa: BLE001 - one bad call must not abort apply - logger.warning('dynamic routing: POST {} {} error: {}', path, label, ex) - return False - if getattr(resp, 'status_code', 0) >= 300: - body = str(getattr(resp, 'text', '')) - if ok_if_missing and 'not found' in body.lower(): - return True - logger.warning( - 'dynamic routing: POST {} {} -> {} {}', - path, label, resp.status_code, body[:200], - ) - return False - return True - # -- published profile (see leasing/profile.py) --------------------------- REVERSE_PROXY_SNAPSHOT = 'reverse-proxy.conf' @@ -2677,31 +2367,6 @@ def observe(self) -> set[str]: services = self._load_sidecar().get('services', {}) return {services[name] for name in running if name in services} - def access(self, endpoints: list[str]) -> dict[str, Any] | None: - """Where a client reaches these endpoints, for the env-file descriptor. - - With the LiteLLM front door, that is one ``base_url`` and the request - model name is the endpoint alias itself. With LiteLLM off there is no - single base URL, but a managed Open WebUI (if on) is still a useful - access point, so report just its URL rather than ``None``. - """ - if not self.litellm: - if self.ui: - return {'ui_url': f'http://127.0.0.1:{self.ui_port}'} - return None - info: dict[str, Any] = { - 'base_url': f'http://127.0.0.1:{self.litellm_port}/v1', - 'api_key_env': API_KEY_ENV, - 'api_key': self.master_key(), - 'request_names': {ep: ep for ep in endpoints}, - } - if self.ui: - info['ui_url'] = f'http://127.0.0.1:{self.ui_port}' - if self.reverse_proxy: - # The unified front door: one origin, UI at / and the API at /v1. - info['proxy_url'] = f'http://127.0.0.1:{self.reverse_proxy_port}' - return info - def _ensure_ollama_tag( self, deployment: Deployment, endpoint: str ) -> str | None: diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index a15d174c..0da21a7f 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -13,13 +13,16 @@ import hashlib import json +import time from pathlib import Path -from typing import Any +from typing import Any, Callable import yaml from ..config import PINNED_IMAGES -from ..env_utils import parse_env_file, write_env_file +from ..config import DEFAULT_PORTS +from ..env_utils import ensure_secret, parse_env_file, write_env_file +from .backend import ConvergeScaffold from .models import Deployment, served_name from .naming import ( OLLAMA_CONTAINER_PORT, @@ -730,3 +733,472 @@ def set_master_key(env_path: Path, key: str) -> None: write_env_file(env_path, values) +class Gateway(ConvergeScaffold): + """The front door's state: settings, the managed keys, the route registry. + + Owned by the backend that runs it (``ComposeBackend.gateway``). The + backend supplies route rows for what it serves; this merges, persists and + renders them, reconciles dynamic routes through LiteLLM's admin API, and + keeps the ``.env`` secrets. It shares the backend's state directory (and + so its converge lock), because the gateway's files live beside the + compose project that runs it. + """ + + def __init__( + self, + state_dir: str | Path, + *, + ports: dict[str, int], + litellm: bool = True, + ui: bool = True, + reverse_proxy: bool = False, + reverse_proxy_port: int = 80, + dynamic_routing: bool = False, + http: Any = None, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + ): + self.state_dir = Path(state_dir) + self.ports = ports + self.litellm = litellm + self.ui = ui + self.reverse_proxy = reverse_proxy + self.reverse_proxy_port = reverse_proxy_port + self.dynamic_routing = dynamic_routing + if http is None: + import requests + http = requests + self.http = http + self._sleep = sleep + self._clock = clock + self.assume_yes = True + + @property + def litellm_port(self) -> int: + return self.ports.get('litellm', DEFAULT_PORTS['litellm']) + + @property + def ui_port(self) -> int: + return self.ports.get('open_webui', DEFAULT_PORTS['open_webui']) + + @property + def _env_path(self) -> Path: + return self.state_dir / '.env' + + @property + def _routes_file(self) -> Path: + return self.state_dir / LITELLM_ROUTES_FILENAME + + @property + def _registry_file(self) -> Path: + return self.state_dir / LITELLM_REGISTRY_FILENAME + + def master_key(self) -> str: + """The managed LiteLLM master key. + + infer-stack manages this secret in the state dir's ``.env``: reused if + already present (you may pin your own ``sk-`` key there), otherwise + generated and persisted. The caller doesn't need to invent or export it + — it is baked into the LiteLLM service, used by the readiness probe, and + shipped in the env-file descriptor (``infer-stack env KEY`` prints it). + """ + existing = parse_env_file(self._env_path) + key = ensure_secret(existing, API_KEY_ENV, prefix='sk-') + if key != existing.get(API_KEY_ENV): + write_env_file(self._env_path, {API_KEY_ENV: key}) + return key + + def rotate_master_key(self) -> dict[str, str | None]: + """Write a fresh master key to the ``.env``; return the values it replaced. + + Only the file: the gateway and Open WebUI pick it up when the next apply + recreates them (their fingerprints hash the key). The caller holds the + publication lock and passes the return value to + :meth:`restore_env` if that apply does not happen. + """ + before = parse_env_file(self._env_path) + set_master_key(self._env_path, ensure_secret({}, API_KEY_ENV, prefix='sk-')) + return {k: before.get(k) for k in (API_KEY_ENV, SALT_KEY_ENV)} + + def restore_env(self, values: dict[str, str | None]) -> None: + """Put back what :meth:`rotate_master_key` replaced.""" + from ..env_utils import remove_env_keys + + write_env_file(self._env_path, {k: v for k, v in values.items() if v is not None}) + remove_env_keys(self._env_path, [k for k, v in values.items() if v is None]) + + def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: + """Does the gateway accept ``key``? ``None`` if it never answered. + + Polls ``/v1/models`` for up to ``wait`` seconds while the gateway is + unreachable or still starting. A definite no is 401/403, or the 400 + LiteLLM actually answers a wrong key with (measured on the pinned image). + """ + deadline = self._clock() + wait + while True: + try: + resp = self.http.get( + f'{self._gateway_base()}/v1/models', + headers={'Authorization': f'Bearer {key}'}, timeout=10.0, + ) + status = getattr(resp, 'status_code', 0) + except Exception: # noqa: BLE001 - not up yet + status = 0 + if status == 200: + return True + if status in (400, 401, 403): + return False + if self._clock() >= deadline: + return None + self._sleep(2.0) + + def db_password(self) -> str: + """The managed Postgres password for LiteLLM's model store. + + Same managed-secret pattern as :meth:`master_key`: reused if already + present in the state-dir ``.env`` (you may pin your own), else generated + and persisted. ``docker compose --env-file`` interpolates it into the + postgres + litellm services, so it never appears literally in the YAML. + Only used when ``dynamic_routing`` is on. ``token_urlsafe`` output is safe + inside the ``postgresql://`` URL (no ``@ : /`` characters). + """ + existing = parse_env_file(self._env_path) + pw = ensure_secret(existing, DB_PASSWORD_ENV) + if pw != existing.get(DB_PASSWORD_ENV): + write_env_file(self._env_path, {DB_PASSWORD_ENV: pw}) + return pw + + def _load_route_registry(self) -> dict[str, Any]: + """Read the route registry, tolerantly (fail-open — a broken registry + must never block a converge). + + Missing file → seed from the live ``litellm_config.yaml`` if present + (upgrade migration, §6), else an empty registry. An *unknown* schema + version whose ``entries`` still parses as a name→row map is preserved + as-is (render what's understood, warn, do NOT rewrite) rather than + reseeded, so a binary rollback doesn't discard the accumulated union. + Only a structurally unusable file (not a map / garbage JSON) falls back + to seeding.""" + from .._log import logger + + if not self._registry_file.exists(): + config = self.state_dir / LITELLM_CONFIG_FILENAME + if config.exists(): + seeded, warnings = _seed_registry_from_litellm_config( + config.read_text() + ) + for w in warnings: + logger.warning(' route registry: {}', w) + logger.info( + ' route registry: seeded {} vLLM route(s) from {}', + len(seeded['entries']), config.name, + ) + return seeded + return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} + try: + data = json.loads(self._registry_file.read_text()) + except (OSError, json.JSONDecodeError) as exc: + logger.warning( + ' route registry: {} is unreadable ({}); rebuilding from seed', + self._registry_file.name, exc, + ) + data = None + if not isinstance(data, dict) or not isinstance( + data.get('entries'), dict + ): + config = self.state_dir / LITELLM_CONFIG_FILENAME + if config.exists(): + seeded, warnings = _seed_registry_from_litellm_config( + config.read_text() + ) + for w in warnings: + logger.warning(' route registry: {}', w) + return seeded + return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} + version = data.get('version') + if version != LITELLM_REGISTRY_VERSION: + logger.warning( + ' route registry: unknown schema version {!r} in {} — ' + 'rendering as-is without rewrite (fields this renderer does ' + 'not understand are ignored)', + version, self._registry_file.name, + ) + return data + + def _save_route_registry(self, merged: dict[str, Any]) -> None: + """Persist a merged registry if it changed (under the converge flock).""" + from .._log import logger + + existing = self._load_route_registry() + if merged == existing: + return + prior = existing.get('entries', {}) if isinstance(existing, dict) else {} + added = sorted(set(merged['entries']) - set(prior)) + updated = sorted( + k for k in merged['entries'] if k in prior and merged['entries'][k] != prior[k] + ) + if added: + logger.info(' route registry: +{} route(s): {}', len(added), ', '.join(added)) + if updated: + logger.info(' route registry: updated route(s): {}', ', '.join(updated)) + self._atomic_write(self._registry_file, _dump_route_registry(merged)) + + def merge_route_registry( + self, incoming: dict[str, dict[str, Any]] + ) -> dict[str, Any]: + """Public write path for out-of-converge registry seeds (``routes seed``). + + Takes the converge flock, read-merge-writes the registry, and returns the + merged dict. ``converge`` only ever merges the invoking process's own + catalog, so a standalone caller (seeding a *sibling* runbook's catalog) + needs this to fold extra rows in before the follow-up ``reconcile`` + renders+applies. The flock here and the one the subsequent converge takes + are sequential acquisitions, not nested — no reentrancy concern.""" + from .._log import logger + + with self._converge_lock(): + existing = self._load_route_registry() + merged, warnings = _merge_route_registry(existing, incoming) + for w in warnings: + logger.warning(' route registry: {}', w) + if merged != existing: + self._atomic_write( + self._registry_file, _dump_route_registry(merged) + ) + return merged + + def _gateway_base(self) -> str: + return f'http://127.0.0.1:{self.litellm_port}' + + def _auth_headers(self) -> dict[str, str]: + return {'Authorization': f'Bearer {self.master_key()}'} + + def _desired_routes(self) -> list[dict[str, Any]]: + """The rendered desired route set (litellm_routes.json), or empty.""" + try: + data = json.loads(self._routes_file.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return [] + return data if isinstance(data, list) else [] + + def _reconcile_routes( + self, *, deadline_s: float = ROUTE_RECONCILE_BOOTSTRAP_S, delay: float = 2.0, + ) -> bool: + """Make the live gateway's managed routes match the rendered route set. + + The render half wrote the desired routes (one per live deployment× + endpoint) to ``litellm_routes.json``; this is the apply half. List the + gateway's current models, add the missing routes and delete the ones no + longer desired -- through the admin API, with **no** container restart -- + then list again to verify both ids and routing semantics, re-diffing and + retrying failed calls until the table matches or the budget runs out. + + Properties this relies on: + + * **Idempotent.** A redundant apply re-diffs to the same set and does + nothing. + * **Drift-healing.** Routes lost to a gateway/DB restart reappear in the + diff and are re-added; stale routes from a prior run (still in the DB) + are deleted because they're no longer desired. + * **Co-existence.** Only routes infer-stack created (id prefix ``isr-``) + are ever deleted, so a model added by hand through the UI/API is left + alone. + + **Bounded and reported.** Everything -- listing retries while the gateway + starts, every POST, and the final verification -- shares one wall-clock + budget, ``deadline_s``. A retry count alone would not bound it: a listing + can take 10 s and a POST 30 s. Returns ``True`` only when the verified + managed route set equals the desired set; any failure is logged and + returns ``False`` rather than raising, so the caller decides whether an + unverified route set blocks anything. + """ + from .._log import logger + + deadline = self._clock() + max(0.0, deadline_s) + desired = { + r['model_info']['id']: r + for r in self._desired_routes() + if isinstance(r.get('model_info'), dict) and r['model_info'].get('id') + } + desired_semantics = {rid: self._route_semantics(route) + for rid, route in desired.items()} + rounds = 0 + while True: + current = self._list_managed_routes(deadline=deadline, delay=delay) + if current is None: + logger.warning( + 'dynamic routing: route set not reconciled and verified within ' + '{:g}s; leaving it for the next apply', deadline_s, + ) + return False + mismatched = sorted( + rid for rid in desired.keys() & current.keys() + if desired_semantics[rid] != current[rid] + ) + to_add_ids = sorted((desired.keys() - current.keys()) | set(mismatched)) + to_delete = sorted((current.keys() - desired.keys()) | set(mismatched)) + to_add = [desired[rid] for rid in to_add_ids] + if not (to_add or to_delete): + return True # this listing is the verification + if rounds: + logger.info('dynamic routing: route set still differs; retrying') + rounds += 1 + ok = True + # A same-id semantic drift must be removed before it can be re-added; + # model/new is not an update API on every LiteLLM release. + for rid in to_delete: + # ok_if_missing: with a shared gateway, another converge may have + # deleted this route already; "not found in db" means the desired + # end-state (route gone) is reached, so don't treat it as an error. + ok &= self._post_route( + '/model/delete', {'id': rid}, rid, ok_if_missing=True, + deadline=deadline, + ) + for route in to_add: + ok &= self._post_route( + '/model/new', route, route.get('model_name'), deadline=deadline, + ) + logger.info( + 'dynamic routing: +{} route(s), -{} route(s), ~{} replacement(s) ' + '(now {} desired)', + len(to_add), len(to_delete), len(mismatched), len(desired), + ) + if not ok: + # A transient admin-API failure: spend the rest of the budget + # re-diffing rather than giving up with most of it unused. + if deadline - self._clock() <= delay: + return False + self._sleep(delay) + + @staticmethod + def _route_semantics(route: dict[str, Any]) -> dict[str, Any]: + """Observable route fields infer-stack owns and must verify. + + LiteLLM's model-info response contains additional database/runtime fields + and may redact credentials. The public alias, upstream model, and + upstream base URL are the routing semantics infer-stack can both set and + reliably observe. A matching managed id with different values here is + drift and is replaced, not accepted as healthy. + """ + params = route.get('litellm_params') or {} + return { + 'model_name': route.get('model_name'), + 'model': params.get('model'), + 'api_base': params.get('api_base'), + } + + def _list_managed_routes( + self, *, deadline: float, delay: float + ) -> dict[str, dict[str, Any]] | None: + """Observable semantics of infer-stack-managed gateway routes. + + Retries while the gateway is unreachable, until ``deadline`` (a value of + ``self._clock``). Each request's own timeout is capped by the time left. + Returns ``None`` if no listing succeeded in time. + """ + while True: + remaining = deadline - self._clock() + if remaining <= 0: + return None + resp = None + try: + resp = self.http.get( + f'{self._gateway_base()}/v1/model/info', + headers=self._auth_headers(), + timeout=min(10.0, remaining), + ) + except Exception: # noqa: BLE001 - the gateway may still be starting + resp = None + if resp is not None and getattr(resp, 'status_code', 0) == 200: + routes: dict[str, dict[str, Any]] = {} + for m in (resp.json().get('data') or []): + rid = (m.get('model_info') or {}).get('id') + if isinstance(rid, str) and rid.startswith(ROUTE_ID_PREFIX): + routes[rid] = self._route_semantics(m) + return routes + if deadline - self._clock() <= delay: + return None + self._sleep(delay) + + def _post_route( + self, + path: str, + payload: dict[str, Any], + label: Any, + *, + ok_if_missing: bool = False, + deadline: float | None = None, + ) -> bool: + """POST one admin-API call (``/model/new`` or ``/model/delete``). + + Returns whether it reached its desired end state. A failure is logged, + not raised, so the remaining calls still run. ``ok_if_missing`` accepts a + "model not found" response (a delete whose target is already gone). + The request timeout is capped by the time left before ``deadline``. + """ + from .._log import logger + + timeout = 30.0 + if deadline is not None: + remaining = deadline - self._clock() + if remaining <= 0: + logger.warning( + 'dynamic routing: POST {} {} skipped: route deadline passed', + path, label, + ) + return False + timeout = min(timeout, remaining) + try: + resp = self.http.post( + f'{self._gateway_base()}{path}', + headers=self._auth_headers(), + json=payload, + timeout=timeout, + ) + except Exception as ex: # noqa: BLE001 - one bad call must not abort apply + logger.warning('dynamic routing: POST {} {} error: {}', path, label, ex) + return False + if getattr(resp, 'status_code', 0) >= 300: + body = str(getattr(resp, 'text', '')) + if ok_if_missing and 'not found' in body.lower(): + return True + logger.warning( + 'dynamic routing: POST {} {} -> {} {}', + path, label, resp.status_code, body[:200], + ) + return False + return True + + def access(self, endpoints: list[str]) -> dict[str, Any] | None: + """Where a client reaches these endpoints, for the env-file descriptor. + + With the LiteLLM front door, that is one ``base_url`` and the request + model name is the endpoint alias itself. With LiteLLM off there is no + single base URL, but a managed Open WebUI (if on) is still a useful + access point, so report just its URL rather than ``None``. + """ + if not self.litellm: + if self.ui: + return {'ui_url': f'http://127.0.0.1:{self.ui_port}'} + return None + info: dict[str, Any] = { + 'base_url': f'http://127.0.0.1:{self.litellm_port}/v1', + 'api_key_env': API_KEY_ENV, + 'api_key': self.master_key(), + 'request_names': {ep: ep for ep in endpoints}, + } + if self.ui: + info['ui_url'] = f'http://127.0.0.1:{self.ui_port}' + if self.reverse_proxy: + # The unified front door: one origin, UI at / and the API at /v1. + info['proxy_url'] = f'http://127.0.0.1:{self.reverse_proxy_port}' + return info + + def merged_route_registry(self, incoming: dict[str, dict[str, Any]]) -> dict[str, Any]: + """The registry with ``incoming`` rows merged in, in memory (no write).""" + from .._log import logger + + merged, warnings = _merge_route_registry(self._load_route_registry(), incoming) + for w in warnings: + logger.warning(' route registry: {}', w) + return merged diff --git a/tests/test_leasing_admission.py b/tests/test_leasing_admission.py index 75630b51..dbbb552f 100644 --- a/tests/test_leasing_admission.py +++ b/tests/test_leasing_admission.py @@ -317,7 +317,7 @@ def test_declined_approval_of_a_coalescing_acquire_changes_nothing(tmp_path): first = ctl.acquire('x', cat.resolve_names(['a']), wait=False) gid = first.deployments[0].id served = dict(ledger.get_deployment(gid).served) - registry = ctl.backend._registry_file.read_text() if ctl.backend._registry_file.exists() else None + registry = ctl.backend.gateway._registry_file.read_text() if ctl.backend.gateway._registry_file.exists() else None def decline(planned): raise ConvergeAborted('no') @@ -327,7 +327,7 @@ def decline(planned): ctl.acquire('y', cat.resolve_names(['b']), wait=False) # would add alias b assert len(ledger.status()[0]) == 1 assert ledger.get_deployment(gid).served == served - now = ctl.backend._registry_file.read_text() if ctl.backend._registry_file.exists() else None + now = ctl.backend.gateway._registry_file.read_text() if ctl.backend.gateway._registry_file.exists() else None assert now == registry diff --git a/tests/test_leasing_dynamic_routing.py b/tests/test_leasing_dynamic_routing.py index 036aa610..4000dd3b 100644 --- a/tests/test_leasing_dynamic_routing.py +++ b/tests/test_leasing_dynamic_routing.py @@ -238,7 +238,7 @@ def test_reconcile_replaces_same_id_route_with_wrong_semantics(tmp_path): # Simulate DB drift / an earlier endpoint definition with the same stable id. gw.models[rid]['litellm_params']['model'] = 'ollama/wrong-tag' before = len(gw.calls) - assert be._reconcile_routes() is True + assert be.gateway._reconcile_routes() is True assert gw.models[rid]['litellm_params']['model'] == 'openai/smol' assert gw.calls[before:] == [('delete', rid), ('new', rid)] @@ -300,12 +300,12 @@ def test_reconcile_leaves_unmanaged_models_alone(tmp_path): def test_db_password_is_persisted_and_reused(tmp_path): be = make_backend(tmp_path, RecordingGateway()) - pw1 = be.db_password() + pw1 = be.gateway.db_password() assert pw1 # rewritten only if missing -> stable across calls and a fresh backend - assert be.db_password() == pw1 + assert be.gateway.db_password() == pw1 be2 = make_backend(tmp_path, RecordingGateway()) - assert be2.db_password() == pw1 + assert be2.gateway.db_password() == pw1 # -- no blip on the UPSTREAMS too (the readiness-killing churn) ------------- @@ -380,9 +380,9 @@ def post(self, url, **kw): monkeypatch.setattr( _log.logger, 'warning', lambda *a, **k: warnings.append((a, k)) ) - be._post_route('/model/delete', {'id': 'isr-x'}, 'isr-x', ok_if_missing=True) + be.gateway._post_route('/model/delete', {'id': 'isr-x'}, 'isr-x', ok_if_missing=True) assert warnings == [] # already gone -> no warning - be._post_route('/model/delete', {'id': 'isr-x'}, 'isr-x') + be.gateway._post_route('/model/delete', {'id': 'isr-x'}, 'isr-x') assert warnings # same response without the flag -> warns @@ -431,9 +431,9 @@ def test_reconcile_reports_success_only_when_routes_verify(tmp_path): gw = RecordingGateway() be = make_backend(tmp_path, gw) be.converge([a], apply=False) - assert be._reconcile_routes() is True + assert be.gateway._reconcile_routes() is True assert _managed(gw) == {_route_id(a.id, 'smol')} - assert be._reconcile_routes() is True # idempotent: nothing to change, still verified + assert be.gateway._reconcile_routes() is True # idempotent: nothing to change, still verified def test_reconcile_against_unreachable_gateway_is_bounded_by_its_deadline(tmp_path): @@ -443,7 +443,7 @@ def test_reconcile_against_unreachable_gateway_is_bounded_by_its_deadline(tmp_pa gw = UnreachableGateway(time) be = _timed_backend(tmp_path, gw, time) be.converge([dep('grp-aaaaaa', served='smol')], apply=False) - assert be._reconcile_routes(deadline_s=25.0) is False + assert be.gateway._reconcile_routes(deadline_s=25.0) is False assert time.now <= 25.0 assert gw.gets >= 2 # it did retry within the budget @@ -458,7 +458,7 @@ def post(self, url, **kw): time = FakeTime() be = _timed_backend(tmp_path, RejectingGateway(), time) be.converge([dep('grp-aaaaaa', served='smol')], apply=False) - assert be._reconcile_routes(deadline_s=20.0) is False + assert be.gateway._reconcile_routes(deadline_s=20.0) is False assert time.now <= 20.0 @@ -477,7 +477,7 @@ def post(self, url, **kw): be = _timed_backend(tmp_path, gw, time) a = dep('grp-aaaaaa', served='smol') be.converge([a], apply=False) - assert be._reconcile_routes(deadline_s=20.0) is True + assert be.gateway._reconcile_routes(deadline_s=20.0) is True assert _managed(gw) == {_route_id(a.id, 'smol')} @@ -496,7 +496,7 @@ def post(self, url, **kw): be = _timed_backend(tmp_path, SlowGateway(), time) be.converge([dep('grp-aaaaaa', served='smol')], apply=False) - be._reconcile_routes(deadline_s=10.0) + be.gateway._reconcile_routes(deadline_s=10.0) assert seen and all(t <= 10.0 - 4.0 for t in seen) # never the default 30 s diff --git a/tests/test_leasing_profile.py b/tests/test_leasing_profile.py index 32b864a9..1a1f5f8a 100644 --- a/tests/test_leasing_profile.py +++ b/tests/test_leasing_profile.py @@ -528,7 +528,7 @@ def test_a_publish_that_never_commits_leaves_no_append_only_state(tmp_path): ledger, ctl = controller(tmp_path, catalog=a) ctl.gc() before = ledger.profile() - registry = ctl.backend._registry_file + registry = ctl.backend.gateway._registry_file registry_before = registry.read_text() if registry.exists() else None def crash(*args, **kw): diff --git a/tests/test_leasing_secrets.py b/tests/test_leasing_secrets.py index 08a2f2d8..f5a730c0 100644 --- a/tests/test_leasing_secrets.py +++ b/tests/test_leasing_secrets.py @@ -55,7 +55,7 @@ def make(tmp_path): def env(ctl): - return parse_env_file(ctl.backend._env_path) + return parse_env_file(ctl.backend.gateway._env_path) def gateway(ctl): From 96ffaf5e594ee67192f2da358cf7ab029859ec1a Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 14:38:49 -0400 Subject: [PATCH 009/141] G-D: render_compose's front door becomes gateway.render_front_door render_compose renders the engine services, then merges in what render_front_door returns: the LiteLLM service and config (static, registry or dynamic routes), Postgres for dynamic routing, Open WebUI and the reverse proxy. The engines pass only what the front door needs from them (their service names for the legacy depends_on, and in-network URLs for a UI without a gateway). Verified byte-identical: six configurations (bare, gateway, gateway+UI+ proxy, route registry with an upstream row, dynamic routing, UI only), 27 services, rendered with the code before and after. Suite 898 passed x3, ty and flake8 clean. compose.py 2483 -> 2360 lines. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/leasing/compose.py | 165 ++++--------------------------- infer_stack/leasing/gateway.py | 173 +++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 144 deletions(-) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index c3fdac03..d564b7f4 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -53,29 +53,16 @@ from ..profile_runtime import simulator_args, vllm_args from .backend import ConvergeScaffold, Readiness from .gateway import ( - Gateway, - API_KEY_ENV, LITELLM_CONFIG_FILENAME, - LITELLM_CONTAINER_PORT, - _litellm_model_list, - _litellm_model_list_from_catalog, - _litellm_model_list_from_registry, - _litellm_routes, - _litellm_service, LITELLM_SERVICE, - _nginx_conf, NGINX_CONFIG_FILENAME, - _nginx_service, - NGINX_SERVICE, - _open_webui_service, - OPEN_WEBUI_SERVICE, - _postgres_service, - POSTGRES_SERVICE, - _registry_incoming_from_catalog, - _registry_incoming_from_deployments, ROUTE_RECONCILE_BOOTSTRAP_S, ROUTE_RECONCILE_STEADY_S, SALT_KEY_ENV, + Gateway, + _registry_incoming_from_catalog, + _registry_incoming_from_deployments, + render_front_door, ) from .launch import env_string, fill, translate_legacy from .models import Deployment, is_reservation, served_name @@ -667,133 +654,23 @@ def render_compose( ollama_native_urls.append(f'http://{name}:{OLLAMA_CONTAINER_PORT}') service_map[name] = deployment.id - litellm_config = None - litellm_routes = None - # The front door (gateway + UI) is rendered whenever it's enabled, even with - # zero models — it's a standing entry point, not a per-model service. So - # releasing/evicting every model leaves an empty gateway (and an empty Open - # WebUI picker) up instead of tearing the whole stack down; only an explicit - # `stack down` removes it. With no models the model_list is simply empty. - if litellm: - # Three route-table strategies, in order of preference: - # * DYNAMIC ROUTING: the rendered config is a STATIC base (empty - # model_list); the real routes live in Postgres and are applied to the - # running gateway via the admin API (see _reconcile_routes). The config - # hash never changes as models come/go, so the gateway is never - # recreated — no blip, and per-deployment routing works (so same-model - # --dedicated deployments each get their own upstream). - # * ROUTE REGISTRY (static-superset default from ComposeBackend): render - # from the whole accumulated registry (every catalog + live deployment - # ever merged, across all runbooks). Byte-stable once seeded, so the - # gateway is never recreated and a cross-catalog converge can no longer - # strip another runbook's routes. The backend loads/merges/writes the - # registry and passes the merged dict in; this function stays pure. - # * STATIC SUPERSET (catalog): one route per catalog endpoint to a - # deterministic host; config depends only on the catalog, so the - # gateway is not recreated as models come/go (no blip) but same-model - # dedicated collapses to one upstream. Unreachable from ComposeBackend - # once the registry is wired; kept for direct callers/tests. - # * LEGACY (no catalog): route only the placed deployments; churns the - # config (and recreates the gateway) on every model change. - if dynamic_routing: - entries: list[dict[str, Any]] = [] - litellm_routes = _litellm_routes(deployments, assignments) - litellm_depends: list[str] = [] - elif route_registry is not None: - entries = _litellm_model_list_from_registry(route_registry) - litellm_depends = [] # no per-model depends_on -> no churn - elif catalog is not None: - entries = _litellm_model_list_from_catalog(catalog) - litellm_depends = [] # no per-model depends_on -> no churn - else: - entries = _litellm_model_list(deployments, assignments) - litellm_depends = list(service_map) - litellm_config = yaml.safe_dump( - { - 'model_list': entries, - 'general_settings': { - 'master_key': f'os.environ/{API_KEY_ENV}' - }, - # An upstream vLLM/Ollama is unreachable only briefly, while it - # loads its model (LiteLLM does not wait for upstream health to - # start). Retry transient connection errors and don't park a - # model in a long cooldown, so the warmup window is self-healing - # instead of surfacing as client 500s ("Connection error. - # Received Model Deployment=…"). - 'router_settings': { - 'num_retries': 3, - 'timeout': 600, - 'cooldown_time': 5, - 'allowed_fails': 100, - }, - }, - sort_keys=False, - ) - config_hash = hashlib.sha256( - litellm_config.encode('utf-8') - ).hexdigest()[:12] - if dynamic_routing: - services[POSTGRES_SERVICE] = _postgres_service(images, state) - services[LITELLM_SERVICE] = _litellm_service( - litellm_depends, - litellm_port, - images, - str(aux_dir or '.'), - master_key=litellm_master_key, - config_hash=config_hash, - dynamic_routing=dynamic_routing, - salt_key=litellm_salt_key, - ) - - # Open WebUI is its own standing front door, rendered whenever ``ui`` is set - # — it does NOT require LiteLLM. Its OpenAI connection prefers the gateway - # (one URL covers every alias) and falls back to the rendered vLLM upstreams' - # own /v1 when there is no gateway. Its native Ollama connection always - # points straight at any Ollama daemon, so you can pull/run models from the - # UI and have the daemon load them on demand — a true drop-in for a - # hand-run ollama + Open WebUI stack. depends_on lists only LiteLLM (the one - # service guaranteed present alongside the UI); the per-model upstreams come - # and go, so the UI tolerates them being absent rather than hard-depending. - # With a gateway the UI is a standing front door (renders even at zero - # models). Without one it is only meaningful pointed at a live upstream, so - # render it only when there is something to connect to — otherwise an empty - # desired set has nothing to run and converge tears the project down. - if ui and (litellm or vllm_v1_urls or ollama_native_urls): - if litellm: - openai_urls = [f'http://{LITELLM_SERVICE}:{LITELLM_CONTAINER_PORT}/v1'] - ui_depends = [LITELLM_SERVICE] - else: - openai_urls = list(vllm_v1_urls) - ui_depends = [] - services[OPEN_WEBUI_SERVICE] = _open_webui_service( - ui_port, - images, - state, - litellm_master_key, - openai_urls=openai_urls, - ollama_urls=ollama_native_urls, - depends_on=ui_depends, - ) - - # Optional single-port HTTP reverse proxy fronting the gateway (+ UI). Needs - # the gateway, so it's only rendered alongside litellm. - nginx_config = None - if reverse_proxy and litellm: - depends = [LITELLM_SERVICE] + ([OPEN_WEBUI_SERVICE] if ui else []) - if reverse_proxy_config: - services[NGINX_SERVICE] = _nginx_service( - reverse_proxy_port, images, aux_dir=str(aux_dir or '.'), - depends_on=depends, config_path=reverse_proxy_config, - ) - else: - nginx_config = _nginx_conf(litellm=litellm, ui=ui) - services[NGINX_SERVICE] = _nginx_service( - reverse_proxy_port, images, aux_dir=str(aux_dir or '.'), - depends_on=depends, - config_hash=hashlib.sha256( - nginx_config.encode('utf-8') - ).hexdigest()[:12], - ) + front = render_front_door( + deployments, assignments, + engine_services=list(service_map), + vllm_v1_urls=vllm_v1_urls, ollama_native_urls=ollama_native_urls, + images=images, state=state, + litellm=litellm, litellm_port=litellm_port, + litellm_master_key=litellm_master_key, litellm_salt_key=litellm_salt_key, + ui=ui, ui_port=ui_port, + reverse_proxy=reverse_proxy, reverse_proxy_port=reverse_proxy_port, + reverse_proxy_config=reverse_proxy_config, aux_dir=aux_dir, + catalog=catalog, route_registry=route_registry, + dynamic_routing=dynamic_routing, + ) + services.update(front.services) + litellm_config = front.litellm_config + nginx_config = front.nginx_config + litellm_routes = front.litellm_routes for name, svc in services.items(): svc.setdefault('labels', {})[SERVICE_LABEL] = name diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 0da21a7f..cf549633 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -14,6 +14,7 @@ import hashlib import json import time +from dataclasses import dataclass from pathlib import Path from typing import Any, Callable @@ -733,6 +734,178 @@ def set_master_key(env_path: Path, key: str) -> None: write_env_file(env_path, values) +@dataclass +class FrontDoor: + """The rendered front door: its Compose services and config files.""" + + services: dict[str, Any] + litellm_config: str | None + nginx_config: str | None + litellm_routes: list[dict[str, Any]] | None + + +def render_front_door( + deployments: list[Deployment], + assignments: dict[str, list[int]], + *, + engine_services: list[str], + vllm_v1_urls: list[str], + ollama_native_urls: list[str], + images: dict[str, str], + state: dict[str, str], + litellm: bool, + litellm_port: int, + litellm_master_key: str | None, + litellm_salt_key: bool, + ui: bool, + ui_port: int, + reverse_proxy: bool, + reverse_proxy_port: int, + reverse_proxy_config: str | None, + aux_dir: str | Path | None, + catalog: Any, + route_registry: dict[str, Any] | None, + dynamic_routing: bool, +) -> FrontDoor: + """Render the gateway, its database, Open WebUI and the reverse proxy. + + The engines are the caller's: it passes the services it rendered + (``engine_services``, only for the legacy per-model ``depends_on``) and + the in-network URLs a UI with no gateway can talk to directly. + """ + services: dict[str, Any] = {} + litellm_config = None + litellm_routes = None + # The front door (gateway + UI) is rendered whenever it's enabled, even with + # zero models — it's a standing entry point, not a per-model service. So + # releasing/evicting every model leaves an empty gateway (and an empty Open + # WebUI picker) up instead of tearing the whole stack down; only an explicit + # `stack down` removes it. With no models the model_list is simply empty. + if litellm: + # Three route-table strategies, in order of preference: + # * DYNAMIC ROUTING: the rendered config is a STATIC base (empty + # model_list); the real routes live in Postgres and are applied to the + # running gateway via the admin API (see _reconcile_routes). The config + # hash never changes as models come/go, so the gateway is never + # recreated — no blip, and per-deployment routing works (so same-model + # --dedicated deployments each get their own upstream). + # * ROUTE REGISTRY (static-superset default from ComposeBackend): render + # from the whole accumulated registry (every catalog + live deployment + # ever merged, across all runbooks). Byte-stable once seeded, so the + # gateway is never recreated and a cross-catalog converge can no longer + # strip another runbook's routes. The backend loads/merges/writes the + # registry and passes the merged dict in; this function stays pure. + # * STATIC SUPERSET (catalog): one route per catalog endpoint to a + # deterministic host; config depends only on the catalog, so the + # gateway is not recreated as models come/go (no blip) but same-model + # dedicated collapses to one upstream. Unreachable from ComposeBackend + # once the registry is wired; kept for direct callers/tests. + # * LEGACY (no catalog): route only the placed deployments; churns the + # config (and recreates the gateway) on every model change. + if dynamic_routing: + entries: list[dict[str, Any]] = [] + litellm_routes = _litellm_routes(deployments, assignments) + litellm_depends: list[str] = [] + elif route_registry is not None: + entries = _litellm_model_list_from_registry(route_registry) + litellm_depends = [] # no per-model depends_on -> no churn + elif catalog is not None: + entries = _litellm_model_list_from_catalog(catalog) + litellm_depends = [] # no per-model depends_on -> no churn + else: + entries = _litellm_model_list(deployments, assignments) + litellm_depends = list(engine_services) + litellm_config = yaml.safe_dump( + { + 'model_list': entries, + 'general_settings': { + 'master_key': f'os.environ/{API_KEY_ENV}' + }, + # An upstream vLLM/Ollama is unreachable only briefly, while it + # loads its model (LiteLLM does not wait for upstream health to + # start). Retry transient connection errors and don't park a + # model in a long cooldown, so the warmup window is self-healing + # instead of surfacing as client 500s ("Connection error. + # Received Model Deployment=…"). + 'router_settings': { + 'num_retries': 3, + 'timeout': 600, + 'cooldown_time': 5, + 'allowed_fails': 100, + }, + }, + sort_keys=False, + ) + config_hash = hashlib.sha256( + litellm_config.encode('utf-8') + ).hexdigest()[:12] + if dynamic_routing: + services[POSTGRES_SERVICE] = _postgres_service(images, state) + services[LITELLM_SERVICE] = _litellm_service( + litellm_depends, + litellm_port, + images, + str(aux_dir or '.'), + master_key=litellm_master_key, + config_hash=config_hash, + dynamic_routing=dynamic_routing, + salt_key=litellm_salt_key, + ) + + # Open WebUI is its own standing front door, rendered whenever ``ui`` is set + # — it does NOT require LiteLLM. Its OpenAI connection prefers the gateway + # (one URL covers every alias) and falls back to the rendered vLLM upstreams' + # own /v1 when there is no gateway. Its native Ollama connection always + # points straight at any Ollama daemon, so you can pull/run models from the + # UI and have the daemon load them on demand — a true drop-in for a + # hand-run ollama + Open WebUI stack. depends_on lists only LiteLLM (the one + # service guaranteed present alongside the UI); the per-model upstreams come + # and go, so the UI tolerates them being absent rather than hard-depending. + # With a gateway the UI is a standing front door (renders even at zero + # models). Without one it is only meaningful pointed at a live upstream, so + # render it only when there is something to connect to — otherwise an empty + # desired set has nothing to run and converge tears the project down. + if ui and (litellm or vllm_v1_urls or ollama_native_urls): + if litellm: + openai_urls = [f'http://{LITELLM_SERVICE}:{LITELLM_CONTAINER_PORT}/v1'] + ui_depends = [LITELLM_SERVICE] + else: + openai_urls = list(vllm_v1_urls) + ui_depends = [] + services[OPEN_WEBUI_SERVICE] = _open_webui_service( + ui_port, + images, + state, + litellm_master_key, + openai_urls=openai_urls, + ollama_urls=ollama_native_urls, + depends_on=ui_depends, + ) + + # Optional single-port HTTP reverse proxy fronting the gateway (+ UI). Needs + # the gateway, so it's only rendered alongside litellm. + nginx_config = None + if reverse_proxy and litellm: + depends = [LITELLM_SERVICE] + ([OPEN_WEBUI_SERVICE] if ui else []) + if reverse_proxy_config: + services[NGINX_SERVICE] = _nginx_service( + reverse_proxy_port, images, aux_dir=str(aux_dir or '.'), + depends_on=depends, config_path=reverse_proxy_config, + ) + else: + nginx_config = _nginx_conf(litellm=litellm, ui=ui) + services[NGINX_SERVICE] = _nginx_service( + reverse_proxy_port, images, aux_dir=str(aux_dir or '.'), + depends_on=depends, + config_hash=hashlib.sha256( + nginx_config.encode('utf-8') + ).hexdigest()[:12], + ) + + return FrontDoor(services=services, litellm_config=litellm_config, + nginx_config=nginx_config, litellm_routes=litellm_routes) + + class Gateway(ConvergeScaffold): """The front door's state: settings, the managed keys, the route registry. From f010ccf728bfdaab724d8f423b20f5c94108611a Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 14:39:35 -0400 Subject: [PATCH 010/141] Record the gateway extraction; drop two dead names in the controller Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 13 ++++++++ dev/journals/claude.md | 32 +++++++++++++++++++ .../plan-backend-unification-2026-09-24.md | 12 +++++++ infer_stack/leasing/controller.py | 4 +-- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 218fa5f7..800fc834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,19 @@ scheduler with its own GPU accounting: a lease held outside it, manual or keep-warm, occupies a GPU the scheduler believes is free, and the job it places there cannot start. +### The gateway is its own module + +The front door (LiteLLM config and service, route registry, dynamic-route +reconciliation, the managed keys, Open WebUI, the reverse proxy) moved out of +`leasing/compose.py` into `leasing/gateway.py`, with a `Gateway` object that +`ComposeBackend` owns (`backend.gateway`). The service naming rules moved to +`leasing/naming.py`. Backends now hand the gateway their route rows rather +than the gateway reading backend state. Rendered output is unchanged +(verified byte for byte), and `compose.py` is about a third smaller. Code +that imported gateway names from `infer_stack.leasing.compose` should import +them from `infer_stack.leasing.gateway`; the public naming helpers are still +importable from `compose`. + ### The kubeai backend fails fast on an engine that cannot start The crash diagnosis (restart count, exit code, and the engine log classified diff --git a/dev/journals/claude.md b/dev/journals/claude.md index 9bb84781..aeb38b4a 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3217,3 +3217,35 @@ evicts idle keep-warm Models when a new one cannot be scheduled. fail on purpose first; GATEWAY=0 is that switch here. (2) "Has method X" is a poor stand-in for "is backend Y": when a second backend gains X, every such check silently changes meaning. + +## 2026-09-24 14:39:35 -0400 + +**Intent.** Make infer-stack more elegant, architecture first and the TUI +after. The user agreed that the Compose backend's real seam is gateway vs. +engines. Model: Claude Opus 5.5 (1M context). + +**What I did.** Four steps on `dev/backend-unification`, each green before +the next: naming rules into `naming.py`, the gateway's module code into +`gateway.py`, a `Gateway` class for its state, and `render_front_door`. +The one real design change: the registry merge no longer reads backend +state. Backends supply rows and the gateway only merges them, which removed +the `upstream_routes` side-channel I had added for KubeAI in K1. Settings the +gateway reads became backend properties onto the single `Gateway`, because +`use_profile` and tests set them on the backend after construction. Two +copies would drift. + +**How I checked a pure refactor.** Unit tests alone are weak evidence for +moved code, so I also rendered six stack configurations with the old and new +code and compared bytes (identical), then ran the real Compose gateway, +`secrets rotate` and the KubeAI e2e. + +**What went wrong.** My first byte comparison "passed" on two empty files, +because the script crashed in both runs. I caught it only because the byte +count was 0. More serious: the weight-floor doctest I wrote earlier wrote +4 GiB of real zeros into /tmp on every run and never cleaned up. A day of +test runs filled 49 GB, k3s went into disk pressure and evicted KubeAI. +Fixed with sparse files. + +**Takeaways.** (1) A comparison of outputs must check the outputs exist: +equality of two failures is not evidence. (2) Test fixtures that need large +files should be sparse. Size was the property under test, not the bytes. diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index ac31001b..ab31e838 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -230,3 +230,15 @@ new modules. Each step leaves the full suite, `ty` and flake8 green. The gateway still runs as a Compose project (it is a container), so KubeAI keeps a `ComposeBackend` with no engines: that is now simply "a Compose project with only the front door", not a special mode. + +**G done 2026-09-24** (on `dev/backend-unification`). A: `naming.py` +(99 lines). B: `gateway.py` module code. C: the `Gateway` class; backends +supply route rows, the gateway merges, persists and renders them (the +`upstream_routes` side-channel is gone), and the backend's gateway settings +are properties onto the one `Gateway`. D: `render_front_door`, verified +byte-identical across six configurations. `compose.py` went from 3651 +lines (on `main`) to 2360; `gateway.py` is 1377. Verified live after C: a +Compose gateway apply and `secrets rotate`, and the KubeAI e2e. + +Found on the way: the weight-floor doctest wrote 4 GiB to `/tmp` per run +(fixed on `main`, 21f5e09). diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 6636c9b6..35c74d55 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1086,7 +1086,7 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: candidate['catalogs'] = merge_catalog_sources( stored.get('catalogs') or [], invocation.get('catalogs') or [] ) - except CatalogConflict as ex: + except CatalogConflict: # Only definitions a resident workload is actually running have to # stay frozen. Redefining anything else -- the normal case while # iterating with `catalog endpoint add --force` -- drops the stale @@ -1224,7 +1224,7 @@ def _render_in_scope(self, context: dict) -> None: scope = getattr(self.backend, 'placement_scope', None) if scope is not None: with scope(context): - rec = self._render() + self._render() self.ledger.clear_placement_context() def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: From 0101a2d8edac9f6920ae30dfaaba5c4f5dfd523b Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 14:39:48 -0400 Subject: [PATCH 011/141] Lesson: size-only test files should be sparse and cleaned up Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/lessons/lessons.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index 9f616c7c..ecdeb16c 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -95,3 +95,15 @@ evidence; prefer append-only; supersede incorrect entries with a new one. absent. - **Applies when:** writing or reviewing any multi-threaded test that uses a barrier, latch, or queue rendezvous. + +- **Lesson:** A test that needs a large file for its *size* should create it + sparse (`file.truncate(n)`), and remove its temp dir. `os.path.getsize` + reports the full size, but no disk is used. Writing real bytes leaks gigabytes + per run unless the test cleans up. +- **Evidence / MWE:** 21f5e09. The `weight_floor_gib` doctest wrote two 2 GiB + files into `mkdtemp()` and never removed them. A day of suite runs left + 49 GB in `/tmp` on the guest, k3s tainted its node for disk pressure, and + KubeAI's pods were evicted. With sparse files the doctest takes 0.15 s and + leaves nothing behind. +- **Applies when:** a test exercises size-dependent logic (VRAM floors, disk + checks, download sizes). From af2e8fcccebefe5e8049f7fbf18a417f1a6d1862 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 15:12:35 -0400 Subject: [PATCH 012/141] Make room for leased demand: idle keep-warm models give way on KubeAI Rule (user decision): a keep-warm model without a lease is always a candidate for eviction when a model with a lease needs a resource. Compose admission already enforces it when placing. Where the runtime schedules (KubeAI), a probe now reports Readiness.needs_room for an Unschedulable pod, and Controller.wait_ready evicts the longest-idle deployment through the normal locked evict, one per ROOM_COOLDOWN_S (30 s), never a leased one. The step runs after the deadline check: a test caught it evicting a second model after the wait had already given up. Verified on k3s: with a profile only one Model fits (cpu-half, added to the test values), acquiring a leased Model evicted the idle keep-warm one within 10 s and was ready in 107 s; `E2E_MAKE_ROOM=1 dev/kubeai_e2e.sh` passes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 11 ++ dev/e2e_tests/kubeai-cpu-values.yaml | 7 ++ dev/kubeai_e2e.sh | 28 +++++ .../plan-backend-unification-2026-09-24.md | 6 + docs/kubeai-backend.md | 5 + infer_stack/backends/kubeai.py | 3 +- infer_stack/leasing/backend.py | 4 + infer_stack/leasing/controller.py | 34 ++++++ tests/test_leasing_make_room.py | 112 ++++++++++++++++++ 9 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 tests/test_leasing_make_room.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 800fc834..2a1cdba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,17 @@ scheduler with its own GPU accounting: a lease held outside it, manual or keep-warm, occupies a GPU the scheduler believes is free, and the job it places there cannot start. +### An idle keep-warm model gives way to a leased one on KubeAI too + +The rule: a keep-warm model that no lease holds is always a candidate for +eviction when a model with a lease needs a resource. Compose admission already +applied it when placing. On KubeAI the cluster schedules and never evicts, so +a leased Model could sit `Unschedulable` behind idle warm ones until its +timeout. A readiness probe can now report `needs_room`; the wait then evicts +the longest-idle deployment, one per 30 s, never a leased one, and never +after the wait's deadline. Verified on k3s: a leased Model that did not fit +beside an idle one was ready 107 s later, with the idle one evicted. + ### The gateway is its own module The front door (LiteLLM config and service, route registry, dynamic-route diff --git a/dev/e2e_tests/kubeai-cpu-values.yaml b/dev/e2e_tests/kubeai-cpu-values.yaml index 20300543..5d308441 100644 --- a/dev/e2e_tests/kubeai-cpu-values.yaml +++ b/dev/e2e_tests/kubeai-cpu-values.yaml @@ -7,3 +7,10 @@ resourceProfiles: requests: cpu: 8 memory: "16Gi" + # Half the guest's 120 cores: only one such Model fits at a time, which is + # how dev/kubeai_e2e.sh's make-room check forces a scheduling conflict. + cpu-half: + imageName: "cpu" + requests: + cpu: 64 + memory: "16Gi" diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index cec829dd..aba5df1b 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -109,4 +109,32 @@ remaining=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ -l infer-stack/managed=true -o name | wc -l) [ "$remaining" = 0 ] || { echo "!! model not pruned"; exit 1; } +if [ "${E2E_MAKE_ROOM:-0}" = 1 ]; then + # Needs a profile only one Model fits at a time (`cpu-half` in + # dev/e2e_tests/kubeai-cpu-values.yaml): E2E_ROOM_PROFILE=cpu-half. + echo '== an idle keep-warm model gives way to a leased one' + cat >> "$WORK/config/catalog.yaml" <&1 | tee "$WORK/big.log" | grep -q 'ready: True'; then + echo '!! the leased model never became ready' >&2; exit 1 + fi + grep -q 'making room for leased demand' "$WORK/big.log" \ + || { echo '!! no idle model was evicted to make room' >&2; exit 1; } + echo ' the idle keep-warm model was evicted; the leased one is ready' + run_is release --env-file "$WORK/big.env" --yes +fi + echo 'PASS: kubeai backend end-to-end lifecycle' diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index ab31e838..130bf9dc 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -160,6 +160,12 @@ backends, `_admission_mode()` is always true for real backends. The non-admission branches are then deleted. Memory and Null backends get a trivial residency so tests use the same path. +**Decision (user, 2026-09-24):** "Keep warm models without a lease should +always be candidates for eviction if a model with a lease needs a resource." +Implemented backend-neutrally ahead of K3: `Readiness.needs_room`, and +`Controller._make_room` in the wait. The one acquire path itself (K3) is +still open. + ### K4. One failure diagnosis Move `startup_failure` and the log classifier behind the K2 snapshot. Compose diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index df1f4e8f..59adb851 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -108,6 +108,11 @@ infer-stack release --env-file lease.env capacity (the cluster schedules); a Model the cluster cannot place sits not-ready until the acquire's `--timeout`, which then rolls the lease back. The wait says why (`pod: Unschedulable`, `pod: ImagePullBackOff`). +- **A keep-warm model without a lease gives way to one with a lease.** When + a leased Model's pod is `Unschedulable`, the wait evicts the longest-idle + keep-warm deployment, one per 30 s, until it fits. Compose applies the same + rule when placing. A leased model is never evicted this way. Verified on + k3s: `E2E_MAKE_ROOM=1 dev/kubeai_e2e.sh` with the `cpu-half` profile. - **An engine that cannot start fails the acquire at once**, as on compose: the crash diagnosis reads the pods (`kubectl get pods`, restart count, last exit) and the previous run's log, and quotes the engine's error with a diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 2385bd10..a65bdf0c 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -638,7 +638,8 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: if failure is not None: return Readiness(False, failure, fatal=True) waiting = self._waiting_reason(deployment) - return Readiness(False, f'{reason} (pod: {waiting})' if waiting else reason) + return Readiness(False, f'{reason} (pod: {waiting})' if waiting else reason, + needs_room='Unschedulable' in waiting) def access(self, endpoints: list[str]) -> dict[str, Any] | None: """Where a client reaches these endpoints (env-file descriptor). diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 59f7736b..0c1db433 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -33,6 +33,10 @@ class Readiness: ready: bool detail: str = '' fatal: bool = False + #: The endpoint is waiting for resources held by others (a Kubernetes pod + #: the scheduler cannot place). The controller then frees room by + #: evicting idle keep-warm deployments, which no lease holds. + needs_room: bool = False class BackendTimeout(RuntimeError): diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 35c74d55..de969761 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -37,6 +37,9 @@ _T = TypeVar('_T') KEEP_WARM = 'keep-warm' +#: Between two evictions made to fit leased demand: long enough for the +#: freed instance to terminate and the scheduler to retry. +ROOM_COOLDOWN_S = 30.0 #: After an interrupted apply, how long to wait for the runtime to stop changing #: before applying again, and how often to sample it. Held under the lock. @@ -1393,14 +1396,17 @@ def wait_ready( if endpoints is None or ep in endpoints ] deadline = self.clock() + timeout + last_room = float('-inf') while True: pending = [] failures = [] + needs_room = False for (g, ep) in pairs: probe = self.backend.probe_ready(g, ep) if probe.ready: continue pending.append((g, ep)) + needs_room = needs_room or probe.needs_room if probe.fatal: failures.append((g.id, ep, probe.detail)) if not pending: @@ -1416,9 +1422,37 @@ def wait_ready( ready=False, pending=[(g.id, ep) for g, ep in pending], ) + # After the deadline check: a wait that has given up evicts nothing. + if needs_room and self.clock() - last_room >= ROOM_COOLDOWN_S: + # A leased model is waiting on resources: a model without a + # lease is always a candidate to give them up. + if self._make_room(): + last_room = self.clock() self.sleep(interval) pairs = pending + def _make_room(self) -> str | None: + """Evict the longest-idle deployment for leased demand; its id, or ``None``. + + Idle means no lease holds it (a keep-warm model left resident). One at + a time: the runtime cannot say how much room is needed, and every + warm model kept is a load avoided. Compose backends never ask, since + their admission already moves idle models aside; this serves backends + where the runtime schedules (KubeAI). + """ + from .._log import logger + + _, deployments = self.ledger.status(virtual_expiry=True) + idle = sorted((g for g in deployments if g.state == DeploymentState.IDLE), + key=lambda g: (g.updated_at, g.id)) + if not idle: + return None + victim = idle[0].id + logger.info('making room for leased demand: evicting idle keep-warm {} ({})', + victim, ', '.join(sorted(idle[0].served))) + self.evict([victim]) + return victim + def _never_ran(self, deployment_ids: list[str]) -> list[str]: """Which of these deployments definitely have no container at all. diff --git a/tests/test_leasing_make_room.py b/tests/test_leasing_make_room.py new file mode 100644 index 00000000..ec6aeb05 --- /dev/null +++ b/tests/test_leasing_make_room.py @@ -0,0 +1,112 @@ +"""A keep-warm model without a lease gives way to a model with one. + +The rule (decided 2026-09-24): an idle keep-warm deployment -- resident, but +held by no lease -- is always a candidate for eviction when a leased model +needs a resource. Compose admission enforces it when placing. Where the +runtime schedules instead (KubeAI), the probe says `needs_room` and the wait +evicts idle models, the longest idle first, one per cooldown. +""" + +from __future__ import annotations + +from infer_stack.leasing import ( + Controller, + DeploymentState, + Ledger, + MemoryBackend, + SqliteStore, +) +from infer_stack.leasing.backend import Readiness +from infer_stack.leasing.controller import ROOM_COOLDOWN_S +from test_leasing_controller import FakeClock, _id_factory, vreq + + +class OneSlotBackend(MemoryBackend): + """Room for `slots` models; a model that does not fit waits for room.""" + + def __init__(self, slots=1): + super().__init__(ready=True) + self.slots = slots + + def probe_ready(self, deployment, endpoint): + others = [gid for gid in self.realized if gid != deployment.id] + if deployment.id in self.realized and len(others) >= self.slots: + return Readiness(False, 'pod: Unschedulable', needs_room=True) + return super().probe_ready(deployment, endpoint) + + +def make(slots=1): + clock = FakeClock() + ledger = Ledger(SqliteStore(':memory:'), clock=clock, id_factory=_id_factory()) + backend = OneSlotBackend(slots) + ctl = Controller(ledger, backend, clock=clock, sleep=clock.advance) + return ctl, ledger, backend, clock + + +def state(ledger, gid): + return ledger.get_deployment(gid).state + + +def test_an_idle_keep_warm_model_gives_way_to_a_leased_one(): + ctl, ledger, backend, _ = make() + warm = ctl.acquire('a', [vreq('warm')], wait=False) + ctl.release(warm.lease.id) # keep-warm: still resident + warm_id = warm.deployments[0].id + assert state(ledger, warm_id) == DeploymentState.IDLE + + out = ctl.acquire('b', [vreq('big')], wait=True, timeout=600, interval=5) + + assert out.wait.ready + assert state(ledger, warm_id) == DeploymentState.STOPPED + assert warm_id not in backend.realized + + +def test_the_longest_idle_goes_first_and_one_at_a_time(): + ctl, ledger, backend, clock = make(slots=2) + older = ctl.acquire('a', [vreq('older')], wait=False) + ctl.release(older.lease.id) + clock.advance(60) + newer = ctl.acquire('a', [vreq('newer')], wait=False) + ctl.release(newer.lease.id) + + out = ctl.acquire('b', [vreq('big')], wait=True, timeout=600, interval=5) + + assert out.wait.ready + assert state(ledger, older.deployments[0].id) == DeploymentState.STOPPED + # One eviction freed the slot, so the more recently used model stays warm. + assert state(ledger, newer.deployments[0].id) == DeploymentState.IDLE + + +def test_evictions_wait_for_the_cooldown(): + ctl, ledger, backend, clock = make(slots=0) # nothing ever fits + for name in ('w1', 'w2'): + out = ctl.acquire('a', [vreq(name)], wait=False) + ctl.release(out.lease.id) + big = ctl.acquire('b', [vreq('big')], wait=False) + + ctl.wait_ready(big.deployments, timeout=ROOM_COOLDOWN_S - 1, interval=5) + stopped = [g for g in ledger.status()[1] if g.state == DeploymentState.STOPPED] + assert len(stopped) == 1 # not both within one cooldown + + +def test_a_leased_model_is_never_evicted_to_make_room(): + ctl, ledger, backend, _ = make() + held = ctl.acquire('a', [vreq('held')], wait=False) # still leased + out = ctl.acquire('b', [vreq('big')], wait=True, timeout=120, interval=5) + + assert not out.wait.ready # waits, then times out + assert state(ledger, held.deployments[0].id) == DeploymentState.LIVE + + +def test_an_unschedulable_kubeai_pod_asks_for_room(tmp_path): + from test_leasing_kubeai import FakeHttp, _pod, make_pod_backend, vllm + + be, kubectl = make_pod_backend(tmp_path) + dep = vllm('grp-big', served='big') + be.converge([dep]) + kubectl.pods = [_pod('model-big-1', 'grp-big', statuses=False, conditions=[ + {'type': 'PodScheduled', 'status': 'False', 'reason': 'Unschedulable'}])] + be.http.post = lambda url, **kw: FakeHttp._Resp(503, {'detail': 'not ready'}) + + probe = be.probe_ready(dep, 'grp-big') + assert probe.needs_room and not probe.fatal and not probe.ready From 08e61d52165f3eb502907c089a9d19b156c94934 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 17:25:22 -0400 Subject: [PATCH 013/141] Serialize every use of the shared ledger connection SqliteStore shares one sqlite3 connection across threads (check_same_thread=False) but only write transactions took its lock, on the belief that sqlite's serialized mode makes concurrent reads safe. It does not for a shared Python connection: two threads stepping the same cached statement interleave. The TUI reads the ledger from its refresh worker and the UI thread at once, and intermittently got a lease whose deployment ids included None (TypeError drawing the leases table; seen as a flaky TUI test). Every statement now goes through _SerializedConnection, which holds the store's re-entrant lock until the statement has run and its rows are fetched, so no cursor outlives the lock and transaction() nests. New test: eight threads reading one ledger 150 times each; without the fix it fails with InterfaceError('bad parameter or other API misuse') and more. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/leasing/store.py | 64 +++++++++++++++++++++++++---- tests/test_leasing_store_threads.py | 45 ++++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 tests/test_leasing_store_threads.py diff --git a/infer_stack/leasing/store.py b/infer_stack/leasing/store.py index 052d352f..a0c393a1 100644 --- a/infer_stack/leasing/store.py +++ b/infer_stack/leasing/store.py @@ -93,6 +93,54 @@ def _dumps(value: Any) -> str: return json.dumps(value, separators=(',', ':'), sort_keys=True) +class _Result: + """A statement's rows, fetched while the connection lock was held.""" + + def __init__(self, rows: list, rowcount: int, lastrowid: int | None): + self._rows = rows + self.rowcount = rowcount + self.lastrowid = lastrowid + + def fetchone(self): + return self._rows[0] if self._rows else None + + def fetchall(self) -> list: + return list(self._rows) + + def __iter__(self): + return iter(self._rows) + + +class _SerializedConnection: + """One sqlite connection, safe to share between threads. + + Python's sqlite3 connection is not: two threads stepping the same cached + statement interleave and return garbage rows. The TUI hit this reading the + ledger from its refresh worker and the UI thread at once (a lease whose + deployment ids included ``None``). Each call here holds the store lock + until the statement has run and every row is fetched, so no cursor ever + outlives it. The lock is re-entrant, so :meth:`SqliteStore.transaction` + (which holds it for the whole transaction) nests. + """ + + def __init__(self, conn: sqlite3.Connection, lock): + self._conn = conn + self._lock = lock + + def execute(self, sql: str, params=()) -> _Result: + with self._lock: + cur = self._conn.execute(sql, params) + return _Result(cur.fetchall(), cur.rowcount, cur.lastrowid) + + def executescript(self, script: str) -> None: + with self._lock: + self._conn.executescript(script) + + def close(self) -> None: + with self._lock: + self._conn.close() + + class SqliteStore: """Thin sqlite wrapper exposing ledger row operations.""" @@ -101,17 +149,19 @@ def __init__(self, path: str | Path = ':memory:', *, busy_timeout_ms: int = 5000 if self.path != ':memory:': Path(self.path).expanduser().parent.mkdir(parents=True, exist_ok=True) # check_same_thread=False lets a long-running process (e.g. the TUI) use - # this connection from a worker thread for converge-while-monitoring; - # ``_lock`` serializes write transactions so two threads can't both - # ``BEGIN IMMEDIATE`` on the one connection. sqlite itself is built - # serialized, so individual reads across threads are safe. - self._conn = sqlite3.connect( + # this connection from worker threads. That is only safe because every + # statement runs under ``_lock`` (see _SerializedConnection): sqlite's + # own serialized mode does NOT make a shared Python connection safe for + # concurrent reads. + raw = sqlite3.connect( self.path, isolation_level=None, timeout=busy_timeout_ms / 1000, check_same_thread=False, ) + raw.row_factory = sqlite3.Row self._lock = threading.RLock() + # Every use of the connection holds the lock: see _SerializedConnection. + self._conn = _SerializedConnection(raw, self._lock) self._busy_timeout_ms = busy_timeout_ms - self._conn.row_factory = sqlite3.Row self._conn.execute('PRAGMA foreign_keys = ON') self._conn.execute(f'PRAGMA busy_timeout = {busy_timeout_ms}') if self.path != ':memory:': @@ -212,7 +262,7 @@ def __del__(self) -> None: # -- transactions ------------------------------------------------------ @contextlib.contextmanager - def transaction(self) -> Iterator[sqlite3.Connection]: + def transaction(self) -> Iterator[_SerializedConnection]: """Take the write lock up front and commit/rollback atomically. ``BEGIN IMMEDIATE`` is what makes the ledger's find-or-create-deployment diff --git a/tests/test_leasing_store_threads.py b/tests/test_leasing_store_threads.py new file mode 100644 index 00000000..3e926ce1 --- /dev/null +++ b/tests/test_leasing_store_threads.py @@ -0,0 +1,45 @@ +"""One ledger connection shared by threads must never return garbage rows. + +The TUI reads the ledger from its refresh worker and from the UI thread at +the same moment. Python's sqlite3 connection is not safe for that on its own: +two threads stepping the same cached statement interleave, and a lease came +back with ``None`` among its deployment ids (TypeError in the leases table). +""" + +from __future__ import annotations + +import threading + +from infer_stack.leasing import Catalog, Controller, Ledger, NullBackend, SqliteStore + + +def test_concurrent_readers_see_consistent_rows(tmp_path): + catalog = Catalog.from_dict({ + 'models': {'m': {'source': 'hf://org/m'}}, + 'endpoints': {f'ep-{i:02d}': {'engine': 'vllm', 'model': 'm'} for i in range(30)}, + }) + ledger = Ledger(SqliteStore(str(tmp_path / 'ledger.db'))) + controller = Controller(ledger, NullBackend()) + for i in range(30): + controller.acquire(f'u{i:02d}', catalog.resolve_names([f'ep-{i:02d}'])) + + problems: list[str] = [] + + def read(): + try: + for _ in range(150): + leases, deployments = ledger.status(virtual_expiry=True) + if len(leases) != 30 or len(deployments) != 30: + problems.append(f'{len(leases)} leases, {len(deployments)} deployments') + for le in leases: + if not le.deployment_ids or None in le.deployment_ids: + problems.append(f'{le.id}: deployment ids {le.deployment_ids}') + except Exception as ex: # noqa: BLE001 - any error is the failure + problems.append(repr(ex)) + + threads = [threading.Thread(target=read) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert problems == [] From f03f6e981b5ec4c2c22ec73fc2dbf6cccda0b416 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 17:25:22 -0400 Subject: [PATCH 014/141] TUI: stop repainting unchanged panes; batch log drawing; report stalls Measured with dev/profile_tui.py (headless and on a real terminal) and a real `infer-stack tui` session in tmux: - The lease and deployment panes were repainted every refresh tick: Textual repaints a widget whenever its border title is set, even to the same text. Titles are now set only on change. Idle terminal output fell from 6922 to 75 bytes/s (the header clock), and from 5223 to 442 with the docker pane open, which is what an SSH session feels. - Streamed log lines went to the UI one call_from_thread per line, capping a loading engine at ~1800 lines/s. Workers now append to a deque the UI drains every 100 ms, at most 200 lines per tick (2000 in one tick stalled the loop 218 ms), keeping the newest lines when a flood outruns the pane. The line mirror is trimmed instead of growing without bound. - The compose file's service list is parsed only when the file changes, not every tick. - A watchdog reports any UI-thread stall over 0.5 s in the TUI log, with the stack sampled while it was stuck written to the error log file, so a freeze on the host comes with what caused it. The scroll-offset test waits for the mount-time refresh; its intermittent failure was the ledger race fixed in the previous commit. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/profile_tui.py | 151 ++++++++++++++++++++++++++++++++++++++++++ infer_stack/tui.py | 162 ++++++++++++++++++++++++++++++++++++++++----- tests/test_tui.py | 65 +++++++++++++++++- 3 files changed, 361 insertions(+), 17 deletions(-) create mode 100644 dev/profile_tui.py diff --git a/dev/profile_tui.py b/dev/profile_tui.py new file mode 100644 index 00000000..88c252f8 --- /dev/null +++ b/dev/profile_tui.py @@ -0,0 +1,151 @@ +"""Measure how responsive the TUI's event loop is, and what blocks it. + +Runs the real app headless against a real ComposeBackend (over the tests' +fake Docker, so the numbers are the TUI's own cost, not Docker's), with a +ledger holding a few leases and deployments. A 50 ms timer on the app's own +loop measures lag: how late it fires is how late a keypress would be +handled. Scenarios: + + idle the dashboard refreshing on its normal timer + docker the docker pane open on the Logs tab (service list, log stream) + flood a log stream emitting lines as fast as it can (an engine loading) + + python dev/profile_tui.py [--profile] # --profile: cProfile the UI thread +""" + +from __future__ import annotations + +import asyncio +import cProfile +import io +import pstats +import statistics +import sys +import tempfile +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(HERE / 'tests')) + +from test_leasing_admission import CAT, acquire, make # noqa: E402 + +from infer_stack.tui import InferStackTUI # noqa: E402 + +FLOOD_LINES = 20000 + + +class FloodProc: + """A log process that writes FLOOD_LINES lines as fast as it can.""" + + def __init__(self, n): + self.n = n + + @property + def stdout(self): + for i in range(self.n): + yield f'vllm-one | INFO 09-24 12:00:00 loader.py:1 loading shard {i}\n' + + def terminate(self): + pass + + +async def lag_probe(seconds: float, period: float = 0.05) -> list[float]: + lags = [] + end = time.perf_counter() + seconds + while time.perf_counter() < end: + t0 = time.perf_counter() + await asyncio.sleep(period) + lags.append(time.perf_counter() - t0 - period) + return lags + + +def summary(name: str, lags: list[float]) -> str: + ms = sorted(x * 1000 for x in lags) + p95 = ms[int(len(ms) * 0.95) - 1] if ms else 0 + return (f'{name:8s} samples={len(ms):4d} median={statistics.median(ms):7.1f} ms ' + f'p95={p95:7.1f} ms max={max(ms):7.1f} ms') + + +def main() -> None: + do_profile = '--profile' in sys.argv + # --docker-latency=S: each docker call takes S seconds, like a loaded host. + latency = next((float(a.split('=', 1)[1]) for a in sys.argv + if a.startswith('--docker-latency=')), 0.0) + # --tty: run on a real terminal (not headless), so drawing is measured too. + tty = '--tty' in sys.argv + import threading + tmp = Path(tempfile.mkdtemp()) + ledger, ctl, docker = make(tmp) + for name in ('one', 'two'): + out = acquire(ctl, name) + if name == 'two': + ctl.release(out.lease.id) # an idle keep-warm resident + flood = {'on': False} + if latency: + real_run = ctl.backend.run + + def slow_run(args, **kw): + time.sleep(latency) + return real_run(args, **kw) + ctl.backend.run = slow_run + + def proc_factory(service): + return FloodProc(FLOOD_LINES) if flood['on'] else FloodProc(0) + + results = [] + profiler = cProfile.Profile() + + async def scenario(): + app = InferStackTUI(ctl, CAT, interval=1.0, proc_factory=proc_factory) + async with app.run_test(size=(200, 60)) as pilot: + await pilot.pause(1.0) + if do_profile: + profiler.enable() + results.append(summary('idle', await lag_probe(8))) + app.query_one('#docker').collapsed = False + await pilot.pause(0.5) + results.append(summary('docker', await lag_probe(8))) + flood['on'] = True + app._restart_logs(app._log_service) + t0 = time.perf_counter() + results.append(summary('flood', await lag_probe(8))) + results.append(f'flood: {len(app._log_lines)} of {FLOOD_LINES} lines shown ' + f'after {time.perf_counter() - t0:.1f}s') + results.append(f'threads alive at end: {threading.active_count()}') + if do_profile: + profiler.disable() + + if tty: + # The app on the real terminal, driven by its own timers. + app = InferStackTUI(ctl, CAT, interval=1.0, proc_factory=proc_factory) + + async def drive(): + await asyncio.sleep(1.5) + results.append(summary('idle', await lag_probe(8))) + app.query_one('#docker').collapsed = False + await asyncio.sleep(0.5) + results.append(summary('docker', await lag_probe(8))) + flood['on'] = True + app._restart_logs(app._log_service) + t0 = time.perf_counter() + results.append(summary('flood', await lag_probe(8))) + results.append(f'flood: {len(app._log_lines)} of {FLOOD_LINES} lines shown ' + f'after {time.perf_counter() - t0:.1f}s') + results.append(f'threads alive at end: {threading.active_count()}') + app.exit() + + app.call_later(lambda: asyncio.ensure_future(drive())) + app.run() + else: + asyncio.run(scenario()) + for line in results: + print(line) + if do_profile: + out = io.StringIO() + pstats.Stats(profiler, stream=out).sort_stats('cumulative').print_stats(35) + print(out.getvalue()) + + +if __name__ == '__main__': + main() diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 51f45f92..3fc5b0de 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -96,6 +96,17 @@ #: Title of the tab that shows what the TUI itself did, and its errors. APP_LOG_TAB_TITLE = 'TUI log' +#: The docker log pane shows at most this many lines (RichLog ``max_lines``). +LOG_PANE_LINES = 2000 +#: How often streamed log lines are drawn: in one batch, not one UI message per +#: line (which capped a loading engine's log at ~1800 lines/s). +LOG_DRAIN_S = 0.1 +#: At most this many lines per drain: RichLog costs ~0.1 ms a line, and 2000 in +#: one tick stalled the UI for 218 ms. The rest waits for the next tick. +LOG_DRAIN_LINES = 200 +#: A UI-thread stall at least this long is reported, with what was running. +STALL_REPORT_S = 0.5 + SELECT_BLANK = next( ( candidate @@ -982,6 +993,19 @@ def __init__( # visible; only the current generation is allowed to append. self._log_generation = 0 self._log_lines: list[str] = [] # mirror of the docker log pane, for tests + # Lines streamed by the log worker, drawn by the UI every LOG_DRAIN_S. + # (generation, line): a line from a replaced stream is dropped. + import collections + self._log_pending: collections.deque = collections.deque() + # Lines taken from _log_pending, not yet drawn; only the newest + # LOG_PANE_LINES are kept, since the pane shows no more. + self._log_backlog: collections.deque = collections.deque(maxlen=LOG_PANE_LINES) + self._log_dropped = 0 + # Parsed compose service names, keyed by the file's (mtime, size). + self._service_names_cache: tuple[tuple[float, int], list[str]] | None = None + # The UI loop's heartbeat, read by the stall watchdog thread. + self._beat = time.monotonic() + self._watchdog_stop = None self._app_log_lines: list[str] = [] # mirror of the TUI log pane, for tests self._app_log_errors = 0 self._api_lines: list[str] = [] # mirror of the API output, for tests @@ -1110,7 +1134,7 @@ def _compose_dashboard(self) -> ComposeResult: id='logsvc', ) yield RichLog(id='logs', highlight=False, - markup=False, max_lines=2000, + markup=False, max_lines=LOG_PANE_LINES, wrap=False) with TabPane('Containers', id='tab-containers'): yield DataTable(id='ps', cursor_type='row', @@ -1289,10 +1313,76 @@ def on_mount(self) -> None: self._refresh_timer = self.set_interval( self.ledger_interval, self.action_refresh ) + self.set_interval(LOG_DRAIN_S, self._drain_logs) + self._start_stall_watchdog() self.query_one('#endpoints', DataTable).focus() def on_unmount(self) -> None: self._terminate_logs() + if self._watchdog_stop is not None: + self._watchdog_stop.set() + + # -- responsiveness ------------------------------------------------------ + + def _start_stall_watchdog(self) -> None: + """Report any UI-thread stall, with the stack that caused it. + + A 0.1 s timer on the UI loop moves a heartbeat; a watchdog thread notices + when it stops, samples the UI thread's stack while it is stuck, and + reports the stall once it ends: its length in the TUI log, the stack in + the error log file. "The TUI froze" then comes with what it was doing. + """ + import sys + import threading + + ui_thread = threading.get_ident() + stop = threading.Event() + self._watchdog_stop = stop + + def beat() -> None: + self._beat = time.monotonic() + + self.set_interval(0.1, beat) + + def watch() -> None: + import traceback + + stalled_since = None + stack: list[str] = [] + while not stop.wait(0.1): + behind = time.monotonic() - self._beat + if behind >= STALL_REPORT_S: + if stalled_since is None: + stalled_since = self._beat + # The first sample, half a second in, is inside whatever + # blocks; later ones can catch the loop catching up. + frame = sys._current_frames().get(ui_thread) + if frame is not None: + stack = traceback.format_stack(frame) + elif stalled_since is not None: + length = self._beat - stalled_since + try: + self.call_from_thread(self._report_stall, length, stack) + except Exception: # noqa: BLE001 - app shutting down + return + stalled_since, stack = None, [] + + threading.Thread(target=watch, name='tui-stall-watchdog', daemon=True).start() + + def _report_stall(self, length: float, stack: list[str]) -> None: + # The innermost frames in infer_stack say what to fix; the file keeps all. + ours = [f for f in stack if 'infer_stack' in f] or stack + where = ours[-1].strip().splitlines()[0] if ours else '(no stack)' + path = self.error_log_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('a', encoding='utf-8') as handle: + handle.write(f'\n=== {time.strftime("%Y-%m-%d %H:%M:%S")} ' + f'UI stalled {length:.1f}s\n') + handle.writelines(stack) + except OSError: + pass + self.app_log(f'UI stalled {length:.1f}s in {where} (stack: {path})', level='warn') # -- theming for docker output bleed ---------------------------------- @@ -1640,16 +1730,23 @@ def _update_summary(self, leases, deployments, observed) -> None: # "0 running" — say we're still observing instead of silently lying. observing = self._observed_at is None running_label = 'observing…' if observing else f'{running} running' + # Assign only on change: Textual repaints a pane whenever its border + # title is set, even to the same text, and these two panes hold the + # lease and deployment tables -- most of the screen, every tick + # (~7 KB/s of terminal output while nothing changed). try: - self.query_one('#docker', Collapsible).title = ( - f'docker — {running_label}' - ) - self.query_one('#leases-pane').border_title = ( - f'leases — {active} active / {len(leases)}' - ) - self.query_one('#deployments-pane').border_title = ( - f'deployments — {running_label} / {len(deployments)}' - ) + docker = self.query_one('#docker', Collapsible) + title = f'docker — {running_label}' + if docker.title != title: + docker.title = title + for pane, text in ( + ('#leases-pane', f'leases — {active} active / {len(leases)}'), + ('#deployments-pane', + f'deployments — {running_label} / {len(deployments)}'), + ): + widget = self.query_one(pane) + if widget.border_title != text: + widget.border_title = text except Exception: # noqa: BLE001 pass @@ -1895,9 +1992,16 @@ def _service_names(self) -> list[str]: if not path: return [] try: + stat = Path(path).stat() + key = (stat.st_mtime, stat.st_size) + cached = self._service_names_cache + if cached is not None and cached[0] == key: + return list(cached[1]) # checked every refresh; parse on change import yaml data = yaml.safe_load(Path(path).read_text()) or {} - return sorted((data.get('services') or {}).keys()) + names = sorted((data.get('services') or {}).keys()) + self._service_names_cache = (key, names) + return list(names) except Exception: # noqa: BLE001 return [] @@ -1973,6 +2077,8 @@ def _restart_logs(self, service: str) -> None: log = self.query_one('#logs', RichLog) log.clear() self._log_lines = [] + self._log_backlog.clear() + self._log_dropped = 0 target, label = self._resolve_log_target(service) if target is NO_LOG_TARGET: log.write(f'— {label} —') @@ -2033,9 +2139,10 @@ def _stream_logs(self, service, generation: int) -> None: self._log_proc = proc try: for line in compact_litellm_tracebacks(proc.stdout): - self.call_from_thread( - self._append_log_if_current, generation, line.rstrip('\n') - ) + if generation != self._log_generation: + break + # Drawn in batches by _drain_logs; deque.append is thread-safe. + self._log_pending.append((generation, line.rstrip('\n'))) except Exception: # noqa: BLE001 - stream ends when the proc dies pass finally: @@ -2048,8 +2155,31 @@ def _append_log_if_current(self, generation: int, line: str) -> None: self._append_log(line) def _append_log(self, line: str) -> None: - self._log_lines.append(line) - self.query_one('#logs', RichLog).write(line) + self._write_log_lines([line]) + + def _write_log_lines(self, lines: list[str]) -> None: + self._log_lines.extend(lines) + if len(self._log_lines) > 2 * LOG_PANE_LINES: + del self._log_lines[:-LOG_PANE_LINES] + self.query_one('#logs', RichLog).write('\n'.join(lines)) + + def _drain_logs(self) -> None: + """Draw streamed lines: a bounded batch per tick, newest lines kept.""" + pending, backlog = self._log_pending, self._log_backlog + while pending: + generation, line = pending.popleft() + if generation != self._log_generation: + continue + if len(backlog) == backlog.maxlen: + self._log_dropped += 1 # the oldest undrawn line falls off + backlog.append(line) + if not backlog: + return + batch = [backlog.popleft() for _ in range(min(LOG_DRAIN_LINES, len(backlog)))] + if self._log_dropped: + batch.insert(0, f'… {self._log_dropped} earlier line(s) not shown') + self._log_dropped = 0 + self._write_log_lines(batch) # -- the TUI's own log ------------------------------------------------- diff --git a/tests/test_tui.py b/tests/test_tui.py index 6e2e4620..ce742ddd 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -1407,7 +1407,8 @@ async def scenario(): async with app.run_test() as pilot: await pilot.pause() app._restart_logs(ALL_SERVICES) # the gateway's own lines - await app.workers.wait_for_complete() # drain the log stream + await app.workers.wait_for_complete() # the stream has ended + app._drain_logs() # lines are drawn in batches await pilot.pause() assert any('ready' in line for line in app._log_lines) @@ -1436,6 +1437,7 @@ async def scenario(): await pilot.pause() app._restart_logs(ALL_SERVICES) # the gateway's own lines await app.workers.wait_for_complete() + app._drain_logs() # lines are drawn in batches await pilot.pause() text = '\n'.join(app._log_lines) assert 'Traceback (most recent call last):' not in text @@ -1563,6 +1565,10 @@ async def scenario(): app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None) async with app.run_test() as pilot: + await pilot.pause() + # The mount-time background refresh must land first, or it can + # repaint all 40 rows after the rebuild below (an intermittent fail). + await app.workers.wait_for_complete() await pilot.pause() table = app.query_one('#leases', DataTable) assert table.row_count == 40 @@ -2003,3 +2009,60 @@ async def scenario(): _run(scenario) assert 'CLI: infer-stack release --all --yes' in seen['applog'] assert 'pulling img: 1 of 4 layers' in seen['applog'] + + +def test_a_ui_thread_stall_is_reported_with_where_it_happened(tmp_path, monkeypatch): + """"The TUI froze" must come with what it was doing.""" + import time + + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + monkeypatch.setattr(InferStackTUI, 'error_log_path', lambda self: tmp_path / 'e.log') + seen = {} + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test() as pilot: + await pilot.pause(0.3) + original = app._update_summary + + def slow(*args, **kwargs): + time.sleep(0.9) # a blocking call on the UI thread + return original(*args, **kwargs) + + app._update_summary = slow + app._refresh_now() + app._update_summary = original + await pilot.pause(0.5) # the watchdog reports after it ends + seen['applog'] = '\n'.join(app._app_log_lines) + + _run(scenario) + assert 'UI stalled' in seen['applog'] + assert 'in slow' in seen['applog'] or 'tui.py' in seen['applog'] + assert 'time.sleep(0.9)' in (tmp_path / 'e.log').read_text() + + +def test_a_log_flood_is_drawn_in_bounded_batches(): + from infer_stack.tui import ALL_SERVICES, LOG_PANE_LINES, InferStackTUI + + controller, catalog = _ctx() + lines = [f'vllm-x | loading shard {i}\n' for i in range(5000)] + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: _FakeProc(lines)) + async with app.run_test() as pilot: + await pilot.pause() + app._restart_logs(ALL_SERVICES) + await app.workers.wait_for_complete() + for _ in range(20): # a bounded batch per tick + app._drain_logs() + await pilot.pause() + shown = list(app._log_lines) + assert len(shown) <= 2 * LOG_PANE_LINES + assert shown[-1].endswith('loading shard 4999') # newest kept + assert any('earlier line(s) not shown' in s for s in shown) + + _run(scenario) From da718469cb15fbbf1367f6f7aca22bf23e5ce837 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 17:29:30 -0400 Subject: [PATCH 015/141] Changelog: the TUI stops repainting what did not change Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef0d357..640f0de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### The TUI stops repainting what did not change + +Every refresh tick repainted the lease and deployment panes, because setting +a pane's border title repaints it even when the text is unchanged: about +7 KB/s of terminal output with nothing on screen changing, which is what an +SSH session feels. Titles are now set only on change (75 bytes/s idle). +Streamed engine logs are drawn in batches of up to 200 lines every 100 ms +instead of one UI message per line, and a flood keeps its newest lines. A +watchdog reports any UI-thread stall over 0.5 s in the TUI log, with the +stack it was stuck in written to the error log file. `dev/profile_tui.py` +measures event-loop lag for idle, docker-pane and log-flood cases. + ### Ledger reads are safe across threads The ledger shares one SQLite connection between threads, but only write From 4feff29dcc32e0d6c4f144bc1a3fdbb223b8c9d1 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 17:46:17 -0400 Subject: [PATCH 016/141] TUI: an activity line for running actions; faster launch and quit The user's report: launch felt laggy, and a slow action gave no sign it was working. Measured on a real terminal with the new `infer-stack tui --exit_after_paint` (quits once the first frame is drawn, then prints when that was after process start): - Activity line: while an action runs in a background worker (acquire, release, evict, apply, save endpoint, inspecting GPUs, API calls), a line above the status bar shows a spinner, what, for how long, and the newest backend progress (image-pull layers). Driven by worker state changes, so no action handler had to change; status hints cannot overwrite it. - `requests` is imported on first use instead of when the backend is built: opening the controller went from 127 ms to 33 ms. - The API, UI and Settings tabs are built right after the first frame (textual Lazy) instead of before it. - On exit, in-flight read-only docker queries (ps, inspect, logs, ...) are killed so quitting does not wait for the background refresh's `docker compose ps`. Mutating commands are never killed. Real terminal, guest: launch+quit 1.15 s -> ~1.0 s, first frame ~0.74 s -> ~0.69 s. The rest is imports (textual, the CLI) and Textual's own mount. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/cli/commands_leasing.py | 7 ++ infer_stack/leasing/compose.py | 55 ++++++++- infer_stack/leasing/gateway.py | 19 ++- infer_stack/tui.py | 173 +++++++++++++++++++++++++++- tests/test_tui.py | 38 ++++++ 5 files changed, 279 insertions(+), 13 deletions(-) diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index e5bbc64c..18d1f3ff 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1702,6 +1702,12 @@ class TuiCLI(_LeasingCommonMixin): catalog = scfg.Value(None, type=str, help='Path to catalog.yaml.') interval = scfg.Value(3.0, type=float, help='Auto-refresh interval (s).') + exit_after_paint = scfg.Value( + False, isflag=True, + help='Quit as soon as the first frame is drawn, then print when that ' + 'was (for measuring startup: `time infer-stack tui ' + '--exit_after_paint`).', + ) @classmethod def main(cls, argv=True, **kwargs): @@ -1730,6 +1736,7 @@ def main(cls, argv=True, **kwargs): return run_tui( controller, catalog, interval=float(config.interval), catalog_path=str(catalog_path), + exit_after_paint=bool(config.exit_after_paint), ) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index d564b7f4..ea44570f 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -746,6 +746,52 @@ def docker_environment(environ: dict[str, str] | None = None) -> dict[str, str]: return env +#: Read-only docker queries in flight (ps, inspect, ...). A long-running +#: process that is exiting (the TUI) kills these rather than wait for them; +#: nothing that changes state is ever in this set. +_RUNNING_QUERIES: set = set() + + +_READ_VERBS = frozenset({'ps', 'inspect', 'logs', 'version', 'images', 'ls', + 'info', 'port', 'top', 'config'}) +_MUTATING_VERBS = frozenset({'up', 'down', 'rm', 'create', 'pull', 'run', 'exec', + 'start', 'stop', 'kill', 'pause', 'unpause', 'connect', + 'disconnect', 'prune', 'build', 'push', 'tag', 'restart'}) + + +def _read_only(args: list[str]) -> bool: + """A docker command that only reads state, and so is safe to abandon. + + Example: + >>> _read_only(['docker', 'compose', '-p', 'x', 'ps', '--format', 'json']) + True + >>> _read_only(['docker', 'network', 'create', 'n']) + False + """ + words = set(args) + return bool(words & _READ_VERBS) and not words & _MUTATING_VERBS + + +def cancel_running_queries() -> int: + """Kill every in-flight read-only docker query; how many were killed. + + For a process that is exiting: its background refresh would otherwise + hold the exit until ``docker compose ps`` returns (measured 0.66 s on the + guest, more on a loaded host). Mutations (up, rm, pull) are never killed. + """ + import os + import signal + + killed = 0 + for proc in list(_RUNNING_QUERIES): + try: + os.killpg(proc.pid, signal.SIGKILL) + killed += 1 + except (ProcessLookupError, PermissionError): + pass + return killed + + def _default_docker_run( args: list[str], *, timeout: float | None = None, stderr_lines: Callable[[str], None] | None = None, @@ -771,6 +817,9 @@ def _default_docker_run( env=docker_environment(), stderr=subprocess.PIPE if stderr_lines is not None else None, ) + query = _read_only(args) + if query: + _RUNNING_QUERIES.add(proc) def kill_group(): try: os.killpg(proc.pid, signal.SIGKILL) @@ -794,6 +843,9 @@ def kill_group(): # session, so without this it would keep running unattended. kill_group() raise + finally: + if query: + _RUNNING_QUERIES.discard(proc) if stderr_lines is not None: for line in (err or '').splitlines(): if line.strip(): @@ -967,9 +1019,6 @@ def __init__( # before the first frame. self._inventory = inventory self.run = run or _default_docker_run - if http is None: - import requests - http = requests # The front door: gateway settings, keys, routes. Created before the # settings below, which are stored on it. self.gateway = Gateway( diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index cf549633..433ee108 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -938,14 +938,25 @@ def __init__( self.reverse_proxy = reverse_proxy self.reverse_proxy_port = reverse_proxy_port self.dynamic_routing = dynamic_routing - if http is None: - import requests - http = requests - self.http = http + # None: `requests`, imported on first use. It costs ~75 ms, and a TUI + # launch or a ledger-only command never makes a request. + self._http = http self._sleep = sleep self._clock = clock self.assume_yes = True + @property + def http(self) -> Any: + if self._http is None: + import requests + + self._http = requests + return self._http + + @http.setter + def http(self, value: Any) -> None: + self._http = value + @property def litellm_port(self) -> int: return self.ports.get('litellm', DEFAULT_PORTS['litellm']) diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 3fc5b0de..68fe105e 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -38,6 +38,9 @@ from textual.binding import Binding from textual.coordinate import Coordinate from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.css.query import NoMatches +from textual.lazy import Lazy +from textual.widget import Widget from textual.screen import ModalScreen from textual.theme import Theme from textual.widgets import ( @@ -107,6 +110,48 @@ #: A UI-thread stall at least this long is reported, with what was running. STALL_REPORT_S = 0.5 +#: What a running background action is doing, shown in the activity line, by +#: worker method. A worker not listed here (the refresh, log streams) is +#: routine and shows nothing. +ACTIVITY_VERBS = { + '_do_acquire': 'acquiring', + '_do_release': 'releasing', + '_do_release_all': 'releasing all leases', + '_do_evict': 'evicting', + '_do_evict_all': 'evicting idle deployments', + '_do_cleanup': 'cleaning up the ledger', + '_do_apply': 'applying', + '_do_compose': 'docker compose', + '_save_endpoint': 'saving endpoint', + '_do_suggest': 'inspecting GPUs for a suggestion', + '_prepare_endpoint_editor': 'inspecting GPUs', + '_do_api_send': 'waiting for the model', + '_do_api_test_all': 'testing every model', + '_do_api_list': 'listing models', +} +SPINNER = '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏' + + +def activity_label(description: str) -> str | None: + """``'acquiring qwen'`` from a worker description like ``_do_acquire('qwen')``. + + Example: + >>> activity_label("_do_acquire('qwen')") + 'acquiring qwen' + >>> activity_label('_do_apply()') + 'applying' + >>> activity_label('_refresh_bg()') is None + True + """ + import re + + method, _, rest = str(description).partition('(') + verb = ACTIVITY_VERBS.get(method.strip()) + if verb is None: + return None + first = re.match(r"\s*'([^']*)'", rest) + return f'{verb} {first.group(1)}' if first else verb + SELECT_BLANK = next( ( candidate @@ -793,6 +838,30 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss(event.button.id == 'ok') +class _Section(Widget): + """A tab's content, built from ``compose_fn`` when it mounts. + + Wrapped in :class:`textual.lazy.Lazy` for tabs hidden at launch, so the + first frame does not wait for widgets nobody can see yet (CSS and layout + of the whole tree was most of the launch time). The app is told when the + content exists, to fill anything it set before then. + """ + + DEFAULT_CSS = '_Section { height: 1fr; }' + + def __init__(self, compose_fn: Callable[[], ComposeResult], *, id: str): + super().__init__(id=id) + self._compose_fn = compose_fn + + def compose(self) -> ComposeResult: + yield from self._compose_fn() + + def on_mount(self) -> None: + on_section = getattr(self.app, '_on_section_mounted', None) + if on_section is not None: + on_section(self.id) + + class InferStackTUI(App): """Monitor + control the leasing stack across panes.""" @@ -886,6 +955,7 @@ class InferStackTUI(App): #api-extra Button { margin: 0 1 0 0; min-width: 12; } #status { dock: bottom; height: 1; padding: 0 2; color: $text-muted; } + #activity { dock: bottom; height: 1; padding: 0 2; color: $warning; display: none; } #settings, #ui-settings { padding: 1 2; } #settings Label, #ui-settings Label { margin: 1 0 0 0; color: $text-muted; } @@ -936,8 +1006,13 @@ def __init__( proc_factory: Callable[[str | None], Any] | None = None, catalog_path: str | Path | None = None, http: Any = None, + exit_after_paint: bool = False, ) -> None: super().__init__() + #: Quit once the first frame is drawn (``tui --exit_after_paint``). + self._exit_after_paint = exit_after_paint + #: Seconds from process start to the first drawn frame, once known. + self.first_frame_s: float | None = None self.controller = controller self.catalog = catalog self.interval = interval @@ -1003,6 +1078,10 @@ def __init__( self._log_dropped = 0 # Parsed compose service names, keyed by the file's (mtime, size). self._service_names_cache: tuple[tuple[float, int], list[str]] | None = None + # Running background actions: worker -> (label, started). Drawn by + # _draw_activity, with the newest backend progress message. + self._activity: dict[Any, tuple[str, float]] = {} + self._activity_note = '' # The UI loop's heartbeat, read by the stall watchdog thread. self._beat = time.monotonic() self._watchdog_stop = None @@ -1037,15 +1116,17 @@ def compose(self) -> ComposeResult: with TabbedContent(id='top'): with TabPane('Dashboard', id='tab-dashboard'): yield from self._compose_dashboard() + # Hidden at launch: built right after the first frame, not before. with TabPane('API', id='tab-api'): - yield from self._compose_api() + yield Lazy(_Section(self._compose_api, id='section-api')) with TabPane('UI', id='tab-ui'): - yield from self._compose_ui_settings() + yield Lazy(_Section(self._compose_ui_settings, id='section-ui')) with TabPane('Settings', id='tab-settings'): - yield from self._compose_settings() + yield Lazy(_Section(self._compose_settings, id='section-settings')) with TabPane(APP_LOG_TAB_TITLE, id='tab-applog'): yield from self._compose_app_log() yield Static('', id='status') + yield Static('', id='activity') yield Footer() def _compose_app_log(self) -> ComposeResult: @@ -1314,13 +1395,28 @@ def on_mount(self) -> None: self.ledger_interval, self.action_refresh ) self.set_interval(LOG_DRAIN_S, self._drain_logs) + # Held directly: query_one searches the active screen, which is a + # dialog's while one is open. + self._activity_widget = self.query_one('#activity', Static) + self.set_interval(0.1, self._draw_activity) self._start_stall_watchdog() self.query_one('#endpoints', DataTable).focus() + self.call_after_refresh(self._first_frame_drawn) + + def _first_frame_drawn(self) -> None: + self.first_frame_s = _seconds_since_process_start() + if self._exit_after_paint: + self.exit() def on_unmount(self) -> None: self._terminate_logs() if self._watchdog_stop is not None: self._watchdog_stop.set() + # Quit must not wait for the background refresh's `docker compose ps`: + # Python joins worker threads at exit. Read-only queries only. + from .leasing.compose import cancel_running_queries + + cancel_running_queries() # -- responsiveness ------------------------------------------------------ @@ -1416,8 +1512,34 @@ def _backend_progress(self, message: str) -> None: except RuntimeError: # already on the UI thread self._show_progress(message) + def _on_section_mounted(self, section_id: str | None) -> None: + """A deferred tab now exists: fill what was set before it did.""" + if section_id == 'section-api': + self._update_api_urls() + self._update_api_curl() + self._sync_api_models(list(getattr(self, '_api_models_wanted', []))) + def _show_progress(self, message: str) -> None: self._status(message) # also recorded in the TUI log + self._activity_note = message # and beside the running action + + def _draw_activity(self) -> None: + """The activity line: what is running, for how long, and its progress.""" + widget = getattr(self, '_activity_widget', None) + if widget is None: + return + if not self._activity: + if widget.display: + widget.display = False + self._activity_note = '' + return + now = time.monotonic() + spin = SPINNER[int(now * 10) % len(SPINNER)] + parts = [f'{label} · {now - started:.0f}s' + for label, started in sorted(self._activity.values(), key=lambda v: v[1])] + note = f' — {self._activity_note}' if self._activity_note else '' + widget.update(f'{spin} ' + ' | '.join(parts) + note) + widget.display = True # -- resizable panes --------------------------------------------------- @@ -1534,10 +1656,14 @@ def _cached_label(source: str, hub: Path | None) -> str: def _sync_api_models(self, names: list[str]) -> None: """Point the API model selector at the currently-ready endpoints only.""" + self._api_models_wanted = names if names == self._ready_endpoints: return + try: + select = self.query_one('#api-model', Select) + except NoMatches: + return # the API tab is not built yet; filled when it is self._ready_endpoints = names - select = self.query_one('#api-model', Select) current = None if _select_is_blank(select.value) else select.value select.set_options([(n, n) for n in names]) if current in names: @@ -2292,6 +2418,14 @@ def on_worker_state_changed(self, event) -> None: worker = event.worker name = worker.name or worker.group or 'worker' + if event.state is WorkerState.RUNNING: + label = activity_label(worker.description) + if label is not None: + self._activity[worker] = (label, time.monotonic()) + self._draw_activity() + elif event.state in (WorkerState.SUCCESS, WorkerState.ERROR, + WorkerState.CANCELLED): + self._activity.pop(worker, None) if event.state is WorkerState.ERROR: error = getattr(worker, 'error', None) if error is not None: @@ -3220,7 +3354,10 @@ def _api_log(self, line: str) -> None: self.query_one('#api-out', RichLog).write(line) def _selected_api_model(self) -> str | None: - value = self.query_one('#api-model', Select).value + try: + value = self.query_one('#api-model', Select).value + except NoMatches: + return None # the API tab is built right after the first frame return None if _select_is_blank(value) else str(value) def action_api_send(self) -> None: @@ -3417,12 +3554,29 @@ def _fmt_ports(row: dict) -> str: return ', '.join(bits) +def _seconds_since_process_start() -> float | None: + """Wall time since this process started (Linux ``/proc``), else ``None``.""" + import os + + try: + with open('/proc/self/stat') as handle: + # Field 22 counts clock ticks since boot; the command name (field 2) + # may contain spaces, so split after its closing parenthesis. + start_ticks = int(handle.read().rsplit(')', 1)[1].split()[19]) + with open('/proc/uptime') as handle: + uptime = float(handle.read().split()[0]) + return uptime - start_ticks / os.sysconf('SC_CLK_TCK') + except (OSError, ValueError, IndexError): + return None + + def run_tui( controller, catalog, *, interval: float = 3.0, catalog_path: str | Path | None = None, + exit_after_paint: bool = False, ) -> int: """Run the TUI against a built controller + catalog. Returns an exit code.""" # The narration loguru sink writes to stderr, which would corrupt the @@ -3434,7 +3588,14 @@ def run_tui( except Exception: # noqa: BLE001 pass app: Any = InferStackTUI( - controller, catalog, interval=interval, catalog_path=catalog_path + controller, catalog, interval=interval, catalog_path=catalog_path, + exit_after_paint=exit_after_paint, ) app.run() + if exit_after_paint: + first = app.first_frame_s + exited = _seconds_since_process_start() + print(f'first frame {first:.2f}s after process start; exited at {exited:.2f}s' + if first is not None and exited is not None + else 'first frame drawn (process start time unavailable)') return 0 diff --git a/tests/test_tui.py b/tests/test_tui.py index ce742ddd..66a111cd 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -2066,3 +2066,41 @@ async def scenario(): assert any('earlier line(s) not shown' in s for s in shown) _run(scenario) + + +def test_a_running_action_shows_in_the_activity_line_until_it_ends(): + """A slow action is visibly working: spinner, what, and for how long.""" + import threading + + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + name = next(iter(catalog.endpoints)) + gate = threading.Event() + real_acquire = controller.acquire + + def slow_acquire(*args, **kwargs): + gate.wait(5) + return real_acquire(*args, **kwargs) + + controller.acquire = slow_acquire + seen = {} + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test() as pilot: + await pilot.pause() + app._start_acquire(name) + await pilot.pause(0.3) + line = app.query_one('#activity') + seen['during'] = (line.display, str(line.render())) + gate.set() + await app.workers.wait_for_complete() + await pilot.pause(0.3) + seen['after'] = line.display + + _run(scenario) + shown, text = seen['during'] + assert shown and f'acquiring {name}' in text and 's' in text + assert seen['after'] is False From a3d18c10c0c678bb8968cb16d19c99c98401801f Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 17:57:39 -0400 Subject: [PATCH 017/141] Switch the CLI from scriptconfig to kwconf kwconf is scriptconfig's successor and what the sibling packages (aiq-magnet, cmd_queue, kwdagger) already use. Mechanical where the APIs match: `import kwconf as kw`, scfg.Value -> kw.Value, scfg.ModalCLI -> kw.ModalCLI, scfg.DataConfig -> kw.Config (233 uses in 9 files). Differences that needed a change: - kwconf's automatic conversion reads `null` as None, so `--backend null` failed its choices; the 8 options with string choices get type=str. - ModalCLI names the program after the class ("ManageCLI"); `__prog__` keeps `infer-stack` in usage and error messages. - The TUI's activity_label doctest needs textual, which CI does not install; it is marked `+REQUIRES(module:textual)`. Dependency: kwconf>=0.11.0. The repo's uv exclude-newer cutoff (2026-06-04) predates every kwconf release, so kwconf alone is admitted up to 2026-08-08 (0.11.0, the version tested) via exclude-newer-package; uv.lock and the exported test lock only gain kwconf and lose scriptconfig. Checked: full suite 909 passed; CI's checks (flake8, ty, the package plus tests with only the tests extra) pass on 3.14 and 3.10 (838 each); CLI smoke: help, --version, subcommands, --no-wait, unknown flags rejected. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 15 +++ README.md | 2 +- docs/source/conf.py | 2 +- infer_stack/cli/__init__.py | 16 ++- infer_stack/cli/commands_catalog.py | 128 +++++++++--------- infer_stack/cli/commands_leasing.py | 200 ++++++++++++++-------------- infer_stack/cli/commands_meta.py | 32 ++--- infer_stack/cli/commands_mock.py | 24 ++-- infer_stack/cli/commands_runtime.py | 40 +++--- infer_stack/cli/context.py | 2 +- infer_stack/cli/options.py | 54 ++++---- infer_stack/tui.py | 1 + pyproject.toml | 6 +- requirements/locks/tests.txt | 12 +- tests/test_cli_equivalent.py | 4 +- uv.lock | 31 +++-- 16 files changed, 294 insertions(+), 275 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 640f0de6..571b00d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### The CLI is built on kwconf instead of scriptconfig + +kwconf is scriptconfig's successor, which aiq-magnet, cmd_queue and kwdagger +already use. `Value`, `ModalCLI` and `__command__` carry over; `DataConfig` +is now `Config`. One behavioural difference needed care: kwconf reads the +strings `null`, `true` and numbers on the command line as values, so +`--backend null` arrived as `None`. Options whose choices are strings now +take `type=str`. The program name stays `infer-stack` in usage and errors +(`__prog__`). + +The repository's uv `exclude-newer` cutoff (2026-06-04) predates every kwconf +release, so kwconf alone gets an exception up to 0.11.0, the version tested. +`uv.lock` and `requirements/locks/tests.txt` were regenerated: kwconf 0.11.0 +added, scriptconfig removed. + ### The TUI stops repainting what did not change Every refresh tick repainted the lease and deployment panes, because setting diff --git a/README.md b/README.md index 642c5ee3..fc51de66 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ infer-stack version # print the installed version infer-stack config paths # show where config / artifacts / caches live ``` -The CLI is built on [`scriptconfig`](https://gitlab.kitware.com/utils/scriptconfig), +The CLI is built on [`kwconf`](https://github.com/Erotemic/kwconf), so every subcommand is also importable as a Python class — useful for notebooks, tests, and other scripts: diff --git a/docs/source/conf.py b/docs/source/conf.py index d28cf6dc..0818d132 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -235,7 +235,7 @@ def visit_Assign(self, node): 'ubelt': ('https://ubelt.readthedocs.io/en/latest/', None), 'xdoctest': ('https://xdoctest.readthedocs.io/en/latest/', None), 'networkx': ('https://networkx.org/documentation/stable/', None), - 'scriptconfig': ('https://scriptconfig.readthedocs.io/en/latest/', None), + 'kwconf': ('https://kwconf.readthedocs.io/en/latest/', None), 'kwconf': ('https://kwconf.readthedocs.io/en/latest/', None), 'rich': ('https://rich.readthedocs.io/en/latest/', None), 'numpy': ('https://numpy.org/doc/stable/', None), diff --git a/infer_stack/cli/__init__.py b/infer_stack/cli/__init__.py index 94c81a8d..eb00a120 100644 --- a/infer_stack/cli/__init__.py +++ b/infer_stack/cli/__init__.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # PYTHON_ARGCOMPLETE_OK -"""scriptconfig-based CLI for infer-stack. +"""kwconf-based CLI for infer-stack. -Each subcommand is a ``scfg.DataConfig`` subclass; ``ManageCLI`` composes -them into a single ``scfg.ModalCLI`` exposed as the ``infer-stack`` entry +Each subcommand is a ``kw.Config`` subclass; ``ManageCLI`` composes +them into a single ``kw.ModalCLI`` exposed as the ``infer-stack`` entry point. Because every subcommand is a ``DataConfig``, the same class can be invoked from the shell (``infer-stack render --profile X``) or from Python (``RenderCLI.main(argv=False, profile='X')``). @@ -22,7 +22,7 @@ from __future__ import annotations -import scriptconfig as scfg +import kwconf as kw from .. import __version__ @@ -78,7 +78,11 @@ # --------------------------------------------------------------------------- -class ManageCLI(scfg.ModalCLI): +class ManageCLI(kw.ModalCLI): + # The program name in usage and error messages (kwconf would use the class + # name, "ManageCLI"). + __prog__ = 'infer-stack' + description = ( 'Lease, acquire, and run LLM endpoints. Primary workflow: ' 'catalog -> acquire/run. See `infer-stack help tree`.' @@ -104,7 +108,7 @@ class ManageCLI(scfg.ModalCLI): actual. Run `infer-stack help tree` for the whole command surface. """ - # Backs the modal ``--version`` flag (scriptconfig reads ``__version__``). + # Backs the modal ``--version`` flag (kwconf reads ``__version__``). # The ``version`` *subcommand* is registered below under a non-colliding # attribute name; its CLI name comes from ``VersionCLI.__command__``. __version__ = __version__ diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index 38a5abcc..d68e62e0 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -16,7 +16,7 @@ from pathlib import Path from typing import Any -import scriptconfig as scfg +import kwconf as kw import yaml from ..leasing import Catalog, CatalogError @@ -223,8 +223,8 @@ def _parse_kv(items) -> dict[str, Any]: class _CatalogCommon(_PathOverridesMixin): - catalog = scfg.Value(None, type=str, help='Catalog path (default: config dir).') - dry_run = scfg.Value( + catalog = kw.Value(None, type=str, help='Catalog path (default: config dir).') + dry_run = kw.Value( False, isflag=True, help='Print the resulting YAML, do not write.' ) @@ -233,7 +233,7 @@ class CatalogInitCLI(_CatalogCommon): """Write a starter catalog.yaml (empty sections) if none exists.""" __command__ = 'init' - force = scfg.Value(False, isflag=True, help='Overwrite an existing catalog.') + force = kw.Value(False, isflag=True, help='Overwrite an existing catalog.') @classmethod def main(cls, argv=True, **kwargs): @@ -275,12 +275,12 @@ class CatalogSuggestCLI( """ __command__ = 'suggest' - catalog = scfg.Value(None, type=str, help='Catalog path (default: config dir).') - apply = scfg.Value( + catalog = kw.Value(None, type=str, help='Catalog path (default: config dir).') + apply = kw.Value( False, isflag=True, help='Merge the suggestion into the catalog (default: render only).', ) - force = scfg.Value( + force = kw.Value( False, isflag=True, help='With --apply, overwrite catalog entries that already exist.', ) @@ -363,7 +363,7 @@ class CatalogPathCLI(_PathOverridesMixin): """Print the catalog path.""" __command__ = 'path' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -392,8 +392,8 @@ class CatalogShowCLI(_PathOverridesMixin): """Pretty-print the whole catalog (or one named entry across sections).""" __command__ = 'show' - catalog = scfg.Value(None, type=str) - name = scfg.Value(None, position=1, type=str, help='Optional entry name.') + catalog = kw.Value(None, type=str) + name = kw.Value(None, position=1, type=str, help='Optional entry name.') @classmethod def main(cls, argv=True, **kwargs): @@ -417,7 +417,7 @@ class CatalogValidateCLI(_PathOverridesMixin): """Parse + cross-reference check the catalog.""" __command__ = 'validate' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -440,7 +440,7 @@ class CatalogEditCLI(_PathOverridesMixin): """Open the catalog in $EDITOR (escape hatch), then validate it.""" __command__ = 'edit' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -470,12 +470,12 @@ class ModelAddCLI(_CatalogCommon): """Add (or --force overwrite) a model: a Hugging Face / local weight source.""" __command__ = 'add' - name = scfg.Value(None, position=1, type=str) - source = scfg.Value(None, type=str, help='e.g. hf://org/Model or a path.') - revision = scfg.Value(None, type=str) - quantization = scfg.Value(None, type=str) - dtype = scfg.Value(None, type=str) - force = scfg.Value(False, isflag=True) + name = kw.Value(None, position=1, type=str) + source = kw.Value(None, type=str, help='e.g. hf://org/Model or a path.') + revision = kw.Value(None, type=str) + quantization = kw.Value(None, type=str) + dtype = kw.Value(None, type=str) + force = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -503,7 +503,7 @@ def main(cls, argv=True, **kwargs): class ModelListCLI(_PathOverridesMixin): """List model names.""" __command__ = 'list' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -513,8 +513,8 @@ def main(cls, argv=True, **kwargs): class ModelShowCLI(_PathOverridesMixin): """Show a model entry, or all of them when no NAME is given.""" __command__ = 'show' - catalog = scfg.Value(None, type=str) - name = scfg.Value(None, position=1, type=str, + catalog = kw.Value(None, type=str) + name = kw.Value(None, position=1, type=str, help='Model name; omit to show every model.') @classmethod @@ -526,7 +526,7 @@ def main(cls, argv=True, **kwargs): class ModelRmCLI(_CatalogCommon): """Remove one or more models by name.""" __command__ = 'rm' - names = scfg.Value([], nargs='+', position=1, type=str, + names = kw.Value([], nargs='+', position=1, type=str, help='Model name(s) to remove.') @classmethod @@ -535,7 +535,7 @@ def main(cls, argv=True, **kwargs): return _rm(config, 'models', config.names) -class CatalogModelCLI(scfg.ModalCLI): +class CatalogModelCLI(kw.ModalCLI): """Manage catalog models.""" __command__ = 'model' add = ModelAddCLI @@ -562,55 +562,55 @@ class EndpointAddCLI(_CatalogCommon): """ __command__ = 'add' - name = scfg.Value( + name = kw.Value( None, position=1, type=str, help='Endpoint alias (default: {model}-N, auto-incrementing).', ) - engine = scfg.Value('vllm', choices=['vllm', 'ollama']) - model = scfg.Value(None, type=str, help='Model name (vllm) or tag (ollama).') - host = scfg.Value(None, type=str, help='Runtime host (ollama).') - public_name = scfg.Value( + engine = kw.Value('vllm', type=str, choices=['vllm', 'ollama']) + model = kw.Value(None, type=str, help='Model name (vllm) or tag (ollama).') + host = kw.Value(None, type=str, help='Runtime host (ollama).') + public_name = kw.Value( None, type=str, help='Served/public name (for coalescing aliases).' ) - reclaim = scfg.Value( - None, choices=['keep-warm', 'stop', 'scale-to-zero'], + reclaim = kw.Value( + None, type=str, choices=['keep-warm', 'stop', 'scale-to-zero'], help='Reclaim policy when idle.', ) - protocol = scfg.Value( - None, choices=['chat', 'completions'], + protocol = kw.Value( + None, type=str, choices=['chat', 'completions'], help='Which OpenAI surface this endpoint serves. Load-bearing twice: ' 'a base model has no chat template, and the readiness probe ' 'follows this — declaring chat for a completions-only serve ' 'blocks `acquire` until the TTL. Default (unset): chat.', ) - min_vram_gib = scfg.Value( + min_vram_gib = kw.Value( None, type=float, help='placement.min_vram_gib — the VRAM this endpoint needs, so the ' 'planner can pick any eligible free GPU. Declaring this is what ' 'lets one catalog be correct on every host.', ) - gpu = scfg.Value( + gpu = kw.Value( [], nargs='*', type=int, help='placement.gpu_indices — exact physical GPU index/indices. Omit ' 'for automatic VRAM-aware placement. This is a local operator ' 'override and is intentionally less portable than --min-vram-gib.', ) # vLLM runtime conveniences - max_model_len = scfg.Value(None, type=int) - gpu_mem = scfg.Value( + max_model_len = kw.Value(None, type=int) + gpu_mem = kw.Value( None, type=float, help='gpu_memory_utilization (0-1).' ) - tensor_parallel = scfg.Value(None, type=int) - extra_args = scfg.Value( + tensor_parallel = kw.Value(None, type=int) + extra_args = kw.Value( None, type=str, help="Raw vLLM flags as one string (shell-split), " "e.g. --extra-args='--dtype=half --enforce-eager'.", ) - runtime = scfg.Value( + runtime = kw.Value( [], nargs='*', type=str, help='Extra runtime KEY=VALUE pairs (YAML-typed).', ) - force = scfg.Value(False, isflag=True) + force = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -681,7 +681,7 @@ def main(cls, argv=True, **kwargs): class EndpointListCLI(_PathOverridesMixin): """List endpoint names.""" __command__ = 'list' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -691,8 +691,8 @@ def main(cls, argv=True, **kwargs): class EndpointShowCLI(_PathOverridesMixin): """Show an endpoint entry, or all of them when no NAME is given.""" __command__ = 'show' - catalog = scfg.Value(None, type=str) - name = scfg.Value(None, position=1, type=str, + catalog = kw.Value(None, type=str) + name = kw.Value(None, position=1, type=str, help='Endpoint name; omit to show every endpoint.') @classmethod @@ -704,7 +704,7 @@ def main(cls, argv=True, **kwargs): class EndpointRmCLI(_CatalogCommon): """Remove one or more endpoints by name.""" __command__ = 'rm' - names = scfg.Value([], nargs='+', position=1, type=str, + names = kw.Value([], nargs='+', position=1, type=str, help='Endpoint name(s) to remove.') @classmethod @@ -713,7 +713,7 @@ def main(cls, argv=True, **kwargs): return _rm(config, 'endpoints', config.names) -class CatalogEndpointCLI(scfg.ModalCLI): +class CatalogEndpointCLI(kw.ModalCLI): """Manage catalog endpoints.""" __command__ = 'endpoint' add = EndpointAddCLI @@ -731,15 +731,15 @@ class HostAddCLI(_CatalogCommon): """Add (or --force overwrite) a runtime host (e.g. an Ollama daemon).""" __command__ = 'add' - name = scfg.Value(None, position=1, type=str) - engine = scfg.Value('ollama', choices=['ollama']) - gpu = scfg.Value([], nargs='*', type=int, help='GPU index/indices.') - keep_alive = scfg.Value(None, type=str, help='Ollama keep_alive, e.g. 5m.') - num_parallel = scfg.Value(None, type=int) - max_loaded_models = scfg.Value(None, type=int) - context_length = scfg.Value(None, type=int) - image = scfg.Value(None, type=str) - force = scfg.Value(False, isflag=True) + name = kw.Value(None, position=1, type=str) + engine = kw.Value('ollama', type=str, choices=['ollama']) + gpu = kw.Value([], nargs='*', type=int, help='GPU index/indices.') + keep_alive = kw.Value(None, type=str, help='Ollama keep_alive, e.g. 5m.') + num_parallel = kw.Value(None, type=int) + max_loaded_models = kw.Value(None, type=int) + context_length = kw.Value(None, type=int) + image = kw.Value(None, type=str) + force = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -780,7 +780,7 @@ def main(cls, argv=True, **kwargs): class HostListCLI(_PathOverridesMixin): """List runtime-host names.""" __command__ = 'list' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -790,7 +790,7 @@ def main(cls, argv=True, **kwargs): class HostRmCLI(_CatalogCommon): """Remove one or more runtime hosts by name.""" __command__ = 'rm' - names = scfg.Value([], nargs='+', position=1, type=str, + names = kw.Value([], nargs='+', position=1, type=str, help='Runtime-host name(s) to remove.') @classmethod @@ -799,7 +799,7 @@ def main(cls, argv=True, **kwargs): return _rm(config, 'runtime_hosts', config.names) -class CatalogHostCLI(scfg.ModalCLI): +class CatalogHostCLI(kw.ModalCLI): """Manage runtime hosts (Ollama daemons / placement).""" __command__ = 'host' add = HostAddCLI @@ -816,9 +816,9 @@ class BundleAddCLI(_CatalogCommon): """Add (or --force overwrite) a bundle: a named group of endpoints.""" __command__ = 'add' - name = scfg.Value(None, position=1, type=str) - members = scfg.Value([], nargs='*', position=2, type=str) - force = scfg.Value(False, isflag=True) + name = kw.Value(None, position=1, type=str) + members = kw.Value([], nargs='*', position=2, type=str) + force = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -844,7 +844,7 @@ def main(cls, argv=True, **kwargs): class BundleListCLI(_PathOverridesMixin): """List bundle names.""" __command__ = 'list' - catalog = scfg.Value(None, type=str) + catalog = kw.Value(None, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -854,7 +854,7 @@ def main(cls, argv=True, **kwargs): class BundleRmCLI(_CatalogCommon): """Remove one or more bundles by name.""" __command__ = 'rm' - names = scfg.Value([], nargs='+', position=1, type=str, + names = kw.Value([], nargs='+', position=1, type=str, help='Bundle name(s) to remove.') @classmethod @@ -863,7 +863,7 @@ def main(cls, argv=True, **kwargs): return _rm(config, 'bundles', config.names) -class CatalogBundleCLI(scfg.ModalCLI): +class CatalogBundleCLI(kw.ModalCLI): """Manage endpoint bundles.""" __command__ = 'bundle' add = BundleAddCLI @@ -876,7 +876,7 @@ class CatalogBundleCLI(scfg.ModalCLI): # --------------------------------------------------------------------------- -class CatalogModalCLI(scfg.ModalCLI): +class CatalogModalCLI(kw.ModalCLI): """Edit the user serving catalog (models / endpoints / hosts / bundles).""" __command__ = 'catalog' diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 18d1f3ff..14bdb67d 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -26,7 +26,7 @@ from typing import Any from pathlib import Path -import scriptconfig as scfg +import kwconf as kw from ..env_utils import parse_env_file, write_env_file from ..leasing import ( @@ -726,24 +726,24 @@ def _do_acquire(config, *, owner: str, ttl_seconds: float | None) -> int: class _LeasingCommonMixin(_PathOverridesMixin, _AllowedGpusMixin, _DisplayGpuMixin): - backend = scfg.Value( + backend = kw.Value( None, - choices=['null', 'compose', 'kubeai'], + type=str, choices=['null', 'compose', 'kubeai'], help='Serving backend: "null" (dry-run), "compose" (single-host ' 'docker), or "kubeai" (cluster; see docs/kubeai-backend.md). ' 'Defaults to `config set backend …`, else "null".', ) - ledger = scfg.Value( + ledger = kw.Value( None, type=str, help='Path to the lease ledger sqlite db.' ) - require_generation = scfg.Value( + require_generation = kw.Value( False, isflag=True, help='Deprecated/no-op: readiness now ALWAYS verifies a real generation ' '(a listed alias or a running container is not proof the model serves). ' 'Accepted for compatibility.', ) - litellm = scfg.Value( + litellm = kw.Value( None, isflag=True, help='Render the LiteLLM gateway — one OpenAI base_url fronting every ' @@ -751,14 +751,14 @@ class _LeasingCommonMixin(_PathOverridesMixin, _AllowedGpusMixin, _DisplayGpuMix 'a lean stack where Open WebUI talks to the upstreams (e.g. an Ollama ' 'daemon) directly. Overrides `config set litellm …`.', ) - ui = scfg.Value( + ui = kw.Value( None, isflag=True, help='Render a managed Open WebUI in front of the gateway (compose ' 'backend). On by default; use --no-ui to skip. Overrides ' '`config set ui …`.', ) - reverse_proxy = scfg.Value( + reverse_proxy = kw.Value( None, isflag=True, alias=['reverse-proxy'], @@ -767,7 +767,7 @@ class _LeasingCommonMixin(_PathOverridesMixin, _AllowedGpusMixin, _DisplayGpuMix '(localhost / trusted networks only). Port + bring-your-own nginx.conf ' 'live in the `reverse_proxy` setting (`config set` / `config edit`).', ) - dynamic_routing = scfg.Value( + dynamic_routing = kw.Value( None, isflag=True, alias=['dynamic-routing'], @@ -787,14 +787,14 @@ class _ApprovalMixin(_LeasingCommonMixin): terminal; ``--yes`` (or a non-TTY) applies without prompting. """ - catalog = scfg.Value( + catalog = kw.Value( None, type=str, help='Path to catalog.yaml. release/gc/evict reconcile the gateway too, ' 'so pass the same catalog as acquire to keep the static superset route ' 'table (no gateway blip); omitted, it falls back to the default-path ' 'catalog, else legacy per-deployment routing.', ) - yes = scfg.Value( + yes = kw.Value( False, isflag=True, alias=['y'], help='Apply compose changes without showing the diff / prompting ' '(compose backend). Implied when stdout is not a terminal.', @@ -802,21 +802,21 @@ class _ApprovalMixin(_LeasingCommonMixin): class _AcquireFlagsMixin(_LeasingCommonMixin): - catalog = scfg.Value(None, type=str, help='Path to catalog.yaml.') - base_url = scfg.Value( + catalog = kw.Value(None, type=str, help='Path to catalog.yaml.') + base_url = kw.Value( 'http://127.0.0.1:14042/v1', type=str, help='Base URL written into the endpoint descriptor (dry-run placeholder).', ) - api_key_env = scfg.Value( + api_key_env = kw.Value( 'LITELLM_MASTER_KEY', type=str, help='Name of the env var holding the API key (kept out of artifacts).', ) - wait = scfg.Value( + wait = kw.Value( True, isflag=True, help='Block until ready (use --no-wait to skip).' ) - queue = scfg.Value( + queue = kw.Value( False, isflag=True, help='Admission queue: if every GPU is busy, WAIT for one to free ' '(up to --timeout) instead of failing fast. Each retry sweeps the ' @@ -824,7 +824,7 @@ class _AcquireFlagsMixin(_LeasingCommonMixin): 'waiting. Intended for batch/pipeline fan-out; interactive use ' 'defaults off (fail fast with a clear "no GPU" error).', ) - apply = scfg.Value( + apply = kw.Value( True, isflag=True, help='Apply the render (docker compose up). Use --no-apply to *stage* ' @@ -833,17 +833,17 @@ class _AcquireFlagsMixin(_LeasingCommonMixin): '(compose backend). --no-apply implies no readiness wait and no diff ' 'prompt; `release` discards a staged lease.', ) - timeout = scfg.Value(600, type=float, help='Readiness wait timeout (s).') - interval = scfg.Value(5, type=float, help='Readiness poll interval (s).') - env_file = scfg.Value( + timeout = kw.Value(600, type=float, help='Readiness wait timeout (s).') + interval = kw.Value(5, type=float, help='Readiness poll interval (s).') + env_file = kw.Value( None, type=str, help='Write the sourceable endpoint env-file here.' ) - yes = scfg.Value( + yes = kw.Value( False, isflag=True, alias=['y'], help='Apply compose changes without showing the diff / prompting ' '(compose backend). Implied when stdout is not a terminal.', ) - json = scfg.Value(False, isflag=True, help='Emit JSON instead of text.') + json = kw.Value(False, isflag=True, help='Emit JSON instead of text.') # --------------------------------------------------------------------------- @@ -901,19 +901,19 @@ class AcquireCLI(_AcquireFlagsMixin): infer-stack leases """ - names = scfg.Value( + names = kw.Value( [], nargs='*', position=1, type=str, help='Endpoint or bundle names.' ) - ttl = scfg.Value( + ttl = kw.Value( None, type=str, help='Soft TTL (e.g. 2h, 30m); default infinite.' ) - owner = scfg.Value(None, type=str, help='Lease owner (default: $USER).') - dedicated = scfg.Value( + owner = kw.Value(None, type=str, help='Lease owner (default: $USER).') + dedicated = kw.Value( False, isflag=True, help='Force a dedicated deployment instead of coalescing.', ) - reserve_gpus = scfg.Value( + reserve_gpus = kw.Value( 0, type=int, alias=['reserve-gpus'], @@ -949,7 +949,7 @@ class RenderCLI(_LeasingCommonMixin): __command__ = 'render' - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -990,12 +990,12 @@ class ApplyCLI(_ApprovalMixin): __command__ = 'apply' - wait = scfg.Value( + wait = kw.Value( False, isflag=True, help='Also block until ready after bringing it up.' ) - timeout = scfg.Value(600, type=float, help='Readiness wait timeout (s).') - interval = scfg.Value(5, type=float, help='Readiness poll interval (s).') - json = scfg.Value(False, isflag=True) + timeout = kw.Value(600, type=float, help='Readiness wait timeout (s).') + interval = kw.Value(5, type=float, help='Readiness poll interval (s).') + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1075,22 +1075,22 @@ class ReleaseCLI(_ApprovalMixin): __command__ = 'release' - lease = scfg.Value( + lease = kw.Value( None, position=1, type=str, help='Lease id (or use --env-file).' ) - env_file = scfg.Value( + env_file = kw.Value( None, type=str, help='Read the lease id from this env-file.' ) - all = scfg.Value( + all = kw.Value( False, isflag=True, help='Release every active lease (the whole stack idles/tears down).', ) - evict = scfg.Value( + evict = kw.Value( False, isflag=True, help='Also evict (tear down) the released deployment(s) now, even if their ' 'reclaim policy is keep-warm — frees the GPU immediately.', ) - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1183,12 +1183,12 @@ class EvictCLI(_ApprovalMixin): __command__ = 'evict' - names = scfg.Value( + names = kw.Value( [], nargs='*', position=1, type=str, help='Endpoint alias or deployment id to evict.', ) - all = scfg.Value(False, isflag=True, help='Evict every idle deployment.') - json = scfg.Value(False, isflag=True) + all = kw.Value(False, isflag=True, help='Evict every idle deployment.') + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1252,17 +1252,17 @@ class GcCLI(_ApprovalMixin): __command__ = 'gc' - evict = scfg.Value( + evict = kw.Value( False, isflag=True, help='Also tear down idle keep-warm deployments (like `evict --all`), ' 'not just leaked/expired demand.', ) - orphans = scfg.Value( + orphans = kw.Value( False, isflag=True, help='Instead: remove containers in the project that infer-stack does not ' 'manage (listed and confirmed first; --yes skips the prompt).', ) - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1337,16 +1337,16 @@ class CleanCLI(_LeasingCommonMixin): infer-stack clean -f --no-orphans # leave unmanaged containers alone """ - force = scfg.Value( + force = kw.Value( False, isflag=True, short_alias=['f'], help='Actually release and tear down. Without it, clean only reports.', ) - orphans = scfg.Value( + orphans = kw.Value( True, isflag=True, help='Also remove containers in the project that infer-stack does not ' 'manage (--no-orphans keeps them).', ) - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1448,13 +1448,13 @@ class WaitCLI(_LeasingCommonMixin): __command__ = 'wait' - names = scfg.Value( + names = kw.Value( [], nargs='*', position=1, type=str, help='Endpoint names to wait for (default: every live deployment).', ) - timeout = scfg.Value(600, type=float, help='Overall wait timeout (s).') - interval = scfg.Value(5, type=float, help='Readiness poll interval (s).') - json = scfg.Value(False, isflag=True) + timeout = kw.Value(600, type=float, help='Overall wait timeout (s).') + interval = kw.Value(5, type=float, help='Readiness poll interval (s).') + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1523,35 +1523,35 @@ class MeasureCLI(_LeasingCommonMixin): __command__ = 'measure' - endpoint = scfg.Value( + endpoint = kw.Value( None, position=1, required=True, type=str, help='Catalog endpoint to measure.', ) - record = scfg.Value( + record = kw.Value( False, isflag=True, help='Record the result into the measurements overlay ' '(consulted automatically at plan time when the catalog declares ' 'nothing for this endpoint).', ) - kv_gib = scfg.Value( + kv_gib = kw.Value( 2.0, type=float, help='KV-cache budget (GiB) added on top of the non-KV profile. ' 'A serving choice (max_model_len / max_num_seqs), not a model fact.', ) - margin = scfg.Value( + margin = kw.Value( 0.05, type=float, help='Safety-margin fraction over the non-KV profile ' '(allocator fragmentation, engine drift).', ) - timeout = scfg.Value( + timeout = kw.Value( 900, type=float, help='Readiness timeout when the endpoint must be brought up first (s).', ) - catalog = scfg.Value( + catalog = kw.Value( None, type=str, help='Catalog path (default: /catalog.yaml).', ) - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -1700,9 +1700,9 @@ class TuiCLI(_LeasingCommonMixin): __command__ = 'tui' - catalog = scfg.Value(None, type=str, help='Path to catalog.yaml.') - interval = scfg.Value(3.0, type=float, help='Auto-refresh interval (s).') - exit_after_paint = scfg.Value( + catalog = kw.Value(None, type=str, help='Path to catalog.yaml.') + interval = kw.Value(3.0, type=float, help='Auto-refresh interval (s).') + exit_after_paint = kw.Value( False, isflag=True, help='Quit as soon as the first frame is drawn, then print when that ' 'was (for measuring startup: `time infer-stack tui ' @@ -1745,9 +1745,9 @@ class RenewCLI(_LeasingCommonMixin): __command__ = 'renew' - lease = scfg.Value(None, position=1, type=str, help='Lease id.') - env_file = scfg.Value(None, type=str) - ttl = scfg.Value(None, type=str, help='New soft TTL (e.g. 2h); empty=infinite.') + lease = kw.Value(None, position=1, type=str, help='Lease id.') + env_file = kw.Value(None, type=str) + ttl = kw.Value(None, type=str, help='New soft TTL (e.g. 2h); empty=infinite.') @classmethod def main(cls, argv=True, **kwargs): @@ -1791,26 +1791,26 @@ class RunCLI(_LeasingCommonMixin): __command__ = 'run' - catalog = scfg.Value(None, type=str, help='Path to catalog.yaml.') - endpoint = scfg.Value( + catalog = kw.Value(None, type=str, help='Path to catalog.yaml.') + endpoint = kw.Value( None, type=str, alias=['endpoints'], help='Comma-separated endpoint or bundle names.', ) - base_url = scfg.Value('http://127.0.0.1:14042/v1', type=str) - api_key_env = scfg.Value('LITELLM_MASTER_KEY', type=str) - owner = scfg.Value(None, type=str) - ttl = scfg.Value('2h', type=str, help='Soft TTL backstop (default 2h).') - timeout = scfg.Value(600, type=float) - interval = scfg.Value(5, type=float) - queue = scfg.Value( + base_url = kw.Value('http://127.0.0.1:14042/v1', type=str) + api_key_env = kw.Value('LITELLM_MASTER_KEY', type=str) + owner = kw.Value(None, type=str) + ttl = kw.Value('2h', type=str, help='Soft TTL backstop (default 2h).') + timeout = kw.Value(600, type=float) + interval = kw.Value(5, type=float) + queue = kw.Value( False, isflag=True, help='Admission queue: wait (up to --timeout) for a GPU to free ' 'instead of failing fast when the fleet is full. Recommended for ' 'pipeline fan-out, where many jobs contend for a few GPUs.', ) - command = scfg.Value( + command = kw.Value( [], nargs='*', position=1, type=str, help='Command to run (after --).' ) @@ -2028,7 +2028,7 @@ class LeasesCLI(_LeasingCommonMixin): infer-stack leases --json # JSON (adds running + gpus per deployment) """ - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -2186,27 +2186,27 @@ class TestCLI(_PathOverridesMixin): __command__ = 'test' - catalog = scfg.Value( + catalog = kw.Value( None, type=str, help='Catalog path, used only to look up the endpoint protocol.', ) - name = scfg.Value( + name = kw.Value( None, position=1, type=str, help='Endpoint alias to test (e.g. chat).' ) - prompt = scfg.Value( + prompt = kw.Value( 'Reply with the single word: ready.', type=str, help='Prompt to send.' ) - max_tokens = scfg.Value(32, type=int) - timeout = scfg.Value(60, type=float, help='Request timeout (s).') - base_url = scfg.Value( + max_tokens = kw.Value(32, type=int) + timeout = kw.Value(60, type=float, help='Request timeout (s).') + base_url = kw.Value( None, type=str, help='Override the gateway base URL (…/v1).' ) - port = scfg.Value( + port = kw.Value( None, type=int, help='Override the gateway port (default: 14042).' ) - json = scfg.Value(False, isflag=True, help='Emit JSON instead of text.') - protocol = scfg.Value( - None, choices=['chat', 'completions'], + json = kw.Value(False, isflag=True, help='Emit JSON instead of text.') + protocol = kw.Value( + None, type=str, choices=['chat', 'completions'], help="Which surface to hit. Default: the endpoint's declared " '`protocol` from the catalog, falling back to chat.', ) @@ -2348,11 +2348,11 @@ class EnvCLI(_PathOverridesMixin): __command__ = 'env' - arg = scfg.Value( + arg = kw.Value( None, position=1, type=str, help='KEY to read its value, or KEY=VALUE to set it. Empty = path.', ) - export = scfg.Value( + export = kw.Value( False, isflag=True, help='Print every entry as `export KEY=value`.' ) @@ -2478,7 +2478,7 @@ class RoutesListCLI(_LeasingCommonMixin): __command__ = 'list' - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -2559,7 +2559,7 @@ class RoutesPruneCLI(_ApprovalMixin): __command__ = 'prune' - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -2656,12 +2656,12 @@ class RoutesSeedCLI(_ApprovalMixin): __command__ = 'seed' - catalogs = scfg.Value( + catalogs = kw.Value( None, nargs='+', position=1, type=str, help='One or more catalog.yaml files whose endpoints to merge into the ' 'route registry.', ) - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -2735,16 +2735,16 @@ class ConfigPublishCLI(_ApprovalMixin): __command__ = 'publish' - catalogs = scfg.Value( + catalogs = kw.Value( [], nargs='*', position=1, type=str, help='Catalog files to publish as one union (default: --catalog, or the ' 'default-path catalog).', ) - pull = scfg.Value( + pull = kw.Value( True, isflag=True, help='Pre-pull every image the profile references (default; --no-pull skips).', ) - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -2815,8 +2815,8 @@ class NetworkMigrateCLI(_ApprovalMixin): __command__ = 'migrate' - subnet = scfg.Value(None, type=str, help='IPv4 subnet, e.g. 172.30.0.0/24 (required).') - force = scfg.Value(False, isflag=True, help='Migrate even with active leases.') + subnet = kw.Value(None, type=str, help='IPv4 subnet, e.g. 172.30.0.0/24 (required).') + force = kw.Value(False, isflag=True, help='Migrate even with active leases.') @classmethod def main(cls, argv=True, **kwargs): @@ -2851,7 +2851,7 @@ class NetworkCheckCLI(_LeasingCommonMixin): __command__ = 'check' - json = scfg.Value(False, isflag=True) + json = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -2881,7 +2881,7 @@ class SecretsRotateCLI(_ApprovalMixin): __command__ = 'rotate' - force = scfg.Value(False, isflag=True, help='Rotate even with active leases.') + force = kw.Value(False, isflag=True, help='Rotate even with active leases.') @classmethod def main(cls, argv=True, **kwargs): @@ -2921,7 +2921,7 @@ def main(cls, argv=True, **kwargs): return 0 -class SecretsModalCLI(scfg.ModalCLI): +class SecretsModalCLI(kw.ModalCLI): """Manage the gateway's secrets.""" __command__ = 'secrets' @@ -2929,7 +2929,7 @@ class SecretsModalCLI(scfg.ModalCLI): rotate = SecretsRotateCLI -class NetworkModalCLI(scfg.ModalCLI): +class NetworkModalCLI(kw.ModalCLI): """Stable per-service addressing (migrate) and the upstream routing check.""" __command__ = 'network' @@ -2938,7 +2938,7 @@ class NetworkModalCLI(scfg.ModalCLI): check = NetworkCheckCLI -class RoutesModalCLI(scfg.ModalCLI): +class RoutesModalCLI(kw.ModalCLI): """Inspect + manage the LiteLLM route registry (static-superset mode). The registry accumulates every catalog's and every live deployment's routes diff --git a/infer_stack/cli/commands_meta.py b/infer_stack/cli/commands_meta.py index 8d382139..a0986fc3 100644 --- a/infer_stack/cli/commands_meta.py +++ b/infer_stack/cli/commands_meta.py @@ -16,7 +16,7 @@ from pathlib import Path from typing import Any -import scriptconfig as scfg +import kwconf as kw from .. import __version__ from ..paths import config_root, data_root, settings_path @@ -24,7 +24,7 @@ from .options import _PathOverridesMixin -class VersionCLI(scfg.DataConfig): +class VersionCLI(kw.Config): """Print the installed infer-stack version.""" __command__ = 'version' @@ -48,7 +48,7 @@ def _iter_subcommands(modal: Any): if attr.startswith('_'): continue if isinstance(val, type) and issubclass( - val, (scfg.DataConfig, scfg.ModalCLI) + val, (kw.Config, kw.ModalCLI) ): name = getattr(val, '__command__', None) or attr.replace('_', '-') out[name] = val @@ -64,7 +64,7 @@ def _doc_one_line(cls: Any) -> str: def _is_group(sub: Any) -> bool: - return isinstance(sub, type) and issubclass(sub, scfg.ModalCLI) + return isinstance(sub, type) and issubclass(sub, kw.ModalCLI) def _build_tree(modal: Any, node: Any) -> None: @@ -86,7 +86,7 @@ def _build_tree(modal: Any, node: Any) -> None: _build_tree(sub, child) -class HelpTreeCLI(scfg.DataConfig): +class HelpTreeCLI(kw.Config): """Print the full nested command tree with one-line descriptions.""" __command__ = 'tree' @@ -106,7 +106,7 @@ def main(cls, argv=True, **kwargs): return 0 -class HelpModalCLI(scfg.ModalCLI): +class HelpModalCLI(kw.ModalCLI): """Help utilities (use ``infer-stack --help`` for per-command help).""" __command__ = 'help' @@ -207,12 +207,12 @@ class ConfigPathsCLI(_PathOverridesMixin): __command__ = 'paths' - target: Any = scfg.Value( + target: Any = kw.Value( 'all', position=1, help='Path group to show: all, config, data, or leasing.', ) - json: Any = scfg.Value( + json: Any = kw.Value( False, isflag=True, help='Emit the path groups as JSON instead of human-readable text.', @@ -392,17 +392,17 @@ class ConfigInitCLI(_PathOverridesMixin): """ __command__ = 'init' - yes = scfg.Value( + yes = kw.Value( False, isflag=True, alias=['y'], help='Non-interactive: write without prompting/confirming.', ) - fresh = scfg.Value( + fresh = kw.Value( False, isflag=True, help='Start over: ignore any existing config and write a clean one from ' 'defaults (discards other persisted settings too).', ) - backend = scfg.Value( - None, choices=['compose', 'kubeai', 'null'], + backend = kw.Value( + None, type=str, choices=['compose', 'kubeai', 'null'], help='Preset the default backend (skips that prompt).', ) @@ -493,8 +493,8 @@ class ConfigSetCLI(_PathOverridesMixin): """Persist a durable default, e.g. ``config set backend compose``.""" __command__ = 'set' - key = scfg.Value(None, position=1, type=str) - value = scfg.Value(None, position=2, type=str) + key = kw.Value(None, position=1, type=str) + value = kw.Value(None, position=2, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -521,7 +521,7 @@ class ConfigGetCLI(_PathOverridesMixin): """Print one setting's value (or all settings).""" __command__ = 'get' - key = scfg.Value(None, position=1, type=str) + key = kw.Value(None, position=1, type=str) @classmethod def main(cls, argv=True, **kwargs): @@ -588,7 +588,7 @@ def main(cls, argv=True, **kwargs): from .commands_leasing import ConfigPublishCLI # noqa: E402 -class ConfigModalCLI(scfg.ModalCLI): +class ConfigModalCLI(kw.ModalCLI): """Inspect + manage infer-stack configuration (paths + durable settings).""" __command__ = 'config' diff --git a/infer_stack/cli/commands_mock.py b/infer_stack/cli/commands_mock.py index c425fa5b..16732d42 100644 --- a/infer_stack/cli/commands_mock.py +++ b/infer_stack/cli/commands_mock.py @@ -10,18 +10,18 @@ from typing import Any -import scriptconfig as scfg +import kwconf as kw import ubelt as ub -class MockServeCLI(scfg.DataConfig): +class MockServeCLI(kw.Config): """ Serve a deterministic OpenAI-compatible mock inference endpoint. """ __command__ = 'serve' - config_fpath = scfg.Value( + config_fpath = kw.Value( None, position=1, help=ub.paragraph( @@ -35,14 +35,14 @@ class MockServeCLI(scfg.DataConfig): ), ) - host = scfg.Value('127.0.0.1', help='Bind address.') + host = kw.Value('127.0.0.1', help='Bind address.') - port = scfg.Value( + port = kw.Value( 8100, help='Bind port. Use 0 to pick a free port and print it.', ) - mode = scfg.Value( + mode = kw.Value( None, help=ub.paragraph( """ @@ -57,13 +57,13 @@ class MockServeCLI(scfg.DataConfig): ), ) - require_auth = scfg.Value( + require_auth = kw.Value( False, isflag=True, help='Reject requests without a valid bearer token.', ) - api_key = scfg.Value( + api_key = kw.Value( None, help=ub.paragraph( """ @@ -74,11 +74,11 @@ class MockServeCLI(scfg.DataConfig): ), ) - list_modes = scfg.Value( + list_modes = kw.Value( False, isflag=True, help='Print the available response modes and exit.', ) - seed = scfg.Value( + seed = kw.Value( None, help=ub.paragraph( """ @@ -88,7 +88,7 @@ class MockServeCLI(scfg.DataConfig): ), ) - print_url = scfg.Value( + print_url = kw.Value( True, isflag=True, help='Print the bound base URL on startup.', @@ -158,7 +158,7 @@ def main(cls, argv=None, **kwargs): server.stop() -class MockModalCLI(scfg.ModalCLI): +class MockModalCLI(kw.ModalCLI): """ Deterministic mock inference server for tests and dry runs. """ diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 99fa46c6..c37d7336 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -import scriptconfig as scfg +import kwconf as kw from ..log_filter import compact_litellm_tracebacks from ..paths import config_root, data_root, get_setting, settings_path @@ -365,7 +365,7 @@ class StatusCLI(_PathOverridesMixin): (active leases / live deployments), with pointers to dig deeper.""" __command__ = 'status' - catalog = scfg.Value( + catalog = kw.Value( None, type=str, help='Catalog path (default: config dir).' ) @@ -393,7 +393,7 @@ class _ComposeWrapperBase(_PathOverridesMixin): """Common fields for ``docker compose `` wrappers over the leasing Compose deployment.""" - services = scfg.Value( + services = kw.Value( None, nargs='*', position=1, @@ -435,17 +435,17 @@ class LogsCLI(_ComposeWrapperBase): __command__ = 'logs' - follow = scfg.Value( + follow = kw.Value( False, isflag=True, short_alias=['f'], help='Stream logs (docker compose logs -f).', ) - tail = scfg.Value( + tail = kw.Value( None, type=str, help="Tail the last N lines (default: all). Pass a number or 'all'.", ) - timestamps = scfg.Value(False, isflag=True) - no_color = scfg.Value(False, isflag=True) - raw = scfg.Value( + timestamps = kw.Value(False, isflag=True) + no_color = kw.Value(False, isflag=True) + raw = kw.Value( False, isflag=True, help='Show raw followed logs without known LiteLLM traceback compaction.', @@ -487,14 +487,14 @@ class PsCLI(_ComposeWrapperBase): __command__ = 'ps' - all = scfg.Value( + all = kw.Value( False, isflag=True, short_alias=['a'], help='Include stopped containers.' ) - services_only = scfg.Value( + services_only = kw.Value( False, isflag=True, help='Print only service names (passes --services to docker compose).', ) - quiet = scfg.Value( + quiet = kw.Value( False, isflag=True, short_alias=['q'], help='Print only container IDs.' ) @@ -515,7 +515,7 @@ def main(cls, argv=True, **kwargs): class RestartCLI(_ComposeWrapperBase): """``docker compose restart [services...]``.""" - timeout = scfg.Value(None, type=int, help='Stop timeout in seconds.') + timeout = kw.Value(None, type=int, help='Stop timeout in seconds.') @classmethod def main(cls, argv=True, **kwargs): @@ -530,8 +530,8 @@ def main(cls, argv=True, **kwargs): class PullCLI(_ComposeWrapperBase): """``docker compose pull [services...]``.""" - quiet = scfg.Value(False, isflag=True, short_alias=['q']) - ignore_pull_failures = scfg.Value(False, isflag=True) + quiet = kw.Value(False, isflag=True, short_alias=['q']) + ignore_pull_failures = kw.Value(False, isflag=True) @classmethod def main(cls, argv=True, **kwargs): @@ -559,7 +559,7 @@ def main(cls, argv=True, **kwargs): class StopCLI(_ComposeWrapperBase): """``docker compose stop [services...]``.""" - timeout = scfg.Value(None, type=int) + timeout = kw.Value(None, type=int) @classmethod def main(cls, argv=True, **kwargs): @@ -578,7 +578,7 @@ class StackDownCLI(_ComposeWrapperBase): automatically on release; this is the manual escape hatch. """ - volumes = scfg.Value( + volumes = kw.Value( False, isflag=True, help='Also remove named volumes (--volumes).' ) @@ -608,7 +608,7 @@ def main(cls, argv=True, **kwargs): return int(subprocess.run(cmd, env=_docker_env()).returncode) -class StackModalCLI(scfg.ModalCLI): +class StackModalCLI(kw.ModalCLI): """Day-2 ops on the running leasing deployment.""" __command__ = 'stack' @@ -645,13 +645,13 @@ class DoctorCLI(_PathOverridesMixin): __command__ = 'doctor' - backend = scfg.Value( + backend = kw.Value( None, type=str, help='Backend to check (default: the configured `backend` setting).', ) - gpu = scfg.Value(False, isflag=True, + gpu = kw.Value(False, isflag=True, help='Also check host GPUs (implied by --sudo).') - sudo = scfg.Value(False, isflag=True, + sudo = kw.Value(False, isflag=True, help='Run the GPU holder scan as root. Without it that ' 'check reports "not checked", never "clear".') diff --git a/infer_stack/cli/context.py b/infer_stack/cli/context.py index 3e01b14b..8d86ebcf 100644 --- a/infer_stack/cli/context.py +++ b/infer_stack/cli/context.py @@ -18,7 +18,7 @@ def _as_mapping(args: Any) -> dict[str, Any]: """Coerce a CLI args object into a plain dict. - Works for ``None``, ``argparse.Namespace``, and ``scfg.DataConfig`` + Works for ``None``, ``argparse.Namespace``, and ``kw.Config`` instances. Used to side-step name clashes between user-declared fields and ``DataConfig`` builtins. """ diff --git a/infer_stack/cli/options.py b/infer_stack/cli/options.py index ced620ff..f4a1c3ce 100644 --- a/infer_stack/cli/options.py +++ b/infer_stack/cli/options.py @@ -2,17 +2,17 @@ from ..paths import CONFIG_DIR_ENV from ..paths import DATA_DIR_ENV -import scriptconfig as scfg +import kwconf as kw # --------------------------------------------------------------------------- # DataConfig mixins for common override flags # --------------------------------------------------------------------------- -class _PathOverridesMixin(scfg.DataConfig): +class _PathOverridesMixin(kw.Config): """Adds global ``--config-dir`` / ``--data-dir`` to a subcommand.""" - config_dir = scfg.Value( + config_dir = kw.Value( None, type=str, help=( @@ -20,7 +20,7 @@ class _PathOverridesMixin(scfg.DataConfig): f'~/.config/infer_stack (XDG_CONFIG_HOME) or ${CONFIG_DIR_ENV} when set.' ), ) - data_dir = scfg.Value( + data_dir = kw.Value( None, type=str, help=( @@ -30,42 +30,42 @@ class _PathOverridesMixin(scfg.DataConfig): ) -class _BackendOverrideMixin(scfg.DataConfig): - backend = scfg.Value( - None, choices=['compose', 'kubeai'], help='Active backend override.' +class _BackendOverrideMixin(kw.Config): + backend = kw.Value( + None, type=str, choices=['compose', 'kubeai'], help='Active backend override.' ) -class _ComposeOverrideMixin(scfg.DataConfig): - compose_cmd = scfg.Value( +class _ComposeOverrideMixin(kw.Config): + compose_cmd = kw.Value( None, type=str, help="Docker compose command override (e.g. 'podman compose').", ) -class _ProfileOverrideMixin(scfg.DataConfig): - profile = scfg.Value( +class _ProfileOverrideMixin(kw.Config): + profile = kw.Value( None, type=str, help='Active profile override (sets config.active_profile).', ) -class _PortOverridesMixin(scfg.DataConfig): - litellm_port = scfg.Value(None, type=int) - open_webui_port = scfg.Value(None, type=int) - postgres_port = scfg.Value(None, type=int) +class _PortOverridesMixin(kw.Config): + litellm_port = kw.Value(None, type=int) + open_webui_port = kw.Value(None, type=int) + postgres_port = kw.Value(None, type=int) -class _ClusterOverridesMixin(scfg.DataConfig): - namespace = scfg.Value( +class _ClusterOverridesMixin(kw.Config): + namespace = kw.Value( None, type=str, help='Kubernetes namespace for kubeai deployments.' ) - ingress_host = scfg.Value( + ingress_host = kw.Value( None, type=str, help='Ingress host (kubeai only).' ) - ingress_enabled = scfg.Value( + ingress_enabled = kw.Value( None, isflag=True, alias=['ingress'], @@ -73,24 +73,24 @@ class _ClusterOverridesMixin(scfg.DataConfig): ) -class _AllowUnsupportedMixin(scfg.DataConfig): - allow_unsupported = scfg.Value( +class _AllowUnsupportedMixin(kw.Config): + allow_unsupported = kw.Value( False, isflag=True, help='Allow validation errors when planning/rendering.', ) -class _SimulateHardwareMixin(scfg.DataConfig): - simulate_hardware = scfg.Value( +class _SimulateHardwareMixin(kw.Config): + simulate_hardware = kw.Value( None, type=str, help='Simulate GPUs: comma-separated NxM or M entries (e.g. 4x96, 2x80, "48,16" for a heterogeneous host). Useful for planning on smaller machines.', ) -class _AllowedGpusMixin(scfg.DataConfig): - allowed_gpus = scfg.Value( +class _AllowedGpusMixin(kw.Config): + allowed_gpus = kw.Value( None, type=str, help=( @@ -102,8 +102,8 @@ class _AllowedGpusMixin(scfg.DataConfig): ) -class _DisplayGpuMixin(scfg.DataConfig): - skip_display_gpus = scfg.Value( +class _DisplayGpuMixin(kw.Config): + skip_display_gpus = kw.Value( None, isflag=True, alias=['skip-display-gpus'], diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 68fe105e..82ad51e7 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -136,6 +136,7 @@ def activity_label(description: str) -> str | None: """``'acquiring qwen'`` from a worker description like ``_do_acquire('qwen')``. Example: + >>> # xdoctest: +REQUIRES(module:textual) >>> activity_label("_do_acquire('qwen')") 'acquiring qwen' >>> activity_label('_do_apply()') diff --git a/pyproject.toml b/pyproject.toml index a08506f8..1c47d3e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,11 +33,11 @@ dynamic = [ ] dependencies = [ "jinja2>=3.1", + "kwconf>=0.11.0", "loguru>=0.7", "pyyaml>=6.0", "requests>=2.31", "rich>=13.0", - "scriptconfig>=0.9", "ubelt>=1.3", ] optional-dependencies.tests = [ @@ -72,6 +72,10 @@ packages.find.include = [ [tool.uv] exclude-newer = "2026-06-04T04:00:00Z" +# kwconf (the CLI library, successor to scriptconfig) has no release before +# the cutoff. Admit it up to 0.11.0 (2026-08-07), the version tested; the +# cutoff still holds for every other package. +exclude-newer-package = { kwconf = "2026-08-08T00:00:00Z" } [tool.ruff] target-version = "py310" diff --git a/requirements/locks/tests.txt b/requirements/locks/tests.txt index 03dd8137..f78fc55b 100644 --- a/requirements/locks/tests.txt +++ b/requirements/locks/tests.txt @@ -18,6 +18,8 @@ iniconfig==2.3.0 # via pytest jinja2==3.1.6 # via infer-stack +kwconf==0.11.0 + # via infer-stack loguru==0.7.3 # via infer-stack markdown-it-py==4.2.0 @@ -46,15 +48,11 @@ pytest-codeblocks==0.17.0 pytest-cov==7.1.0 # via infer-stack pyyaml==6.0.3 - # via - # infer-stack - # scriptconfig + # via infer-stack requests==2.34.2 # via infer-stack rich==15.0.0 # via infer-stack -scriptconfig==0.9.1 - # via infer-stack tomli==2.4.1 ; python_full_version <= '3.11' # via # coverage @@ -62,9 +60,7 @@ tomli==2.4.1 ; python_full_version <= '3.11' typing-extensions==4.15.0 ; python_full_version < '3.11' # via exceptiongroup ubelt==1.4.2 - # via - # infer-stack - # scriptconfig + # via infer-stack urllib3==2.7.0 # via requests win32-setctime==1.2.0 ; sys_platform == 'win32' diff --git a/tests/test_cli_equivalent.py b/tests/test_cli_equivalent.py index 9d0e2e79..c90aa8c5 100644 --- a/tests/test_cli_equivalent.py +++ b/tests/test_cli_equivalent.py @@ -9,7 +9,7 @@ import shlex import pytest -import scriptconfig as scfg +import kwconf as kw from infer_stack import cli_equivalent as cli from infer_stack.cli import ManageCLI @@ -20,7 +20,7 @@ def parse(text: str): words = shlex.split(text.split(' #', 1)[0]) assert words[0] == 'infer-stack' node, rest = ManageCLI, words[1:] - while isinstance(node, type) and issubclass(node, scfg.ModalCLI): + while isinstance(node, type) and issubclass(node, kw.ModalCLI): node = getattr(node, rest[0].replace('-', '_')) rest = rest[1:] return node, node.cli(argv=rest, strict=True) diff --git a/uv.lock b/uv.lock index 558a181a..c9a7262a 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,9 @@ resolution-markers = [ [options] exclude-newer = "2026-06-04T04:00:00Z" +[options.exclude-newer-package] +kwconf = "2026-08-08T00:00:00Z" + [[package]] name = "certifi" version = "2026.5.20" @@ -257,7 +260,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -278,11 +281,11 @@ name = "infer-stack" source = { editable = "." } dependencies = [ { name = "jinja2" }, + { name = "kwconf" }, { name = "loguru" }, { name = "pyyaml" }, { name = "requests" }, { name = "rich" }, - { name = "scriptconfig" }, { name = "ubelt" }, ] @@ -300,6 +303,7 @@ tui = [ [package.metadata] requires-dist = [ { name = "jinja2", specifier = ">=3.1" }, + { name = "kwconf", specifier = ">=0.11.0" }, { name = "loguru", specifier = ">=0.7" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-codeblocks", marker = "extra == 'tests'", specifier = ">=0.17" }, @@ -307,7 +311,6 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.31" }, { name = "rich", specifier = ">=13.0" }, - { name = "scriptconfig", specifier = ">=0.9" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=0.50" }, { name = "ubelt", specifier = ">=1.3" }, { name = "xdoctest", marker = "extra == 'tests'", specifier = ">=1.1.5" }, @@ -335,6 +338,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "kwconf" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/77/20ce9cf65bc20e769db9a506e72e291f7a38ee766393110858a15e93ea1b/kwconf-0.11.0.tar.gz", hash = "sha256:cd8e9192a3c0e54750368980f9bf6b02676f75b76ac3830b656e977cc55f988f", size = 172077, upload-time = "2026-08-07T15:25:34.385Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/7a/4b4a1c503cc621eb2256e5d18385ccfb546f536008cdb4db0189426f5e77/kwconf-0.11.0-py3-none-any.whl", hash = "sha256:4691afdf7173ca0d42302a2cf17910917ac3bd02729e0b2f2cb2aa1e3b2ee18b", size = 110488, upload-time = "2026-08-07T15:25:32.964Z" }, +] + [[package]] name = "linkify-it-py" version = "2.1.0" @@ -655,19 +667,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] -[[package]] -name = "scriptconfig" -version = "0.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "python_full_version < '4'" }, - { name = "ubelt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a4/4c/6d0ecc5e29b498dc6b5402a0c561e97dd00ce1aeebdc5e494cc5fa8a0b6b/scriptconfig-0.9.1.tar.gz", hash = "sha256:9766c2ee601c4b8d97753ba13a914f8c0a212da4f6d2aec18ad628fd8b53d7ef", size = 112424, upload-time = "2026-03-02T17:33:34.405Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/00/2bf691dfc2b0170013d466fcdbbfc9d036e26d6049d74efaf7775401321d/scriptconfig-0.9.1-py3-none-any.whl", hash = "sha256:4b69e2f4eb681ec4480e9bb5d6c82d2f8c82a9badc703a261ee8940736101d86", size = 87103, upload-time = "2026-03-02T17:33:31.394Z" }, -] - [[package]] name = "textual" version = "8.2.7" From 4def7e7eac74cb4de89a7c343981a856ee25c7db Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 24 Sep 2026 21:19:49 -0400 Subject: [PATCH 018/141] Remove the uv exclude-newer cutoff The 2026-06-04 cutoff predated every kwconf release and needed a per-package exception. Removed at the user's request, with the exception. Relocked without --upgrade: uv.lock loses only the cutoff header, the locked versions and requirements/locks/tests.txt are unchanged. A fresh unpinned install now gets kwconf 0.12.0, and the CI-equivalent suite passes on it (838 passed, 3.14, tests extra only). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 5 ++--- pyproject.toml | 7 ------- uv.lock | 6 ------ 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 571b00d9..908ab1ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,9 @@ strings `null`, `true` and numbers on the command line as values, so take `type=str`. The program name stays `infer-stack` in usage and errors (`__prog__`). -The repository's uv `exclude-newer` cutoff (2026-06-04) predates every kwconf -release, so kwconf alone gets an exception up to 0.11.0, the version tested. `uv.lock` and `requirements/locks/tests.txt` were regenerated: kwconf 0.11.0 -added, scriptconfig removed. +added, scriptconfig removed. The uv `exclude-newer` cutoff (2026-06-04), which +predated every kwconf release, is removed; the locked versions are unchanged. ### The TUI stops repainting what did not change diff --git a/pyproject.toml b/pyproject.toml index 1c47d3e8..906273c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,13 +70,6 @@ packages.find.include = [ "infer_stack*", ] -[tool.uv] -exclude-newer = "2026-06-04T04:00:00Z" -# kwconf (the CLI library, successor to scriptconfig) has no release before -# the cutoff. Admit it up to 0.11.0 (2026-08-07), the version tested; the -# cutoff still holds for every other package. -exclude-newer-package = { kwconf = "2026-08-08T00:00:00Z" } - [tool.ruff] target-version = "py310" line-length = 80 diff --git a/uv.lock b/uv.lock index c9a7262a..0028f947 100644 --- a/uv.lock +++ b/uv.lock @@ -8,12 +8,6 @@ resolution-markers = [ "python_full_version >= '4'", ] -[options] -exclude-newer = "2026-06-04T04:00:00Z" - -[options.exclude-newer-package] -kwconf = "2026-08-08T00:00:00Z" - [[package]] name = "certifi" version = "2026.5.20" From 271d6458bb8eb739f24112d34c68a77a6f0d795d Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Fri, 25 Sep 2026 13:54:52 -0400 Subject: [PATCH 019/141] TUI: one Clean up key in the footer; gc --forget; log CLI equivalents Both Clean up buttons ran the same ledger prune. It is now a single x binding in the footer, backed by a new `infer-stack gc --forget`. The TUI log also names the command for refresh, settings save and the API tab. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 13 ++++++++++++ infer_stack/cli/commands_leasing.py | 15 ++++++++++++++ infer_stack/cli_equivalent.py | 5 +++++ infer_stack/tui.py | 31 +++++++++++++++++++---------- tests/test_cli_leasing.py | 16 +++++++++++++++ tests/test_tui.py | 30 ++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908ab1ea..378a48e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### One Clean up, in the footer, with a CLI command behind it + +The leases and deployments panes each had a Clean up button, and both did the +same thing: forget released/expired leases and stopped deployments. Nothing +running changes. That is now one `x Clean up` in the footer, and the same +action on the CLI is `infer-stack gc --forget`. + +The TUI log now names the CLI command for the actions that were missing one: +`r` (`status`), Clean up (`gc --forget`), saving settings (`config set …`), +and the API tab's send, test-all (`test `) and model list (a `curl` +that reads the key through `infer-stack env` and never prints it). Poll +intervals are TUI-only, and the log says so. + ### The CLI is built on kwconf instead of scriptconfig kwconf is scriptconfig's successor, which aiq-magnet, cmd_queue and kwdagger diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 14bdb67d..507b3ff9 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1248,6 +1248,7 @@ class GcCLI(_ApprovalMixin): blocking ``acquire`` (``--queue``) already does this implicitly while it waits. ``--evict`` additionally tears down idle *keep-warm* deployments (like ``evict --all``). On a terminal the teardown is shown and confirmed (``--yes`` skips). + ``--forget`` only drops finished rows from the ledger (the TUI's Clean up). """ __command__ = 'gc' @@ -1262,6 +1263,11 @@ class GcCLI(_ApprovalMixin): help='Instead: remove containers in the project that infer-stack does not ' 'manage (listed and confirmed first; --yes skips the prompt).', ) + forget = kw.Value( + False, isflag=True, + help='Instead: forget released/expired leases and stopped deployments ' + 'from the ledger. History only; nothing running changes.', + ) json = kw.Value(False, isflag=True) @classmethod @@ -1270,6 +1276,15 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config, interactive=True) + if config.forget: + n_leases, n_deployments = controller.prune() + if config.json: + print(json.dumps({'leases': n_leases, + 'deployments': n_deployments}, indent=2)) + else: + print(f'gc --forget: forgot {n_leases} released/expired lease(s), ' + f'{n_deployments} stopped deployment(s)') + return 0 if config.orphans: if not isinstance(controller.backend, ComposeBackend): raise SystemExit('gc --orphans needs the compose backend') diff --git a/infer_stack/cli_equivalent.py b/infer_stack/cli_equivalent.py index 5d271d5e..a4b93448 100644 --- a/infer_stack/cli_equivalent.py +++ b/infer_stack/cli_equivalent.py @@ -26,6 +26,11 @@ def _kv_value(value: Any) -> str: return json.dumps(value) +#: What the gateway routes; the key comes from the CLI, never the log. +MODELS_CURL = ('curl -s "$(infer-stack env OPENAI_BASE_URL)/models" ' + '-H "Authorization: Bearer $(infer-stack env LITELLM_MASTER_KEY)"') + + #: Endpoint fields `catalog endpoint add` can express. _ENDPOINT_FIELDS = {'engine', 'model', 'host', 'public_name', 'reclaim', 'protocol', 'placement', 'runtime'} diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 82ad51e7..96add27e 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -970,9 +970,11 @@ class InferStackTUI(App): BINDINGS = [ # Truly global controls stay in the footer. - ('r', 'refresh', 'Refresh'), + ('r', 'refresh_now', 'Refresh'), # Global on purpose: an error can happen while any tab is in front. ('l', 'show_app_log', 'TUI log'), + # Forgets finished history in both tables, so it belongs to neither pane. + ('x', 'cleanup', 'Clean up'), ('tab', 'focus_next', 'Next pane'), ('q', 'quit', 'Quit'), # Pane-scoped actions: keys still work, but they live as buttons under @@ -981,7 +983,6 @@ class InferStackTUI(App): Binding('d', 'release', 'Release', show=False), Binding('e', 'evict', 'Evict', show=False), Binding('a', 'release_all', 'Release all', show=False), - Binding('x', 'cleanup', 'Clean up', show=False), # Multi-select: space toggles the cursor row in the focused leases/ # deployments table; release/evict then act on every checked row. Binding('space', 'toggle_select', 'Select row', show=False), @@ -1188,7 +1189,6 @@ def _compose_dashboard(self) -> ComposeResult: with Horizontal(id='lease-actions'): yield Button('Release', id='btn-release') yield Button('Release all', id='btn-release-all') - yield Button('Clean up', id='btn-cleanup') yield _Divider('y', self._drag_tables, id='tsplit') with Vertical(id='deployments-pane'): yield Static( @@ -1196,15 +1196,15 @@ def _compose_dashboard(self) -> ComposeResult: "The 'leases' column is how many leases hold each. " 'Evict an idle one to free its GPU (cursor row, or ' 'rows checked with space / ctrl/shift-click); Evict ' - 'all idle clears every kept-warm one; Clean up forgets ' - 'stopped ones.', classes='desc', + 'all idle clears every kept-warm one. x (Clean up) ' + 'forgets stopped deployments and finished leases.', + classes='desc', ) yield DataTable(id='deployments', cursor_type='row', zebra_stripes=True) with Horizontal(id='deployment-actions'): yield Button('Evict', id='btn-evict') yield Button('Evict all idle', id='btn-evict-all') - yield Button('Clean up', id='btn-cleanup-deployments') yield _Divider('y', self._drag_logs, id='hsplit') with Collapsible(title='docker', collapsed=True, id='docker'): with TabbedContent(id='docker-tabs'): @@ -1845,6 +1845,12 @@ def _refresh_bg(self) -> None: data = self._collect() self.call_from_thread(self._render, data) + def action_refresh_now(self) -> None: + """The `r` key: a refresh the user asked for (the timer calls + ``action_refresh`` directly, and must not fill the TUI log).""" + self._cli(cli.command('status')) + self.action_refresh() + def action_refresh(self) -> None: self._sync_pane_state() # capture pane state on the UI thread first self._refresh_bg() @@ -2706,8 +2712,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: 'btn-release-all': self.action_release_all, 'btn-evict': self.action_evict, 'btn-evict-all': self.action_evict_all, - 'btn-cleanup': self.action_cleanup, - 'btn-cleanup-deployments': self.action_cleanup, 'btn-suggest': self.action_suggest, 'btn-add-model': self.action_add_model, 'btn-add-endpoint': self.action_add_endpoint, @@ -2748,6 +2752,7 @@ def _on_apply_ui_settings(self) -> None: return self.ledger_interval = ledger self.observe_interval = max(observe, ledger) # observe never beats ledger + self.app_log('CLI: none; poll intervals are a TUI-only preference') self._apply_poll_settings() prefs = load_tui_settings() prefs['ledger_interval'] = self.ledger_interval @@ -2777,6 +2782,10 @@ def _on_save_settings(self) -> None: ) path = save_settings(s) self._status(f'saved settings → {path}') + self._cli(*(cli.command('config', 'set', key, str(s[key]).lower() + if isinstance(s[key], bool) else s[key]) + for key in ('backend', 'data_dir', 'ui', 'reverse_proxy', + 'skip_display_gpus') if key in s)) except Exception as ex: # noqa: BLE001 self._status(f'save settings failed: {ex}') @@ -2814,8 +2823,7 @@ def action_evict_all(self) -> None: def action_cleanup(self) -> None: self._status('cleaning up released/expired leases + stopped deployments…') - self.app_log('CLI: none yet; this only forgets finished rows in ' - 'the ledger (nothing running changes)') + self._cli(cli.command('gc', '--forget')) self._do_cleanup() # -- docker compose control ------------------------------------------- @@ -3369,6 +3377,7 @@ def action_api_send(self) -> None: prompt = (self.query_one('#api-prompt', Input).value.strip() or 'Say hello in one short sentence.') self._api_log(f'> [{model}] {prompt}') + self._cli(cli.command('test', model, '--prompt', prompt)) self._do_api_send(model, prompt) def action_api_test_all(self) -> None: @@ -3377,10 +3386,12 @@ def action_api_test_all(self) -> None: self._refuse('no ready models to test (acquire one first)') return self._api_log(f'— testing {len(models)} ready model(s) —') + self._cli(*(cli.command('test', model) for model in models)) self._do_api_test_all(models) def action_api_list_models(self) -> None: self._api_log('> GET /v1/models (what the gateway routes)') + self._cli(cli.MODELS_CURL) self._do_api_list() def action_api_copy_curl(self) -> None: diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 9f612adc..d007f7c9 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -225,6 +225,22 @@ def test_release_evict_tears_down_immediately(env, capsys): assert _leases_json(env, capsys)['deployments'][0]['state'] == 'stopped' +def test_gc_forget_drops_only_finished_rows(env, capsys): + """`gc --forget` is the TUI's Clean up: history goes, live rows stay.""" + from infer_stack.cli.commands_leasing import GcCLI + + AcquireCLI.main(argv=['qwen-coder', *_base(env), '--owner', 'a']) + ReleaseCLI.main(argv=['--ledger', env.db, '--all', '--evict']) + AcquireCLI.main(argv=['reranker', *_base(env), '--owner', 'b']) + capsys.readouterr() + assert GcCLI.main(argv=['--ledger', env.db, '--forget', '--json']) == 0 + assert json.loads(capsys.readouterr().out) == {'leases': 1, 'deployments': 1} + data = _leases_json(env, capsys) + assert [le['owner'] for le in data['leases']] == ['b'] + assert [g['state'] for g in data['deployments']] != ['stopped'] + assert len(data['deployments']) == 1 + + def test_acquire_without_ttl_is_standing_lease(env, capsys): # No --ttl -> an infinite (standing-service) lease owned by the caller. AcquireCLI.main(argv=['qwen-coder', *_base(env)]) diff --git a/tests/test_tui.py b/tests/test_tui.py index 66a111cd..aedd48c1 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -1231,6 +1231,36 @@ async def scenario(): assert not any(str(le.state) == 'released' for le in leases) +def test_clean_up_is_one_footer_key_that_logs_its_cli_command(): + """One Clean up, on `x` in the footer, and each action names its command.""" + from textual.widgets import Button + + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + seen = {} + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test() as pilot: + await pilot.pause() + seen['buttons'] = [b.id for b in app.query(Button) + if 'cleanup' in (b.id or '')] + seen['footer'] = [b for b in app.BINDINGS if isinstance(b, tuple)] + await pilot.press('x') + await app.workers.wait_for_complete() + await pilot.press('r') + await pilot.pause() + seen['applog'] = '\n'.join(app._app_log_lines) + + _run(scenario) + assert seen['buttons'] == [] + assert ('x', 'cleanup', 'Clean up') in seen['footer'] # tuples show in the footer + assert 'CLI: infer-stack gc --forget' in seen['applog'] + assert 'CLI: infer-stack status' in seen['applog'] + + def test_tui_evict_all_idle_button(): from infer_stack.leasing import DeploymentState from infer_stack.tui import InferStackTUI From d672bfa05c46931ab4f696a71b696dd55f129932 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Fri, 25 Sep 2026 14:05:03 -0400 Subject: [PATCH 020/141] TUI: shorten the four pane descriptions Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/tui.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 96add27e..a07a2440 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1149,8 +1149,7 @@ def _compose_dashboard(self) -> ComposeResult: with Horizontal(id='body'): with Vertical(id='sidebar'): yield Static( - 'Endpoints — runnable model + engine configs. Acquire one to ' - 'serve it, or Suggest a set sized to your GPUs.', classes='desc', + 'Endpoints: acquirable configurations', classes='desc', ) yield Static('', id='catalog-help') yield _EndpointTable(id='endpoints', cursor_type='row', @@ -1165,8 +1164,7 @@ def _compose_dashboard(self) -> ComposeResult: yield Button('✨ Suggest from my GPUs', id='btn-suggest') yield _Divider('y', self._drag_models, id='csplit') yield Static( - 'Models — weights an endpoint can serve. Add models here, ' - 'then point an endpoint at one.', classes='desc', + 'Models: servable weights', classes='desc', ) yield DataTable(id='models', cursor_type='row', zebra_stripes=True) @@ -1178,10 +1176,8 @@ def _compose_dashboard(self) -> ComposeResult: with Vertical(id='tables'): with Vertical(id='leases-pane'): yield Static( - 'Reservations you hold. Each maps to one deployment ' - 'below (see the deployment column); many leases can ' - 'share one. Release acts on the cursor row, or on ' - 'every row you check (space, or ctrl/shift-click).', + 'Reservations that map to a deployment. ' + '(space or ctrl/shift-click to multiselect).', classes='desc', ) yield DataTable(id='leases', cursor_type='row', @@ -1192,13 +1188,7 @@ def _compose_dashboard(self) -> ComposeResult: yield _Divider('y', self._drag_tables, id='tsplit') with Vertical(id='deployments-pane'): yield Static( - 'Running model deployments and the GPUs they hold. ' - "The 'leases' column is how many leases hold each. " - 'Evict an idle one to free its GPU (cursor row, or ' - 'rows checked with space / ctrl/shift-click); Evict ' - 'all idle clears every kept-warm one. x (Clean up) ' - 'forgets stopped deployments and finished leases.', - classes='desc', + 'Models running.', classes='desc', ) yield DataTable(id='deployments', cursor_type='row', zebra_stripes=True) From 3fd3a3785162309e3b17b28b7f739f5cf7577818 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Fri, 25 Sep 2026 14:10:38 -0400 Subject: [PATCH 021/141] TUI: show endpoints|models and leases|deployments as tabs Each pair is laid out by one helper that builds either tabs or the old draggable split from the same pane functions, so either layout is one class switch. Tab labels carry the lease and deployment counts. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 9 +++ infer_stack/tui.py | 137 ++++++++++++++++++++++++++++++--------------- tests/test_tui.py | 62 ++++++++++++++------ 3 files changed, 145 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 378a48e8..bd01f31a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### Endpoints|models and leases|deployments are tabs + +The sidebar shows Endpoints and Models as two tabs instead of two panes split +by a divider, so the endpoint list gets the whole height; the main area does +the same for Leases and Deployments. A tab's label carries its counts +(`Leases 1/3`), so the hidden one is still readable. Each pair is one switch +(`InferStackTUI.TABBED_CATALOG`, `TABBED_TABLES`); setting it to `False` +restores the draggable split, and both layouts are tested. + ### One Clean up, in the footer, with a CLI command behind it The leases and deployments panes each had a Clean up button, and both did the diff --git a/infer_stack/tui.py b/infer_stack/tui.py index a07a2440..17272da9 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -940,6 +940,10 @@ class InferStackTUI(App): #leases-pane { height: 14; min-height: 6; } /* height set via _apply_sizes */ #deployments-pane { height: 1fr; min-height: 6; } #leases, #deployments { height: 1fr; min-height: 3; } + /* tabbed pairs (_pair): the visible pane takes all the room */ + .pair, .pair ContentSwitcher, .pair TabPane { height: 1fr; } + .pair TabPane { padding: 0; } + .pair #models, .pair #leases-pane { height: 1fr; } #docker-tabs { height: 16; min-height: 8; } #logsvc { margin: 0 0 1 0; } #logs, #ps { height: 1fr; background: $surface; } @@ -968,6 +972,11 @@ class InferStackTUI(App): #compose-actions Button { margin: 0 1 0 0; } """ + #: Endpoints|models and leases|deployments as tabs (True), or as two panes + #: split by a draggable divider (False). Each is one switch to flip back. + TABBED_CATALOG = True + TABBED_TABLES = True + BINDINGS = [ # Truly global controls stay in the footer. ('r', 'refresh_now', 'Refresh'), @@ -1145,56 +1154,78 @@ def _compose_app_log(self) -> ComposeResult: yield RichLog(id='applog', highlight=False, markup=True, max_lines=4000, wrap=True) + def _pair(self, tabbed: bool, tabs_id: str, split_id: str, drag, + first: tuple[str, Any], second: tuple[str, Any]) -> ComposeResult: + """Two panes, as tabs or stacked around a draggable divider. + + ``first``/``second`` are ``(tab label, compose function)``. Only the + container differs between the layouts: the panes, and every id the + rest of the app queries, are the same either way. + """ + if tabbed: + with TabbedContent(id=tabs_id, classes='pair'): + for label, body in (first, second): + with TabPane(label, id=f'pane-{label.lower()}'): + yield from body() + else: + yield from first[1]() + yield _Divider('y', drag, id=split_id) + yield from second[1]() + + def _compose_endpoints(self) -> ComposeResult: + yield Static('Endpoints: acquirable configurations', classes='desc') + yield Static('', id='catalog-help') + yield _EndpointTable(id='endpoints', cursor_type='row', + zebra_stripes=True) + with Horizontal(id='endpoint-actions'): + yield Button('Acquire', id='btn-acquire', variant='primary', + action='app.acquire') + yield Button('Add', id='btn-add-endpoint') + yield Button('Edit', id='btn-edit-endpoint') + yield Button('Remove', id='btn-remove-endpoint') + with Horizontal(id='suggest-actions'): + yield Button('✨ Suggest from my GPUs', id='btn-suggest') + + def _compose_models(self) -> ComposeResult: + yield Static('Models: servable weights', classes='desc') + yield DataTable(id='models', cursor_type='row', zebra_stripes=True) + with Horizontal(id='model-actions'): + yield Button('Add', id='btn-add-model') + yield Button('Remove', id='btn-remove-model') + + def _compose_leases(self) -> ComposeResult: + with Vertical(id='leases-pane'): + yield Static('Reservations that map to a deployment. ' + '(space or ctrl/shift-click to multiselect).', + classes='desc') + yield DataTable(id='leases', cursor_type='row', zebra_stripes=True) + with Horizontal(id='lease-actions'): + yield Button('Release', id='btn-release') + yield Button('Release all', id='btn-release-all') + + def _compose_deployments(self) -> ComposeResult: + with Vertical(id='deployments-pane'): + yield Static('Models running.', classes='desc') + yield DataTable(id='deployments', cursor_type='row', + zebra_stripes=True) + with Horizontal(id='deployment-actions'): + yield Button('Evict', id='btn-evict') + yield Button('Evict all idle', id='btn-evict-all') + def _compose_dashboard(self) -> ComposeResult: with Horizontal(id='body'): with Vertical(id='sidebar'): - yield Static( - 'Endpoints: acquirable configurations', classes='desc', - ) - yield Static('', id='catalog-help') - yield _EndpointTable(id='endpoints', cursor_type='row', - zebra_stripes=True) - with Horizontal(id='endpoint-actions'): - yield Button('Acquire', id='btn-acquire', variant='primary', - action='app.acquire') - yield Button('Add', id='btn-add-endpoint') - yield Button('Edit', id='btn-edit-endpoint') - yield Button('Remove', id='btn-remove-endpoint') - with Horizontal(id='suggest-actions'): - yield Button('✨ Suggest from my GPUs', id='btn-suggest') - yield _Divider('y', self._drag_models, id='csplit') - yield Static( - 'Models: servable weights', classes='desc', - ) - yield DataTable(id='models', cursor_type='row', - zebra_stripes=True) - with Horizontal(id='model-actions'): - yield Button('Add', id='btn-add-model') - yield Button('Remove', id='btn-remove-model') + yield from self._pair( + self.TABBED_CATALOG, 'catalog-tabs', 'csplit', self._drag_models, + ('Endpoints', self._compose_endpoints), + ('Models', self._compose_models)) yield _Divider('x', self._drag_sidebar, id='vsplit') with Vertical(id='main'): with Vertical(id='tables'): - with Vertical(id='leases-pane'): - yield Static( - 'Reservations that map to a deployment. ' - '(space or ctrl/shift-click to multiselect).', - classes='desc', - ) - yield DataTable(id='leases', cursor_type='row', - zebra_stripes=True) - with Horizontal(id='lease-actions'): - yield Button('Release', id='btn-release') - yield Button('Release all', id='btn-release-all') - yield _Divider('y', self._drag_tables, id='tsplit') - with Vertical(id='deployments-pane'): - yield Static( - 'Models running.', classes='desc', - ) - yield DataTable(id='deployments', cursor_type='row', - zebra_stripes=True) - with Horizontal(id='deployment-actions'): - yield Button('Evict', id='btn-evict') - yield Button('Evict all idle', id='btn-evict-all') + yield from self._pair( + self.TABBED_TABLES, 'table-tabs', 'tsplit', self._drag_tables, + ('Leases', self._compose_leases), + ('Deployments', self._compose_deployments)) yield _Divider('y', self._drag_logs, id='hsplit') with Collapsible(title='docker', collapsed=True, id='docker'): with TabbedContent(id='docker-tabs'): @@ -1537,8 +1568,11 @@ def _draw_activity(self) -> None: def _apply_sizes(self) -> None: self.query_one('#sidebar').styles.width = self._sidebar_w self.query_one('#docker-tabs').styles.height = self._log_h - self.query_one('#models').styles.height = self._models_h - self.query_one('#leases-pane').styles.height = self._leases_h + # Fixed heights only matter beside a divider; a tab fills its pane. + if not self.TABBED_CATALOG: + self.query_one('#models').styles.height = self._models_h + if not self.TABBED_TABLES: + self.query_one('#leases-pane').styles.height = self._leases_h def _drag_sidebar(self, delta: int) -> None: # Allow the full width range (down to a sliver, up to nearly all of it), @@ -1870,6 +1904,17 @@ def _update_summary(self, leases, deployments, observed) -> None: widget = self.query_one(pane) if widget.border_title != text: widget.border_title = text + if self.TABBED_TABLES: + # A hidden tab's counts show on its label. + tabs = self.query_one('#table-tabs', TabbedContent) + for pane_id, text in ( + ('pane-leases', f'Leases {active}/{len(leases)}'), + ('pane-deployments', f'Deployments {"…" if observing else running}' + f'/{len(deployments)}'), + ): + tab = tabs.get_tab(pane_id) + if str(tab.label) != text: + tab.label = text except Exception: # noqa: BLE001 pass diff --git a/tests/test_tui.py b/tests/test_tui.py index aedd48c1..e1557e87 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -373,35 +373,65 @@ async def scenario(): _run(scenario) -def test_tui_leases_deployments_are_separate_panes(): +def _split_layout(): + """The TUI with both pane pairs stacked around dividers instead of tabbed.""" + from infer_stack.tui import InferStackTUI + + class Split(InferStackTUI): + TABBED_CATALOG = False + TABBED_TABLES = False + return Split + + +@pytest.mark.parametrize('tabbed', [True, False]) +def test_tui_pane_pairs_are_tabs_or_split(tabbed): + """Either layout has the same panes and ids; only the container differs.""" from textual.containers import Vertical + from textual.css.query import NoMatches + from textual.widgets import TabbedContent from infer_stack.tui import InferStackTUI controller, catalog = _ctx() + out = controller.acquire('bob', catalog.resolve_names(['qwen-coder'])) + assert out.lease + cls = InferStackTUI if tabbed else _split_layout() async def scenario(): - app = InferStackTUI(controller, catalog, interval=999, - proc_factory=lambda svc: None) - async with app.run_test() as pilot: + app = cls(controller, catalog, interval=999, proc_factory=lambda svc: None) + async with app.run_test(size=(140, 45)) as pilot: await pilot.pause() - # leases/deployments are their own panes split by a drag handle, - # not collapsibles. - assert isinstance(app.query_one('#leases-pane'), Vertical) - assert isinstance(app.query_one('#deployments-pane'), Vertical) - assert app.query_one('#tsplit') + app.action_refresh() + await app.workers.wait_for_complete() + await pilot.pause() + for pane in ('#leases-pane', '#deployments-pane'): + assert isinstance(app.query_one(pane), Vertical) + for table in ('#endpoints', '#models', '#leases', '#deployments'): + assert app.query_one(table) + dividers = [] + for divider in ('#csplit', '#tsplit'): + try: + dividers.append(app.query_one(divider)) + except NoMatches: + pass + if tabbed: + assert not dividers + tabs = app.query_one('#table-tabs', TabbedContent) + assert str(tabs.get_tab('pane-leases').label) == 'Leases 1/1' + # the pane in front gets the whole height, not a fixed 14 rows + assert app.query_one('#leases-pane').size.height > 14 + else: + assert len(dividers) == 2 _run(scenario) def test_tui_panes_drag_resize(): - from infer_stack.tui import InferStackTUI - controller, catalog = _ctx() async def scenario(): - app = InferStackTUI(controller, catalog, interval=999, - proc_factory=lambda svc: None) + app = _split_layout()(controller, catalog, interval=999, + proc_factory=lambda svc: None) async with app.run_test(size=(120, 40)) as pilot: await pilot.pause() w0, h0 = app._sidebar_w, app._log_h @@ -420,13 +450,11 @@ async def scenario(): def test_tui_dividers_have_a_grab_area(): - from infer_stack.tui import InferStackTUI - controller, catalog = _ctx() async def scenario(): - app = InferStackTUI(controller, catalog, interval=999, - proc_factory=lambda svc: None) + app = _split_layout()(controller, catalog, interval=999, + proc_factory=lambda svc: None) async with app.run_test(size=(120, 40)) as pilot: await pilot.pause() # A 0-size divider can't be grabbed; both must span their cross-axis. From fe88f740dd243fcdd3cfb697429d2dd771b6a0fc Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Fri, 25 Sep 2026 21:05:39 -0400 Subject: [PATCH 022/141] Docs: Compose/KubeAI parity matrix and a roadmap to close the gaps docs/backend-parity.md states the intended relationship (KubeAI is Compose plus a scheduler: more information supplied, more tools installed, the same catalog, verbs, env file and TUI) and records row by row where that holds, where it does not, and which differences are deliberate. docs/planning/backend-parity-roadmap.md plans the rest in phases with exit criteria: one acquire path (K3), day-2 verbs and the TUI through the backend seam, gateway feature parity, placement information, and the gateway inside the cluster for a multi-workstation run. The README's KubeAI section described the pre-leasing setup/deploy workflow and a live kubectl patch that is no longer needed; it now points at the current docs and keeps the cluster prerequisites and debugging checks. The readiness-diagnosis subsection used commands that no longer exist. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 13 + README.md | 425 ++++-------------- .../plan-backend-unification-2026-09-24.md | 5 + docs/backend-parity.md | 148 ++++++ docs/kubeai-backend.md | 4 + docs/planning/backend-parity-roadmap.md | 178 ++++++++ 6 files changed, 429 insertions(+), 344 deletions(-) create mode 100644 docs/backend-parity.md create mode 100644 docs/planning/backend-parity-roadmap.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bd01f31a..819244cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### Compose and KubeAI: a parity matrix and a roadmap + +`docs/backend-parity.md` states the relationship the two backends are meant +to have (KubeAI is Compose plus a scheduler: more information supplied, more +tools installed, the same catalog, verbs, env file and TUI) and records, row +by row, where that holds today, where it does not, and which differences are +deliberate. `docs/planning/backend-parity-roadmap.md` plans the rest in five +phases with exit criteria, from one acquire path to the gateway inside the +cluster for a multi-workstation run. The README's KubeAI section, which still +described the pre-leasing `setup` / `deploy` workflow and a live patch that +is no longer needed, now points at the current docs and keeps only the +cluster prerequisites and the `kubectl` debugging checks. + ### Endpoints|models and leases|deployments are tabs The sidebar shows Endpoints and Models as two tabs instead of two panes split diff --git a/README.md b/README.md index fc51de66..6c76e9aa 100644 --- a/README.md +++ b/README.md @@ -702,27 +702,33 @@ chat settings. ## Backend 2: KubeAI -Use KubeAI when you want Kubernetes-managed serving. +`--backend kubeai` runs the same leasing verbs against a Kubernetes cluster +running [KubeAI](https://www.kubeai.org): the same catalog, ledger, TTLs, +env file and TUI, with `Model` custom resources in place of compose +services and the cluster scheduler in place of the local GPU planner. The +LiteLLM gateway still fronts everything, so a card sees one `OPENAI_BASE_URL`, +the managed key and the endpoint alias on either backend. -### Important rules +* Setup, settings and semantics: [docs/kubeai-backend.md](docs/kubeai-backend.md). +* What matches Compose, what does not yet, and what is deliberate: + [docs/backend-parity.md](docs/backend-parity.md); the plan to close the + rest: [docs/planning/backend-parity-roadmap.md](docs/planning/backend-parity-roadmap.md). -1. **Use the same namespace everywhere.** The namespace in `infer-stack setup --namespace ...` must match the namespace where the KubeAI Helm release already exists. -2. **Prefer the repo-driven path.** The normal path is `setup` -> `validate` -> `render` -> `deploy` -> `status`. -3. **`kubectl port-forward` stays in the foreground.** Leave it running in one terminal and send requests from another. -4. **The first request can take a while.** `/openai/v1/models` may work before chat completions work. The first completion may trigger pod creation, image pull, model load, and compile warmup. -5. **On the current repo version, KubeAI still needs a live workaround after deploy.** The renderer currently produces a `Model` spec that needs a small manual patch to work with the KubeAI version used in these notes. +The short version: -### KubeAI prerequisites - -You need: - -* a working Kubernetes cluster -* `kubectl` -* Helm +```bash +./scripts/bootstrap_k3s.sh # a one-host cluster (k3s + helm) +./scripts/install_kubeai.sh kubeai-values.yaml # the chart, with your resourceProfiles +kubectl -n kubeai port-forward svc/kubeai 8000:80 & +infer-stack config set backend kubeai +infer-stack doctor # cluster -> CRD -> namespace -> gateway +infer-stack acquire --ttl 2h --env-file lease.env --yes +``` -If you want a quick local single-node cluster, K3s is a good option. +### KubeAI prerequisites -Install K3s: +You need a Kubernetes cluster, `kubectl` and Helm. `scripts/bootstrap_k3s.sh` +installs k3s and helm on one host; the steps it runs, by hand: ```bash curl -sfL https://get.k3s.io | sh - @@ -773,332 +779,78 @@ kubectl get node "$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')" kubectl get nodes --show-labels | tr ',' '\n' | grep 'nvidia.com/' || true ``` -You want to see a non-empty `nvidia.com/gpu` count and `nvidia.com/*` labels such as product and memory. - -### KubeAI Helm repository +You want a non-empty `nvidia.com/gpu` count and `nvidia.com/*` labels such as product and memory. -```bash -helm repo add kubeai https://www.kubeai.org -helm repo update -``` +### Resource profiles -## Determine which namespace to use - -Before doing anything else, discover whether a `kubeai` release already exists and which namespace it uses. - -```bash -KUBEAI_NAMESPACE="$(helm list -A | awk '$1=="kubeai"{print $2; exit}')" -if [ -z "${KUBEAI_NAMESPACE}" ]; then - KUBEAI_NAMESPACE=default -fi -echo "Using KubeAI namespace: ${KUBEAI_NAMESPACE}" -``` - -If a release already exists, **reuse that namespace**. - -Sanity-check the cluster: - -```bash -kubectl get nodes -kubectl get crd models.kubeai.org || true -helm list -A | grep kubeai || true -kubectl -n "${KUBEAI_NAMESPACE}" get pods || true -kubectl get node "$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')" \ - -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}' -``` - ---- - -## Generate the local KubeAI resource-profile file - -Generate a local KubeAI resource-profile file from the labels on this machine. - -For the built-in serving profiles in this repo, keep these names aligned: - -* `gpu-single-default` -* `gpu-tp2-balanced` -* `gpu-tp2-maxctx` - -**Important:** include GPU `requests`, GPU `limits`, and `runtimeClassName: nvidia`. Without those, the model pod can land on the GPU node but still start without `libcuda.so.1` available inside the container. +A `resourceProfile` is what one "GPU unit" means on your cluster; the +catalog's `runtime.resource_profile` (or the `kubeai_resource_profile` +setting) names one, and infer-stack appends the GPU count. Include GPU +`requests`, GPU `limits` and `runtimeClassName: nvidia`: without them the +pod can land on the GPU node and still start without `libcuda.so.1`. ```bash PRODUCT="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.product}')" -MEMORY="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.memory}')" -cat > values-kubeai-local-gpu.yaml < kubeai-values.yaml <:` - * append `:1` in the live `Model` spec -* `libcuda.so.1: cannot open shared object file` - * the pod landed on the GPU node without actually requesting a GPU; fix the resource-profile file to include GPU requests, limits, and `runtimeClassName: nvidia` -* `/models` works but completions 404 with `The model ... does not exist.` - * the served model name does not match the public profile name; apply the live args patch above -* startup probe fails with `connection refused` - * the model pod may still be pulling the image, loading the model, or warming up +* `libcuda.so.1: cannot open shared object file`: the pod did not request a + GPU; fix the resource profile (requests, limits, `runtimeClassName`). +* startup probe fails with `connection refused`: still pulling the image, + loading the model or warming up; the acquire's wait reports which. +* `/models` works but completions 404: the request bypassed the gateway and + used the alias; through the gateway the alias is the model name, directly + against KubeAI the Model name is (`INFER_STACK_ENDPOINT_*` in the env file). --- ## Which backend should I start with? -Start with **Compose** if you want: - -* the fastest path to a working local server -* easy inspection of generated files -* simple single-host iteration - -Move to **KubeAI** when you want: - -* vLLM runtimes on Kubernetes -* KubeAI’s OpenAI-compatible front door -* profile deployment through Kubernetes artifacts - -KubeAI rendering is vLLM-only for now. Profiles that enable Ollama, LiteLLM, or Open WebUI are rejected for `--backend kubeai`. - -A good workflow is: - -1. inspect a profile with `describe-profile` -2. run it with Compose when you want the simplest local deployment -3. move to KubeAI when you want Kubernetes-backed serving - -Compose is the better fit when you already know which profile you want. KubeAI has more first-request overhead because it may need to create pods, pull images, load the model, and warm up the backend. +**Compose** when one workstation is enough: it is the fastest path to a +working server, everything it renders is a file you can read, and it needs +only Docker. +**KubeAI** when the models must run on more than one machine, or a cluster +already exists. It is Compose plus a scheduler: the same catalog, verbs and +env file, with a cluster and `resourceProfiles` supplied. Expect more +first-request overhead (pod creation, image pull, model load). +A catalog written for Compose runs on KubeAI, with two exceptions: ollama +endpoints and custom container launches (`runtime.command` / `mounts`), +which stay Compose-only. [docs/backend-parity.md](docs/backend-parity.md) +has the full matrix. ## vLLM startup caches @@ -1107,39 +859,24 @@ Triton, and CUDA JIT caches. Warm starts avoid redownloading and redoing many compile/JIT steps, but a vLLM model swap still creates a new engine process and must reload weights into GPU memory. -### Diagnosing profile switches and readiness - -`docker compose` health only means that a container-level healthcheck passed. It -is not the same thing as "the routed model can answer a request through the -active access surface." This matters most when switching between two vLLM -profiles that reuse the same runtime service name: the old vLLM process exits, -Docker starts the replacement process, and LiteLLM may remain up while returning -upstream connection errors until vLLM finishes loading the new model. - -Use the dedicated readiness and diagnostics commands after a switch: - -```bash -infer-stack switch gpt2-single --apply --yes -infer-stack wait-ready --model gpt2 -infer-stack smoke-test --model gpt2 -``` +### Diagnosing readiness -For debugging, use: +`docker compose` health only means a container-level healthcheck passed. It +is not "the routed model answers a request through the front door", which is +what `acquire` and `wait` check: a model swap starts a new engine process, +and LiteLLM stays up while returning upstream connection errors until vLLM +has loaded the weights. ```bash -infer-stack diagnose --model gpt2 --generation -infer-stack diagnose --logs --tail 80 +infer-stack acquire --yes # waits for a real generation +infer-stack wait # after acquire --no-wait +infer-stack test # one generation through the gateway +infer-stack status # desired vs running, per deployment +infer-stack logs --tail 80 # the engine or gateway log ``` -`diagnose` prints the resolved provider/gateway/frontend graph, rendered Compose -service state, LiteLLM route probes, direct provider probes, and optional recent -logs. It is intended to distinguish an actual LiteLLM outage from the more -common case where LiteLLM is running but its upstream vLLM runtime is still -booting. - -The Compose service-state diagnostics include Docker's exit code, OOM-killed -flag, restart count, and actual container name. This is important because -`litellm exited with code 137` usually means Docker sent SIGKILL, commonly from -an OOM kill or a forced container replacement, whereas LiteLLM returning HTTP -500 with `Cannot connect to host vllm-*` means LiteLLM is still running but the -upstream vLLM runtime is not ready yet. +A crash-looping engine fails the acquire at once with its error quoted, so +the timeout is only for a model that is loading. Reading Docker's own +signals: `litellm exited with code 137` is a SIGKILL (an OOM kill or a forced +replacement), whereas LiteLLM returning HTTP 500 with `Cannot connect to host +vllm-*` means LiteLLM is running and its upstream vLLM is not ready yet. diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 130bf9dc..528d61a1 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -1,6 +1,11 @@ # Plan: one set of authorities for the Compose and KubeAI backends Status: draft for review, 2026-09-24. Nothing here is implemented. +Update 2026-09-25: K0, K1, K2, K4, K5 (except TUI logs) and G are done; +what remains (K3, TUI logs, dynamic routing on KubeAI, the in-cluster +gateway) is planned as phases in `docs/planning/backend-parity-roadmap.md`, +and the current state is `docs/backend-parity.md`. This file stays as the +audit record. ## Goal diff --git a/docs/backend-parity.md b/docs/backend-parity.md new file mode 100644 index 00000000..520b7044 --- /dev/null +++ b/docs/backend-parity.md @@ -0,0 +1,148 @@ +# Compose and KubeAI: parity and deviations + +infer-stack has one control plane and two realization layers. Everything +above the backend seam is shared: the catalog, the ledger (leases, +deployments, TTLs), the controller (`acquire` / `release` / `wait` / `evict` +/ `gc` / `clean` / `renew`, the admission queue, keep-warm reclaim), the +LiteLLM front door, the env-file contract, and the TUI. A backend only makes +the desired set real: Compose as containers on the local Docker daemon, +KubeAI as `Model` custom resources on a cluster. + +The intended relationship is **KubeAI = Compose + a scheduler**. A catalog +that runs on Compose runs on KubeAI once you say what a GPU unit means on +the cluster and install the cluster tooling. A card does not change: the same +`OPENAI_BASE_URL`, the same managed key, the endpoint alias as the model +name. + +This page records where that holds today (2026-09-25), where it does not, +and which gaps are deliberate. The plan for the rest is +[planning/backend-parity-roadmap.md](planning/backend-parity-roadmap.md); +cluster setup is [kubeai-backend.md](kubeai-backend.md). + +## What you add to move from Compose to KubeAI + +| | Compose | KubeAI adds | +|---|---|---| +| tools | Docker with the NVIDIA runtime | a cluster (`scripts/bootstrap_k3s.sh` for one host, `scripts/join_agent.sh` for another), `kubectl`, `helm`, the KubeAI chart (`scripts/install_kubeai.sh`), the NVIDIA device plugin on GPU nodes | +| information | none: GPUs are discovered with `nvidia-smi` | `resourceProfiles` in the chart values (what one GPU unit requests, and on which nodes), and per endpoint `runtime.resource_profile` or a default `kubeai_resource_profile` | +| settings | `backend compose` | `backend kubeai`; `kubeai_namespace`, `kubeai_base_url`, `kubeai_gateway_upstream` when the defaults (namespace `kubeai`, a port-forward on 8000, the Service's cluster IP) do not hold | +| preflight | none needed | `infer-stack doctor`: cluster → CRD → namespace → gateway | + +The catalog, `acquire … --env-file`, `release`, the TUI and the env file a +card sources are the same on both. + +## Layers + +``` +catalog ──► ledger ──► controller ──► gateway (LiteLLM: aliases, key, routes) + │ + ▼ backend seam: leasing/backend.py + ┌──────────┴───────────┐ + ComposeBackend KubeaiBackend + docker compose kubectl apply of Model CRs + local GPU planner the cluster scheduler + leasing/compose/ leasing/kubeai/ + leasing/kubeai-gateway/ +``` + +On KubeAI the gateway is a Compose project with no engines +(`infer-stack-gateway`) on the host running infer-stack. It routes each +alias to the cluster's KubeAI Service under the Model's name, so `secrets +rotate` and the static superset route table work unchanged. + +## Parity matrix + +**same**: one code path, or verified equivalent. **≈**: the same outcome by +a different mechanism. **gap**: missing on one side and on the roadmap. +**n/a**: does not apply there. **boundary**: deliberately unsupported +(see [planning/known-limitations.md](planning/known-limitations.md)). + +### Client contract + +| | Compose | KubeAI | +|---|---|---| +| one `OPENAI_BASE_URL`, the managed key, the alias as model name | same | same through the gateway; with `litellm false`, Model names and no key | +| `secrets rotate` | same | same | +| env file (`INFER_STACK_*`, `OPENAI_*`) | same | same | +| readiness is a real generation through the front door | same | same | + +### Lifecycle + +| | Compose | KubeAI | +|---|---|---| +| `acquire` / `release` / `wait` / `evict` / `gc` / `clean` / `renew` / `run` / `test` | same | same | +| admission: lease and GPUs committed atomically; a `--queue`d acquire holds nothing | yes | **gap** (roadmap P1): the lease is committed first, then rendered; a Model the cluster cannot place waits out `--timeout`, then rolls back | +| `config publish` | pure preview, then commit | render, then commit (same gap) | +| `--no-apply` / `apply` / `render` | same | same | +| unleased keep-warm yields to leased demand | at placement | ≈ during the wait: `needs_room` evicts the longest-idle, one per 30 s | +| crash-loop fail-fast, the engine's error quoted | same | same (pods, `kubectl logs --previous`) | +| image-pull progress | yes | n/a: the kubelet pulls; `ImagePullBackOff` is a reported wait reason | +| strict `residency()` for decisions, lenient `observe()` for reports | same | same | +| recovery after an interrupted apply (settle check) | yes | ≈ `kubectl apply` is idempotent; nothing is left half-created | + +### Placement + +| | Compose | KubeAI | +|---|---|---| +| where a deployment lands | local planner over `nvidia-smi` | the cluster scheduler, via `resource_profile:` | +| GPU count from TP × PP × DP | same | same | +| `placement.gpu_indices`, `allowed_gpus`, `skip_display_gpus` | yes | n/a: no host indices; node-scoped resource profiles are the equivalent | +| `placement.min_vram_gib`, `infer-stack measure` | yes | warned and ignored; `measure` refused (**gap**, P4) | +| GPU allocations recorded in the ledger | yes | none: the cluster owns them | +| more than one host | boundary | yes: the reason the backend exists | + +### Catalog features + +| | Compose | KubeAI | +|---|---|---| +| vLLM endpoints, `runtime.*` flags, `extra_args`, `env` | same | same (one argument pipeline) | +| served-name rules | same | same (`leasing/naming.py`) | +| `runtime.command`, `runtime.mounts` (custom launchers) | yes | boundary: a stock vLLM Model has no place for them; the render refuses loudly | +| ollama endpoints (`runtime_hosts`) | yes | boundary: daemon-shaped, not model-shaped | +| `min_replicas` / `max_replicas` | n/a | pass-through, default 1/1 | +| weight cache | the host HF cache, mounted | whatever the chart provides; cache profiles are not rendered | + +### Gateway + +| | Compose | KubeAI | +|---|---|---| +| static superset routes, no blip on model churn | same | same | +| `routes` inspect / seed / prune | yes | **gap** (P3): refused, although the registry exists | +| `dynamic_routing` (admin API + Postgres; distinct upstreams for same-model `--dedicated`) | yes | **gap** (P3) | +| Open WebUI (`ui`), reverse proxy | yes | **gap** (P3): the gateway project is rendered with `ui` off | +| `network migrate` / `network check` | yes | n/a: Service addresses are stable | +| where the gateway runs | this host | this host, so it is in every request's path (P5 moves it into the cluster) | + +### Day-2 operations and the TUI + +| | Compose | KubeAI | +|---|---|---| +| `doctor` | nothing to check | four checks | +| `logs`, `ps`, `stack up` / `stack down` | docker compose wrappers | **gap** (P2): they say so and point at `kubectl` | +| TUI: leases, deployments, catalog editing, acquire / release / evict, API tab, settings | same | same | +| TUI: engine log follow, the docker pane, the Up / Down buttons | `docker logs`, `docker compose ps` | **gap** (P2): empty, or "nothing rendered yet" | +| TUI GPU pane | `nvidia-smi` on this host | this host, not the cluster | +| `gc --orphans` (also inside `clean`) | yes | n/a: unlabeled Models are never touched | + +### Testing + +| | Compose | KubeAI | +|---|---|---| +| unit suite with fake runtimes | yes | yes | +| end to end | `dev/e2e_tests/run.sh` (tiers; `--gpu` for serving) | `dev/kubeai_e2e.sh` against a real cluster; k3s with CPU vLLM needs no GPU | + +## Deviations that stay + +- **Custom container launches and ollama daemons** do not run on KubeAI. A + KubeAI Model is stock vLLM. Use `--backend compose` for those endpoints. +- **GPU indices** have no cluster meaning. Kubernetes places by resource, not + by index; the per-node equivalent is a resource profile with a node + selector. +- **One control plane never spans a Compose host and a cluster.** Pick a + backend per data root. + +## Where the gaps close + +[planning/backend-parity-roadmap.md](planning/backend-parity-roadmap.md) +has the phases and their exit criteria. The audit that produced this page, +with what has already landed, is +`dev/tmp/plan-backend-unification-2026-09-24.md`. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 59adb851..1878dace 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -31,6 +31,10 @@ semantics carry over: an idle `keep-warm` deployment keeps its CR (model stays resident); `stop` deployments are pruned on release; `evict`/`gc` free the cluster. Hand-applied Models without the managed label are never touched. +Where the two backends match and where they still differ, row by row: +[backend-parity.md](backend-parity.md); the plan for the rest is +[planning/backend-parity-roadmap.md](planning/backend-parity-roadmap.md). + ## One-time cluster setup ```bash diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md new file mode 100644 index 00000000..47373b75 --- /dev/null +++ b/docs/planning/backend-parity-roadmap.md @@ -0,0 +1,178 @@ +# Backend parity roadmap: KubeAI as a superset of Compose + +**Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on +`dev/backend-unification` · P1–P5 not started · P6 is ongoing. +**Current state:** [../backend-parity.md](../backend-parity.md). +**Origin:** the scale-up run needs more than one workstation, and the +KubeAI backend had drifted from Compose for three months before the +2026-09-24 audit (`dev/tmp/plan-backend-unification-2026-09-24.md`). + +--- + +## Objective (read this first; it outlives the plan) + +A card, a catalog and an operator's habits move from one workstation to a +cluster with one change: naming what a GPU unit is. Concretely, + +1. every row of the parity matrix reads *same*, *≈*, *n/a* or *boundary*; + none reads *gap*; +2. a feature is written once. No code outside the backend modules and the + CLI's construction of them branches on the backend kind. + +Compose stays the fastest local path and loses nothing. KubeAI is Compose +plus a scheduler, with more information supplied and more tools installed; +it is not a sibling with its own habits. + +## Principles + +- **Superset, not sibling.** KubeAI gains what Compose has. Where a Compose + feature has no cluster meaning (GPU indices, `network migrate`), the + matrix says *n/a* and the CLI says why; it does not grow a second design. +- **The seam is the only place kind matters.** `isinstance(…, ComposeBackend)` + in the CLI or TUI is a smell. Each phase removes some. The Compose-only + escape hatches (`stack up`/`down` as raw compose) stay, and say so. +- **Verified on a real cluster.** A phase is done when its step in + `dev/kubeai_e2e.sh` passes on k3s, not when its fakes pass. This is how + the backend drifted last time. +- **One reviewable change per phase**, the Compose suite green throughout. + +## Phases + +### P0. One contract, one liveness view, one diagnosis — done + +The 2026-09-24 audit landed: an e2e on k3s (K0); the LiteLLM gateway +fronting both backends, so the alias and key are the same everywhere (K1); +a strict `residency()` built from pods (K2); one startup diagnosis over +containers or pods (K4); one served-name rule, one GPU count, `runtime.env` +on KubeAI (K5); the gateway as its own module (G); and unleased keep-warm +yielding to leased demand on both backends. + +### P1. One acquire path + +**Closes:** the admission and `config publish` rows. + +Today `Controller._admission_mode()` is true only for a backend with +`residency` + `preview` + `converge`, which is Compose. KubeAI takes the +pre-September branch in five places (render, acquire, `observe_state`, +`config publish`, `renew`). + +- `KubeaiBackend.preview(desired, placement, approve)`: render the Models + to memory with a digest, write nothing. Placement inputs are accepted and + ignored; `unplaced` is exactly the unrenderable set (no profile, engine, + custom launch). +- Admission accepts a backend with no GPU accounting: Compose reports + assignments, KubeAI reports none, and the ledger's allocation table stays + empty for it. A Pending pod is a wait reason, never an unplaced error. +- `--queue` on KubeAI: admitted at once; the cluster is the queue. Say so + in the option's help. +- `MemoryBackend` / `NullBackend` get a trivial `residency` and `preview` + so the tests run the one path. +- Delete the non-admission branches and `_admission_mode()`. + +**Exit:** `_admission_mode` is gone; a render failure on KubeAI rolls the +lease back before any `kubectl`; e2e passes; `test_controller` runs its +acquire scenarios on all three fakes. +**Size:** medium. The controller is the most-changed module; start from a +green e2e and keep the diff to the five sites. + +### P2. Day-2 verbs and the TUI through the seam + +**Closes:** `logs` / `ps` / `stack`, the TUI's log follow, docker pane and +Up / Down, and `measure`. + +- Two backend methods: `instances()` (from `residency()`: deployment, unit + name, state, restarts, age) and `stream_logs(target, *, follow, tail)` + (`docker logs -f` / `kubectl logs -f`). +- `ps` prints `instances()`. `logs` streams. `stack up` is `apply`; `stack + down` is `backend.down()`, which both backends have. The raw compose form + moves under `stack compose …` and stays Compose-only by name. +- TUI: the docker pane becomes an instances pane; the log follower uses + `stream_logs`; Up / Down call `apply` / `down`. +- `measure`: lift the guard (`deployment_logs` already exists on KubeAI) + after checking vLLM's memory line reaches the pod log. + +**Exit:** on k3s the TUI follows a Model's log and lists its pod; `infer-stack +logs ` and `ps` work on both backends with the same output shape. +**Size:** medium, mostly plumbing; no controller change. + +### P3. Gateway feature parity + +**Closes:** `routes` on KubeAI, `dynamic_routing`, Open WebUI and the reverse +proxy in front of a cluster. + +- The `routes` commands and `secrets rotate` resolve `controller.backend.gateway` + on either backend instead of checking the backend kind. +- The KubeAI gateway project is rendered with the same `ui`, `reverse_proxy` + and `dynamic_routing` settings as Compose. Postgres for dynamic routing is + a gateway-project service, so nothing engine-side changes. + +**Exit:** `routes list` on KubeAI; Open WebUI in front of a cluster; an e2e +step that acquires the same model `--dedicated` twice under dynamic routing +and gets two Models and two routes. +**Size:** small to medium; the `Gateway` class already owns the pieces. + +### P4. Placement information parity (optional) + +**Closes:** `min_vram_gib` / `measure`; less hand-written cluster +information. + +- `catalog suggest` on KubeAI reads the device plugin's node labels + (`nvidia.com/gpu.product`, `.memory`) and proposes `resourceProfiles` + (today the README does this by hand). +- With several profile sizes, `min_vram_gib` picks the smallest profile + whose GPU memory fits when an endpoint names none; otherwise the + warn-and-ignore stays. + +**Exit:** a catalog with `min_vram_gib` and no `resource_profile` lands on +the right profile on a cluster with two sizes. +**Size:** small. Shrinks "the information you add"; skip if nobody runs a +mixed-GPU cluster. + +### P5. The multi-workstation shape + +**Closes:** the gateway on one host, the port-forward default. + +- Render the gateway in the cluster (a Deployment + Service behind an + ingress) as the second gateway target; the Compose-project gateway stays + the single-host default. `secrets rotate` becomes a Secret update and a + rollout; `doctor` checks the ingress; `kubeai_base_url` defaults to it. +- A second workstation joins with `scripts/join_agent.sh`; resource + profiles with node selectors say which GPUs live where. Write the "add a + workstation" runbook into `kubeai-backend.md`. + +**Exit:** on two nodes, a card on node A leases a model that lands on node +B and talks to it through the in-cluster gateway; `secrets rotate` works. +**Size:** medium to large. The only phase that adds a second renderer for +the gateway, and the only one that needs a second machine. Last. + +### P6. One test surface (ongoing) + +- Parametrize the controller tests over the Memory, fake-Compose and + fake-KubeAI backends; `tests/test_parity.py` runs each *same* row's + scenario on both fakes. +- `dev/kubeai_e2e.sh` gains one step per phase. +- Finishing a phase updates the matrix in `backend-parity.md`; the plan is + done when the matrix has no *gap* row. + +## Not in scope + +- more than one cluster, or scheduling across a Compose host and a cluster; +- KubeAI autoscaling beyond `min_replicas` / `max_replicas`: the lease + decides residency; +- custom container launches and ollama daemons on KubeAI (a boundary in + [known-limitations.md](known-limitations.md)); +- Slurm, which is its own track ([../slurm-compatibility.md](../slurm-compatibility.md)); +- moving the Compose backend onto Kubernetes primitives. + +## Order and size + +| phase | size | needs | gives | +|---|---|---|---| +| P1 | medium | a green e2e | one controller; atomic acquire on a cluster | +| P2 | medium | P1 (instances from residency) | the TUI and day-2 verbs on a cluster | +| P3 | small–medium | none | dynamic routing, Open WebUI on a cluster | +| P4 | small | a mixed-GPU cluster to verify | less hand-written cluster information | +| P5 | medium–large | a second machine | the gateway off the operator's host | + +P1 → P2 → P3 is single-host-cluster parity: k3s on one workstation, with +everything a Compose user has. P5 is the multi-workstation step. From d6a19c48058acf0dddccf307ad2e0d264ab50241 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Fri, 25 Sep 2026 21:27:45 -0400 Subject: [PATCH 023/141] P1a: KubeAI acquires go through admission, as Compose's do KubeAI gains an in-memory preview (and plan_on_idle_host) over one shared render, and converge accepts placement=, so Controller._admission_mode() is true for both real backends. An unrenderable acquire now writes no lease and runs no kubectl; a queued one fails at once. Duplicate authorities removed on the way: - "does this backend allocate GPUs" is allocates_gpus(), read by the controller's accounting and the CLI's status view; on KubeAI every deployment commits an empty allocation, so nothing is unresolved. - the approval digest and pre-approval move from ComposeBackend into ConvergeScaffold. Compose's stable-address network and container adoption are handed only to a backend that declares them. The roadmap splits P1 after review: P1b (delete the legacy branch, which only test fakes use) goes with P6; deferred duplicates are tabled there. Verified: full suite, and dev/kubeai_e2e.sh on k3s with a new admission step and the make-room step. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 17 ++++++ dev/kubeai_e2e.sh | 24 ++++++++ docs/backend-parity.md | 5 +- docs/kubeai-backend.md | 16 +++-- docs/planning/backend-parity-roadmap.md | 79 +++++++++++++++++++------ infer_stack/backends/kubeai.py | 59 +++++++++++++++--- infer_stack/cli/commands_leasing.py | 10 +++- infer_stack/leasing/backend.py | 56 ++++++++++++++++-- infer_stack/leasing/compose.py | 24 +------- infer_stack/leasing/controller.py | 43 +++++++++----- tests/test_leasing_kubeai.py | 49 +++++++++++++-- 11 files changed, 299 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 819244cb..f7acb9fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### KubeAI acquires go through admission, as Compose's do + +An acquire on the KubeAI backend is now previewed in memory and commits its +lease only if every deployment renders, the same path Compose takes. A +refused acquire (no resource profile, an ollama endpoint, a served-name +collision) writes no lease and runs no `kubectl`; before, the lease was +committed, rendered, and then rolled back. A `--queue` acquire of such an +endpoint fails at once instead of waiting out its timeout. The cluster +schedules, so a KubeAI deployment commits an empty GPU allocation; `leases` +still shows no GPU for it, and a `--no-apply` acquire now reports it as +cluster-scheduled rather than unplaced. An idle +keep-warm Model is kept only while its pod has started, as a keep-warm +container is on Compose. The approval digest moved into the scaffold both +backends share, and one function now answers whether a backend allocates +GPUs. Removing the older acquire branch, which only test fakes still use, +is the next step (roadmap P1b). + ### Compose and KubeAI: a parity matrix and a roadmap `docs/backend-parity.md` states the relationship the two backends are meant diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index aba5df1b..f735e1e2 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -109,6 +109,30 @@ remaining=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ -l infer-stack/managed=true -o name | wc -l) [ "$remaining" = 0 ] || { echo "!! model not pruned"; exit 1; } +echo '== an unrenderable endpoint is refused before anything is written' +# No resource_profile and no kubeai_resource_profile default: admission's +# preview refuses it, so no lease is committed and kubectl applies nothing; +# --queue fails at once, since waiting cannot make it renderable. +cat >> "$WORK/config/catalog.yaml" < "$WORK/bad.log" 2>&1; then + echo '!! an endpoint with no resource profile was admitted' >&2; exit 1 +fi +grep -q 'resource profile' "$WORK/bad.log" \ + || { cat "$WORK/bad.log" >&2; echo '!! the refusal did not name the cause' >&2; exit 1; } +active=$(run_is leases --json | python3 -c \ + 'import json,sys; print(sum(le["state"] == "active" for le in json.load(sys.stdin)["leases"]))') +[ "$active" = 0 ] || { echo "!! $active lease(s) left active" >&2; exit 1; } +remaining=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ + -l infer-stack/managed=true -o name | wc -l) +[ "$remaining" = 0 ] || { echo '!! a Model was applied for it' >&2; exit 1; } +echo ' refused at admission: no lease, no Model' + if [ "${E2E_MAKE_ROOM:-0}" = 1 ]; then # Needs a profile only one Model fits at a time (`cpu-half` in # dev/e2e_tests/kubeai-cpu-values.yaml): E2E_ROOM_PROFILE=cpu-half. diff --git a/docs/backend-parity.md b/docs/backend-parity.md index 520b7044..f4741e44 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -70,8 +70,9 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | `acquire` / `release` / `wait` / `evict` / `gc` / `clean` / `renew` / `run` / `test` | same | same | -| admission: lease and GPUs committed atomically; a `--queue`d acquire holds nothing | yes | **gap** (roadmap P1): the lease is committed first, then rendered; a Model the cluster cannot place waits out `--timeout`, then rolls back | -| `config publish` | pure preview, then commit | render, then commit (same gap) | +| admission: previewed in memory, then the lease committed with its allocation; a refused acquire writes nothing | yes | same; the allocation is empty (the cluster places), so admission decides renderability, and a pod the cluster cannot place is a wait reason | +| `--queue` | waits for a free GPU | ≈ admitted at once, the cluster is the queue; an unrenderable endpoint fails at once | +| `config publish` | pure preview, then commit | same | | `--no-apply` / `apply` / `render` | same | same | | unleased keep-warm yields to leased demand | at placement | ≈ during the wait: `needs_room` evicts the longest-idle, one per 30 s | | crash-loop fail-fast, the engine's error quoted | same | same (pods, `kubectl logs --previous`) | diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 1878dace..3be5b96e 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -108,10 +108,18 @@ infer-stack release --env-file lease.env - **Readiness is a real generation** through the gateway (same philosophy as compose): a Model CR existing — even with ready replicas — is not proof it can serve. -- **Placement can only fail at admission.** The render never rejects for - capacity (the cluster schedules); a Model the cluster cannot place sits - not-ready until the acquire's `--timeout`, which then rolls the lease back. - The wait says why (`pod: Unschedulable`, `pod: ImagePullBackOff`). +- **Admission is the same as on compose, minus GPU accounting.** An acquire + is previewed in memory and commits its lease only if every deployment + renders; a refused one (no resource profile, an ollama endpoint, a + served-name collision) writes no lease and runs no `kubectl`. The cluster + schedules, so capacity is never an admission reason and `--queue` is + admitted at once. A Model the cluster cannot place sits not-ready until + the acquire's `--timeout`, which rolls the lease back; the wait says why + (`pod: Unschedulable`, `pod: ImagePullBackOff`). +- **An idle keep-warm Model stays only while its pod has started**, as an + idle keep-warm container does on compose (a crash-looping pod counts, like + a restarting container). One whose pod is gone, still Pending or finished + is pruned at the next render. - **A keep-warm model without a lease gives way to one with a lease.** When a leased Model's pod is `Unschedulable`, the wait evicts the longest-idle keep-warm deployment, one per 30 s, until it fits. Compose applies the same diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 47373b75..cc554423 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,8 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · P1–P5 not started · P6 is ongoing. +`dev/backend-unification` · **P1a done** 2026-09-25 · P1b–P5 not started · +P6 is ongoing. **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the @@ -56,24 +57,50 @@ Today `Controller._admission_mode()` is true only for a backend with pre-September branch in five places (render, acquire, `observe_state`, `config publish`, `renew`). +Review 2026-09-25 against the code split this in two. The admission path +assumes GPU accounting in three places (`_backfill_allocations`, +`_unresolved_allocations`, `_admission_view` keeps a LIVE deployment only +with committed GPUs), so a `preview` alone would drop every KubeAI lease from +the render. And about fifteen test fakes drive the legacy path on purpose +(queue, lock, serialised publication), so deleting it is test work, not +controller work. + +**P1a. KubeAI on the admission path.** + +- One authority for "this backend allocates GPUs": `allocates_gpus` + (Compose true, KubeAI false). The controller's accounting reads it + through one helper; on KubeAI every deployment commits an empty + allocation, so nothing is ever unresolved. - `KubeaiBackend.preview(desired, placement, approve)`: render the Models to memory with a digest, write nothing. Placement inputs are accepted and - ignored; `unplaced` is exactly the unrenderable set (no profile, engine, - custom launch). -- Admission accepts a backend with no GPU accounting: Compose reports - assignments, KubeAI reports none, and the ledger's allocation table stays - empty for it. A Pending pod is a wait reason, never an unplaced error. -- `--queue` on KubeAI: admitted at once; the cluster is the queue. Say so - in the option's help. -- `MemoryBackend` / `NullBackend` get a trivial `residency` and `preview` - so the tests run the one path. -- Delete the non-admission branches and `_admission_mode()`. - -**Exit:** `_admission_mode` is gone; a render failure on KubeAI rolls the -lease back before any `kubectl`; e2e passes; `test_controller` runs its -acquire scenarios on all three fakes. -**Size:** medium. The controller is the most-changed module; start from a -green e2e and keep the diff to the five sites. + ignored; the plan assigns every renderable deployment no GPUs. + `plan_on_idle_host` is the same render, so a `--queue` acquire of an + unrenderable endpoint fails at once instead of waiting out the timeout. +- `converge(..., placement=)` on KubeAI. Without it the controller's + `TypeError` fallback would render **and apply** in one step. +- The approved-digest guard (`_planned_digest`, pre-approval, + `last_planned_digest` / `last_preview_digest`) moves from `ComposeBackend` + to `ConvergeScaffold`: one approval mechanism, not a copy. +- Compose's stable addresses and container adoption stay Compose-only by + capability (`network`, `adopted`), not by joining KubeAI to them. +- `--queue` on KubeAI: admitted at once; the cluster is the queue. + +**P1a done 2026-09-25.** Verified on k3s by `dev/kubeai_e2e.sh`, which +gained the step "an unrenderable endpoint is refused before anything is +written" (a `--queue` acquire, no lease, no Model), and still passes the +make-room step, which now runs on the admission path. + +**P1b. Delete the legacy branch** (with P6). +`MemoryBackend` / `NullBackend` and the test fakes get a trivial +`residency` and `preview`; then the non-admission branches and +`_admission_mode()` go. Until then the legacy path serves only test fakes +and `realize`/`teardown` backends, and no new code may depend on it. + +**Exit:** P1a: `_admission_mode()` is true for both real backends; a render +failure on KubeAI rolls the lease back before any `kubectl`; e2e passes. +P1b: `_admission_mode` is gone; `test_controller` runs its acquire scenarios +on all three fakes. +**Size:** P1a medium, P1b medium (mostly tests). ### P2. Day-2 verbs and the TUI through the seam @@ -154,6 +181,21 @@ the gateway, and the only one that needs a second machine. Last. - Finishing a phase updates the matrix in `backend-parity.md`; the plan is done when the matrix has no *gap* row. +## Duplicate authorities + +The rule while executing: a duplicate authority found on the way is +refactored when it is small, recorded here when it is not, and never made +worse. A blocker is fixed whatever its size. + +| authority | where it was | status | +|---|---|---| +| "this backend allocates GPUs" | inferred from which methods a backend has | **fixed** (P1a): `allocates_gpus()` in `leasing/backend.py`, read by the controller and the CLI | +| approval digest and pre-approval | a copy inside `ComposeBackend` | **fixed** (P1a): `ConvergeScaffold`, shared | +| KubeAI render vs its plan | `converge` rendered inline | **fixed** (P1a): one `_render_documents` behind `converge`, `preview` and `plan_on_idle_host` | +| the acquire path | admission and a legacy branch in five places | deferred to P1b | +| `_render`'s one-shot `converge(desired)` fallback, which applies | a second render contract for old backends | deferred to P1b: it goes with the legacy branch | +| the KubeAI gateway's approval | the gateway project asks its own diff approval at render, after the lease commits, not in the admission preview | deferred to P3. Same result under `--yes`; interactively, a declined gateway change rolls the lease back after the commit | + ## Not in scope - more than one cluster, or scheduling across a Compose host and a cluster; @@ -168,7 +210,8 @@ the gateway, and the only one that needs a second machine. Last. | phase | size | needs | gives | |---|---|---|---| -| P1 | medium | a green e2e | one controller; atomic acquire on a cluster | +| P1a | medium | a green e2e | atomic acquire on a cluster | +| P1b | medium | P1a; with P6 | one controller path | | P2 | medium | P1 (instances from residency) | the TUI and day-2 verbs on a cluster | | P3 | small–medium | none | dynamic routing, Open WebUI on a cluster | | P4 | small | a mixed-GPU cluster to verify | less hand-written cluster information | diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index a65bdf0c..94f0af2c 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -392,8 +392,52 @@ def _render_gateway(self, rendered: RenderedModels) -> None: # -- converge-style surface ------------------------------------------------ - def converge(self, desired: list[Deployment], *, apply: bool = True): - """Render the desired Model set, then optionally apply it.""" + #: The cluster schedules: no host GPU indices are allocated or recorded, + #: and admission commits an empty allocation for every deployment. + allocates_gpus = False + + def _render_documents(self, desired: list[Deployment]): + """``(plan, rendered, planned)`` in memory: the one KubeAI render. + + The plan assigns every renderable deployment no GPUs; the + unrenderable ones are left out, with the render's reasons. + """ + from ..leasing.placement import GpuPlan + + rendered = render_models( + list(desired), + namespace=self.namespace, + default_resource_profile=self.default_resource_profile, + ) + plan = GpuPlan( + assignments={g.id: [] for g in desired if g.id not in rendered.unrenderable}, + errors=list(rendered.errors), + ) + return plan, rendered, {self.models_file: rendered.text} + + def preview(self, desired: list[Deployment], placement=None, *, approve: bool = False): + """Render ``desired`` as :meth:`converge` would; write nothing. + + ``placement`` is accepted for the admission interface and ignored: + the cluster places. Returns ``(plan, rendered)``. + """ + plan, rendered, planned = self._render_documents(desired) + self._preview_approval(planned, approve=approve) + return plan, rendered + + def plan_on_idle_host(self, desired: list[Deployment]): + """Whether ``desired`` could ever be served here: renderable or not. + + Capacity is the cluster's to decide, so only a render failure is + permanent; that lets a queued acquire of one fail at once. + """ + return self._render_documents(desired)[0] + + def converge(self, desired: list[Deployment], *, apply: bool = True, placement=None): + """Render the desired Model set, then optionally apply it. + + ``placement`` (admission inputs) is ignored: the cluster places. + """ from .._log import logger desired = list(desired) @@ -416,17 +460,14 @@ def converge(self, desired: list[Deployment], *, apply: bool = True): 'requirement via the resource profile instead)', g.id, ) - rendered = render_models( - desired, - namespace=self.namespace, - default_resource_profile=self.default_resource_profile, - ) + plan, rendered, planned = self._render_documents(desired) self.last_errors = list(rendered.errors) self.last_unplaced = set(rendered.unrenderable) - self.last_assignments = {} + self.last_assignments = {} # the cluster places for err in rendered.errors: logger.warning(' render: {}', err) - self._approve_changes({self.models_file: rendered.text}) + self.last_planned_digest = self._planned_digest(planned) + self._approve_changes(planned) self._atomic_write(self.models_file, rendered.text) self._save_sidecar( { diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 507b3ff9..cce3120d 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -511,9 +511,13 @@ def _emit_staged(config, controller, outcome) -> int: }, indent=2)) return 0 print(f'staged {outcome.lease.id} (owner={outcome.lease.owner}) — not applied') + from ..leasing.backend import allocates_gpus + + local = allocates_gpus(controller.backend) for g in outcome.deployments: eps = ', '.join(sorted(g.served)) or g.id - print(f' {eps}: {_gpu_where(assignments.get(g.id))} ({g.id})') + where = _gpu_where(assignments.get(g.id)) if local else 'cluster-scheduled' + print(f' {eps}: {where} ({g.id})') path = _compose_file_path(controller) if path: print(f' compose: {path}') @@ -1898,6 +1902,10 @@ def _placement_view(controller): except Exception: # noqa: BLE001 - status must never crash pass assignments: dict[str, list[int]] = {} + from ..leasing.backend import allocates_gpus + + if not allocates_gpus(backend): + return observed, assignments # the cluster places; no GPU indices if controller._admission_mode(): # Committed allocations, and idle residents' physical GPUs; the # legacy planner view would show placements admission would not make. diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 0c1db433..28356ddc 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -85,6 +85,17 @@ def __init__(self, deployment_ids, reasons): ) +def allocates_gpus(backend) -> bool: + """Whether ``backend`` allocates host GPU indices itself. + + True for Compose, which places on this host and records the GPUs each + deployment holds. False for a backend whose cluster schedules (KubeAI): + admission commits an empty allocation, and no GPU is ever "unresolved". + The one place the controller and the CLI ask this. + """ + return bool(getattr(backend, 'allocates_gpus', True)) + + @runtime_checkable class Backend(Protocol): """What the :class:`Controller` needs from a serving backend. @@ -156,12 +167,14 @@ def apply(self) -> bool | None: class AdmissionBackend(ConvergeBackend, Protocol): """The surface admission mode uses (see ``Controller._admission_mode``). - Today only :class:`~infer_stack.leasing.compose.ComposeBackend` has it: - strict residency, an in-memory placement ``preview``, and the Compose - network and adoption state the controller hands it. The controller - reaches these through ``Controller._admitting``, only after the - capability check, so the type checker sees one named capability instead - of attributes a minimal :class:`Backend` does not have. + Both real backends have it: strict residency and an in-memory + ``preview``. Compose also takes the stable-address network and the + container-adoption table from the controller (``network``, + ``on_addresses``, ``adopted``); those are Compose-only, and the + controller hands them over only to a backend that declares them. The + controller reaches these through ``Controller._admitting``, only after + the capability check, so the type checker sees one named capability + instead of attributes a minimal :class:`Backend` does not have. """ network: dict[str, Any] | None @@ -254,9 +267,37 @@ def _save_sidecar(self, data: dict) -> None: self._atomic_write(self._state_file, json.dumps(data, indent=2)) + #: Digest of files an admission preview already had approved. + _preapproved: str | None = None + #: Digest of the files the last render produced (approved-digest guard). + last_planned_digest: str | None = None + #: Digest of the files the last preview produced. + last_preview_digest: str | None = None + + @staticmethod + def _planned_digest(planned: dict) -> str: + import hashlib + import json + + material = json.dumps({str(k): v for k, v in planned.items()}, sort_keys=True) + return hashlib.sha256(material.encode('utf-8')).hexdigest() + + def _preview_approval(self, planned: dict, *, approve: bool) -> None: + """Record a preview's digest; with ``approve``, ask now, not after commit. + + The render that follows the commit skips the prompt when it produces + the same files (see :meth:`_approve_changes`). + """ + self.last_preview_digest = self._planned_digest(planned) + if approve: + self._approve_changes(planned) + self._preapproved = self.last_preview_digest + def _approve_changes(self, planned: dict) -> None: """Show pending rendered-state changes and confirm them. + Files a preview already had approved (same digest) pass silently once. + ``planned`` maps target paths to their new content. When nothing actually changed, this is a quiet no-op. When ``assume_yes`` (scripts / non-interactive / ``--yes``), it applies after a one-line log. @@ -265,6 +306,9 @@ def _approve_changes(self, planned: dict) -> None: """ from .._log import logger + preapproved, self._preapproved = self._preapproved, None + if preapproved is not None and self._planned_digest(planned) == preapproved: + return changed = { p: text for p, text in planned.items() diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index ea44570f..ff209d2c 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1389,10 +1389,7 @@ def preview(self, desired: list[Deployment], placement=None, *, approve: bool = then does not ask again as long as it produces the same files. """ docs = self._render_documents(list(desired), placement) - self.last_preview_digest = self._planned_digest(docs['planned']) - if approve: - self._approve_changes(docs['planned']) - self._preapproved = self.last_preview_digest + self._preview_approval(docs['planned'], approve=approve) return docs['plan'], docs['rendered'] def _render_documents(self, desired: list[Deployment], placement) -> dict[str, Any]: @@ -1455,18 +1452,6 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A #: newly allocated addresses (append-only). on_addresses: Any = None - @staticmethod - def _planned_digest(planned: dict) -> str: - material = json.dumps({str(k): v for k, v in planned.items()}, sort_keys=True) - return hashlib.sha256(material.encode('utf-8')).hexdigest() - - #: Digest of files an admission preview already had approved. - _preapproved: str | None = None - #: Digest of the files the last render produced (approved-digest guard). - last_planned_digest: str | None = None - #: Digest of the files the last preview produced. - last_preview_digest: str | None = None - def pull_images(self, images) -> list[str]: """Pull ``images``; ``config publish`` passes :func:`profile_images`.""" from .._log import logger @@ -1476,13 +1461,6 @@ def pull_images(self, images) -> list[str]: self.run(['docker', 'pull', image]) return sorted(set(images)) - def _approve_changes(self, planned: dict) -> None: - if self._preapproved is not None and self._planned_digest(planned) == self._preapproved: - self._preapproved = None - return - self._preapproved = None - super()._approve_changes(planned) - def plan_on_idle_host(self, desired: list[Deployment]): """Placement for ``desired`` alone, as if nothing else were running. diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index de969761..7e0c29f6 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -485,7 +485,8 @@ def _render(self) -> ReconcileResult: self._backfill_allocations(residency) self._prepare_network() desired, placement = self._admission_view(residency) - self._admitting.adopted = self._prune_adopted(residency) + if hasattr(self.backend, 'adopted'): + self._admitting.adopted = self._prune_adopted(residency) else: desired = self.desired_deployments() # Bound once rather than probed with hasattr: the capability check is @@ -520,7 +521,7 @@ def _render(self) -> ReconcileResult: displaced=list(getattr(self.backend, 'last_displaced', ()) or ()), degraded=list(getattr(self.backend, 'last_degraded', ()) or ()), ) - if placement is not None: + if placement is not None and hasattr(self.backend, 'adopted'): self._adopt_existing(residency) return rec desired_ids = {g.id for g in desired} @@ -554,16 +555,19 @@ def _render(self) -> ReconcileResult: # -- admission (plan steps P5, P6, P9) ------------------------------------ # - # Backends with strict residency and an in-memory preview (Compose) get - # admission semantics: + # Backends with strict residency and an in-memory preview (Compose and + # KubeAI) get admission semantics: # * a LIVE deployment holds a committed allocation (assigned_gpus); # * an IDLE keep-warm deployment is only an optional candidate, and only # while it is uniquely resident; it yields its GPUs to demand and is # never started; # * an acquire is previewed in memory (placement AND render) and commits # its lease together with its allocations, or commits nothing. - # Other backends (KubeAI, where the cluster schedules; test fakes) keep the - # previous behaviour. + # Where the cluster schedules (KubeAI, ``allocates_gpus`` false) every + # deployment commits an empty allocation: admission then decides only + # renderability, and a Pending pod is a wait reason, not an unplaced + # error. Test fakes without residency/preview keep the previous path + # (roadmap P1b removes it). def _stored_state(self, lease_id: str): """A lease's state as stored (not virtually expired), or ``None``.""" @@ -628,7 +632,13 @@ def _admission_view( return [*required.values(), *optional], inputs def _prepare_network(self) -> None: - """Give the backend the stable-address table, once a network is migrated.""" + """Give the backend the stable-address table, once a network is migrated. + + Compose only: a backend without a ``network`` attribute has no + host network to stamp addresses on. + """ + if not hasattr(self.backend, 'network'): + return config = self.ledger.network_config() if config is None: self._admitting.network = None @@ -859,6 +869,13 @@ def remove_orphans(self, confirm: Callable[[list], bool]) -> list: self._admitting.run(['docker', 'rm', '-f', *[c.container_id for c in orphans]]) return orphans + def _gpu_units(self, deployment) -> int: + """GPUs admission must account for: none where the cluster schedules.""" + from .backend import allocates_gpus + from .placement import required_gpu_count + + return required_gpu_count(deployment) if allocates_gpus(self.backend) else 0 + def _backfill_allocations(self, residency) -> list[str]: """Adopt allocations for LIVE deployments that predate them. @@ -867,14 +884,12 @@ def _backfill_allocations(self, residency) -> list[str]: GPUs stays unresolved: it keeps running where it is, but no new GPU is allocated to anyone until it is released. Returns the unresolved ids. """ - from .placement import required_gpu_count - unresolved = [] _, deployments = self.ledger.status() for deployment in deployments: if deployment.state != DeploymentState.LIVE or deployment.assigned_gpus is not None: continue - if required_gpu_count(deployment) == 0: + if self._gpu_units(deployment) == 0: self.ledger.set_allocation(deployment.id, []) continue resident = residency.resident(deployment.id) @@ -891,15 +906,13 @@ def _admit(self, overlay, residency): commit for the candidate's new and revived deployments. Nothing is written here. """ - from .placement import required_gpu_count - adopted: dict[str, list[int]] = {} need: list[str] = [] for gid, deployment in overlay.deployments.items(): fresh = gid in overlay.created or gid in overlay.revived if not fresh: continue - if required_gpu_count(deployment) == 0: + if self._gpu_units(deployment) == 0: continue resident = residency.resident(gid) if ( residency is not None and gid in overlay.revived) else None @@ -980,13 +993,11 @@ def _gpu_holders(self, plan) -> list[str]: return out def _unresolved_allocations(self, *, exclude: AbstractSet[str] = frozenset()) -> list[str]: - from .placement import required_gpu_count - _, deployments = self.ledger.status() return [ g.id for g in deployments if g.state == DeploymentState.LIVE and g.assigned_gpus is None - and g.id not in exclude and required_gpu_count(g) > 0 + and g.id not in exclude and self._gpu_units(g) > 0 ] def set_invocation_catalog(self, catalog) -> None: diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index de85cbc9..19071644 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -70,6 +70,11 @@ def __call__(self, args: list[str]) -> str: if doc: self.applied[doc['metadata']['name']] = doc return '' + if verb == 'get' and args[4] == 'pods': + # KubeAI runs one pod per applied Model; here it is up at once. + return json.dumps({'items': [ + _pod(f'model-{name}-0', doc['metadata']['labels']['infer-stack/deployment']) + for name, doc in self.applied.items()]}) if verb == 'get': items = list(self.applied.values()) if '-l' in args: # emulate the label selector @@ -386,19 +391,55 @@ def test_acquire_release_lifecycle(tmp_path): assert rel.reconcile is not None -def test_acquire_missing_profile_rolls_back(tmp_path): - """An unrenderable deployment behaves exactly like a placement failure: - the acquire fails loudly and the lease is rolled back.""" +def test_acquire_missing_profile_commits_nothing(tmp_path): + """An unrenderable deployment fails admission, as an unplaceable one does + on compose: the preview refuses it before any lease is written or any + kubectl apply runs.""" from infer_stack.leasing.backend import PlacementError ctl, be, kubectl = make_controller(tmp_path) with pytest.raises(PlacementError, match='resource profile'): ctl.acquire('alice', [_req('qwen', profile=None)], wait=False) leases, deployments = ctl.ledger.status() - assert [le.state for le in leases] == [LeaseState.RELEASED] + assert leases == [] + assert not any('apply' in call for call in kubectl.calls) assert kubectl.applied == {} +def test_kubeai_takes_the_admission_path(tmp_path): + """One acquire path: KubeAI admits by preview and commits no GPUs.""" + ctl, be, kubectl = make_controller(tmp_path) + assert ctl._admission_mode() + out = ctl.acquire('alice', [_req('qwen')], wait=False) + (deployment,) = out.deployments + assert deployment.assigned_gpus == [] # committed, and empty + assert ctl._unresolved_allocations() == [] + + +def test_status_shows_no_gpu_for_a_cluster_scheduled_deployment(tmp_path): + """The committed allocation is empty, which must not read as "cpu".""" + from infer_stack.cli.commands_leasing import _gpu_label, _placement_view + + ctl, be, kubectl = make_controller(tmp_path) + out = ctl.acquire('alice', [_req('qwen')], wait=False) + observed, assignments = _placement_view(ctl) + (deployment,) = out.deployments + assert _gpu_label(deployment.id, observed, assignments) == '-' + + +def test_queued_acquire_of_an_unrenderable_endpoint_fails_at_once(tmp_path): + """The cluster is the queue, but it cannot queue what it cannot render.""" + from infer_stack.leasing.backend import PlacementError + + ctl, be, kubectl = make_controller(tmp_path) + slept = [] + ctl.sleep = slept.append + with pytest.raises(PlacementError, match='resource profile'): + ctl.acquire('alice', [_req('qwen', profile=None)], wait=False, + wait_for_placement=True, timeout=600) + assert slept == [] + + def test_keep_warm_stays_resident_after_release(tmp_path): ctl, be, kubectl = make_controller(tmp_path) out = ctl.acquire( From d3ee494107746f95c90c2901df546fd8e4b85bcc Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Fri, 25 Sep 2026 21:31:18 -0400 Subject: [PATCH 024/141] Docs: a work queue for backend parity, ending in a UX audit loop docs/queue.md orders what can be done and verified without a GPU or a second machine, with a done-when per item, a rule for adding unforeseen items, and a UX audit that must pass before the queue is finished. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 1 + docs/queue.md | 146 ++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 docs/queue.md diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index cc554423..268f8d76 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -4,6 +4,7 @@ `dev/backend-unification` · **P1a done** 2026-09-25 · P1b–P5 not started · P6 is ongoing. **Current state:** [../backend-parity.md](../backend-parity.md). +**Execution order:** [../queue.md](../queue.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the 2026-09-24 audit (`dev/tmp/plan-backend-unification-2026-09-24.md`). diff --git a/docs/queue.md b/docs/queue.md new file mode 100644 index 00000000..08f8fa05 --- /dev/null +++ b/docs/queue.md @@ -0,0 +1,146 @@ +# Work queue: backend parity + +The executable order for [planning/backend-parity-roadmap.md](planning/backend-parity-roadmap.md), +limited to what can be done and verified without a GPU or a second physical +machine: a guest VM running k3s with CPU vLLM. Items that end in a hardware +check produce a script to hand over rather than a claim of done. + +Work top to bottom. An item is done when its **Done when** holds and its +commit is pushed; mark it `[x]` with the date and commit. Update +[backend-parity.md](backend-parity.md) and the roadmap's status in the same +commit as the change they describe. + +## Rules + +- **Unforeseen work gets added here, not done silently.** When an item turns + out to need something the plan did not foresee, add it as a new item in + its place in the order, with a *Why* line naming what was found. A blocker + goes directly above the item it blocks. +- **Duplicate authorities** found on the way: refactor if small, otherwise + add to the roadmap's *Duplicate authorities* table as deferred. Never make + one worse. A blocker is fixed whatever its size. +- **Verified on k3s, not on fakes.** Every item that changes KubeAI + behaviour adds or extends a step in `dev/kubeai_e2e.sh` and passes it. +- **The queue is not done until item 9 passes.** Do not stop at the last + feature item. + +## Items + +### 1. [x] P1a: KubeAI acquires go through admission + +Done 2026-09-25, `d6a19c4`. + +### 2. [ ] P1b: delete the legacy acquire branch + +`MemoryBackend`, `NullBackend` and the test fakes (queue, lock, serialised +publication) get a trivial `residency` and `preview`. Then the +non-admission branches, `_render`'s one-shot `converge(desired)` fallback +and `_admission_mode()` go. + +**Done when:** `_admission_mode` does not exist; the full suite passes; +queue-semantics tests still assert the same behaviour. + +### 3. [ ] P2: day-2 commands and the TUI through the backend + +`instances()` and `stream_logs(target, *, follow, tail)` on both backends; +`ps`, `logs`, `stack up` / `stack down` use them, the raw compose form +moves under `stack compose …`; the TUI's docker pane, log follower and +Up / Down go through the backend; `measure` works on KubeAI if vLLM's +memory line reaches the pod log. + +**Done when:** on k3s, `infer-stack ps` and `infer-stack logs ` work +with the same output shape as on Compose, and the TUI follows a Model's log +and lists its pod (checked in a real terminal, not only in tests). + +### 4. [ ] P3: gateway feature parity + +`routes` and `secrets rotate` resolve the backend's gateway instead of +checking its kind; the KubeAI gateway project honours `ui`, +`reverse_proxy` and `dynamic_routing`. Move the KubeAI gateway's approval +into the admission preview (deferred duplicate). + +**Done when:** `routes list` works on KubeAI; Open WebUI answers in front of +the cluster; an e2e step acquires the same model `--dedicated` twice under +dynamic routing and gets two Models and two routes. + +### 5. [ ] P6: one test surface + +Parametrize the controller's acquire scenarios over the Memory, +fake-Compose and fake-KubeAI backends; `tests/test_parity.py` runs each +*same* row of the parity matrix on both fakes. + +**Done when:** every *same* row has a parity test, and the CI suite runs it. + +### 6. [ ] P4: placement from node labels (verified with faked labels) + +`catalog suggest` on KubeAI proposes `resourceProfiles` from +`nvidia.com/gpu.product` / `.memory` node labels; `min_vram_gib` picks the +smallest fitting profile when an endpoint names none. + +**Done when:** on k3s with hand-set labels for two GPU sizes (CPU-backed +profiles), a catalog with `min_vram_gib` and no `resource_profile` lands on +the right profile; `dev/handover/p4_gpu_labels.sh` exists for one run on a +real GPU node. + +### 7. [ ] P5: in-cluster gateway and a second node + +Render the gateway as a Deployment + Service behind an ingress; `secrets +rotate` becomes a Secret update and a rollout; `doctor` checks the ingress. +Try a second k3s agent in a Docker container on the guest; write the +"add a workstation" runbook into `kubeai-backend.md`. + +**Done when:** on one node, a card reaches a Model through the in-cluster +gateway and `secrets rotate` works. If the simulated second node fits in the +VM, a Model pinned to it by node selector is served through the same +gateway. `dev/handover/p5_two_hosts.sh` exists for one run across two real +machines. + +### 8. [ ] The README's Compose sections + +About 33 references to verbs that no longer exist (`setup`, `up -d`, +`switch`, `describe-profile`, `smoke-test`, `wait-ready`, `diagnose`). + +**Done when:** every command in the README runs as written against the +current CLI, and `grep` finds none of those verbs. + +### 9. [ ] UX audit loop: do not stop without a passing audit + +Polishing the UX can take hours; that is expected, and this item is the +reason the queue exists. Loop: + +1. Audit (below) on **both** backends, from a fresh data root. +2. Write every finding as a sub-item here, with the exact command and + output. +3. Fix them, one commit each, tests added where the behaviour is testable. +4. Repeat from 1. The audit passes when a full pass finds nothing new. + +The audit covers: + +- **First run.** Following the README and `kubeai-backend.md` literally + from nothing reaches a working request on each backend. +- **Every verb** (`acquire`, `release`, `wait`, `evict`, `gc`, `clean`, + `renew`, `run`, `test`, `status`, `leases`, `ps`, `logs`, `routes`, + `doctor`, `config`, `catalog`, `secrets`, `stack`): `--help` is accurate + and its examples run; output has the same shape on both backends. +- **Every failure.** Each error names the cause and the next command to run; + none mentions Docker or Compose on KubeAI, or `kubectl` on Compose; no + traceback reaches the user for an expected condition (port in use, no + cluster, unrenderable endpoint, refused approval). +- **The TUI**, in a real terminal at 80x24 and wide: every pane, tab, key + and button works on both backends; every action logs its CLI equivalent; + nothing is empty without saying why. +- **Consistency.** The same thing has the same name in the CLI, the TUI and + the docs. + +Seed findings, already known: + +- [ ] `r` does not reload catalogs, and edits to a catalog file are not + picked up until restart. +- [ ] `infer-stack logs` should accept a container name, a prefixed name or + an endpoint name. +- [ ] `status` shows STALE during an apply instead of "apply in progress". + +### 10. [ ] Handover + +Summarize for the operator: what was verified here, and the two handover +scripts (items 6 and 7) with what each run proves. From 184289cae8f03ff5a57d4737b1c2a717bc02c460 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 11:30:21 -0400 Subject: [PATCH 025/141] P1b: one acquire path; the pre-admission branch is gone SimpleAdmission gives a realize/teardown backend (dry-run, tests) the admission surface: residency from observe(), a no-GPU preview, and converge/apply over realize/teardown. Fakes that emulate capacity override plan(); one that emulates a render failure overrides refuse(). With every backend on it, the controller loses _admission_mode(), the legacy acquire, render, renew and publish branches, the one-shot converge(desired) fallback, desired_deployments() and the placement-scope recovery render. Duplicate authorities fixed on the way: - unknown residency: _admit admitted requests needing no new GPU, whose render then failed after commit; now nothing is admitted. - the desired set: routes prune used desired_deployments() instead of the admission view the render uses. Rollback after a commit keeps its tests, on a fake whose render refuses what its preview admitted. Verified: full suite, compose e2e (non-GPU tiers) and the kubeai e2e on k3s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 10 + docs/planning/backend-parity-roadmap.md | 25 +- docs/queue.md | 4 +- infer_stack/cli/commands_leasing.py | 38 +- infer_stack/leasing/backend.py | 116 ++++- infer_stack/leasing/compose.py | 22 - infer_stack/leasing/controller.py | 457 +++++-------------- infer_stack/leasing/models.py | 4 +- tests/test_cli_leasing.py | 2 +- tests/test_leasing_admission.py | 11 +- tests/test_leasing_compose.py | 13 +- tests/test_leasing_controller_lock.py | 9 +- tests/test_leasing_controller_queue.py | 39 +- tests/test_leasing_kubeai.py | 1 - tests/test_leasing_profile.py | 19 +- tests/test_leasing_serialised_publication.py | 103 +++-- 16 files changed, 385 insertions(+), 488 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7acb9fd..f27c8d3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### One acquire path + +The controller's pre-admission branch is gone: every backend, including the +dry-run backend, acquires, renders, renews and publishes through admission. +A backend with only `realize` / `teardown` gets the admission surface from +`SimpleAdmission`. Two behaviours changed with it. While residency cannot +be read, no acquire is admitted; before, a request needing no new GPU was +admitted and then failed at its render, after the lease was committed. And +`routes prune` computes the desired set with the same view the render uses. + ### KubeAI acquires go through admission, as Compose's do An acquire on the KubeAI backend is now previewed in memory and commits its diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 268f8d76..f1aa3530 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,10 +1,9 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **P1a done** 2026-09-25 · P1b–P5 not started · -P6 is ongoing. +`dev/backend-unification` · **P1 done** 2026-09-26 · P2–P5 not started · +P6 is ongoing. Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). -**Execution order:** [../queue.md](../queue.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the 2026-09-24 audit (`dev/tmp/plan-backend-unification-2026-09-24.md`). @@ -91,11 +90,18 @@ gained the step "an unrenderable endpoint is refused before anything is written" (a `--queue` acquire, no lease, no Model), and still passes the make-room step, which now runs on the admission path. -**P1b. Delete the legacy branch** (with P6). +**P1b. Delete the legacy branch.** `MemoryBackend` / `NullBackend` and the test fakes get a trivial `residency` and `preview`; then the non-admission branches and -`_admission_mode()` go. Until then the legacy path serves only test fakes -and `realize`/`teardown` backends, and no new code may depend on it. +`_admission_mode()` go. + +**P1b done 2026-09-26.** `SimpleAdmission` (in `leasing/backend.py`) gives +a backend with only `realize` / `teardown` the admission surface; a fake +that emulates capacity overrides its `plan`, one that emulates a render +failure its `refuse`. The controller has one acquire, render, renew and +publish path. Rollback after a commit is still reachable (the runtime can +change between preview and render) and keeps its tests, on a fake whose +render refuses what its preview admitted. **Exit:** P1a: `_admission_mode()` is true for both real backends; a render failure on KubeAI rolls the lease back before any `kubectl`; e2e passes. @@ -193,8 +199,11 @@ worse. A blocker is fixed whatever its size. | "this backend allocates GPUs" | inferred from which methods a backend has | **fixed** (P1a): `allocates_gpus()` in `leasing/backend.py`, read by the controller and the CLI | | approval digest and pre-approval | a copy inside `ComposeBackend` | **fixed** (P1a): `ConvergeScaffold`, shared | | KubeAI render vs its plan | `converge` rendered inline | **fixed** (P1a): one `_render_documents` behind `converge`, `preview` and `plan_on_idle_host` | -| the acquire path | admission and a legacy branch in five places | deferred to P1b | -| `_render`'s one-shot `converge(desired)` fallback, which applies | a second render contract for old backends | deferred to P1b: it goes with the legacy branch | +| the acquire path | admission and a legacy branch in five places | **fixed** (P1b) | +| `_render`'s one-shot `converge(desired)` fallback, which applies | a second render contract for old backends | **fixed** (P1b): removed | +| "can anything be admitted while residency is unknown" | `_admit` admitted requests needing no new GPU; the render after the commit then failed without residency | **fixed** (P1b): nothing is admitted, and nothing is committed | +| the desired set | `desired_deployments()` beside the admission view; `routes prune` used the former | **fixed** (P1b): one view, `_admission_view` | +| a crashed acquire's placement scope | recorded in the marker and re-applied by a recovery render, although admission never records one | **fixed** (P1b): a stale scope is dropped | | the KubeAI gateway's approval | the gateway project asks its own diff approval at render, after the lease commits, not in the admission preview | deferred to P3. Same result under `--yes`; interactively, a declined gateway change rolls the lease back after the commit | ## Not in scope diff --git a/docs/queue.md b/docs/queue.md index 08f8fa05..84ebcaed 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -30,7 +30,9 @@ commit as the change they describe. Done 2026-09-25, `d6a19c4`. -### 2. [ ] P1b: delete the legacy acquire branch +### 2. [x] P1b: delete the legacy acquire branch + +Done 2026-09-26. `MemoryBackend`, `NullBackend` and the test fakes (queue, lock, serialised publication) get a trivial `residency` and `preview`. Then the diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index cce3120d..71507813 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1906,28 +1906,19 @@ def _placement_view(controller): if not allocates_gpus(backend): return observed, assignments # the cluster places; no GPU indices - if controller._admission_mode(): - # Committed allocations, and idle residents' physical GPUs; the - # legacy planner view would show placements admission would not make. - _, deployments = controller.ledger.status(virtual_expiry=True) + # Committed allocations, and idle residents' physical GPUs. + _, deployments = controller.ledger.status(virtual_expiry=True) + for g in deployments: + if g.assigned_gpus is not None: + assignments[g.id] = list(g.assigned_gpus) + try: + residency = backend.residency() for g in deployments: - if g.assigned_gpus is not None: - assignments[g.id] = list(g.assigned_gpus) - try: - residency = backend.residency() - for g in deployments: - c = residency.resident(g.id) - if g.id not in assignments and c is not None: - assignments[g.id] = list(c.gpus) - except Exception: # noqa: BLE001 - pass - return observed, assignments - plan = getattr(backend, 'plan', None) - if plan is not None: - try: - assignments = dict(plan(controller.desired_deployments()).assignments) - except Exception: # noqa: BLE001 - pass + c = residency.resident(g.id) + if g.id not in assignments and c is not None: + assignments[g.id] = list(c.gpus) + except Exception: # noqa: BLE001 + pass return observed, assignments @@ -2602,8 +2593,9 @@ def main(cls, argv=True, **kwargs): backend = _require_compose_backend(controller) def prune_plan() -> tuple[dict, dict, list[str]]: - desired = controller.desired_deployments() - plan = backend.plan(desired) + # The desired set exactly as the next render sees it. + desired, inputs = controller._admission_view(backend.residency()) + plan = backend.plan(desired, inputs) keep: dict = {} if backend.catalog is not None: keep.update(_registry_incoming_from_catalog(backend.catalog)) diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 28356ddc..9755130c 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -141,15 +141,16 @@ class ConvergeBackend(Backend, Protocol): * ``last_assignments`` — deployment id -> GPU indices (empty for backends where the cluster schedules). - Backends without this surface fall back to the per-deployment - ``realize``/``teardown`` path in :meth:`Controller._render`. + A backend with only ``realize``/``teardown`` gets this surface from + :class:`SimpleAdmission`. """ last_unplaced: set[str] last_errors: list[str] last_assignments: dict[str, list[int]] - def converge(self, desired: list[Deployment], *, apply: bool = True): + def converge(self, desired: list[Deployment], *, apply: bool = True, + placement: Any = None): """Render the desired set to backend state; optionally apply it.""" ... @@ -165,10 +166,11 @@ def apply(self) -> bool | None: @runtime_checkable class AdmissionBackend(ConvergeBackend, Protocol): - """The surface admission mode uses (see ``Controller._admission_mode``). + """The surface the controller drives: every acquire goes through admission. Both real backends have it: strict residency and an in-memory - ``preview``. Compose also takes the stable-address network and the + ``preview``; :class:`SimpleAdmission` supplies it for backends that + neither place nor inspect (the dry-run and test backends). Compose also takes the stable-address network and the container-adoption table from the controller (``network``, ``on_addresses``, ``adopted``); those are Compose-only, and the controller hands them over only to a backend that declares them. The @@ -331,7 +333,107 @@ def _approve_changes(self, planned: dict) -> None: ) -class MemoryBackend: +@dataclass +class RenderPreview: + """The render half of a :meth:`SimpleAdmission.preview`: what it refused.""" + + unrenderable: set[str] + errors: list[str] + + +class SimpleAdmission: + """The admission surface for a backend that neither places nor inspects. + + For in-process backends (the dry-run and test backends): they allocate no + GPUs, their :meth:`residency` is what :meth:`observe` reports (each + deployment one warm instance), and converge/apply is ``realize`` / + ``teardown`` over the rendered set. A subclass that emulates capacity + overrides :meth:`plan`; one that emulates render failures overrides + :meth:`refuse`. Everything else goes through the one admission path the + real backends use. + """ + + allocates_gpus = False + last_preview_digest: str | None = None + + def plan(self, desired: list[Deployment], placement: Any = None): + """Which of ``desired`` fit; here, all of them, on no GPU.""" + from .placement import GpuPlan + + return GpuPlan(assignments={g.id: [] for g in desired}) + + def refuse(self, desired: list[Deployment]) -> dict[str, str]: + """``{deployment id: reason}`` for what cannot be rendered; none here.""" + return {} + + def residency(self): + from .residency import Container, Residency + + return Residency(by_deployment={ + gid: (Container(container_id=gid, deployment_id=gid, + state='running', labelled=True),) + for gid in sorted(self.observe()) + }) + + def _preview(self, desired: list[Deployment], placement: Any = None): + desired = list(desired) + plan = self.plan(desired, placement) + refused = self.refuse(desired) + return plan, RenderPreview(set(refused), [f'{g}: {why}' for g, why in refused.items()]) + + def preview(self, desired: list[Deployment], placement: Any = None, *, + approve: bool = False): + plan, rendered = self._preview(desired, placement) + self.last_preview_digest = repr(sorted( + (gid, list(gpus)) for gid, gpus in plan.assignments.items() + if gid not in rendered.unrenderable)) + return plan, rendered + + def converge(self, desired: list[Deployment], *, apply: bool = True, + placement: Any = None): + """Record the renderable, placed part of ``desired``; ``apply`` realizes it.""" + desired = list(desired) + plan, rendered = self._preview(desired, placement) + self.last_errors = list(plan.errors) + list(rendered.errors) + self.last_unplaced = { + g.id for g in desired + if g.id not in plan.assignments or g.id in rendered.unrenderable} + self.last_assignments = (dict(plan.assignments) + if self.allocates_gpus else {}) + self._rendered = {g.id: g for g in desired if g.id not in self.last_unplaced} + known = getattr(self, '_known', {}) + known.update({g.id: g for g in desired}) + self._known = known + if apply: + self.apply() + + def apply(self) -> None: + """Realize what the last converge rendered; tear down the rest.""" + rendered = getattr(self, '_rendered', {}) + for gid in sorted(self.observe() - set(rendered)): + deployment = getattr(self, '_known', {}).get(gid) + if deployment is not None: + self.teardown(deployment) + for gid, deployment in rendered.items(): + if gid not in self.observe(): + self.realize(deployment) + + # Set by converge (the controller reads them with a default before then). + last_unplaced: set[str] + last_errors: list[str] + last_assignments: dict[str, list[int]] + + def observe(self) -> set[str]: # pragma: no cover - provided by the backend + raise NotImplementedError + + def realize(self, deployment: Deployment) -> None: # pragma: no cover + raise NotImplementedError + + def teardown(self, deployment: Deployment) -> None: # pragma: no cover + raise NotImplementedError + + +class MemoryBackend(SimpleAdmission): """In-memory backend that records calls and has configurable readiness. Not a real serving backend — it never starts a process. It exists so the @@ -395,7 +497,7 @@ def set_ready( self.ready_overrides[(deployment_id, endpoint)] = ready -class NullBackend: +class NullBackend(SimpleAdmission): """A no-op backend that serves nothing — for ``--dry-run`` and ``leases``. It never starts a process. ``observe`` returns the empty set, so the diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index ff209d2c..1ea628bd 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -2141,28 +2141,6 @@ def use_profile(self, profile: dict[str, Any]) -> None: sources = profile.get('catalogs') or [] self.catalog = CatalogUnion.from_sources(sources) if sources else None - def placement_context(self) -> dict[str, Any] | None: - """This caller's admission scope, stored with a pending acquire.""" - if self.allowed_gpus is None: - return None - return {'allowed_gpus': list(self.allowed_gpus)} - - def placement_scope(self, context: dict[str, Any] | None): - """Temporarily render with another caller's admission scope.""" - import contextlib - - @contextlib.contextmanager - def scope(): - saved = self.allowed_gpus - if context and 'allowed_gpus' in context: - self.allowed_gpus = context['allowed_gpus'] - try: - yield - finally: - self.allowed_gpus = saved - - return scope() - def validate_requests(self, requests) -> None: """Refuse requests the published catalog union does not define identically.""" from .profile import validate_requests_against diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 7e0c29f6..8d5c3a36 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -419,19 +419,6 @@ def _global_lock(self): # -- reconcile --------------------------------------------------------- - def desired_deployments(self) -> list[Deployment]: - """Deployments that should currently be running.""" - _, deployments = self.ledger.status() - desired: list[Deployment] = [] - for deployment in deployments: - if deployment.state == DeploymentState.LIVE: - desired.append(deployment) - elif deployment.state == DeploymentState.IDLE: - policy = deployment.spec.get('reclaim', self.reclaim_default) - if policy == KEEP_WARM: - desired.append(deployment) - return desired - def _infeasible_alone(self, deployments, requested: set) -> dict: """Which of this lease's deployments cannot be placed even on an idle host. @@ -469,77 +456,36 @@ def _infeasible_alone(self, deployments, requested: set) -> dict: def _render(self) -> ReconcileResult: """Render the desired state to disk WITHOUT the slow apply. - The caller MUST hold :meth:`_global_lock`. For a converge-style backend - (Compose) this writes the compose project (placement + files) but does - not ``docker compose up`` -- that is :meth:`_apply_pending`, under the - same lock hold. A per-deployment ``realize``/``teardown`` backend has no - such split, so it realizes here; those backends expose no ``apply``. + The caller MUST hold :meth:`_global_lock`. This writes the backend's + rendered state (the compose project, the Model manifests) but does not + bring it up -- that is :meth:`_apply_pending`, under the same lock hold. """ self.ledger.sweep() - placement = None - if self._admission_mode(): - # Residency decides which idle keep-warm deployments are candidates - # at all; unknown residency fails the render (the change stays - # pending) rather than guessing which warm models exist. - residency = self._admitting.residency() - self._backfill_allocations(residency) - self._prepare_network() - desired, placement = self._admission_view(residency) - if hasattr(self.backend, 'adopted'): - self._admitting.adopted = self._prune_adopted(residency) - else: - desired = self.desired_deployments() - # Bound once rather than probed with hasattr: the capability check is - # the same, and the bound method keeps its type instead of narrowing - # to `object` the way an attribute reached through hasattr does. - converge = getattr(self.backend, 'converge', None) - if converge is not None: - before = set(self.backend.observe()) - try: - if placement is not None: - converge(desired, apply=False, placement=placement) - else: - converge(desired, apply=False) - except TypeError: - # Legacy converge(desired) with no apply kwarg renders+applies - # in one shot (no separate apply()); accept that here. - converge(desired) - after = set(self.backend.observe()) - rec = ReconcileResult( - realized=sorted(after - before), - torn_down=sorted(before - after), - unplaced=sorted( - getattr(self.backend, 'last_unplaced', ()) or () - ), - placement_errors=list( - getattr(self.backend, 'last_errors', ()) or () - ), - assignments=dict( - getattr(self.backend, 'last_assignments', {}) or {} - ), - applied=False, - displaced=list(getattr(self.backend, 'last_displaced', ()) or ()), - degraded=list(getattr(self.backend, 'last_degraded', ()) or ()), - ) - if placement is not None and hasattr(self.backend, 'adopted'): - self._adopt_existing(residency) - return rec - desired_ids = {g.id for g in desired} - actual = self.backend.observe() - result = ReconcileResult() - for deployment in desired: - if deployment.id not in actual: - self.backend.realize(deployment) - result.realized.append(deployment.id) - stale = actual - desired_ids - if stale: - by_id = {g.id: g for g in self.ledger.status()[1]} - for gid in stale: - deployment = by_id.get(gid) - if deployment is not None: - self.backend.teardown(deployment) - result.torn_down.append(gid) - return result + # Residency decides which idle keep-warm deployments are candidates at + # all; unknown residency fails the render (the change stays pending) + # rather than guessing which warm models exist. + residency = self._admitting.residency() + self._backfill_allocations(residency) + self._prepare_network() + desired, placement = self._admission_view(residency) + if hasattr(self.backend, 'adopted'): + self._admitting.adopted = self._prune_adopted(residency) + before = set(self.backend.observe()) + self._admitting.converge(desired, apply=False, placement=placement) + after = set(self.backend.observe()) + rec = ReconcileResult( + realized=sorted(after - before), + torn_down=sorted(before - after), + unplaced=sorted(getattr(self.backend, 'last_unplaced', ()) or ()), + placement_errors=list(getattr(self.backend, 'last_errors', ()) or ()), + assignments=dict(getattr(self.backend, 'last_assignments', {}) or {}), + applied=False, + displaced=list(getattr(self.backend, 'last_displaced', ()) or ()), + degraded=list(getattr(self.backend, 'last_degraded', ()) or ()), + ) + if hasattr(self.backend, 'adopted'): + self._adopt_existing(residency) + return rec # -- serialised publication -------------------------------------------- # @@ -555,8 +501,7 @@ def _render(self) -> ReconcileResult: # -- admission (plan steps P5, P6, P9) ------------------------------------ # - # Backends with strict residency and an in-memory preview (Compose and - # KubeAI) get admission semantics: + # Every acquire, renew and publication goes through admission: # * a LIVE deployment holds a committed allocation (assigned_gpus); # * an IDLE keep-warm deployment is only an optional candidate, and only # while it is uniquely resident; it yields its GPUs to demand and is @@ -566,8 +511,8 @@ def _render(self) -> ReconcileResult: # Where the cluster schedules (KubeAI, ``allocates_gpus`` false) every # deployment commits an empty allocation: admission then decides only # renderability, and a Pending pod is a wait reason, not an unplaced - # error. Test fakes without residency/preview keep the previous path - # (roadmap P1b removes it). + # error. Backends that neither place nor inspect (dry-run, tests) get the + # same surface from SimpleAdmission. def _stored_state(self, lease_id: str): """A lease's state as stored (not virtually expired), or ``None``.""" @@ -578,17 +523,11 @@ def _stored_state(self, lease_id: str): def _admitting(self) -> AdmissionBackend: """``self.backend`` as the admission surface. - Only for code that runs after :meth:`_admission_mode` (or an - equivalent capability check) said the backend has it. + Every backend the controller drives has it (the real ones, and + :class:`~infer_stack.leasing.backend.SimpleAdmission` for the rest). """ return cast(AdmissionBackend, self.backend) - def _admission_mode(self) -> bool: - return all( - callable(getattr(self.backend, name, None)) - for name in ('residency', 'preview', 'converge') - ) - def _admission_view( self, residency, *, overlay=None, virtual_expiry: bool = False ): @@ -753,11 +692,10 @@ def observe_state(self) -> dict: except Exception: # noqa: BLE001 - a view must always render sidecar = {} residency, residency_error = None, None - if callable(getattr(self.backend, 'residency', None)): - try: - residency = self._admitting.residency() - except ResidencyUnknown as ex: - residency_error = str(ex) + try: + residency = self._admitting.residency() + except ResidencyUnknown as ex: + residency_error = str(ex) degraded = set(sidecar.get('degraded') or ()) displaced = set(sidecar.get('displaced') or ()) rows = [] @@ -773,7 +711,7 @@ def observe_state(self) -> dict: elif g.id in displaced and g.state == DeploymentState.IDLE: condition = 'displaced' elif (g.state == DeploymentState.LIVE and g.assigned_gpus is None - and self._admission_mode() and residency is not None + and residency is not None and residency.resident(g.id) is None): condition = 'unresolved' elif residency is not None: @@ -818,7 +756,7 @@ def _prune_adopted(self, residency) -> dict: def _adopt_existing(self, residency) -> None: """One-time migration: adopt project containers from before ownership labels. - Runs after the first admission-mode render on a ledger. A container + Runs after the first render on a ledger. A container without infer-stack's service and fingerprint labels is adopted, with the fingerprint of this render, if it is infrastructure the render still has, or a deployment container whose deployment is LIVE on the @@ -920,10 +858,13 @@ def _admit(self, overlay, residency): adopted[gid] = list(resident.gpus) # IDLE->LIVE keeps its GPUs else: need.append(gid) - if residency is None and (need or adopted): + if residency is None: + # The render after the commit needs residency too, so nothing can + # be admitted without it; refusing here commits nothing. return {}, [ - 'Docker residency is unknown, so only requests that need no new ' - 'GPU are admitted; retry when `docker ps` works' + 'what is running cannot be read right now (the runtime did not ' + 'answer), so nothing is admitted; retry when `infer-stack leases` ' + 'shows no residency error' ] if need: unresolved = self._unresolved_allocations(exclude=set(overlay.deployments)) @@ -1205,8 +1146,7 @@ def _sync_profile(self, *, create: bool) -> None: self._applied_profile = stored def _mark_pending( - self, *, apply: bool, placement_context: dict | None = None, - create_profile: bool = True, + self, *, apply: bool, create_profile: bool = True, ) -> dict: """Record that desired state is about to change (caller holds the lock). @@ -1215,31 +1155,16 @@ def _mark_pending( ever turns ``apply_requested`` on: a staged change never cancels an apply already requested (promotion, see the plan's D23). - If an earlier acquire died between committing and its first render, its - placement scope is still in the marker: render once with that scope - first, so its deployment is placed where that caller was allowed. + A placement scope left in the marker by pre-admission code is dropped: + a row that code committed without an allocation stays unresolved and is + never placed, so no render needs that caller's scope. """ if create_profile: self._sync_profile(create=True) current = self.ledger.publication_pending() if current and current.get('placement_context'): - self._render_in_scope(current['placement_context']) - return self.ledger.mark_publication_pending( - apply_requested=apply, placement_context=placement_context, - ) - - def _render_in_scope(self, context: dict) -> None: - """Render with another caller's admission scope, then forget the scope. - - The scope is cleared only after a render that succeeded (and so pinned - the placement). A declined or failed render propagates and keeps it, - so no later caller can place that deployment within its own scope. - """ - scope = getattr(self.backend, 'placement_scope', None) - if scope is not None: - with scope(context): - self._render() - self.ledger.clear_placement_context() + self.ledger.clear_placement_context() + return self.ledger.mark_publication_pending(apply_requested=apply) def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: """Apply the last render if the marker requests it; clear on success. @@ -1254,12 +1179,7 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: marker = self.ledger.publication_pending() if marker is None: return rec - apply_fn = getattr(self.backend, 'apply', None) - if apply_fn is None: - # realize/teardown backends applied during the render itself. - self.ledger.clear_publication_pending(marker['version']) - rec.publication_pending = False - return rec + apply_fn = self._admitting.apply if not marker['apply_requested']: rec.publication_pending = True # staged; never applied here return rec @@ -1467,19 +1387,14 @@ def _make_room(self) -> str | None: def _never_ran(self, deployment_ids: list[str]) -> list[str]: """Which of these deployments definitely have no container at all. - Uses strict residency where the backend has it: a deployment with any - container, in any state, is kept, and if Docker cannot be read nothing - is reported (so nothing warm is ever evicted on a failed look). Backends - without residency fall back to ``observe()``. + Uses strict residency: a deployment with any container, in any state, + is kept, and if the runtime cannot be read nothing is reported (so + nothing warm is ever evicted on a failed look). """ - residency = getattr(self.backend, 'residency', None) - if residency is None: - running = set(self.backend.observe()) - return [gid for gid in deployment_ids if gid not in running] from .residency import ResidencyUnknown try: - snap = residency() + snap = self._admitting.residency() except ResidencyUnknown: return [] return [gid for gid in deployment_ids if not snap.containers(gid)] @@ -1593,106 +1508,12 @@ def acquire( grabs. Head-of-line GPU reservation is a follow-up; for the small-fleet case (few GPUs, rare multi-GPU jobs) plain queueing is sufficient. """ - from .backend import ConvergeAborted, PlacementError - - if self._admission_mode(): - result, rec = self._acquire_by_admission( - owner, requests, ttl_seconds=ttl_seconds, apply=apply, - wait_for_placement=wait_for_placement, - placement_timeout=timeout if placement_timeout is None else placement_timeout, - placement_interval=interval if placement_interval is None else placement_interval, - ) - return self._finish_acquire(result, rec, apply=apply, wait=wait, - timeout=timeout, interval=interval) - - # Intent, ledger write, render and (once placed) apply all under one lock - # hold, so a second caller blocks before touching sqlite and no render - # can change the files this apply reads. The readiness wait and the - # admission-queue sleep stay OUTSIDE the lock. - with self._global_lock(): - self._sync_profile(create=True) - stored_profile = self.ledger.profile() - candidate_profile = self._acquire_profile_candidate() - if candidate_profile is not None: - self._use_profile_candidate(candidate_profile) - try: - validate = getattr(self.backend, 'validate_requests', None) - if validate is not None: - validate(requests) # before desired state is written - except BaseException: - if candidate_profile is not None: - self._restore_stored_profile(stored_profile) - raise - if candidate_profile is not None: - # The user catalog is authoritative. Persist the compatible - # recovery inputs now; the desired-state marker belongs to the - # acquire immediately below, not to this snapshot refresh. - self._commit_profile_candidate(candidate_profile) - context = getattr(self.backend, 'placement_context', lambda: None)() - self._mark_pending(apply=apply, placement_context=context) - result = self.ledger.acquire( - owner, requests, ttl_seconds=ttl_seconds - ) - try: - try: - rec = self._render() - finally: - # Rendered (placement pinned) or about to roll back: either - # way a recovery no longer needs this caller's scope. - if context is not None: - self.ledger.clear_placement_context() - except ConvergeAborted: - # The operator declined the compose changes -- don't leave the - # just-created lease dangling in the ledger. - self._rollback_acquire(result.lease.id, apply=apply) - raise - # If a deployment this lease just requested could not be placed (e.g. no - # free GPU), either queue for one (wait_for_placement) or -- the default -- - # roll the lease back and report the planner's reason, so the deployment - # never lingers as a phantom ``live`` with nothing behind it. - requested = {g.id for g in result.deployments} - unplaced = requested & set(rec.unplaced) - if not unplaced: - rec = self._apply_admitted(rec, result.lease.id, apply=apply) - if unplaced and wait_for_placement and apply: - # Never queue for capacity that cannot exist. Re-plan this lease's - # deployments ALONE on an idle host: if they do not fit there, no - # amount of waiting will help, and waiting is actively harmful -- - # the lease holds whatever it did place for the whole timeout, so a - # request that was never satisfiable can block ones that are. - # - # Only the aggregate case needs this. A single deployment too large - # for any card is already caught by the planner's permanent branch; - # what is missed is a lease whose deployments cannot fit TOGETHER, - # e.g. a 4-GPU model plus a 1-GPU extractor on a 4-GPU host. - infeasible = self._infeasible_alone(result.deployments, requested) - if infeasible: - self._rollback_acquire(result.lease.id, apply=apply) - raise PlacementError(sorted(infeasible.keys()), - sorted(infeasible.values())) - p_timeout = timeout if placement_timeout is None else placement_timeout - p_interval = ( - interval if placement_interval is None else placement_interval - ) - deadline = self.clock() + p_timeout - while unplaced and self.clock() < deadline: - self.sleep(p_interval) - # Re-render under the lock: each retry sweeps (reclaiming a crashed - # job's TTL-expired lease) and re-plans against the freed GPUs. - with self._global_lock(): - self._mark_pending(apply=True) # the render sweeps - rec = self._render() - unplaced = requested & set(rec.unplaced) - if not unplaced: - rec = self._apply_admitted(rec, result.lease.id, apply=True) - if unplaced: - self._rollback_acquire(result.lease.id, apply=apply) - reasons = [ - e - for e in rec.placement_errors - if any(e.startswith(gid) for gid in unplaced) - ] - raise PlacementError(sorted(unplaced), reasons) + result, rec = self._acquire_by_admission( + owner, requests, ttl_seconds=ttl_seconds, apply=apply, + wait_for_placement=wait_for_placement, + placement_timeout=timeout if placement_timeout is None else placement_timeout, + placement_interval=interval if placement_interval is None else placement_interval, + ) return self._finish_acquire(result, rec, apply=apply, wait=wait, timeout=timeout, interval=interval) @@ -1700,7 +1521,7 @@ def _acquire_by_admission( self, owner, requests, *, ttl_seconds, apply, wait_for_placement, placement_timeout, placement_interval, ): - """Admission-mode acquire: preview in memory, commit only if admissible. + """The acquire: preview in memory, commit only if admissible. Each attempt runs under the lock: sweep (itself a published mutation), observe residency, overlay the request on the ledger, and preview @@ -1750,7 +1571,6 @@ def _acquire_by_admission( self._commit_profile_candidate(candidate_profile) # Allocations are committed with the lease, so no placement # scope needs recording for recovery. - context = None self._mark_pending(apply=apply) try: result = self.ledger.acquire( @@ -1763,11 +1583,7 @@ def _acquire_by_admission( except AdmissionConflict: continue # the ledger moved; preview again try: - try: - rec = self._render() - finally: - if context is not None: - self.ledger.clear_placement_context() + rec = self._render() except ConvergeAborted: self._rollback_acquire(result.lease.id, apply=apply) raise @@ -1946,68 +1762,43 @@ def publish_profile(self, profile: dict) -> ReconcileResult: f'config publish needs a quiescent stack: {len(active)} active ' f'lease(s) ({", ".join(active[:3])}); release them first' ) - residency = getattr(self.backend, 'residency', None) - if residency is not None: - try: - snap = residency() - except ResidencyUnknown as ex: - raise ProfileMismatch( - f'config publish cannot confirm the stack is quiescent: {ex}' - ) from ex - running = sorted({c.deployment_id for c in snap.all_containers() - if c.deployment_id}) - if running: - raise ProfileMismatch( - 'config publish needs a quiescent stack: deployment ' - f'container(s) exist for {", ".join(running[:3])}; ' - '`infer-stack evict --all` first' - ) - if self._admission_mode(): - # A PURE preview first: the real render persists append-only - # state (route registry, addresses), which must not happen for - # a candidate whose publication has not committed. - previous = self._applied_profile - use(profile) - try: - residency = self._admitting.residency() - self._prepare_network() - desired, inputs = self._admission_view( - residency, virtual_expiry=True - ) - self._admitting.preview(desired, inputs, approve=True) - except BaseException: - if previous is not None: - use(previous) - raise - self.ledger.store.publish_profile( - profile, approved_digest=self._admitting.last_preview_digest) - self._profile_error = None - self._applied_profile = profile - self._invocation_profile = profile - self._profile_drift_warned = False - return self._publish() - # Backends without a preview (KubeAI): render, then commit. - # No implicit profile here: on a fresh ledger a declined preview - # must leave neither a profile nor a marker behind. - existed = self.ledger.publication_pending() is not None - marker = self._mark_pending(apply=True, create_profile=False) + try: + snap = self._admitting.residency() + except ResidencyUnknown as ex: + raise ProfileMismatch( + f'config publish cannot confirm the stack is quiescent: {ex}' + ) from ex + running = sorted({c.deployment_id for c in snap.all_containers() + if c.deployment_id}) + if running: + raise ProfileMismatch( + 'config publish needs a quiescent stack: deployment ' + f'container(s) exist for {", ".join(running[:3])}; ' + '`infer-stack evict --all` first' + ) + # A PURE preview first: the real render persists append-only + # state (route registry, addresses), which must not happen for + # a candidate whose publication has not committed. previous = self._applied_profile use(profile) try: - rec = self._render() + residency = self._admitting.residency() + self._prepare_network() + desired, inputs = self._admission_view( + residency, virtual_expiry=True + ) + self._admitting.preview(desired, inputs, approve=True) except BaseException: if previous is not None: use(previous) - if not existed: - self.ledger.clear_publication_pending(marker['version']) raise self.ledger.store.publish_profile( - profile, approved_digest=getattr(self.backend, 'last_planned_digest', None)) + profile, approved_digest=self._admitting.last_preview_digest) self._profile_error = None self._applied_profile = profile self._invocation_profile = profile self._profile_drift_warned = False - return self._apply_pending(rec) + return self._publish() def prune(self) -> tuple[int, int]: """Forget released/expired leases and stopped deployments, under the lock. @@ -2027,7 +1818,7 @@ def renew(self, lease_id: str, *, ttl_seconds: float | None) -> RenewOutcome: nothing. **Slow path, under the lock:** a deployment went IDLE, so the renew is a - desired-state change. In admission mode it is re-admitted: the IDLE + desired-state change, so it is re-admitted: the IDLE deployment adopts its resident GPUs, or is placed fresh; if neither is possible the renew fails with :class:`PlacementError` and writes nothing. The lease is re-validated as ACTIVE under the lock first. @@ -2038,52 +1829,36 @@ def renew(self, lease_id: str, *, ttl_seconds: float | None) -> RenewOutcome: if fast is not False: return RenewOutcome(fast, []) with self._global_lock(): - if self._admission_mode(): - lease = self.ledger.get_lease(lease_id) - if lease is None or lease.state != LeaseState.ACTIVE: - return RenewOutcome(None, []) - from .residency import ResidencyUnknown - - try: - residency = self._admitting.residency() - except ResidencyUnknown: - residency = None - overlay = self.ledger.plan_acquire([]) - for gid in dict.fromkeys(lease.deployment_ids): - deployment = self.ledger.get_deployment(gid) - if deployment is not None and deployment.state == DeploymentState.IDLE: - deployment.state = DeploymentState.LIVE - overlay.deployments[gid] = deployment - overlay.revived.append(gid) - if not overlay.revived: - return RenewOutcome( - self.ledger.renew(lease_id, ttl_seconds=ttl_seconds), []) - allocations, reasons = self._admit(overlay, residency) - if reasons: - raise PlacementError(list(overlay.revived), reasons) - self._mark_pending(apply=True) - if self._admission_digest: - self.ledger.mark_publication_pending( - apply_requested=True, approved_digest=self._admission_digest) - renewed = self.ledger.renew( - lease_id, ttl_seconds=ttl_seconds, allocations=allocations) - rec = self._publish() - return RenewOutcome(renewed, list(overlay.revived), rec) lease = self.ledger.get_lease(lease_id) - reviving = [] - if lease is not None and lease.state == LeaseState.ACTIVE: - for gid in dict.fromkeys(lease.deployment_ids): - deployment = self.ledger.get_deployment(gid) - if deployment is not None and deployment.state == DeploymentState.IDLE: - reviving.append(gid) - if not reviving: + if lease is None or lease.state != LeaseState.ACTIVE: + return RenewOutcome(None, []) + from .residency import ResidencyUnknown + + try: + residency = self._admitting.residency() + except ResidencyUnknown: + residency = None + overlay = self.ledger.plan_acquire([]) + for gid in dict.fromkeys(lease.deployment_ids): + deployment = self.ledger.get_deployment(gid) + if deployment is not None and deployment.state == DeploymentState.IDLE: + deployment.state = DeploymentState.LIVE + overlay.deployments[gid] = deployment + overlay.revived.append(gid) + if not overlay.revived: return RenewOutcome( - self.ledger.renew(lease_id, ttl_seconds=ttl_seconds), [], - ) + self.ledger.renew(lease_id, ttl_seconds=ttl_seconds), []) + allocations, reasons = self._admit(overlay, residency) + if reasons: + raise PlacementError(list(overlay.revived), reasons) self._mark_pending(apply=True) - renewed = self.ledger.renew(lease_id, ttl_seconds=ttl_seconds) + if self._admission_digest: + self.ledger.mark_publication_pending( + apply_requested=True, approved_digest=self._admission_digest) + renewed = self.ledger.renew( + lease_id, ttl_seconds=ttl_seconds, allocations=allocations) rec = self._publish() - return RenewOutcome(renewed, reviving, rec) + return RenewOutcome(renewed, list(overlay.revived), rec) def evict(self, deployment_ids: Iterable[str] | None = None) -> EvictOutcome: """Force-evict idle (released) deployments now, overriding keep-warm. diff --git a/infer_stack/leasing/models.py b/infer_stack/leasing/models.py index ab24d8c9..d7a98022 100644 --- a/infer_stack/leasing/models.py +++ b/infer_stack/leasing/models.py @@ -362,8 +362,8 @@ class Deployment: created_at: float updated_at: float demand: int = 0 - # The committed GPU allocation of a LIVE deployment (admission-mode - # backends). ``None`` when not LIVE, or LIVE but unresolved (a ledger from + # The committed GPU allocation of a LIVE deployment (empty where the + # cluster schedules). ``None`` when not LIVE, or LIVE but unresolved (a ledger from # before allocations existed). Cleared in the same transaction as any # transition out of LIVE. assigned_gpus: list[int] | None = None diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index d007f7c9..91bb3e03 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -985,7 +985,7 @@ def test_apply_exits_nonzero_while_publication_stays_pending(env, capsys, monkey from infer_stack.leasing.backend import MemoryBackend class RoutesNeverVerify(MemoryBackend): - def converge(self, desired, *, apply=True): + def converge(self, desired, *, apply=True, placement=None): self.last_unplaced, self.last_errors, self.last_assignments = [], [], {} def apply(self): diff --git a/tests/test_leasing_admission.py b/tests/test_leasing_admission.py index dbbb552f..2ebd2494 100644 --- a/tests/test_leasing_admission.py +++ b/tests/test_leasing_admission.py @@ -179,7 +179,9 @@ def preview(self, desired, placement=None, **kw): assert len(calls) == 2 and ledger.get_deployment(out.deployments[0].id).assigned_gpus == [0] -def test_unknown_residency_admits_only_resource_neutral_requests(tmp_path): # 35 +def test_unknown_residency_admits_nothing(tmp_path): # 35 + """The render after a commit needs residency, so admission refuses without + it, even for a request that needs no new GPU.""" from infer_stack.leasing.residency import ResidencyUnknown ledger, ctl, _ = make(tmp_path) @@ -189,12 +191,11 @@ def unknown(): raise ResidencyUnknown('docker ps failed') ctl.backend.residency = unknown - with pytest.raises(PlacementError, match='residency is unknown'): + with pytest.raises(PlacementError, match='cannot be read'): ctl.acquire('x', CAT.resolve_names(['two']), wait=False, apply=False) - # Coalescing onto the LIVE, allocated deployment needs no new GPU. Its render - # still needs residency, so stage it without rendering residency-dependent state. overlay = ledger.plan_acquire(CAT.resolve_names(['one'])) - assert ctl._admit(overlay, None) == ({}, []) + allocations, reasons = ctl._admit(overlay, None) + assert allocations == {} and reasons # -- P5: allocations and renew ---------------------------------------------------------- diff --git a/tests/test_leasing_compose.py b/tests/test_leasing_compose.py index 232c7537..25173891 100644 --- a/tests/test_leasing_compose.py +++ b/tests/test_leasing_compose.py @@ -1108,11 +1108,20 @@ def test_acquire_rolls_back_lease_on_decline(tmp_path): from infer_stack.leasing import EndpointRequest, LeaseState, vllm_structural from infer_stack.leasing.backend import ConvergeAborted - class DeclineBackend: + from infer_stack.leasing.backend import SimpleAdmission + + class DeclineBackend(SimpleAdmission): + """The operator declines the diff, which admission asks at preview.""" + def observe(self): return set() - def converge(self, desired): + def preview(self, desired, placement=None, *, approve=False): + if approve: + raise ConvergeAborted('declined') + return super().preview(desired, placement) + + def converge(self, desired, *, apply=True, placement=None): raise ConvergeAborted('declined') led = Ledger(SqliteStore(tmp_path / 'ledger.db')) diff --git a/tests/test_leasing_controller_lock.py b/tests/test_leasing_controller_lock.py index 264be200..74d10cb0 100644 --- a/tests/test_leasing_controller_lock.py +++ b/tests/test_leasing_controller_lock.py @@ -14,6 +14,7 @@ import pytest +from infer_stack.leasing.backend import SimpleAdmission from infer_stack.leasing import ( Controller, EndpointRequest, @@ -32,7 +33,7 @@ THREAD_TIMEOUT_S = 30 -class OverlapBackend: +class OverlapBackend(SimpleAdmission): """A converge-style backend that records max concurrent converges.""" def __init__(self) -> None: @@ -43,7 +44,7 @@ def __init__(self) -> None: self.last_errors: tuple = () self.last_assignments: dict = {} - def converge(self, desired, *, apply: bool = True) -> None: + def converge(self, desired, *, apply: bool = True, placement=None) -> None: with self._guard: self.active += 1 self.max_active = max(self.max_active, self.active) @@ -172,7 +173,7 @@ def test_created_lock_file_and_dir_are_group_writable(tmp_path): assert dmode & stat.S_ISGID, f'lock dir missing setgid: {oct(dmode)}' -class SharedOverlapBackend: +class SharedOverlapBackend(SimpleAdmission): """Like OverlapBackend but records overlap into a shared counter, so two *separate* controllers' converges can be compared.""" @@ -184,7 +185,7 @@ def __init__(self, shared: dict, guard: threading.Lock) -> None: self.last_assignments: dict = {} self._ids: set = set() - def converge(self, desired, *, apply: bool = True) -> None: + def converge(self, desired, *, apply: bool = True, placement=None) -> None: with self.guard: self.shared['active'] += 1 self.shared['max'] = max(self.shared['max'], self.shared['active']) diff --git a/tests/test_leasing_controller_queue.py b/tests/test_leasing_controller_queue.py index 833fb9df..2afa912e 100644 --- a/tests/test_leasing_controller_queue.py +++ b/tests/test_leasing_controller_queue.py @@ -19,7 +19,7 @@ SqliteStore, vllm_structural, ) -from infer_stack.leasing.backend import PlacementError, Readiness +from infer_stack.leasing.backend import PlacementError, Readiness, SimpleAdmission from infer_stack.leasing.placement import GpuPlan @@ -45,39 +45,32 @@ def factory(prefix: str) -> str: return factory -class BudgetBackend: +class BudgetBackend(SimpleAdmission): """Places up to ``budget`` deployments (1 slot each); the rest are unplaced. - A converge-style fake: ``converge`` recomputes the placed/unplaced split from - the desired set each call, so freeing demand (a release) lets a queued - deployment in on the next reconcile. + Capacity is its :meth:`plan`, which admission consults before committing + anything, so freeing demand (a release) lets a queued deployment in on + the next attempt. It allocates GPUs, so admission accounts for them. """ + allocates_gpus = True + def __init__(self, budget: int): self.budget = budget self.realized: dict[str, object] = {} # what is "running" (post-apply) - self._placed: dict[str, object] = {} # what the last render decided to run self.apply_calls = 0 - self.last_unplaced: list[str] = [] - self.last_errors: list[str] = [] - self.last_assignments: dict[str, list[int]] = {} - def converge(self, desired, apply: bool = True) -> None: + def plan(self, desired, placement=None): placed = list(desired)[: self.budget] - unplaced = list(desired)[self.budget :] - # Render always records the placed set; bringing it "up" is apply()'s job - # (the render/apply split). Legacy apply=True still realizes in one shot. - self._placed = {g.id: g for g in placed} - if apply: - self.realized = dict(self._placed) - self.last_assignments = {g.id: [i] for i, g in enumerate(placed)} - self.last_unplaced = [g.id for g in unplaced] - self.last_errors = [f'{g.id}: no free GPU' for g in unplaced] + return GpuPlan( + assignments={g.id: [i] for i, g in enumerate(placed)}, + errors=[f'{g.id}: no free GPU' for g in list(desired)[self.budget:]], + ) def apply(self) -> None: """Bring the last-rendered set 'up' (idempotent).""" self.apply_calls += 1 - self.realized = dict(self._placed) + super().apply() def observe(self) -> set[str]: return set(self.realized) @@ -85,11 +78,11 @@ def observe(self) -> set[str]: def probe_ready(self, deployment, endpoint) -> Readiness: return Readiness(deployment.id in self.realized, 'ok') - def realize(self, deployment) -> None: # unused by converge backends - pass + def realize(self, deployment) -> None: + self.realized[deployment.id] = deployment def teardown(self, deployment) -> None: - pass + self.realized.pop(deployment.id, None) def vreq(endpoint, *, reclaim='stop'): diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 19071644..b162318e 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -409,7 +409,6 @@ def test_acquire_missing_profile_commits_nothing(tmp_path): def test_kubeai_takes_the_admission_path(tmp_path): """One acquire path: KubeAI admits by preview and commits no GPUs.""" ctl, be, kubectl = make_controller(tmp_path) - assert ctl._admission_mode() out = ctl.acquire('alice', [_req('qwen')], wait=False) (deployment,) = out.deployments assert deployment.assigned_gpus == [] # committed, and empty diff --git a/tests/test_leasing_profile.py b/tests/test_leasing_profile.py index 1a1f5f8a..231fa82c 100644 --- a/tests/test_leasing_profile.py +++ b/tests/test_leasing_profile.py @@ -281,25 +281,16 @@ def fail(planned): assert ledger.publication_pending() is None -def test_a_declined_recovery_render_keeps_the_crashed_acquires_scope(tmp_path): - from infer_stack.leasing.backend import ConvergeAborted - +def test_a_stale_scope_from_pre_admission_code_is_dropped(tmp_path): + """No render needs a crashed legacy acquire's scope: its row stays + unresolved (see above), so the next change simply clears it.""" a = Catalog.from_dict(cat('alpha')) ledger, ctl = controller(tmp_path, catalog=a) ctl.gc() ledger.mark_publication_pending(apply_requested=True, placement_context={'allowed_gpus': [3]}) - ledger.acquire('x', a.resolve_names(['alpha'])) - _, ctl2 = controller(tmp_path, catalog=a, allowed_gpus=[0]) - - def decline(planned): - raise ConvergeAborted('no') - - ctl2.backend._approve_changes = decline - with pytest.raises(ConvergeAborted): - ctl2.gc() - assert ledger.publication_pending()['placement_context'] == {'allowed_gpus': [3]} - + ctl.acquire('x', a.resolve_names(['alpha']), wait=False, apply=False) + assert ledger.publication_pending()['placement_context'] is None def test_cli_catalog_edit_then_acquire_needs_no_publish_step(tmp_path, monkeypatch): from infer_stack.cli import commands_leasing as cl diff --git a/tests/test_leasing_serialised_publication.py b/tests/test_leasing_serialised_publication.py index fb3a378a..945b8a4c 100644 --- a/tests/test_leasing_serialised_publication.py +++ b/tests/test_leasing_serialised_publication.py @@ -25,7 +25,7 @@ SqliteStore, vllm_structural, ) -from infer_stack.leasing.backend import Readiness +from infer_stack.leasing.backend import Readiness, SimpleAdmission @@ -46,7 +46,7 @@ def _vreq(endpoint: str) -> EndpointRequest: ) -class SharedStackBackend: +class SharedStackBackend(SimpleAdmission): """Models the one shared compose project across separate controllers. ``converge(apply=False)`` (render) writes the desired union to shared @@ -63,7 +63,7 @@ def __init__(self, shared: dict, guard: threading.Lock, apply_sleep: float = 0.0 self.last_errors: list[str] = [] self.last_assignments: dict[str, list[int]] = {} - def converge(self, desired, *, apply: bool = True) -> None: + def converge(self, desired, *, apply: bool = True, placement=None) -> None: ids = {g.id for g in desired} with self.guard: self.shared['rendered'] = set(ids) @@ -440,20 +440,38 @@ def test_no_apply_rollback_never_applies_over_an_older_pending_apply(tmp_path): class _NoRoomBackend(SharedStackBackend): - """Converge fake that can never place an endpoint named ``big``.""" + """Can never place an endpoint named ``big``; preview and render agree.""" - def converge(self, desired, *, apply: bool = True) -> None: - placeable = [g for g in desired if 'big' not in g.served] - self.last_unplaced = [g.id for g in desired if 'big' in g.served] - super().converge(placeable, apply=apply) + def refuse(self, desired): + return {g.id: 'no room' for g in desired if 'big' in g.served} + def converge(self, desired, *, apply: bool = True, placement=None) -> None: + refused = self.refuse(desired) + self.last_unplaced = list(refused) + super().converge([g for g in desired if g.id not in refused], apply=apply) -def test_placement_rollback_evicts_and_rerenders(tmp_path): - """Regression: a failed placement must fully roll back — lease released, the - never-ran deployments evicted (not left idle-keep-warm, which would pin them - in the desired set), the placed sibling removed from the on-disk render, and - the publication marker cleared.""" - from infer_stack.leasing import DeploymentState, LeaseState + +class _RenderRefusesBackend(_NoRoomBackend): + """Its preview admits ``big``; its render then refuses it. + + The one way an admitted acquire still fails after its lease is committed + (the runtime changed between preview and render), so the rollback tests + use it. + """ + + def refuse(self, desired): + return {} + + def converge(self, desired, *, apply: bool = True, placement=None) -> None: + refused = [g.id for g in desired if 'big' in g.served] + SharedStackBackend.converge( + self, [g for g in desired if g.id not in refused], apply=apply) + self.last_unplaced = refused + + +def test_a_refused_acquire_commits_nothing(tmp_path): + """Admission refuses before anything is written: no lease, no deployment, + nothing rendered, no marker.""" from infer_stack.leasing.backend import PlacementError db = str(tmp_path / 'ledger.db') @@ -461,13 +479,34 @@ def test_placement_rollback_evicts_and_rerenders(tmp_path): ledger = Ledger(SqliteStore(db)) ctl = Controller(ledger, _NoRoomBackend(shared, threading.Lock())) + with pytest.raises(PlacementError, match='no room'): + ctl.acquire('alice', [_vreq('ok'), _vreq('big')], wait=False) + + assert ledger.status() == ([], []) + assert shared['rendered'] == set() + assert shared['realized'] == set() + assert ledger.publication_pending() is None + + +def test_a_render_refusal_after_commit_rolls_back_and_rerenders(tmp_path): + """Regression: a render that refuses what admission admitted must fully + roll back: lease released, the never-ran deployments evicted (not left + idle keep-warm, which would pin them in the desired set), the placed + sibling removed from the render, and the publication marker cleared.""" + from infer_stack.leasing import DeploymentState, LeaseState + from infer_stack.leasing.backend import PlacementError + + db = str(tmp_path / 'ledger.db') + shared = {'rendered': set(), 'realized': set(), 'apply_calls': 0} + ledger = Ledger(SqliteStore(db)) + ctl = Controller(ledger, _RenderRefusesBackend(shared, threading.Lock())) + with pytest.raises(PlacementError): ctl.acquire('alice', [_vreq('ok'), _vreq('big')], wait=False) leases, deployments = ledger.status() assert [le.state for le in leases] == [LeaseState.RELEASED] assert {g.state for g in deployments} == {DeploymentState.STOPPED} - # 'ok' was rendered by the failed acquire; the rollback re-render removed it. assert shared['rendered'] == set() assert shared['realized'] == set() assert ledger.publication_pending() is None @@ -475,8 +514,8 @@ def test_placement_rollback_evicts_and_rerenders(tmp_path): def test_rollback_keeps_coalesced_warm_deployment_resident(tmp_path): """A failed acquire that coalesced onto a pre-existing warm (idle keep-warm) - deployment must roll that deployment back to IDLE — not evict the resident - model someone else may still want warm.""" + deployment must leave it IDLE — not evict the resident model someone else + may still want warm.""" from infer_stack.leasing import DeploymentState from infer_stack.leasing.backend import PlacementError @@ -514,7 +553,7 @@ def __init__(self, *args, **kwargs): self.converge_calls = 0 self.declined = False - def converge(self, desired, *, apply: bool = True) -> None: + def converge(self, desired, *, apply: bool = True, placement=None) -> None: self.converge_calls += 1 if desired and not self.declined: self.declined = True @@ -558,13 +597,13 @@ def _warm_then_failed_acquire(tmp_path, residency): shared = _shared() - class Backend(_NoRoomBackend): + class Backend(_RenderRefusesBackend): pass - Backend.residency = lambda self: residency(self) ledger, ctl = _fresh(str(tmp_path / 'ledger.db'), shared, Backend) warm = ctl.acquire('alice', [_vreq('m')], wait=False) ctl.release(warm.lease.id) + Backend.residency = lambda self: residency(self) shared['realized'] = set() # observe() now says "nothing" (as on a docker error) with pytest.raises(PlacementError): ctl.acquire('bob', [_vreq('m'), _vreq('big')], wait=False) @@ -743,16 +782,13 @@ def test_release_all_releases_every_active_lease_in_one_apply(tmp_path): # -- transitional: rollback under unknown residency keeps a phantom warm candidate -- -def test_transitional_unknown_residency_rollback_keeps_an_idle_candidate(tmp_path): - """Behaviour of backends WITHOUT admission mode (no preview), kept by design. +def test_unknown_residency_admits_nothing(tmp_path): + """Without a strict view of what runs, nothing is admitted or written. - A brand-new acquire fails; rollback cannot read residency, so it refuses to - evict (the safe direction). On such backends the desired set still contains - that IDLE deployment, so a later apply starts it with no lease behind it. - Admission-mode backends (Compose) never commit a failed acquire and never - start idle deployments: see tests/test_leasing_admission.py. + (Before one acquire path, a backend without admission committed the + lease, failed, and left an IDLE candidate a later apply started with no + lease behind it.) """ - from infer_stack.leasing import DeploymentState from infer_stack.leasing.backend import PlacementError from infer_stack.leasing.residency import ResidencyUnknown @@ -763,9 +799,8 @@ def residency(self): raise ResidencyUnknown('docker ps failed') ledger, ctl = _fresh(str(tmp_path / 'ledger.db'), shared, Backend) - with pytest.raises(PlacementError): - ctl.acquire('alice', [_vreq('fresh'), _vreq('big')], wait=False) - fresh = next(g for g in ledger.status()[1] if 'fresh' in g.served) - assert fresh.state == DeploymentState.IDLE # not evicted - ctl.apply_now() - assert fresh.id in shared['realized'] # started without a lease (P9 fixes) + with pytest.raises(PlacementError, match='cannot be read'): + ctl.acquire('alice', [_vreq('fresh')], wait=False) + assert ledger.status() == ([], []) + assert ledger.publication_pending() is None + assert shared['realized'] == set() From 64f1e595d4f71be8303588c6bd08e2e9984b5129 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 11:30:21 -0400 Subject: [PATCH 026/141] Queue: record the P1b commit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 84ebcaed..f78b8b06 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -32,7 +32,7 @@ Done 2026-09-25, `d6a19c4`. ### 2. [x] P1b: delete the legacy acquire branch -Done 2026-09-26. +Done 2026-09-26, `184289c`. `MemoryBackend`, `NullBackend` and the test fakes (queue, lock, serialised publication) get a trivial `residency` and `preview`. Then the From c2d393b443d2df519c809711f7d02e67c1b5cb89 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 11:55:41 -0400 Subject: [PATCH 027/141] P2: ps, logs, status and the TUI read the backend, on either backend leasing/instances.py: an Instance per container or pod, built by each backend's instances() from its strict residency, the log commands for each runtime, target resolution (alias, name, id prefix, deployment id), and one LogFollower shared by `logs -f` and the TUI. - ps: one table on both backends, naming what each instance serves. - logs: by alias/name/id; -f follows instances that appear or restart. - status: health from residency (and `starting` before ready). - stack up = apply; stack down = backend.down(); the raw Compose verbs (stack compose, restart, pull, start, stop) act on the backend's Compose project on this host, the gateway's on kubeai. - TUI: the runtime pane lists instances, follows pods, Apply/Down work. Duplicate authorities removed: status's own `docker compose ps` liveness, the TUI's name-hint engine detection (it counted Open WebUI and Postgres), and the day-2 verbs' hard-coded project path. Bugs fixed: the TUI replaced kubeai's kubectl runner with Docker's (no KUBECONFIG, every kubectl call failed), and a kwconf flag swallowed the positional after it on every command (`logs -f qwen`, `acquire --yes qwen`). Verified on k3s (CPU vLLM) and on compose (the simulator catalog): ps, logs, logs -f during a request, status, stack compose, and the TUI in a real terminal (instances, pod log follow, Down then Apply). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 20 + README.md | 6 +- docs/backend-parity.md | 8 +- docs/kubeai-backend.md | 4 + docs/planning/backend-parity-roadmap.md | 16 +- docs/queue.md | 30 +- infer_stack/backends/kubeai.py | 19 + infer_stack/cli/commands_runtime.py | 588 +++++++++++++++--------- infer_stack/cli/options.py | 48 +- infer_stack/leasing/backend.py | 13 + infer_stack/leasing/compose.py | 38 +- infer_stack/leasing/instances.py | 278 +++++++++++ infer_stack/leasing/residency.py | 23 + infer_stack/tui.py | 428 +++++------------ tests/test_cli_meta.py | 10 +- tests/test_day2.py | 204 ++++++++ tests/test_log_filter.py | 101 ++-- tests/test_tui.py | 120 +++-- 18 files changed, 1275 insertions(+), 679 deletions(-) create mode 100644 infer_stack/leasing/instances.py create mode 100644 tests/test_day2.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f27c8d3c..aacb24f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### `ps`, `logs`, `status` and the TUI read the backend, on either backend + +`infer-stack ps` lists what the backend runs, containers or pods and the +gateway, in one table that says what each serves; `infer-stack logs` takes an +endpoint alias, an instance name, a container id prefix or a deployment id, +and `-f` keeps following instances that start or restart later. Both read the +same strict residency the controller decides with, so they work on KubeAI; +`status` does too, and says `starting` for an instance that is up but not yet +ready. `stack up` is now `apply` and `stack down` stops everything on either +backend; the raw Compose verbs (`stack compose -- …`, `restart`, `pull`, +`start`, `stop`) act on the Compose project on this host, which is the +gateway's on KubeAI. The TUI's runtime pane (formerly "docker") follows pod +logs, lists pods, and its Apply / Down buttons work on KubeAI. + +Fixed on the way: every kubectl call from the TUI failed, because the TUI +replaced the backend's command runner with Docker's, whose environment has no +`KUBECONFIG`; the TUI's "engines" log view included Open WebUI and Postgres; +and a flag before a positional ate it (`logs -f qwen` followed everything, +`acquire --yes qwen` named no endpoint), on every command. + ### One acquire path The controller's pre-admission branch is gone: every backend, including the diff --git a/README.md b/README.md index 6c76e9aa..b7fdfd56 100644 --- a/README.md +++ b/README.md @@ -812,8 +812,10 @@ If a `kubeai` release already exists, reuse its namespace ### Debugging checks `infer-stack acquire` reports pod-level failures itself (`ImagePullBackOff`, -`Unschedulable`, a crash with the engine's error quoted). For anything else, -with `NS` the namespace and `MODEL` the Model's name (`kubectl -n $NS get models`): +`Unschedulable`, a crash with the engine's error quoted). `infer-stack ps` lists +the pods and `infer-stack logs -f ` follows one, as on compose. For +anything else, with `NS` the namespace and `MODEL` the Model's name +(`kubectl -n $NS get models`): ```bash kubectl -n "$NS" describe model "$MODEL" diff --git a/docs/backend-parity.md b/docs/backend-parity.md index f4741e44..711ccd42 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -87,7 +87,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | where a deployment lands | local planner over `nvidia-smi` | the cluster scheduler, via `resource_profile:` | | GPU count from TP × PP × DP | same | same | | `placement.gpu_indices`, `allowed_gpus`, `skip_display_gpus` | yes | n/a: no host indices; node-scoped resource profiles are the equivalent | -| `placement.min_vram_gib`, `infer-stack measure` | yes | warned and ignored; `measure` refused (**gap**, P4) | +| `placement.min_vram_gib`, `infer-stack measure` | yes | **gap** (P4): `min_vram_gib` is warned and ignored; `measure` runs but is unverified on a GPU, and `--record` writes an overlay KubeAI does not read | | GPU allocations recorded in the ledger | yes | none: the cluster owns them | | more than one host | boundary | yes: the reason the backend exists | @@ -118,9 +118,11 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | `doctor` | nothing to check | four checks | -| `logs`, `ps`, `stack up` / `stack down` | docker compose wrappers | **gap** (P2): they say so and point at `kubectl` | +| `ps`, `logs [-f] `, `status` health | containers | same shape; pods, and the gateway's containers | +| `stack up` (= `apply`), `stack down` | same | same | +| `stack compose …`, `restart` / `pull` / `start` / `stop` | the stack's Compose project | the gateway's Compose project (the engines are pods) | | TUI: leases, deployments, catalog editing, acquire / release / evict, API tab, settings | same | same | -| TUI: engine log follow, the docker pane, the Up / Down buttons | `docker logs`, `docker compose ps` | **gap** (P2): empty, or "nothing rendered yet" | +| TUI: log follow, the Instances tab, Apply / Down | containers | same, over pods | | TUI GPU pane | `nvidia-smi` on this host | this host, not the cluster | | `gc --orphans` (also inside `clean`) | yes | n/a: unlabeled Models are never touched | diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 3be5b96e..48e418ef 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -143,6 +143,10 @@ infer-stack release --env-file lease.env on new setups. Without a GPU, install the chart with `dev/e2e_tests/kubeai-cpu-values.yaml` (real vLLM on CPU; needs AVX-512) and run it with `E2E_RESOURCE_PROFILE=cpu`. Verified on k3s 2026-09-24. +- `infer-stack ps`, `infer-stack logs [-f] `, `status` and the TUI's + runtime pane read the pods (and the gateway's containers), in the same shape + as on compose. `stack compose …` and `stack restart` act on the gateway's + Compose project; the engines are pods, restarted by the kubelet. - The gateway runs on the host running infer-stack, so that host is in every request's path. Dynamic routing (`dynamic_routing`) is compose-only; the KubeAI gateway uses static routes. diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index f1aa3530..4d05d473 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,7 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **P1 done** 2026-09-26 · P2–P5 not started · +`dev/backend-unification` · **P1, P2 done** 2026-09-26 · P3–P5 not started · P6 is ongoing. Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the @@ -129,6 +129,16 @@ Up / Down, and `measure`. logs ` and `ps` work on both backends with the same output shape. **Size:** medium, mostly plumbing; no controller change. +**P2 done 2026-09-26.** `leasing/instances.py`: an `Instance` per container +or pod, built from residency by each backend's `instances()`, and one +`LogFollower` for the CLI and the TUI. `stack up` is `apply`, `stack down` is +`backend.down()`, and the raw Compose verbs (`stack compose …`) act on +whichever Compose project the backend has on this host, the gateway's on +KubeAI. Verified on k3s and on compose (the simulator catalog), in a real +terminal. `measure` was never refused on KubeAI (its guard was +`deployment_logs`, which KubeAI has), but it reads GPU memory-profiling lines +that CPU vLLM does not print: it moves to P4 with its GPU handover. + ### P3. Gateway feature parity **Closes:** `routes` on KubeAI, `dynamic_routing`, Open WebUI and the reverse @@ -204,6 +214,10 @@ worse. A blocker is fixed whatever its size. | "can anything be admitted while residency is unknown" | `_admit` admitted requests needing no new GPU; the render after the commit then failed without residency | **fixed** (P1b): nothing is admitted, and nothing is committed | | the desired set | `desired_deployments()` beside the admission view; `routes prune` used the former | **fixed** (P1b): one view, `_admission_view` | | a crashed acquire's placement scope | recorded in the marker and re-applied by a recovery render, although admission never records one | **fixed** (P1b): a stale scope is dropped | +| "is it running" for `status` | `docker compose ps` service names beside residency (so KubeAI read `unverified`) | **fixed** (P2): residency | +| engine vs gateway in the TUI | a `litellm` name hint (Open WebUI and Postgres counted as engines) | **fixed** (P2): an instance serves a deployment or it does not | +| where the day-2 verbs find the Compose project | a hard-coded path and project name | **fixed** (P2): the backend's `compose_project()` and `compose_argv()` | +| how the TUI runs a runtime command | it replaced the backend's runner with Docker's, whose allowlisted environment has no `KUBECONFIG`: every kubectl call from the TUI failed | **fixed** (P2): only `docker` commands are wrapped | | the KubeAI gateway's approval | the gateway project asks its own diff approval at render, after the lease commits, not in the admission preview | deferred to P3. Same result under `--yes`; interactively, a declined gateway change rolls the lease back after the commit | ## Not in scope diff --git a/docs/queue.md b/docs/queue.md index f78b8b06..d5e57cb5 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -42,7 +42,9 @@ and `_admission_mode()` go. **Done when:** `_admission_mode` does not exist; the full suite passes; queue-semantics tests still assert the same behaviour. -### 3. [ ] P2: day-2 commands and the TUI through the backend +### 3. [x] P2: day-2 commands and the TUI through the backend + +Done 2026-09-26. `instances()` and `stream_logs(target, *, follow, tail)` on both backends; `ps`, `logs`, `stack up` / `stack down` use them, the raw compose form @@ -79,10 +81,15 @@ fake-Compose and fake-KubeAI backends; `tests/test_parity.py` runs each `nvidia.com/gpu.product` / `.memory` node labels; `min_vram_gib` picks the smallest fitting profile when an endpoint names none. +`measure` on KubeAI (moved here from P2): it already runs, but CPU vLLM prints +no GPU memory-profiling lines, and `--record` writes Compose's measurements +overlay, which KubeAI does not read. Decide where a cluster measurement is +recorded, and verify `measure` in the GPU handover. + **Done when:** on k3s with hand-set labels for two GPU sizes (CPU-backed profiles), a catalog with `min_vram_gib` and no `resource_profile` lands on the right profile; `dev/handover/p4_gpu_labels.sh` exists for one run on a -real GPU node. +real GPU node, and runs `measure` there. ### 7. [ ] P5: in-cluster gateway and a second node @@ -97,13 +104,17 @@ VM, a Model pinned to it by node selector is served through the same gateway. `dev/handover/p5_two_hosts.sh` exists for one run across two real machines. -### 8. [ ] The README's Compose sections +### 8. [ ] The README's Compose sections, and the other stale docs About 33 references to verbs that no longer exist (`setup`, `up -d`, -`switch`, `describe-profile`, `smoke-test`, `wait-ready`, `diagnose`). +`switch`, `describe-profile`, `smoke-test`, `wait-ready`, `diagnose`), and +top-level `restart` / `stop` / `start` / `pull` (they live under `stack`). +*Why widened (2026-09-26):* the same verbs appear in +`docs/persistent-caches-and-warm-restarts.md` and +`docs/stack-graph-profiles.md`. -**Done when:** every command in the README runs as written against the -current CLI, and `grep` finds none of those verbs. +**Done when:** every command in the README and `docs/` runs as written +against the current CLI, and `grep` finds none of those verbs. ### 9. [ ] UX audit loop: do not stop without a passing audit @@ -138,8 +149,11 @@ Seed findings, already known: - [ ] `r` does not reload catalogs, and edits to a catalog file are not picked up until restart. -- [ ] `infer-stack logs` should accept a container name, a prefixed name or - an endpoint name. +- [x] `infer-stack logs` should accept a container name, a prefixed name or + an endpoint name. Done with P2: an instance name, a container id + prefix, a deployment id or an endpoint alias. +- [x] `infer-stack logs -f qwen` followed every instance: a kwconf flag took + the next word as its value. Fixed for every command (P2). - [ ] `status` shows STALE during an apply instead of "apply in progress". ### 10. [ ] Handover diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 94f0af2c..d3117e99 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -308,6 +308,13 @@ def __init__( def models_file(self) -> Path: return self.state_dir / MODELS_FILENAME + #: The file a render writes, whatever the backend (``status`` shows it). + rendered_file = models_file + + def compose_project(self): + """The Compose project on this host: the gateway's, or ``None``.""" + return self.gateway + @property def _state_file(self) -> Path: return self.state_dir / STATE_FILENAME @@ -588,6 +595,18 @@ def residency(self): raise ResidencyUnknown(f'kubectl get pods failed: {ex}') from ex return residency_from_pods(raw) + def instances(self): + """The managed Models' pods, then the gateway's containers. + + Raises :class:`~infer_stack.leasing.residency.ResidencyUnknown` when + the cluster cannot be read. + """ + from ..leasing.instances import KUBERNETES, from_residency + + pods = from_residency(self.residency(), runtime=KUBERNETES, + namespace=self.namespace) + return pods + (self.gateway.instances() if self.gateway is not None else []) + def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: """Recent engine logs: each pod's current run, then its previous one. diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index c37d7336..0063014d 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -18,11 +18,12 @@ from ..log_filter import compact_litellm_tracebacks from ..paths import config_root, data_root, get_setting, settings_path +from .commands_leasing import ApplyCLI from .context import _apply_path_overrides from .options import _PathOverridesMixin # --------------------------------------------------------------------------- -# leasing compose project helpers (the target of the day-2 wrappers) +# the backend behind the day-2 verbs # --------------------------------------------------------------------------- @@ -32,43 +33,61 @@ def _docker_env() -> dict[str, str]: return docker_environment() -def _leasing_compose_file() -> Path: - from ..leasing.compose import COMPOSE_FILENAME +def _day2_backend(config): + """The configured backend, built as the leasing verbs build it.""" + from .commands_leasing import _make_backend - return data_root() / 'leasing' / 'compose' / COMPOSE_FILENAME + _apply_path_overrides(config) + return _make_backend(config) -def _day2_compose_base(config, command: str) -> list[str]: - """``docker compose -p -f `` base for the wrappers. +def _served_by_deployment() -> dict[str, list[str]]: + """Deployment id -> the endpoint aliases it serves (read-only ledger).""" + from ..leasing import Ledger, SqliteStore, default_ledger_path - Targets the leasing Compose deployment (no ``config.yaml`` needed). Raises a - helpful error when nothing has been deployed yet — the file is written by - ``acquire`` / ``apply``. - """ - from ..leasing.compose import LEASING_PROJECT + path = default_ledger_path() + if not path.exists(): + return {} + try: + _, deployments = Ledger(SqliteStore(str(path))).status(virtual_expiry=True) + except Exception: # noqa: BLE001 - a view never fails on the ledger + return {} + return {d.id: sorted(d.served) for d in deployments} - _apply_path_overrides(config) - compose_file = _leasing_compose_file() - if not compose_file.exists(): - hint = '' - if (get_setting('backend') or '') == 'kubeai': - hint = ( - ' Note: the configured backend is kubeai — these verbs manage ' - 'the docker compose stack only; inspect the cluster with ' - '`infer-stack doctor` / `kubectl -n get models`.' - ) + +def _instances(backend) -> list: + """What the backend runs, or a clean exit when it cannot be read.""" + from ..leasing.residency import ResidencyUnknown + + try: + return list(backend.instances()) + except ResidencyUnknown as ex: + raise SystemExit(f'cannot read what is running: {ex}') + + +def _compose_argv(config) -> list[str]: + """``docker compose ...`` for the Compose project on this host. + + That is the stack itself on the compose backend, and the gateway on the + kubeai backend. Exits when there is none, or nothing is rendered yet. + """ + backend = _day2_backend(config) + project = getattr(backend, 'compose_project', lambda: None)() + if project is None: raise SystemExit( - f'nothing deployed yet (no {compose_file}). ' - f'Bring a model up first, e.g. `infer-stack acquire `.' - f'{hint}' - ) - base = ['docker', 'compose'] - # The same managed .env the backend passes, so `stack up` interpolates the - # master key, DB password and HF_TOKEN from it (never the caller's shell). - env_file = compose_file.parent / '.env' - if env_file.exists(): - base += ['--env-file', str(env_file)] - return [*base, '-p', LEASING_PROJECT, '-f', str(compose_file)] + f'the {_backend_name(backend)} backend has no Compose project on this ' + 'host; use `infer-stack ps` and `infer-stack logs`') + if not project.compose_file.exists(): + raise SystemExit( + f'nothing rendered yet (no {project.compose_file}); bring a model up ' + 'first, e.g. `infer-stack acquire `') + return project.compose_argv() + + +def _backend_name(backend) -> str: + name = type(backend).__name__ + return {'ComposeBackend': 'compose', 'KubeaiBackend': 'kubeai', + 'NullBackend': 'null (dry-run)'}.get(name, name) # --------------------------------------------------------------------------- @@ -122,82 +141,65 @@ def _leasing_status() -> dict[str, Any]: for d in deployments ] out['summary'] = (active, len(leases), live, len(deployments)) - out['served'] = _served_models(deployments) + out['live_deployments'] = [d for d in deployments if d.state == DeploymentState.LIVE] return out -def _running_services() -> set[str] | None: - """Compose service names currently running, or None if docker can't say. +def _served_models(deployments, backend=None) -> list[tuple[str, str, str, str]]: + """``(endpoint, model, engine, health)`` rows for what is serving. - A single cheap `docker compose ps`. It exists because the ledger's LIVE is - a *belief*: a converge that fails partway (an unpullable image, a daemon - hiccup) leaves deployments recorded LIVE with no container behind them, and - the only way to notice used to be an acquire that hung on readiness. - Distinguishing "recorded" from "running" is the whole point of showing this. - - None (not an empty set) when docker is unreachable, so the caller can say - "unverified" instead of falsely reporting everything down. - """ - from ..leasing.compose import LEASING_PROJECT - - compose_file = _leasing_compose_file() - if not compose_file.exists(): - return None - try: - proc = subprocess.run( - ['docker', 'compose', '-p', LEASING_PROJECT, '-f', str(compose_file), - 'ps', '--services', '--filter', 'status=running'], - capture_output=True, text=True, timeout=15, env=_docker_env(), - ) - except Exception: # noqa: BLE001 - status must never fail on this - return None - if proc.returncode != 0: - return None - return {ln.strip() for ln in proc.stdout.splitlines() if ln.strip()} - - -def _served_models(deployments) -> list[tuple[str, str, str, str]]: - """``(endpoint, model, gpus_or_engine, health)`` rows for what is serving. - - Reads the ledger for what *should* be up and reconciles against the - containers that actually are, so `status` answers "what can I send a - request to right now" without the user having to cross-read `leases` - against `stack ps`. + Reads the ledger for what *should* be up and the backend's strict + residency for what is, so `status` answers "what can I send a request to + right now" without cross-reading `leases` against `ps`. """ from ..leasing import DeploymentState - from ..leasing.compose import ollama_service_name_for, vllm_service_name_for + from ..leasing.residency import ResidencyUnknown live = [d for d in deployments if d.state == DeploymentState.LIVE] if not live: return [] - running = _running_services() + residency = None + if backend is not None: + try: + residency = backend.residency() + except (ResidencyUnknown, Exception): # noqa: BLE001 - status never fails here + residency = None rows: list[tuple[str, str, str, str]] = [] for d in live: + if residency is None: + health = 'unverified' + elif residency.resident(d.id) is not None: + # Warm; a health check that has not passed yet means still loading. + health = ('starting' if residency.resident(d.id).health == 'starting' + else 'up') + elif residency.containers(d.id): + # There, but not warm: starting, crashed, or ambiguous. + health = residency.containers(d.id)[0].state + else: + # The ledger says live and nothing exists. Almost always an apply + # that failed after the record was written. + health = 'STALE' for endpoint in sorted(d.served): payload = d.served.get(endpoint) or {} model = (payload.get('hf_model_id') or payload.get('model') or d.spec.get('hf_model_id') or '-') - if d.engine == 'ollama': - service = ollama_service_name_for(d.spec.get('host') or endpoint) - else: - service = vllm_service_name_for( - payload.get('served_model_name') or endpoint) - if running is None: - health = 'unverified' - elif service in running: - health = 'up' - else: - # The ledger says live and nothing is running. Almost always a - # converge that failed after the record was written. - health = 'STALE' rows.append((endpoint, model, d.engine, health)) return rows def _gather_status(config) -> dict[str, Any]: - compose_file = _leasing_compose_file() + try: + backend = _day2_backend(config) + except (SystemExit, Exception): # noqa: BLE001 - status never fails on this + backend = None + rendered = getattr(backend, 'rendered_file', None) + leasing = _leasing_status() + if leasing.get('live_deployments'): + leasing['served'] = _served_models(leasing.pop('live_deployments'), backend) + else: + leasing.pop('live_deployments', None) return { 'backend': str(get_setting('backend') or 'null'), 'data_dir': str(data_root()), @@ -206,16 +208,17 @@ def _gather_status(config) -> dict[str, Any]: 'settings': {'path': str(settings_path()), 'exists': settings_path().exists()}, 'catalog': _catalog_summary(config), - 'compose': {'path': str(compose_file), 'exists': compose_file.exists()}, - 'leasing': _leasing_status(), + 'rendered': {'path': str(rendered) if rendered else None, + 'exists': bool(rendered and rendered.exists())}, + 'leasing': leasing, } _DIG_DEEPER = ( ('infer-stack leases', 'full lease + deployment tables'), ('infer-stack tui', 'live dashboard (opt-in: infer-stack[tui])'), - ('infer-stack stack ps', 'running containers'), - ('infer-stack logs -f', 'tail service logs'), + ('infer-stack ps', 'what is running (containers or pods)'), + ('infer-stack logs -f ', 'follow an engine\'s log'), ('infer-stack catalog show', 'what you can serve'), ) @@ -237,11 +240,14 @@ def _served_lines(served: list[tuple[str, str, str, str]]) -> list[str]: for endpoint, model, _engine, health in served: out.append(f' {endpoint.ljust(w_ep)} {model.ljust(w_mo)} {health}') if any(r[3] == 'STALE' for r in served): - out.append(' STALE = the ledger records this live but no container is ' - 'running; `infer-stack apply` or `gc`') + out.append(' STALE = the ledger records this live but nothing is ' + 'running for it; `infer-stack apply` or `gc`') + if any(r[3] == 'starting' for r in served): + out.append(' starting = up, but not ready yet (loading the model); ' + '`infer-stack wait `') if any(r[3] == 'unverified' for r in served): - out.append(' unverified = could not reach docker to confirm; the ' - 'ledger says live') + out.append(' unverified = the runtime could not be read to confirm; ' + 'the ledger says live') return out @@ -262,8 +268,9 @@ def _print_status_plain(d: dict[str, Any]) -> None: f'{"" if d["configured"] else " (run infer-stack config init)"}') lz = d['leasing'] print(f' ledger: {lz["path"]}{"" if lz["exists"] else " (none yet)"}') - print(f' compose: {d["compose"]["path"]}' - f'{"" if d["compose"]["exists"] else " (not rendered yet)"}') + if d['rendered']['path']: + print(f' rendered: {d["rendered"]["path"]}' + f'{"" if d["rendered"]["exists"] else " (not rendered yet)"}') if lz['summary']: active, total_l, live, total_d = lz['summary'] print() @@ -309,10 +316,11 @@ def _print_status_rich(d: dict[str, Any], console) -> None: if not lz['exists']: ledger.append(' (none yet)', style='dim') table.add_row('ledger', ledger) - compose = Text(d['compose']['path'], style='cyan') - if not d['compose']['exists']: - compose.append(' (not rendered yet)', style='dim') - table.add_row('compose', compose) + if d['rendered']['path']: + rendered = Text(d['rendered']['path'], style='cyan') + if not d['rendered']['exists']: + rendered.append(' (not rendered yet)', style='dim') + table.add_row('rendered', rendered) console.print(table) if lz['summary']: @@ -332,7 +340,8 @@ def _print_status_rich(d: dict[str, Any], console) -> None: served_table.add_column('model', overflow='fold') served_table.add_column('engine', style='dim', no_wrap=True) served_table.add_column('health', no_wrap=True) - styles = {'up': 'green', 'STALE': 'red', 'unverified': 'yellow'} + styles = {'up': 'green', 'starting': 'yellow', 'STALE': 'red', + 'unverified': 'yellow'} for endpoint, model, engine, health in served: served_table.add_row( endpoint, model, engine, @@ -343,7 +352,7 @@ def _print_status_rich(d: dict[str, Any], console) -> None: console.print(served_table) if any(r[3] == 'STALE' for r in served): console.print(Text( - ' STALE = recorded live but no container is running; ' + ' STALE = recorded live but nothing is running for it; ' '`infer-stack apply` or `gc`', style='dim')) console.print() @@ -385,66 +394,161 @@ def main(cls, argv=True, **kwargs): # --------------------------------------------------------------------------- -# stack — docker compose day-2-ops wrappers over the leasing project +# ps / logs / stack: what the backend runs, on either backend # --------------------------------------------------------------------------- -class _ComposeWrapperBase(_PathOverridesMixin): - """Common fields for ``docker compose `` wrappers over the leasing - Compose deployment.""" +class _InstancesBase(_PathOverridesMixin): + """Options shared by the verbs that read the backend's instances.""" services = kw.Value( None, nargs='*', position=1, - help='Optional service names to filter (empty = all).', + help='Which instances: a service or pod name, a container id (prefix), ' + 'a deployment id, or an endpoint alias. Empty = all.', + ) + backend = kw.Value( + None, type=str, + help='Backend to read (default: the configured `backend` setting).', ) -def _run_compacted_follow(cmd: list[str]) -> int: - """Stream Compose logs through the conservative LiteLLM compactor.""" - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - text=True, - errors='replace', - bufsize=1, - env=_docker_env(), +#: States `ps` hides unless --all: finished, or on the way out. +_PS_HIDDEN = frozenset({'exited', 'dead', 'removing'}) + + +def _ps_rows(instances, served) -> list[dict[str, Any]]: + rows = [] + for inst in instances: + rows.append({ + 'name': inst.name, + 'id': inst.id, + 'deployment': inst.deployment_id or None, + 'serves': served.get(inst.deployment_id, []) if inst.deployment_id else [], + 'state': inst.state, + 'status': inst.status, + 'restarts': inst.restarts, + 'gpus': list(inst.gpus), + 'started': inst.started or None, + 'ports': inst.ports or None, + 'runtime': inst.runtime, + }) + return rows + + +def _print_ps(rows) -> None: + def cell(row): + serves = ', '.join(row['serves']) or ('-' if row['deployment'] else '(front door)') + gpus = ','.join(map(str, row['gpus'])) or '-' + ident = row['id'][:12] if row['runtime'] == 'docker' else '-' + return (row['name'], row['status'], serves, gpus, + (row['started'] or '-')[:19], ident, row['ports'] or '-') + + head = ('NAME', 'STATUS', 'SERVES', 'GPUS', 'STARTED', 'ID', 'PORTS') + table = [head, *(cell(r) for r in rows)] + widths = [max(len(str(r[i])) for r in table) for i in range(len(head))] + for r in table: + print(' '.join(str(v).ljust(w) for v, w in zip(r, widths)).rstrip()) + + +class PsCLI(_InstancesBase): + """What the backend is running: engine containers or pods, and the gateway. + + One shape on every backend. An engine row names the endpoints it serves; + the gateway, UI and proxy show as the front door. Reads the same strict + residency the controller decides with, so a row here is what admission + sees. + """ + + __command__ = 'ps' + + all = kw.Value( + False, isflag=True, short_alias=['a'], + help='Include exited and dying instances.', ) - try: - if proc.stdout is None: # pragma: no cover - PIPE guarantees stdout - return int(proc.wait()) - for line in compact_litellm_tracebacks(proc.stdout): - sys.stdout.write(line) - return int(proc.wait()) - except KeyboardInterrupt: - # The child normally receives the same SIGINT. If it is still alive, - # make sure an interrupted follow does not leave Compose behind. - if proc.poll() is None: - proc.terminate() + services_only = kw.Value( + False, isflag=True, help='Print only instance names.', + ) + quiet = kw.Value( + False, isflag=True, short_alias=['q'], + help='Print only ids (container ids, or pod names).', + ) + json = kw.Value(False, isflag=True, help='Print the rows as JSON.') + + @classmethod + def main(cls, argv=True, **kwargs): + import json + + from ..leasing.instances import UnknownTarget, resolve + + config = cls.cli(argv=argv, data=kwargs) + backend = _day2_backend(config) + instances = _instances(backend) + served = _served_by_deployment() + if config.services: try: - proc.wait(timeout=2) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - return 130 + instances = resolve(instances, config.services, served) + except UnknownTarget as ex: + raise SystemExit(str(ex)) + if not config.all: + instances = [i for i in instances if i.state not in _PS_HIDDEN] + rows = _ps_rows(instances, served) + if config.json: + print(json.dumps(rows, indent=2)) + elif config.quiet: + for row in rows: + print(row['id']) + elif config.services_only: + for row in rows: + print(row['name']) + elif not rows: + print(f'nothing running ({_backend_name(backend)} backend); ' + 'bring a model up with `infer-stack acquire `') + else: + _print_ps(rows) + return 0 + +#: Prefix colors for `logs` on a terminal, cycled per instance name. +_LOG_COLORS = ('36', '33', '32', '35', '34', '96', '93', '92', '95', '94') -class LogsCLI(_ComposeWrapperBase): - """Tail leasing Compose service logs without typing the full docker path.""" + +def _colorize(lines, *, enabled: bool): + """Color each ``name | `` prefix, one color per name, like Compose did.""" + if not enabled: + yield from lines + return + colors: dict[str, str] = {} + for line in lines: + name, sep, rest = line.partition(' | ') + if not sep: + yield line + continue + code = colors.setdefault(name, _LOG_COLORS[len(colors) % len(_LOG_COLORS)]) + yield f'\x1b[{code}m{name} |\x1b[0m {rest}' + + +class LogsCLI(_InstancesBase): + """Engine and gateway logs, by instance, deployment or endpoint alias. + + ``infer-stack logs qwen -f`` follows whatever serves the ``qwen`` + endpoint, container or pod; with no names, every instance. Following + picks up instances that start later and follows a restarted one again. + """ __command__ = 'logs' follow = kw.Value( False, isflag=True, short_alias=['f'], - help='Stream logs (docker compose logs -f).', + help='Keep streaming, including instances that start later.', ) tail = kw.Value( None, type=str, - help="Tail the last N lines (default: all). Pass a number or 'all'.", + help="Only the last N lines of each (default: all). A number or 'all'.", ) timestamps = kw.Value(False, isflag=True) - no_color = kw.Value(False, isflag=True) + no_color = kw.Value(False, isflag=True, help='Do not color the name prefixes.') raw = kw.Value( False, isflag=True, @@ -453,74 +557,96 @@ class LogsCLI(_ComposeWrapperBase): @classmethod def main(cls, argv=True, **kwargs): + from ..leasing.instances import ( + LogFollower, + UnknownTarget, + history_argv, + resolve, + runtime_env, + ) + config = cls.cli(argv=argv, data=kwargs) + backend = _day2_backend(config) + served = _served_by_deployment() + names = list(config.services or []) - # The compacted path pipes Compose stdout through Python. Without an - # explicit ANSI mode Compose sees a non-TTY pipe and drops the service - # colors before the compactor can preserve them. Force ANSI only for - # the human-facing compacted view; --no-color remains authoritative. - compact = config.follow and not config.raw and sys.stdout.isatty() - cmd = _day2_compose_base(config, 'logs') - if compact and not config.no_color: - cmd.extend(['--ansi', 'always']) - cmd.append('logs') - if config.follow: - cmd.append('--follow') - if config.tail is not None: - cmd.extend(['--tail', str(config.tail)]) - if config.no_color: - cmd.append('--no-color') - if config.timestamps: - cmd.append('--timestamps') - cmd.extend(config.services or []) + def pick(instances): + return resolve(instances, names, served) if names else instances - # Compact only the human-facing live view. Captures and pipelines keep - # Docker's exact bytes unless a future explicit compact-output mode is - # added; ``--raw`` is also available for interactive LiteLLM debugging. - if compact: - return _run_compacted_follow(cmd) - return int(subprocess.run(cmd, env=_docker_env()).returncode) + try: + chosen = pick(_instances(backend)) + except UnknownTarget as ex: + raise SystemExit(str(ex)) + color = sys.stdout.isatty() and not config.no_color + if config.follow: + def listing(): + try: + return pick(backend.instances()) + except UnknownTarget: + return [] # the named one is between restarts + follower = LogFollower(listing, history=config.tail or 'all', + timestamps=bool(config.timestamps)) + lines = follower.stdout + if sys.stdout.isatty() and not config.raw: + lines = compact_litellm_tracebacks(lines) + try: + for line in _colorize(lines, enabled=color): + sys.stdout.write(line) + sys.stdout.flush() + except KeyboardInterrupt: + return 130 + finally: + follower.terminate() + return 0 + if not chosen: + print(f'nothing running ({_backend_name(backend)} backend)') + return 0 + prefix = len(chosen) > 1 + status = 0 + for inst in chosen: + argv_ = history_argv(inst, tail=config.tail, + timestamps=bool(config.timestamps)) + if argv_ is None: + print(f'{inst.name}: the {_backend_name(backend)} backend keeps no logs') + continue + proc = subprocess.run(argv_, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=runtime_env(inst)) + text = proc.stdout.decode('utf-8', 'replace') + lines = [f'{inst.name} | {ln}\n' if prefix else f'{ln}\n' + for ln in text.splitlines()] + for line in _colorize(lines, enabled=color and prefix): + sys.stdout.write(line) + status = status or proc.returncode + return int(status) -class PsCLI(_ComposeWrapperBase): - """``docker compose ps`` for the leasing deployment.""" +class _ComposeWrapperBase(_PathOverridesMixin): + """``docker compose `` over the Compose project on this host. - __command__ = 'ps' + The stack itself on the compose backend; the gateway on kubeai. + """ - all = kw.Value( - False, isflag=True, short_alias=['a'], help='Include stopped containers.' - ) - services_only = kw.Value( - False, isflag=True, - help='Print only service names (passes --services to docker compose).', + services = kw.Value( + None, + nargs='*', + position=1, + help='Optional service names to filter (empty = all).', ) - quiet = kw.Value( - False, isflag=True, short_alias=['q'], help='Print only container IDs.' + backend = kw.Value( + None, type=str, + help='Backend (default: the configured `backend` setting).', ) - @classmethod - def main(cls, argv=True, **kwargs): - config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'ps') + ['ps'] - if config.all: - cmd.append('--all') - if config.services_only: - cmd.append('--services') - if config.quiet: - cmd.append('--quiet') - cmd.extend(config.services or []) - return int(subprocess.run(cmd, env=_docker_env()).returncode) - class RestartCLI(_ComposeWrapperBase): - """``docker compose restart [services...]``.""" + """``docker compose restart [services...]`` (the Compose project on this host).""" timeout = kw.Value(None, type=int, help='Stop timeout in seconds.') @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'restart') + ['restart'] + cmd = _compose_argv(config) + ['restart'] if config.timeout is not None: cmd.extend(['--timeout', str(config.timeout)]) cmd.extend(config.services or []) @@ -528,7 +654,7 @@ def main(cls, argv=True, **kwargs): class PullCLI(_ComposeWrapperBase): - """``docker compose pull [services...]``.""" + """``docker compose pull [services...]`` (the Compose project on this host).""" quiet = kw.Value(False, isflag=True, short_alias=['q']) ignore_pull_failures = kw.Value(False, isflag=True) @@ -536,7 +662,7 @@ class PullCLI(_ComposeWrapperBase): @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'pull') + ['pull'] + cmd = _compose_argv(config) + ['pull'] if config.quiet: cmd.append('--quiet') if config.ignore_pull_failures: @@ -546,81 +672,119 @@ def main(cls, argv=True, **kwargs): class StartCLI(_ComposeWrapperBase): - """``docker compose start [services...]``.""" + """``docker compose start [services...]`` (the Compose project on this host).""" @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'start') + ['start'] + cmd = _compose_argv(config) + ['start'] cmd.extend(config.services or []) return int(subprocess.run(cmd, env=_docker_env()).returncode) class StopCLI(_ComposeWrapperBase): - """``docker compose stop [services...]``.""" + """``docker compose stop [services...]`` (the Compose project on this host).""" timeout = kw.Value(None, type=int) @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'stop') + ['stop'] + cmd = _compose_argv(config) + ['stop'] if config.timeout is not None: cmd.extend(['--timeout', str(config.timeout)]) cmd.extend(config.services or []) return int(subprocess.run(cmd, env=_docker_env()).returncode) -class StackDownCLI(_ComposeWrapperBase): - """``docker compose down`` the leasing deployment. +class StackComposeCLI(_PathOverridesMixin): + """Run any ``docker compose`` command on the Compose project on this host. - Tears the whole project down. Leasing's reconcile manages teardown - automatically on release; this is the manual escape hatch. + The raw escape hatch: the stack itself on the compose backend, the gateway + on kubeai. It bypasses the ledger and renders nothing, e.g. + ``infer-stack stack compose -- up -d litellm``. """ - volumes = kw.Value( - False, isflag=True, help='Also remove named volumes (--volumes).' + __command__ = 'compose' + + args = kw.Value(None, nargs='*', position=1, + help='Arguments for docker compose (put them after --).') + backend = kw.Value( + None, type=str, + help='Backend (default: the configured `backend` setting).', ) @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'down') + ['down', '--remove-orphans'] - if config.volumes: - cmd.append('--volumes') + cmd = _compose_argv(config) + list(config.args or []) return int(subprocess.run(cmd, env=_docker_env()).returncode) -class StackUpCLI(_ComposeWrapperBase): - """``docker compose up -d`` exactly what is on disk — the raw escape hatch. +class StackDownCLI(_PathOverridesMixin): + """Stop everything the backend runs, bypassing the ledger. - Brings up the on-disk compose file as-is, without touching the ledger or - re-rendering. Prefer ``infer-stack apply``, which re-renders from intent - first. Reach for ``stack up`` only to run a *hand-edited* compose file - verbatim. + The manual escape hatch: releases no lease, so a later publish brings + leased models back. Compose: ``docker compose down``; kubeai: deletes every + managed Model, then the gateway. ``--volumes`` (Compose only) also removes + named volumes. """ + __command__ = 'down' + + volumes = kw.Value( + False, isflag=True, help='Also remove named volumes (Compose only).' + ) + backend = kw.Value( + None, type=str, + help='Backend (default: the configured `backend` setting).', + ) + @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - cmd = _day2_compose_base(config, 'up') + ['up', '-d', '--remove-orphans'] - cmd.extend(config.services or []) - return int(subprocess.run(cmd, env=_docker_env()).returncode) + if config.volumes: + cmd = _compose_argv(config) + ['down', '--remove-orphans', '--volumes'] + return int(subprocess.run(cmd, env=_docker_env()).returncode) + backend = _day2_backend(config) + down = getattr(backend, 'down', None) + if down is None: + print(f'the {_backend_name(backend)} backend runs nothing to bring down') + return 0 + down() + return 0 + + +class StackUpCLI(ApplyCLI): + """``apply``: bring up what the ledger says should run (both backends). + + The raw form, ``docker compose up`` of the file on disk, is + ``infer-stack stack compose -- up -d``. + """ + + __command__ = 'up' class StackModalCLI(kw.ModalCLI): - """Day-2 ops on the running leasing deployment.""" + """Day-2 ops on what the backend runs. + + ``up`` is ``apply`` and ``down`` stops everything, on either backend. The + ``docker compose`` verbs (restart, pull, start, stop, and ``compose`` for + anything else) act on the Compose project on this host: the stack itself + on compose, the gateway on kubeai. + """ __command__ = 'stack' up = StackUpCLI logs = LogsCLI ps = PsCLI + down = StackDownCLI + compose = StackComposeCLI restart = RestartCLI pull = PullCLI start = StartCLI stop = StopCLI - down = StackDownCLI class DoctorCLI(_PathOverridesMixin): diff --git a/infer_stack/cli/options.py b/infer_stack/cli/options.py index f4a1c3ce..1acb1413 100644 --- a/infer_stack/cli/options.py +++ b/infer_stack/cli/options.py @@ -9,7 +9,53 @@ # --------------------------------------------------------------------------- -class _PathOverridesMixin(kw.Config): +#: Values a bare flag may carry (``--yes false``); anything else was a positional. +_BOOL_WORDS = frozenset({'true', 'false', 'yes', 'no', 'on', 'off', '1', '0'}) + + +def reclaim_swallowed_positionals(config) -> None: + """Give back a positional argument that a preceding flag consumed. + + kwconf flags take an optional value, so ``logs -f qwen`` parsed as + ``follow='qwen'`` and no names: the command then acted on everything. + Every infer-stack flag is a boolean, so a flag holding any other string + was handed a positional. It becomes ``True``, and the string goes back to + the front of the command's positional list (or its single positional, + when that is still empty). + """ + defaults = type(config).__default__ + positional = sorted((k for k, v in defaults.items() if getattr(v, 'position', None)), + key=lambda k: defaults[k].position) + for key, value in defaults.items(): + if not getattr(value, 'isflag', False) or value.isflag == 'counter': + continue + got = config[key] + if not isinstance(got, str) or got.strip().lower() in _BOOL_WORDS: + continue + config[key] = True + if not positional: + raise SystemExit(f'--{key} takes no value (got {got!r})') + target = positional[0] + many = defaults[target].parsekw.get('nargs') in ('*', '+') + if many: + config[target] = [got, *(config[target] or [])] + elif config[target] in (None, ''): + config[target] = got + else: + raise SystemExit(f'--{key} takes no value (got {got!r})') + + +class _FlagSafeMixin(kw.Config): + """Parses ``--flag positional`` as a flag and a positional (see above).""" + + @classmethod + def cli(cls, *args, **kwargs): + config = super().cli(*args, **kwargs) + reclaim_swallowed_positionals(config) + return config + + +class _PathOverridesMixin(_FlagSafeMixin): """Adds global ``--config-dir`` / ``--data-dir`` to a subcommand.""" config_dir = kw.Value( diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 9755130c..d50454d6 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -188,6 +188,13 @@ def residency(self) -> Any: """A strict :class:`~infer_stack.leasing.residency.Residency`, or raise.""" ... + def instances(self) -> list[Any]: + """Every running unit, as :class:`~infer_stack.leasing.instances.Instance`. + + Built from :meth:`residency`, so it raises when that does. + """ + ... + def preview(self, desired: list[Deployment], placement: Any = None, *, approve: bool = False) -> Any: """Place and render ``desired`` without writing; ``(plan, rendered)``.""" @@ -375,6 +382,12 @@ def residency(self): for gid in sorted(self.observe()) }) + def instances(self): + """One in-process instance per realized deployment; no log to read.""" + from .instances import MEMORY, from_residency + + return from_residency(self.residency(), runtime=MEMORY) + def _preview(self, desired: list[Deployment], placement: Any = None): desired = list(desired) plan = self.plan(desired, placement) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 1ea628bd..0470a36f 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1093,6 +1093,26 @@ def inventory(self, value: dict[str, Any] | None) -> None: def compose_file(self) -> Path: return self.state_dir / COMPOSE_FILENAME + #: The file a render writes, whatever the backend (``status`` shows it). + rendered_file = compose_file + + def compose_project(self): + """The Compose project on this host: this one.""" + return self + + def compose_argv(self) -> list[str]: + """``docker compose [--env-file ...] -p -f ``: the one base. + + The managed ``.env`` beside the compose file resolves the master key + and the other managed secrets; docker compose's own ``.env`` discovery + keys off the caller's working directory, so it is passed explicitly, + and only when present (a missing ``--env-file`` is a hard error). + """ + cmd = ['docker', 'compose'] + if self.gateway._env_path.exists(): + cmd += ['--env-file', str(self.gateway._env_path)] + return [*cmd, '-p', self.project, '-f', str(self.compose_file)] + # -- the front door (leasing.gateway) ------------------------------------ # Settings the gateway owns. Kept as attributes of the backend because # profiles and callers set them here; they are stored in one place. @@ -1337,17 +1357,7 @@ def doctor(self) -> list[tuple[str, bool, str]]: def _compose(self, args: list[str]) -> str: self._ensure_state_dir() - cmd = ['docker', 'compose'] - # Resolve ${LITELLM_MASTER_KEY} (and any other managed secret) from the - # sidecar .env beside the compose file, so secrets stay out of the YAML. - # docker compose's default .env discovery keys off the *current working - # directory* (wherever infer-stack was invoked), not the state dir, so we - # point it explicitly. Only when present: a litellm-less stack never - # writes one, and a missing --env-file path is a hard error. - if self.gateway._env_path.exists(): - cmd += ['--env-file', str(self.gateway._env_path)] - cmd += ['-p', self.project, '-f', str(self.compose_file)] - return self.run([*cmd, *args]) + return self.run([*self.compose_argv(), *args]) def plan(self, desired: list[Deployment], placement=None): """Compute GPU placement for ``desired`` without writing or applying. @@ -2202,6 +2212,12 @@ def settle_snapshot(self) -> tuple[tuple[str, str], ...]: pairs.append((parts[0], parts[1].lower())) return tuple(sorted(pairs)) + def instances(self): + """Every container of this project, engines first (raises if unknown).""" + from .instances import DOCKER, from_residency + + return from_residency(self.residency(), runtime=DOCKER) + def residency(self) -> Residency: """Strict snapshot of this project's deployment containers and their GPUs. diff --git a/infer_stack/leasing/instances.py b/infer_stack/leasing/instances.py new file mode 100644 index 00000000..3b4f9df6 --- /dev/null +++ b/infer_stack/leasing/instances.py @@ -0,0 +1,278 @@ +"""What a backend is running, one row per unit, and how to read its log. + +An :class:`Instance` is a container (Compose, or the KubeAI gateway) or a pod +(KubeAI). Backends build them from their strict residency +(:meth:`instances`), so ``ps``, ``logs``, ``status`` and the TUI read the +same view the controller decides with. The runtime an instance lives in is +recorded on it, so reading its log needs no knowledge of which backend +produced it: :func:`history_argv` and :func:`follow_argv` are the only place +that knows ``docker logs`` from ``kubectl logs``. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from typing import Callable, Iterable, Iterator, Mapping + +#: Runtimes an instance can live in. +DOCKER = 'docker' +KUBERNETES = 'kubernetes' +#: In-process backends (dry-run, tests): nothing to read a log from. +MEMORY = 'memory' + + +@dataclass(frozen=True) +class Instance: + """One running unit: an engine container or pod, or a front-door service.""" + + #: What a user types: the Compose service, or the pod name. + name: str + #: The runtime's own id: container id, or the pod name. + id: str + #: The deployment it serves; empty for the gateway, UI, database, proxy. + deployment_id: str + state: str + restarts: int = 0 + #: Why it is not running, when the runtime says (``CrashLoopBackOff``...). + reason: str = '' + #: ``healthy`` / ``starting`` / ``unhealthy``, when the runtime has a check. + health: str = '' + gpus: tuple[int, ...] = () + started: str = '' + ports: str = '' + runtime: str = DOCKER + #: Kubernetes only: where the pod lives and which container is the engine. + namespace: str = '' + container: str = '' + + @property + def is_engine(self) -> bool: + return bool(self.deployment_id) + + @property + def status(self) -> str: + """``running``, ``restarting (CrashLoopBackOff, 3 restarts)``, ...""" + bits = [b for b in (self.reason, self.health if self.health != 'healthy' else '') + if b] + if self.restarts: + bits.append(f'{self.restarts} restart{"s" if self.restarts != 1 else ""}') + return f'{self.state} ({", ".join(bits)})' if bits else self.state + + +def from_residency(residency, *, runtime: str = DOCKER, namespace: str = '', + container: str = '') -> list[Instance]: + """Every instance in a :class:`~infer_stack.leasing.residency.Residency`.""" + out = [] + for c in residency.all_containers(): + name = c.service if runtime == DOCKER and c.service else c.container_id + out.append(Instance( + name=name, id=c.container_id, deployment_id=c.deployment_id, + state=c.state, restarts=c.restart_count, reason=c.reason, + health=c.health, gpus=tuple(c.gpus), started=c.started, ports=c.ports, + runtime=runtime, namespace=namespace, container=container, + )) + return sorted(out, key=lambda i: (not i.is_engine, i.name, i.id)) + + +def history_argv(instance: Instance, *, tail: str | int | None = None, + timestamps: bool = False) -> list[str] | None: + """The command that prints ``instance``'s log so far, or ``None``.""" + flags = [] if tail is None else ['--tail', str(tail)] + if timestamps: + flags.append('--timestamps') + if instance.runtime == DOCKER: + return ['docker', 'logs', *flags, instance.id] + if instance.runtime == KUBERNETES: + return ['kubectl', '-n', instance.namespace, 'logs', *flags, instance.id, + *(['-c', instance.container] if instance.container else [])] + return None + + +def follow_argv(instance: Instance, *, timestamps: bool = False) -> list[str] | None: + r"""The command that streams ``instance``'s output from now on, or ``None``. + + Docker: ``docker attach`` with stdin closed and signals not forwarded, not + ``docker logs -f``: the log driver holds a partial line until its newline, + so a progress bar redrawn with ``\r`` shows nothing for minutes (measured). + Ending the attach never touches the container. Kubernetes has no such + driver in the way: ``kubectl logs -f --tail 0``. + """ + if instance.runtime == DOCKER: + return ['docker', 'attach', '--no-stdin', '--sig-proxy=false', instance.id] + if instance.runtime == KUBERNETES: + argv = history_argv(instance, tail=0, timestamps=timestamps) + assert argv is not None + return [*argv, '--follow'] + return None + + +def runtime_env(instance: Instance) -> dict[str, str] | None: + """The environment to run the instance's log commands in (None: inherit).""" + if instance.runtime == DOCKER: + from .compose import docker_environment + + return docker_environment() + return None + + +class UnknownTarget(ValueError): + """A name that matches no instance; the message lists what exists.""" + + +def resolve(instances: list[Instance], names: Iterable[str], + served: Mapping[str, Iterable[str]] | None = None) -> list[Instance]: + """The instances ``names`` refer to, in order, without repeats. + + A name matches an instance's name, its id (a prefix of at least four + characters, as Docker accepts), its deployment id, or an endpoint alias + the deployment serves (``served``: deployment id -> aliases). One alias + or deployment can match several instances (a pod and its restart). + + >>> a = Instance('vllm-qwen', 'c0ffee12', 'grp-1', 'running') + >>> b = Instance('litellm', 'deadbeef', '', 'running') + >>> [i.name for i in resolve([a, b], ['qwen', 'litellm'], {'grp-1': ['qwen']})] + ['vllm-qwen', 'litellm'] + >>> [i.name for i in resolve([a, b], ['c0ff'])] + ['vllm-qwen'] + >>> resolve([a, b], ['nope']) + Traceback (most recent call last): + ... + infer_stack.leasing.instances.UnknownTarget: no instance matches 'nope'; running: litellm, vllm-qwen + """ + served = {gid: set(aliases) for gid, aliases in (served or {}).items()} + picked: list[Instance] = [] + for name in names: + found = [ + i for i in instances + if name in (i.name, i.id, i.deployment_id) + or (len(name) >= 4 and i.id.startswith(name)) + or (i.deployment_id and name in served.get(i.deployment_id, ())) + ] + if not found: + known = ', '.join(sorted({i.name for i in instances})) or 'nothing' + raise UnknownTarget(f'no instance matches {name!r}; running: {known}') + picked.extend(i for i in found if i not in picked) + return picked + + +class LogFollower: + r"""Follow instances' logs: recent history, then live output. + + ``list_instances`` is called every ``poll`` seconds, so an instance that + appears later (a model starting, a recreate) is followed from its first + line, and one that restarted is followed again. ``stdout`` yields + ``name | line``. Output written between the history read and the live + stream (a few milliseconds) can be missed. + """ + + poll = 3.0 + + def __init__(self, list_instances: Callable[[], list[Instance]], *, + history: str | int = 200, timestamps: bool = False): + import queue + import threading + + self._list = list_instances + self._history = history + self._timestamps = timestamps + self._lines: queue.Queue = queue.Queue() + self._stop = threading.Event() + self._seen: set[str] = set() + self._live: dict[str, subprocess.Popen] = {} + threading.Thread(target=self._watch, daemon=True).start() + + def _watch(self) -> None: + import threading + + first = True + while not self._stop.is_set(): + try: + found = self._list() + except Exception: # noqa: BLE001 - runtime unreachable: try again + found = [] + for inst in found: + running = inst.state == 'running' + live = self._live.get(inst.id) + attached = live is not None and live.poll() is None + if inst.id not in self._seen: + # Existing instances: the recent tail. A new one: all of it. + history: str | int | None = self._history if first else 'all' + elif running and not attached: + history = None # restarted: follow again, no repeat + else: + continue + self._seen.add(inst.id) + threading.Thread(target=self._follow, args=(inst, history, running), + daemon=True).start() + first = False + self._stop.wait(self.poll) + + def _follow(self, inst: Instance, history, running: bool) -> None: + from ..log_filter import LogLineSplitter + + prefix = f'{inst.name} | ' + env = runtime_env(inst) + if history is not None: + argv = history_argv(inst, tail=history, timestamps=self._timestamps) + old = '' + if argv is not None: + try: + old = subprocess.run( + argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=60, env=env, + ).stdout.decode('utf-8', 'replace') + except Exception: # noqa: BLE001 + old = '' + split = LogLineSplitter(every=0.0) + for line in split.feed(old) + split.flush(): + self._lines.put(prefix + line) + argv = follow_argv(inst, timestamps=self._timestamps) + if not running or argv is None or self._stop.is_set(): + return + # Engines log to stderr, which docker attach passes out on its own stderr. + proc = subprocess.Popen(argv, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=env) + self._live[inst.id] = proc + self._pump(proc, prefix) + + def _pump(self, proc: subprocess.Popen, prefix: str) -> None: + import codecs + import os + + from ..log_filter import LogLineSplitter + + decode = codecs.getincrementaldecoder('utf-8')('replace').decode + split = LogLineSplitter() + assert proc.stdout is not None # Popen(stdout=PIPE) + fd = proc.stdout.fileno() + try: + while True: + chunk = os.read(fd, 65536) + if not chunk: + break + for line in split.feed(decode(chunk)): + self._lines.put(prefix + line) + finally: + proc.stdout.close() + for line in split.flush(): + self._lines.put(prefix + line) + + @property + def stdout(self) -> Iterator[str]: + import queue + + while not self._stop.is_set(): + try: + yield self._lines.get(timeout=0.5) + '\n' + except queue.Empty: + continue + + def terminate(self) -> None: + self._stop.set() + for proc in list(self._live.values()): + try: + proc.terminate() + proc.wait(timeout=2) + except Exception: # noqa: BLE001 + proc.kill() diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index bc5e9446..51ccbabf 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -58,6 +58,21 @@ COMPOSE_PROJECT_LABEL = 'com.docker.compose.project' COMPOSE_SERVICE_LABEL = 'com.docker.compose.service' +def _published_ports(ports) -> str: + """``14042->4000/tcp`` for each published port of a ``docker inspect``. + + >>> _published_ports({'4000/tcp': [{'HostIp': '0.0.0.0', 'HostPort': '14042'}, + ... {'HostIp': '::', 'HostPort': '14042'}], + ... '8000/tcp': None}) + '14042->4000/tcp' + """ + out = [] + for inner, bindings in sorted((ports or {}).items()): + for host in sorted({b.get('HostPort') for b in bindings or [] if b.get('HostPort')}): + out.append(f'{host}->{inner}') + return ', '.join(out) + + #: Container states that hold, or will reclaim on their own, a warm model: the #: process is up, is being restarted by Docker's restart policy, or is paused #: (a paused process keeps its GPU memory). ``created``, ``exited``, @@ -112,6 +127,10 @@ class Container: #: ``CrashLoopBackOff``, ``ImagePullBackOff``, ``OOMKilled`` (Kubernetes). #: Empty when there is nothing to say, or the runtime does not say. reason: str = '' + #: When the current run started (the runtime's timestamp), for display. + started: str = '' + #: Published ports, ``host->container/proto`` joined by ``, ``; display only. + ports: str = '' @property def warm(self) -> bool: @@ -261,6 +280,8 @@ def residency_from_inspect(raw: str, *, project: str) -> Residency: or {}).get('MaximumRetryCount')) or 0), exit_code=(None if (item.get('State') or {}).get('ExitCode') is None else int((item.get('State') or {})['ExitCode'])), + started=str((item.get('State') or {}).get('StartedAt') or ''), + ports=_published_ports((item.get('NetworkSettings') or {}).get('Ports')), ips=tuple(sorted( str(n.get('IPAddress')) for n in (((item.get('NetworkSettings') or {}).get('Networks')) or {}).values() @@ -362,6 +383,8 @@ def residency_from_pods(raw: str) -> Residency: # A Deployment's pods are always restarted by the kubelet. restart_policy='always', reason=waiting or str(ended.get('reason') or ''), + started=str((current.get('running') or {}).get('startedAt') + or status.get('startTime') or ''), ) if container.deployment_id: grouped.setdefault(container.deployment_id, []).append(container) diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 17272da9..439afd00 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -12,8 +12,9 @@ it in Open WebUI. * **Leases** + **Deployments** (center) — the live ledger (desired *state* vs what's actually *running*, and which GPUs), with Release / Evict / Clean-up. -* **docker** — a collapsible pane with **Logs** and **Containers** (the - ``docker ps`` view: status/uptime, created, id, ports) tabs (collapsed by +* **runtime** — a collapsible pane with **Logs**, **Instances** (what + ``infer-stack ps`` shows: containers or pods, status, what each serves, + ports) and **Control** (Apply / Down) tabs, on either backend (collapsed by default; ``c`` toggles it). * **system** — live ``nvidia-smi`` GPUs + host CPU/mem (collapsed by default). * **api** — send a prompt to a *ready* model through the LiteLLM gateway @@ -31,7 +32,7 @@ import subprocess import time from pathlib import Path -from typing import Any, Callable, Iterable +from typing import Any, Callable from textual import events, work from textual.app import App, ComposeResult @@ -69,7 +70,7 @@ from rich.markup import escape as escape_markup from . import cli_equivalent as cli -from .log_filter import LogLineSplitter, compact_litellm_tracebacks +from .log_filter import compact_litellm_tracebacks ALL_SERVICES = '' # the Select value meaning "every service" # The Select value meaning "every service EXCEPT the gateway". This is the @@ -78,12 +79,8 @@ # pane before anyone can read it. The gateway's own logs are one selection away # when they are what you want. ENGINE_SERVICES = '\x00engines' -# Substring rather than equality: compose names the gateway service `litellm` -# today, but deployment naming has carried suffixes before and a missed match -# silently restores the noisy view. -GATEWAY_SERVICE_HINT = 'litellm' # What `_resolve_log_target` returns when a view has nothing to follow. Not -# None: None means "every service" to `docker compose logs`. +# None: None means "every instance". NO_LOG_TARGET = object() @@ -167,15 +164,6 @@ def _select_is_blank(value: object) -> bool: return value is SELECT_BLANK -def is_gateway_service(name: str) -> bool: - """Is this compose service the LiteLLM gateway rather than an engine?""" - return GATEWAY_SERVICE_HINT in str(name).lower() - - -def engine_services(names) -> list[str]: - """Every service that is not the gateway, in the given order.""" - return [n for n in names if not is_gateway_service(n)] - SELECT_MARK = '✓' # multi-select marker in the leases/deployments tables DEFAULT_THEME = 'textual-dark' @@ -257,151 +245,6 @@ def on_key(self, event: events.Key) -> None: ) -class _DockerLogProc: - r"""Follow the project's containers: recent history, then live output. - - Not ``docker compose logs -f``, and not ``docker logs -f`` either: Docker's - log driver stores output line by line and holds a partial line until its - newline, so a download bar redrawn with ``\r`` showed nothing for many - minutes and then arrived all at once (measured with both). ``docker - attach`` reads the container's output as it is written, so each container - gets its recent ``docker logs --tail`` (complete lines), then an attach - with stdin closed and signals not forwarded -- ending it never touches the - container. - - Containers are re-listed every ``poll`` seconds: one created later (a model - starting, a recreate) is followed from its first line, and one that - restarted is attached again. ``stdout`` yields ``service | line`` like - Compose. Output written between the history read and the attach (a few - milliseconds) can be missed. - """ - - poll = 3.0 - - def __init__(self, project: str, compose_file: str, service=None): - import queue - import threading - - del compose_file # the project label is enough to find them - self.project = project - if isinstance(service, (list, tuple)): - self.services = {str(s) for s in service} - else: - self.services = {str(service)} if service else None - self._lines: queue.Queue = queue.Queue() - self._stop = threading.Event() - self._seen: set[str] = set() - self._live: dict[str, subprocess.Popen] = {} - threading.Thread(target=self._watch, daemon=True).start() - - def _containers(self) -> list[tuple[str, str, bool]]: - """(id, service, running) for this project's containers in view.""" - from .leasing.compose import docker_environment - - out = subprocess.run( - ['docker', 'ps', '-a', '--filter', - f'label=com.docker.compose.project={self.project}', - '--format', '{{.ID}} {{.State}} {{.Label "com.docker.compose.service"}}'], - capture_output=True, text=True, timeout=30, env=docker_environment(), - ).stdout - found = [] - for row in out.splitlines(): - cid, _, rest = row.strip().partition(' ') - state, _, svc = rest.partition(' ') - if cid and (self.services is None or svc in self.services): - found.append((cid, svc, state == 'running')) - return sorted(found, key=lambda row: row[1]) - - def _watch(self) -> None: - import threading - - first = True - while not self._stop.is_set(): - try: - found = self._containers() - except Exception: # noqa: BLE001 - docker unreachable: try again - found = [] - for cid, svc, running in found: - attached = cid in self._live and self._live[cid].poll() is None - if cid not in self._seen: - # Existing containers: the recent tail. A new one: all of it. - history = '200' if first else 'all' - elif running and not attached: - history = None # restarted: attach again, no repeat - else: - continue - self._seen.add(cid) - threading.Thread(target=self._follow, args=(cid, svc, history, running), - daemon=True).start() - first = False - self._stop.wait(self.poll) - - def _follow(self, cid: str, service: str, history: str | None, running: bool) -> None: - from .leasing.compose import docker_environment - - prefix = f'{service} | ' - if history is not None: - try: - old = subprocess.run( - ['docker', 'logs', '--tail', history, cid], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60, - env=docker_environment(), - ).stdout.decode('utf-8', 'replace') - except Exception: # noqa: BLE001 - old = '' - split = LogLineSplitter(every=0.0) - for line in split.feed(old) + split.flush(): - self._lines.put(prefix + line) - if not running or self._stop.is_set(): - return - proc = subprocess.Popen( - ['docker', 'attach', '--no-stdin', '--sig-proxy=false', cid], - # Engines log to stderr, which attach passes out on its own stderr. - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=docker_environment(), - ) - self._live[cid] = proc - self._pump(proc, prefix) - - def _pump(self, proc: subprocess.Popen, prefix: str) -> None: - import codecs - import os - - decode = codecs.getincrementaldecoder('utf-8')('replace').decode - split = LogLineSplitter() - assert proc.stdout is not None # Popen(stdout=PIPE) - fd = proc.stdout.fileno() - try: - while True: - chunk = os.read(fd, 65536) - if not chunk: - break - for line in split.feed(decode(chunk)): - self._lines.put(prefix + line) - finally: - proc.stdout.close() - for line in split.flush(): - self._lines.put(prefix + line) - - @property - def stdout(self) -> Iterable[str]: - import queue - - while not self._stop.is_set(): - try: - yield self._lines.get(timeout=0.5) + '\n' - except queue.Empty: - continue - - def terminate(self) -> None: - self._stop.set() - for proc in list(self._live.values()): - try: - proc.terminate() - proc.wait(timeout=2) - except Exception: # noqa: BLE001 - proc.kill() - - class _Divider(Static): """A draggable splitter bar. @@ -1000,7 +843,7 @@ class InferStackTUI(App): Binding('n', 'add_endpoint', 'Add endpoint', show=False), Binding('o', 'open', 'Open in browser', show=False), Binding('y', 'copy_status', 'Copy status', show=False), - Binding('c', 'toggle_docker', 'Toggle docker', show=False), + Binding('c', 'toggle_docker', 'Toggle runtime', show=False), Binding('left_square_bracket', 'sidebar_narrower', 'sidebar -', show=False), Binding('right_square_bracket', 'sidebar_wider', 'sidebar +', show=False), Binding('minus', 'logs_shorter', 'logs -', show=False), @@ -1088,7 +931,7 @@ def __init__( self._log_backlog: collections.deque = collections.deque(maxlen=LOG_PANE_LINES) self._log_dropped = 0 # Parsed compose service names, keyed by the file's (mtime, size). - self._service_names_cache: tuple[tuple[float, int], list[str]] | None = None + self._last_instances: list = [] # Running background actions: worker -> (label, started). Drawn by # _draw_activity, with the newest backend progress message. self._activity: dict[Any, tuple[str, float]] = {} @@ -1227,31 +1070,32 @@ def _compose_dashboard(self) -> ComposeResult: ('Leases', self._compose_leases), ('Deployments', self._compose_deployments)) yield _Divider('y', self._drag_logs, id='hsplit') - with Collapsible(title='docker', collapsed=True, id='docker'): + with Collapsible(title='runtime', collapsed=True, id='docker'): with TabbedContent(id='docker-tabs'): with TabPane('Logs', id='tab-logs'): yield Select( - [('(engines — no litellm)', ENGINE_SERVICES), - ('(all services)', ALL_SERVICES)], + [('(engines)', ENGINE_SERVICES), + ('(everything)', ALL_SERVICES)], value=ENGINE_SERVICES, allow_blank=False, id='logsvc', ) yield RichLog(id='logs', highlight=False, markup=False, max_lines=LOG_PANE_LINES, wrap=False) - with TabPane('Containers', id='tab-containers'): + with TabPane('Instances', id='tab-containers'): yield DataTable(id='ps', cursor_type='row', zebra_stripes=True) with TabPane('Control', id='tab-control'): yield Static( - 'Bring the rendered compose project up or down. ' + 'Apply brings up what the ledger says should run. ' + 'Down stops everything and releases no lease. ' 'Output appears in the Logs tab.', classes='hint', ) yield Static('', id='compose-path') with Horizontal(id='compose-actions'): - yield Button('Compose up', id='btn-compose-up', + yield Button('Apply', id='btn-compose-up', variant='primary') - yield Button('Compose down', + yield Button('Down', id='btn-compose-down') with Collapsible(title='system', collapsed=True, id='system'): yield Static( @@ -1376,7 +1220,7 @@ def on_mount(self) -> None: 'leases', 'held by' ) self.query_one('#ps', DataTable).add_columns( - 'service', 'status (uptime)', 'created', 'container id', 'ports' + 'name', 'status', 'serves', 'started', 'ports' ) self.query_one('#gpus', DataTable).add_columns( 'gpu', 'name', 'util%', 'mem (used/total)', 'temp' @@ -1395,9 +1239,9 @@ def on_mount(self) -> None: '…', '(loading…)', '-', '-', '-' ) self._gpus_rows_cache = [('__loading__',) * 5] - compose_file = getattr(self.controller.backend, 'compose_file', None) + rendered = getattr(self.controller.backend, 'rendered_file', None) self.query_one('#compose-path', Static).update( - f'compose file: {compose_file or "(not rendered yet)"}' + f'rendered: {rendered or "(this backend renders no file)"}' ) # Capture docker's own chatter (up/down progress on stderr) into the # logs pane instead of letting it bleed onto the full-screen terminal. @@ -1505,25 +1349,40 @@ def _report_stall(self, length: float, stack: list[str]) -> None: # -- theming for docker output bleed ---------------------------------- def _install_quiet_docker(self) -> None: - """Route ``docker compose`` output to the logs pane, not the terminal.""" - backend = self.controller.backend - if not hasattr(backend, 'run'): - return + """Route ``docker`` output to the logs pane, not the terminal. + Only ``docker`` commands: the kubeai backend runs ``kubectl`` through + the same seam, and the Docker runner's allowlisted environment has no + ``KUBECONFIG`` (every kubectl call from the TUI used to fail). The + kubeai backend's gateway is a Compose project of its own, so it is + wrapped too. + """ from .leasing.compose import _default_docker_run - def quiet_run(args: list[str], **kwargs) -> str: - # Same bounded, explicit-environment runner as the CLI; only stderr - # is redirected to the logs pane, unless the caller takes it. - noisy = not any(a == 'ps' for a in args) - sink = ( - (lambda line: self.call_from_thread(self._append_log, line)) - if noisy else (lambda line: None) - ) - kwargs.setdefault('stderr_lines', sink) - return _default_docker_run(args, **kwargs) + def quiet(target) -> None: + original = getattr(target, 'run', None) + if original is None: + return + + def quiet_run(args: list[str], **kwargs) -> str: + if not args or args[0] != 'docker': + return original(args, **kwargs) + # Same bounded, explicit-environment runner as the CLI; only + # stderr is redirected to the logs pane, unless the caller + # takes it. + noisy = not any(a == 'ps' for a in args) + sink = ( + (lambda line: self.call_from_thread(self._append_log, line)) + if noisy else (lambda line: None) + ) + kwargs.setdefault('stderr_lines', sink) + return _default_docker_run(args, **kwargs) - backend.run = quiet_run + target.run = quiet_run + + backend = self.controller.backend + quiet(backend) + quiet(getattr(backend, 'gateway', None)) if hasattr(backend, 'progress'): backend.progress = self._backend_progress @@ -1750,8 +1609,8 @@ def _collect(self) -> dict[str, Any]: 'leases': leases, 'deployments': deployments, 'observed': self._observed, 'assignments': self._assignments, } - if not self._collapsed['docker'] and self._active_tab == 'tab-containers': - data['ps'] = self._compose_ps_rows() + if not self._collapsed['docker']: + data['instances'] = self._instance_list() if not self._collapsed['system']: data['gpus'] = self._gpu_rows() data['sysinfo'] = self._system_line() @@ -1769,8 +1628,11 @@ def _render(self, data: dict[str, Any]) -> None: self._fill_deployments( data['deployments'], data['observed'], data['assignments'], data['leases'] ) - if 'ps' in data: - self._fill_ps(data['ps']) + if 'instances' in data: + if data['instances'] is not None: + self._last_instances = data['instances'] + if self._active_tab == 'tab-containers': + self._fill_ps(self._ps_rows(data['instances'])) if 'gpus' in data: self._fill_gpus(data['gpus']) self.query_one('#sysinfo', Static).update(data.get('sysinfo', '')) @@ -1893,7 +1755,7 @@ def _update_summary(self, leases, deployments, observed) -> None: # (~7 KB/s of terminal output while nothing changed). try: docker = self.query_one('#docker', Collapsible) - title = f'docker — {running_label}' + title = f'runtime — {running_label}' if docker.title != title: docker.title = title for pane, text in ( @@ -1997,12 +1859,15 @@ def _fill_deployments(self, deployments, observed, assignments, leases) -> None: def _fill_ps(self, rows) -> None: table = self.query_one('#ps', DataTable) - new_rows = [ - (row['service'], row['status'], row['created'] or '-', - row['id'] or '-', row['ports'] or '-') - for row in rows - ] - if not rows: + if rows is None: + new_rows = [('(cannot read the runtime)', '-', '-', '-', '-')] + else: + new_rows = [ + (row['name'], row['status'], row['serves'] or '-', + row['started'] or '-', row['ports'] or '-') + for row in rows + ] + if rows is not None and not rows: new_rows = [('(nothing running)', '-', '-', '-', '-')] self._diff_fill(table, new_rows, '_ps_rows_cache', id_index=0) @@ -2103,75 +1968,34 @@ def _system_line(self) -> str: # -- docker ps --------------------------------------------------------- - def _compose_ps_rows(self) -> list[dict[str, str]]: - """Best-effort ``docker compose ps`` rows.""" - import json - - backend = self.controller.backend - path = getattr(backend, 'compose_file', None) - run = getattr(backend, 'run', None) - if not path or not run or not Path(path).exists(): - return [] - project = getattr(backend, 'project', 'infer-stack') - try: - out = run([ - 'docker', 'compose', '-p', str(project), '-f', str(path), - 'ps', '--format', 'json', - ]) - except Exception: # noqa: BLE001 - ps is best-effort - return [] - rows: list[dict[str, Any]] = [] - out = (out or '').strip() - if not out: - return [] + def _instance_list(self): + """What the backend runs (worker thread), or ``None`` if unreadable.""" try: - parsed = json.loads(out) - rows = parsed if isinstance(parsed, list) else [parsed] - except json.JSONDecodeError: - for line in out.splitlines(): - line = line.strip() - if line: - try: - rows.append(json.loads(line)) - except json.JSONDecodeError: - pass - result = [] - for row in rows: - ports = _fmt_ports(row) or str(row.get('Ports') or '') - cid = str(row.get('ID') or '')[:12] - # `Status` is the docker-ps STATUS column ("Up 3 minutes") — it - # carries the uptime; `CreatedAt`/`RunningFor` give the age. - created = str(row.get('CreatedAt') or row.get('RunningFor') or '') - result.append({ - 'service': str(row.get('Service') or row.get('Name') or '?'), - 'status': str(row.get('Status') or row.get('State') or '?'), - 'created': created, - 'id': cid, - 'ports': ports, - }) - return sorted(result, key=lambda r: r['service']) + return list(self.controller.backend.instances()) + except Exception: # noqa: BLE001 - a monitor must never crash + return None - # -- logs -------------------------------------------------------------- + def _ps_rows(self, instances) -> list[dict[str, str]] | None: + """Rows for the Instances table (same shape as ``infer-stack ps``).""" + if instances is None: + return None + served = {g.id: sorted(g.served) for g in (self._last_deployments or [])} + return [{ + 'name': i.name, + 'status': i.status, + 'serves': (', '.join(served.get(i.deployment_id, [])) + if i.deployment_id else '(front door)'), + 'started': (i.started or '')[:19].replace('T', ' '), + 'ports': i.ports, + } for i in instances] def _service_names(self) -> list[str]: - """Service names from the on-disk compose file (best-effort).""" - backend = self.controller.backend - path = getattr(backend, 'compose_file', None) - if not path: - return [] - try: - stat = Path(path).stat() - key = (stat.st_mtime, stat.st_size) - cached = self._service_names_cache - if cached is not None and cached[0] == key: - return list(cached[1]) # checked every refresh; parse on change - import yaml - data = yaml.safe_load(Path(path).read_text()) or {} - names = sorted((data.get('services') or {}).keys()) - self._service_names_cache = (key, names) - return list(names) - except Exception: # noqa: BLE001 - return [] + """Names of the instances last seen (refreshed by the worker).""" + return sorted({i.name for i in self._last_instances}) + + def _engine_names(self) -> list[str]: + """Names of the last-seen instances that serve a deployment.""" + return sorted({i.name for i in self._last_instances if i.is_engine}) def _sync_log_services(self) -> None: names = self._service_names() @@ -2180,8 +2004,8 @@ def _sync_log_services(self) -> None: self._service_options = names select = self.query_one('#logsvc', Select) options = [ - ('(engines — no litellm)', ENGINE_SERVICES), - ('(all services)', ALL_SERVICES), + ('(engines)', ENGINE_SERVICES), + ('(everything)', ALL_SERVICES), ] + [(n, n) for n in names] # Keep whatever is selected. The two sentinels are not service names, # so they have to be allowed through explicitly or refreshing the @@ -2224,13 +2048,20 @@ def on_input_changed(self, event: Input.Changed) -> None: self._update_api_curl() def _default_proc_factory(self) -> Callable[[str | None], Any]: - def factory(service: str | None): + def factory(service): + from .leasing.instances import LogFollower + backend = self.controller.backend - path = getattr(backend, 'compose_file', None) - project = getattr(backend, 'project', 'infer-stack') - if not path: - return None - return _DockerLogProc(str(project), str(path), service) + if isinstance(service, (list, tuple)): + wanted: set[str] | None = {str(s) for s in service} + else: + wanted = {str(service)} if service else None + + def listing(): + found = backend.instances() + return [i for i in found if wanted is None or i.name in wanted] + + return LogFollower(listing) return factory @@ -2264,14 +2095,14 @@ def _resolve_log_target(self, service: str): its absence, and it stays that way once engines do appear. """ if service == ENGINE_SERVICES: - names = engine_services(self._service_names()) + names = self._engine_names() if not names: return NO_LOG_TARGET, ( - 'no engine services yet; choose (all services) ' + 'no engines running yet; choose (everything) ' 'for the gateway') return names, f'engines: {", ".join(names)}' if not service: - return None, 'all services' + return None, 'everything' return service, service def _stop_log_proc(self) -> None: @@ -2295,7 +2126,7 @@ def _stream_logs(self, service, generation: int) -> None: if proc is None: self.call_from_thread( self._append_log_if_current, generation, - '(no compose project yet — acquire a model)' + '(nothing to follow — acquire a model)' ) return if generation != self._log_generation: @@ -2863,14 +2694,10 @@ def action_cleanup(self) -> None: # -- docker compose control ------------------------------------------- - def _compose_target(self) -> tuple[Any, str, str] | None: - """(run, project, compose_file) for the leasing project, or None.""" - backend = self.controller.backend - path = getattr(backend, 'compose_file', None) - run = getattr(backend, 'run', None) - if not path or not run or not Path(path).exists(): - return None - return run, str(getattr(backend, 'project', 'infer-stack')), str(path) + def _compose_target(self): + """The backend's rendered file, when a render exists; else ``None``.""" + rendered = getattr(self.controller.backend, 'rendered_file', None) + return rendered if rendered is not None and Path(rendered).exists() else None def action_compose_up(self) -> None: if self._compose_target() is None: @@ -2897,22 +2724,17 @@ def action_compose_down(self) -> None: if self._compose_target() is None: self._refuse('nothing rendered yet — nothing to bring down') return - self._status('docker compose down (raw: bypasses leases; releases nothing)…') + self._status('down (bypasses leases; releases nothing)…') self._cli(cli.command('stack', 'down')) - self._do_compose(['down', '--remove-orphans'], 'down') + self._do_down() @work(thread=True, exclusive=True, group='mutate') - def _do_compose(self, args: list[str], label: str) -> None: - target = self._compose_target() - if target is None: - self._after_mutation('nothing rendered yet') - return - run, project, path = target + def _do_down(self) -> None: try: - run(['docker', 'compose', '-p', project, '-f', path, *args]) - msg = f'compose {label} done' + self.controller.backend.down() + msg = 'down done' except Exception as ex: # noqa: BLE001 - msg = f'compose {label} failed: {ex}' + msg = f'down failed: {ex}' self._after_mutation(msg) # -- open in browser --------------------------------------------------- @@ -3589,18 +3411,6 @@ def _parse_kv_str(text: str) -> dict[str, Any]: return out -def _fmt_ports(row: dict) -> str: - """Compact published-ports string from a compose ps JSON row.""" - pubs = row.get('Publishers') or [] - bits = [] - for pub in pubs: - published = pub.get('PublishedPort') - target = pub.get('TargetPort') - if published: - bits.append(f'{published}->{target}') - return ', '.join(bits) - - def _seconds_since_process_start() -> float | None: """Wall time since this process started (Linux ``/proc``), else ``None``.""" import os diff --git a/tests/test_cli_meta.py b/tests/test_cli_meta.py index d843cb8e..08030a34 100644 --- a/tests/test_cli_meta.py +++ b/tests/test_cli_meta.py @@ -139,15 +139,17 @@ def test_render_rich_colorizes_status() -> None: assert '\x1b[' in out -def test_day2_compose_base_prefers_leasing(tmp_path: Path, monkeypatch) -> None: - from infer_stack.cli.commands_runtime import _day2_compose_base +def test_day2_compose_verbs_target_the_backends_project(tmp_path: Path, monkeypatch) -> None: + from types import SimpleNamespace + + from infer_stack.cli.commands_runtime import _compose_argv monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) compose_file = tmp_path / 'leasing' / 'compose' / 'docker-compose.yml' compose_file.parent.mkdir(parents=True) compose_file.write_text('services: {}\n') - # leasing compose present -> targets it without needing config.yaml - base = _day2_compose_base(None, 'logs') + # The compose backend's own project; no config.yaml needed. + base = _compose_argv(SimpleNamespace(backend='compose')) assert base == [ 'docker', 'compose', '-p', 'infer-stack', '-f', str(compose_file) ] diff --git a/tests/test_day2.py b/tests/test_day2.py new file mode 100644 index 00000000..92fbe895 --- /dev/null +++ b/tests/test_day2.py @@ -0,0 +1,204 @@ +"""Day-2 verbs (ps, logs, status, stack) read the backend, on either backend.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from infer_stack.leasing.instances import ( + DOCKER, + KUBERNETES, + MEMORY, + Instance, + UnknownTarget, + follow_argv, + from_residency, + history_argv, + resolve, +) +from infer_stack.leasing.residency import Container, Residency, ResidencyUnknown + +POD = Instance('model-qwen-0-abc', 'model-qwen-0-abc', 'grp-1', 'running', + runtime=KUBERNETES, namespace='kubeai') +GATEWAY = Instance('litellm', 'c0ffee1234', '', 'running', ports='14042->4000/tcp') + + +def test_log_commands_follow_the_runtime(): + assert history_argv(POD, tail=50) == [ + 'kubectl', '-n', 'kubeai', 'logs', '--tail', '50', 'model-qwen-0-abc'] + assert follow_argv(POD) == [ + 'kubectl', '-n', 'kubeai', 'logs', '--tail', '0', 'model-qwen-0-abc', '--follow'] + assert history_argv(GATEWAY, tail='all', timestamps=True) == [ + 'docker', 'logs', '--tail', 'all', '--timestamps', 'c0ffee1234'] + assert follow_argv(GATEWAY)[:2] == ['docker', 'attach'] + memory = Instance('grp-1', 'grp-1', 'grp-1', 'running', runtime=MEMORY) + assert history_argv(memory) is None and follow_argv(memory) is None + + +def test_instances_come_from_residency_engines_first(): + residency = Residency( + by_deployment={'grp-1': (Container('pod-1', 'grp-1', 'restarting', + restart_count=3, reason='CrashLoopBackOff'),)}, + others=(Container('gw', '', 'running', service='litellm'),), + ) + pods = from_residency(residency, runtime=KUBERNETES, namespace='kubeai') + assert [i.name for i in pods] == ['pod-1', 'gw'] # pods are named by pod + assert pods[0].status == 'restarting (CrashLoopBackOff, 3 restarts)' + containers = from_residency(residency, runtime=DOCKER) + assert [i.name for i in containers] == ['pod-1', 'litellm'] # service names + + +def test_a_target_resolves_by_alias_deployment_name_or_id(): + served = {'grp-1': ['qwen']} + both = [POD, GATEWAY] + assert resolve(both, ['qwen'], served) == [POD] + assert resolve(both, ['grp-1', 'litellm'], served) == [POD, GATEWAY] + assert resolve(both, ['c0ff'], served) == [GATEWAY] + with pytest.raises(UnknownTarget, match='running: litellm, model-qwen-0-abc'): + resolve(both, ['nope'], served) + + +class _Backend: + """A backend as the day-2 verbs see it.""" + + def __init__(self, instances, *, residency=None): + self._instances = instances + self._residency = residency + self.down_calls = 0 + + def instances(self): + if self._instances is None: + raise ResidencyUnknown('kubectl get pods failed') + return list(self._instances) + + def residency(self): + if self._residency is None: + raise ResidencyUnknown('kubectl get pods failed') + return self._residency + + def down(self): + self.down_calls += 1 + + +def _cli(monkeypatch, backend, served=None): + from infer_stack.cli import commands_runtime as rt + + monkeypatch.setattr(rt, '_day2_backend', lambda config: backend) + monkeypatch.setattr(rt, '_served_by_deployment', lambda: dict(served or {})) + return rt + + +def test_ps_has_one_shape_and_names_what_each_serves(monkeypatch, capsys): + rt = _cli(monkeypatch, _Backend([POD, GATEWAY]), {'grp-1': ['qwen']}) + assert rt.PsCLI.main(argv=[]) == 0 + out = capsys.readouterr().out.splitlines() + assert out[0].split() == ['NAME', 'STATUS', 'SERVES', 'GPUS', 'STARTED', 'ID', 'PORTS'] + assert 'qwen' in out[1] and 'model-qwen-0-abc' in out[1] + assert '(front door)' in out[2] and '14042->4000/tcp' in out[2] + + assert rt.PsCLI.main(argv=['qwen', '--json']) == 0 + rows = json.loads(capsys.readouterr().out) + assert [r['name'] for r in rows] == ['model-qwen-0-abc'] + assert rows[0]['serves'] == ['qwen'] and rows[0]['runtime'] == KUBERNETES + + +def test_ps_hides_finished_instances_unless_all(monkeypatch, capsys): + done = Instance('old', 'dead0000', 'grp-2', 'exited') + rt = _cli(monkeypatch, _Backend([POD, done])) + rt.PsCLI.main(argv=['-q']) + assert capsys.readouterr().out.split() == ['model-qwen-0-abc'] + rt.PsCLI.main(argv=['-q', '--all']) + assert capsys.readouterr().out.split() == ['model-qwen-0-abc', 'dead0000'] + + +def test_ps_says_why_when_the_runtime_cannot_be_read(monkeypatch): + rt = _cli(monkeypatch, _Backend(None)) + with pytest.raises(SystemExit, match='cannot read what is running'): + rt.PsCLI.main(argv=[]) + + +def test_logs_reads_the_named_endpoint_through_its_runtime(monkeypatch, capsys): + import subprocess + + rt = _cli(monkeypatch, _Backend([POD, GATEWAY]), {'grp-1': ['qwen']}) + ran = [] + + def fake_run(argv, **kw): + ran.append(argv) + return SimpleNamespace(stdout=b'INFO: loaded\n', returncode=0) + + monkeypatch.setattr(subprocess, 'run', fake_run) + assert rt.LogsCLI.main(argv=['qwen', '--tail', '5']) == 0 + assert ran == [['kubectl', '-n', 'kubeai', 'logs', '--tail', '5', 'model-qwen-0-abc']] + assert capsys.readouterr().out == 'INFO: loaded\n' # one instance: no prefix + with pytest.raises(SystemExit, match="no instance matches 'nope'"): + rt.LogsCLI.main(argv=['nope']) + + +def test_status_health_comes_from_residency(monkeypatch): + from infer_stack.cli.commands_runtime import _served_models + from infer_stack.leasing import DeploymentState + + def dep(gid): + return SimpleNamespace(id=gid, state=DeploymentState.LIVE, engine='vllm', + served={gid: {'hf_model_id': 'org/m'}}, spec={}) + + residency = Residency(by_deployment={ + 'up': (Container('c1', 'up', 'running'),), + 'loading': (Container('c3', 'loading', 'running', health='starting'),), + 'crashing': (Container('c2', 'crashing', 'exited'),), + }) + rows = _served_models([dep('up'), dep('loading'), dep('crashing'), dep('gone')], + _Backend([], residency=residency)) + assert {r[0]: r[3] for r in rows} == {'up': 'up', 'loading': 'starting', + 'crashing': 'exited', 'gone': 'STALE'} + rows = _served_models([dep('up')], _Backend([], residency=None)) + assert rows[0][3] == 'unverified' + + +def test_stack_down_stops_the_backend_on_either_backend(monkeypatch): + backend = _Backend([]) + rt = _cli(monkeypatch, backend) + assert rt.StackDownCLI.main(argv=[]) == 0 + assert backend.down_calls == 1 + + +def test_stack_compose_targets_the_compose_project_on_this_host(monkeypatch, tmp_path): + from infer_stack.cli import commands_runtime as rt + + compose_file = tmp_path / 'docker-compose.yml' + project = SimpleNamespace(compose_file=compose_file, + compose_argv=lambda: ['docker', 'compose', '-p', 'gw']) + kubeai = SimpleNamespace(compose_project=lambda: project) + monkeypatch.setattr(rt, '_day2_backend', lambda config: kubeai) + with pytest.raises(SystemExit, match='nothing rendered yet'): + rt._compose_argv(SimpleNamespace()) + compose_file.write_text('services: {}\n') + assert rt._compose_argv(SimpleNamespace()) == ['docker', 'compose', '-p', 'gw'] + + monkeypatch.setattr(rt, '_day2_backend', lambda config: SimpleNamespace()) + with pytest.raises(SystemExit, match='no Compose project on this host'): + rt._compose_argv(SimpleNamespace()) + + +@pytest.mark.parametrize(('argv', 'follow', 'names'), [ + (['-f', 'qwen'], True, ['qwen']), + (['qwen', '-f'], True, ['qwen']), + (['--follow', 'qwen', 'litellm'], True, ['qwen', 'litellm']), + (['-f', 'false'], False, []), +]) +def test_a_flag_never_swallows_the_positional_after_it(argv, follow, names): + """kwconf flags take an optional value; `logs -f qwen` followed everything.""" + from infer_stack.cli.commands_runtime import LogsCLI + + config = LogsCLI.cli(argv=argv) + assert config.follow is follow and list(config.services or []) == names + + +def test_acquire_yes_before_the_endpoint_still_names_it(): + from infer_stack.cli.commands_leasing import AcquireCLI + + config = AcquireCLI.cli(argv=['--yes', 'qwen']) + assert config.yes is True and config.names == ['qwen'] diff --git a/tests/test_log_filter.py b/tests/test_log_filter.py index 46e26760..281ace38 100644 --- a/tests/test_log_filter.py +++ b/tests/test_log_filter.py @@ -262,84 +262,51 @@ def test_eof_mid_traceback_fails_open(): assert list(compact_litellm_tracebacks(original)) == original -def test_cli_follow_compaction_helper_streams_known_trace(monkeypatch): +def _follow_logs(monkeypatch, lines, *, no_color, tty=True): + """Run `infer-stack logs -f` over a fake instance whose log is ``lines``.""" from infer_stack.cli import commands_runtime + from infer_stack.leasing import instances as instances_mod + from infer_stack.leasing.instances import Instance - source = _known_segments()[0] + config = SimpleNamespace(follow=True, raw=False, no_color=no_color, tail=None, + timestamps=False, services=None) - class FakeProc: - def __init__(self): - self.stdout = iter(source) + class Backend: + def instances(self): + return [Instance('litellm', 'c1', '', 'running')] - def wait(self, timeout=None): - return 0 - - def poll(self): - return 0 + class Follower: + def __init__(self, listing, **kw): + assert [i.name for i in listing()] == ['litellm'] + self.stdout = iter(lines) def terminate(self): - raise AssertionError('completed process should not be terminated') + pass - def kill(self): - raise AssertionError('completed process should not be killed') + class Out(io.StringIO): + def isatty(self): + return tty - monkeypatch.setattr(commands_runtime.subprocess, 'Popen', lambda *a, **kw: FakeProc()) - stream = io.StringIO() + monkeypatch.setattr(commands_runtime.LogsCLI, 'cli', lambda *a, **kw: config) + monkeypatch.setattr(commands_runtime, '_day2_backend', lambda config: Backend()) + monkeypatch.setattr(commands_runtime, '_served_by_deployment', lambda: {}) + monkeypatch.setattr(instances_mod, 'LogFollower', Follower) + stream = Out() monkeypatch.setattr(commands_runtime.sys, 'stdout', stream) + assert commands_runtime.LogsCLI.main(argv=False) == 0 + return stream.getvalue() - rc = commands_runtime._run_compacted_follow(['docker', 'compose', 'logs', '-f']) - assert rc == 0 - assert 'Traceback (most recent call last):' not in stream.getvalue() - assert 'ConnectionRefusedError: [Errno 111]' in stream.getvalue() - -@pytest.mark.parametrize(('no_color', 'expected_ansi'), [ - (False, True), - (True, False), -]) -def test_cli_compacted_follow_preserves_compose_color_mode( - monkeypatch, no_color, expected_ansi -): - from infer_stack.cli import commands_runtime - - config = SimpleNamespace( - follow=True, - raw=False, - no_color=no_color, - tail=None, - timestamps=False, - services=None, - ) - - class TTY(io.StringIO): - def isatty(self): - return True - - captured = {} - monkeypatch.setattr( - commands_runtime.LogsCLI, 'cli', lambda *args, **kwargs: config - ) - monkeypatch.setattr( - commands_runtime, - '_day2_compose_base', - lambda config, purpose: [ - 'docker', 'compose', '-p', 'infer-stack', '-f', 'compose.yml' - ], - ) - def fake_follow(cmd): - captured['cmd'] = cmd - return 0 +def test_cli_follow_compacts_a_known_trace(monkeypatch): + lines = [line.replace('litellm-1 | ', 'litellm | ') for line in _known_segments()[0]] + out = _follow_logs(monkeypatch, lines, no_color=True) + assert 'Traceback (most recent call last):' not in out + assert 'ConnectionRefusedError: [Errno 111]' in out - monkeypatch.setattr(commands_runtime, '_run_compacted_follow', fake_follow) - monkeypatch.setattr(commands_runtime.sys, 'stdout', TTY()) - assert commands_runtime.LogsCLI.main(argv=False) == 0 - cmd = captured['cmd'] - logs_index = cmd.index('logs') - assert ('--ansi' in cmd) is expected_ansi - if expected_ansi: - ansi_index = cmd.index('--ansi') - assert cmd[ansi_index + 1] == 'always' - assert ansi_index < logs_index - assert ('--no-color' in cmd) is no_color +@pytest.mark.parametrize('no_color', [False, True]) +def test_cli_follow_colors_prefixes_unless_asked_not_to(monkeypatch, no_color): + out = _follow_logs(monkeypatch, ['litellm | hello\n'], no_color=no_color) + assert ('\x1b[' in out) is (not no_color) + assert 'hello' in out diff --git a/tests/test_tui.py b/tests/test_tui.py index e1557e87..ecccb8b1 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import json import pytest @@ -253,7 +252,7 @@ async def scenario(): proc_factory=lambda svc: None) async with app.run_test() as pilot: await pilot.pause() - # docker is a collapsible pane with Logs/Containers/Control tabs; + # runtime is a collapsible pane with Logs/Instances/Control tabs; # system is its own collapsed pane; API is a top-level tab now. tabs = app.query_one('#docker-tabs', TabbedContent) assert {p.id for p in tabs.query('TabPane')} == { @@ -263,12 +262,10 @@ async def scenario(): assert app.query_one('#system', Collapsible).collapsed top = app.query_one('#top', TabbedContent) assert 'tab-api' in {p.id for p in top.query('TabPane')} - # the containers ps view carries the docker-ps columns + # the instances view carries what `infer-stack ps` shows ps = app.query_one('#ps', DataTable) labels = [str(c.label) for c in ps.columns.values()] - assert 'status (uptime)' in labels - assert 'created' in labels - assert 'container id' in labels + assert labels == ['name', 'status', 'serves', 'started', 'ports'] _run(scenario) @@ -915,35 +912,33 @@ async def scenario(): _run(scenario) -def test_tui_ps_rows_parse_status_created_and_id(): +def test_tui_instance_rows_say_what_each_serves(): + from types import SimpleNamespace + + from infer_stack.leasing.instances import Instance from infer_stack.tui import InferStackTUI controller, catalog = _ctx() - sample = json.dumps([ - {'Service': 'litellm', 'Status': 'Up 3 minutes', 'State': 'running', - 'CreatedAt': '2026-06-19 00:00:00 -0400', 'ID': 'abcdef1234567890', - 'Publishers': [{'PublishedPort': 14042, 'TargetPort': 4000}]}, - ]) + instances = [ + Instance('vllm-qwen', 'abcdef1234567890', 'grp-1', 'restarting', + restarts=2, reason='CrashLoopBackOff', + started='2026-06-19T00:00:00.123Z'), + Instance('litellm', 'fedcba', '', 'running', ports='14042->4000/tcp'), + ] async def scenario(): app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None) async with app.run_test() as pilot: await pilot.pause() - # stub the backend's compose seam - backend = controller.backend - import tempfile - f = tempfile.NamedTemporaryFile('w', suffix='.yml', delete=False) - f.write('services: {}\n') - f.close() - backend.compose_file = f.name - backend.project = 'infer-stack' - backend.run = lambda args: sample - rows = app._compose_ps_rows() - assert rows[0]['status'] == 'Up 3 minutes' - assert rows[0]['created'].startswith('2026-06-19') - assert rows[0]['id'] == 'abcdef123456' # truncated to 12 - assert '14042->4000' in rows[0]['ports'] + app._last_deployments = [SimpleNamespace(id='grp-1', served={'qwen': {}})] + engine, gateway = app._ps_rows(instances) + assert engine['serves'] == 'qwen' + assert engine['status'] == 'restarting (CrashLoopBackOff, 2 restarts)' + assert engine['started'] == '2026-06-19 00:00:00' + assert gateway['serves'] == '(front door)' + assert gateway['ports'] == '14042->4000/tcp' + assert app._ps_rows(None) is None # the runtime was unreadable _run(scenario) @@ -980,11 +975,11 @@ async def scenario(): await pilot.pause() app._active_tab = 'tab-containers' app._collapsed['docker'] = True - assert 'ps' not in app._collect() # collapsed -> no ps poll + assert 'instances' not in app._collect() # collapsed -> no poll app._collapsed['docker'] = False - assert 'ps' in app._collect() # visible -> polled + assert 'instances' in app._collect() # visible -> polled app._active_tab = 'tab-logs' - assert 'ps' not in app._collect() # other tab -> no ps poll + assert 'instances' in app._collect() # the log picker needs names app._collapsed['system'] = True assert 'gpus' not in app._collect() # system collapsed app._collapsed['system'] = False @@ -1441,8 +1436,7 @@ async def scenario(): async with app.run_test() as pilot: await pilot.pause() backend = controller.backend - backend.compose_file = compose_file - backend.project = 'infer-stack' + backend.rendered_file = compose_file backend.run = lambda args: calls.append(args) or '' app.action_compose_up() await app.workers.wait_for_complete() @@ -1704,27 +1698,22 @@ def test_gateway_services_are_excluded_from_the_default_log_view(): """The logs pane defaults to engines, not everything. LiteLLM logs a line per proxied request, so on a busy host it scrolls the - engine output -- where errors actually appear -- out of the pane. + engine output -- where errors actually appear -- out of the pane. An + engine is an instance that serves a deployment; the gateway, UI, database + and proxy serve none, whatever they are named (a name hint used to let + Open WebUI and Postgres into the engines view). """ - from infer_stack.tui import ( - ALL_SERVICES, - ENGINE_SERVICES, - engine_services, - is_gateway_service, - ) + from infer_stack.leasing.instances import Instance + from infer_stack.tui import ALL_SERVICES, ENGINE_SERVICES - assert is_gateway_service('litellm') - # Substring, not equality: a suffixed gateway service must still match, or - # the noisy view comes back silently. - assert is_gateway_service('infer-stack-litellm-1') - assert not is_gateway_service('vllm-qwen-qwen3-8-27b') - - names = ['litellm', 'vllm-a', 'vllm-b'] - assert engine_services(names) == ['vllm-a', 'vllm-b'] + engine = Instance('vllm-a', 'c1', 'grp-1', 'running') + assert engine.is_engine + for name in ('litellm', 'infer-stack-litellm-1', 'open-webui', 'postgres'): + assert not Instance(name, 'c2', '', 'running').is_engine # The two sentinels must stay distinguishable from each other and from any - # real service name. + # real instance name. assert ENGINE_SERVICES != ALL_SERVICES - assert ENGINE_SERVICES not in names + assert ENGINE_SERVICES not in ('litellm', 'vllm-a') def test_named_log_process_follows_only_that_service(monkeypatch): @@ -1733,7 +1722,8 @@ def test_named_log_process_follows_only_that_service(monkeypatch): import subprocess import time - from infer_stack.tui import _DockerLogProc + from infer_stack.leasing.instances import Instance + from infer_stack.tui import InferStackTUI service = 'vllm-qwen3-8-27b-dbirks-hyperqwen' touched = [] @@ -1743,8 +1733,6 @@ def __init__(self, stdout): self.stdout = stdout def fake_run(cmd, **kwargs): - if cmd[:2] == ['docker', 'ps']: - return _Done(f'c1 running {service}\nc2 running litellm\n') touched.append(cmd[-1]) # docker logs --tail N return _Done(b'') @@ -1764,9 +1752,15 @@ def terminate(self): def wait(self, timeout=None): return 0 + controller, catalog = _ctx() + controller.backend.instances = lambda: [ + Instance(service, 'c1', 'grp-1', 'running'), + Instance('litellm', 'c2', '', 'running'), + ] monkeypatch.setattr(subprocess, 'run', fake_run) monkeypatch.setattr(subprocess, 'Popen', lambda cmd, **kw: _Proc(cmd)) - proc = _DockerLogProc('infer-stack', '/tmp/docker-compose.yml', service) + app = InferStackTUI(controller, catalog, interval=999) + proc = app._default_proc_factory()(service) deadline = time.monotonic() + 5 while len(touched) < 2 and time.monotonic() < deadline: time.sleep(0.05) @@ -1811,7 +1805,10 @@ async def scenario(): proc_factory=lambda svc: None) async with app.run_test() as pilot: await pilot.pause() - app._service_names = lambda: ['litellm', 'vllm-a', 'vllm-b'] + from infer_stack.leasing.instances import Instance + app._last_instances = [Instance('litellm', 'c0', '', 'running'), + Instance('vllm-a', 'c1', 'g1', 'running'), + Instance('vllm-b', 'c2', 'g2', 'running')] target, label = app._resolve_log_target(ENGINE_SERVICES) assert target == ['vllm-a', 'vllm-b'] @@ -1820,16 +1817,16 @@ async def scenario(): # A named service is passed straight through. assert app._resolve_log_target('litellm') == ('litellm', 'litellm') - # "all" stays None so docker compose logs gets no service argument. + # "everything" stays None: the follower takes every instance. target, label = app._resolve_log_target(ALL_SERVICES) - assert target is None and label == 'all services' + assert target is None and label == 'everything' # With no engines there is nothing to follow. Falling back to - # every service would show the gateway under the engines label. + # every instance would show the gateway under the engines label. from infer_stack.tui import NO_LOG_TARGET - app._service_names = lambda: ['litellm'] + app._last_instances = [Instance('litellm', 'c0', '', 'running')] target, label = app._resolve_log_target(ENGINE_SERVICES) - assert target is NO_LOG_TARGET and 'no engine services' in label + assert target is NO_LOG_TARGET and 'no engines' in label _run(scenario) @@ -1855,15 +1852,16 @@ async def scenario(): proc_factory=factory) async with app.run_test() as pilot: await pilot.pause() - services = ['litellm'] - app._service_names = lambda: list(services) + from infer_stack.leasing.instances import Instance + app._last_instances = [Instance('litellm', 'c0', '', 'running')] app._collapsed['docker'] = False app._sync_log_services() app._restart_logs(app._log_service) # the pane opens await pilot.pause(0.2) assert started == [] # nothing to follow yet - services.append('vllm-a') # an engine is deployed + app._last_instances = [*app._last_instances, # an engine is deployed + Instance('vllm-a', 'c1', 'g1', 'running')] app._sync_log_services() await pilot.pause(0.2) await pilot.pause() From 46b49b7a2cd49adcb4a34068d2c7df26e19cf6ac Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 11:55:41 -0400 Subject: [PATCH 028/141] Queue: record the P2 commit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index d5e57cb5..cf6c75e7 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -44,7 +44,7 @@ queue-semantics tests still assert the same behaviour. ### 3. [x] P2: day-2 commands and the TUI through the backend -Done 2026-09-26. +Done 2026-09-26, `c2d393b`. `instances()` and `stream_logs(target, *, follow, tail)` on both backends; `ps`, `logs`, `stack up` / `stack down` use them, the raw compose form From 7a322c1fb1cfb842fc958569b5198f7fb74e4096 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 12:19:37 -0400 Subject: [PATCH 029/141] P3: the gateway in front of a cluster is the compose gateway The kubeai gateway project takes the compose front-door settings (ui, reverse_proxy, dynamic_routing), recorded in the recovery profile as the gateway project's own profile. The kubeai backend hands the gateway explicit render inputs (registry rows for the catalog and every Model, or per-deployment dynamic routes via the shared upstream_route) instead of writing rows into the registry before approval, so the gateway's diff is previewed and approved with the acquire's. Under dynamic routing each deployment is its own Model, named with compose's deployment tail. Duplicate authorities removed: a route's upstream (render vs routes list), a deployment's Model name (five places), and isinstance(ComposeBackend) checks in routes, gc/clean --orphans, network migrate and secrets rotate, now the capability each needs. Verified: full suite; compose e2e non-GPU tiers; dev/kubeai_e2e.sh on k3s with new steps for routes list, Open WebUI, and two --dedicated Models behind one alias under dynamic routing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 13 +++ dev/kubeai_e2e.sh | 53 +++++++++ docs/backend-parity.md | 7 +- docs/kubeai-backend.md | 7 +- docs/planning/backend-parity-roadmap.md | 20 +++- docs/queue.md | 4 +- infer_stack/backends/kubeai.py | 134 +++++++++++++++++++---- infer_stack/cli/commands_leasing.py | 102 +++++++++--------- infer_stack/leasing/compose.py | 28 ++++- infer_stack/leasing/gateway.py | 83 ++++++++------ infer_stack/leasing/naming.py | 12 ++- tests/test_leasing_kubeai.py | 138 +++++++++++++++++++++++- 12 files changed, 475 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aacb24f9..1bde84cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### The gateway in front of a cluster is the compose gateway + +On the kubeai backend the gateway now takes the same settings as on compose: +Open WebUI (`ui`, on by default), the reverse proxy, and dynamic routing, +under which each deployment is its own Model (`-`), so the +same model acquired `--dedicated` twice runs as two Models behind one alias. +Catalog endpoints are routed before they run, as on compose, so a new Model +no longer recreates the gateway. The gateway's changes are shown and +approved with the acquire's, before its lease commits. `routes list / seed / +prune` work on kubeai and show a cluster route's upstream (they showed `?`), +and `gc --orphans` and `clean --orphans` run there (and find nothing: the +backend only reads its own labelled pods). + ### `ps`, `logs`, `status` and the TUI read the backend, on either backend `infer-stack ps` lists what the backend runs, containers or pods and the diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index f735e1e2..e106ee75 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -24,6 +24,9 @@ # GATEWAY 1 (default): clients go through infer-stack's LiteLLM # gateway, as on the compose backend. 0: straight to # KubeAI, where the alias below does NOT route (404). +# E2E_DYNAMIC 1 (default): finish with dynamic routing, two +# --dedicated leases of one model (two Models at once). +# E2E_UI_PORT Open WebUI's port on this host (default 13000). set -euo pipefail MODEL="${E2E_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" @@ -45,6 +48,7 @@ cleanup() { set +e run_is release --all --yes >/dev/null 2>&1 run_is gc --evict --yes >/dev/null 2>&1 + run_is stack down >/dev/null 2>&1 # the gateway, UI and database too # nothing managed may remain on the cluster, pass or fail leftover=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ -l infer-stack/managed=true -o name 2>/dev/null | wc -l) @@ -103,6 +107,26 @@ else exit 1 fi +if [ "${GATEWAY:-1}" = 1 ]; then + echo '== the gateway fronts the cluster as on compose: routes, Open WebUI' + if ! run_is routes list --json | python3 -c ' +import json, sys +alias = sys.argv[1] +row = next(r for r in json.load(sys.stdin)["routes"] if r["name"] == alias) +assert row["engine"] == "upstream" and row["live"], row +print(" routes list:", alias, "->", row["target"], "at", row["upstream"])' "$ALIAS"; then + echo '!! routes list does not route the alias to the cluster' >&2; exit 1 + fi + ui='' + for _ in $(seq 60); do + ui=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${E2E_UI_PORT:-13000}/" || true) + [ "$ui" = 200 ] && break + sleep 2 + done + [ "$ui" = 200 ] || { echo "!! Open WebUI did not answer (HTTP $ui)" >&2; exit 1; } + echo ' Open WebUI answers in front of the cluster' +fi + echo '== release prunes the Model (reclaim: stop)' run_is release --env-file "$WORK/lease.env" --yes remaining=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ @@ -161,4 +185,33 @@ EOF run_is release --env-file "$WORK/big.env" --yes fi +if [ "${GATEWAY:-1}" = 1 ] && [ "${E2E_DYNAMIC:-1}" = 1 ]; then + echo '== dynamic routing: two --dedicated leases on one model, two Models, one alias' + run_is stack down >/dev/null 2>&1 # the gateway comes back with Postgres + run_is config set dynamic_routing true + for n in 1 2; do + run_is acquire "$ALIAS" --dedicated --yes --ttl 30m --timeout "$TIMEOUT" \ + --env-file "$WORK/dyn$n.env" + done + models=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ + -l infer-stack/managed=true -o name | wc -l) + [ "$models" = 2 ] || { echo "!! $models Model(s), expected 2" >&2; exit 1; } + # shellcheck disable=SC1090 + source "$WORK/dyn1.env" + routes=$(curl -s "$OPENAI_BASE_URL/model/info" -H "Authorization: Bearer $OPENAI_API_KEY" \ + | python3 -c 'import json,sys; print(sum(m["model_name"] == sys.argv[1] for m in json.load(sys.stdin)["data"]))' "$ALIAS") + [ "$routes" = 2 ] || { echo "!! $routes route(s) for $ALIAS, expected 2" >&2; exit 1; } + if curl -sS --fail-with-body "$OPENAI_BASE_URL/chat/completions" \ + -H "Authorization: Bearer $OPENAI_API_KEY" -H 'Content-Type: application/json' \ + -d "{\"model\": \"$ALIAS\", \"max_tokens\": 4, + \"messages\": [{\"role\": \"user\", \"content\": \"say ok\"}]}" \ + | grep -q 'choices'; then + echo ' two Models, two routes under one alias, and it answers' + else + echo "!! no generation for $ALIAS under dynamic routing" >&2; exit 1 + fi + run_is release --env-file "$WORK/dyn1.env" --yes + run_is release --env-file "$WORK/dyn2.env" --yes +fi + echo 'PASS: kubeai backend end-to-end lifecycle' diff --git a/docs/backend-parity.md b/docs/backend-parity.md index 711ccd42..9f399cc4 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -107,9 +107,10 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | static superset routes, no blip on model churn | same | same | -| `routes` inspect / seed / prune | yes | **gap** (P3): refused, although the registry exists | -| `dynamic_routing` (admin API + Postgres; distinct upstreams for same-model `--dedicated`) | yes | **gap** (P3) | -| Open WebUI (`ui`), reverse proxy | yes | **gap** (P3): the gateway project is rendered with `ui` off | +| `routes` inspect / seed / prune | yes | same; a route points at the cluster under the Model's name | +| `dynamic_routing` (admin API + Postgres; distinct upstreams for same-model `--dedicated`) | yes | same: each deployment is its own Model (`-`) | +| Open WebUI (`ui`), reverse proxy | yes | same, in the gateway's project on this host | +| the gateway's changes approved with the acquire's, before the lease commits | yes | same | | `network migrate` / `network check` | yes | n/a: Service addresses are stable | | where the gateway runs | this host | this host, so it is in every request's path (P5 moves it into the cluster) | diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 48e418ef..8cf548f1 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -148,5 +148,8 @@ infer-stack release --env-file lease.env as on compose. `stack compose …` and `stack restart` act on the gateway's Compose project; the engines are pods, restarted by the kubelet. - The gateway runs on the host running infer-stack, so that host is in every - request's path. Dynamic routing (`dynamic_routing`) is compose-only; the - KubeAI gateway uses static routes. + request's path. It takes the same settings as on compose: `ui` (Open WebUI, + on by default), `reverse_proxy`, and `dynamic_routing`, under which each + deployment is its own Model (`-`), so `--dedicated` twice + gives two Models behind one alias. The gateway's config changes are shown + and approved with the acquire's, before its lease commits. diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 4d05d473..6eacb09a 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,7 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **P1, P2 done** 2026-09-26 · P3–P5 not started · +`dev/backend-unification` · **P1–P3 done** 2026-09-26 · P4, P5 not started · P6 is ongoing. Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the @@ -155,6 +155,18 @@ step that acquires the same model `--dedicated` twice under dynamic routing and gets two Models and two routes. **Size:** small to medium; the `Gateway` class already owns the pieces. +**P3 done 2026-09-26.** The KubeAI gateway takes the compose settings (`ui`, +`reverse_proxy`, `dynamic_routing`) and records them in the recovery profile +as the gateway project's own profile. The kubeai backend hands the gateway +explicit render inputs (registry rows for the catalog and every Model, or +per-deployment dynamic routes) instead of writing rows into the registry +before approval, so the gateway's changes are previewed and approved with +the acquire's. Under dynamic routing each deployment is its own Model, +named with the deployment tail compose uses for its services. The `routes` +commands resolve the backend's Compose project and ask the backend for its +rows. Verified by `dev/kubeai_e2e.sh` on k3s: routes, Open WebUI, and two +`--dedicated` Models behind one alias. + ### P4. Placement information parity (optional) **Closes:** `min_vram_gib` / `measure`; less hand-written cluster @@ -218,7 +230,11 @@ worse. A blocker is fixed whatever its size. | engine vs gateway in the TUI | a `litellm` name hint (Open WebUI and Postgres counted as engines) | **fixed** (P2): an instance serves a deployment or it does not | | where the day-2 verbs find the Compose project | a hard-coded path and project name | **fixed** (P2): the backend's `compose_project()` and `compose_argv()` | | how the TUI runs a runtime command | it replaced the backend's runner with Docker's, whose allowlisted environment has no `KUBECONFIG`: every kubectl call from the TUI failed | **fixed** (P2): only `docker` commands are wrapped | -| the KubeAI gateway's approval | the gateway project asks its own diff approval at render, after the lease commits, not in the admission preview | deferred to P3. Same result under `--yes`; interactively, a declined gateway change rolls the lease back after the commit | +| the KubeAI gateway's approval | the gateway project asked its own diff approval at render, after the lease committed | **fixed** (P3): previewed and approved with the acquire's | +| KubeAI's route rows | written into the registry before any approval, and only for live Models (so every new Model recreated the gateway) | **fixed** (P3): render inputs, persisted after approval; the catalog's rows too, so no blip | +| a route's upstream | derived twice, in the render and in `routes list` (which showed `?` for a cluster route) | **fixed** (P3): `registry_route_entry` | +| a deployment's Model name | derived in five places | **fixed** (P3): `model_name()` | +| "is this the compose backend" in the CLI | `isinstance(…, ComposeBackend)` for `routes`, `gc --orphans`, `clean`, `network migrate`, `secrets rotate` | **fixed** (P3): the capability each needs (`compose_project()`, residency labels, `network`, `litellm`) | ## Not in scope diff --git a/docs/queue.md b/docs/queue.md index cf6c75e7..8bdd3b36 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -56,7 +56,9 @@ memory line reaches the pod log. with the same output shape as on Compose, and the TUI follows a Model's log and lists its pod (checked in a real terminal, not only in tests). -### 4. [ ] P3: gateway feature parity +### 4. [x] P3: gateway feature parity + +Done 2026-09-26. `routes` and `secrets rotate` resolve the backend's gateway instead of checking its kind; the KubeAI gateway project honours `ui`, diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index d3117e99..1852860c 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -53,14 +53,32 @@ DEFAULT_BASE_URL = 'http://127.0.0.1:8000/openai/v1' -def model_name_for(served: str) -> str: +def model_name_for(served: str, deployment_id: str | None = None) -> str: """Deterministic Model CR name for a served model name: ````. Derived purely from the served name (like the compose service name), so it is identical across releases/re-acquires of the same endpoint — the request - name clients use through the KubeAI gateway never changes. + name clients use through the KubeAI gateway never changes. With + ``deployment_id`` (dynamic routing), the deployment's tail is appended, as + for a compose service, so same-model ``--dedicated`` deployments are + separate Models; the gateway addresses each by name. + + >>> model_name_for('Qwen/Qwen3-8B') + 'qwen-qwen3-8b' + >>> model_name_for('Qwen/Qwen3-8B', 'grp-0123456789ab') + 'qwen-qwen3-8b-01234567' """ - return dns_slug(served) + if deployment_id is None: + return dns_slug(served) + from ..leasing.naming import deployment_tail + + tail = deployment_tail(deployment_id) + return f'{dns_slug(served)[:62 - len(tail)].rstrip("-")}-{tail}' + + +def model_name(deployment: Deployment, *, unique: bool = False) -> str: + """The Model CR name for ``deployment``: the one place it is derived.""" + return model_name_for(_served_name(deployment), deployment.id if unique else None) def _served_name(deployment: Deployment) -> str: @@ -81,6 +99,7 @@ def _model_doc( *, namespace: str, resource_profile: str, + name: str | None = None, ) -> dict[str, Any]: """Build one KubeAI ``Model`` CR for a vLLM deployment. @@ -90,7 +109,7 @@ def _model_doc( engine here too. ``served_model_name`` is overridden to the CR name so the gateway's request name and vLLM's served name agree. """ - name = model_name_for(_served_name(deployment)) + name = name or model_name(deployment) svc = vllm_service_dict(deployment) svc['served_model_name'] = name profile = resource_profile @@ -159,6 +178,7 @@ def render_models( *, namespace: str, default_resource_profile: str | None, + unique_names: bool = False, ) -> RenderedModels: """Render the desired set into KubeAI ``Model`` docs (pure, no I/O). @@ -208,7 +228,7 @@ def render_models( 'kubeai_resource_profile ` as the default.' ) continue - name = model_name_for(_served_name(deployment)) + name = model_name(deployment, unique=unique_names) if name in out.models: out.unrenderable.add(deployment.id) out.errors.append( @@ -222,6 +242,7 @@ def render_models( deployment, namespace=namespace, resource_profile=str(profile), + name=name, ) ) out.models[name] = deployment.id @@ -383,19 +404,68 @@ def _upstream_url(self) -> str: self.gateway_upstream = f'http://{ip}/openai/v1' return self.gateway_upstream.rstrip('/') - def _render_gateway(self, rendered: RenderedModels) -> None: - """Route each endpoint alias through the gateway to its Model.""" - from ..leasing.gateway import UPSTREAM_ROUTE + @property + def dynamic_routing(self) -> bool: + """The gateway manages routes live, so each deployment is its own Model.""" + return bool(self.gateway is not None and getattr(self.gateway, 'dynamic_routing', False)) + + def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: + """Registry rows sending each vLLM catalog endpoint to its Model. + The static superset, as on compose: the gateway's config then stays + byte-stable as models come and go, so it is never recreated for one. + """ + from ..leasing.gateway import UPSTREAM_ROUTE, _registry_incoming_from_catalog + + if catalog is None or self.gateway is None: + return {} base = self._upstream_url() - # This backend's route rows, persisted like any other (the registry is - # append-only, so a released Model stays routable, as on compose). - self.gateway.merge_route_registry({ + return { + alias: {'engine': UPSTREAM_ROUTE, + 'served': model_name_for(row.get('served') or alias), + 'api_base': base} + for alias, row in _registry_incoming_from_catalog(catalog).items() + if row.get('engine') == 'vllm' + } + + def route_rows(self, desired: list[Deployment], placement=None) -> dict[str, dict[str, Any]]: + """The registry rows a render of ``desired`` merges (``routes prune`` + keeps exactly these): the catalog's, and every rendered Model's.""" + return self._front_door_inputs(desired, self._render_documents(desired)[1])[0] + + def _front_door_inputs(self, desired, rendered: RenderedModels): + """``(registry rows, dynamic routes)`` for the gateway, from a render. + + Static routing: one row per alias, to the Model serving it. Dynamic + routing: one route per (deployment, endpoint), each to its own Model, + so same-model ``--dedicated`` Models share the alias and LiteLLM + balances across them. + """ + from ..leasing.gateway import UPSTREAM_ROUTE, upstream_route + + if self.gateway is None: + return {}, [] + base = self._upstream_url() + if self.dynamic_routing: + by_id = {g.id: g for g in desired} + routes = [ + upstream_route(gid, endpoint, name, base) + for name, gid in sorted(rendered.models.items()) + for endpoint in sorted(by_id[gid].served) + ] + return {}, routes + rows = self.catalog_route_rows(self.catalog) + rows.update({ alias: {'engine': UPSTREAM_ROUTE, 'served': name, 'api_base': base} for alias, name in rendered.request_names.items() }) - # Gateway only: no engines on this host, so nothing to place. - self.gateway.converge([], apply=False) + return rows, [] + + def _set_front_door(self, desired, rendered: RenderedModels) -> None: + """Hand the gateway its inputs for the next render or preview.""" + rows, routes = self._front_door_inputs(desired, rendered) + self.gateway.upstream_rows = rows + self.gateway.upstream_routes = routes # -- converge-style surface ------------------------------------------------ @@ -415,6 +485,7 @@ def _render_documents(self, desired: list[Deployment]): list(desired), namespace=self.namespace, default_resource_profile=self.default_resource_profile, + unique_names=self.dynamic_routing, ) plan = GpuPlan( assignments={g.id: [] for g in desired if g.id not in rendered.unrenderable}, @@ -430,8 +501,21 @@ def preview(self, desired: list[Deployment], placement=None, *, approve: bool = """ plan, rendered, planned = self._render_documents(desired) self._preview_approval(planned, approve=approve) + if self.gateway is not None: + # The gateway's changes are part of the same approval: shown now, + # before the lease commits, and not asked again at the render. + self._set_front_door(desired, rendered) + self.gateway.preview([], None, approve=approve) + self.last_preview_digest = self._combined_digest( + self.last_preview_digest, self.gateway.last_preview_digest) return plan, rendered + def _combined_digest(self, models: str | None, gateway: str | None) -> str | None: + """One digest for the Models and the gateway's files together.""" + if gateway is None: + return models + return self._planned_digest({'models': models or '', 'gateway': gateway}) + def plan_on_idle_host(self, desired: list[Deployment]): """Whether ``desired`` could ever be served here: renderable or not. @@ -473,7 +557,8 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N self.last_assignments = {} # the cluster places for err in rendered.errors: logger.warning(' render: {}', err) - self.last_planned_digest = self._planned_digest(planned) + models_digest = self._planned_digest(planned) + self.last_planned_digest = models_digest self._approve_changes(planned) self._atomic_write(self.models_file, rendered.text) self._save_sidecar( @@ -483,7 +568,13 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N } ) if self.gateway is not None: - self._render_gateway(rendered) + # Gateway only: no engines on this host, so nothing to place. + # Its converge persists the merged route registry after its + # own approval (pre-approved by the preview when there was one). + self._set_front_door(desired, rendered) + self.gateway.converge([], apply=False) + self.last_planned_digest = self._combined_digest( + models_digest, self.gateway.last_planned_digest) if not apply: logger.info( 'rendered {} Model(s) to {} (not applied; ' @@ -508,7 +599,9 @@ def render_profile(self) -> dict: 'namespace': self.namespace, 'base_url': self.base_url, 'resource_profile': self.default_resource_profile, - 'gateway': self.gateway is not None, + # The front door's own settings (UI, proxy, routing mode), in the + # gateway project's profile; None without a gateway. + 'gateway': self.gateway.render_profile() if self.gateway is not None else None, 'gateway_upstream': self.gateway_upstream, 'catalogs': catalog_sources(self.catalog), } @@ -534,6 +627,11 @@ def use_profile(self, profile: dict) -> None: self.gateway_upstream = profile.get('gateway_upstream') or self.gateway_upstream sources = profile.get('catalogs') or [] self.catalog = CatalogUnion.from_sources(sources) if sources else None + front = profile.get('gateway') + # A profile from before the front door's settings were recorded holds + # only True/False here; it then keeps this process's settings. + if isinstance(front, dict) and self.gateway is not None: + self.gateway.use_profile(front) def apply(self) -> None: """Converge the cluster to the last render: apply + prune. @@ -679,7 +777,7 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: base, model = f'{front._gateway_base()}/v1', endpoint headers = front._auth_headers() else: - base, model = self.base_url, model_name_for(_served_name(deployment)) + base, model = self.base_url, model_name(deployment) headers = None ok, reason = openai_ready( base_url=base, @@ -732,7 +830,7 @@ def realize(self, deployment: Deployment) -> None: # pragma: no cover pass def teardown(self, deployment: Deployment) -> None: - name = model_name_for(_served_name(deployment)) + name = model_name(deployment, unique=self.dynamic_routing) self._kubectl( ['delete', 'models.kubeai.org', name, '--ignore-not-found'] ) diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 71507813..6acb5e48 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -283,12 +283,19 @@ def _make_backend(config, *, interactive: bool = False): # cluster: one base_url, the managed key, and endpoint aliases as # request names. Its own state dir and compose project, so it can # never touch a compose stack's containers on the same host. + # The front door takes the same settings as on compose (UI, + # reverse proxy, dynamic routing); only the engines are elsewhere. + rp_enabled, rp_port, rp_config = _resolve_reverse_proxy(config) gateway = ComposeBackend( state_dir=data_root() / 'leasing' / 'kubeai-gateway', inventory={'gpu_count': 0, 'gpus': []}, project='infer-stack-gateway', litellm=True, - ui=False, + ui=_resolve_ui(config), + reverse_proxy=rp_enabled, + reverse_proxy_port=rp_port, + reverse_proxy_config=rp_config, + dynamic_routing=_resolve_dynamic_routing(config), assume_yes=_resolve_assume_yes(config, interactive=interactive), ) backend = KubeaiBackend( @@ -1290,8 +1297,8 @@ def main(cls, argv=True, **kwargs): f'{n_deployments} stopped deployment(s)') return 0 if config.orphans: - if not isinstance(controller.backend, ComposeBackend): - raise SystemExit('gc --orphans needs the compose backend') + # Unlabelled units in residency. The kubeai backend reads only the + # pods carrying its label, so it never has any. def confirm(found): print(f'gc --orphans: {len(found)} unmanaged container(s):') @@ -1379,8 +1386,8 @@ def main(cls, argv=True, **kwargs): held = [g for g in deployments if g.state in (DeploymentState.LIVE, DeploymentState.IDLE)] observed, assignments = _placement_view(controller) - can_orphan = bool(config.orphans) and isinstance( - controller.backend, ComposeBackend) + # On any backend: one that selects its units by label (kubeai) has none. + can_orphan = bool(config.orphans) found: list = [] @@ -2456,17 +2463,17 @@ def main(cls, argv=True, **kwargs): def _require_compose_backend(controller): - """The controller's ComposeBackend, or a SystemExit for other backends. + """The Compose project holding the gateway's route registry. - The route registry is a compose-backend concept (it feeds the static-superset - LiteLLM gateway); ``--backend null``/``kubeai`` have no registry to touch.""" - backend = controller.backend - if not isinstance(backend, ComposeBackend): + The stack itself on the compose backend, the gateway's project on kubeai; + a SystemExit for a backend with no gateway (null, or ``litellm false``).""" + project = getattr(controller.backend, 'compose_project', lambda: None)() + if project is None or not getattr(project, 'litellm', False): raise SystemExit( - 'the `routes` commands require the compose backend ' - '(set `--backend compose` or `config set backend compose`)' + 'the `routes` commands need a LiteLLM gateway (the compose or kubeai ' + 'backend, with `litellm` on)' ) - return backend + return project def _live_endpoints(controller) -> set[str]: @@ -2484,7 +2491,8 @@ class RoutesListCLI(_LeasingCommonMixin): """Print the accumulated LiteLLM route registry (static-superset mode). One row per persisted route: its alias, engine, served-model/tag, the - upstream compose service it derives, and whether a live deployment is + upstream it routes to (a compose service, or a KubeAI cluster's gateway + under the Model's name), and whether a live deployment is currently backing it. Routes with no live backer still list (that is the point — a released endpoint stays routable/testable); their upstream simply errors until something serves it. @@ -2496,12 +2504,7 @@ class RoutesListCLI(_LeasingCommonMixin): @classmethod def main(cls, argv=True, **kwargs): - from ..leasing.compose import ( - OLLAMA_CONTAINER_PORT, - VLLM_CONTAINER_PORT, - ollama_service_name_for, - vllm_service_name_for, - ) + from ..leasing.gateway import registry_route_entry config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) @@ -2514,22 +2517,13 @@ def main(cls, argv=True, **kwargs): for name in sorted(entries): row = entries[name] engine = row.get('engine') - if engine == 'vllm': - served = row.get('served') or name - upstream = ( - f'http://{vllm_service_name_for(served)}:' - f'{VLLM_CONTAINER_PORT}/v1' - ) - target = served - elif engine == 'ollama': - target = row.get('model') or name - host = row.get('host') or name - upstream = ( - f'http://{ollama_service_name_for(host)}:' - f'{OLLAMA_CONTAINER_PORT}' - ) - else: + entry = registry_route_entry(name, row) # what the gateway renders + if entry is None: target, upstream = '?', '?' + else: + params = entry['litellm_params'] + target = params['model'].split('/', 1)[-1] + upstream = params['api_base'] rows.append({ 'name': name, 'engine': engine, @@ -2579,12 +2573,7 @@ class RoutesPruneCLI(_ApprovalMixin): def main(cls, argv=True, **kwargs): from ..diff_prompt import confirm_writes from ..leasing.backend import ConvergeAborted - from ..leasing.gateway import ( - LITELLM_REGISTRY_VERSION, - _dump_route_registry, - _registry_incoming_from_catalog, - _registry_incoming_from_deployments, - ) + from ..leasing.gateway import LITELLM_REGISTRY_VERSION, _dump_route_registry config = cls.cli(argv=argv, data=kwargs) # interactive=False so reconcile auto-applies the compose diff; the @@ -2593,13 +2582,13 @@ def main(cls, argv=True, **kwargs): backend = _require_compose_backend(controller) def prune_plan() -> tuple[dict, dict, list[str]]: - # The desired set exactly as the next render sees it. - desired, inputs = controller._admission_view(backend.residency()) - plan = backend.plan(desired, inputs) - keep: dict = {} - if backend.catalog is not None: - keep.update(_registry_incoming_from_catalog(backend.catalog)) - keep.update(_registry_incoming_from_deployments(desired, plan.assignments)) + # The desired set exactly as the next render sees it, and the rows + # that render merges: the catalog's and the live deployments'. + # Any: _require_compose_backend above confirmed a gateway, and + # both backends that have one supply route rows. + engines: Any = controller.backend + desired, inputs = controller._admission_view(engines.residency()) + keep = dict(engines.route_rows(desired, inputs)) current = backend.gateway._load_route_registry().get('entries', {}) return current, keep, sorted(set(current) - set(keep)) @@ -2681,7 +2670,6 @@ class RoutesSeedCLI(_ApprovalMixin): @classmethod def main(cls, argv=True, **kwargs): from ..leasing.backend import ConvergeAborted - from ..leasing.compose import _registry_incoming_from_catalog config = cls.cli(argv=argv, data=kwargs) paths = _collect_names(config.catalogs) @@ -2691,6 +2679,8 @@ def main(cls, argv=True, **kwargs): # there is no destructive gate to confirm). controller = _open_controller(config, interactive=False) backend = _require_compose_backend(controller) + # Any: a gateway exists (checked above), and its backend supplies rows. + engines: Any = controller.backend incoming: dict = {} for raw in paths: @@ -2701,7 +2691,9 @@ def main(cls, argv=True, **kwargs): cat = Catalog.load(path) except CatalogError as ex: raise SystemExit(f'invalid catalog {path}: {ex}') - incoming.update(_registry_incoming_from_catalog(cat)) + # The engine backend's rows for it: compose upstreams on compose, + # the cluster's Models on kubeai. + incoming.update(engines.catalog_route_rows(cat)) if not incoming: raise SystemExit( 'routes seed: the named catalog(s) resolved no routable endpoints' @@ -2842,8 +2834,9 @@ def main(cls, argv=True, **kwargs): if not config.subnet: raise SystemExit('network migrate: --subnet is required') controller = _open_controller(config, interactive=True) - if not isinstance(controller.backend, ComposeBackend): - raise SystemExit('network migrate needs the compose backend') + if not hasattr(controller.backend, 'network'): + raise SystemExit('network migrate gives compose containers stable ' + 'addresses; this backend runs none (n/a on kubeai)') try: rec = controller.network_migrate(config.subnet, force=bool(config.force)) except ProfileMismatch as ex: @@ -2908,7 +2901,7 @@ def main(cls, argv=True, **kwargs): # Any: rotate_gateway_key below refuses a backend without a gateway, # so past it these gateway methods exist. backend: Any = controller.backend - old = backend.master_key() if isinstance(backend, ComposeBackend) else None + old = backend.master_key() if getattr(backend, 'litellm', False) else None try: rec = controller.rotate_gateway_key(force=bool(config.force)) except ProfileMismatch as ex: @@ -2930,7 +2923,8 @@ def main(cls, argv=True, **kwargs): 'rejects the OLD key') else: print(' gateway: new key accepted, old key rejected') - if getattr(backend, 'ui', False): + front = getattr(backend, 'compose_project', lambda: None)() + if getattr(front, 'ui', False): print(' Open WebUI may keep the old key in its own settings: ' 'update it under Admin > Settings > Connections') return 0 diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 0470a36f..c44793cd 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -578,6 +578,7 @@ def render_compose( catalog: Any = None, route_registry: dict[str, Any] | None = None, dynamic_routing: bool = False, + upstream_routes: list[dict[str, Any]] | None = None, ) -> RenderedCompose: """Render a compose project for the placed deployments. @@ -665,7 +666,7 @@ def render_compose( reverse_proxy=reverse_proxy, reverse_proxy_port=reverse_proxy_port, reverse_proxy_config=reverse_proxy_config, aux_dir=aux_dir, catalog=catalog, route_registry=route_registry, - dynamic_routing=dynamic_routing, + dynamic_routing=dynamic_routing, upstream_routes=upstream_routes, ) services.update(front.services) litellm_config = front.litellm_config @@ -1426,6 +1427,7 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A reverse_proxy_config=self.reverse_proxy_config, aux_dir=self.state_dir, project=self.project, catalog=self.catalog, route_registry=route_registry, dynamic_routing=self.dynamic_routing, + upstream_routes=self.upstream_routes, ) addresses = None if self.network is not None: @@ -1622,12 +1624,30 @@ def _merged_route_registry( placed deployment (``desired`` spans all runbooks via the shared ledger, so a live cross-runbook deployment stays routable, and past release). """ - incoming: dict[str, dict[str, Any]] = {} - if self.catalog is not None: - incoming.update(_registry_incoming_from_catalog(self.catalog)) + incoming = self.catalog_route_rows(self.catalog) incoming.update(_registry_incoming_from_deployments(desired, assignments)) + incoming.update(self.upstream_rows) return self.gateway.merged_route_registry(incoming) + #: Render inputs from the owner of engines this project does not run (the + #: kubeai backend, whose gateway this is): static route-registry rows and + #: dynamic routes, both pointing at its servers. Set before each render. + upstream_rows: dict[str, dict[str, Any]] = {} + upstream_routes: list[dict[str, Any]] = [] + + def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: + """Route-registry rows for every endpoint of ``catalog``.""" + return {} if catalog is None else _registry_incoming_from_catalog(catalog) + + def route_rows(self, desired: list[Deployment], placement=None) -> dict[str, dict[str, Any]]: + """The rows a render of ``desired`` merges: catalog, placed deployments, + and the owner's upstream rows (``routes prune`` keeps exactly these).""" + assignments = self.plan(desired, placement).assignments + rows = self.catalog_route_rows(self.catalog) + rows.update(_registry_incoming_from_deployments(desired, assignments)) + rows.update(self.upstream_rows) + return rows + def _update_route_registry( self, desired: list[Deployment], assignments: dict[str, list[int]] ) -> dict[str, Any]: diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 433ee108..daaa48f6 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -261,32 +261,51 @@ def _litellm_model_list_from_registry( entries: list[dict[str, Any]] = [] rows = registry.get('entries', {}) if isinstance(registry, dict) else {} for name in sorted(rows): - row = rows[name] - if not isinstance(row, dict): - continue - engine = row.get('engine') - if engine == 'vllm': - served = row.get('served') or name - api_base = ( - f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' - ) - entries.append(_vllm_route_entry(name, served, api_base)) - elif engine == 'ollama': - tag = row.get('model') or name - host = row.get('host') or name - api_base = ( - f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' - ) - entries.append(_ollama_route_entry(name, tag, api_base)) - elif engine == UPSTREAM_ROUTE and row.get('api_base'): - # An OpenAI-compatible server this project does not run (a KubeAI - # cluster's gateway): the row carries its address and the name it - # serves the model under. - entries.append(_vllm_route_entry(name, row.get('served') or name, - str(row['api_base']))) + entry = registry_route_entry(name, rows[name]) + if entry is not None: + entries.append(entry) return entries +def registry_route_entry(name: str, row: Any) -> dict[str, Any] | None: + """The LiteLLM entry one registry row renders to, or ``None`` if it cannot. + + The one derivation of a row's upstream: the render uses it, and so does + ``routes list``. Upstreams come from the live naming helpers, so the + registry never becomes a rendered-config parse surface. + """ + if not isinstance(row, dict): + return None + engine = row.get('engine') + if engine == 'vllm': + served = row.get('served') or name + api_base = f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' + return _vllm_route_entry(name, served, api_base) + if engine == 'ollama': + tag = row.get('model') or name + host = row.get('host') or name + api_base = f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' + return _ollama_route_entry(name, tag, api_base) + if engine == UPSTREAM_ROUTE and row.get('api_base'): + # An OpenAI-compatible server this project does not run (a KubeAI + # cluster's gateway): the row carries its address and the name it + # serves the model under. + return _vllm_route_entry(name, row.get('served') or name, str(row['api_base'])) + return None + + +def upstream_route(deployment_id: str, endpoint: str, served: str, + api_base: str) -> dict[str, Any]: + """A dynamic route to a server this project does not run (a KubeAI Model). + + The same entry shape, and the same deterministic id, as a Compose engine's + dynamic route (:func:`_litellm_routes`). + """ + entry = _vllm_route_entry(endpoint, served, api_base) + entry['model_info'] = {'id': _route_id(deployment_id, endpoint)} + return entry + + def _merge_route_registry( existing: dict[str, Any], incoming: dict[str, dict[str, Any]] ) -> tuple[dict[str, Any], list[str]]: @@ -399,17 +418,7 @@ def _litellm_routes( f':{VLLM_CONTAINER_PORT}/v1' ) for endpoint in sorted(deployment.served): - entries.append( - { - 'model_name': endpoint, - 'litellm_params': { - 'model': f'openai/{served}', - 'api_base': api_base, - 'api_key': 'EMPTY', - }, - 'model_info': {'id': _route_id(deployment.id, endpoint)}, - } - ) + entries.append(upstream_route(deployment.id, endpoint, served, api_base)) elif deployment.engine == 'ollama': api_base = ( f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' @@ -766,12 +775,15 @@ def render_front_door( catalog: Any, route_registry: dict[str, Any] | None, dynamic_routing: bool, + upstream_routes: list[dict[str, Any]] | None = None, ) -> FrontDoor: """Render the gateway, its database, Open WebUI and the reverse proxy. The engines are the caller's: it passes the services it rendered (``engine_services``, only for the legacy per-model ``depends_on``) and the in-network URLs a UI with no gateway can talk to directly. + ``upstream_routes`` are dynamic routes to servers this project does not + run (see :func:`upstream_route`). """ services: dict[str, Any] = {} litellm_config = None @@ -804,7 +816,8 @@ def render_front_door( # config (and recreates the gateway) on every model change. if dynamic_routing: entries: list[dict[str, Any]] = [] - litellm_routes = _litellm_routes(deployments, assignments) + litellm_routes = (_litellm_routes(deployments, assignments) + + list(upstream_routes or [])) litellm_depends: list[str] = [] elif route_registry is not None: entries = _litellm_model_list_from_registry(route_registry) diff --git a/infer_stack/leasing/naming.py b/infer_stack/leasing/naming.py index c2d58c82..c89515f8 100644 --- a/infer_stack/leasing/naming.py +++ b/infer_stack/leasing/naming.py @@ -54,8 +54,16 @@ def _unique_vllm_service_name(served: str, deployment_id: str) -> str: so N dedicated deployments of one model become N containers on N GPUs. The suffix is the deployment id's hex tail, keeping the name short and DNS-safe. """ - tail = deployment_id.rsplit('-', 1)[-1][:8] or 'x' - return f'{vllm_service_name_for(served)}-{_dns_slug(tail)}' + return f'{vllm_service_name_for(served)}-{deployment_tail(deployment_id)}' + + +def deployment_tail(deployment_id: str) -> str: + """The short, DNS-safe suffix that makes a per-deployment name unique. + + >>> deployment_tail('grp-0123456789ab') + '01234567' + """ + return _dns_slug(deployment_id.rsplit('-', 1)[-1][:8] or 'x') def vllm_service_name(deployment: Deployment, *, unique: bool = False) -> str: diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index b162318e..91b1d950 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -773,15 +773,18 @@ def test_a_slow_start_is_not_a_failure(tmp_path): assert not probe.ready and not probe.fatal -def test_compose_only_commands_refuse_kubeai_explicitly(tmp_path, monkeypatch): - """`gc --orphans` used "has residency" to mean compose; kubeai has it now.""" +def test_gc_orphans_on_kubeai_finds_nothing_to_remove(tmp_path, monkeypatch, capsys): + """Residency selects pods by infer-stack's label, so none is an orphan: + the command runs on kubeai and never deletes anything.""" from infer_stack.cli import commands_leasing - be, _ = make_pod_backend(tmp_path) + be, kubectl = make_pod_backend(tmp_path) + kubectl.pods = [_pod('model-a-0', 'grp-1', ready=True)] ctl = Controller(Ledger(SqliteStore(str(tmp_path / 'l.db'))), be) monkeypatch.setattr(commands_leasing, '_open_controller', lambda *a, **k: ctl) - with pytest.raises(SystemExit, match='needs the compose backend'): - commands_leasing.GcCLI.main(argv=['--orphans', '--yes']) + assert commands_leasing.GcCLI.main(argv=['--orphans', '--yes']) == 0 + assert 'removed 0' in capsys.readouterr().out + assert not any('delete' in call or 'rm' in call for call in kubectl.calls) def test_runtime_env_reaches_the_model_like_it_reaches_a_container(): @@ -793,3 +796,128 @@ def test_runtime_env_reaches_the_model_like_it_reaches_a_container(): (doc,) = rendered.docs assert doc['spec']['env'] == {'MODE': 'fast', 'CTX': '4096', 'ON': 'true', 'VLLM_ATTENTION_BACKEND': 'TORCH_SDPA'} + + +# -- the front door: same settings, same approval, same routes (P3) ----------- + + +def make_front_door_backend(tmp_path, **gateway_kw): + from infer_stack.leasing.compose import ComposeBackend + from test_leasing_compose import FakeDocker + + kubectl = FakeKubectl() + gateway = ComposeBackend( + state_dir=tmp_path / 'gateway', inventory={'gpu_count': 0, 'gpus': []}, + run=FakeDocker(), http=GatewayHttp(kubectl, tmp_path / 'gateway'), + project='infer-stack-gateway', litellm=True, + images={'litellm': 'litellm:test', 'open-webui': 'owui:test', + 'nginx': 'nginx:test', 'postgres': 'pg:test'}, + **gateway_kw, + ) + be = KubeaiBackend(state_dir=tmp_path / 'kubeai', run=kubectl, + http=gateway.http, gateway=gateway, gateway_upstream=UPSTREAM) + return be, kubectl + + +def _services(be): + doc = yaml.safe_load(be.gateway.compose_file.read_text()) + return set(doc['services']) + + +def test_the_gateway_project_takes_the_front_door_settings(tmp_path): + be, _ = make_front_door_backend(tmp_path, ui=True, reverse_proxy=True) + from infer_stack.leasing.gateway import NGINX_SERVICE, OPEN_WEBUI_SERVICE + + be.converge([vllm('grp-a', served='tiny')], apply=False) + assert {'litellm', OPEN_WEBUI_SERVICE, NGINX_SERVICE} <= _services(be) + + +def test_dynamic_routing_gives_each_dedicated_deployment_its_own_model(tmp_path): + be, kubectl = make_front_door_backend(tmp_path, dynamic_routing=True) + a = vllm('grp-aaaaaaaa1111', served='tiny', t=1.0) + b = vllm('grp-bbbbbbbb2222', served='tiny', t=2.0) + for dep in (a, b): + dep.served = {'tiny': {'served_model_name': 'tiny', 'protocol': 'chat'}} + be.converge([a, b], apply=False) + models = yaml.safe_load_all(be.models_file.read_text()) + assert {m['metadata']['name'] for m in models} == {'tiny-aaaaaaaa', 'tiny-bbbbbbbb'} + routes = json.loads((be.gateway.state_dir / 'litellm_routes.json').read_text()) + assert sorted(r['litellm_params']['model'] for r in routes) == [ + 'openai/tiny-aaaaaaaa', 'openai/tiny-bbbbbbbb'] + assert {r['model_name'] for r in routes} == {'tiny'} # one alias, two upstreams + assert 'postgres-litellm' in _services(be) + + +def test_catalog_endpoints_are_routed_before_they_run(tmp_path): + """The static superset, as on compose: acquiring a catalog endpoint does + not change the gateway's config, so the gateway is not recreated.""" + from infer_stack.leasing import Catalog + + be, _ = make_front_door_backend(tmp_path) + be.catalog = Catalog.from_dict({ + 'models': {'m': {'source': 'hf://org/m'}}, + 'endpoints': {'tiny': {'engine': 'vllm', 'model': 'm'}}, + }) + be.converge([], apply=False) + before = (be.gateway.state_dir / 'litellm_config.yaml').read_text() + assert 'tiny' in before + dep = vllm('grp-a', served='tiny') + dep.served = {'tiny': {'served_model_name': 'tiny', 'protocol': 'chat'}} + be.converge([dep], apply=False) + assert (be.gateway.state_dir / 'litellm_config.yaml').read_text() == before + + +def test_the_gateways_changes_are_approved_in_the_preview(tmp_path, monkeypatch): + """One approval, before the lease commits: a decline writes neither the + Models nor the gateway's route registry.""" + from infer_stack import diff_prompt as dp + from infer_stack.leasing.backend import ConvergeAborted + + be, _ = make_front_door_backend(tmp_path) + be.assume_yes = be.gateway.assume_yes = False + asked = [] + monkeypatch.setattr(dp, 'confirm_writes', lambda changed, **kw: asked.append( + sorted(p.name for p in changed)) or False) + with pytest.raises(ConvergeAborted): + be.preview([vllm('grp-a', served='tiny')], approve=True) + assert not be.models_file.exists() + assert not be.gateway.gateway._registry_file.exists() + + asked.clear() + monkeypatch.setattr(dp, 'confirm_writes', lambda changed, **kw: asked.append( + sorted(p.name for p in changed)) or True) + desired = [vllm('grp-a', served='tiny')] + be.preview(desired, approve=True) + approved = be.last_preview_digest + assert any('litellm_config.yaml' in names for names in asked) + asked.clear() + be.converge(desired, apply=False) + assert asked == [] # not asked a second time + assert be.last_planned_digest == approved # the approved-digest guard holds + + +def test_routes_list_shows_kubeai_upstreams(tmp_path, monkeypatch, capsys): + from infer_stack.cli import commands_leasing + + be, _ = make_front_door_backend(tmp_path) + dep = vllm('grp-a', served='Qwen/Qwen2.5-0.5B') + dep.served = {'tiny': {'served_model_name': 'Qwen/Qwen2.5-0.5B', 'protocol': 'chat'}} + be.converge([dep], apply=False) + ctl = Controller(Ledger(SqliteStore(str(tmp_path / 'l.db'))), be) + monkeypatch.setattr(commands_leasing, '_open_controller', lambda *a, **k: ctl) + capsys.readouterr() # the render's own output + assert commands_leasing.RoutesListCLI.main(argv=['--json']) == 0 + (row,) = json.loads(capsys.readouterr().out)['routes'] + assert row['name'] == 'tiny' and row['engine'] == 'upstream' + assert row['target'] == 'qwen-qwen2-5-0-5b' and row['upstream'] == UPSTREAM + + +def test_the_front_doors_settings_are_in_the_recovery_profile(tmp_path): + be, _ = make_front_door_backend(tmp_path, ui=True, dynamic_routing=True) + profile = be.render_profile() + assert profile['gateway']['ui'] is True and profile['gateway']['dynamic_routing'] is True + other, _ = make_front_door_backend(tmp_path / 'other') + other.use_profile(profile) + assert other.gateway.ui is True and other.dynamic_routing is True + other.use_profile({**profile, 'gateway': True}) # a profile from before + assert other.dynamic_routing is True # keeps what it has From 487cab54124800bab4ca8c5a0dcda54aad824d35 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 12:19:37 -0400 Subject: [PATCH 030/141] Queue: record the P3 commit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 8bdd3b36..c662ea27 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -58,7 +58,7 @@ and lists its pod (checked in a real terminal, not only in tests). ### 4. [x] P3: gateway feature parity -Done 2026-09-26. +Done 2026-09-26, `7a322c1`. `routes` and `secrets rotate` resolve the backend's gateway instead of checking its kind; the KubeAI gateway project honours `ui`, From 1a4e4b266b8645aa17e2a5e52183d45ec468f7de Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 12:37:15 -0400 Subject: [PATCH 031/141] P6: a parity suite, one test per row the matrix marks same tests/test_parity.py builds the same stack over a fake Docker (compose) and a fake kubectl (kubeai, with its gateway) and runs one scenario per *same* row of docs/backend-parity.md on both: the client contract, the lifecycle verbs, admission, config publish, --no-apply, crash-loop diagnosis, strict residency, GPU count, the argument pipeline, served names, static and dynamic routes, routes list, the UI and proxy, approval, instances, stack down, and the TUI. Found on the way: the TUI replaced an injected Docker runner with the real one, so a TUI on a test backend talked to the host's Docker; it now wraps only the default runner. The matrix's gc --orphans row is corrected (it runs on kubeai and finds none). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 7 + docs/backend-parity.md | 6 +- docs/planning/backend-parity-roadmap.md | 14 +- docs/queue.md | 4 +- infer_stack/tui.py | 15 +- tests/test_parity.py | 449 ++++++++++++++++++++++++ 6 files changed, 482 insertions(+), 13 deletions(-) create mode 100644 tests/test_parity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bde84cb..d1f6b58a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### A parity suite for the two backends + +`tests/test_parity.py` runs one scenario per row that `docs/backend-parity.md` +marks *same*, on the compose backend over a fake Docker and on the kubeai +backend over a fake kubectl, from the client contract to the TUI. The TUI no +longer swaps an injected Docker runner for the real one. + ### The gateway in front of a cluster is the compose gateway On the kubeai backend the gateway now takes the same settings as on compose: diff --git a/docs/backend-parity.md b/docs/backend-parity.md index 9f399cc4..de60913f 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -51,7 +51,8 @@ rotate` and the static superset route table work unchanged. ## Parity matrix -**same**: one code path, or verified equivalent. **≈**: the same outcome by +**same**: one code path, or verified equivalent; every *same* row has a test +in `tests/test_parity.py` that runs the same scenario on both backends. **≈**: the same outcome by a different mechanism. **gap**: missing on one side and on the roadmap. **n/a**: does not apply there. **boundary**: deliberately unsupported (see [planning/known-limitations.md](planning/known-limitations.md)). @@ -125,13 +126,14 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | TUI: leases, deployments, catalog editing, acquire / release / evict, API tab, settings | same | same | | TUI: log follow, the Instances tab, Apply / Down | containers | same, over pods | | TUI GPU pane | `nvidia-smi` on this host | this host, not the cluster | -| `gc --orphans` (also inside `clean`) | yes | n/a: unlabeled Models are never touched | +| `gc --orphans` (also inside `clean`) | yes | runs, and finds none: pods are read by infer-stack's label, so unlabelled Models are never touched | ### Testing | | Compose | KubeAI | |---|---|---| | unit suite with fake runtimes | yes | yes | +| parity suite (`tests/test_parity.py`: each *same* row, both backends) | yes | yes | | end to end | `dev/e2e_tests/run.sh` (tiers; `--gpu` for serving) | `dev/kubeai_e2e.sh` against a real cluster; k3s with CPU vLLM needs no GPU | ## Deviations that stay diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 6eacb09a..6e5aee23 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,8 +1,8 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **P1–P3 done** 2026-09-26 · P4, P5 not started · -P6 is ongoing. Execution order: [../queue.md](../queue.md). +`dev/backend-unification` · **P1–P3, P6 done** 2026-09-26 · P4, P5 not +started. Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the @@ -210,6 +210,16 @@ the gateway, and the only one that needs a second machine. Last. - Finishing a phase updates the matrix in `backend-parity.md`; the plan is done when the matrix has no *gap* row. +**P6 done 2026-09-26** as a suite that stays open: `tests/test_parity.py` +builds the same stack over a fake Docker and a fake kubectl and runs one +test per *same* row on both (21 rows, the TUI included). A row turned *same* +gets its test in the same change. The Memory and Null backends run the +controller's one path through `SimpleAdmission` (P1b), so the controller +suites cover them without a third parametrization. Found on the way: the +TUI replaced an injected Docker runner with the real one, so a TUI started +on a test backend talked to the host's Docker; it now wraps only the +default runner. + ## Duplicate authorities The rule while executing: a duplicate authority found on the way is diff --git a/docs/queue.md b/docs/queue.md index c662ea27..7c89a151 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -69,7 +69,9 @@ into the admission preview (deferred duplicate). the cluster; an e2e step acquires the same model `--dedicated` twice under dynamic routing and gets two Models and two routes. -### 5. [ ] P6: one test surface +### 5. [x] P6: one test surface + +Done 2026-09-26. Parametrize the controller's acquire scenarios over the Memory, fake-Compose and fake-KubeAI backends; `tests/test_parity.py` runs each diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 439afd00..dc1b6687 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1351,22 +1351,21 @@ def _report_stall(self, length: float, stack: list[str]) -> None: def _install_quiet_docker(self) -> None: """Route ``docker`` output to the logs pane, not the terminal. - Only ``docker`` commands: the kubeai backend runs ``kubectl`` through - the same seam, and the Docker runner's allowlisted environment has no - ``KUBECONFIG`` (every kubectl call from the TUI used to fail). The - kubeai backend's gateway is a Compose project of its own, so it is - wrapped too. + Only where the backend runs Docker with the default runner: the kubeai + backend runs ``kubectl`` through the same seam, and the Docker + runner's allowlisted environment has no ``KUBECONFIG`` (every kubectl + call from the TUI used to fail); an injected runner is the caller's + and is left alone. The kubeai backend's gateway is a Compose project + of its own, so it is wrapped too. """ from .leasing.compose import _default_docker_run def quiet(target) -> None: original = getattr(target, 'run', None) - if original is None: + if original is not _default_docker_run: return def quiet_run(args: list[str], **kwargs) -> str: - if not args or args[0] != 'docker': - return original(args, **kwargs) # Same bounded, explicit-environment runner as the CLI; only # stderr is redirected to the logs pane, unless the caller # takes it. diff --git a/tests/test_parity.py b/tests/test_parity.py new file mode 100644 index 00000000..34087a31 --- /dev/null +++ b/tests/test_parity.py @@ -0,0 +1,449 @@ +"""Every row of docs/backend-parity.md that reads *same*, run on both backends. + +One harness builds the same stack twice: the compose backend over a fake +Docker, and the kubeai backend (with its LiteLLM gateway) over a fake +kubectl. Each test is one row of the matrix, named after it, and runs the +same scenario on both. A row whose test is here and passes on both is what +*same* means; a behaviour that only one backend can show belongs in that +backend's own tests. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +import pytest +import yaml + +from infer_stack.leasing import Catalog, Controller, LeaseState, Ledger, SqliteStore +from infer_stack.leasing.backend import PlacementError +from infer_stack.leasing.residency import DEPLOYMENT_LABEL, ResidencyUnknown + +CATALOG = { + 'models': { + 'tiny': {'source': 'hf://org/tiny'}, + 'big': {'source': 'hf://org/big'}, + }, + 'endpoints': { + 'one': {'engine': 'vllm', 'model': 'tiny', 'reclaim': {'policy': 'stop'}, + 'runtime': {'max_model_len': 2048, 'extra_args': ['--seed', '7']}}, + 'wide': {'engine': 'vllm', 'model': 'big', + 'runtime': {'tensor_parallel_size': 2, 'max_model_len': 4096}}, + 'warm': {'engine': 'vllm', 'model': 'big', 'reclaim': {'policy': 'keep-warm'}, + 'runtime': {'max_model_len': 1024}}, + }, +} +CAT = Catalog.from_dict(CATALOG) +UPSTREAM = 'http://10.43.0.9/openai/v1' +CRASH_LOG = ('INFO vLLM API server version 0.25.1\n' + 'ValueError: The checkpoint has model type `example_moe_v1` but ' + 'Transformers does not recognize this architecture. If the model is ' + 'custom, set trust_remote_code=True.\n') + + +@dataclass +class Stack: + """One backend under test, and the handles a scenario needs.""" + + kind: str + ctl: Controller + backend: Any + runtime: Any # the fake Docker or the fake kubectl + front: Any # the ComposeBackend holding the gateway + + def acquire(self, *names, dedicated: bool = False, **kw): + from infer_stack.leasing import Sharing + + kw.setdefault('wait', False) + sharing = Sharing.DEDICATED if dedicated else None + requests = CAT.resolve_names(list(names), sharing=sharing) + return self.ctl.acquire('alice', requests, **kw) + + def running(self) -> set[str]: + return {gid for gid, found in self.backend.residency().by_deployment.items() + if any(c.warm for c in found)} + + def crash(self, gid: str) -> None: + """Make the deployment's engine crash-loop with CRASH_LOG.""" + if self.kind == 'compose': + from infer_stack.leasing.compose import CRASH_LOOP_RESTARTS + + for c in self.runtime.containers.values(): + if c['labels'].get(DEPLOYMENT_LABEL) == gid: + c.update(state='restarting', restart_count=CRASH_LOOP_RESTARTS, + exit_code=1) + else: + self.runtime.crash[gid] = CRASH_LOG + # A crash-looping pod answers nothing, as a restarting container. + post = self.backend.http.post + self.backend.http.post = lambda url, **kw: ( + self.backend.http._Resp(503, {'detail': 'upstream down'}) + if url.startswith('http://127.0.0.1:') else post(url, **kw)) + self.backend.deployment_logs = lambda deployment, tail=400: CRASH_LOG + + def break_runtime(self) -> None: + """The runtime stops answering (docker or kubectl down).""" + def broken(args, **kw): + raise RuntimeError('connection refused') + self.backend.run = broken + + def gateway_routes(self) -> dict[str, dict]: + cfg = self.front.state_dir / 'litellm_config.yaml' + doc = yaml.safe_load(cfg.read_text()) if cfg.exists() else {} + return {e['model_name']: e['litellm_params'] for e in doc.get('model_list') or []} + + +def _compose(tmp_path, **gateway_kw) -> Stack: + from infer_stack.hardware import simulate_inventory + from infer_stack.leasing.compose import ComposeBackend + from test_leasing_compose import IMAGES, PORTS, STATE, FakeDocker, FakeHttp + + docker = FakeDocker() + backend = ComposeBackend( + state_dir=tmp_path / 'compose', inventory=simulate_inventory('4x80'), + run=docker, http=FakeHttp(tmp_path / 'compose'), + images={**IMAGES, 'open-webui': 'owui:test', 'nginx': 'nginx:test', + 'postgres': 'pg:test'}, + ports=PORTS, state=STATE, litellm=True, catalog=CAT, **gateway_kw, + ) + ctl = Controller(Ledger(SqliteStore(str(tmp_path / 'ledger.db'))), backend) + return Stack('compose', ctl, backend, docker, backend) + + +class _Kubectl: + """kubectl over applied Model CRs; each Model runs one pod.""" + + def __init__(self): + from test_leasing_kubeai import FakeKubectl + + self.inner = FakeKubectl() + self.crash: dict[str, str] = {} + + @property + def applied(self): + return self.inner.applied + + def __call__(self, args: list[str]) -> str: + from test_leasing_kubeai import _pod + + if len(args) > 4 and args[3] == 'get' and args[4] == 'pods': + pods = [] + for name, doc in self.inner.applied.items(): + gid = doc['metadata']['labels']['infer-stack/deployment'] + if gid in self.crash: + pods.append(_pod(f'model-{name}-0', gid, restarts=3, + state={'waiting': {'reason': 'CrashLoopBackOff'}}, + last={'exitCode': 1, 'reason': 'Error'})) + else: + pods.append(_pod(f'model-{name}-0', gid, ready=True)) + return json.dumps({'items': pods}) + return self.inner(args) + + +def _kubeai(tmp_path, **gateway_kw) -> Stack: + from infer_stack.backends.kubeai import KubeaiBackend + from infer_stack.leasing.compose import ComposeBackend + from test_leasing_compose import FakeDocker + from test_leasing_kubeai import GatewayHttp + + kubectl = _Kubectl() + gateway = ComposeBackend( + state_dir=tmp_path / 'gateway', inventory={'gpu_count': 0, 'gpus': []}, + run=FakeDocker(), http=GatewayHttp(kubectl.inner, tmp_path / 'gateway'), + project='infer-stack-gateway', litellm=True, + images={'litellm': 'litellm:test', 'open-webui': 'owui:test', + 'nginx': 'nginx:test', 'postgres': 'pg:test'}, + **gateway_kw, + ) + backend = KubeaiBackend(state_dir=tmp_path / 'kubeai', run=kubectl, + http=gateway.http, gateway=gateway, + gateway_upstream=UPSTREAM, default_resource_profile='gpu') + backend.catalog = CAT + ctl = Controller(Ledger(SqliteStore(str(tmp_path / 'ledger.db'))), backend) + return Stack('kubeai', ctl, backend, kubectl, gateway) + + +BUILDERS = {'compose': _compose, 'kubeai': _kubeai} + + +@pytest.fixture(params=sorted(BUILDERS)) +def make_stack(request, tmp_path): + def make(**gateway_kw) -> Stack: + return BUILDERS[request.param](tmp_path, **gateway_kw) + return make + + +# -- client contract ----------------------------------------------------------- + + +def test_one_base_url_the_managed_key_and_the_alias(make_stack): + stack = make_stack() + stack.acquire('one') + info = stack.backend.access(['one']) + assert info['base_url'].startswith('http://127.0.0.1:') + assert info['api_key'] == stack.front.master_key() + assert info['request_names'] == {'one': 'one'} + + +def test_env_file_has_the_same_keys(make_stack): + from types import SimpleNamespace + + from infer_stack.cli.commands_leasing import _descriptor_for + from infer_stack.leasing.envfile import descriptor_env + + stack = make_stack() + out = stack.acquire('one') + config = SimpleNamespace(base_url='unused', api_key_env='LITELLM_MASTER_KEY') + env = descriptor_env(_descriptor_for(stack.ctl, out.lease, out.deployments, config)) + assert set(env) >= {'INFER_STACK_LEASE_ID', 'OPENAI_BASE_URL', + 'INFER_STACK_ENDPOINT_ONE', 'INFER_STACK_MODELS'} + assert env['INFER_STACK_ENDPOINT_ONE'] == 'one' # the alias, on both + + +def test_readiness_is_a_generation_through_the_front_door(make_stack): + stack = make_stack() + out = stack.acquire('one', wait=True, timeout=10, interval=1) + assert out.wait is not None and out.wait.ready + + +def test_secrets_rotate(make_stack): + stack = make_stack() + old = stack.front.master_key() + stack.ctl.rotate_gateway_key() + assert stack.front.master_key() != old + + +# -- lifecycle ------------------------------------------------------------------- + + +def test_acquire_release_evict_gc_renew(make_stack): + stack = make_stack() + one = stack.acquire('one') + warm = stack.acquire('warm') + gid_one, gid_warm = one.deployments[0].id, warm.deployments[0].id + assert {gid_one, gid_warm} <= stack.running() + stack.ctl.renew(one.lease.id, ttl_seconds=3600) + stack.ctl.release(one.lease.id) + stack.ctl.release(warm.lease.id) + assert gid_one not in stack.running() # reclaim: stop + assert gid_warm in stack.running() # keep-warm stays resident + stack.ctl.gc() + assert gid_warm in stack.running() # plain gc leaves it + stack.ctl.evict(None) + assert stack.running() == set() + + +def test_a_refused_acquire_writes_nothing(make_stack): + stack = make_stack() + # What each backend cannot serve: more GPUs than the host has, or a + # Model with no resource profile. + if stack.kind == 'compose': + stack.backend.inventory = {'gpu_count': 1, 'gpus': [ + {'index': 0, 'memory_total_mib': 81920}]} + else: + stack.backend.default_resource_profile = None + with pytest.raises(PlacementError): + stack.acquire('wide') + assert stack.ctl.ledger.status() == ([], []) + assert stack.running() == set() + + +def test_config_publish_previews_then_commits(make_stack): + stack = make_stack() + profile = stack.backend.render_profile() + stack.ctl.publish_profile(profile) + assert stack.ctl.ledger.profile() == profile + + +def test_no_apply_stages_and_apply_brings_it_up(make_stack): + stack = make_stack() + out = stack.acquire('one', apply=False) + gid = out.deployments[0].id + assert gid not in stack.running() + assert stack.ctl.ledger.publication_pending() is not None + stack.ctl.apply_now() + assert gid in stack.running() + + +def test_a_crash_looping_engine_fails_fast_with_its_error(make_stack): + stack = make_stack() + out = stack.acquire('one') + gid = out.deployments[0].id + stack.crash(gid) + probe = stack.backend.probe_ready(stack.ctl.ledger.get_deployment(gid), 'one') + assert probe.ready is False and probe.fatal is True + assert 'trust_remote_code' in probe.detail + + +def test_strict_residency_and_lenient_observe(make_stack): + stack = make_stack() + stack.acquire('one') + stack.break_runtime() + with pytest.raises(ResidencyUnknown): + stack.backend.residency() + assert stack.backend.observe() == set() + + +# -- placement and the catalog ----------------------------------------------------- + + +def test_gpu_count_from_tp_pp_dp(make_stack): + stack = make_stack() + out = stack.acquire('wide') + gid = out.deployments[0].id + if stack.kind == 'compose': + assert len(stack.ctl.ledger.get_deployment(gid).assigned_gpus) == 2 + else: + (doc,) = [d for d in stack.runtime.applied.values() + if d['metadata']['labels']['infer-stack/deployment'] == gid] + assert doc['spec']['resourceProfile'] == 'gpu:2' + + +def _engine_args(stack, gid) -> list[str]: + if stack.kind == 'compose': + doc = yaml.safe_load(stack.backend.compose_file.read_text()) + (svc,) = [s for s in doc['services'].values() + if s.get('labels', {}).get(DEPLOYMENT_LABEL) == gid] + return [str(a) for a in svc['command']] + (doc,) = [d for d in stack.runtime.applied.values() + if d['metadata']['labels']['infer-stack/deployment'] == gid] + return [str(a) for a in doc['spec']['args']] + + +def test_runtime_flags_and_extra_args_reach_the_engine(make_stack): + stack = make_stack() + out = stack.acquire('one') + args = ' '.join(_engine_args(stack, out.deployments[0].id)) + assert '--max-model-len=2048' in args + assert '--seed 7' in args + + +def test_served_names_come_from_one_rule(make_stack): + from infer_stack.leasing.naming import dns_slug + + stack = make_stack() + out = stack.acquire('one') + gid = out.deployments[0].id + if stack.kind == 'compose': + doc = yaml.safe_load(stack.backend.compose_file.read_text()) + names = [n for n, s in doc['services'].items() + if s.get('labels', {}).get(DEPLOYMENT_LABEL) == gid] + assert names == [f'vllm-{dns_slug("one")}'] + else: + assert f'{dns_slug("one")}' in stack.runtime.applied + + +# -- the gateway ------------------------------------------------------------------ + + +def test_catalog_routes_exist_before_models_run_and_do_not_churn(make_stack): + stack = make_stack() + stack.ctl.apply_now() + before = stack.gateway_routes() + assert {'one', 'wide', 'warm'} <= set(before) + stack.acquire('one') + assert stack.gateway_routes() == before + + +def test_routes_list(make_stack, monkeypatch, capsys): + from infer_stack.cli import commands_leasing + + stack = make_stack() + stack.acquire('one') + monkeypatch.setattr(commands_leasing, '_open_controller', lambda *a, **k: stack.ctl) + capsys.readouterr() + assert commands_leasing.RoutesListCLI.main(argv=['--json']) == 0 + rows = {r['name']: r for r in json.loads(capsys.readouterr().out)['routes']} + assert rows['one']['live'] is True and rows['one']['upstream'] != '?' + + +def test_dynamic_routing_gives_dedicated_deployments_their_own_upstreams(make_stack): + stack = make_stack(dynamic_routing=True) + first = stack.acquire('one', dedicated=True, apply=False) + second = stack.acquire('one', dedicated=True, apply=False) + assert first.deployments[0].id != second.deployments[0].id + routes = json.loads((stack.front.state_dir / 'litellm_routes.json').read_text()) + ones = [r for r in routes if r['model_name'] == 'one'] + assert len(ones) == 2 # one alias, two upstreams + assert len({(r['litellm_params']['model'], r['litellm_params']['api_base']) + for r in ones}) == 2 + +def test_open_webui_and_the_reverse_proxy(make_stack): + from infer_stack.leasing.gateway import NGINX_SERVICE, OPEN_WEBUI_SERVICE + + stack = make_stack(ui=True, reverse_proxy=True) + stack.acquire('one', apply=False) + doc = yaml.safe_load(stack.front.compose_file.read_text()) + assert {OPEN_WEBUI_SERVICE, NGINX_SERVICE} <= set(doc['services']) + + +def test_the_gateways_changes_are_approved_with_the_acquire(make_stack, monkeypatch): + from infer_stack import diff_prompt as dp + + stack = make_stack() + for b in {id(stack.backend): stack.backend, id(stack.front): stack.front}.values(): + b.assume_yes = False + asked = [] + monkeypatch.setattr(dp, 'confirm_writes', + lambda changed, **kw: asked.append(sorted(p.name for p in changed)) + or True) + out = stack.acquire('one') + assert out.lease.state == LeaseState.ACTIVE + shown = [name for names in asked for name in names] + assert 'litellm_config.yaml' in shown # the gateway's change too + # Each file was shown once, before the commit, never again at the render. + assert len(shown) == len(set(shown)) + + +# -- day-2 ------------------------------------------------------------------------- + + +def test_instances_have_one_shape(make_stack): + stack = make_stack() + out = stack.acquire('one') + gid = out.deployments[0].id + instances = stack.backend.instances() + engines = [i for i in instances if i.deployment_id == gid] + assert len(engines) == 1 and engines[0].state == 'running' + assert any(not i.is_engine for i in instances) # the gateway + + +def test_stack_down_stops_everything(make_stack): + stack = make_stack() + stack.acquire('one') + stack.backend.down() + assert stack.running() == set() + + +def test_the_tui_reads_either_backend(make_stack): + """Leases, deployments, the Instances tab and the engines log view.""" + pytest.importorskip('textual') + import asyncio + + from infer_stack.tui import ENGINE_SERVICES, InferStackTUI + + stack = make_stack() + out = stack.acquire('one') + gid = out.deployments[0].id + + async def scenario(): + app = InferStackTUI(stack.ctl, CAT, interval=999, proc_factory=lambda s: None) + async with app.run_test() as pilot: + await pilot.pause() + await app.workers.wait_for_complete() # the startup refresh + app._collapsed['docker'] = False + data = app._collect() + assert [le.id for le in data['leases']] == [out.lease.id] + assert gid in {g.id for g in data['deployments']} + # What _render keeps, without racing the app's own refresh. + app._last_deployments = data['deployments'] + app._last_instances = data['instances'] + (engine,) = [r for r in app._ps_rows(data['instances']) + if r['serves'] == 'one'] + assert engine['status'] == 'running' + target, _ = app._resolve_log_target(ENGINE_SERVICES) + assert target == [engine['name']] + + asyncio.run(scenario()) From c430cafa2da0e0a881b8746169e0f43f4dd939de Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 12:37:15 -0400 Subject: [PATCH 032/141] Queue: record the P6 commit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 7c89a151..49d92e8d 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -71,7 +71,7 @@ dynamic routing and gets two Models and two routes. ### 5. [x] P6: one test surface -Done 2026-09-26. +Done 2026-09-26, `1a4e4b2`. Parametrize the controller's acquire scenarios over the Memory, fake-Compose and fake-KubeAI backends; `tests/test_parity.py` runs each From 94c263de5c96928dfe0afa874cd034465a895e5b Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:08:28 -0400 Subject: [PATCH 033/141] P4: KubeAI picks a resource profile by GPU size An endpoint with placement.min_vram_gib and no resource_profile gets the smallest resource profile whose nodes' GPUs are that large: a profile's size is the nvidia.com/gpu.memory label (GPU Feature Discovery) of the nodes its nodeSelector selects. The chart's selector-less profiles have no size. KubeAI keeps its own measurements overlay, filled by `measure --record`, and a measurement feeds the same choice, as on compose. `catalog suggest --backend kubeai` sizes the catalog to the largest GPU node and proposes a resource profile per GPU product. dev/k3s_agent_container.sh adds a tainted second node on this host; the e2e gains E2E_SIZED=1 (two nodes, fake GPU labels): 40 GiB lands on the 80 GiB profile's node, 10 GiB on the 24 GiB one and answers. The real-GPU check is dev/handover/p4_gpu_labels.sh. The e2e's `grep -q` checks no longer race pipefail, and the dynamic phase waits for a quiescent stack. Verified: pytest infer_stack tests; ty; dev/kubeai_e2e.sh with every phase on k3s. Journal and two lessons added. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 12 ++ dev/e2e_tests/kubeai-cpu-values.yaml | 22 +++ dev/handover/p4_gpu_labels.sh | 108 ++++++++++++ dev/journals/claude.md | 41 +++++ dev/k3s_agent_container.sh | 49 ++++++ dev/kubeai_e2e.sh | 84 ++++++++- dev/lessons/lessons.md | 18 ++ dev/lessons/mwe/grep_q_pipefail.sh | 8 + docs/backend-parity.md | 3 +- docs/kubeai-backend.md | 17 ++ docs/planning/backend-parity-roadmap.md | 15 +- docs/queue.md | 13 +- infer_stack/backends/kubeai.py | 223 ++++++++++++++++++++++-- infer_stack/cli/commands_catalog.py | 53 +++++- infer_stack/cli/commands_leasing.py | 7 +- tests/test_leasing_kubeai.py | 89 ++++++++++ 16 files changed, 732 insertions(+), 30 deletions(-) create mode 100755 dev/handover/p4_gpu_labels.sh create mode 100755 dev/k3s_agent_container.sh create mode 100644 dev/lessons/mwe/grep_q_pipefail.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f6b58a..a5ce85a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### KubeAI picks a resource profile by GPU size + +On the kubeai backend, an endpoint with `placement.min_vram_gib` and no +`resource_profile` gets the smallest resource profile whose nodes' GPUs are +that large, read from GPU Feature Discovery's `nvidia.com/gpu.memory` label; +before, `min_vram_gib` was warned about and ignored. `infer-stack measure +--record` works there too, and a recorded measurement feeds the same choice. +`infer-stack catalog suggest` on kubeai sizes the catalog to the cluster's +largest GPU node and proposes a resource profile per GPU product for the +helm values. `dev/k3s_agent_container.sh` adds a second (simulated) node to +a development cluster. + ### A parity suite for the two backends `tests/test_parity.py` runs one scenario per row that `docs/backend-parity.md` diff --git a/dev/e2e_tests/kubeai-cpu-values.yaml b/dev/e2e_tests/kubeai-cpu-values.yaml index 5d308441..8d5fb354 100644 --- a/dev/e2e_tests/kubeai-cpu-values.yaml +++ b/dev/e2e_tests/kubeai-cpu-values.yaml @@ -14,3 +14,25 @@ resourceProfiles: requests: cpu: 64 memory: "16Gi" + # Two "GPU sizes" on CPU, for the sizing check (E2E_SIZED=1 in + # dev/kubeai_e2e.sh): each selects the nodes labelled with one fake GPU + # product, as a real profile selects nvidia.com/gpu.product. `sized-80g` + # tolerates dev/k3s_agent_container.sh's taint, so it lands on that node. + sized-24g: + imageName: "cpu" + requests: + cpu: 8 + memory: "16Gi" + nodeSelector: + nvidia.com/gpu.product: "FAKE-24G" + sized-80g: + imageName: "cpu" + requests: + cpu: 8 + memory: "16Gi" + nodeSelector: + nvidia.com/gpu.product: "FAKE-80G" + tolerations: + - key: infer-stack.test/simulated + operator: Exists + effect: NoSchedule diff --git a/dev/handover/p4_gpu_labels.sh b/dev/handover/p4_gpu_labels.sh new file mode 100755 index 00000000..e47e2cfe --- /dev/null +++ b/dev/handover/p4_gpu_labels.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# P4 on real GPUs: what the CPU-only development cluster cannot show. +# +# Run on a GPU machine that is a k3s node with the NVIDIA device plugin and +# GPU Feature Discovery (README: "NVIDIA GPU support"), with the KubeAI chart +# installed and `kubectl -n kubeai port-forward svc/kubeai 8000:80` running. +# It serves one small model on one GPU for a few minutes: run it when that +# GPU is free. +# +# It checks, in order: +# 1. GPU Feature Discovery labels each GPU node (product, memory) and the +# device plugin advertises nvidia.com/gpu: the facts sizing reads. +# 2. `catalog suggest --backend kubeai` proposes a resource profile for +# each GPU product, from those labels. +# 3. With the proposed profiles installed, an endpoint that declares only +# `min_vram_gib` gets the right profile and answers a request on a GPU. +# 4. `infer-stack measure --record` finds vLLM's memory-profiling +# lines in the pod's log and records a min_vram_gib. +# +# It uses its own config and data directories (never this host's infer-stack +# state). It leaves the proposed profiles in the chart's values: they +# describe this cluster's GPUs and are what min_vram_gib needs. +# +# dev/handover/p4_gpu_labels.sh 2>&1 | tee p4-gpu.log +# +# Knobs: E2E_MODEL (default Qwen/Qwen2.5-0.5B-Instruct), NAMESPACE (kubeai), +# E2E_TIMEOUT (900 s). +set -euo pipefail + +MODEL="${E2E_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" +NAMESPACE="${NAMESPACE:-kubeai}" +TIMEOUT="${E2E_TIMEOUT:-900}" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/infer-stack-p4.XXXXXX")" +IS_ENV="INFER_STACK_CONFIG_DIR=$WORK/config INFER_STACK_DATA_DIR=$WORK/data" +run_is() { env $IS_ENV infer-stack "$@"; } +pass() { echo "PASS $*"; } +fail() { echo "FAIL $*" >&2; exit 1; } + +cleanup() { + set +e + run_is release --all --yes >/dev/null 2>&1 + run_is stack down >/dev/null 2>&1 + echo "work dir kept for the report: $WORK" +} +trap cleanup EXIT +mkdir -p "$WORK/config" + +echo '== 1. the GPU facts sizing reads' +kubectl get nodes -o json | python3 -c ' +import json, sys +found = 0 +for n in json.load(sys.stdin)["items"]: + labels = n["metadata"].get("labels", {}) + gpus = n["status"].get("allocatable", {}).get("nvidia.com/gpu") + product, memory = labels.get("nvidia.com/gpu.product"), labels.get("nvidia.com/gpu.memory") + print(f" {n[\"metadata\"][\"name\"]}: gpus={gpus} product={product} memory_mib={memory}") + found += bool(gpus and product and memory) +sys.exit(0 if found else 1)' || fail 'no node has nvidia.com/gpu and the GFD product/memory labels' +pass 'GPU nodes are labelled' + +echo '== 2. catalog suggest proposes profiles from the labels' +run_is config set backend kubeai >/dev/null +run_is config set kubeai_namespace "$NAMESPACE" >/dev/null +run_is catalog suggest --backend kubeai > "$WORK/suggest.yaml" 2> "$WORK/suggest.err" \ + || { cat "$WORK/suggest.err" >&2; fail 'catalog suggest failed'; } +python3 - "$WORK/suggest.err" "$WORK/profiles.yaml" <<'PY' || fail 'suggest printed no resourceProfiles' +import sys, yaml +text = open(sys.argv[1]).read() +block = yaml.safe_load(text[text.index('resourceProfiles:'):]) +open(sys.argv[2], 'w').write(yaml.safe_dump(block, sort_keys=False)) +print(yaml.safe_dump(block, sort_keys=False)) +PY +pass 'suggest proposed a profile per GPU product' + +echo '== 3. min_vram_gib picks the proposed profile and serves on a GPU' +version=$(helm list -n "$NAMESPACE" -o json | python3 -c \ + 'import json,sys; print(json.load(sys.stdin)[0]["chart"].rsplit("-", 1)[1])') +helm upgrade kubeai kubeai/kubeai -n "$NAMESPACE" --version "$version" \ + --reuse-values -f "$WORK/profiles.yaml" --wait >/dev/null +cat > "$WORK/config/catalog.yaml" < "$WORK/acquire.log" 2>&1 || true +grep -q 'ready: True' "$WORK/acquire.log" || fail 'p4-small never became ready' +got=$(kubectl -n "$NAMESPACE" get models.kubeai.org -l infer-stack/managed=true \ + -o jsonpath='{.items[0].spec.resourceProfile}') +echo " Model resourceProfile: $got (profiles proposed: $(tr '\n' ' ' < "$WORK/profiles.yaml" | head -c 200))" +case "$got" in nvidia-*:1) pass "min_vram_gib 8 -> $got, and it answers" ;; + *) fail "unexpected resource profile $got" ;; esac + +echo '== 4. measure reads the pod log on a GPU' +run_is measure p4-small --record | tee "$WORK/measure.log" +grep -q 'min_vram_gib' "$WORK/measure.log" || fail 'measure printed no min_vram_gib' +test -s "$WORK/data/leasing/kubeai/measurements.json" || fail 'measure --record wrote nothing' +pass 'measure recorded a min_vram_gib on kubeai' + +echo 'ALL PASS' diff --git a/dev/journals/claude.md b/dev/journals/claude.md index aeb38b4a..c2441faa 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3249,3 +3249,44 @@ Fixed with sparse files. **Takeaways.** (1) A comparison of outputs must check the outputs exist: equality of two failures is not evidence. (2) Test fixtures that need large files should be sparse. Size was the property under test, not the bytes. + +## 2026-09-26 13:05:00 -0400 + +**Intent.** Execute the backend-parity queue (`docs/queue.md`): the roadmap +phases that can be built and verified without a GPU or a second machine, +refactoring duplicate authorities on the way, and not stopping before a +passing UX audit. Model: Claude Opus 5.5 (Claude Code). + +**What landed.** P1b deleted the pre-admission acquire branch: a +`SimpleAdmission` mixin gives the dry-run and test backends the admission +surface, so the controller has one path. P2 put `ps`, `logs`, `status` and +the TUI behind an `Instance` view built from residency, on both backends. +P3 made the gateway in front of a cluster the compose gateway (UI, proxy, +dynamic routing, one approval). P6 is a parity suite, one test per *same* +row. P4 picks a KubeAI resource profile by GPU size from node labels. + +**Decisions.** The review split P1: a `preview` alone would have dropped +every KubeAI lease, because the admission view assumed GPU accounting, so +"does this backend allocate GPUs" became one function. Under dynamic +routing a KubeAI Model is named per deployment with compose's own tail +rule, rather than a new scheme. A profile's size is what its node selector +selects; the chart's selector-less profiles deliberately have none, because +guessing a size for "anywhere" would pick wrong silently. + +**What surprised me.** Three bugs older than this work surfaced only in a +real terminal or a real cluster: the TUI replaced kubeai's kubectl runner +with Docker's (no KUBECONFIG, so every kubectl call failed), a kwconf flag +swallowed the positional after it on every command, and the TUI swapped an +injected runner for the real Docker. Fakes passed throughout; each was +found by running the thing. The e2e also failed once on its own +`grep -q` + pipefail pattern (lessons.md). + +**Risks.** The kubeai recovery profile now nests the gateway's profile; +old profiles (a bare boolean) keep this process's settings. P4 is verified +with fake labels only; `dev/handover/p4_gpu_labels.sh` is the real test. + +**Takeaways.** (1) When a refactor removes a branch, first run the whole +suite with the other branch forced on: the failures list exactly what to +migrate. (2) A seam that wraps a runner must wrap only what it owns; a +wrapper that replaces is a second authority. (3) Put every e2e check's +output in a file before testing it. diff --git a/dev/k3s_agent_container.sh b/dev/k3s_agent_container.sh new file mode 100755 index 00000000..379594af --- /dev/null +++ b/dev/k3s_agent_container.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# A second k3s node on this host, in a Docker container: enough to exercise +# anything that depends on the cluster having more than one node (node +# labels, node selectors, where a pod lands) without a second machine. +# +# dev/k3s_agent_container.sh up [NAME] # join, taint, print the node +# dev/k3s_agent_container.sh down [NAME] # drain it out of the cluster +# +# The node is tainted `infer-stack.test/simulated=true:NoSchedule`, so only +# pods that tolerate it land there: an ordinary Model never does (its image +# would be pulled inside the container's own containerd). Give a test +# resource profile that toleration to put a Model on it. +# +# Needs: this host running the k3s server (scripts/bootstrap_k3s.sh), sudo +# to read the node token, and Docker. The image tag follows the server's. +set -euo pipefail + +ACTION="${1:-up}" +NAME="${2:-k3s-agent-b}" +CONTAINER="infer-stack-${NAME}" + +case "$ACTION" in + up) + version="$(k3s --version | awk 'NR==1 {print $3}')" # v1.36.4+k3s1 + image="rancher/k3s:${version/+/-}" + server_ip="$(ip -4 route get 1.1.1.1 | awk '{for (i=1;i/dev/null 2>&1 || true + docker run -d --name "$CONTAINER" --privileged --hostname "$NAME" \ + -e K3S_URL="https://${server_ip}:6443" -e K3S_TOKEN="$token" \ + --tmpfs /run --tmpfs /var/run "$image" agent --node-name "$NAME" >/dev/null + for _ in $(seq 60); do + kubectl get node "$NAME" >/dev/null 2>&1 && break + sleep 2 + done + kubectl wait --for=condition=Ready "node/$NAME" --timeout=120s >/dev/null + kubectl taint node "$NAME" infer-stack.test/simulated=true:NoSchedule --overwrite >/dev/null + kubectl get node "$NAME" -o wide + ;; + down) + kubectl delete node "$NAME" --ignore-not-found >/dev/null + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + echo "removed $NAME" + ;; + *) + echo "usage: $0 up|down [NAME]" >&2 + exit 2 + ;; +esac diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index e106ee75..5999338d 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -27,6 +27,10 @@ # E2E_DYNAMIC 1 (default): finish with dynamic routing, two # --dedicated leases of one model (two Models at once). # E2E_UI_PORT Open WebUI's port on this host (default 13000). +# E2E_SIZED 1: check min_vram_gib picks a profile by GPU size, +# on two nodes with fake GPU labels. Needs +# dev/k3s_agent_container.sh up (E2E_SIZED_NODE names +# the node; default k3s-agent-b). set -euo pipefail MODEL="${E2E_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" @@ -49,6 +53,13 @@ cleanup() { run_is release --all --yes >/dev/null 2>&1 run_is gc --evict --yes >/dev/null 2>&1 run_is stack down >/dev/null 2>&1 # the gateway, UI and database too + if [ "${SIZED_LABELLED:-0}" = 1 ]; then # the fake GPU labels, off again + for node in "$NODE_A" "$NODE_B"; do + kubectl label node "$node" nvidia.com/gpu.product- nvidia.com/gpu.memory- >/dev/null 2>&1 + done + kubectl patch node "$NODE_B" --subresource=status --type=json \ + -p '[{"op":"remove","path":"/status/capacity/nvidia.com~1gpu"}]' >/dev/null 2>&1 + fi # nothing managed may remain on the cluster, pass or fail leftover=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ -l infer-stack/managed=true -o name 2>/dev/null | wc -l) @@ -175,8 +186,11 @@ if [ "${E2E_MAKE_ROOM:-0}" = 1 ]; then EOF run_is acquire e2e-warm --yes --timeout "$TIMEOUT" --env-file "$WORK/warm.env" run_is release --env-file "$WORK/warm.env" --yes # idle, still resident - if ! run_is acquire e2e-big --yes --timeout "$TIMEOUT" \ - --env-file "$WORK/big.env" 2>&1 | tee "$WORK/big.log" | grep -q 'ready: True'; then + # Into a file first: `grep -q` quits at its match, and under pipefail the + # writer's SIGPIPE would fail the check. + run_is acquire e2e-big --yes --timeout "$TIMEOUT" \ + --env-file "$WORK/big.env" > "$WORK/big.log" 2>&1 || true + if ! grep -q 'ready: True' "$WORK/big.log"; then echo '!! the leased model never became ready' >&2; exit 1 fi grep -q 'making room for leased demand' "$WORK/big.log" \ @@ -185,9 +199,75 @@ EOF run_is release --env-file "$WORK/big.env" --yes fi +if [ "${E2E_SIZED:-0}" = 1 ]; then + # Needs a second node (dev/k3s_agent_container.sh up) and the sized-* + # profiles of dev/e2e_tests/kubeai-cpu-values.yaml. Fake GPU labels stand + # in for GPU Feature Discovery's; cleanup removes them. + echo '== min_vram_gib picks the smallest resource profile whose GPUs fit' + NODE_A=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[0].metadata.name}') + NODE_B="${E2E_SIZED_NODE:-k3s-agent-b}" + kubectl label node "$NODE_A" nvidia.com/gpu.product=FAKE-24G nvidia.com/gpu.memory=24576 --overwrite >/dev/null + kubectl label node "$NODE_B" nvidia.com/gpu.product=FAKE-80G nvidia.com/gpu.memory=81920 --overwrite >/dev/null + kubectl patch node "$NODE_B" --subresource=status --type=merge \ + -p '{"status":{"capacity":{"nvidia.com/gpu":"2"}}}' >/dev/null + SIZED_LABELLED=1 + cat >> "$WORK/config/catalog.yaml" < -> " " of its Model's pod + for _ in $(seq 60); do + out=$(kubectl -n "$NAMESPACE" get models.kubeai.org -l infer-stack/managed=true -o json \ + | python3 -c ' +import json, sys +for m in json.load(sys.stdin)["items"]: + if sys.argv[1].replace("/", "-").lower() in m["metadata"]["name"]: + print(m["metadata"]["name"], m["spec"]["resourceProfile"])' "$1") + name=${out%% *}; profile=${out##* } + node=$(kubectl -n "$NAMESPACE" get pods -l "model=$name" -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null || true) + [ -n "$node" ] && { echo "$profile $node"; return; } + sleep 2 + done + echo "$profile unscheduled" + } + run_is acquire e2e-sized-big --wait false --yes --env-file "$WORK/big-sized.env" >/dev/null + got=$(where e2e-sized-big) + [ "$got" = "sized-80g:1 $NODE_B" ] || { echo "!! 40 GiB went to: $got" >&2; exit 1; } + echo " min_vram_gib 40 -> $got" + run_is acquire e2e-sized-small --yes --timeout "$TIMEOUT" --env-file "$WORK/small-sized.env" \ + > "$WORK/small-sized.log" 2>&1 || true + grep -q 'ready: True' "$WORK/small-sized.log" \ + || { echo '!! the 10 GiB endpoint never became ready' >&2; exit 1; } + got=$(where e2e-sized-small) + [ "$got" = "sized-24g:1 $NODE_A" ] || { echo "!! 10 GiB went to: $got" >&2; exit 1; } + echo " min_vram_gib 10 -> $got, and it answers" + run_is catalog suggest --backend kubeai > "$WORK/suggest.yaml" 2> "$WORK/suggest.err" + grep -q 'nvidia-fake-80g' "$WORK/suggest.err" \ + || { cat "$WORK/suggest.err" >&2; echo '!! suggest proposed no profile for FAKE-80G' >&2; exit 1; } + echo ' catalog suggest proposes a resource profile per GPU product' + run_is release --env-file "$WORK/big-sized.env" --yes >/dev/null + run_is release --env-file "$WORK/small-sized.env" --yes >/dev/null +fi + if [ "${GATEWAY:-1}" = 1 ] && [ "${E2E_DYNAMIC:-1}" = 1 ]; then echo '== dynamic routing: two --dedicated leases on one model, two Models, one alias' run_is stack down >/dev/null 2>&1 # the gateway comes back with Postgres + # A routing-mode change is adopted only by a quiescent stack: wait for the + # last managed pod (a released Model's) to be gone. + for _ in $(seq 90); do + [ -z "$(kubectl -n "$NAMESPACE" get pods -l infer-stack/managed=true -o name)" ] && break + sleep 2 + done run_is config set dynamic_routing true for n in 1 2; do run_is acquire "$ALIAS" --dedicated --yes --ttl 30m --timeout "$TIMEOUT" \ diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index ecdeb16c..31b2bbfb 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -107,3 +107,21 @@ evidence; prefer append-only; supersede incorrect entries with a new one. leaves nothing behind. - **Applies when:** a test exercises size-dependent logic (VRAM floors, disk checks, download sizes). + +- **Lesson:** A kwconf flag (`isflag=True`) takes an optional value, so a + flag written before a positional swallows it: `logs -f qwen` parsed as + `follow='qwen'` with no names. infer-stack's flags are all boolean, so + `_FlagSafeMixin` hands a non-boolean string back to the positional list. +- **Evidence / MWE:** `tests/test_day2.py::test_a_flag_never_swallows_the_positional_after_it`; + `infer_stack/cli/options.py` (`reclaim_swallowed_positionals`). Found live + 2026-09-26: `logs -f ` followed every instance. +- **Applies when:** a kwconf command has both flags and positional + arguments. + +- **Lesson:** `producer | grep -q pattern` under `set -o pipefail` can fail + although the pattern matched: `grep -q` exits at the first match and the + producer's next write dies of SIGPIPE (141). Write to a file, then grep it. +- **Evidence / MWE:** `dev/lessons/mwe/grep_q_pipefail.sh` prints 141 for + the pipeline and 0 for the file. +- **Applies when:** an e2e or handover script checks a command's output with + `grep -q` under `set -euo pipefail`. diff --git a/dev/lessons/mwe/grep_q_pipefail.sh b/dev/lessons/mwe/grep_q_pipefail.sh new file mode 100644 index 00000000..d85acfd8 --- /dev/null +++ b/dev/lessons/mwe/grep_q_pipefail.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# `producer | grep -q pattern` under `set -o pipefail` can report failure +# although the pattern matched: grep -q exits at its first match, the +# producer's next write gets SIGPIPE (exit 141), and pipefail reports that. +set -o pipefail +seq 1 1000000 | grep -q '^1$'; echo "pipeline: $? (141 = the producer's SIGPIPE, not a miss)" +seq 1 1000000 > /tmp/grep-q-mwe.txt; grep -q '^1$' /tmp/grep-q-mwe.txt; echo "file: $?" +rm -f /tmp/grep-q-mwe.txt diff --git a/docs/backend-parity.md b/docs/backend-parity.md index de60913f..e776a625 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -88,7 +88,8 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | where a deployment lands | local planner over `nvidia-smi` | the cluster scheduler, via `resource_profile:` | | GPU count from TP × PP × DP | same | same | | `placement.gpu_indices`, `allowed_gpus`, `skip_display_gpus` | yes | n/a: no host indices; node-scoped resource profiles are the equivalent | -| `placement.min_vram_gib`, `infer-stack measure` | yes | **gap** (P4): `min_vram_gib` is warned and ignored; `measure` runs but is unverified on a GPU, and `--record` writes an overlay KubeAI does not read | +| `placement.min_vram_gib` (declared, or recorded by `measure --record`) | picks GPUs that large | ≈ picks the smallest resource profile whose nodes' GPUs are that large (GFD's `nvidia.com/gpu.memory` label); verified with fake labels, a real-GPU run is `dev/handover/p4_gpu_labels.sh` | +| `catalog suggest` | sized to this host's GPUs | ≈ sized to the largest GPU node, plus a `resourceProfiles` block per GPU product | | GPU allocations recorded in the ledger | yes | none: the cluster owns them | | more than one host | boundary | yes: the reason the backend exists | diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 8cf548f1..7524f12f 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -87,6 +87,23 @@ endpoints: resource_profile: nvidia-gpu-rtx-4090 # -> nvidia-gpu-rtx-4090:2 ``` +### Sizing: `min_vram_gib` picks a profile + +An endpoint that names no `resource_profile` but declares +`placement.min_vram_gib` (or has one recorded by `infer-stack measure +--record`) gets the smallest resource profile whose GPUs are that large. A +profile's GPU size is the `nvidia.com/gpu.memory` label (set by GPU Feature +Discovery) of the nodes its `nodeSelector` selects; a profile without a +`nodeSelector`, like the chart's generic ones, has no size and is never +picked this way. With none large enough, the `kubeai_resource_profile` +default is used, or the acquire is refused with the sizes it found. +`infer-stack catalog suggest` proposes one sized profile per GPU product in +the cluster, ready for the helm values: + +```bash +infer-stack catalog suggest --backend kubeai # catalog on stdout, profiles on stderr +``` + Verify the setup before the first acquire — `doctor` checks the chain in dependency order (cluster reachable → CRD installed → namespace → gateway): diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 6e5aee23..503f2975 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,8 +1,8 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **P1–P3, P6 done** 2026-09-26 · P4, P5 not -started. Execution order: [../queue.md](../queue.md). +`dev/backend-unification` · **P1–P4, P6 done** 2026-09-26 (P4's GPU run is +a handover) · P5 not started. Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the @@ -184,6 +184,17 @@ the right profile on a cluster with two sizes. **Size:** small. Shrinks "the information you add"; skip if nobody runs a mixed-GPU cluster. +**P4 done 2026-09-26** on two k3s nodes (the second a container, +`dev/k3s_agent_container.sh`) with fake GPU labels: `min_vram_gib 40` got +the 80 GiB profile and scheduled on that node, `10` got the 24 GiB one and +answered, and `catalog suggest` proposed a profile per product +(`E2E_SIZED=1 dev/kubeai_e2e.sh`). A profile's size is its selected nodes' +`nvidia.com/gpu.memory`; the chart's generic profiles have no selector, so +no size. KubeAI keeps its own measurements overlay and fills a missing +`min_vram_gib` from it, as compose does, so `measure --record` feeds the +choice. Real GPU labels, a GPU serving the chosen profile, and `measure` +reading vLLM's memory lines are `dev/handover/p4_gpu_labels.sh`. + ### P5. The multi-workstation shape **Closes:** the gateway on one host, the port-forward default. diff --git a/docs/queue.md b/docs/queue.md index 49d92e8d..9bb99396 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -79,7 +79,12 @@ fake-Compose and fake-KubeAI backends; `tests/test_parity.py` runs each **Done when:** every *same* row has a parity test, and the CI suite runs it. -### 6. [ ] P4: placement from node labels (verified with faked labels) +### 6. [x] P4: placement from node labels (verified with faked labels) + +Done 2026-09-26; the GPU run is `dev/handover/p4_gpu_labels.sh`. +*Why reordered:* two GPU sizes need two nodes (a node has one +`nvidia.com/gpu.memory` label), so item 7's simulated second node +(`dev/k3s_agent_container.sh`) was built first, here. `catalog suggest` on KubeAI proposes `resourceProfiles` from `nvidia.com/gpu.product` / `.memory` node labels; `min_vram_gib` picks the @@ -159,6 +164,12 @@ Seed findings, already known: - [x] `infer-stack logs -f qwen` followed every instance: a kwconf flag took the next word as its value. Fixed for every command (P2). - [ ] `status` shows STALE during an apply instead of "apply in progress". +- [ ] Open WebUI writes its data directory as root: removing a data root + (the e2e's cleanup, or an operator's `rm -rf`) fails with permission + denied. Found by `dev/kubeai_e2e.sh` 2026-09-26. +- [ ] A refused acquire always ends with "free a GPU first — …", also when + the reason is a render refusal (a served-name collision) or the + backend is kubeai, where no GPU is ours to free. Found 2026-09-26. ### 10. [ ] Handover diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 1852860c..c9382288 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -156,6 +156,103 @@ def _model_doc( return doc +#: GPU Feature Discovery's node labels: the GPU model, and per-GPU memory (MiB). +GPU_PRODUCT_LABEL = 'nvidia.com/gpu.product' +GPU_MEMORY_LABEL = 'nvidia.com/gpu.memory' +GPU_RESOURCE = 'nvidia.com/gpu' + + +def node_gpus(nodes: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Each node's GPUs, from its labels and allocatable: ``{node: facts}``. + + ``facts`` is ``{'product', 'memory_gib', 'count', 'labels'}``; a node with + no memory label has ``memory_gib`` None. + + >>> n = {'metadata': {'name': 'a', 'labels': {GPU_PRODUCT_LABEL: 'L4', + ... GPU_MEMORY_LABEL: '23034'}}, + ... 'status': {'allocatable': {GPU_RESOURCE: '2'}}} + >>> node_gpus([n])['a']['memory_gib'], node_gpus([n])['a']['count'] + (22.49, 2) + """ + out = {} + for node in nodes: + meta = node.get('metadata') or {} + labels = dict(meta.get('labels') or {}) + memory = labels.get(GPU_MEMORY_LABEL) + try: + memory_gib = round(int(memory) / 1024, 2) if memory else None + except ValueError: + memory_gib = None + count = str(((node.get('status') or {}).get('allocatable') or {}).get(GPU_RESOURCE) or '') + out[str(meta.get('name'))] = { + 'product': labels.get(GPU_PRODUCT_LABEL), + 'memory_gib': memory_gib, + 'count': int(count) if count.isdigit() else 0, + 'labels': labels, + } + return out + + +def sized_profiles(profiles: dict[str, Any], nodes: dict[str, dict[str, Any]]) -> dict[str, float]: + """Per-GPU memory (GiB) of each resource profile that says where it runs. + + A profile's size is the smallest ``nvidia.com/gpu.memory`` among the nodes + its ``nodeSelector`` selects. A profile without a ``nodeSelector`` (the + chart's generic ones) could land anywhere, so it has no size and is never + picked by size. + + >>> nodes = {'a': {'memory_gib': 24.0, 'labels': {'gpu': 'small'}}, + ... 'b': {'memory_gib': 80.0, 'labels': {'gpu': 'big'}}} + >>> sized_profiles({'small': {'nodeSelector': {'gpu': 'small'}}, + ... 'big': {'nodeSelector': {'gpu': 'big'}}, + ... 'anywhere': {}}, nodes) + {'big': 80.0, 'small': 24.0} + """ + sizes = {} + for name, profile in sorted((profiles or {}).items()): + selector = (profile or {}).get('nodeSelector') or {} + if not selector: + continue + found = [facts['memory_gib'] for facts in nodes.values() + if facts.get('memory_gib') + and all(facts['labels'].get(k) == str(v) for k, v in selector.items())] + if found: + sizes[name] = min(found) + return sizes + + +def pick_profile(explicit: str | None, min_vram_gib: float | None, + sized: dict[str, float], default: str | None) -> tuple[str | None, str]: + """``(profile, why)`` for one deployment. + + An endpoint's own ``resource_profile`` wins. Else, with ``min_vram_gib``, + the smallest sized profile whose GPUs have that much memory. Else the + ``kubeai_resource_profile`` default. ``why`` says which rule chose, or + why nothing could. + + >>> sized = {'small': 24.0, 'big': 80.0} + >>> pick_profile(None, 40, sized, 'cpu') + ('big', 'min_vram_gib 40 GiB -> big (80 GiB GPUs)') + >>> pick_profile(None, 10, sized, None)[0], pick_profile('mine', 40, sized, None)[0] + ('small', 'mine') + >>> pick_profile(None, 100, sized, None) + (None, 'min_vram_gib 100 GiB: no resource profile has GPUs that large (big 80 GiB, small 24 GiB)') + """ + if explicit: + return explicit, 'runtime.resource_profile' + if min_vram_gib: + fitting = sorted((size, name) for name, size in sized.items() + if size >= float(min_vram_gib)) + if fitting: + size, name = fitting[0] + return name, f'min_vram_gib {min_vram_gib:g} GiB -> {name} ({size:g} GiB GPUs)' + if not default: + known = ', '.join(f'{n} {g:g} GiB' for n, g in sorted(sized.items())) or 'none sized' + return None, (f'min_vram_gib {min_vram_gib:g} GiB: no resource profile has ' + f'GPUs that large ({known})') + return default, 'kubeai_resource_profile' + + class RenderedModels: """Output of the render half: manifest text + bookkeeping maps.""" @@ -165,6 +262,8 @@ def __init__(self) -> None: self.request_names: dict[str, str] = {} # endpoint -> CR name self.unrenderable: set[str] = set() self.errors: list[str] = [] + #: deployment id -> which rule chose its resource profile. + self.profile_reasons: dict[str, str] = {} @property def text(self) -> str: @@ -179,6 +278,7 @@ def render_models( namespace: str, default_resource_profile: str | None, unique_names: bool = False, + sized: dict[str, float] | None = None, ) -> RenderedModels: """Render the desired set into KubeAI ``Model`` docs (pure, no I/O). @@ -215,10 +315,10 @@ def render_models( 'VLLM Model does not. Use --backend compose for this endpoint.' ) continue - profile = ( - runtime.get('resource_profile') or default_resource_profile or '' - ) - if not str(profile).strip(): + min_vram = (deployment.spec.get('placement') or {}).get('min_vram_gib') + profile, why = pick_profile(runtime.get('resource_profile'), min_vram, + sized or {}, default_resource_profile) + if not str(profile or '').strip(): out.unrenderable.add(deployment.id) out.errors.append( f'{deployment.id}: no resource profile — set ' @@ -226,8 +326,10 @@ def render_models( 'resourceProfiles key from your KubeAI helm values, e.g. ' "'nvidia-gpu-rtx-4090') or `config set " 'kubeai_resource_profile ` as the default.' + + (f' ({why})' if min_vram else '') ) continue + out.profile_reasons[deployment.id] = why name = model_name(deployment, unique=unique_names) if name in out.models: out.unrenderable.add(deployment.id) @@ -322,6 +424,12 @@ def __init__( # IP, which containers on a cluster node can reach; a gateway off the # cluster needs an ingress URL here. self.gateway_upstream = gateway_upstream + # `infer-stack measure --record` writes here, as on compose; a + # measurement fills an endpoint's missing min_vram_gib, which picks + # its resource profile by size. + from ..leasing.vram import Measurements + + self.measurements = Measurements(self.state_dir / 'measurements.json') # -- state-dir plumbing -------------------------------------------------- @@ -473,6 +581,92 @@ def _set_front_door(self, desired, rendered: RenderedModels) -> None: #: and admission commits an empty allocation for every deployment. allocates_gpus = False + #: Seconds a read of the cluster's GPU facts is reused: resource profiles + #: and node labels change rarely, and a render reads them once. + GPU_FACTS_TTL = 60.0 + #: The KubeAI chart's configuration, which holds its resourceProfiles. + CONFIG_MAP = 'kubeai-config' + + def gpu_facts(self) -> tuple[dict[str, dict[str, Any]], dict[str, float]]: + """``(each node's GPUs, each sized profile's GPU GiB)`` from the cluster. + + Empty when the cluster cannot say (no GPU labels, no chart config): + sizing then falls back to the default profile, as before. + """ + import time + + cached = getattr(self, '_gpu_facts_cache', None) + if cached is not None and time.monotonic() - cached[0] < self.GPU_FACTS_TTL: + return cached[1] + try: + nodes = (json.loads(self._kubectl(['get', 'nodes', '-o', 'json']) or '{}') + .get('items') or []) + config = json.loads(self._kubectl( + ['get', 'configmap', self.CONFIG_MAP, '-o', 'json']) or '{}') + system = yaml.safe_load(((config.get('data') or {}).get('system.yaml')) or '') or {} + facts = node_gpus(nodes) + sized = sized_profiles(system.get('resourceProfiles') or {}, facts) + except Exception: # noqa: BLE001 - sizing is best-effort, never fatal + facts, sized = {}, {} + self._gpu_facts_cache = (time.monotonic(), (facts, sized)) + return facts, sized + + def suggestion_inventory(self) -> tuple[dict[str, Any], dict[str, Any]]: + """``(inventory, resourceProfiles)`` for ``catalog suggest``. + + The inventory is the GPUs of the largest GPU node (a Model runs on one + node, so that is the biggest thing it can use), shaped like a host + inventory. The profiles are one per GPU product in the cluster, each + selecting that product's nodes: what the helm values need for + ``min_vram_gib`` to choose by size. + """ + facts, _ = self.gpu_facts() + gpu_nodes = [f for f in facts.values() if f['count'] and f['memory_gib']] + if not gpu_nodes: + return {'gpu_count': 0, 'gpus': []}, {} + best = max(gpu_nodes, key=lambda f: (f['count'] * f['memory_gib'], f['memory_gib'])) + gpus = [{'index': i, 'name': best['product'] or 'GPU', + 'memory_gib': best['memory_gib'], + 'memory_mib': int(best['memory_gib'] * 1024), + 'display_active': False} for i in range(best['count'])] + profiles = {} + for f in sorted(gpu_nodes, key=lambda f: f['memory_gib']): + product = f['product'] or 'gpu' + profiles[f'nvidia-{dns_slug(product)}'] = { + 'imageName': 'nvidia-gpu', + 'runtimeClassName': 'nvidia', + 'requests': {GPU_RESOURCE: '1'}, + 'limits': {GPU_RESOURCE: '1'}, + 'nodeSelector': {GPU_PRODUCT_LABEL: product}, + } + return {'gpu_count': len(gpus), 'gpus': gpus}, profiles + + def _enrich_min_vram(self, desired) -> None: + """Fill a missing ``min_vram_gib`` from a recorded measurement (in memory). + + The compose resolution order: a catalog-declared value wins, else the + measurements overlay. Never persisted. + """ + from ..leasing.vram import measurement_key_for_spec + + for g in desired: + placement = dict(g.spec.get('placement') or {}) + if g.engine != 'vllm' or placement.get('min_vram_gib'): + continue + measured = self.measurements.get_min_vram_gib(measurement_key_for_spec(g.spec)) + if measured: + placement.update(min_vram_gib=measured, min_vram_source='measured') + g.spec['placement'] = placement + + def _needs_sizing(self, desired) -> bool: + """Whether any deployment asks for its profile by GPU memory.""" + for g in desired: + runtime = g.spec.get('runtime') or {} + if (g.spec.get('placement') or {}).get('min_vram_gib') and not runtime.get( + 'resource_profile'): + return True + return False + def _render_documents(self, desired: list[Deployment]): """``(plan, rendered, planned)`` in memory: the one KubeAI render. @@ -481,11 +675,14 @@ def _render_documents(self, desired: list[Deployment]): """ from ..leasing.placement import GpuPlan + desired = list(desired) + self._enrich_min_vram(desired) rendered = render_models( - list(desired), + desired, namespace=self.namespace, default_resource_profile=self.default_resource_profile, unique_names=self.dynamic_routing, + sized=self.gpu_facts()[1] if self._needs_sizing(desired) else None, ) plan = GpuPlan( assignments={g.id: [] for g in desired if g.id not in rendered.unrenderable}, @@ -539,19 +736,11 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N self.namespace, ', '.join(sorted(g.id for g in desired)) or '(none)', ) - for g in desired: - # Warn-and-ignore by decision (vram-aware-placement.md, - # Resolutions #3): k8s owns placement on this backend; the - # equivalent mechanism is the resourceProfile / resource - # requests, not our single-host planner. - if (g.spec.get('placement') or {}).get('min_vram_gib'): - logger.warning( - ' {}: placement.min_vram_gib is ignored on the ' - 'kubeai backend (k8s owns placement — express the ' - 'requirement via the resource profile instead)', - g.id, - ) plan, rendered, planned = self._render_documents(desired) + for gid, why in sorted(rendered.profile_reasons.items()): + if why.startswith('min_vram_gib'): + # The cluster places, within the profile chosen by size. + logger.info(' {}: resource profile by {}', gid, why) self.last_errors = list(rendered.errors) self.last_unplaced = set(rendered.unrenderable) self.last_assignments = {} # the cluster places diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index d68e62e0..83b0c5e7 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -269,6 +269,12 @@ class CatalogSuggestCLI( Pure + offline: ``--simulate-hardware 2x80`` suggests for hardware you do not have in front of you. + On the kubeai backend the hardware is the cluster's: GPU Feature + Discovery's node labels (``nvidia.com/gpu.product`` / ``.memory``) and each + node's allocatable GPUs. The catalog is sized to the largest GPU node, and + a ``resourceProfiles`` block (one per GPU product, selecting its nodes) is + printed for the helm values, so ``min_vram_gib`` can pick a profile by size. + infer-stack catalog suggest # render only (no write) infer-stack catalog suggest --simulate-hardware 4x48 infer-stack catalog suggest --apply # merge into the catalog @@ -284,6 +290,11 @@ class CatalogSuggestCLI( False, isflag=True, help='With --apply, overwrite catalog entries that already exist.', ) + backend = kw.Value( + None, type=str, + help='Whose hardware (default: the configured `backend` setting): this ' + "host's GPUs, or on kubeai the cluster's.", + ) @classmethod def main(cls, argv=True, **kwargs): @@ -292,16 +303,40 @@ def main(cls, argv=True, **kwargs): migrate_known_suggestion_aliases, suggest_catalog, ) - from .commands_leasing import _resolve_skip_display + from ..paths import get_setting + from .commands_leasing import _make_backend, _resolve_skip_display from .context import effective_inventory config = cls.cli(argv=argv, data=kwargs) - inventory = effective_inventory(config) or detect_inventory() + profiles: dict = {} + cluster = (config.backend or get_setting('backend')) == 'kubeai' + inventory = effective_inventory(config) + if inventory is None and cluster: + backend = _make_backend(config) + inventory, profiles = backend.suggestion_inventory() + if not inventory['gpus']: + print( + 'no cluster node reports GPUs: suggest needs each GPU node\'s ' + 'allocatable nvidia.com/gpu and GPU Feature Discovery\'s ' + 'nvidia.com/gpu.product / .memory labels (the NVIDIA device ' + 'plugin with gfd.enabled=true; see the README). Pass ' + '--simulate-hardware NxM to plan without them.', + file=sys.stderr, + ) + return 1 + inventory = inventory or detect_inventory() gpus = inventory.get('gpus') or [] - skip_display = _resolve_skip_display(config) + skip_display = _resolve_skip_display(config) and not cluster frag = suggest_catalog( inventory, reserve_display_gpu='auto' if skip_display else False ) + if cluster: + # A cluster has no host GPU indices: the profile says where. + for endpoint in frag['endpoints'].values(): + placement = endpoint.get('placement') or {} + placement.pop('gpu_indices', None) + if not placement: + endpoint.pop('placement', None) max_mem = max((g.get('memory_gib') or 0 for g in gpus), default=0) n_display = sum(1 for g in gpus if g.get('display_active')) @@ -322,6 +357,10 @@ def main(cls, argv=True, **kwargs): text = yaml.safe_dump(frag, sort_keys=False, default_flow_style=False) + if cluster: + hw = f'the cluster\'s largest GPU node: {hw}' + values = (yaml.safe_dump({'resourceProfiles': profiles}, sort_keys=False) + if profiles else '') if not config.apply: print( f'# suggested for: {hw} ({usable} usable)\n' @@ -330,6 +369,10 @@ def main(cls, argv=True, **kwargs): file=sys.stderr, ) _print_yaml(text) + if values: + print('# helm values for the KubeAI chart (merge, then ' + '`scripts/install_kubeai.sh `):\n' + values, + file=sys.stderr) return 0 # --apply: additive merge into the catalog (keep existing entries). @@ -356,6 +399,10 @@ def main(cls, argv=True, **kwargs): f' kept existing (pass --force to overwrite): ' f'{", ".join(skipped)}' ) + if values: + print('resource profiles for the KubeAI chart (merge into your helm ' + 'values, then `scripts/install_kubeai.sh `):') + print(values, end='') return 0 diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 6acb5e48..922a6ea0 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1594,8 +1594,8 @@ def main(cls, argv=True, **kwargs): logs_fn = getattr(backend, 'deployment_logs', None) if logs_fn is None: raise SystemExit( - 'measure needs the compose backend ' - '(the engine container log is the measurement source).' + 'measure reads the engine\'s log, and this backend runs no engine ' + '(set `--backend compose` or `kubeai`).' ) catalog = _requests_catalog(controller, config) name = config.endpoint @@ -1669,8 +1669,7 @@ def main(cls, argv=True, **kwargs): store = getattr(backend, 'measurements', None) if store is None: raise SystemExit( - '--record needs the compose backend measurements ' - 'overlay.' + '--record: this backend keeps no measurements overlay.' ) store.record( key, value, endpoint=name, profile=profile, diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 91b1d950..dbc16253 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -921,3 +921,92 @@ def test_the_front_doors_settings_are_in_the_recovery_profile(tmp_path): assert other.gateway.ui is True and other.dynamic_routing is True other.use_profile({**profile, 'gateway': True}) # a profile from before assert other.dynamic_routing is True # keeps what it has + + +# -- placement information: profiles chosen by GPU size (P4) ------------------ + +SIZED_PROFILES = { + 'small': {'nodeSelector': {'nvidia.com/gpu.product': 'FAKE-24G'}}, + 'big': {'nodeSelector': {'nvidia.com/gpu.product': 'FAKE-80G'}}, + 'generic': {'limits': {'nvidia.com/gpu': '1'}}, # the chart's kind +} + + +def _gpu_node(name, product, mib, count=2): + return {'metadata': {'name': name, 'labels': { + 'nvidia.com/gpu.product': product, 'nvidia.com/gpu.memory': str(mib)}}, + 'status': {'allocatable': {'nvidia.com/gpu': str(count)}}} + + +class SizedKubectl(FakeKubectl): + """A cluster with a 24 GiB node and an 80 GiB node, and the chart's config.""" + + def __call__(self, args): + if len(args) > 4 and args[3] == 'get' and args[4] == 'nodes': + return json.dumps({'items': [_gpu_node('a', 'FAKE-24G', 24576), + _gpu_node('b', 'FAKE-80G', 81920)]}) + if len(args) > 4 and args[3] == 'get' and args[4] == 'configmap': + system = yaml.safe_dump({'resourceProfiles': SIZED_PROFILES}) + return json.dumps({'data': {'system.yaml': system}}) + return super().__call__(args) + + +def _sized(gid, min_vram, **runtime): + dep = vllm(gid, profile=None, **runtime) + dep.spec['placement'] = {'min_vram_gib': min_vram} + return dep + + +def test_min_vram_gib_picks_the_smallest_profile_that_fits(tmp_path): + kubectl = SizedKubectl() + be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=FakeHttp(kubectl)) + be.converge([_sized('grp-a', 40), _sized('grp-b', 10)]) + profiles = {doc['metadata']['labels']['infer-stack/deployment']: doc['spec']['resourceProfile'] + for doc in kubectl.applied.values()} + assert profiles == {'grp-a': 'big:1', 'grp-b': 'small:1'} + + +def test_an_explicit_profile_wins_and_too_large_is_refused(tmp_path): + kubectl = SizedKubectl() + be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=FakeHttp(kubectl)) + explicit = _sized('grp-a', 40, resource_profile='generic') + be.converge([explicit, _sized('grp-b', 200)], apply=False) + assert be.last_unplaced == {'grp-b'} + assert any('no resource profile has GPUs that large' in e for e in be.last_errors) + (doc,) = yaml.safe_load_all(be.models_file.read_text()) + assert doc['spec']['resourceProfile'] == 'generic:1' + + +def test_sizing_reads_the_cluster_only_when_asked(tmp_path): + kubectl = SizedKubectl() + be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=FakeHttp(kubectl)) + be.converge([vllm('grp-a')], apply=False) + assert not any('nodes' in call or 'configmap' in call for call in kubectl.calls) + + +def test_catalog_suggest_on_kubeai_sizes_to_the_cluster(tmp_path, monkeypatch, capsys): + from infer_stack.cli import commands_catalog, commands_leasing + + kubectl = SizedKubectl() + be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=FakeHttp(kubectl)) + monkeypatch.setattr(commands_leasing, '_make_backend', lambda config, **kw: be) + monkeypatch.setenv('INFER_STACK_CONFIG_DIR', str(tmp_path / 'config')) + assert commands_catalog.CatalogSuggestCLI.main(argv=['--backend', 'kubeai']) == 0 + out, err = capsys.readouterr() + assert "largest GPU node" in err + values = yaml.safe_load(err[err.index('resourceProfiles:'):])['resourceProfiles'] + assert values['nvidia-fake-80g']['nodeSelector'] == {'nvidia.com/gpu.product': 'FAKE-80G'} + assert 'gpu_indices' not in out # no host indices on a cluster + + +def test_a_recorded_measurement_picks_the_profile(tmp_path): + """`measure --record` works on kubeai: the overlay fills min_vram_gib.""" + from infer_stack.leasing.vram import measurement_key_for_spec + + kubectl = SizedKubectl() + be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=FakeHttp(kubectl)) + dep = vllm('grp-a', profile=None) + be.measurements.record(measurement_key_for_spec(dep.spec), 50.0, endpoint='grp-a') + be.converge([dep], apply=False) + (doc,) = yaml.safe_load_all(be.models_file.read_text()) + assert doc['spec']['resourceProfile'] == 'big:1' From 08dccd95a727df9fd8ff2137f29f4f0f9cd7c907 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:08:28 -0400 Subject: [PATCH 034/141] Queue: record the P4 commit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 9bb99396..0426a973 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -81,7 +81,7 @@ fake-Compose and fake-KubeAI backends; `tests/test_parity.py` runs each ### 6. [x] P4: placement from node labels (verified with faked labels) -Done 2026-09-26; the GPU run is `dev/handover/p4_gpu_labels.sh`. +Done 2026-09-26, `94c263d`; the GPU run is `dev/handover/p4_gpu_labels.sh`. *Why reordered:* two GPU sizes need two nodes (a node has one `nvidia.com/gpu.memory` label), so item 7's simulated second node (`dev/k3s_agent_container.sh`) was built first, here. From 49e70ddd85598f2097072461f57a2f44469c3a5e Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:30:56 -0400 Subject: [PATCH 035/141] P5: the gateway can run inside the cluster `kubeai_gateway cluster` renders the LiteLLM gateway as a Deployment and a NodePort Service in the KubeAI namespace (backends/kubeai_gateway.py): the same Gateway (image, config, key, route registry), reaching KubeAI by its Service's DNS name, so no port-forward is needed and any node answers. The key is a Secret applied from the state dir and never shown in a diff; its hash in the pod template rolls the pods on rotation. `doctor` checks it, `stack down` removes it. The recovery profile says which placement renders, so the setting is adopted like any other once quiescent (the CLI gives the backend a factory per placement). Gateway gains an optional base URL; `front_door()` (keys and routes) is split from `compose_project()` (a real Compose project). Verified: pytest; dev/kubeai_e2e.sh on k3s with a second node in a container: a NodePort generation, doctor, a Model on the second node answering through the gateway, secrets rotate, stack down. The two-machine run is dev/handover/p5_two_hosts.sh; the runbook is "Add a workstation". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 12 + dev/handover/p5_two_hosts.sh | 116 +++++++++ dev/kubeai_e2e.sh | 62 +++++ docs/backend-parity.md | 6 +- docs/kubeai-backend.md | 60 ++++- docs/planning/backend-parity-roadmap.md | 20 +- docs/queue.md | 4 +- infer_stack/backends/kubeai.py | 31 ++- infer_stack/backends/kubeai_gateway.py | 319 ++++++++++++++++++++++++ infer_stack/cli/commands_leasing.py | 38 ++- infer_stack/cli/commands_meta.py | 9 + infer_stack/leasing/compose.py | 4 + infer_stack/leasing/gateway.py | 16 +- tests/test_leasing_kubeai.py | 101 ++++++++ 14 files changed, 777 insertions(+), 21 deletions(-) create mode 100755 dev/handover/p5_two_hosts.sh create mode 100644 infer_stack/backends/kubeai_gateway.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ce85a8..57289221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### The gateway can run inside the cluster + +`infer-stack config set kubeai_gateway cluster` runs the LiteLLM gateway as a +Deployment and a NodePort Service in the KubeAI namespace instead of on the +host running infer-stack, so no single host is in every request's path and +no `kubectl port-forward` is needed: an env file's `OPENAI_BASE_URL` is a +node's address on the NodePort (or `kubeai_gateway_url`), and every node +answers. It is the same gateway, key and route registry; `secrets rotate` +updates its Secret and rolls it, `doctor` checks it, and `stack down` +removes it. It takes static routes only. `docs/kubeai-backend.md` gains +"Add a workstation", the runbook for a second GPU machine. + ### KubeAI picks a resource profile by GPU size On the kubeai backend, an endpoint with `placement.min_vram_gib` and no diff --git a/dev/handover/p5_two_hosts.sh b/dev/handover/p5_two_hosts.sh new file mode 100755 index 00000000..527e4b78 --- /dev/null +++ b/dev/handover/p5_two_hosts.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# P5 across two real machines: what a second node in a container cannot show. +# +# Run on the cluster's first node (kubeconfig working) once a second GPU +# workstation has joined it (docs/kubeai-backend.md, "Add a workstation"), +# with the NVIDIA device plugin and GPU Feature Discovery running and the +# KubeAI chart installed. It serves one small model on the second node's GPU +# for a few minutes: run it when that GPU is free. +# +# It checks, in order: +# 1. two Ready nodes, and the second reports GPUs with GFD's labels; +# 2. with the gateway in the cluster, an endpoint whose resource profile +# selects the second node's GPU product lands there and answers through +# the gateway: pod-to-pod traffic across the real network (flannel); +# 3. the NodePort answers on the second node's own address too, so a card +# on either workstation uses the same env file; +# 4. `secrets rotate` (a Secret and a rollout) with the Model still served. +# +# Own config and data directories; nothing of this host's infer-stack state +# is touched. It adds one resource profile for the second node's GPU product +# to the chart's values and leaves it: that is what the node needs. +# +# dev/handover/p5_two_hosts.sh [SECOND_NODE_NAME] 2>&1 | tee p5-two-hosts.log +set -euo pipefail + +NODE_B="${1:-}" +MODEL="${E2E_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" +NAMESPACE="${NAMESPACE:-kubeai}" +TIMEOUT="${E2E_TIMEOUT:-900}" +PORT="${NODE_PORT:-30442}" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/infer-stack-p5.XXXXXX")" +IS_ENV="INFER_STACK_CONFIG_DIR=$WORK/config INFER_STACK_DATA_DIR=$WORK/data" +run_is() { env $IS_ENV infer-stack "$@"; } +pass() { echo "PASS $*"; } +fail() { echo "FAIL $*" >&2; exit 1; } + +cleanup() { + set +e + run_is release --all --yes >/dev/null 2>&1 + run_is stack down >/dev/null 2>&1 + echo "work dir kept for the report: $WORK" +} +trap cleanup EXIT +mkdir -p "$WORK/config" + +echo '== 1. two nodes, the second with GPUs' +if [ -z "$NODE_B" ]; then + NODE_B=$(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' \ + -o jsonpath='{.items[0].metadata.name}') +fi +[ -n "$NODE_B" ] || fail 'no second node (pass its name as the first argument)' +kubectl wait --for=condition=Ready "node/$NODE_B" --timeout=30s >/dev/null \ + || fail "$NODE_B is not Ready" +PRODUCT=$(kubectl get node "$NODE_B" -o jsonpath='{.metadata.labels.nvidia\.com/gpu\.product}') +GPUS=$(kubectl get node "$NODE_B" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}') +ADDR_B=$(kubectl get node "$NODE_B" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') +[ -n "$PRODUCT" ] && [ "${GPUS:-0}" -gt 0 ] \ + || fail "$NODE_B reports no GPUs or no nvidia.com/gpu.product label" +pass "$NODE_B at $ADDR_B: $GPUS x $PRODUCT" + +echo '== 2. a Model on the second node, through the in-cluster gateway' +profile="p5-$(echo "$PRODUCT" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9\n' '-')" +cat > "$WORK/profile.yaml" </dev/null +run_is config set backend kubeai >/dev/null +run_is config set kubeai_namespace "$NAMESPACE" >/dev/null +run_is config set kubeai_gateway cluster >/dev/null +cat > "$WORK/config/catalog.yaml" < "$WORK/acquire.log" 2>&1 || true +grep -q 'ready: True' "$WORK/acquire.log" || { tail -30 "$WORK/acquire.log"; fail 'not ready'; } +node=$(kubectl -n "$NAMESPACE" get pods -l infer-stack/managed=true \ + -o jsonpath='{.items[0].spec.nodeName}') +[ "$node" = "$NODE_B" ] || fail "the Model runs on $node, not $NODE_B" +# shellcheck disable=SC1090 +source "$WORK/lease.env" +ask() { # -> a generation for p5-remote + curl -sS --fail-with-body "$1/chat/completions" -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"model": "p5-remote", "max_tokens": 4, "messages": [{"role": "user", "content": "say ok"}]}' \ + > "$WORK/ask.json" && grep -q choices "$WORK/ask.json" +} +ask "$OPENAI_BASE_URL" || fail "no generation via $OPENAI_BASE_URL" +pass "the Model on $NODE_B answers via $OPENAI_BASE_URL" + +echo '== 3. the NodePort answers on the second node too' +ask "http://$ADDR_B:$PORT/v1" || fail "no generation via $ADDR_B:$PORT (firewall on the NodePort?)" +pass "a card on $NODE_B can use http://$ADDR_B:$PORT/v1" + +echo '== 4. secrets rotate across the cluster' +run_is secrets rotate --force > "$WORK/rotate.log" 2>&1 || { cat "$WORK/rotate.log"; fail 'rotate'; } +grep -q 'new key accepted, old key rejected' "$WORK/rotate.log" || { cat "$WORK/rotate.log"; fail 'rotate'; } +pass 'secrets rotate: new key accepted, old key rejected' + +echo 'ALL PASS' diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 5999338d..5430c807 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -31,6 +31,12 @@ # on two nodes with fake GPU labels. Needs # dev/k3s_agent_container.sh up (E2E_SIZED_NODE names # the node; default k3s-agent-b). +# E2E_CLUSTER_GATEWAY 1 (default): finish with the gateway in the cluster +# (kubeai_gateway cluster): a NodePort, doctor, +# secrets rotate, stack down. +# E2E_REMOTE_NODE 1 (with E2E_SIZED=1): also serve a Model on the +# second node through that gateway (pulls the vLLM +# CPU image inside the node container once). set -euo pipefail MODEL="${E2E_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" @@ -294,4 +300,60 @@ if [ "${GATEWAY:-1}" = 1 ] && [ "${E2E_DYNAMIC:-1}" = 1 ]; then run_is release --env-file "$WORK/dyn2.env" --yes fi +if [ "${GATEWAY:-1}" = 1 ] && [ "${E2E_CLUSTER_GATEWAY:-1}" = 1 ]; then + echo '== the gateway inside the cluster: a card reaches the Model on a NodePort' + run_is stack down >/dev/null 2>&1 # the host gateway, and any Models + for _ in $(seq 90); do + [ -z "$(kubectl -n "$NAMESPACE" get pods -l infer-stack/managed=true -o name)" ] && break + sleep 2 + done + run_is config set dynamic_routing false >/dev/null + run_is config set kubeai_gateway cluster + run_is acquire "$ALIAS" --yes --ttl 30m --timeout "$TIMEOUT" \ + --env-file "$WORK/cluster.env" > "$WORK/cluster.log" 2>&1 || true + grep -q 'ready: True' "$WORK/cluster.log" \ + || { tail -20 "$WORK/cluster.log" >&2; echo '!! not ready through the in-cluster gateway' >&2; exit 1; } + # shellcheck disable=SC1090 + source "$WORK/cluster.env" + case "$OPENAI_BASE_URL" in + *:"${E2E_NODE_PORT:-30442}"/v1) ;; + *) echo "!! the env file points at $OPENAI_BASE_URL, not a NodePort" >&2; exit 1 ;; + esac + ask() { # -> a generation through the env file's gateway + curl -sS --fail-with-body "$OPENAI_BASE_URL/chat/completions" \ + -H "Authorization: Bearer $OPENAI_API_KEY" -H 'Content-Type: application/json' \ + -d "{\"model\": \"$1\", \"max_tokens\": 4, + \"messages\": [{\"role\": \"user\", \"content\": \"say ok\"}]}" > "$WORK/ask.json" \ + && grep -q choices "$WORK/ask.json" + } + ask "$ALIAS" || { echo "!! no generation via $OPENAI_BASE_URL" >&2; exit 1; } + echo " generation ok via $OPENAI_BASE_URL (a node's NodePort, no port-forward)" + run_is doctor > "$WORK/doctor.log" 2>&1 || { cat "$WORK/doctor.log" >&2; exit 1; } + grep -q '\[ok \] in-cluster gateway' "$WORK/doctor.log" \ + || { cat "$WORK/doctor.log" >&2; echo '!! doctor did not check the gateway' >&2; exit 1; } + echo ' doctor checks the in-cluster gateway' + if [ "${E2E_SIZED:-0}" = 1 ] && [ "${E2E_REMOTE_NODE:-0}" = 1 ]; then + run_is acquire e2e-sized-big --yes --timeout "$TIMEOUT" \ + --env-file "$WORK/remote.env" > "$WORK/remote.log" 2>&1 || true + grep -q 'ready: True' "$WORK/remote.log" \ + || { tail -20 "$WORK/remote.log" >&2; echo '!! the Model on the second node never became ready' >&2; exit 1; } + got=$(where e2e-sized-big) + [ "$got" = "sized-80g:1 $NODE_B" ] || { echo "!! the remote Model is at: $got" >&2; exit 1; } + ask e2e-sized-big || { echo '!! no generation from the second node' >&2; exit 1; } + echo " a Model on $NODE_B answers through the same gateway" + run_is release --env-file "$WORK/remote.env" --yes >/dev/null + fi + run_is release --env-file "$WORK/cluster.env" --yes >/dev/null + run_is secrets rotate > "$WORK/rotate.log" 2>&1 \ + || { cat "$WORK/rotate.log" >&2; echo '!! secrets rotate failed' >&2; exit 1; } + grep -q 'new key accepted, old key rejected' "$WORK/rotate.log" \ + || { cat "$WORK/rotate.log" >&2; exit 1; } + echo ' secrets rotate: new key accepted, old key rejected (a Secret and a rollout)' + run_is stack down >/dev/null + if kubectl -n "$NAMESPACE" get deployment infer-stack-gateway >/dev/null 2>&1; then + echo '!! stack down left the gateway Deployment' >&2; exit 1 + fi + echo ' stack down removes it' +fi + echo 'PASS: kubeai backend end-to-end lifecycle' diff --git a/docs/backend-parity.md b/docs/backend-parity.md index e776a625..31a0da56 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -24,8 +24,8 @@ cluster setup is [kubeai-backend.md](kubeai-backend.md). | | Compose | KubeAI adds | |---|---|---| | tools | Docker with the NVIDIA runtime | a cluster (`scripts/bootstrap_k3s.sh` for one host, `scripts/join_agent.sh` for another), `kubectl`, `helm`, the KubeAI chart (`scripts/install_kubeai.sh`), the NVIDIA device plugin on GPU nodes | -| information | none: GPUs are discovered with `nvidia-smi` | `resourceProfiles` in the chart values (what one GPU unit requests, and on which nodes), and per endpoint `runtime.resource_profile` or a default `kubeai_resource_profile` | -| settings | `backend compose` | `backend kubeai`; `kubeai_namespace`, `kubeai_base_url`, `kubeai_gateway_upstream` when the defaults (namespace `kubeai`, a port-forward on 8000, the Service's cluster IP) do not hold | +| information | none: GPUs are discovered with `nvidia-smi` | `resourceProfiles` in the chart values (what one GPU unit requests, and on which nodes; `catalog suggest --backend kubeai` proposes them from the nodes' GPU labels), and per endpoint `runtime.resource_profile`, a `min_vram_gib` that picks one by size, or a default `kubeai_resource_profile` | +| settings | `backend compose` | `backend kubeai`; `kubeai_gateway cluster` for more than one workstation; `kubeai_namespace`, `kubeai_base_url`, `kubeai_gateway_upstream` when the defaults (namespace `kubeai`, a port-forward on 8000, the Service's cluster IP) do not hold | | preflight | none needed | `infer-stack doctor`: cluster → CRD → namespace → gateway | The catalog, `acquire … --env-file`, `release`, the TUI and the env file a @@ -114,7 +114,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | Open WebUI (`ui`), reverse proxy | yes | same, in the gateway's project on this host | | the gateway's changes approved with the acquire's, before the lease commits | yes | same | | `network migrate` / `network check` | yes | n/a: Service addresses are stable | -| where the gateway runs | this host | this host, so it is in every request's path (P5 moves it into the cluster) | +| where the gateway runs | this host | this host (default), or the cluster (`kubeai_gateway cluster`: a Deployment and a NodePort any node answers on; static routes, no UI) | ### Day-2 operations and the TUI diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 7524f12f..8353bb23 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -120,6 +120,61 @@ source lease.env # OPENAI_BASE_URL + OPENAI_API_KEY -> the gateway infer-stack release --env-file lease.env ``` +## The gateway inside the cluster + +By default the gateway is a Compose project on the host running infer-stack, +so that host is in every request's path. For more than one workstation, put +it in the cluster: + +```bash +infer-stack stack down # the host gateway (and any Models) +infer-stack config set kubeai_gateway cluster +infer-stack acquire --env-file lease.env --yes +# OPENAI_BASE_URL is now http://:30442/v1: any node answers +``` + +It is the same gateway (image, config, managed key, route registry) as a +Deployment and a NodePort Service in the KubeAI namespace, and it reaches +KubeAI by the Service's cluster DNS name, so no `kubectl port-forward` is +needed. `secrets rotate` updates its Secret and rolls it; `doctor` checks it; +`stack down` removes it. Settings: `kubeai_gateway_node_port` (30442) and +`kubeai_gateway_url` (an ingress URL clients should use instead). Static +routes only: dynamic routing and Open WebUI need the host placement. + +## Add a workstation + +The cluster's first node is the one `scripts/bootstrap_k3s.sh` set up. To add +a GPU workstation as a second node: + +1. On the first node, collect the join facts: + ```bash + sudo cat /var/lib/rancher/k3s/server/node-token # the token + k3s --version # join with the same version + ``` +2. On the new workstation (NVIDIA driver and container toolkit installed), + join with the server's version: + ```bash + INSTALL_K3S_VERSION='v1.36.4+k3s1' \ + scripts/join_agent.sh https://:6443 + ``` + Between the nodes, open 6443/tcp (to the first node), 8472/udp (flannel) + and 10250/tcp; for clients, the NodePort (30442/tcp). +3. Back on the first node: the NVIDIA device plugin is a DaemonSet, so it + starts on the new node by itself. Check the new node reports its GPUs and + labels, then add a sized profile for its GPU product: + ```bash + kubectl get node -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}' + infer-stack catalog suggest --backend kubeai # profiles per GPU product (stderr) + ``` + Merge the proposed `resourceProfiles` into your helm values and rerun + `scripts/install_kubeai.sh `. An endpoint that names that profile, + or declares a `min_vram_gib` only that GPU meets, lands on the new node. +4. With the gateway in the cluster (above), a card on either workstation uses + the env file as it is: the NodePort answers on every node. + +`dev/k3s_agent_container.sh` makes a second node out of a container on one +host, for development; `dev/handover/p5_two_hosts.sh` checks a real one. + ## Semantics + limitations - **Readiness is a real generation** through the gateway (same philosophy as @@ -164,8 +219,9 @@ infer-stack release --env-file lease.env runtime pane read the pods (and the gateway's containers), in the same shape as on compose. `stack compose …` and `stack restart` act on the gateway's Compose project; the engines are pods, restarted by the kubelet. -- The gateway runs on the host running infer-stack, so that host is in every - request's path. It takes the same settings as on compose: `ui` (Open WebUI, +- By default the gateway runs on the host running infer-stack, so that host is + in every request's path (`kubeai_gateway cluster` moves it; see above). It + takes the same settings as on compose: `ui` (Open WebUI, on by default), `reverse_proxy`, and `dynamic_routing`, under which each deployment is its own Model (`-`), so `--dedicated` twice gives two Models behind one alias. The gateway's config changes are shown diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 503f2975..4c85f48f 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,8 +1,8 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **P1–P4, P6 done** 2026-09-26 (P4's GPU run is -a handover) · P5 not started. Execution order: [../queue.md](../queue.md). +`dev/backend-unification` · **all phases done** 2026-09-26; P4 and P5 end in +a handover run on real hardware (`dev/handover/`). Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the @@ -209,6 +209,22 @@ reading vLLM's memory lines are `dev/handover/p4_gpu_labels.sh`. **Exit:** on two nodes, a card on node A leases a model that lands on node B and talks to it through the in-cluster gateway; `secrets rotate` works. + +**P5 done 2026-09-26** with the second node a container on the same host +(`dev/k3s_agent_container.sh`): `kubeai_gateway cluster` renders the gateway +as a Deployment and a NodePort Service (`backends/kubeai_gateway.py`), +reaching KubeAI by its Service's DNS name, so no port-forward is needed; +the key is a Secret applied from the state dir, never shown in a diff, and +its hash rolls the pods; `doctor` checks it. The recovery profile says which +placement renders, so changing the setting is adopted like any other once +the stack is quiescent. `dev/kubeai_e2e.sh` (with `E2E_SIZED=1 +E2E_REMOTE_NODE=1`) served a Model on the second node through that gateway, +rotated the key, and removed it with `stack down`. Deferred: dynamic +routing and Open WebUI in the cluster (they need Postgres and a UI there), +and an Ingress (no controller on the development cluster; `kubeai_gateway_url` +takes one). The run across two real machines is +`dev/handover/p5_two_hosts.sh`, and the runbook is "Add a workstation" in +`kubeai-backend.md`. **Size:** medium to large. The only phase that adds a second renderer for the gateway, and the only one that needs a second machine. Last. diff --git a/docs/queue.md b/docs/queue.md index 0426a973..93564072 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -100,7 +100,9 @@ profiles), a catalog with `min_vram_gib` and no `resource_profile` lands on the right profile; `dev/handover/p4_gpu_labels.sh` exists for one run on a real GPU node, and runs `measure` there. -### 7. [ ] P5: in-cluster gateway and a second node +### 7. [x] P5: in-cluster gateway and a second node + +Done 2026-09-26; the two-machine run is `dev/handover/p5_two_hosts.sh`. Render the gateway as a Deployment + Service behind an ingress; `secrets rotate` becomes a Secret update and a rollout; `doctor` checks the ingress. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index c9382288..72fb990b 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -398,6 +398,7 @@ def __init__( assume_yes: bool = True, gateway: Any = None, gateway_upstream: str | None = None, + gateway_factory: Callable[[str], Any] | None = None, ): self.state_dir = Path(state_dir) self.state_dir.mkdir(parents=True, exist_ok=True) @@ -420,6 +421,9 @@ def __init__( # request names (the endpoint aliases) as on the compose backend. # Without it, clients talk to KubeAI directly under its Model names. self.gateway = gateway + # Builds the gateway for a placement ('host' or 'cluster'), so a + # recovery profile can say where it runs (see use_profile). + self.gateway_factory = gateway_factory # How that gateway reaches KubeAI. Unset: the KubeAI Service's cluster # IP, which containers on a cluster node can reach; a gateway off the # cluster needs an ingress URL here. @@ -441,7 +445,12 @@ def models_file(self) -> Path: rendered_file = models_file def compose_project(self): - """The Compose project on this host: the gateway's, or ``None``.""" + """The Compose project on this host: the gateway's, or ``None`` + (no gateway, or the gateway runs in the cluster).""" + return self.gateway.compose_project() if self.gateway is not None else None + + def front_door(self): + """What holds the gateway's keys and route registry: the gateway.""" return self.gateway @property @@ -503,6 +512,10 @@ def gateway_accepts(self, key: str, *, wait: float = 0.0): def _upstream_url(self) -> str: """Where the gateway sends requests for this cluster's Models.""" + if not self.gateway_upstream and getattr(self.gateway, 'in_cluster', False): + # Inside the cluster: the Service's DNS name, stable across + # reinstalls, unlike its cluster IP. + return f'http://kubeai.{self.namespace}.svc.cluster.local/openai/v1' if not self.gateway_upstream: ip = self._kubectl(['get', 'service', 'kubeai', '-o', 'jsonpath={.spec.clusterIP}']).strip() @@ -820,6 +833,18 @@ def use_profile(self, profile: dict) -> None: # A profile from before the front door's settings were recorded holds # only True/False here; it then keeps this process's settings. if isinstance(front, dict) and self.gateway is not None: + wanted = 'cluster' if front.get('placement') == 'cluster' else 'host' + have = 'cluster' if getattr(self.gateway, 'in_cluster', False) else 'host' + if wanted != have: + # The snapshot decides where the gateway runs, as it decides + # everything else it renders. + if self.gateway_factory is None: + from ..leasing.profile import ProfileMismatch + + raise ProfileMismatch( + f'the active recovery snapshot has the gateway on the {wanted}, ' + f'this process on the {have}') + self.gateway = self.gateway_factory(wanted) self.gateway.use_profile(front) def apply(self) -> None: @@ -1072,6 +1097,10 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: f'kubectl create namespace {self.namespace} (or install the ' 'chart there)', ) + if getattr(self.gateway, 'in_cluster', False): + # Clients use the gateway in the cluster; no port-forward needed. + checks.append(self.gateway.doctor_check()) + return checks try: resp = self.http.get(f'{self.base_url}/models', timeout=10) code = getattr(resp, 'status_code', 0) diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py new file mode 100644 index 00000000..95d24b54 --- /dev/null +++ b/infer_stack/backends/kubeai_gateway.py @@ -0,0 +1,319 @@ +"""The LiteLLM gateway inside the cluster, for the kubeai backend. + +The default front door for a cluster is a one-service Compose project on the +host running infer-stack, so that host is in every request's path. This is +the other placement: the same gateway (the same image, config, key and route +registry, through :class:`~infer_stack.leasing.gateway.Gateway`) rendered as +Kubernetes objects in the KubeAI namespace and reached on a NodePort of any +node, or an ingress URL. + +It answers the calls the kubeai backend makes on its gateway (keys, routes, +preview / converge / apply, down, instances), so the backend does not know +which placement it has. Static routes only: dynamic routing and Open WebUI +need Postgres and a UI in the cluster, which stay with the host placement. + +Objects, all named ``infer-stack-gateway``: a ConfigMap (the LiteLLM config), +a Secret (the master key, applied from the state dir's ``.env`` and never +shown in a diff), a Deployment whose pod template carries the config's and +the key's hashes (so either change rolls the pods), and a NodePort Service. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Callable + +import yaml + +from ..config import PINNED_IMAGES +from ..leasing.backend import ConvergeScaffold +from ..leasing.gateway import ( + API_KEY_ENV, + LITELLM_CONTAINER_PORT, + Gateway, + render_front_door, +) + +NAME = 'infer-stack-gateway' +#: The pod label the Service selects and ``instances`` lists by. +APP_LABEL = 'app.kubernetes.io/name' +MANIFESTS_FILENAME = 'gateway.yaml' +SECRET_FILENAME = 'gateway-secret.yaml' # 0600, like the .env it copies +STATE_FILENAME = 'gateway-state.json' +DEFAULT_NODE_PORT = 30442 + + +class ClusterGateway(ConvergeScaffold): + """The LiteLLM gateway as a Deployment + NodePort Service in the cluster.""" + + _approve_title = 'infer-stack will update the in-cluster gateway' + _state_noun = 'gateway manifests' + #: Where this gateway runs, for the kubeai backend's routing decisions. + in_cluster = True + #: The settings a host gateway has and this one does not (see module doc). + litellm = True + ui = False + reverse_proxy = False + dynamic_routing = False + + def __init__( + self, + *, + state_dir: str | Path, + namespace: str, + run: Callable[[list[str]], str], + node_port: int = DEFAULT_NODE_PORT, + url: str | None = None, + images: dict[str, str] | None = None, + http: Any = None, + assume_yes: bool = True, + ): + self.state_dir = Path(state_dir) + self.state_dir.mkdir(parents=True, exist_ok=True) + self.namespace = namespace + self.run = run + self.node_port = int(node_port) + self.url = url + self.images = {**PINNED_IMAGES, **(images or {})} + self.assume_yes = assume_yes + self.gateway = Gateway(self.state_dir, ports={'litellm': self.node_port}, + litellm=True, ui=False, http=http, + base_url=self.base_url) + self.catalog = None + #: Render inputs from the kubeai backend (see ComposeBackend's). + self.upstream_rows: dict[str, dict[str, Any]] = {} + self.upstream_routes: list[dict[str, Any]] = [] + self._node_address: str | None = None + + # -- where it is --------------------------------------------------------- + + @property + def http(self) -> Any: + return self.gateway.http + + @http.setter + def http(self, value: Any) -> None: + self.gateway.http = value + + @property + def manifests_file(self) -> Path: + return self.state_dir / MANIFESTS_FILENAME + + rendered_file = manifests_file + + @property + def _state_file(self) -> Path: + return self.state_dir / STATE_FILENAME + + @property + def litellm_port(self) -> int: + return self.node_port + + def _kubectl(self, args: list[str]) -> str: + return self.run(['kubectl', '-n', self.namespace, *args]) + + def base_url(self) -> str: + """Where clients reach it: the ``url`` setting, else a node's NodePort. + + A NodePort answers on every node, so any node's address works; the + first node's InternalIP is used. + """ + if self.url: + return self.url.rstrip('/') + if self._node_address is None: + nodes = json.loads(self._kubectl(['get', 'nodes', '-o', 'json']) or '{}') + addresses = [a['address'] for n in nodes.get('items') or [] + for a in (n.get('status') or {}).get('addresses') or [] + if a.get('type') == 'InternalIP'] + self._node_address = addresses[0] if addresses else '127.0.0.1' + return f'http://{self._node_address}:{self.node_port}' + + def compose_project(self): + """No Compose project: this gateway is Kubernetes objects.""" + return None + + def front_door(self): + return self + + # -- keys and routes: the one Gateway ------------------------------------ + + def master_key(self) -> str: + return self.gateway.master_key() + + def rotate_master_key(self): + return self.gateway.rotate_master_key() + + def restore_env(self, values) -> None: + self.gateway.restore_env(values) + + def gateway_accepts(self, key: str, *, wait: float = 0.0): + return self.gateway.gateway_accepts(key, wait=wait) + + def access(self, endpoints: list[str]) -> dict[str, Any] | None: + return self.gateway.access(endpoints) + + def merge_route_registry(self, incoming): + return self.gateway.merge_route_registry(incoming) + + def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: + return {} # the kubeai backend supplies its own rows + + # -- render / apply ----------------------------------------------------- + + def _render_documents(self) -> tuple[dict[Path, str], dict[str, Any]]: + """``(planned files, merged registry)`` in memory: the one render.""" + registry = self.gateway.merged_route_registry(dict(self.upstream_rows)) + front = render_front_door( + [], {}, engine_services=[], vllm_v1_urls=[], ollama_native_urls=[], + images=self.images, state={}, litellm=True, + litellm_port=LITELLM_CONTAINER_PORT, litellm_master_key=None, + litellm_salt_key=False, ui=False, ui_port=0, reverse_proxy=False, + reverse_proxy_port=0, reverse_proxy_config=None, aux_dir=self.state_dir, + catalog=None, route_registry=registry, dynamic_routing=False, + ) + config = front.litellm_config or '' + key_hash = hashlib.sha256(self.master_key().encode()).hexdigest()[:12] + docs = gateway_manifests( + namespace=self.namespace, image=self.images['litellm'], config=config, + key_hash=key_hash, node_port=self.node_port) + text = '---\n'.join(yaml.safe_dump(d, sort_keys=False) for d in docs) + return {self.manifests_file: text}, registry + + def preview(self, desired=(), placement=None, *, approve: bool = False): + """Render without writing; with ``approve``, show the diff now.""" + planned, _ = self._render_documents() + self._preview_approval(planned, approve=approve) + return None, None + + def converge(self, desired=(), *, apply: bool = True, placement=None): + """Render the gateway's objects (and persist the registry after approval).""" + with self._converge_lock(): + planned, registry = self._render_documents() + self.last_planned_digest = self._planned_digest(planned) + self._approve_changes(planned) + self.gateway._save_route_registry(registry) + for path, text in planned.items(): + self._atomic_write(path, text) + if apply: + self.apply() + + def apply(self) -> None: + """Apply the key's Secret, then the objects, and wait for the rollout.""" + import os + + from .._log import logger + + if not self.manifests_file.exists(): + return + secret = yaml.safe_dump(secret_manifest(self.namespace, self.master_key())) + path = self.state_dir / SECRET_FILENAME + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w') as handle: + handle.write(secret) + logger.info('kubectl apply (the in-cluster gateway)') + self._kubectl(['apply', '-f', str(path)]) + self._kubectl(['apply', '-f', str(self.manifests_file)]) + self._kubectl(['rollout', 'status', f'deployment/{NAME}', '--timeout=300s']) + + def down(self) -> None: + self._kubectl(['delete', 'deployment,service,configmap,secret', NAME, + '--ignore-not-found']) + + def instances(self): + from ..leasing.instances import KUBERNETES, from_residency + from ..leasing.residency import ResidencyUnknown, residency_from_pods + + try: + raw = self._kubectl(['get', 'pods', '-l', f'{APP_LABEL}={NAME}', '-o', 'json']) + except Exception as ex: # noqa: BLE001 - any failure is "unknown" + raise ResidencyUnknown(f'kubectl get pods failed: {ex}') from ex + return from_residency(residency_from_pods(raw), runtime=KUBERNETES, + namespace=self.namespace) + + def doctor_check(self) -> tuple[str, bool, str]: + """``(check, ok, detail)``: does the gateway answer with the managed key?""" + base = self.base_url() + accepted = self.gateway_accepts(self.master_key()) + if accepted: + return (f'in-cluster gateway at {base}/v1', True, '') + return (f'in-cluster gateway at {base}/v1', False, + 'not answering; `infer-stack apply` renders and starts it, and ' + '`kubectl -n get pods -l ' + f'{APP_LABEL}={NAME}` shows why it is not up') + + # -- the recovery profile ------------------------------------------------ + + def render_profile(self) -> dict[str, Any]: + return {'placement': 'cluster', 'node_port': self.node_port, 'url': self.url, + 'image': self.images['litellm']} + + def use_profile(self, profile: dict[str, Any]) -> None: + if profile.get('placement') != 'cluster': + from ..leasing.profile import ProfileMismatch + + raise ProfileMismatch( + 'the active recovery snapshot has the gateway on this host; ' + '`infer-stack stack down` it before moving it into the cluster') + self.node_port = int(profile.get('node_port') or self.node_port) + self.url = profile.get('url') or self.url + self.images['litellm'] = profile.get('image') or self.images['litellm'] + + +def secret_manifest(namespace: str, key: str) -> dict[str, Any]: + """The master key as a Secret (applied, never rendered into a diff).""" + return {'apiVersion': 'v1', 'kind': 'Secret', + 'metadata': {'name': NAME, 'namespace': namespace}, + 'type': 'Opaque', 'stringData': {API_KEY_ENV: key}} + + +def gateway_manifests(*, namespace: str, image: str, config: str, key_hash: str, + node_port: int) -> list[dict[str, Any]]: + """ConfigMap, Deployment and NodePort Service for the in-cluster gateway. + + >>> docs = gateway_manifests(namespace='kubeai', image='litellm:x', + ... config='model_list: []\\n', key_hash='abc', node_port=30442) + >>> [d['kind'] for d in docs] + ['ConfigMap', 'Deployment', 'Service'] + >>> docs[2]['spec']['ports'][0]['nodePort'] + 30442 + """ + config_hash = hashlib.sha256(config.encode()).hexdigest()[:12] + labels = {APP_LABEL: NAME} + meta = {'name': NAME, 'namespace': namespace, 'labels': labels} + return [ + {'apiVersion': 'v1', 'kind': 'ConfigMap', 'metadata': meta, + 'data': {'config.yaml': config}}, + {'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': meta, + 'spec': { + 'replicas': 1, + 'selector': {'matchLabels': labels}, + 'template': { + 'metadata': {'labels': labels, 'annotations': { + # LiteLLM reads both once at startup: a change must roll. + 'infer-stack/config-hash': config_hash, + 'infer-stack/key-hash': key_hash}}, + 'spec': {'containers': [{ + 'name': 'litellm', + 'image': image, + 'args': ['--config', '/etc/litellm/config.yaml', + '--port', str(LITELLM_CONTAINER_PORT)], + 'envFrom': [{'secretRef': {'name': NAME}}], + 'ports': [{'containerPort': LITELLM_CONTAINER_PORT}], + 'readinessProbe': { + 'httpGet': {'path': '/health/liveliness', + 'port': LITELLM_CONTAINER_PORT}, + 'periodSeconds': 5}, + 'volumeMounts': [{'name': 'config', 'mountPath': '/etc/litellm', + 'readOnly': True}], + }], + 'volumes': [{'name': 'config', 'configMap': {'name': NAME}}]}, + }, + }}, + {'apiVersion': 'v1', 'kind': 'Service', 'metadata': meta, + 'spec': {'type': 'NodePort', 'selector': labels, + 'ports': [{'port': LITELLM_CONTAINER_PORT, + 'targetPort': LITELLM_CONTAINER_PORT, + 'nodePort': node_port}]}}, + ] diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 922a6ea0..4aee2d4c 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -277,16 +277,29 @@ def _make_backend(config, *, interactive: bool = False): if name == 'kubeai': from ..backends.kubeai import KubeaiBackend - gateway = None - if _resolve_litellm(config): + def make_gateway(placement: str): + """The LiteLLM front door for a placement: 'host' or 'cluster'.""" + if placement == 'cluster': + from ..backends.kubeai import _default_kubectl_run + from ..backends.kubeai_gateway import DEFAULT_NODE_PORT, ClusterGateway + + return ClusterGateway( + state_dir=data_root() / 'leasing' / 'kubeai-cluster-gateway', + namespace=get_setting('kubeai_namespace') or 'kubeai', + run=_default_kubectl_run, + node_port=int(get_setting('kubeai_gateway_node_port') + or DEFAULT_NODE_PORT), + url=get_setting('kubeai_gateway_url') or None, + assume_yes=_resolve_assume_yes(config, interactive=interactive), + ) # The same LiteLLM front door as the compose backend, fronting the # cluster: one base_url, the managed key, and endpoint aliases as # request names. Its own state dir and compose project, so it can - # never touch a compose stack's containers on the same host. - # The front door takes the same settings as on compose (UI, - # reverse proxy, dynamic routing); only the engines are elsewhere. + # never touch a compose stack's containers on the same host. It + # takes the same settings as on compose (UI, reverse proxy, + # dynamic routing); only the engines are elsewhere. rp_enabled, rp_port, rp_config = _resolve_reverse_proxy(config) - gateway = ComposeBackend( + return ComposeBackend( state_dir=data_root() / 'leasing' / 'kubeai-gateway', inventory={'gpu_count': 0, 'gpus': []}, project='infer-stack-gateway', @@ -298,6 +311,9 @@ def _make_backend(config, *, interactive: bool = False): dynamic_routing=_resolve_dynamic_routing(config), assume_yes=_resolve_assume_yes(config, interactive=interactive), ) + + gateway = (make_gateway(get_setting('kubeai_gateway') or 'host') + if _resolve_litellm(config) else None) backend = KubeaiBackend( state_dir=data_root() / 'leasing' / 'kubeai', namespace=get_setting('kubeai_namespace') or 'kubeai', @@ -307,6 +323,7 @@ def _make_backend(config, *, interactive: bool = False): assume_yes=_resolve_assume_yes(config, interactive=interactive), gateway=gateway, gateway_upstream=get_setting('kubeai_gateway_upstream') or None, + gateway_factory=make_gateway if gateway is not None else None, ) try: backend.catalog = _load_catalog(config) # frozen into the profile @@ -2462,11 +2479,12 @@ def main(cls, argv=True, **kwargs): def _require_compose_backend(controller): - """The Compose project holding the gateway's route registry. + """What holds the gateway's route registry (its front door). - The stack itself on the compose backend, the gateway's project on kubeai; - a SystemExit for a backend with no gateway (null, or ``litellm false``).""" - project = getattr(controller.backend, 'compose_project', lambda: None)() + The stack itself on the compose backend; on kubeai the gateway, on this + host or in the cluster. A SystemExit for a backend with no gateway (null, + or ``litellm false``).""" + project = getattr(controller.backend, 'front_door', lambda: None)() if project is None or not getattr(project, 'litellm', False): raise SystemExit( 'the `routes` commands need a LiteLLM gateway (the compose or kubeai ' diff --git a/infer_stack/cli/commands_meta.py b/infer_stack/cli/commands_meta.py index a0986fc3..17be8587 100644 --- a/infer_stack/cli/commands_meta.py +++ b/infer_stack/cli/commands_meta.py @@ -366,6 +366,15 @@ class _Setting: 'URL the LiteLLM gateway uses to reach KubeAI (default: the kubeai ' "Service's cluster IP, reachable from a cluster node; set an ingress " 'URL when the gateway runs off the cluster).', + 'kubeai_gateway': + 'Where the LiteLLM gateway runs: `host` (default; a Compose project on ' + 'this host) or `cluster` (a Deployment + NodePort Service in the KubeAI ' + 'namespace, reachable from every node).', + 'kubeai_gateway_node_port': + 'NodePort of the in-cluster gateway (default: 30442).', + 'kubeai_gateway_url': + 'Where clients reach the in-cluster gateway, e.g. an ingress URL ' + "(default: the first node's address on the NodePort).", } diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index c44793cd..80ce7110 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1101,6 +1101,10 @@ def compose_project(self): """The Compose project on this host: this one.""" return self + def front_door(self): + """What holds the gateway's keys and route registry: this project.""" + return self + def compose_argv(self) -> list[str]: """``docker compose [--env-file ...] -p -f ``: the one base. diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index daaa48f6..53689755 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -943,8 +943,13 @@ def __init__( http: Any = None, sleep: Callable[[float], None] = time.sleep, clock: Callable[[], float] = time.monotonic, + base_url: str | Callable[[], str] | None = None, ): self.state_dir = Path(state_dir) + # Where clients reach the gateway (no ``/v1``). None: this host, on the + # published port. The in-cluster gateway passes a callable, so the + # node address is looked up only when something asks. + self.base_url: str | Callable[[], str] | None = base_url self.ports = ports self.litellm = litellm self.ui = ui @@ -1165,7 +1170,14 @@ def merge_route_registry( return merged def _gateway_base(self) -> str: - return f'http://127.0.0.1:{self.litellm_port}' + where = self.base_url + if where is None: + base = f'http://127.0.0.1:{self.litellm_port}' + elif isinstance(where, str): + base = where + else: + base = where() + return base.rstrip('/') def _auth_headers(self) -> dict[str, str]: return {'Authorization': f'Bearer {self.master_key()}'} @@ -1379,7 +1391,7 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: return {'ui_url': f'http://127.0.0.1:{self.ui_port}'} return None info: dict[str, Any] = { - 'base_url': f'http://127.0.0.1:{self.litellm_port}/v1', + 'base_url': f'{self._gateway_base()}/v1', 'api_key_env': API_KEY_ENV, 'api_key': self.master_key(), 'request_names': {ep: ep for ep in endpoints}, diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index dbc16253..44bccc2b 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -1010,3 +1010,104 @@ def test_a_recorded_measurement_picks_the_profile(tmp_path): be.converge([dep], apply=False) (doc,) = yaml.safe_load_all(be.models_file.read_text()) assert doc['spec']['resourceProfile'] == 'big:1' + + +# -- the gateway inside the cluster (P5) ---------------------------------------- + + +class ClusterKubectl(FakeKubectl): + """FakeKubectl that keeps the gateway's objects apart from the Models.""" + + def __init__(self): + super().__init__() + self.gateway_objects: dict[str, dict] = {} + self.rollouts = 0 + + def __call__(self, args): + if len(args) > 4 and args[3] == 'get' and args[4] == 'nodes': + return json.dumps({'items': [{'status': {'addresses': [ + {'type': 'InternalIP', 'address': '10.0.0.7'}]}}]}) + if len(args) > 3 and args[3] == 'rollout': + self.rollouts += 1 + return '' + if len(args) > 3 and args[3] == 'apply': + path = Path(args[args.index('-f') + 1]) + docs = [d for d in yaml.safe_load_all(path.read_text()) if d] + if all(d.get('kind') != 'Model' for d in docs): + for d in docs: + self.gateway_objects[d['kind']] = d + return '' + return super().__call__(args) + + +def make_cluster_gateway_backend(tmp_path): + from infer_stack.backends.kubeai_gateway import ClusterGateway + + kubectl = ClusterKubectl() + gateway = ClusterGateway(state_dir=tmp_path / 'gw', namespace='kubeai', run=kubectl, + images={'litellm': 'litellm:test'}) + be = KubeaiBackend(state_dir=tmp_path / 'kubeai', run=kubectl, http=FakeHttp(kubectl), + gateway=gateway) + return be, kubectl + + +def test_the_in_cluster_gateway_routes_by_cluster_dns(tmp_path): + be, kubectl = make_cluster_gateway_backend(tmp_path) + dep = vllm('grp-a', served='tiny') + dep.served = {'tiny': {'served_model_name': 'tiny', 'protocol': 'chat'}} + be.converge([dep]) + config = yaml.safe_load(kubectl.gateway_objects['ConfigMap']['data']['config.yaml']) + (route,) = config['model_list'] + assert route['model_name'] == 'tiny' + assert route['litellm_params']['api_base'] == \ + 'http://kubeai.kubeai.svc.cluster.local/openai/v1' + assert kubectl.gateway_objects['Service']['spec']['type'] == 'NodePort' + assert kubectl.rollouts == 1 + assert be.access(['tiny'])['base_url'] == 'http://10.0.0.7:30442/v1' + assert be.compose_project() is None and be.front_door() is be.gateway + + +def test_the_key_is_a_secret_never_a_diff_and_rotating_it_rolls_the_pods(tmp_path): + import stat + + be, kubectl = make_cluster_gateway_backend(tmp_path) + be.converge([]) + key = be.master_key() + assert key not in be.gateway.manifests_file.read_text() + secret = be.gateway.state_dir / 'gateway-secret.yaml' + assert stat.S_IMODE(secret.stat().st_mode) == 0o600 + assert kubectl.gateway_objects['Secret']['stringData']['LITELLM_MASTER_KEY'] == key + + def key_hash(): + tmpl = kubectl.gateway_objects['Deployment']['spec']['template'] + return tmpl['metadata']['annotations']['infer-stack/key-hash'] + + before = key_hash() + be.rotate_master_key() + be.converge([]) + assert key_hash() != before + + +def test_doctor_checks_the_in_cluster_gateway_not_a_port_forward(tmp_path): + be, _ = make_cluster_gateway_backend(tmp_path) + be.gateway.gateway_accepts = lambda key, wait=0.0: True + names = [name for name, ok, _ in be.doctor()] + assert names[-1] == 'in-cluster gateway at http://10.0.0.7:30442/v1' + assert not any('port-forward' in name for name in names) + + +def test_the_recovery_profile_says_where_the_gateway_runs(tmp_path): + """Changing `kubeai_gateway` is adopted like any setting: the snapshot + decides which gateway renders, host or cluster, and switches it.""" + from infer_stack.backends.kubeai_gateway import ClusterGateway + + cluster_be, _ = make_cluster_gateway_backend(tmp_path / 'c') + host_be, _ = make_front_door_backend(tmp_path / 'h') + host_gateway = host_be.gateway + host_be.gateway_factory = lambda placement: ( + cluster_be.gateway if placement == 'cluster' else host_gateway) + host_be.use_profile(cluster_be.render_profile()) + assert isinstance(host_be.gateway, ClusterGateway) + host_be.use_profile({**cluster_be.render_profile(), + 'gateway': host_gateway.render_profile()}) + assert host_be.gateway is host_gateway From daf24765ca3efe31f79ce0aa384139cb993695e4 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:30:56 -0400 Subject: [PATCH 036/141] Queue: record the P5 commit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 93564072..e6a65da1 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -102,7 +102,7 @@ real GPU node, and runs `measure` there. ### 7. [x] P5: in-cluster gateway and a second node -Done 2026-09-26; the two-machine run is `dev/handover/p5_two_hosts.sh`. +Done 2026-09-26, `49e70dd`; the two-machine run is `dev/handover/p5_two_hosts.sh`. Render the gateway as a Deployment + Service behind an ingress; `secrets rotate` becomes a Secret update and a rollout; `doctor` checks the ingress. From 76c84aa6af801a1e2a9a563512a87783fd8145fc Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:34:02 -0400 Subject: [PATCH 037/141] UX: "free a GPU first" only when a free GPU would have helped A refused acquire always ended with advice to free a GPU, also for a render refusal (a served-name collision, no resource profile) and on kubeai, where no GPU is ours to free. PlacementError now carries `capacity`: set by admission only when GPU placement refused a deployment on a backend that allocates GPUs; a host too small for the request is not capacity either. The CLI gives the hint (now also naming --queue) only then. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 5 +++-- infer_stack/cli/commands_leasing.py | 12 +++++++----- infer_stack/leasing/backend.py | 7 ++++++- infer_stack/leasing/controller.py | 20 ++++++++++++++++---- tests/test_leasing_controller_queue.py | 3 ++- tests/test_leasing_kubeai.py | 10 ++++++++++ 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index e6a65da1..af34a527 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -169,9 +169,10 @@ Seed findings, already known: - [ ] Open WebUI writes its data directory as root: removing a data root (the e2e's cleanup, or an operator's `rm -rf`) fails with permission denied. Found by `dev/kubeai_e2e.sh` 2026-09-26. -- [ ] A refused acquire always ends with "free a GPU first — …", also when +- [x] A refused acquire always ends with "free a GPU first — …", also when the reason is a render refusal (a served-name collision) or the - backend is kubeai, where no GPU is ours to free. Found 2026-09-26. + backend is kubeai, where no GPU is ours to free. Found 2026-09-26; + fixed: `PlacementError.capacity` says whether room would have helped. ### 10. [ ] Handover diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 4aee2d4c..8dce849d 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -739,11 +739,13 @@ def _do_acquire(config, *, owner: str, ttl_seconds: float | None) -> int: lines += [f' {r}' for r in ex.reasons] or [ f' {", ".join(ex.deployment_ids)}' ] - lines.append( - ' free a GPU first — `infer-stack leases` to see what holds them, ' - 'then `infer-stack release`/`evict`. (Every GPU, including any ' - 'display-attached one, is used unless you set --skip-display-gpus.)' - ) + if ex.capacity: + lines.append( + ' free a GPU first — `infer-stack leases` to see what holds them, ' + 'then `infer-stack release`/`evict`, or wait for one with --queue. ' + '(Every GPU, including any display-attached one, is used unless ' + 'you set --skip-display-gpus.)' + ) raise SystemExit('\n'.join(lines)) return _emit_acquire(config, controller, outcome) diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index d50454d6..d7023a9f 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -74,11 +74,16 @@ class PlacementError(Exception): controller rolls back the just-created lease before raising, so a request that cannot be satisfied does not linger as a phantom ``live`` deployment with no container behind it. ``reasons`` carries the planner's per-deployment messages. + + ``capacity`` says whether free GPUs would have let it in, as opposed to a + request no amount of room admits (a render refusal, an unreadable + runtime, a host too small): only then is "free a GPU" the advice. """ - def __init__(self, deployment_ids, reasons): + def __init__(self, deployment_ids, reasons, *, capacity: bool = True): self.deployment_ids = list(deployment_ids) self.reasons = list(reasons) + self.capacity = bool(capacity) super().__init__( '; '.join(self.reasons) or f'could not place: {", ".join(self.deployment_ids)}' diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 8d5c3a36..e7666b05 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -199,6 +199,8 @@ def __init__( # that differs from an earlier approved digest. self._explicit_apply = False self._admission_digest: str | None = None + #: Whether the last refused admission was for lack of GPUs. + self._admission_capacity = False from .profile import ProfileMismatch try: @@ -844,6 +846,7 @@ def _admit(self, overlay, residency): commit for the candidate's new and revived deployments. Nothing is written here. """ + self._admission_capacity = False adopted: dict[str, list[int]] = {} need: list[str] = [] for gid, deployment in overlay.deployments.items(): @@ -879,7 +882,10 @@ def _admit(self, overlay, residency): self._prepare_network() desired, inputs = self._admission_view(residency, overlay=overlay) plan, rendered = self._admitting.preview(desired, inputs) + from .backend import allocates_gpus + reasons = [] + capacity = False # did any refusal come from GPU placement? unresolved = set(self._unresolved_allocations()) # EVERY deployment the candidate claims must be placed and renderable, # including an existing one it only coalesces onto (whose served @@ -893,9 +899,12 @@ def _admit(self, overlay, residency): continue if gid in plan.degraded: reasons.append(f'{gid}: its GPUs are no longer available') + capacity = True elif gid not in plan.assignments: why = [e for e in plan.errors if e.startswith(gid)] reasons.extend(why or [f'{gid}: could not be placed']) + # Where the cluster places, a plan without it is a refusal. + capacity = capacity or allocates_gpus(self.backend) elif gid in set(rendered.unrenderable): why = [e for e in rendered.errors if gid in e] or [ f'{gid}: could not be rendered ({e})' for e in rendered.errors] @@ -907,6 +916,7 @@ def _admit(self, overlay, residency): if holders: reasons.append('GPUs held by admitted demand: ' + '; '.join(holders)) self._admission_digest = None + self._admission_capacity = capacity if not reasons: # Approval happens now, before anything is committed; the render # after the commit produces the same files and does not ask again. @@ -1611,15 +1621,16 @@ def _acquire_by_admission( if gid in overlay.created or gid in overlay.revived ) if not (wait_for_placement and apply): - raise PlacementError(blocked, reasons) + raise PlacementError(blocked, reasons, capacity=self._admission_capacity) if not checked_feasible: checked_feasible = True infeasible = self._infeasible_alone( list(overlay.deployments.values()), set(blocked)) if infeasible: - raise PlacementError(sorted(infeasible), sorted(infeasible.values())) + raise PlacementError(sorted(infeasible), sorted(infeasible.values()), + capacity=False) if self.clock() + placement_interval > deadline: - raise PlacementError(blocked, reasons) + raise PlacementError(blocked, reasons, capacity=self._admission_capacity) self.sleep(placement_interval) def _finish_acquire(self, result, rec, *, apply, wait, timeout, interval) -> AcquireOutcome: @@ -1850,7 +1861,8 @@ def renew(self, lease_id: str, *, ttl_seconds: float | None) -> RenewOutcome: self.ledger.renew(lease_id, ttl_seconds=ttl_seconds), []) allocations, reasons = self._admit(overlay, residency) if reasons: - raise PlacementError(list(overlay.revived), reasons) + raise PlacementError(list(overlay.revived), reasons, + capacity=self._admission_capacity) self._mark_pending(apply=True) if self._admission_digest: self.ledger.mark_publication_pending( diff --git a/tests/test_leasing_controller_queue.py b/tests/test_leasing_controller_queue.py index 2afa912e..2f05038f 100644 --- a/tests/test_leasing_controller_queue.py +++ b/tests/test_leasing_controller_queue.py @@ -115,8 +115,9 @@ def sleep(clock, ledger): ctl, backend, _, _ = _make(budget=1, sleep=sleep) ctl.acquire('alice', [vreq('A')]) # fills the single slot - with pytest.raises(PlacementError): + with pytest.raises(PlacementError) as ei: ctl.acquire('bob', [vreq('B')]) # no wait -> fail fast + assert ei.value.capacity # a free GPU would have let it in def test_acquire_queues_until_a_gpu_frees(): diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 44bccc2b..acbe6362 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -1111,3 +1111,13 @@ def test_the_recovery_profile_says_where_the_gateway_runs(tmp_path): host_be.use_profile({**cluster_be.render_profile(), 'gateway': host_gateway.render_profile()}) assert host_be.gateway is host_gateway + + +def test_a_render_refusal_is_not_a_capacity_problem(tmp_path): + """"Free a GPU first" is advice only for a lack of GPUs.""" + from infer_stack.leasing.backend import PlacementError + + ctl, be, kubectl = make_controller(tmp_path) + with pytest.raises(PlacementError) as ei: + ctl.acquire('alice', [_req('qwen', profile=None)], wait=False) + assert ei.value.capacity is False From 7fe76b342091eccc60d8c268374993a00cc8fb33 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:39:06 -0400 Subject: [PATCH 038/141] Plan doc: a generic model name in the naming example Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/tmp/plan-backend-unification-2026-09-24.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tmp/plan-backend-unification-2026-09-24.md b/dev/tmp/plan-backend-unification-2026-09-24.md index 528d61a1..0b1097c4 100644 --- a/dev/tmp/plan-backend-unification-2026-09-24.md +++ b/dev/tmp/plan-backend-unification-2026-09-24.md @@ -25,7 +25,7 @@ launches (`runtime.command` / `mounts`) stay Compose-only. | | Compose | KubeAI | |---|---|---| | front door | LiteLLM gateway, one `base_url` | KubeAI's own gateway | -| request name | the **endpoint alias** | a DNS slug of the served name (`Qwen/Qwen3.8-27B` → `qwen-qwen3-8-27b`) | +| request name | the **endpoint alias** | a DNS slug of the served name (`Org/Model-7B` → `org-model-7b`) | | auth | managed master key | none (`EMPTY`) | Cards and the pipeline send the endpoint alias as `model=`. Only one magnet From e1c68f30976ad64b371caff1791be9acab037dbf Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:39:06 -0400 Subject: [PATCH 039/141] Docs: the README and two docs describe the current CLI The README's Compose half, docs/persistent-caches-and-warm-restarts.md and docs/stack-graph-profiles.md still documented the pre-leasing CLI (setup, up, deploy, switch, describe-profile, smoke-test, wait-ready, top-level restart/stop/pull/down) and named stack profiles. Rewritten against the current commands (acquire, test, wait, stack ...), each checked with --help; catalog examples pass `catalog validate`. Sections for removed features (chat_compat, the LDAP and TLS proxy fields) are removed, and the Open WebUI auth and reverse-proxy sections state what the leasing renderer actually does. stack-graph-profiles.md is now a short note on what replaced profiles. Stale cache paths and a claimed `ipc: host` in the caches doc are corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- README.md | 726 +++++++------------- docs/persistent-caches-and-warm-restarts.md | 82 +-- docs/stack-graph-profiles.md | 296 +------- 3 files changed, 324 insertions(+), 780 deletions(-) diff --git a/README.md b/README.md index b7fdfd56..5cc10e60 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,10 @@ [![Python versions](https://img.shields.io/pypi/pyversions/infer-stack.svg)](https://pypi.org/project/infer-stack/) [![License](https://img.shields.io/pypi/l/infer-stack.svg)](https://github.com/AIQ-Kitware/infer_stack/blob/main/LICENSE) -> **Heads up — the leasing model is now the primary workflow.** Declare models -> in a catalog (`infer-stack catalog …`) and `acquire`/`run` endpoints -> on demand; see `docs/source/manual/` (the Ollama + Open WebUI tutorial and the -> leasing demo) and `infer-stack help tree`. The -> **named stack profiles** documented below (`setup`/`render`/`up`/`switch`/…) -> are the pre-leasing model and now live under **`infer-stack legacy `** -> (e.g. `infer-stack legacy render`). This README still describes that legacy -> flow; a leasing-oriented rewrite is pending. +Declare models in a catalog (`infer-stack catalog …`) and `acquire` or `run` +endpoints on demand. `infer-stack help tree` prints the whole command surface; +[docs/source/manual/](docs/source/manual/) has the Ollama + Open WebUI +tutorial and the leasing demo. ## Primary leasing workflow @@ -23,7 +19,8 @@ infer-stack catalog suggest --apply infer-stack acquire ``` -`config.yaml` and `catalog.yaml` are the user configuration. Leasing keeps an +`settings.yaml` (`infer-stack config …`) and `catalog.yaml` are the user +configuration. Leasing keeps an internal frozen recovery snapshot so a crash cannot re-render committed state with different settings, but ordinary `acquire` advances that snapshot automatically. Compatible catalog additions can be acquired while other models @@ -62,49 +59,46 @@ Operational and security constraints that are accepted during the current planning-stage release are tracked in [docs/planning/known-limitations.md](docs/planning/known-limitations.md). -`infer_stack` manages **named stack profiles** for local and Kubernetes-backed inference. +`infer-stack` serves the endpoints declared in a catalog. `acquire` takes a +lease on an endpoint; the controller places its engine on free GPUs and +reconciles the backend to run it: -A stack profile is a small graph made from: +* **engines**: vLLM (one container per deployment) and Ollama (one daemon per + `runtime_hosts` entry, serving many tags); +* **LiteLLM gateway**: one OpenAI base URL, `http://127.0.0.1:14042/v1`, in + front of every endpoint alias. On by default; `config set litellm false` + drops it; +* **Open WebUI**: on by default at `http://127.0.0.1:13000`; + `config set ui false` or `acquire --no-ui` drops it; +* **reverse proxy**: an optional single-port nginx in front of both. -* **providers** — inference runtimes such as vLLM and Ollama -* **gateways** — optional API routers such as LiteLLM -* **frontends** — optional UIs such as Open WebUI -* **routes** — optional public model aliases exposed through a gateway - -This repo can render those profiles through two backends: - -* **Compose** for local single-host serving. Compose supports vLLM, Ollama, optional LiteLLM, and optional Open WebUI. -* **KubeAI** for Kubernetes-backed vLLM serving. KubeAI support is vLLM-only for now. - -The direct Ollama path can run without LiteLLM and without predeclaring models. vLLM profiles still use explicit runtimes, placement, and runtime settings. +Two backends run this: **Compose** (single host; vLLM and Ollama) and +**KubeAI** (a Kubernetes cluster; vLLM only). ## Main commands ```bash -infer-stack setup --backend compose --profile ollama-direct -# or: infer-stack setup --backend compose --profile qwen2-5-7b-instruct-turbo-default -infer-stack list-profiles -infer-stack describe-profile -infer-stack validate -infer-stack render -infer-stack up -d -infer-stack deploy -infer-stack switch --apply # re-render and converge; no separate up needed -infer-stack status -infer-stack smoke-test -infer-stack version # print the installed version -infer-stack config paths # show where config / artifacts / caches live +infer-stack config init # data dir + default backend -> settings.yaml +infer-stack catalog suggest --apply # seed catalog.yaml from this host's GPUs +infer-stack catalog show # what can be acquired +infer-stack acquire # lease, render, bring up, wait for a real generation +infer-stack test # one generation through the gateway +infer-stack leases # desired vs running, per deployment +infer-stack status # paths, backend and a lease summary +infer-stack release --all # drop every lease +infer-stack paths # where settings, catalog, ledger and caches live +infer-stack version +infer-stack help tree # the whole command surface ``` The CLI is built on [`kwconf`](https://github.com/Erotemic/kwconf), -so every subcommand is also importable as a Python class — useful for -notebooks, tests, and other scripts: +so every subcommand is also importable as a Python class: ```python -from infer_stack.cli import RenderCLI, SmokeTestCLI +from infer_stack.cli import AcquireCLI, TestCLI -RenderCLI.main(argv=False, profile="qwen2-5-7b-instruct-turbo-default", yes=True) -SmokeTestCLI.main(argv=False, model="qwen/qwen2.5-7b-instruct-turbo") +AcquireCLI.main(argv=False, names=['smol135-1'], yes=True) +TestCLI.main(argv=False, name='smol135-1') ``` `manage.py` and `infer-stack` are aliases for the same entry point; @@ -112,247 +106,216 @@ shell examples below use `infer-stack`. ## Operating the rendered Compose stack -Once the stack is up, common docker compose operations are available as -`infer-stack` subcommands so you don't have to `cd` into the rendered -output directory or repeat the `-f docker-compose.yml --env-file .env` -flags. They all resolve the rendered location via the same -`output.generated_dir` chain as the rest of the CLI. +`ps` and `logs` read the backend directly; `stack` wraps `docker compose` on +the rendered project, so you never `cd` into it or repeat `-f`/`--env-file`. ```bash -infer-stack ps # docker compose ps -infer-stack ps -a # include stopped -infer-stack logs -f open-webui # follow one service -infer-stack logs --tail=200 litellm vllm-* # tailored backlog -infer-stack logs -f --raw litellm # full LiteLLM tracebacks -infer-stack restart open-webui # restart specific services -infer-stack stop # stop everything (no remove) -infer-stack start # start back up -infer-stack pull # refresh images -``` - -Interactive ``infer-stack logs -f`` compacts only explicitly registered, known-noisy -LiteLLM traceback shapes; unknown tracebacks pass through unchanged. Redirected or -piped output stays raw, and ``--raw`` disables compaction in an interactive follow. -The compacted CLI path preserves Compose ANSI service colors when attached to a TTY; -``--no-color`` still disables them. The TUI uses the same conservative compactor when -LiteLLM logs are visible. - -For Ollama model management inside the rendered Ollama service, prefer the -CLI wrappers: +infer-stack ps # engines, gateway, UI +infer-stack ps -a # include exited instances +infer-stack logs -f # follow whatever serves an endpoint +infer-stack logs --tail 200 litellm # the gateway's backlog +infer-stack logs -f --raw litellm # full LiteLLM tracebacks +infer-stack stack restart open-webui # docker compose restart +infer-stack stack stop # stop everything (no remove) +infer-stack stack start # start it back up +infer-stack stack pull # refresh images +infer-stack stack compose -- ps --format json # any other docker compose command +``` + +`logs` accepts a service or pod name, a container id prefix, a deployment id +or an endpoint alias. Interactive `infer-stack logs -f` compacts only +explicitly registered, known-noisy LiteLLM traceback shapes; unknown +tracebacks pass through unchanged. Redirected or piped output stays raw, and +`--raw` disables compaction in an interactive follow. `--no_color` drops the +name-prefix colors. The TUI uses the same compactor. + +Ollama tags are pulled into the daemon on the first `acquire` of an endpoint +that serves them. `stack compose` reaches the daemon for anything else, e.g. +`infer-stack stack compose -- exec ollama-local-ollama ollama list`. + +On the KubeAI backend `ps` and `logs` read pods, and the `stack` Compose verbs +act on the gateway's Compose project on this host. + +## Inspect an endpoint before running it ```bash -infer-stack ollama-pull smollm2:135m -infer-stack ollama-list -infer-stack ollama-ps +infer-stack catalog show +infer-stack acquire --no-apply # declare + write the compose project; start nothing +infer-stack paths leasing # where docker-compose.yml landed +infer-stack apply # start it (or `release --all` to discard) ``` -For other interactive one-shot commands inside a container, use -`infer-stack logs`, `infer-stack ps`, `infer-stack restart`, or fall back to raw -Compose only when no wrapper exists. +## Catalog model -On the KubeAI backend these wrappers raise ``NotImplementedError`` — -use the equivalent ``kubectl`` commands in the meantime. +`catalog.yaml` is the one user-edited description of what can run. Its +sections: -## Inspect a profile before running it +* `models`: weight sources (`hf://org/name`, with optional `revision`, + `quantization`, `dtype`); +* `endpoints`: served API names. Each picks an `engine` (`vllm` or `ollama`), + a `model` (a `models` key, or an Ollama tag), and optionally `runtime` + (vLLM settings), `protocol`, `placement`, `sharing` and `reclaim`; +* `runtime_hosts`: Ollama daemons, each with its GPUs and daemon settings; +* `bundles`: named lists of endpoints to acquire together. -```bash -infer-stack describe-profile qwen2-5-7b-instruct-turbo-default --format yaml -``` - -## Stack profile model +```yaml +models: + smol135: + source: hf://HuggingFaceTB/SmolLM2-135M-Instruct -Profiles are written as stack graphs. The main sections are `providers`, `gateways`, `frontends`, and `routes`. For details and examples, see [docs/stack-graph-profiles.md](docs/stack-graph-profiles.md). +endpoints: + smol135-1: + engine: vllm + model: smol135 + runtime: {max_model_len: 8192} + chat: + engine: ollama + host: local-ollama + model: qwen3.5:4b -Common shapes: +runtime_hosts: + local-ollama: + engine: ollama + placement: {gpu_indices: [0]} + settings: {keep_alive: 30m} -```text -Open WebUI -> Ollama # ollama-direct, no LiteLLM -Open WebUI -> LiteLLM -> vLLM # classic vLLM compose profiles -Open WebUI -> LiteLLM -> Ollama # Ollama with stable aliases -Open WebUI -> LiteLLM -> Ollama + vLLM # mixed migration / test stacks -Ollama API + vLLM API directly # raw backend profiles +bundles: + both: [smol135-1, chat] ``` -Custom provider models and custom profiles live in the configured `catalog.user_models_file`, which defaults to `~/.config/infer_stack/models.yaml`. New files should prefer provider-specific top-level keys: +Edit it with `infer-stack catalog model|endpoint|host|bundle add …`, or by +hand with `infer-stack catalog edit`; `infer-stack catalog validate` checks it. +The schema reference is the docstring of `infer_stack/leasing/catalog.py`. -```yaml -vllm_models: - my-vllm-model: - hf_model_id: org/model - -ollama_models: - my-ollama-model: - tag: qwen3.5:4b - -profiles: - my-stack: - providers: {} - gateways: {} - frontends: {} - routes: {} +Shapes the stack renders: + +```text +Open WebUI -> LiteLLM -> vLLM / Ollama # the default +Open WebUI -> vLLM / Ollama # config set litellm false ``` -`models:` is still interpreted as a vLLM model catalog for convenience, but new docs and recipes use `vllm_models:` / `ollama_models:`. +The named stack profiles of earlier releases (`setup`, `switch`, +`--profile`) are gone; see +[docs/stack-graph-profiles.md](docs/stack-graph-profiles.md). ## Where config and rendered artifacts live -`infer-stack` follows XDG basedir conventions, so where you invoke it -from never changes which config it reads or where it writes rendered -artifacts: - -There are exactly two path roots: +`infer-stack` follows XDG basedir conventions, so the directory you invoke it +from never changes which config it reads or where it writes. There are two +roots: | What | Default location | How to relocate | | --- | --- | --- | -| `config.yaml`, `models.yaml`, `kubeai-values.local.yaml` | `~/.config/infer_stack/` (resp. `$XDG_CONFIG_HOME`) | `--config-dir` (or `INFER_STACK_CONFIG_DIR`) | -| **Everything generated** — `generated/` (docker-compose.yml, .env, plan.yaml, kubeai/*) **and** `state/` (hf-cache, postgres volumes, Ollama store, runtime bind mounts) | `~/.local/share/infer_stack/` (resp. `$XDG_DATA_HOME`) | `--data-dir` (or `INFER_STACK_DATA_DIR`) | +| `settings.yaml`, `catalog.yaml` | `~/.config/infer_stack/` (resp. `$XDG_CONFIG_HOME`) | `--config-dir` or `INFER_STACK_CONFIG_DIR` | +| **Everything generated**: `leasing/` (the ledger, the compose project, its `.env`) and the bind-mounted state (`hf-cache/`, `vllm-cache/`, `open-webui/`, `ollama/`, …) | `~/.local/share/infer_stack/` (resp. `$XDG_DATA_HOME`) | `config set data_dir `, `--data-dir` or `INFER_STACK_DATA_DIR` | + +`infer-stack paths` prints every resolved path and whether it exists. -`--data-dir` is the single knob for "put everything I generate in one -directory." It **relocates the one infer-stack installation controlling a +The data dir **relocates the one infer-stack installation controlling a host/backend; it does not create an isolated second installation**. Do not run controllers from multiple config/data roots against the same Docker host or Kubernetes namespace. See the [single-owner limitation](docs/planning/known-limitations.md#one-control-plane-per-host-or-backend-namespace). -Set it once at `setup`; it is baked into the absolute -`state.*` and `output.generated_dir` paths written to `config.yaml`, so -later commands don't need it again: - ```bash -# All rendered artifacts and bind-mount state land under one directory. -infer-stack setup \ - --backend compose \ - --profile ollama-direct \ - --data-dir /data/service/docker/vllm-stack +# Persist it once; later commands read it from settings.yaml. +infer-stack config set data_dir /data/service/docker/infer-stack -infer-stack render --yes +# Or keep both roots in a checkout for an ad-hoc experiment. +export INFER_STACK_CONFIG_DIR=$PWD/cfg INFER_STACK_DATA_DIR=$PWD/stack +infer-stack paths ``` -```bash -# Keep config.yaml in a checkout for ad-hoc experiments. -infer-stack setup --config-dir $PWD --backend compose --profile

--data-dir $PWD/stack -``` - -`--config-dir` / `--data-dir` live on every subcommand, so they appear -**after** the subcommand name. For "set once for the whole shell" use the -env vars instead. For a bespoke split layout (e.g. big `state/` on a data -disk, artifacts elsewhere), edit `state.*` / `output.generated_dir` in -`config.yaml` directly. +`--config-dir` / `--data-dir` are accepted by every subcommand, after the +subcommand name. ## Constraining placement to specific GPUs -If some of your GPUs are tied up by other work, restrict the planner -(and the rendered ``device_ids``) to the subset you want it to use: - ```bash # Only place onto GPU 1 (e.g. GPU 0 is running a display). -infer-stack render --yes --profile test-single-11gb --allowed-gpus 1 +infer-stack acquire --allowed-gpus 1 -# Or pin a TP=2 profile to physical GPUs 1 and 3. -infer-stack render --yes --profile test-multi-gpu --allowed-gpus 1,3 +# Or confine a TP=2 endpoint to physical GPUs 1 and 3. +infer-stack acquire --allowed-gpus 1,3 ``` -``--allowed-gpus`` (or ``INFER_STACK_ALLOWED_GPUS=1,3``) filters the -detected inventory before placement — real indices are preserved, so -the rendered compose stack pins ``device_ids: ["1", "3"]`` to those -exact physical GPUs. Useful for integration tests that need to share a -host with other jobs. +`--allowed-gpus` (or `INFER_STACK_ALLOWED_GPUS=1,3`) filters the detected +inventory before placement for that call only. Real indices are preserved, so +the rendered compose stack pins `device_ids` to those exact GPUs. The durable +forms live in data: -## Demos / integration recipes - -End-to-end examples under [docs/demos/](docs/demos/) are written as -markdown tutorials. The CI smoke test is runnable with pytest-codeblocks: - -```bash -pytest --codeblocks docs/demos/ci_smoke_test.md -``` +* `placement: {gpu_indices: [1]}` on a vLLM endpoint pins it exactly (the list + length must equal tp×pp×dp); Ollama daemons pin through their + `runtime_hosts` entry; +* `placement: {min_vram_gib: 24}` makes smaller GPUs ineligible; +* `config set skip_display_gpus true` (or `--skip-display-gpus`) leaves the + GPU driving a monitor free. -Each ``bash`` block is a self-contained shell snippet you can also -copy-paste into a terminal. See -[docs/demos/ci_smoke_test.md](docs/demos/ci_smoke_test.md) for the -``setup → describe → validate → render`` flow on the smallest test -profiles. - -For a real running vLLM stack on a workstation, see -[docs/demos/quickstart.md](docs/demos/quickstart.md). For direct Ollama on a dual GTX 1080 Ti style host, see -[docs/demos/ollama_direct_quickstart.md](docs/demos/ollama_direct_quickstart.md). For a focused GPU-1 backend switch test, see [docs/demos/smollm2_gpu1_backend_switch.md](docs/demos/smollm2_gpu1_backend_switch.md). +## Demos / integration recipes -User-supplied paths on the CLI (`--file`, `--from-file`, -`--resource-profiles-file`, `--output-dir`) still resolve against the -current working directory — they're meant to behave as typed. +The user manual under [docs/source/manual/](docs/source/manual/) has two +walkthroughs on the current CLI: +[the Ollama + Open WebUI tutorial](docs/source/manual/ollama-openwebui-tutorial.md) +and [the leasing demo](docs/source/manual/leasing-demo.md) (standing service, +Open WebUI, several models side by side). --- ## Backend 1: Compose -Use Compose for local single-host deployments. It can render direct Ollama stacks, vLLM stacks, mixed Ollama+vLLM stacks, and raw backend-only stacks. +Use Compose for single-host serving. It runs vLLM and Ollama engines, mixed +freely, behind the optional gateway and UI. ### Getting started -Prerequisite: Docker and the `docker compose` plugin must be installed. +Prerequisite: Docker and the `docker compose` plugin. ```bash -# Direct Ollama, no LiteLLM and no predeclared models. -infer-stack setup --backend compose --profile ollama-direct -infer-stack validate --simulate-hardware 2x11 -infer-stack render --yes --simulate-hardware 2x11 -infer-stack up -d - -# Classic vLLM through LiteLLM/Open WebUI. -infer-stack setup --backend compose --profile qwen2-5-7b-instruct-turbo-default -infer-stack validate -infer-stack render -infer-stack up -d -``` - -### Test that it is responding +infer-stack config init --backend compose +infer-stack doctor --gpu +infer-stack catalog init -When LiteLLM is enabled, the default Compose front door is: +# A vLLM endpoint. +infer-stack catalog model add smol135 --source hf://HuggingFaceTB/SmolLM2-135M-Instruct +infer-stack catalog endpoint add --model smol135 # -> smol135-1 +infer-stack acquire smol135-1 -```text -http://127.0.0.1:14042/v1 +# An Ollama endpoint, on a daemon pinned to GPU 0. +infer-stack catalog host add local-ollama --engine ollama --gpu 0 +infer-stack catalog endpoint add chat --engine ollama --host local-ollama --model qwen3.5:4b +infer-stack acquire chat ``` -When using `ollama-direct`, Open WebUI talks to Ollama directly and the Ollama API is available at: - -```text -http://127.0.0.1:11434 -http://127.0.0.1:11434/v1 -``` +`infer-stack catalog suggest --apply` fills the catalog with endpoints sized +for the detected GPUs instead. -unless you changed the relevant ports in config. +### Test that it is responding -Wait until the active profile can serve a real request through its resolved default endpoint: +With LiteLLM enabled, every endpoint is reachable by its alias at: -```bash -infer-stack wait-ready +```text +http://127.0.0.1:14042/v1 ``` -`wait-ready` is stronger than Docker Compose health: it probes the user-facing -LiteLLM, Ollama, or direct vLLM access surface and, by default, requires a tiny -generation/completion to succeed. The smoke test runs this readiness probe by -default before issuing its normal test request: +`acquire` already blocks until the endpoint returns a real generation through +that front door, which is stronger than Docker's container health. After +`acquire --no-wait`, block separately; to check again later, send one request: ```bash -infer-stack smoke-test +infer-stack wait smol135-1 +infer-stack test smol135-1 +infer-stack test chat --prompt "Name three colors." --max-tokens 32 ``` -For direct Ollama profiles, pull a model first and then smoke-test that model: +Clients read the front door and the managed key from the env file: ```bash -infer-stack ollama-pull qwen3.5:4b -infer-stack ollama-list -infer-stack smoke-test --model qwen3.5:4b +export OPENAI_BASE_URL=$(infer-stack env OPENAI_BASE_URL) +export OPENAI_API_KEY=$(infer-stack env LITELLM_MASTER_KEY) ``` -For LiteLLM profiles, `smoke-test` reads the rendered `.env` automatically and -uses the active profile's resolved OpenAI-compatible front door. You can inspect -individual secrets when needed: - -```bash -infer-stack env LITELLM_MASTER_KEY -infer-stack env VLLM_BACKEND_API_KEY -``` +or get both, plus per-endpoint names, from +`infer-stack acquire --env-file lease.env`. To replace the gateway's master key (refused while leases are active; the gateway restarts, and clients must fetch the key again): @@ -361,244 +324,111 @@ gateway restarts, and clients must fetch the key again): infer-stack secrets rotate ``` -When you intentionally want the old quick behavior, skip the readiness wait: - -```bash -infer-stack smoke-test --no-wait --model gpt2 -``` - ### Stop it ```bash -infer-stack down +infer-stack release --all # drop every lease +infer-stack release --all --evict # ...and stop the engines now +infer-stack clean -f # no leases, nothing on a GPU; the gateway stays +infer-stack stack down # docker compose down, bypassing the ledger ``` -`down` never removes named volumes. The Postgres data directory and the -Open WebUI volume are preserved across `down`, `up`, `switch`, and `render`. +After a plain `release`, `keep-warm` endpoints (the default `reclaim` policy) +stay loaded until another lease needs their GPUs or `infer-stack evict` stops +them. `stack down` releases no lease, so the next `apply` or `acquire` brings +leased models back. -### Open WebUI authentication - -By default Open WebUI runs with `WEBUI_AUTH=False` — no login screen, -anyone who can reach the port gets straight into the UI. This is the -expected behavior for a local dev box. To re-enable login/signup, set -in `config.yaml`: +All state is bind-mounted from the data dir, so none of these delete it, +including `stack down --volumes`. For a destructive reset, remove the +directories `infer-stack paths` lists. -```yaml -open_webui: - auth: true -``` +### Open WebUI authentication -and re-render. Existing accounts stored in the `postgres-open-webui` -volume are preserved across the toggle. +Open WebUI runs with `WEBUI_AUTH=False`: no login screen, and anyone who can +reach port 13000 gets the UI. No setting changes that. Keep the host on a +trusted network, or run without the UI (`config set ui false`). -### Reverse proxy (TLS) and LDAP +### Reverse proxy -Open WebUI can be fronted by an opt-in nginx TLS reverse proxy, and its -login can be backed by an LDAP directory. Both are off by default and -configured as ordinary config fields. The built-in `openwebui-tls-ldap` -profile wires them together as a worked example (Ollama + Open WebUI -behind nginx, no public Open WebUI/Ollama ports); see -[`examples/openwebui-tls-ldap/`](examples/openwebui-tls-ldap/). +An optional nginx service publishes one HTTP port with the UI at `/` and the +API at `/v1`. It needs the LiteLLM gateway. ```bash -infer-stack setup --backend compose --profile openwebui-tls-ldap +infer-stack config set reverse_proxy true # port 80 +infer-stack config set reverse_proxy '{enabled: true, port: 8080}' ``` -**Reverse proxy.** Enable it under `frontends.reverse_proxy`. It renders -an nginx service plus a generated `state.runtime/nginx.conf`: +`acquire --reverse-proxy` turns it on for one call. Add `config_path: +/path/to/nginx.conf` to the block to mount your own config at +`/etc/nginx/conf.d/default.conf` instead of the generated one. -```yaml -frontends: - reverse_proxy: - enabled: true - target: open_webui # or litellm / ollama / a custom upstream - server_name: host.example.com - ssl: - enabled: true - certificate: ./certs/site.crt - certificate_key: ./certs/site.key - dhparam: ./dhparam.pem # optional -``` - -When `ssl.enabled` is true, port 80 redirects to HTTPS (`force_https`) -and the cert/key/dhparam host paths are bind-mounted read-only. When -`ssl.enabled` is false, only HTTP is published (HTTPS publishing is -gated on TLS so you never get a `:443` mapping with nothing listening). - -> **Path caveat.** Relative `certificate`/`certificate_key`/`dhparam`/ -> `config_path` values are written verbatim into the generated -> `docker-compose.yml`, so Docker Compose resolves them **relative to the -> generated directory** (where the compose file lives), not your CWD. -> `infer-stack render` warns when a referenced cert or config file is not -> found. Use absolute paths if you want to avoid the ambiguity. - -**LDAP.** Enable it under `frontends.open_webui.ldap`. The directory -settings render as Open WebUI `LDAP_*` environment variables, and -secrets/site-specific values are emitted as `.env` placeholders -(`LDAP_HOST`, `LDAP_PASSWD`, `LDAP_SEARCH_BASE`, …) so you can fill them -in after the first render without re-touching the compose YAML: - -```yaml -frontends: - open_webui: - ldap: - enabled: true - env_defaults: - LDAP_PORT: '636' - LDAP_USE_TLS: 'true' - LDAP_ATTRIBUTE_FOR_USERNAME: uid -``` - -**Manual escape hatches.** When the typed renderer is not enough, drop -to manual control without leaving infer-stack: - -* `frontends.reverse_proxy.config_path` — mount an existing nginx config - file instead of rendering one. -* `frontends.reverse_proxy.extra_config` — inject extra directives into - the rendered HTTPS `server` block. -* Every rendered service (`ollama`, vLLM runtimes, `litellm`, - `open_webui`, `reverse_proxy`) accepts generic overrides: - `extra_env`, `env_file`, `extra_volumes`, `extra_hosts`, `labels`, - `additional_ports`, and `gpus` (scalar `all`/count or a structured - device-request list). - -Field precedence (lowest to highest) is: top-level config section -(`reverse_proxy:` / `open_webui:` / `ollama:`) → the matching -`frontends.*` / `providers.*` / `gateways.*` section → the active -profile. Newer configs should prefer the `frontends.*` / `providers.*` -form shown above. +It does no TLS and no authentication. Only the one port is published and no +certificate is mounted, so terminate TLS in a proxy in front of it. The TLS +and LDAP settings of the pre-leasing profiles no longer exist. ### Persistent state and database layout -Compose renders stateful services only when their components are enabled: - -* `postgres-open-webui` — rendered only when Open WebUI is enabled. It stores chats, accounts, and settings in `state.postgres_open_webui`. -* `postgres-litellm` — rendered only when LiteLLM is enabled. It stores router state in `state.postgres_litellm`. -* `ollama` — rendered only when the Ollama provider is enabled. Its model store is `state.ollama`, mounted at `/root/.ollama`. -* vLLM runtimes mount `state.hf_cache` for Hugging Face weights and `state.vllm_cache` for compiled artifacts. +Everything lives under the data dir (`infer-stack paths`): -Each Postgres container has its own `POSTGRES_DB`, `POSTGRES_USER`, and -`POSTGRES_PASSWORD`, sourced from component-specific `.env` keys. There is no shared Postgres instance and no `postgres-init` bootstrap service. +* `open-webui/`: Open WebUI's data directory (accounts, chats, settings), + mounted at `/app/backend/data`; +* `postgres-litellm/`: LiteLLM's route store, rendered only with + `config set dynamic_routing true`; +* `ollama/`: the Ollama model store, mounted at `/root/.ollama`; +* `hf-cache/`, `vllm-cache/`, `torch-cache/`, `triton-cache/`, `cuda-cache/`: + vLLM weights and compile caches (see + [docs/persistent-caches-and-warm-restarts.md](docs/persistent-caches-and-warm-restarts.md)); +* `runtime/`: directories an endpoint's `runtime.mounts` asks for; +* `leasing/`: the ledger and the rendered compose project. -Open WebUI chat history is **not** tied to the model currently being served, so after a profile switch old chats may reference model IDs the current gateway no longer advertises — that is expected. - -### Operational tips - -Prefer scoping commands to specific services rather than relying on -container names. Use only the services rendered by the active profile: - -```bash -# LiteLLM gateway profile -infer-stack logs -f litellm - -# Direct Ollama profile -infer-stack logs -f ollama - -# Ollama model store helpers -infer-stack ollama-list -infer-stack ollama-ps -``` - -You do not need to delete any volume during normal operation. If you -ever want a destructive reset, do it explicitly with -`docker compose down -v` against `generated/docker-compose.yml` — the -toolchain itself never does this. +Open WebUI chat history is not tied to the models currently served, so old +chats may name aliases the gateway no longer advertises. That is expected. ### Custom .env values are preserved -`generated/.env` is rewritten non-destructively. Any `KEY=value` pair -you add manually (for example `VERBOSE=1`, `HF_HOME=/data/hf`, or any -key this program does not yet know about) is preserved across -`render`, `setup`, `switch`, `up`, and `deploy`. Comments and the order -of existing lines are preserved where practical. +The compose project's `.env` holds the managed secrets (`LITELLM_MASTER_KEY`, +`HF_TOKEN`, …). `infer-stack env KEY=VALUE` merges a value into it, and keys +you add are kept across renders. Compose uses the file for interpolation, so a +key reaches a container only when the rendered service references it (as +`HF_TOKEN` does for vLLM). Set `HF_TOKEN` before the first `acquire` of a +gated model. -### Switching profiles +### Switching models + +There is no single active model to switch. `acquire` another endpoint and it +runs beside the first; release the first when you are done: ```bash -infer-stack switch --apply +infer-stack acquire smol135-1 +infer-stack acquire chat # now both are served +infer-stack leases # find smol135-1's lease id +infer-stack release ``` -`switch --apply` re-renders from the updated `config.yaml`, then brings the -stack up convergently with `--remove-orphans` so a separate `infer-stack up` is -not needed. Components/runtimes that are no longer in the rendered compose file -are dropped. Compose preserves existing containers whose service definitions did -not change. For vLLM-to-vLLM profile switches, unchanged Open WebUI stays up; -LiteLLM is refreshed through its admin API when possible. The live refresh -path treats LiteLLM's "model not found in db" response for config-backed -models as non-fatal, so switching aliases can add the new route without -tearing LiteLLM down. That can temporarily leave stale config-backed aliases -in `/v1/models`; restart LiteLLM manually only when you want to clean those -up. If Compose already created or recreated LiteLLM while converging the new -stack, no extra router refresh is attempted because the new container has -already loaded the freshly rendered YAML. Profiles that do not render -LiteLLM, such as direct Ollama profiles, skip the router refresh path even if -an old `runtime/litellm_config.yaml` file remains from a previous profile. -Switches that change Open WebUI's provider wiring, such as -`Open WebUI -> LiteLLM` to `Open WebUI -> Ollama`, necessarily recreate -Open WebUI because its environment changes. Postgres volumes and provider -caches are left untouched. vLLM runtime containers are named after their -Compose service, for example `vllm-chat`, so `docker ps` and -`infer-stack logs vllm-chat` clearly identify them as vLLM containers. +The gateway carries a route for every catalog endpoint, so adding or removing +a model does not recreate LiteLLM, and Open WebUI stays up. When GPUs are +short, `acquire` fails fast; `--queue` waits for a GPU instead, and idle +`keep-warm` deployments are evicted to make room. vLLM containers are named +after the served alias (`vllm-`), so `docker ps` and +`infer-stack logs vllm-` identify them. ### Protocol modes for base vs. instruct models -Profiles declare a `protocol_mode` (`chat` or `completions`) that the -served model must support. Models also declare which protocols they -support via `supported_protocols`. Validation runs before render and -fails with an actionable message if a profile asks for `chat` on a -completions-only model. - -Practical guidance: - -* Instruct/chat models (with a chat template) can use either, but - default to `chat`. -* Base models like Pythia, Llama-2 base, Mistral-v0.1 base, and Falcon - base do not define a chat template. Their HELM profiles use - `protocol_mode: completions` and the `smoke-test` command will - exercise `/v1/completions` for them. -* The rendered LiteLLM config uses `text-completion-openai/` - as the upstream provider for completions-only services. That means - even chat-shaped requests sent through Open WebUI to a Pythia model - get translated by LiteLLM into upstream `/v1/completions` calls — no - second vLLM container is needed to support Open WebUI for a - completions-only model. -* Open WebUI is still a chat UI, so prompt formatting matters. - HELM/eval clients should call `/v1/completions` directly for exact - prompt control rather than going through the chat frontend. - -### Chat-shaped clients on top of completions models - -Some clients (e.g. InspectAI / Inspect Evals stock MMLU tasks) only -speak `/v1/chat/completions` and cannot be reconfigured. For those -cases, profiles can opt into a LiteLLM-only adapter: - -```yaml -chat_compat: - enabled: true - strategy: flat_messages -``` - -When set on a `protocol_mode: completions` service, the rendered -LiteLLM config keeps the `text-completion-openai/` upstream -and adds LiteLLM's documented prompt-template fields -(`initial_prompt_value` / `roles` / `final_prompt_value`) so chat -messages get flattened into a plain prompt — no role labels, messages -joined by `\n` — before being forwarded to vLLM `/v1/completions`. - -This is **not** a chat tune; the model is still a base model and -prompt formatting still matters for evaluation. Use it only when a -chat-shaped client cannot be changed. The vLLM container is not -restarted, no `--chat-template` is rendered, and the adapter takes -effect after a `litellm`-only restart: +An endpoint's `protocol` is `chat` (the default) or `completions`. It decides +which surface the readiness probe and `infer-stack test` use, so a base model +without a chat template must declare `completions` or its `acquire` never sees +a ready generation: ```bash -infer-stack render -infer-stack restart litellm +infer-stack catalog model add pythia-160m --source hf://EleutherAI/pythia-160m +infer-stack catalog endpoint add --model pythia-160m --protocol completions +infer-stack acquire pythia-160m-1 +infer-stack test pythia-160m-1 # hits /v1/completions ``` -The built-in `pythia-inspect-mmlu-compat` profile is a ready-made -example; see -[`recipies/compose_pythia_inspect_mmlu_compat.md`](recipies/compose_pythia_inspect_mmlu_compat.md). +The gateway forwards `/v1/completions` unchanged, so evaluation clients that +need exact prompt control should call it directly. Open WebUI is a chat UI and +sends chat requests, which a base model cannot answer. ### Images with their own launcher @@ -608,34 +438,25 @@ endpoint's `runtime`; infer-stack has no model-specific code for it: ```yaml runtime: - image: ghcr.io/syv-ai/hyperqwen:sha-684e927 + image: example.org/my-vllm-launcher:1.0 max_model_len: 65536 gpu_memory_utilization: 0.93 command: [single] # replaces `vllm serve MODEL ` env: # container environment PORT: '{port}' - SPEC: dflash2 - CTX: fast - PREFIX_CACHE: 1 MAX_LEN: '{max_model_len}' # filled from the field above GPU_UTIL: '{gpu_memory_utilization}' EXTRA_ARGS: '--served-model-name={served_model_name}' mounts: # persisted under the runtime data dir - /app/models: hyperqwen/qwen3.8-27b/models - /cache: hyperqwen/qwen3.8-27b/cache + /app/models: my-launcher/models ``` -Switching that image to another context profile is a data edit, in the catalog -or the TUI's endpoint editor. HyperQwen's measured 3090 profiles are: - -- fast/default: `max_model_len: 65536`, `SPEC: dflash2`, `CTX: fast`; -- long: `max_model_len: 150000`, `SPEC: mtp`, `CTX: long`; -- huge: `max_model_len: 245760`, `SPEC: dflash2`, `CTX: huge`. - -On an RTX 3090, `catalog suggest` emits the latter two as `-long` and `-huge` -endpoint variants. The hardware check happens only while suggesting: the -resulting catalog contains ordinary explicit runtime data, so `apply`/`acquire` -never silently retunes an endpoint after the fact. +Changing the launcher's mode is a data edit, in the catalog or the TUI's +endpoint editor. The HyperQwen suggestion (see "Related work") is a worked +example: on an RTX 3090, `catalog suggest` emits its measured context +profiles as separate `-long` and `-huge` endpoints. The hardware check +happens only while suggesting; the resulting catalog holds ordinary explicit +runtime data, so `apply`/`acquire` never retunes an endpoint after the fact. - `{max_model_len}`, `{gpu_memory_utilization}`, `{served_model_name}` and `{port}` are filled in from the endpoint, so a launcher that takes them @@ -656,42 +477,29 @@ never silently retunes an endpoint after the fact. ### Reasoning / thinking models -Models can declare reasoning support in the catalog: +vLLM separates a reasoning trace from the answer when it is started with a +reasoning parser. Pass the flag through `runtime.extra_args`: ```yaml -reasoning: - enabled: true - parser: qwen3 - expose_to_openwebui: true +endpoints: + qwen3-think: + engine: vllm + model: qwen3-0.6b + runtime: + extra_args: [--reasoning-parser=qwen3] ``` -Profiles can override or set the same field per service. When a -service has `reasoning.enabled: true` and a `parser`, the renderer -adds `--reasoning-parser ` to that vLLM container's command -line — that flag alone enables reasoning extraction in the current -vLLM CLI. You do not need to repeat it by hand in `extra_args`. - -Open WebUI sees reasoning content via two paths: - -1. Inline `...` tags emitted by the model. -2. Structured `reasoning_content` fields when LiteLLM normalizes them. - -The LiteLLM template keeps `merge_reasoning_content_in_choices: true` -on chat-mode entries so Open WebUI can display reasoning in the -streamed response. To test reasoning end-to-end: +The parser name depends on the model family and the vLLM version (`vllm serve +--help` lists them). To test end to end: ```bash -# Non-streaming CLI smoke test: -infer-stack smoke-test \ - --model qwen3.6-35b-a3b \ - --prompt "Think step by step: 17*23" - -# For streaming inspection, read the key with the CLI wrapper: -LITELLM_MASTER_KEY=$(infer-stack env LITELLM_MASTER_KEY) -curl -N http://127.0.0.1:14042/v1/chat/completions \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ +infer-stack test qwen3-think --prompt "Think step by step: 17*23" --max-tokens 512 + +# Streaming, through the gateway: +curl -N "$(infer-stack env OPENAI_BASE_URL)/chat/completions" \ + -H "Authorization: Bearer $(infer-stack env LITELLM_MASTER_KEY)" \ -H 'Content-Type: application/json' \ - -d '{"model":"qwen3.6-35b-a3b","stream":true, + -d '{"model":"qwen3-think","stream":true, "messages":[{"role":"user","content":"Think step by step: 17*23"}]}' ``` diff --git a/docs/persistent-caches-and-warm-restarts.md b/docs/persistent-caches-and-warm-restarts.md index 9426297e..e5cd8f21 100644 --- a/docs/persistent-caches-and-warm-restarts.md +++ b/docs/persistent-caches-and-warm-restarts.md @@ -11,9 +11,8 @@ process loads a model. ### Ollama model store — `state.ollama -> /root/.ollama` Ollama downloads GGUF/model blobs into `/root/.ollama`. The Compose template -mounts `state.ollama` there whenever the Ollama provider is enabled. Direct -Ollama profiles therefore survive `down`, `up`, `switch`, and `render` without -re-pulling models. +mounts `/ollama` there for every Ollama daemon, so pulled tags +survive container replacement, `release`, and `infer-stack stack down`. Ollama model residency is controlled by daemon/request settings such as `keep_alive` / `OLLAMA_KEEP_ALIVE`. A short keep-alive can let a mostly idle @@ -40,10 +39,10 @@ dtype, etc.). On a cold container with an empty cache, vLLM has to re-compile; on a warm restart against the same configuration, those artifacts are reused. -Default host path: `/data/service/docker/infer-stack/vllm-cache`. Override via -`state.vllm_cache` in `config.yaml` or by editing the rendered deployment -plan. The path is created on first volume mount; no manual `mkdir` is -required. +Host path: `/vllm-cache/cfg-`, one subdirectory per serve +configuration (`infer-stack paths` shows the data dir). It moves with the data +dir (`infer-stack config set data_dir `); there is no separate setting. +The path is created on first volume mount; no manual `mkdir` is required. The cache key is keyed on the engine configuration. Changing `max_model_len`, `tensor_parallel_size`, `gpu_memory_utilization`, the optimization level, @@ -58,11 +57,13 @@ The Compose template also persists the other obvious startup caches that sit outside `VLLM_CACHE_ROOT`: - `state.torch_cache -> /root/.cache/torch` for PyTorch / TorchInductor - artifacts, with `TORCH_HOME` and `TORCHINDUCTOR_CACHE_DIR` set explicitly. -- `state.triton_cache -> /root/.cache/triton` for Triton kernels, with - `TRITON_CACHE_DIR` set explicitly. -- `state.cuda_cache -> /root/.cache/nvidia/ComputeCache` for NVIDIA driver - JIT artifacts, with `CUDA_CACHE_PATH` set explicitly. + artifacts. +- `state.triton_cache -> /root/.triton` for Triton kernels. +- `state.cuda_cache -> /root/.nv` for NVIDIA driver JIT artifacts. + +Each is mounted at the tool's default location; no cache environment +variables are set. Unlike the vLLM cache, these are shared across serve +configurations, because their entries are content-addressed. These caches reduce repeated compile/JIT work, but they do not make model switching instantaneous. A vLLM process still has to import Python modules, @@ -81,53 +82,44 @@ the vLLM engine process, because vLLM serves one model configuration per process in this stack. That is why even tiny models can take tens of seconds to come back healthy: the expensive work is not just downloading weights. -## Shared memory: `ipc: host` - -vLLM uses shared memory for tensor-parallel communication and worker IPC. -Docker's default `--shm-size` (64 MiB) is far too small. The Compose -template sets `ipc: host` on every vLLM service, matching the upstream vLLM -Docker guidance and giving the engine the host's shared-memory budget. +## Shared memory -If your environment forbids host IPC (multi-tenant cluster policy, etc.), -replace `ipc: host` in the rendered Compose with an explicit `shm_size` -sized for your tensor-parallel topology — typically a few GiB for TP > 1. +vLLM uses shared memory for tensor-parallel communication and worker IPC, and +upstream vLLM's Docker guidance is `ipc: host` or a `shm_size` of a few GiB +for TP > 1. The rendered Compose currently sets neither, so vLLM containers get +Docker's default 64 MiB `/dev/shm`. If an endpoint, most likely a +tensor-parallel one, fails on shared memory, this is the first thing to +check. ## Minimal-restart workflow -LiteLLM deliberately does **not** depend on provider health in the rendered -Compose file. That prevents Compose from restarting LiteLLM every time a vLLM -runtime container is replaced during a model swap. The CLI refreshes LiteLLM's -route table through its admin API when possible; smoke tests retry briefly -while the selected upstream model finishes loading. +LiteLLM deliberately does **not** depend on engine health in the rendered +Compose file, and its route table already names every catalog endpoint. So +replacing a vLLM container during a model swap neither restarts LiteLLM nor +rewrites its config. -When you are iterating on a single profile and don't want to bounce the whole -stack: +To refresh one engine without bouncing the gateway or Open WebUI: ```bash -# Pull refreshed images through the rendered Compose wrapper. -infer-stack pull vllm- +# Pull a refreshed image for one service (names from `infer-stack ps`). +infer-stack stack pull vllm- -# Restart just the one service through the wrapper. This avoids bouncing -# Postgres / LiteLLM / Open WebUI. -infer-stack restart vllm- +# Restart just that service. +infer-stack stack restart vllm- ``` -For a full stack restart (e.g. after `infer-stack render` against a new -profile), `infer-stack down && infer-stack up -d` is the safe path; persistent -volumes and bind mounts (Postgres, Open WebUI data, Ollama model store, and -vLLM caches) are not touched by `down`. - -For direct Ollama model pulls, operate on the shared daemon instead of replacing -the container: +`infer-stack apply` re-renders from the ledger and recreates only services +whose definition changed. `infer-stack stack down` followed by `infer-stack +apply` is a full restart; the bind-mounted state (Open WebUI data, Ollama +model store, vLLM caches) is not touched by `stack down`. -```bash -infer-stack ollama-pull qwen3.5:4b -``` +Ollama tags are pulled into the running daemon when an endpoint that serves +them is acquired, so adding a tag never replaces the container. ## Verifying the cache is being reused -After a warm restart, a populated `state.vllm_cache` directory will contain -subdirectories keyed by configuration hash. The vLLM startup logs print +After a warm restart, `/vllm-cache` contains one `cfg-` +subdirectory per serve configuration. The vLLM startup logs print "Using cached compiled graph" (or similar; exact wording varies by version) when the cache is hit. If you see a long compile pass on every restart, check that the volume mount is actually pointing at the persistent path and diff --git a/docs/stack-graph-profiles.md b/docs/stack-graph-profiles.md index 4880e87f..333e47fd 100644 --- a/docs/stack-graph-profiles.md +++ b/docs/stack-graph-profiles.md @@ -1,276 +1,20 @@ -# Stack graph profiles - -Profiles describe an LLM deployment as a small graph rather than as a single -vLLM-shaped service list. - -The four graph sections are: - -- `providers`: inference servers that can answer model requests, currently - `vllm` and `ollama`. -- `gateways`: optional API routers/proxies, currently `litellm`. -- `frontends`: optional user-facing UIs, currently `open_webui`. -- `routes`: optional public model aliases exposed through a gateway. - -This separation keeps the simple cases simple: - -- Ollama can run as one direct daemon with no predeclared model list. -- vLLM can still run one or more explicit model runtimes. -- LiteLLM is only needed when you want a unified `/v1` model namespace. -- Open WebUI can point either at LiteLLM or directly at Ollama. - -## Mental model - -```text -providers -> raw inference endpoints -routes -> optional public model names -gateways -> optional route surface, usually LiteLLM -frontends -> optional UI, usually Open WebUI -``` - -A profile may have providers without routes. For example, `ollama-direct` -starts Ollama and Open WebUI directly, and you pull models with the Ollama CLI -or from Open WebUI. A mixed profile with both Ollama and vLLM usually enables -LiteLLM so clients have one stable OpenAI-compatible endpoint. - -## Direct Ollama - -`ollama-direct` starts one Ollama daemon and Open WebUI connected directly to -it. It does not render LiteLLM, does not render `postgres-litellm`, and does -not require model declarations. - -```bash -infer-stack setup --backend compose --profile ollama-direct -infer-stack render --yes --simulate-hardware 2x11 -infer-stack up -d - -infer-stack ollama-pull qwen3.5:4b -``` - -Rendered shape: - -```text -Open WebUI -> Ollama -``` - -The profile owns daemon settings such as `keep_alive`, `context_length`, and -`max_loaded_models`; the model store itself lives in `state.ollama` and is -mounted at `/root/.ollama`. - -## Ollama through LiteLLM - -Use this when you want a stable OpenAI-compatible route name that can later be -moved from Ollama to vLLM without changing clients. - -```yaml -profiles: - ollama-qwen3.5-4b-gateway: - providers: - ollama: - enabled: true - gpu_indices: [0, 1] - keep_alive: 2m - context_length: 4096 - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - home-assistant-local: - provider: ollama - model: qwen3.5:4b -``` - -LiteLLM renders that route as `ollama_chat/qwen3.5:4b` with -`api_base: http://ollama:11434`. - -Rendered shape: - -```text -Open WebUI -> LiteLLM -> Ollama -``` - -Routes may reference an entry in `ollama_models`, or a raw Ollama tag such as -`qwen3.5:4b` directly. - -## vLLM through LiteLLM - -vLLM runtimes are explicit because each runtime starts a model-serving process. - -```yaml -vllm_models: - smollm2-135m-instruct: - hf_model_id: HuggingFaceTB/SmolLM2-135M-Instruct - served_model_name: smollm2-135m - supported_protocols: [chat] - min_vram_gib_per_replica: 4 - preferred_gpu_count: 1 - defaults: - max_model_len: 2048 - gpu_memory_utilization: 0.5 - -profiles: - smollm-vllm-compose: - providers: - vllm: - runtimes: - chat: - model: smollm2-135m-instruct - placement: - strategy: first_fit - gpu_count: 1 - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - smollm2: - provider: vllm - runtime: chat -``` - -Rendered shape: - -```text -Open WebUI -> LiteLLM -> vLLM runtime -``` - -## Mixed Ollama + vLLM - -`mixed-ollama-smollm` demonstrates one shared Ollama daemon plus one vLLM -runtime behind LiteLLM. - -```yaml -profiles: - mixed-local: - providers: - ollama: - enabled: true - gpu_indices: [0, 1] - vllm: - runtimes: - smollm: - model: smollm2-135m-instruct - placement: - strategy: first_fit - gpu_count: 1 - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - home-assistant-local: - provider: ollama - model: qwen3.5:4b - smollm2-135m: - provider: vllm - runtime: smollm -``` - -Rendered shape: - -```text - -> Ollama -Open WebUI -> LiteLLM - -> vLLM -``` - -Mixed routes need a gateway for one unified client namespace. Without LiteLLM, -Ollama and vLLM can still run as raw servers, but clients must address them -separately. - -## Raw servers - -`raw-ollama-vllm` starts backend servers without Open WebUI or LiteLLM. This is -useful for debugging direct provider endpoints. - -```yaml -profiles: - raw-ollama-vllm: - providers: - ollama: - enabled: true - publish_port: true - vllm: - runtimes: - smollm: - model: smollm2-135m-instruct - publish_port: true - gateways: - litellm: - enabled: false - frontends: - open_webui: - enabled: false - routes: {} -``` - -Rendered shape: - -```text -Ollama API directly -vLLM API directly -``` - -## Backend support - -Compose supports Ollama, vLLM, LiteLLM, and Open WebUI in valid combinations. -KubeAI currently supports only vLLM runtimes; profiles that enable Ollama, -LiteLLM, or Open WebUI are rejected for `--backend kubeai`. - -## Configuration files - -Custom provider models and stack profiles can come from the configured -`catalog.user_models_file`, which defaults to `~/.config/infer_stack/models.yaml`, -from additional paths in `catalog.model_path` / `catalog.model_paths`, or from -`INFER_STACK_MODEL_PATH` for per-shell overlays. Use provider-specific top-level -keys: - -```yaml -vllm_models: - my-vllm-model: - hf_model_id: org/model - -ollama_models: - my-ollama-model: - tag: qwen3.5:4b - -profiles: - my-stack: - providers: {} - gateways: {} - frontends: {} - routes: {} -``` - -`models:` is still interpreted as a vLLM model catalog for convenience, but new -examples should use `vllm_models:` and `ollama_models:`. - -`INFER_STACK_MODEL_PATH` behaves like a PATH variable: entries are separated by -`:` on POSIX systems and by `;` on Windows. Each entry can be a YAML file or a -directory. Directory entries are scanned non-recursively for `models.yaml` / `models.yml`, -`*.models.yaml` / `*.models.yml`, and `*.profiles.yaml` / `*.profiles.yml`. Later files override earlier files, so a -repo-local experiment overlay can temporarily override or extend the normal user -catalog without editing `config.yaml`: - -```bash -export INFER_STACK_MODEL_PATH="${INFER_STACK_MODEL_PATH:+$INFER_STACK_MODEL_PATH:}$PWD/infer_stack_profiles" -infer-stack describe-profile my-experiment-profile -``` - -For persistent extra catalogs, put the same directory or file path in -`config.yaml`: - -```yaml -catalog: - model_path: - - /srv/experiments/infer_stack_profiles -``` +# Stack graph profiles (removed) + +Named stack profiles (`providers` / `gateways` / `frontends` / `routes`, +selected with `--profile` or `active_profile`, applied with `setup`, `switch` +and `up`) no longer exist. The leasing catalog replaced them: `catalog.yaml` +declares `models`, `endpoints`, `runtime_hosts` and `bundles`, and +`infer-stack acquire ` leases an endpoint and brings up its engine +behind the LiteLLM gateway and Open WebUI. Gateway and UI are settings +(`infer-stack config set litellm|ui|reverse_proxy …`), not graph nodes. + +Start with the README's "Primary leasing workflow" and "Catalog model" +sections, then [the user manual](source/manual/index.md) and +[litellm-gateway-routing.md](litellm-gateway-routing.md). + +"Profile" survives in one place: the ledger's **recovery snapshot**, the +frozen copy of every non-ledger render input (settings, image pins, the +published catalog union) that recovery re-renders from. `acquire` advances it +automatically; `infer-stack config publish` pre-seeds or previews it +explicitly. See [ADR 0001](adr/0001-user-config-is-authoritative.md) and the +docstring of `infer_stack/leasing/profile.py`. From 00dbed12b9f1985aff57a6a25aa58d711ce66a70 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:40:34 -0400 Subject: [PATCH 040/141] Docs: mark the pre-leasing recipes as such; drop dead Makefile targets docs/demos/, recipies/ and examples/ describe the removed stack-profile CLI; each now opens with a banner saying its commands no longer run and where the current workflow is. The Makefile's init/deploy/smoke-test and example-single-node targets called removed commands and are gone; install-kubeai takes VALUES. Whether to rewrite or delete the recipes, and whether vLLM containers need ipc/shm settings back, are queued as decisions (docs/queue.md 8a, 8b). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- Makefile | 26 ++++--------------- docs/demos/ci_smoke_test.md | 2 ++ docs/demos/ollama_direct_quickstart.md | 2 ++ docs/demos/quickstart.md | 2 ++ docs/demos/qwen_3090_profiles.md | 2 ++ docs/demos/smollm2_gpu1_backend_switch.md | 2 ++ docs/queue.md | 25 +++++++++++++++++- examples/config.multi-model.yaml | 2 ++ examples/openwebui-tls-ldap/README.md | 2 ++ examples/single-node/README.md | 2 ++ examples/single-node/config.yaml | 2 ++ examples/single-node/models.yaml | 2 ++ .../compose_pythia_inspect_mmlu_compat.md | 1 + .../compose_pythia_qwen36_mixed_4x96GB.md | 1 + recipies/compose_qwen35_122b_4x96GB.md | 2 ++ recipies/compose_qwen35_122b_fp8_4x96GB.md | 2 ++ recipies/compose_qwen36_35b_a3b_4x96GB.md | 2 ++ 17 files changed, 57 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 68465960..4a94f8ec 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,18 @@ PYTHON ?= python -EXAMPLE ?= single-node - -init: - $(PYTHON) manage.py init - -render: - $(PYTHON) manage.py render - -deploy: - $(PYTHON) manage.py deploy +# The KubeAI chart values: your resourceProfiles (see docs/kubeai-backend.md). +VALUES ?= kubeai-values.yaml status: $(PYTHON) manage.py status -smoke-test: - $(PYTHON) manage.py smoke-test - -example-single-node: - cp examples/single-node/config.yaml ./config.yaml - cp examples/single-node/models.yaml ./models.yaml - @echo "Copied single-node example into repo root. Edit hostname/model ids as needed." +render: + $(PYTHON) manage.py render bootstrap-k3s: bash scripts/bootstrap_k3s.sh install-kubeai: - bash scripts/install_kubeai.sh generated/kubeai/kubeai-values.yaml kubeai - -bootstrap-single-node: example-single-node bootstrap-k3s - @echo "Single-node example copied and K3s bootstrapped. Next: python manage.py render && python manage.py deploy" + bash scripts/install_kubeai.sh $(VALUES) kubeai port-forward-kubeai: kubectl -n kubeai port-forward svc/kubeai 8000:80 diff --git a/docs/demos/ci_smoke_test.md b/docs/demos/ci_smoke_test.md index a1b6457d..763c08e8 100644 --- a/docs/demos/ci_smoke_test.md +++ b/docs/demos/ci_smoke_test.md @@ -1,5 +1,7 @@ # CI smoke test +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + Walks through the canonical `infer-stack` workflow end-to-end: pick a profile, render the deployment, inspect the rendered artifacts, and exercise a few read-only inspection commands. diff --git a/docs/demos/ollama_direct_quickstart.md b/docs/demos/ollama_direct_quickstart.md index 3ec204a6..fd790ad8 100644 --- a/docs/demos/ollama_direct_quickstart.md +++ b/docs/demos/ollama_direct_quickstart.md @@ -1,5 +1,7 @@ # Quickstart — direct Ollama on a dual GTX 1080 Ti host +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + This quickstart brings up the simplest local stack for Pascal-era GPUs: ```text diff --git a/docs/demos/quickstart.md b/docs/demos/quickstart.md index 7ef85742..b9fe2169 100644 --- a/docs/demos/quickstart.md +++ b/docs/demos/quickstart.md @@ -1,5 +1,7 @@ # Quickstart — vLLM + LiteLLM on an RTX 3090 workstation +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + This guide brings up the traditional vLLM-backed stack on a single-GPU workstation where GPU 0 may be reserved for the desktop and GPU 1 is free for inference. It starts with the tiniest vLLM model to prove the plumbing works, diff --git a/docs/demos/qwen_3090_profiles.md b/docs/demos/qwen_3090_profiles.md index aebc875b..6cb591ca 100644 --- a/docs/demos/qwen_3090_profiles.md +++ b/docs/demos/qwen_3090_profiles.md @@ -1,5 +1,7 @@ # Qwen profiles for a 3090 workstation +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + These built-in profiles are intended for quick GPU-1 testing on a workstation-class RTX 3090 host. diff --git a/docs/demos/smollm2_gpu1_backend_switch.md b/docs/demos/smollm2_gpu1_backend_switch.md index b7db1e86..53641c8f 100644 --- a/docs/demos/smollm2_gpu1_backend_switch.md +++ b/docs/demos/smollm2_gpu1_backend_switch.md @@ -1,5 +1,7 @@ # SmolLM2 backend switch test on GPU 1 +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + This is a copy/paste smoke path for a workstation where GPU 1 is available. It uses only built-in profiles and `infer-stack` runtime commands. diff --git a/docs/queue.md b/docs/queue.md index af34a527..f07df172 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -115,7 +115,9 @@ VM, a Model pinned to it by node selector is served through the same gateway. `dev/handover/p5_two_hosts.sh` exists for one run across two real machines. -### 8. [ ] The README's Compose sections, and the other stale docs +### 8. [x] The README's Compose sections, and the other stale docs + +Done 2026-09-26, `e1c68f3`. About 33 references to verbs that no longer exist (`setup`, `up -d`, `switch`, `describe-profile`, `smoke-test`, `wait-ready`, `diagnose`), and @@ -127,6 +129,24 @@ top-level `restart` / `stop` / `start` / `pull` (they live under `stack`). **Done when:** every command in the README and `docs/` runs as written against the current CLI, and `grep` finds none of those verbs. +### 8a. [ ] Decide: rewrite or delete the pre-leasing recipes + +*Why added (2026-09-26):* `docs/demos/`, `recipies/` and `examples/` (about +2,000 lines) are hardware recipes for the removed profile CLI; the Makefile +targets that used them are gone. Each now opens with a "Pre-leasing" banner +saying its commands no longer run. Rewriting them as catalog entries needs +the hardware they describe; deleting them loses tuning notes. **The +operator's call.** + +### 8b. [ ] Decide: `/dev/shm` for vLLM containers + +*Why added (2026-09-26):* the leasing renderer sets neither `ipc: host` nor +`shm_size`, so a vLLM container gets Docker's 64 MiB `/dev/shm`; the +pre-leasing template had `ipc: host`, and vLLM's own Docker instructions use +it for tensor parallelism. Adding it changes every vLLM service's +fingerprint, so the next apply recreates running engines. Needs a TP=2 run +on a GPU host to confirm the symptom first. **The operator's call.** + ### 9. [ ] UX audit loop: do not stop without a passing audit Polishing the UX can take hours; that is expected, and this item is the @@ -163,6 +183,9 @@ Seed findings, already known: - [x] `infer-stack logs` should accept a container name, a prefixed name or an endpoint name. Done with P2: an instance name, a container id prefix, a deployment id or an endpoint alias. +- [x] `logs --no-color` silently kept color: kwconf reads a leading `no-` as + negation, so a flag *named* `no_color` never saw it. Now `color`, whose + `--no-color` works. Found by the README rewrite 2026-09-26. - [x] `infer-stack logs -f qwen` followed every instance: a kwconf flag took the next word as its value. Fixed for every command (P2). - [ ] `status` shows STALE during an apply instead of "apply in progress". diff --git a/examples/config.multi-model.yaml b/examples/config.multi-model.yaml index 512ce8f7..da0c81e3 100644 --- a/examples/config.multi-model.yaml +++ b/examples/config.multi-model.yaml @@ -1,3 +1,5 @@ +# Pre-leasing: a stack-profile config for the removed profile CLI; it is +# not read by infer-stack any more. See the README for the catalog. name: aiq-multi-model backend: compose active_profile: qwen-mixed diff --git a/examples/openwebui-tls-ldap/README.md b/examples/openwebui-tls-ldap/README.md index 39404028..753f329c 100644 --- a/examples/openwebui-tls-ldap/README.md +++ b/examples/openwebui-tls-ldap/README.md @@ -1,5 +1,7 @@ # Open WebUI with TLS reverse proxy and LDAP +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + This example runs the built-in `openwebui-tls-ldap` profile: an opt-in Compose stack with diff --git a/examples/single-node/README.md b/examples/single-node/README.md index 160b6527..feb685ef 100644 --- a/examples/single-node/README.md +++ b/examples/single-node/README.md @@ -1,5 +1,7 @@ # Single-node KubeAI example +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). + This example uses the stack-graph schema, but KubeAI currently renders only the `providers.vllm.runtimes` section. Ollama, LiteLLM, and Open WebUI are disabled in this example because they are Compose-only for now. diff --git a/examples/single-node/config.yaml b/examples/single-node/config.yaml index 09a54427..93bac038 100644 --- a/examples/single-node/config.yaml +++ b/examples/single-node/config.yaml @@ -1,3 +1,5 @@ +# Pre-leasing: a stack-profile config for the removed profile CLI; it is +# not read by infer-stack any more. See the README for the catalog. name: single-node-kubeai backend: kubeai active_profile: qwen-single-node diff --git a/examples/single-node/models.yaml b/examples/single-node/models.yaml index c233e6a9..7ec5f4b9 100644 --- a/examples/single-node/models.yaml +++ b/examples/single-node/models.yaml @@ -1,3 +1,5 @@ +# Pre-leasing: a stack-profile config for the removed profile CLI; it is +# not read by infer-stack any more. See the README for the catalog. vllm_models: qwen-small: hf_model_id: Qwen/Qwen2.5-7B-Instruct diff --git a/recipies/compose_pythia_inspect_mmlu_compat.md b/recipies/compose_pythia_inspect_mmlu_compat.md index b7ca7131..8687a7bc 100644 --- a/recipies/compose_pythia_inspect_mmlu_compat.md +++ b/recipies/compose_pythia_inspect_mmlu_compat.md @@ -1,5 +1,6 @@ # Compose recipe: Pythia with LiteLLM-only chat compatibility (Inspect / MMLU) +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). > Schema note: built-in profiles now resolve to stack-graph profiles. vLLM runtimes live under `providers.vllm.runtimes`, LiteLLM routes live under `routes`, and direct Ollama profiles can run without LiteLLM. diff --git a/recipies/compose_pythia_qwen36_mixed_4x96GB.md b/recipies/compose_pythia_qwen36_mixed_4x96GB.md index 73d48bb9..464f67d0 100644 --- a/recipies/compose_pythia_qwen36_mixed_4x96GB.md +++ b/recipies/compose_pythia_qwen36_mixed_4x96GB.md @@ -1,5 +1,6 @@ # Compose recipe: Pythia 6.9B + Pythia 2.8B + Qwen3.6-35B-A3B on a 4x96GB host +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). > Schema note: built-in profiles now resolve to stack-graph profiles. vLLM runtimes live under `providers.vllm.runtimes`, LiteLLM routes live under `routes`, and direct Ollama profiles can run without LiteLLM. diff --git a/recipies/compose_qwen35_122b_4x96GB.md b/recipies/compose_qwen35_122b_4x96GB.md index ea747ca9..9a0fcc6e 100644 --- a/recipies/compose_qwen35_122b_4x96GB.md +++ b/recipies/compose_qwen35_122b_4x96GB.md @@ -1,5 +1,7 @@ # Compose recipe: Qwen3.5-122B-A10B on a 4x96GB host +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). + This is the shortest working end-to-end example for serving **Qwen/Qwen3.5-122B-A10B** on a machine with **4 x 96GB GPUs** using the **Compose** backend. This profile uses: diff --git a/recipies/compose_qwen35_122b_fp8_4x96GB.md b/recipies/compose_qwen35_122b_fp8_4x96GB.md index 334c02ca..b01417d5 100644 --- a/recipies/compose_qwen35_122b_fp8_4x96GB.md +++ b/recipies/compose_qwen35_122b_fp8_4x96GB.md @@ -1,5 +1,7 @@ # Compose recipe: Qwen3.5-122B-A10B-FP8 on a 4x96GB host +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). + This is the shortest working end-to-end example for serving **Qwen/Qwen3.5-122B-A10B-FP8** on a machine with **4 x 96GB GPUs** using the **Compose** backend. This profile uses: diff --git a/recipies/compose_qwen36_35b_a3b_4x96GB.md b/recipies/compose_qwen36_35b_a3b_4x96GB.md index a2c41f16..c9cbd4a2 100644 --- a/recipies/compose_qwen36_35b_a3b_4x96GB.md +++ b/recipies/compose_qwen36_35b_a3b_4x96GB.md @@ -1,5 +1,7 @@ # Compose recipe: Qwen3.6-35B-A3B on a 4x96GB host +> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). + This is the shortest end-to-end example for serving **Qwen/Qwen3.6-35B-A3B** on a machine with **4 x 96GB GPUs** using the **Compose** backend. This profile uses: From 9eba980c47f404b23c36c593b51c423fef3cd599 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:40:34 -0400 Subject: [PATCH 041/141] UX: `logs --no-color` works kwconf reads a leading `no-` as negation, so a flag named `no_color` never saw `--no-color` (only the undocumented `--no_color`). The flag is now `color`, on by default, and `--no-color` turns it off. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- README.md | 2 +- infer_stack/cli/commands_runtime.py | 7 +++++-- tests/test_day2.py | 8 ++++++++ tests/test_log_filter.py | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5cc10e60..b1a16c8a 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ infer-stack stack compose -- ps --format json # any other docker compose comma or an endpoint alias. Interactive `infer-stack logs -f` compacts only explicitly registered, known-noisy LiteLLM traceback shapes; unknown tracebacks pass through unchanged. Redirected or piped output stays raw, and -`--raw` disables compaction in an interactive follow. `--no_color` drops the +`--raw` disables compaction in an interactive follow. `--no-color` drops the name-prefix colors. The TUI uses the same compactor. Ollama tags are pulled into the daemon on the first `acquire` of an endpoint diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 0063014d..f82b3e83 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -548,7 +548,10 @@ class LogsCLI(_InstancesBase): help="Only the last N lines of each (default: all). A number or 'all'.", ) timestamps = kw.Value(False, isflag=True) - no_color = kw.Value(False, isflag=True, help='Do not color the name prefixes.') + # A positive flag: kwconf reads a leading `no-` as negation, so a flag + # named `no_color` silently ignored `--no-color`. + color = kw.Value(True, isflag=True, + help='Color the name prefixes on a terminal (`--no-color` to not).') raw = kw.Value( False, isflag=True, @@ -577,7 +580,7 @@ def pick(instances): chosen = pick(_instances(backend)) except UnknownTarget as ex: raise SystemExit(str(ex)) - color = sys.stdout.isatty() and not config.no_color + color = sys.stdout.isatty() and bool(config.color) if config.follow: def listing(): try: diff --git a/tests/test_day2.py b/tests/test_day2.py index 92fbe895..58d032f1 100644 --- a/tests/test_day2.py +++ b/tests/test_day2.py @@ -202,3 +202,11 @@ def test_acquire_yes_before_the_endpoint_still_names_it(): config = AcquireCLI.cli(argv=['--yes', 'qwen']) assert config.yes is True and config.names == ['qwen'] + + +def test_no_color_turns_color_off(): + """kwconf reads a leading `no-` as negation: `--no-color` must negate `color`.""" + from infer_stack.cli.commands_runtime import LogsCLI + + assert LogsCLI.cli(argv=[]).color is True + assert LogsCLI.cli(argv=['--no-color']).color is False diff --git a/tests/test_log_filter.py b/tests/test_log_filter.py index 281ace38..d2a06a51 100644 --- a/tests/test_log_filter.py +++ b/tests/test_log_filter.py @@ -268,7 +268,7 @@ def _follow_logs(monkeypatch, lines, *, no_color, tty=True): from infer_stack.leasing import instances as instances_mod from infer_stack.leasing.instances import Instance - config = SimpleNamespace(follow=True, raw=False, no_color=no_color, tail=None, + config = SimpleNamespace(follow=True, raw=False, color=not no_color, tail=None, timestamps=False, services=None) class Backend: From ce48cc48f48ed9ac8c5fa630fc33903afc2df9f9 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:46:37 -0400 Subject: [PATCH 042/141] UX: Open WebUI runs as its data directory's owner As root, Open WebUI left files in the data root that only root could delete: removing a data root (an operator's, or the e2e's cleanup) failed with permission denied. It now runs as the owner of its data directory, with a managed session key (WEBUI_SECRET_KEY in the gateway's .env, so a recreate no longer signs everyone out) and its static assets under its data (the image's own directory is not writable as a user). A missing bind source is made by infer-stack before `docker compose up`, as that user. A directory a root container already wrote to keeps running as root, and the log says how to chown it. Verified live on compose: the container runs as the user, the UI answers, and the data root is removed without sudo. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 7 ++-- infer_stack/leasing/compose.py | 24 ++++++++++++- infer_stack/leasing/gateway.py | 63 ++++++++++++++++++++++++++++++++++ tests/test_leasing_compose.py | 20 +++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index f07df172..83dfc74c 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -189,9 +189,12 @@ Seed findings, already known: - [x] `infer-stack logs -f qwen` followed every instance: a kwconf flag took the next word as its value. Fixed for every command (P2). - [ ] `status` shows STALE during an apply instead of "apply in progress". -- [ ] Open WebUI writes its data directory as root: removing a data root +- [x] Open WebUI writes its data directory as root: removing a data root (the e2e's cleanup, or an operator's `rm -rf`) fails with permission - denied. Found by `dev/kubeai_e2e.sh` 2026-09-26. + denied. Found by `dev/kubeai_e2e.sh` 2026-09-26; fixed: it runs as the + directory's owner (a directory root already wrote keeps root, and the + log says how to `chown` it). Engine caches (vLLM as root) are the same + shape; not yet looked at. - [x] A refused acquire always ends with "free a GPU first — …", also when the reason is a render refusal (a served-name collision) or the backend is kubeai, where no GPU is ours to free. Found 2026-09-26; diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 80ce7110..c2659e1a 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -579,6 +579,7 @@ def render_compose( route_registry: dict[str, Any] | None = None, dynamic_routing: bool = False, upstream_routes: list[dict[str, Any]] | None = None, + ui_run_as: str | None = None, ) -> RenderedCompose: """Render a compose project for the placed deployments. @@ -667,6 +668,7 @@ def render_compose( reverse_proxy_config=reverse_proxy_config, aux_dir=aux_dir, catalog=catalog, route_registry=route_registry, dynamic_routing=dynamic_routing, upstream_routes=upstream_routes, + ui_run_as=ui_run_as, ) services.update(front.services) litellm_config = front.litellm_config @@ -1414,6 +1416,17 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A # The DB secret must exist before rendering, so docker compose # --env-file can interpolate ${LITELLM_DB_PASSWORD} at apply time. self.gateway.db_password() + ui_run_as = None + if self.ui: + from .gateway import open_webui_run_as + + self.gateway.webui_secret() # interpolated at apply, likewise + ui_run_as, why = open_webui_run_as(self.state['open_webui']) + if ui_run_as is None and not getattr(self, '_ui_root_noted', False): + from .._log import logger + + self._ui_root_noted = True + logger.info('Open WebUI runs as root: {}', why) route_registry = None if self.litellm and not self.dynamic_routing: # Unconditional in static-superset mode: `self.catalog` may be None; @@ -1431,7 +1444,7 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A reverse_proxy_config=self.reverse_proxy_config, aux_dir=self.state_dir, project=self.project, catalog=self.catalog, route_registry=route_registry, dynamic_routing=self.dynamic_routing, - upstream_routes=self.upstream_routes, + upstream_routes=self.upstream_routes, ui_run_as=ui_run_as, ) addresses = None if self.network is not None: @@ -1782,6 +1795,15 @@ def apply(self) -> bool: # A render from before fingerprints: re-render (any mutation) first. logger.warning('apply: the render predates fingerprints; re-render, then apply') return False + # A service that runs as a user needs its bind-mount sources made by + # us, as that user: Docker would make a missing one as root. + for svc in services.values(): + if not svc.get('user'): + continue + for volume in svc.get('volumes') or []: + source = str(volume).split(':', 1)[0] + if source.startswith('/') and not Path(source).exists(): + Path(source).mkdir(parents=True, exist_ok=True) dynamic = bool(self.litellm and self.dynamic_routing) outcome = self.selective_apply( services, fingerprints, diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 53689755..a73dd7de 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -68,6 +68,7 @@ POSTGRES_DB_NAME = 'litellm' POSTGRES_DB_USER = 'litellm' DB_PASSWORD_ENV = 'LITELLM_DB_PASSWORD' # managed secret in the sidecar .env +WEBUI_SECRET_ENV = 'WEBUI_SECRET_KEY' # Open WebUI's session key, likewise # Marks a LiteLLM route as infer-stack-managed, so reconcile only ever deletes # routes it created (never a model added by hand through the UI/admin API). ROUTE_ID_PREFIX = 'isr-' @@ -571,9 +572,15 @@ def _open_webui_service( openai_urls: list[str] | None = None, ollama_urls: list[str] | None = None, depends_on: list[str] | None = None, + run_as: str | None = None, ) -> dict[str, Any]: """A managed Open WebUI pointed at whatever front door is available. + ``run_as`` (``uid:gid``, see :func:`open_webui_run_as`) runs it as the + owner of its data directory, so what it writes there stays that owner's; + None keeps the image's default user (root). + + Open WebUI holds two independent kinds of connection, wired here from the rendered services: @@ -610,7 +617,16 @@ def _open_webui_service( # Single-user workstation default; the port shouldn't be exposed # publicly. Tracked as a knob in dev/leasing-followups.md. 'WEBUI_AUTH': 'False', + # A managed secret (the .env, like the master key): without it the + # image writes a generated one into its own directory, which a + # non-root user cannot, and a recreate would sign everyone out. + WEBUI_SECRET_ENV: '${' + WEBUI_SECRET_ENV + '}', } + if run_as: + # Its bundled static assets are copied at startup; into the data + # directory, which this user can write, not the image's own. + env['STATIC_DIR'] = '/app/backend/data/static' + env['HOME'] = '/tmp' if openai_urls: env['ENABLE_OPENAI_API'] = 'True' if len(openai_urls) == 1: @@ -636,11 +652,48 @@ def _open_webui_service( 'restart': 'unless-stopped', 'labels': {ENGINE_LABEL: 'open-webui'}, } + if run_as: + service['user'] = run_as if depends_on: service['depends_on'] = sorted(depends_on) return service +def open_webui_run_as(data_path: str | Path) -> tuple[str | None, str]: + """``(uid:gid or None, why)``: who Open WebUI should run as. + + Its data directory's owner, so a data root stays removable by whoever + owns it (as root, the container left files only root could delete). A + directory not made yet will be made by this process, so its user. None, + the image default (root), when the directory is root's, or when it or + anything directly in it belongs to someone else: a data directory written + by a root container before, whose files Open WebUI could no longer open. + + >>> import os, tempfile + >>> d = tempfile.mkdtemp() + >>> open_webui_run_as(os.path.join(d, 'open-webui'))[0] == f'{os.getuid()}:{os.stat(d).st_gid}' + True + """ + import os + + path = Path(data_path) + if not path.exists(): + parent = next((p for p in path.parents if p.exists()), Path('/')) + return f'{os.getuid()}:{parent.stat().st_gid}', 'new directory' + owner = path.stat() + if owner.st_uid == 0: + return None, f'{path} is root\'s' + try: + foreign = [p.name for p in path.iterdir() if p.lstat().st_uid != owner.st_uid] + except OSError: + foreign = ['?'] + if foreign: + return None, (f'{path} holds files another user owns ({", ".join(foreign[:3])}); ' + f'`sudo chown -R {owner.st_uid}:{owner.st_gid} {path}` lets Open ' + 'WebUI run as its owner') + return f'{owner.st_uid}:{owner.st_gid}', 'the directory\'s owner' + + def _nginx_conf(*, litellm: bool, ui: bool) -> str: """A minimal HTTP reverse-proxy conf: one origin, path-routed. @@ -776,6 +829,7 @@ def render_front_door( route_registry: dict[str, Any] | None, dynamic_routing: bool, upstream_routes: list[dict[str, Any]] | None = None, + ui_run_as: str | None = None, ) -> FrontDoor: """Render the gateway, its database, Open WebUI and the reverse proxy. @@ -893,6 +947,7 @@ def render_front_door( openai_urls=openai_urls, ollama_urls=ollama_native_urls, depends_on=ui_depends, + run_as=ui_run_as, ) # Optional single-port HTTP reverse proxy fronting the gateway (+ UI). Needs @@ -1054,6 +1109,14 @@ def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: return None self._sleep(2.0) + def webui_secret(self) -> str: + """Open WebUI's managed session key (the .env), made on first use.""" + existing = parse_env_file(self._env_path) + key = ensure_secret(existing, WEBUI_SECRET_ENV) + if key != existing.get(WEBUI_SECRET_ENV): + write_env_file(self._env_path, {WEBUI_SECRET_ENV: key}) + return key + def db_password(self) -> str: """The managed Postgres password for LiteLLM's model store. diff --git a/tests/test_leasing_compose.py b/tests/test_leasing_compose.py index 25173891..8231dc74 100644 --- a/tests/test_leasing_compose.py +++ b/tests/test_leasing_compose.py @@ -1850,3 +1850,23 @@ def test_default_docker_run_can_redirect_stderr_lines(): out = _default_docker_run(['sh', '-c', 'echo out; echo err1 >&2; echo err2 >&2'], timeout=10, stderr_lines=seen.append) assert out.strip() == 'out' and seen == ['err1', 'err2'] + + +def test_open_webui_runs_as_its_data_directorys_owner(tmp_path): + """As root it left files only root could delete (the data root could not + be removed without sudo); now it runs as the directory's owner, with a + managed session key and its static assets inside its data.""" + import os + + be = ComposeBackend( + state_dir=tmp_path / 'state', inventory=simulate_inventory('1x80'), + run=FakeDocker(), http=FakeHttp(tmp_path / 'state'), litellm=True, ui=True, + images={**IMAGES, 'open_webui': 'owui:test'}, ports=PORTS, + state=dict(STATE, open_webui=str(tmp_path / 'open-webui')), + ) + be.converge([], apply=False) + svc = yaml.safe_load(be.compose_file.read_text())['services']['open-webui'] + assert svc['user'] == f'{os.getuid()}:{tmp_path.stat().st_gid}' + assert svc['environment']['STATIC_DIR'] == '/app/backend/data/static' + assert svc['environment']['WEBUI_SECRET_KEY'] == '${WEBUI_SECRET_KEY}' + assert 'WEBUI_SECRET_KEY=' in (tmp_path / 'state' / '.env').read_text() From 977fa4e634cd9aaa845ed1c0ba3627f1eccd52f4 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:47:11 -0400 Subject: [PATCH 043/141] UX: status says "pending" while a change is being applied `status` showed STALE for a deployment recorded live whose engine did not exist yet, including while an apply was still bringing it up. With the ledger's publication marker set (an apply running, or a failed one left for `infer-stack apply`) it now says `pending`; STALE means nothing is pending either. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 4 +++- infer_stack/cli/commands_runtime.py | 24 +++++++++++++++++------- tests/test_day2.py | 3 +++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 83dfc74c..f6ad6cfa 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -188,7 +188,9 @@ Seed findings, already known: `--no-color` works. Found by the README rewrite 2026-09-26. - [x] `infer-stack logs -f qwen` followed every instance: a kwconf flag took the next word as its value. Fixed for every command (P2). -- [ ] `status` shows STALE during an apply instead of "apply in progress". +- [x] `status` shows STALE during an apply instead of "apply in progress". + Fixed: `pending` while the publication marker says the change is not + applied yet; STALE only when nothing is pending. - [x] Open WebUI writes its data directory as root: removing a data root (the e2e's cleanup, or an operator's `rm -rf`) fails with permission denied. Found by `dev/kubeai_e2e.sh` 2026-09-26; fixed: it runs as the diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index f82b3e83..69192289 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -127,7 +127,9 @@ def _leasing_status() -> dict[str, Any]: if not path.exists(): return out try: - leases, deployments = Ledger(SqliteStore(str(path))).status(virtual_expiry=True) + ledger = Ledger(SqliteStore(str(path))) + leases, deployments = ledger.status(virtual_expiry=True) + out['pending'] = ledger.publication_pending() is not None except Exception: # noqa: BLE001 return out active = sum(1 for le in leases if le.state == LeaseState.ACTIVE) @@ -145,7 +147,8 @@ def _leasing_status() -> dict[str, Any]: return out -def _served_models(deployments, backend=None) -> list[tuple[str, str, str, str]]: +def _served_models(deployments, backend=None, *, + pending: bool = False) -> list[tuple[str, str, str, str]]: """``(endpoint, model, engine, health)`` rows for what is serving. Reads the ledger for what *should* be up and the backend's strict @@ -176,9 +179,12 @@ def _served_models(deployments, backend=None) -> list[tuple[str, str, str, str]] elif residency.containers(d.id): # There, but not warm: starting, crashed, or ambiguous. health = residency.containers(d.id)[0].state + elif pending: + # Recorded, not applied yet: an apply is running, or a failed one + # left the change for `infer-stack apply`. + health = 'pending' else: - # The ledger says live and nothing exists. Almost always an apply - # that failed after the record was written. + # The ledger says live, nothing exists, and no change is pending. health = 'STALE' for endpoint in sorted(d.served): payload = d.served.get(endpoint) or {} @@ -197,7 +203,8 @@ def _gather_status(config) -> dict[str, Any]: rendered = getattr(backend, 'rendered_file', None) leasing = _leasing_status() if leasing.get('live_deployments'): - leasing['served'] = _served_models(leasing.pop('live_deployments'), backend) + leasing['served'] = _served_models(leasing.pop('live_deployments'), backend, + pending=bool(leasing.get('pending'))) else: leasing.pop('live_deployments', None) return { @@ -242,6 +249,9 @@ def _served_lines(served: list[tuple[str, str, str, str]]) -> list[str]: if any(r[3] == 'STALE' for r in served): out.append(' STALE = the ledger records this live but nothing is ' 'running for it; `infer-stack apply` or `gc`') + if any(r[3] == 'pending' for r in served): + out.append(' pending = recorded but not applied yet: an apply is running, ' + 'or `infer-stack apply` finishes it') if any(r[3] == 'starting' for r in served): out.append(' starting = up, but not ready yet (loading the model); ' '`infer-stack wait `') @@ -340,8 +350,8 @@ def _print_status_rich(d: dict[str, Any], console) -> None: served_table.add_column('model', overflow='fold') served_table.add_column('engine', style='dim', no_wrap=True) served_table.add_column('health', no_wrap=True) - styles = {'up': 'green', 'starting': 'yellow', 'STALE': 'red', - 'unverified': 'yellow'} + styles = {'up': 'green', 'starting': 'yellow', 'pending': 'yellow', + 'STALE': 'red', 'unverified': 'yellow'} for endpoint, model, engine, health in served: served_table.add_row( endpoint, model, engine, diff --git a/tests/test_day2.py b/tests/test_day2.py index 58d032f1..88bde5f4 100644 --- a/tests/test_day2.py +++ b/tests/test_day2.py @@ -156,6 +156,9 @@ def dep(gid): 'crashing': 'exited', 'gone': 'STALE'} rows = _served_models([dep('up')], _Backend([], residency=None)) assert rows[0][3] == 'unverified' + # Recorded but not applied yet (an apply is running): not STALE. + rows = _served_models([dep('gone')], _Backend([], residency=residency), pending=True) + assert rows[0][3] == 'pending' def test_stack_down_stops_the_backend_on_either_backend(monkeypatch): From 6629c722b31279fa0be66f66df612a9b6a3c79e2 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:48:41 -0400 Subject: [PATCH 044/141] UX: the TUI picks up catalog edits made outside it An edit to catalog.yaml made elsewhere (an editor, `infer-stack catalog endpoint add`) showed only after restarting the TUI, and `r` did not reread it. Each refresh now rereads the catalog when its file changed, a half-saved broken file is reported once rather than every refresh (the last good catalog stays), and `r` rereads it always. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 6 ++++-- infer_stack/tui.py | 23 ++++++++++++++++++++++- tests/test_tui.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index f6ad6cfa..d3db3718 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -178,8 +178,10 @@ The audit covers: Seed findings, already known: -- [ ] `r` does not reload catalogs, and edits to a catalog file are not - picked up until restart. +- [x] `r` does not reload catalogs, and edits to a catalog file are not + picked up until restart. Fixed: each refresh rereads a changed catalog + (a broken save is reported once), and `r` rereads it always. Checked + live: `catalog endpoint add` in another shell shows in the TUI. - [x] `infer-stack logs` should accept a container name, a prefixed name or an endpoint name. Done with P2: an instance name, a container id prefix, a deployment id or an endpoint alias. diff --git a/infer_stack/tui.py b/infer_stack/tui.py index dc1b6687..033032d3 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -895,6 +895,7 @@ def __init__( self._ps_rows_cache: list[tuple] = [] self._gpus_rows_cache: list[tuple] = [] self.catalog_path = Path(catalog_path).expanduser() if catalog_path else None + self._catalog_seen = self._catalog_stamp() # the catalog given is this one self._http = http self._proc_factory = proc_factory or self._default_proc_factory() self._endpoint_names: list[str] = [] @@ -1567,9 +1568,26 @@ def _update_catalog_help(self) -> None: 'open it in Open WebUI.' ) + def _catalog_stamp(self): + """The catalog file's (mtime, size), or None when there is none.""" + try: + st = self.catalog_path.stat() if self.catalog_path else None + except OSError: + return None + return None if st is None else (st.st_mtime_ns, st.st_size) + + def _reload_catalog_if_changed(self) -> None: + """Pick up an edit made outside the TUI (checked on each refresh).""" + stamp = self._catalog_stamp() + if stamp is not None and stamp != getattr(self, '_catalog_seen', None): + self._reload_catalog() + def _reload_catalog(self) -> None: if not self.catalog_path or not self.catalog_path.exists(): return + # Remember this version even if it fails to load: a half-saved edit is + # reported once, not on every refresh until it is fixed. + self._catalog_seen = self._catalog_stamp() try: from .leasing import Catalog self.catalog = Catalog.load(self.catalog_path) @@ -1732,12 +1750,15 @@ def _refresh_bg(self) -> None: def action_refresh_now(self) -> None: """The `r` key: a refresh the user asked for (the timer calls - ``action_refresh`` directly, and must not fill the TUI log).""" + ``action_refresh`` directly, and must not fill the TUI log). It also + rereads the catalog, edited or not.""" self._cli(cli.command('status')) + self._reload_catalog() self.action_refresh() def action_refresh(self) -> None: self._sync_pane_state() # capture pane state on the UI thread first + self._reload_catalog_if_changed() self._refresh_bg() def _update_summary(self, leases, deployments, observed) -> None: diff --git a/tests/test_tui.py b/tests/test_tui.py index ecccb8b1..bf316253 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -2160,3 +2160,40 @@ async def scenario(): shown, text = seen['during'] assert shown and f'acquiring {name}' in text and 's' in text assert seen['after'] is False + + +def test_an_edit_made_outside_the_tui_appears_on_the_next_refresh(tmp_path): + """The catalog file is reread when it changes (and on `r`); a broken save + is reported once, not on every refresh.""" + import copy + + import yaml + + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + catalog_path = tmp_path / 'catalog.yaml' + catalog_path.write_text(yaml.safe_dump(CATALOG)) + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None, + catalog_path=str(catalog_path)) + async with app.run_test() as pilot: + await pilot.pause() + edited = copy.deepcopy(CATALOG) + edited['endpoints']['qwen-extra'] = dict(edited['endpoints']['qwen-fast']) + catalog_path.write_text(yaml.safe_dump(edited)) + app.action_refresh() + await pilot.pause() + assert 'qwen-extra' in app._endpoint_names + + refused = [] + app._refuse = lambda msg, **kw: refused.append(msg) + catalog_path.write_text('endpoints: [not, a, mapping\n') + app.action_refresh() + app.action_refresh() + assert len(refused) == 1 and 'catalog reload failed' in refused[0] + assert 'qwen-extra' in app._endpoint_names # the last good one stays + + _run(scenario) From 0fc3f5db23a99174df8c5c80674c32b425a22d33 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 13:57:43 -0400 Subject: [PATCH 045/141] UX: help and narration say what is true on both backends Audit pass 1 over every command's --help and the examples in them: - five one-line summaries ended mid-sentence in `help tree`; `leases` said "deployment deployments"; - --litellm, --ui and --yes said "(compose backend)"; acquire, apply, --apply and render described only a compose project, and `render` on kubeai printed "(backend has no on-disk project)" (it names the Models file now; JSON keeps compose_file and adds rendered_file); - `apply`'s help still called `stack up` the raw hatch; - every .env write printed `Write .env to ...` on stdout, into --json output too (now a debug log); - the render step logged "not applied; `infer-stack apply` ..." in every acquire, just before the apply. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 16 ++++ infer_stack/backends/kubeai.py | 9 +-- infer_stack/cli/commands_leasing.py | 117 ++++++++++++++++------------ infer_stack/cli/commands_runtime.py | 6 +- infer_stack/env_utils.py | 6 +- infer_stack/leasing/compose.py | 8 +- tests/test_cli_leasing.py | 3 +- 7 files changed, 99 insertions(+), 66 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index d3db3718..4b7eaa16 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -204,6 +204,22 @@ Seed findings, already known: backend is kubeai, where no GPU is ours to free. Found 2026-09-26; fixed: `PlacementError.capacity` says whether room would have helped. +Audit pass 1 (2026-09-26): every command's `--help` collected +(`infer-stack help tree`, 65 leaves) and every example in them run on a +dry-run root. + +- [x] Five one-line summaries ended mid-sentence in `help tree` (`measure`, + `routes prune`, `status`, `tui`, `wait`); `leases` said "deployment + deployments". +- [x] `--litellm`, `--ui` and `--yes` said "(compose backend)", and + `acquire`, `apply`, `--apply` and `render` described only a compose + project; `render` on kubeai printed "(backend has no on-disk project)". + `apply`'s help still called `stack up` the raw hatch. +- [x] Every `.env` write printed `Write .env to …` on stdout, into `--json` + output too. +- [x] The render step logged "(not applied; `infer-stack apply` …)" in + every acquire, right before the apply it said had not happened. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 72fb990b..8a20627c 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -778,12 +778,9 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N self.last_planned_digest = self._combined_digest( models_digest, self.gateway.last_planned_digest) if not apply: - logger.info( - 'rendered {} Model(s) to {} (not applied; ' - '`infer-stack apply` to converge the cluster)', - len(rendered.models), - self.models_file, - ) + # The caller applies next, or (--no-apply, render) says how. + logger.info('rendered {} Model(s) to {}', + len(rendered.models), self.models_file) return None self.apply() return None diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 8dce849d..15628961 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -8,7 +8,7 @@ infer-stack run --endpoint qwen-coder -- python my_node.py infer-stack release --env-file is.env infer-stack acquire qwen-coder # standing service (no --ttl) - infer-stack leases # status of leases + deployment deployments + infer-stack leases # status of leases + deployments Until the Compose/KubeAI backends land, the default ``--backend null`` is a dry-run: the ledger does all the real bookkeeping (coalescing, demand, TTL) but @@ -498,10 +498,17 @@ def _public_descriptor(descriptor: dict) -> dict: def _compose_file_path(controller) -> str | None: + """The compose file, on the compose backend only (JSON's ``compose_file``).""" path = getattr(controller.backend, 'compose_file', None) return str(path) if path else None +def _rendered_path(controller) -> str | None: + """The file a render writes, on any backend: compose project or Models.""" + path = getattr(controller.backend, 'rendered_file', None) + return str(path) if path else None + + def _gpu_where(gpus) -> str: """Human label for a deployment's GPU assignment (or its absence).""" if gpus is None: @@ -527,6 +534,7 @@ def _emit_staged(config, controller, outcome) -> int: 'applied': False, 'descriptor': _public_descriptor(descriptor), 'compose_file': _compose_file_path(controller), + 'rendered_file': _rendered_path(controller), 'placement': [ {'deployment': g.id, 'served': sorted(g.served), 'gpus': assignments.get(g.id)} @@ -542,10 +550,11 @@ def _emit_staged(config, controller, outcome) -> int: eps = ', '.join(sorted(g.served)) or g.id where = _gpu_where(assignments.get(g.id)) if local else 'cluster-scheduled' print(f' {eps}: {where} ({g.id})') - path = _compose_file_path(controller) - if path: - print(f' compose: {path}') + rendered = _rendered_path(controller) + if rendered: + print(f' rendered: {rendered}') print(' apply: infer-stack apply # bring the staged set up') + path = _compose_file_path(controller) if path: # The file carries `name: infer-stack`, so plain docker works too. print(f' ...or: docker compose -f {path} up -d') @@ -777,16 +786,16 @@ class _LeasingCommonMixin(_PathOverridesMixin, _AllowedGpusMixin, _DisplayGpuMix None, isflag=True, help='Render the LiteLLM gateway — one OpenAI base_url fronting every ' - 'endpoint alias (compose backend). On by default; use --no-litellm for ' + 'endpoint alias. On by default; use --no-litellm for ' 'a lean stack where Open WebUI talks to the upstreams (e.g. an Ollama ' 'daemon) directly. Overrides `config set litellm …`.', ) ui = kw.Value( None, isflag=True, - help='Render a managed Open WebUI in front of the gateway (compose ' - 'backend). On by default; use --no-ui to skip. Overrides ' - '`config set ui …`.', + help='Render a managed Open WebUI in front of the gateway (on this host; ' + 'not with kubeai_gateway cluster). On by default; use --no-ui to skip. ' + 'Overrides `config set ui …`.', ) reverse_proxy = kw.Value( None, @@ -826,8 +835,9 @@ class _ApprovalMixin(_LeasingCommonMixin): ) yes = kw.Value( False, isflag=True, alias=['y'], - help='Apply compose changes without showing the diff / prompting ' - '(compose backend). Implied when stdout is not a terminal.', + help='Apply the rendered changes (the compose project, or the KubeAI ' + 'Models and gateway) without showing the diff / prompting. Implied when ' + 'stdout is not a terminal.', ) @@ -852,16 +862,17 @@ class _AcquireFlagsMixin(_LeasingCommonMixin): '(up to --timeout) instead of failing fast. Each retry sweeps the ' 'ledger, so a crashed job\'s TTL-expired lease is reclaimed while ' 'waiting. Intended for batch/pipeline fan-out; interactive use ' - 'defaults off (fail fast with a clear "no GPU" error).', + 'defaults off (fail fast with a clear "no GPU" error). On kubeai the ' + 'cluster is the queue: admitted at once, and a Pending pod waits.', ) apply = kw.Value( True, isflag=True, - help='Apply the render (docker compose up). Use --no-apply to *stage* ' - 'only: declare the lease and write the on-disk compose project + ' - 'placement WITHOUT starting it, then `infer-stack apply` to bring it up ' - '(compose backend). --no-apply implies no readiness wait and no diff ' - 'prompt; `release` discards a staged lease.', + help='Apply the render (bring it up). Use --no-apply to *stage* only: ' + 'declare the lease and write the rendered state (the compose project, ' + 'or the KubeAI Models) WITHOUT starting it, then `infer-stack apply` to ' + 'bring it up. --no-apply implies no readiness wait and no diff prompt; ' + '`release` discards a staged lease.', ) timeout = kw.Value(600, type=float, help='Readiness wait timeout (s).') interval = kw.Value(5, type=float, help='Readiness poll interval (s).') @@ -870,8 +881,9 @@ class _AcquireFlagsMixin(_LeasingCommonMixin): ) yes = kw.Value( False, isflag=True, alias=['y'], - help='Apply compose changes without showing the diff / prompting ' - '(compose backend). Implied when stdout is not a terminal.', + help='Apply the rendered changes (the compose project, or the KubeAI ' + 'Models and gateway) without showing the diff / prompting. Implied when ' + 'stdout is not a terminal.', ) json = kw.Value(False, isflag=True, help='Emit JSON instead of text.') @@ -885,11 +897,11 @@ class AcquireCLI(_AcquireFlagsMixin): """Acquire a lease on one or more endpoints/bundles, bring them up, wait. ``acquire NAME…`` is the everyday verb. It takes a lease on each endpoint or - bundle, renders the compose project (the LiteLLM gateway + one container per - model + a managed Open WebUI), brings it up, and blocks until every endpoint - is ready. Run it again with more names to add models side by side — the - gateway and UI stay put. Placement, ``docker compose``, and readiness are - narrated on stderr. + bundle, renders what serves it (the LiteLLM gateway, one engine per model: + a container on compose, a Model on kubeai, and a managed Open WebUI), + brings it up, and blocks until every endpoint is ready. Run it again with + more names to add models side by side — the gateway and UI stay put. + Placement, the apply, and readiness are narrated on stderr. With no ``--ttl`` the lease is infinite — a standing service you tear down explicitly (``release`` / ``evict``). Pass ``--ttl`` (e.g. ``2h``, ``30m``) @@ -900,13 +912,13 @@ class AcquireCLI(_AcquireFlagsMixin): when done. (For a one-shot "acquire, run a command, release", use ``infer-stack run`` instead.) - The work is render (write the on-disk compose project) then apply (``docker - compose up``). ``--no-apply`` does just the render so you can see what - would run before pulling the trigger (then ``infer-stack apply``). - ``--no-wait`` applies but returns immediately so several models load in - parallel (``wait`` for them later). ``--no-ui`` skips Open WebUI. On a - terminal you are shown the compose diff and asked before applying; ``--yes`` - skips that prompt (and it is skipped automatically off a TTY). + The work is render (write the rendered state to disk) then apply (bring + it up). ``--no-apply`` does just the render so you can see what would run + before pulling the trigger (then ``infer-stack apply``). ``--no-wait`` + applies but returns immediately so several models load in parallel + (``wait`` for them later). ``--no-ui`` skips Open WebUI. On a terminal you + are shown the diff and asked before anything is committed; ``--yes`` skips + that prompt (and it is skipped automatically off a TTY). """ __command__ = 'acquire' @@ -968,10 +980,11 @@ def main(cls, argv=True, **kwargs): class RenderCLI(_LeasingCommonMixin): - """Write the on-disk compose project for the current desired set — no up. + """Write the rendered state for the current desired set, without bringing it up. - Lease-free and idempotent: ``render`` re-materializes the manifest (compose - file + gateway config + GPU placement) from whatever is *already* declared, + Lease-free and idempotent: ``render`` re-materializes what would run (the + compose project and gateway config with GPU placement, or the KubeAI Models + and gateway) from whatever is *already* declared, WITHOUT starting anything — to inspect what would run, or refresh a file you touched by hand. It creates no lease; to stage a *new* endpoint use ``acquire --no-apply`` (which declares it too). Apply with ``infer-stack apply``. @@ -986,16 +999,17 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config, interactive=False) rec = controller.reconcile(apply=False) - path = _compose_file_path(controller) + path = _rendered_path(controller) if config.json: print(json.dumps({ 'applied': False, - 'compose_file': path, + 'compose_file': _compose_file_path(controller), + 'rendered_file': path, 'placement': rec.assignments, 'unplaced': rec.unplaced, }, indent=2)) return 0 - print(f'rendered desired set -> {path or "(backend has no on-disk project)"}') + print(f'rendered desired set -> {path or "(this backend renders no file)"}') for gid, gpus in sorted(rec.assignments.items()): print(f' {gid}: {_gpu_where(gpus)}') for err in rec.placement_errors: @@ -1011,11 +1025,10 @@ class ApplyCLI(_ApprovalMixin): declared (by ``acquire``) onto the backend. It is the *trigger* for a staged ``acquire --no-apply`` and the re-sync button after a manual edit or a backend hiccup; idempotent (a second apply with nothing changed is a - no-op). On a terminal it shows the compose diff and asks (``--yes`` skips). - The rendered file carries ``name: infer-stack``, so ``docker compose -f - up -d`` is an exact equivalent. (``infer-stack stack up`` is the - lower-level "run exactly what is on disk" hatch; ``apply`` re-renders from - intent first.) + no-op). On a terminal it shows the diff and asks (``--yes`` skips). + ``infer-stack stack up`` is the same command. On compose, ``infer-stack + stack compose -- up -d`` runs exactly what is on disk without re-rendering + from intent first. """ __command__ = 'apply' @@ -1478,8 +1491,7 @@ def _list_only(orphans): class WaitCLI(_LeasingCommonMixin): - """Block until served endpoints are ready — the companion to ``acquire - --no-wait``. + """Block until served endpoints are ready (the companion to ``acquire --no-wait``). Fan out, then wait: ``acquire --no-wait smol17b-1`` + ``acquire --no-wait smol135-1`` kick both deployments off in parallel (each converges and starts @@ -1548,8 +1560,10 @@ def main(cls, argv=True, **kwargs): class MeasureCLI(_LeasingCommonMixin): - """Measure an endpoint's real per-GPU VRAM requirement from the engine's - own memory-profiling log (docs/planning/vram-aware-placement.md §3). + """Measure an endpoint's real per-GPU VRAM requirement from its engine's log. + + Reads the engine's own memory-profiling log + (docs/planning/vram-aware-placement.md §3). Parses vLLM's profiling breakdown (weights + non-torch + activation peak) — deliberately NOT ``nvidia-smi memory.used``, which only reflects the @@ -1733,8 +1747,9 @@ def main(cls, argv=True, **kwargs): class TuiCLI(_LeasingCommonMixin): - """Launch the Textual TUI: a live monitor of the stack with controls to - serve / release / evict models. + """Launch the Textual TUI: a live monitor of the stack, with controls. + + Serve, release and evict models, and follow their logs. Mostly a monitor — the lease + deployment tables (desired state vs running, GPUs) refresh live — with key-bound controls: ``s`` serve, ``d`` release, ``a`` @@ -2040,7 +2055,7 @@ def state_style(state) -> str: class LeasesCLI(_LeasingCommonMixin): - """Show current leases and deployment deployments (the leasing-model status). + """Show current leases and deployments (the leasing-model status). Two tables. **leases** are who asked for what (id, owner, state, ttl, endpoints). **deployments** are the actual deployments behind them — one deployment is @@ -2570,8 +2585,10 @@ def main(cls, argv=True, **kwargs): class RoutesPruneCLI(_ApprovalMixin): - """Forget stale routes: rewrite the registry to *invoking catalog ∪ live*, - then converge (one accepted gateway recreate). + """Forget stale routes: keep only the invoking catalog's and the live ones. + + Rewrites the registry to *invoking catalog ∪ live*, then converges (one + accepted gateway recreate). The registry is append-only by design (that is what keeps the gateway config byte-stable), so pruning is the explicit, operator-driven "forget" verb — diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 69192289..8cbb880f 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -380,8 +380,10 @@ def _print_status_rich(d: dict[str, Any], console) -> None: class StatusCLI(_PathOverridesMixin): - """Holistic overview: where things live, the backend, and a leasing summary - (active leases / live deployments), with pointers to dig deeper.""" + """Holistic overview: where things live, the backend, and what is serving. + + A leasing summary (active leases, live deployments) with each endpoint's + health, and pointers to dig deeper.""" __command__ = 'status' catalog = kw.Value( diff --git a/infer_stack/env_utils.py b/infer_stack/env_utils.py index eb6eb580..89bb8f29 100644 --- a/infer_stack/env_utils.py +++ b/infer_stack/env_utils.py @@ -93,7 +93,11 @@ def write_env_file(path: Path, values: dict[str, str]) -> None: if text and not text.endswith('\n'): text += '\n' - print(f'Write .env to {path}') + from ._log import logger + + # Debug, not stdout: a command's output (its --json above all) is not the + # place for an internal file write; `env set` confirms its own. + logger.debug('wrote {}', path) # Atomic replace: Docker Compose (or a fingerprint) must never read a # half-written file. import os diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index c2659e1a..ebf5c245 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1748,11 +1748,9 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N }) services = rendered.compose.get('services') if not apply: - logger.info( - 'rendered {} service(s) to {} (not applied; ' - '`infer-stack apply` to bring it up)', - len(services or {}), self.compose_file, - ) + # The caller applies next, or (--no-apply, render) says how. + logger.info('rendered {} service(s) to {}', + len(services or {}), self.compose_file) return plan # Apply OUTSIDE the converge (render) lock: the controller coalesces and # serializes applies via its own apply-lock, so re-taking the render lock diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 91bb3e03..35c7b2df 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -967,8 +967,7 @@ def test_routes_prune_drops_stale(tmp_path, monkeypatch, capsys): capsys.readouterr() rc = RoutesPruneCLI.main(argv=['--ledger', db, '--yes', '--json']) assert rc == 0 - raw = capsys.readouterr().out # `Write .env to ...` may precede the JSON - out = json.loads(raw[raw.index('{'):]) + out = json.loads(capsys.readouterr().out) # --json output is only the JSON assert out['dropped'] == ['beta'] assert out['kept'] == ['alpha'] From cc2907d51e26f9c197ee37470d87637630b999e3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:05:40 -0400 Subject: [PATCH 046/141] UX: failures name their cause, not a traceback - A refused runtime command (the gateway's port taken, a daemon down) ended in a CalledProcessError traceback. The CLI now prints the command, the runtime's own last lines, and a hint (a port in use names the port). Docker's stderr still reaches the terminal live, through a pipe of our own that keeps its tail for the message. - `evict ` says whether each name is held by a lease or unknown, instead of "no idle deployment for" followed by "nothing to evict". - `test` on a gateway that is not up says so, not a urllib3 dump. - `logs ` with nothing running says "nothing is running". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 8 +++++ infer_stack/cli/__init__.py | 54 +++++++++++++++++++++++++++++ infer_stack/cli/commands_leasing.py | 21 ++++++++--- infer_stack/leasing/compose.py | 51 ++++++++++++++++++++++++++- infer_stack/leasing/instances.py | 5 +-- tests/test_cli_leasing.py | 2 +- 6 files changed, 132 insertions(+), 9 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 4b7eaa16..30a10392 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -219,6 +219,14 @@ dry-run root. output too. - [x] The render step logged "(not applied; `infer-stack apply` …)" in every acquire, right before the apply it said had not happened. +- [x] A runtime refusal (the gateway's port taken, a daemon down) ended in + a Python traceback of `CalledProcessError`. Now the command, docker's + own last lines, and a hint (a port in use names the port); docker's + stderr is still shown live, through a pipe that keeps its tail. +- [x] `evict ` printed "no idle deployment for" then "nothing to + evict"; it now says whether each name is held by a lease or unknown. + `test` on a stopped gateway printed a urllib3 dump; now "nothing is + listening there". `logs` said "running: nothing". ### 10. [ ] Handover diff --git a/infer_stack/cli/__init__.py b/infer_stack/cli/__init__.py index eb00a120..5e3d44a3 100644 --- a/infer_stack/cli/__init__.py +++ b/infer_stack/cli/__init__.py @@ -165,8 +165,55 @@ def __getattr__(name: str): raise AttributeError(f'module {__name__!r} has no attribute {name!r}') +def runtime_failure(exc) -> str: + """A failed ``docker`` / ``kubectl`` command, for a person, not a traceback. + + The command (without its long path arguments), the runtime's own last + words, and a hint for the failures with a known cause. + + >>> import subprocess + >>> ex = subprocess.CalledProcessError(1, ['docker', 'compose', '-p', 'x', 'up', '-d'], + ... stderr='Error: Bind for :::14042 failed: port is already allocated') + >>> print(runtime_failure(ex)) + `docker compose up -d` failed: + Error: Bind for :::14042 failed: port is already allocated + port 14042 is in use by something else (`ss -ltnp | grep :14042` shows what); stop it, then `infer-stack apply` + >>> ex.stderr = 'failed to bind host port 0.0.0.0:13000/tcp: address already in use' + >>> runtime_failure(ex).splitlines()[-1].split(' (')[0] + ' port 13000 is in use by something else' + """ + import re + + cmd = [str(a) for a in (exc.cmd if isinstance(exc.cmd, (list, tuple)) else [exc.cmd])] + # Drop option values that are paths or project names: the verb is what reads. + shown, skip = [], False + for arg in cmd: + if skip: + skip = False + continue + if arg in ('-f', '-p', '--env-file', '-n', '--namespace'): + skip = True + continue + shown.append(arg) + text = '\n'.join(filter(None, [exc.stderr or '', exc.output or ''])) + last = [ln.strip() for ln in str(text).splitlines() if ln.strip()][-3:] + lines = [f'`{" ".join(shown)}` failed:'] + [f' {ln}' for ln in last] + port = re.search(r'Bind for \S*?:(\d+) failed: port is already allocated' + r'|:(\d+)(?:/tcp)?: (?:bind: )?address already in use', str(text)) + if port: + number = port.group(1) or port.group(2) + lines.append(f' port {number} is in use by something else (`ss -ltnp | grep ' + f':{number}` shows what); stop it, then `infer-stack apply`') + elif shown and shown[0] in ('docker', 'kubectl'): + lines.append(' `infer-stack doctor` checks what this backend needs') + return '\n'.join(lines) + + def main(argv=None) -> int: + import subprocess + from ..leasing import LeaseLockError + from ..leasing.backend import BackendTimeout try: rv = ManageCLI.main(argv=argv) @@ -174,6 +221,13 @@ def main(argv=None) -> int: # A mutating verb (acquire/release/gc/evict/apply) could not get the # cross-process lock; surface the actionable diagnosis, not a traceback. raise SystemExit(str(exc)) + except subprocess.CalledProcessError as exc: + # The runtime refused (a port in use, a daemon down): its words and a + # hint, not our stack. The controller already rolled back what it had + # committed, or left the change pending for `infer-stack apply`. + raise SystemExit(runtime_failure(exc)) + except BackendTimeout as exc: + raise SystemExit(str(exc)) return int(rv) if rv is not None else 0 diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 15628961..e63ec468 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1251,17 +1251,22 @@ def main(cls, argv=True, **kwargs): if missing: # Diagnostic, not payload: stdout must stay pure JSON # under --json (and stay grep-able human output without). - print( - f'no idle deployment for: {", ".join(missing)}', - file=sys.stderr, - ) + _, rows = controller.ledger.status(virtual_expiry=True) + held = {name for g in rows if g.state == DeploymentState.LIVE + for name in (g.id, *g.served)} + for name in missing: + why = ('is live: a lease holds it; release it first' + if name in held else + 'is not a deployment or served endpoint ' + '(`infer-stack leases` lists them)') + print(f'evict: {name} {why}', file=sys.stderr) if not targets: if config.json: print(json.dumps( {'evicted': [], 'torn_down': [], 'missing': missing}, indent=2)) else: - print('nothing to evict') + print('nothing evicted') return 0 outcome = controller.evict(targets) except ConvergeAborted: @@ -2309,6 +2314,12 @@ def main(cls, argv=True, **kwargs): json=payload, timeout=float(config.timeout), ) + except requests.exceptions.ConnectionError: + return _test_fail(config, base_url, + 'nothing is listening there (the gateway is not up)') + except requests.exceptions.Timeout: + return _test_fail(config, base_url, + f'no answer within {float(config.timeout):g}s') except requests.exceptions.RequestException as ex: return _test_fail(config, base_url, f'not reachable: {ex}') dt = time.monotonic() - t0 diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index ebf5c245..ff1f25bd 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -815,11 +815,20 @@ def _default_docker_run( from .backend import BackendTimeout bound = _docker_timeout(args) if timeout is None else timeout + # With no stderr handler, stderr still reaches the terminal as it is + # written (docker's progress), through a pipe of our own so its last lines + # can also go into the error: "port is already allocated" belongs in the + # message, not only scrolled past above it. + tee = None + if stderr_lines is None: + tee = _StderrTee() proc = subprocess.Popen( args, stdout=subprocess.PIPE, text=True, start_new_session=True, env=docker_environment(), - stderr=subprocess.PIPE if stderr_lines is not None else None, + stderr=subprocess.PIPE if tee is None else tee.write_end, ) + if tee is not None: + tee.start() query = _read_only(args) if query: _RUNNING_QUERIES.add(proc) @@ -853,11 +862,51 @@ def kill_group(): for line in (err or '').splitlines(): if line.strip(): stderr_lines(line.rstrip()) + if tee is not None: + err = tee.finish() if proc.returncode != 0: raise subprocess.CalledProcessError(proc.returncode, args, output=out, stderr=err) return out +class _StderrTee: + """A child's stderr, echoed to ours as it arrives, with its last lines kept.""" + + def __init__(self, keep: int = 20): + import collections + import os + + self._read_end, self.write_end = os.pipe() + self._tail: collections.deque = collections.deque(maxlen=keep) + self._thread = None + + def start(self) -> None: + import os + import threading + + os.close(self.write_end) # the child holds its copy + + def pump(): + import sys + + with os.fdopen(self._read_end, 'r', errors='replace') as stream: + for line in stream: + self._tail.append(line) + try: + sys.stderr.write(line) + sys.stderr.flush() + except Exception: # noqa: BLE001 - never fail the command + pass + + self._thread = threading.Thread(target=pump, daemon=True) + self._thread.start() + + def finish(self) -> str: + if self._thread is not None: + self._thread.join(timeout=5.0) + return ''.join(self._tail) + + def _communicate_streaming(proc, bound: float, on_line) -> tuple[str, str | None]: """``proc.communicate(timeout=bound)``, handing each stdout line to ``on_line``. diff --git a/infer_stack/leasing/instances.py b/infer_stack/leasing/instances.py index 3b4f9df6..a2577e01 100644 --- a/infer_stack/leasing/instances.py +++ b/infer_stack/leasing/instances.py @@ -150,8 +150,9 @@ def resolve(instances: list[Instance], names: Iterable[str], or (i.deployment_id and name in served.get(i.deployment_id, ())) ] if not found: - known = ', '.join(sorted({i.name for i in instances})) or 'nothing' - raise UnknownTarget(f'no instance matches {name!r}; running: {known}') + known = ', '.join(sorted({i.name for i in instances})) + raise UnknownTarget(f'no instance matches {name!r}; ' + + (f'running: {known}' if known else 'nothing is running')) picked.extend(i for i in found if i not in picked) return picked diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 35c7b2df..9dcb0095 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -809,7 +809,7 @@ def test_evict_json_stdout_is_pure_json(env, capsys): out, err = capsys.readouterr() data = json.loads(out) # must parse: no human text mixed into stdout assert data == {'evicted': [], 'torn_down': [], 'missing': ['ghost']} - assert 'no idle deployment for: ghost' in err + assert 'evict: ghost is not a deployment or served endpoint' in err assert rc == 0 From fdc57c816b5def0365a822af0653fd3860cda8e8 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:08:41 -0400 Subject: [PATCH 047/141] UX: an unreachable cluster says why, in one line On kubeai with no cluster, `acquire` refused with "the runtime did not answer" and no cause, and `ps` / `leases` printed kubectl's klog retry lines. kubectl failures now carry kubectl's own last sentence, and the refusal includes the residency error with `infer-stack doctor` as the next step. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 5 +++++ infer_stack/backends/kubeai.py | 25 ++++++++++++++++++++++++- infer_stack/leasing/controller.py | 16 +++++++++++----- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 30a10392..cdddfd3a 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -227,6 +227,11 @@ dry-run root. evict"; it now says whether each name is held by a lease or unknown. `test` on a stopped gateway printed a urllib3 dump; now "nothing is listening there". `logs` said "running: nothing". +- [x] On kubeai with no cluster, `acquire` refused with "the runtime did + not answer" and no cause; `ps` and `leases` printed kubectl's klog + retries (hundreds of characters). Now kubectl's own last sentence + ("The connection to the server … was refused"), in the refusal too, + with `infer-stack doctor` as the next step. ### 10. [ ] Handover diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 8a20627c..e766e95b 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -366,13 +366,36 @@ def _default_kubectl_run(args: list[str]) -> str: args, capture_output=True, text=True, timeout=120, ) if proc.returncode != 0: - detail = (proc.stderr or proc.stdout or '').strip() + detail = kubectl_complaint(proc.stderr or proc.stdout or '') raise RuntimeError( f'{" ".join(args)} failed ({proc.returncode}): {detail[:500]}' ) return proc.stdout +def kubectl_complaint(stderr: str) -> str: + r"""kubectl's own complaint, without the client library's log lines. + + kubectl prefixes its retries with klog lines (``E0926 14:05:52 ... + memcache.go:265] "Unhandled Error" err="..."``) and ends with the sentence + that says what is wrong; that sentence is the complaint. + + >>> kubectl_complaint('E0926 14:05:52.449219 841241 memcache.go:265] "Unhandled ' + ... 'Error" err="couldn\'t get current server API group list"\n' + ... 'The connection to the server localhost:8080 was refused - ' + ... 'did you specify the right host or port?\n') + 'The connection to the server localhost:8080 was refused - did you specify the right host or port?' + """ + import re + + lines = [ln.strip() for ln in str(stderr).splitlines() if ln.strip()] + plain = [ln for ln in lines if not re.match(r'^[EWIF]\d{4} \d\d:\d\d:\d\d', ln)] + if plain: + return plain[-1] + return lines[-1] if lines else '' + + + class KubeaiBackend(ConvergeScaffold): """Cluster KubeAI backend (converge-style). diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index e7666b05..1cb9429a 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -201,6 +201,8 @@ def __init__( self._admission_digest: str | None = None #: Whether the last refused admission was for lack of GPUs. self._admission_capacity = False + #: Why residency could not be read, for the refusal that follows. + self._residency_error: str | None = None from .profile import ProfileMismatch try: @@ -864,10 +866,10 @@ def _admit(self, overlay, residency): if residency is None: # The render after the commit needs residency too, so nothing can # be admitted without it; refusing here commits nothing. + why = self._residency_error or 'the runtime did not answer' return {}, [ - 'what is running cannot be read right now (the runtime did not ' - 'answer), so nothing is admitted; retry when `infer-stack leases` ' - 'shows no residency error' + f'what is running cannot be read right now ({why}), so nothing is ' + 'admitted; `infer-stack doctor` checks the runtime' ] if need: unresolved = self._unresolved_allocations(exclude=set(overlay.deployments)) @@ -1559,8 +1561,10 @@ def _acquire_by_admission( self._publish() try: residency = self._admitting.residency() - except ResidencyUnknown: + self._residency_error = None + except ResidencyUnknown as ex: residency = None + self._residency_error = str(ex) candidate_profile = self._acquire_profile_candidate( residency=residency ) @@ -1847,8 +1851,10 @@ def renew(self, lease_id: str, *, ttl_seconds: float | None) -> RenewOutcome: try: residency = self._admitting.residency() - except ResidencyUnknown: + self._residency_error = None + except ResidencyUnknown as ex: residency = None + self._residency_error = str(ex) overlay = self.ledger.plan_acquire([]) for gid in dict.fromkeys(lease.deployment_ids): deployment = self.ledger.get_deployment(gid) From be0b1fad257c36465bedd771f33736c1d35b9930 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:15:21 -0400 Subject: [PATCH 048/141] UX: the TUI works at 80x24 with the runtime pane open Opening the runtime pane on a 24-row terminal took every row, so the lease and deployment tables vanished, and each log line spent most of its width on the instance-name prefix. The pane is now capped near half the screen, counting its tab strip and source picker (three log lines at 24 rows, and the tables keep their row); below 32 rows the pane descriptions give way; and the TUI's follower leaves the name off while it follows one instance. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 5 ++++ infer_stack/leasing/instances.py | 26 ++++++++++++++------- infer_stack/tui.py | 28 ++++++++++++++++++++--- tests/test_day2.py | 39 ++++++++++++++++++++++++++++++++ tests/test_tui.py | 24 ++++++++++++++++++++ 5 files changed, 111 insertions(+), 11 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index cdddfd3a..9198be1c 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -232,6 +232,11 @@ dry-run root. retries (hundreds of characters). Now kubectl's own last sentence ("The connection to the server … was refused"), in the refusal too, with `infer-stack doctor` as the next step. +- [x] TUI at 80x24: opening the runtime pane took every row (the lease and + deployment tables vanished), and each log line spent ~20 of its ~33 + columns on the instance-name prefix. The pane is now capped near half + the screen (three log lines at 24 rows), pane descriptions hide below + 32 rows, and one followed instance gets no prefix. ### 10. [ ] Handover diff --git a/infer_stack/leasing/instances.py b/infer_stack/leasing/instances.py index a2577e01..6a9db195 100644 --- a/infer_stack/leasing/instances.py +++ b/infer_stack/leasing/instances.py @@ -163,20 +163,25 @@ class LogFollower: ``list_instances`` is called every ``poll`` seconds, so an instance that appears later (a model starting, a recreate) is followed from its first line, and one that restarted is followed again. ``stdout`` yields - ``name | line``. Output written between the history read and the live - stream (a few milliseconds) can be missed. + ``name | line``; with ``prefix='auto'`` the name is left off while only + one instance has been followed (it says nothing then, and costs a narrow + pane most of its width). Output written between the history read and the + live stream (a few milliseconds) can be missed. """ poll = 3.0 def __init__(self, list_instances: Callable[[], list[Instance]], *, - history: str | int = 200, timestamps: bool = False): + history: str | int = 200, timestamps: bool = False, + prefix: str = 'always'): import queue import threading self._list = list_instances self._history = history self._timestamps = timestamps + self._prefix = prefix + self._names: set[str] = set() self._lines: queue.Queue = queue.Queue() self._stop = threading.Event() self._seen: set[str] = set() @@ -204,6 +209,7 @@ def _watch(self) -> None: else: continue self._seen.add(inst.id) + self._names.add(inst.name) threading.Thread(target=self._follow, args=(inst, history, running), daemon=True).start() first = False @@ -212,7 +218,7 @@ def _watch(self) -> None: def _follow(self, inst: Instance, history, running: bool) -> None: from ..log_filter import LogLineSplitter - prefix = f'{inst.name} | ' + prefix = inst.name env = runtime_env(inst) if history is not None: argv = history_argv(inst, tail=history, timestamps=self._timestamps) @@ -227,7 +233,7 @@ def _follow(self, inst: Instance, history, running: bool) -> None: old = '' split = LogLineSplitter(every=0.0) for line in split.feed(old) + split.flush(): - self._lines.put(prefix + line) + self._lines.put((prefix, line)) argv = follow_argv(inst, timestamps=self._timestamps) if not running or argv is None or self._stop.is_set(): return @@ -253,11 +259,11 @@ def _pump(self, proc: subprocess.Popen, prefix: str) -> None: if not chunk: break for line in split.feed(decode(chunk)): - self._lines.put(prefix + line) + self._lines.put((prefix, line)) finally: proc.stdout.close() for line in split.flush(): - self._lines.put(prefix + line) + self._lines.put((prefix, line)) @property def stdout(self) -> Iterator[str]: @@ -265,9 +271,13 @@ def stdout(self) -> Iterator[str]: while not self._stop.is_set(): try: - yield self._lines.get(timeout=0.5) + '\n' + name, line = self._lines.get(timeout=0.5) except queue.Empty: continue + if self._prefix == 'auto' and len(self._names) <= 1: + yield line + '\n' + else: + yield f'{name} | {line}\n' def terminate(self) -> None: self._stop.set() diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 033032d3..c8dc7f0e 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -731,6 +731,9 @@ class InferStackTUI(App): /* one-line, per-pane descriptions (replaces the old global intro) */ .desc { height: auto; color: $text-muted; padding: 0 1; } + /* A short terminal (see on_resize): rows go to tables and logs. */ + .compact .desc { display: none; } + .compact #logsvc { margin: 0; } #catalog-help { height: auto; color: $text-muted; padding: 0 1; } #endpoint-actions, #lease-actions, #deployment-actions, #model-actions, @@ -787,7 +790,7 @@ class InferStackTUI(App): .pair, .pair ContentSwitcher, .pair TabPane { height: 1fr; } .pair TabPane { padding: 0; } .pair #models, .pair #leases-pane { height: 1fr; } - #docker-tabs { height: 16; min-height: 8; } + #docker-tabs { height: 16; min-height: 8; } /* capped by _log_height */ #logsvc { margin: 0 0 1 0; } #logs, #ps { height: 1fr; background: $surface; } #gpus, #api-out { height: 8; background: $surface; } @@ -1424,9 +1427,28 @@ def _draw_activity(self) -> None: # -- resizable panes --------------------------------------------------- + def _log_height(self) -> int: + """The runtime pane's log height: the chosen one, capped near half the + screen, so on a small terminal the lease and deployment tables keep + rows when the pane opens (its tabs and picker take ~9 rows already).""" + # The tabbed area holds the tab strip (2) and the source picker (3) + # before any log line: 8 is those and three lines. + rows = self.size.height or 50 + return max(8, min(self._log_h, rows // 2 - 4)) + + #: Below this many rows the pane descriptions give way to tables and logs. + COMPACT_ROWS = 32 + + def on_resize(self, event: events.Resize) -> None: + self.set_class(event.size.height < self.COMPACT_ROWS, 'compact') + try: + self._apply_sizes() + except Exception: # noqa: BLE001 - not mounted yet + pass + def _apply_sizes(self) -> None: self.query_one('#sidebar').styles.width = self._sidebar_w - self.query_one('#docker-tabs').styles.height = self._log_h + self.query_one('#docker-tabs').styles.height = self._log_height() # Fixed heights only matter beside a divider; a tab fills its pane. if not self.TABBED_CATALOG: self.query_one('#models').styles.height = self._models_h @@ -2081,7 +2103,7 @@ def listing(): found = backend.instances() return [i for i in found if wanted is None or i.name in wanted] - return LogFollower(listing) + return LogFollower(listing, prefix='auto') return factory diff --git a/tests/test_day2.py b/tests/test_day2.py index 88bde5f4..83be298d 100644 --- a/tests/test_day2.py +++ b/tests/test_day2.py @@ -213,3 +213,42 @@ def test_no_color_turns_color_off(): assert LogsCLI.cli(argv=[]).color is True assert LogsCLI.cli(argv=['--no-color']).color is False + + +def test_one_followed_instance_needs_no_name_prefix(monkeypatch): + """The TUI's follower leaves the name off while it follows one instance: + on a narrow pane the prefix took most of every line.""" + import os + import subprocess + import time + + from infer_stack.leasing.instances import LogFollower + + class _Done: + stdout = b'first\nsecond\n' + + class _Proc: + def __init__(self, *a, **kw): + r, w = os.pipe() + os.close(w) + self.stdout = os.fdopen(r, 'rb') + + def poll(self): + return 0 + + def terminate(self): + pass + + def wait(self, timeout=None): + return 0 + + monkeypatch.setattr(subprocess, 'run', lambda *a, **kw: _Done()) + monkeypatch.setattr(subprocess, 'Popen', _Proc) + follower = LogFollower(lambda: [GATEWAY], prefix='auto') + lines, deadline = [], time.monotonic() + 5 + for line in follower.stdout: + lines.append(line) + if len(lines) == 2 or time.monotonic() > deadline: + break + follower.terminate() + assert lines == ['first\n', 'second\n'] diff --git a/tests/test_tui.py b/tests/test_tui.py index bf316253..be1fc5dd 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -2197,3 +2197,27 @@ async def scenario(): assert 'qwen-extra' in app._endpoint_names # the last good one stays _run(scenario) + + +def test_an_80x24_terminal_shows_logs_and_the_tables_when_the_runtime_opens(): + """At 80x24 the runtime pane used to take every row (the tables vanished), + and then, capped naively, left none for the log itself.""" + from textual.widgets import Collapsible + + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test(size=(80, 24)) as pilot: + await pilot.pause() + assert app.has_class('compact') # descriptions give way + app.query_one('#docker', Collapsible).collapsed = False + await pilot.pause() + await pilot.pause() + assert app.query_one('#logs').region.height >= 3 + assert app.query_one('#tables').region.height >= 1 + + _run(scenario) From 8bd07c804eb1be00945dbb4f351a98fee0028c44 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:17:03 -0400 Subject: [PATCH 049/141] UX: the TUI's API tab never shows the master key The example curl on the API tab carried the gateway's master key in clear text, on a pane that is on screen (and screens get shared). It now reads the key when run, `$(infer-stack env LITELLM_MASTER_KEY)`; the Copy curl button still copies a curl with the literal key. The UI tab's "docker observe interval" is now the runtime observe interval. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 4 ++++ infer_stack/tui.py | 27 ++++++++++++++++++++------- tests/test_tui.py | 25 +++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 9198be1c..148e6abc 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -237,6 +237,10 @@ dry-run root. columns on the instance-name prefix. The pane is now capped near half the screen (three log lines at 24 rows), pane descriptions hide below 32 rows, and one followed instance gets no prefix. +- [x] The TUI's API tab showed the gateway's master key in clear text in + its example curl. The pane now reads it at run time + (`$(infer-stack env LITELLM_MASTER_KEY)`); Copy curl copies the + literal key. The UI tab said "docker observe interval". ### 10. [ ] Handover diff --git a/infer_stack/tui.py b/infer_stack/tui.py index c8dc7f0e..2941d999 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1147,8 +1147,8 @@ def _compose_ui_settings(self) -> ComposeResult: yield Input(value=f'{self.ledger_interval:g}', id='set-ledger-interval') yield Label( - 'docker observe interval (seconds) — the "running" / GPU-placement ' - 'columns; higher = fewer `docker compose ps` calls' + 'runtime observe interval (seconds) — the "running" / GPU columns ' + 'and the runtime pane; higher = fewer runtime queries' ) yield Input(value=f'{self.observe_interval:g}', id='set-observe-interval') @@ -3208,15 +3208,25 @@ def _api_chat(self, model: str, prompt: str) -> str: self._raise_for_body(resp) return self._completion_text(resp.json()) - def _curl_for(self, model: str, prompt: str) -> str: + def _curl_for(self, model: str, prompt: str, *, reveal_key: bool = False) -> str: """The equivalent ``curl`` for a chat- or text-completion, matching the - endpoint's served protocol, against the gateway.""" + endpoint's served protocol, against the gateway. + + The key is read at run time (``infer-stack env LITELLM_MASTER_KEY``) + rather than shown: the pane is on screen, and screens get shared. + ``reveal_key`` puts the literal key in, for the clipboard. + """ import json as _json base, key = self._litellm() if not base: return '# acquire a model first — no LiteLLM gateway yet' - auth = f" -H 'Authorization: Bearer {key}'" if key else '' + if key and reveal_key: + auth = f" -H 'Authorization: Bearer {key}'" + elif key: + auth = ' -H "Authorization: Bearer $(infer-stack env LITELLM_MASTER_KEY)"' + else: + auth = '' if self._protocol_for(model) == 'completions': path = '/v1/completions' body = _json.dumps({ @@ -3294,8 +3304,11 @@ def action_api_list_models(self) -> None: self._do_api_list() def action_api_copy_curl(self) -> None: - text = str(self.query_one('#api-curl', Static).render()) - ok = self._copy(text) + # The clipboard gets the literal key (works in any shell); the pane + # shows only how to read it. + model = self._selected_api_model() or '' + prompt = self.query_one('#api-prompt', Input).value.strip() or 'hello' + ok = self._copy(self._curl_for(model, prompt, reveal_key=True)) self._status('copied curl to clipboard' if ok else 'copy failed — install wl-copy/xclip, or enable OSC 52') diff --git a/tests/test_tui.py b/tests/test_tui.py index be1fc5dd..30ab29f0 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -2221,3 +2221,28 @@ async def scenario(): assert app.query_one('#tables').region.height >= 1 _run(scenario) + + +def test_the_api_tab_never_shows_the_master_key(): + """The curl on screen reads the key when run; the clipboard gets it.""" + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test() as pilot: + await pilot.pause() + app._litellm = lambda: ('http://localhost:14042', 'sk-secret-key') + app._sync_api_models(['qwen-coder']) + app._update_api_curl() + shown = str(app.query_one('#api-curl').render()) + assert 'sk-secret-key' not in shown + assert '$(infer-stack env LITELLM_MASTER_KEY)' in shown + copied = [] + app._copy = lambda text: copied.append(text) or True + app.action_api_copy_curl() + assert copied and 'sk-secret-key' in copied[0] + + _run(scenario) From 224501a30e58c7528880ad50b697724ff8ab59a3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:23:26 -0400 Subject: [PATCH 050/141] UX: the TUI says why a runtime command failed, and shows engine colors - A refused docker/kubectl command showed in the status line as a CalledProcessError repr full of paths; it now reads like the CLI's (the runtime's words and a hint). - Colored engine output (vLLM's "(APIServer pid=1)") was garbled in the logs pane: the escape codes went in as plain text. They are rendered as ANSI now. - "no engines running yet" after a release is "no engines running". Checked on kubeai at 80x24 in a real terminal: acquire, the pod's log, release-all, all from the TUI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 4 ++++ infer_stack/tui.py | 37 +++++++++++++++++++++++++++---------- tests/test_tui.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 148e6abc..df360b20 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -241,6 +241,10 @@ dry-run root. its example curl. The pane now reads it at run time (`$(infer-stack env LITELLM_MASTER_KEY)`); Copy curl copies the literal key. The UI tab said "docker observe interval". +- [x] TUI: a refused runtime command showed as a `CalledProcessError` + repr full of paths (same fix as the CLI's), and colored engine output + (vLLM's `(APIServer pid=1)`) was garbled in the logs pane. Checked on + kubeai at 80x24: acquire, the pod's log, release-all, from the TUI. ### 10. [ ] Handover diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 2941d999..647a4d61 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -160,6 +160,18 @@ def activity_label(description: str) -> str | None: ) +def _why(ex: BaseException) -> str: + """A failure for the status line: a refused runtime command the way the + CLI says it (its words, a hint), anything else as it reads.""" + import subprocess + + if isinstance(ex, subprocess.CalledProcessError): + from .cli import runtime_failure + + return ' — '.join(line.strip() for line in runtime_failure(ex).splitlines()) + return str(ex) + + def _select_is_blank(value: object) -> bool: return value is SELECT_BLANK @@ -2140,7 +2152,7 @@ def _resolve_log_target(self, service: str): names = self._engine_names() if not names: return NO_LOG_TARGET, ( - 'no engines running yet; choose (everything) ' + 'no engines running; choose (everything) ' 'for the gateway') return names, f'engines: {", ".join(names)}' if not service: @@ -2202,7 +2214,12 @@ def _write_log_lines(self, lines: list[str]) -> None: self._log_lines.extend(lines) if len(self._log_lines) > 2 * LOG_PANE_LINES: del self._log_lines[:-LOG_PANE_LINES] - self.query_one('#logs', RichLog).write('\n'.join(lines)) + # Engines color their output (vLLM's "(APIServer pid=1)" prefix): as + # a plain string the escape codes garbled the line; as ANSI they are + # colors, and _log_lines keeps the raw text for search and copy. + from rich.text import Text + + self.query_one('#logs', RichLog).write(Text.from_ansi('\n'.join(lines))) def _drain_logs(self) -> None: """Draw streamed lines: a bounded batch per tick, newest lines kept.""" @@ -2759,7 +2776,7 @@ def _do_apply(self) -> None: msg = ('apply done' if not rec.publication_pending else 'apply did not fully take effect; still pending') except Exception as ex: # noqa: BLE001 - msg = f'apply failed: {ex}' + msg = f'apply failed: {_why(ex)}' self._after_mutation(msg) def action_compose_down(self) -> None: @@ -2776,7 +2793,7 @@ def _do_down(self) -> None: self.controller.backend.down() msg = 'down done' except Exception as ex: # noqa: BLE001 - msg = f'down failed: {ex}' + msg = f'down failed: {_why(ex)}' self._after_mutation(msg) # -- open in browser --------------------------------------------------- @@ -3380,7 +3397,7 @@ def _do_acquire(self, name: str) -> None: 'engine may still be loading' ) except Exception as ex: # noqa: BLE001 - msg = f'acquire {name} failed: {ex}' + msg = f'acquire {name} failed: {_why(ex)}' self.call_from_thread(self._finish_acquire, name, msg) def _finish_acquire(self, name: str, message: str) -> None: @@ -3398,7 +3415,7 @@ def _do_release(self, ids: list[str]) -> None: self._lease_sel.clear() msg = f'released {len(ids)} lease(s)' except Exception as ex: # noqa: BLE001 - msg = f'release failed: {ex}' + msg = f'release failed: {_why(ex)}' self._after_mutation(msg) @work(thread=True, exclusive=True, group='mutate') @@ -3407,7 +3424,7 @@ def _do_release_all(self) -> None: out = self.controller.release_leases(None) msg = f'released {len(out.released_lease_ids)} lease(s)' except Exception as ex: # noqa: BLE001 - msg = f'release --all failed: {ex}' + msg = f'release --all failed: {_why(ex)}' self._after_mutation(msg) @work(thread=True, exclusive=True, group='mutate') @@ -3424,7 +3441,7 @@ def _do_evict(self, ids: list[str]) -> None: msg = (f'none of the {len(ids)} selected were idle — ' 'release their leases first') except Exception as ex: # noqa: BLE001 - msg = f'evict failed: {ex}' + msg = f'evict failed: {_why(ex)}' self._after_mutation(msg) @work(thread=True, exclusive=True, group='mutate') @@ -3436,7 +3453,7 @@ def _do_evict_all(self) -> None: msg = (f'evicted {n} idle deployment(s)' if n else 'no idle deployments to evict') except Exception as ex: # noqa: BLE001 - msg = f'evict all failed: {ex}' + msg = f'evict all failed: {_why(ex)}' self._after_mutation(msg) @work(thread=True, exclusive=True, group='mutate') @@ -3446,7 +3463,7 @@ def _do_cleanup(self) -> None: msg = (f'cleaned up {n_leases} released/expired lease(s) + ' f'{n_deployments} stopped deployment(s)') except Exception as ex: # noqa: BLE001 - msg = f'cleanup failed: {ex}' + msg = f'cleanup failed: {_why(ex)}' self._after_mutation(msg) def _after_mutation(self, message: str) -> None: diff --git a/tests/test_tui.py b/tests/test_tui.py index 30ab29f0..5cc88a85 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -1826,7 +1826,7 @@ async def scenario(): from infer_stack.tui import NO_LOG_TARGET app._last_instances = [Instance('litellm', 'c0', '', 'running')] target, label = app._resolve_log_target(ENGINE_SERVICES) - assert target is NO_LOG_TARGET and 'no engines' in label + assert target is NO_LOG_TARGET and 'no engines running' in label _run(scenario) @@ -2246,3 +2246,30 @@ async def scenario(): assert copied and 'sk-secret-key' in copied[0] _run(scenario) + + +def test_colored_engine_output_reads_cleanly_in_the_logs_pane(): + """vLLM colors its "(APIServer pid=1)" prefix; the escape codes used to + garble the line in the pane.""" + from textual.widgets import RichLog + + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test(size=(160, 40)) as pilot: + await pilot.pause() + from textual.widgets import Collapsible + + app.query_one('#docker', Collapsible).collapsed = False + await pilot.pause() + app._write_log_lines(['\x1b[1;36m(APIServer pid=1)\x1b[0;0m INFO engines: model loaded']) + await pilot.pause() + text = '\n'.join(strip.text for strip in app.query_one('#logs', RichLog).lines) + assert '(APIServer pid=1) INFO engines: model loaded' in text + assert '\x1b' not in text and '[1;36m' not in text + + _run(scenario) From a62e5aef66e6cb64abc593fbed7b5fc3fea4890b Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:27:28 -0400 Subject: [PATCH 051/141] UX: the README's first run works on a host with no GPU Following the README literally on a host without a GPU dead-ended: `catalog suggest` found no model and offered only simulated hardware, whose endpoints cannot run there, so `acquire` had nothing to take. `catalog suggest --simulator` adds `mock-smol`, the simulator endpoint from the mock catalog (vLLM's API, random text, never results); suggest names it when it finds no GPU, and the README says so. `config init` now points at `catalog suggest --apply`, the README's next step, instead of `catalog init`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- README.md | 4 +++ docs/queue.md | 8 ++++++ infer_stack/cli/commands_catalog.py | 44 ++++++++++++++++++++++++++++- infer_stack/cli/commands_meta.py | 3 +- infer_stack/leasing/suggest.py | 22 +++++++++++++++ tests/test_cli_catalog.py | 10 +++++++ 6 files changed, 89 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b1a16c8a..251908b1 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ infer-stack catalog suggest --apply infer-stack acquire ``` +No GPU here? `infer-stack catalog suggest --simulator --apply` adds +`mock-smol`, a simulator that answers like vLLM with random text, so the same +three steps run on any host with Docker ([docs/mock-endpoints.md](docs/mock-endpoints.md)). + `settings.yaml` (`infer-stack config …`) and `catalog.yaml` are the user configuration. Leasing keeps an internal frozen recovery snapshot so a crash cannot re-render committed state diff --git a/docs/queue.md b/docs/queue.md index df360b20..2c8f81be 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -245,6 +245,14 @@ dry-run root. repr full of paths (same fix as the CLI's), and colored engine output (vLLM's `(APIServer pid=1)`) was garbled in the logs pane. Checked on kubeai at 80x24: acquire, the pod's log, release-all, from the TUI. +- [x] First run, README literally, on a host with no GPU: `catalog suggest` + found nothing and offered only simulated hardware (endpoints that + cannot run here), so step 3 had nothing to acquire; `config init` + pointed at `catalog init`, not the README's next step. Now `catalog + suggest --simulator` adds a simulator endpoint (named in suggest's + message when there is no GPU, and in the README), and `config init` + points at `suggest`. Checked: init, suggest --simulator --apply, + acquire, test, release, from nothing. ### 10. [ ] Handover diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index 83b0c5e7..24e90062 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -295,6 +295,12 @@ class CatalogSuggestCLI( help='Whose hardware (default: the configured `backend` setting): this ' "host's GPUs, or on kubeai the cluster's.", ) + simulator = kw.Value( + False, isflag=True, + help='Suggest a simulator endpoint instead (`mock-smol`): it answers ' + "like vLLM with random text and needs no GPU, to try infer-stack's " + 'workflow on any host. Never for results.', + ) @classmethod def main(cls, argv=True, **kwargs): @@ -308,6 +314,13 @@ def main(cls, argv=True, **kwargs): from .context import effective_inventory config = cls.cli(argv=argv, data=kwargs) + if config.simulator: + import copy + + from ..leasing.suggest import SIMULATOR_FRAGMENT + + return _merge_or_print(config, copy.deepcopy(SIMULATOR_FRAGMENT), + 'a simulator: no GPU needed, random text, never results') profiles: dict = {} cluster = (config.backend or get_setting('backend')) == 'kubeai' inventory = effective_inventory(config) @@ -350,7 +363,10 @@ def main(cls, argv=True, **kwargs): print( f'no pooled model fits the detected hardware ({hw}). ' 'Pass --simulate-hardware NxM to plan for a bigger box, or add ' - 'models by hand with `catalog model add`.', + 'models by hand with `catalog model add`.' + + ('' if gpus else ' With no GPU here, `infer-stack catalog suggest ' + '--simulator --apply` adds an endpoint that answers like vLLM ' + '(random text), to try the workflow.'), file=sys.stderr, ) return 0 @@ -406,6 +422,32 @@ def main(cls, argv=True, **kwargs): return 0 +def _merge_or_print(config, frag: dict, what: str) -> int: + """Print a suggested catalog fragment, or with ``--apply`` merge it.""" + text = yaml.safe_dump(frag, sort_keys=False, default_flow_style=False) + if not config.apply: + print(f'# suggested: {what}; re-run with --apply to merge', file=sys.stderr) + _print_yaml(text) + return 0 + path = _catalog_path(config) + data = _load_raw(path) + added, skipped = [], [] + for section in ('models', 'endpoints'): + for name, value in frag[section].items(): + if name in data[section] and not config.force: + skipped.append(f'{section[:-1]}:{name}') + continue + data[section][name] = value + added.append(f'{section[:-1]}:{name}') + _save_raw(path, data, dry_run=False) + print(f'merged suggestion into {path} ({what})') + if added: + print(f' added: {", ".join(added)}') + if skipped: + print(f' kept existing (pass --force to overwrite): {", ".join(skipped)}') + return 0 + + class CatalogPathCLI(_PathOverridesMixin): """Print the catalog path.""" diff --git a/infer_stack/cli/commands_meta.py b/infer_stack/cli/commands_meta.py index 17be8587..7fc3a50a 100644 --- a/infer_stack/cli/commands_meta.py +++ b/infer_stack/cli/commands_meta.py @@ -494,7 +494,8 @@ def _proposed(s: _Setting): base.update(values) save_settings(base) print(f'wrote settings -> {path}') - print('next: `infer-stack catalog init` to add models/endpoints') + print('next: `infer-stack catalog suggest --apply` (a catalog sized to this ' + "host's GPUs), then `infer-stack acquire `") return 0 diff --git a/infer_stack/leasing/suggest.py b/infer_stack/leasing/suggest.py index 83edcacd..428cd978 100644 --- a/infer_stack/leasing/suggest.py +++ b/infer_stack/leasing/suggest.py @@ -379,6 +379,28 @@ def derive_runtime( return runtime +#: A simulator endpoint for a host without a GPU (``catalog suggest +#: --simulator``): llm-d-inference-sim answers like vLLM with random text, so +#: the whole workflow (acquire, the gateway, a request, release) runs here. +#: Never a result. The same entry as dev/e2e_tests/catalog-mock.yaml's. +SIMULATOR_FRAGMENT: dict[str, Any] = { + 'models': {'smol135': {'source': 'hf://HuggingFaceTB/SmolLM2-135M-Instruct'}}, + 'endpoints': {'mock-smol': { + 'engine': 'vllm', + 'model': 'smol135', + 'runtime': { + 'image': 'ghcr.io/llm-d/llm-d-inference-sim:v0.9.0', + 'max_model_len': 2048, + 'max_num_seqs': 8, + 'simulator': {'kind': 'llm-d-sim', 'mode': 'random', 'seed': 20260731, + 'time_to_first_token': '120ms', + 'inter_token_latency': '8ms', 'startup_duration': '10s'}, + }, + 'reclaim': {'policy': 'stop'}, + }}, +} + + # --------------------------------------------------------------------------- # the top-level pure function # --------------------------------------------------------------------------- diff --git a/tests/test_cli_catalog.py b/tests/test_cli_catalog.py index 578e8344..d581083f 100644 --- a/tests/test_cli_catalog.py +++ b/tests/test_cli_catalog.py @@ -382,3 +382,13 @@ def test_suggest_no_fit_writes_nothing(tmp_path, capsys): err = capsys.readouterr().err assert 'no pooled model fits' in err assert not (tmp_path / 'catalog.yaml').exists() + + +def test_suggest_simulator_gives_a_gpu_free_first_run(tmp_path): + """On a host without a GPU the documented first run dead-ended; the + simulator endpoint makes the same steps run anywhere.""" + CatalogInitCLI.main(argv=_opts(tmp_path)) + assert CatalogSuggestCLI.main(argv=[*_opts(tmp_path), '--simulator', '--apply']) == 0 + cat = Catalog.load(cat_path(tmp_path)) + (request,) = cat.resolve_names(['mock-smol']) + assert request.spec['runtime']['simulator']['kind'] == 'llm-d-sim' From a8e66e900a468fc0926e07044f0bc00731ff1327 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:27:59 -0400 Subject: [PATCH 052/141] Docs: the KubeAI profile example requests a GPU, as the README warns it must The setup example set only limits; without the GPU request and runtimeClassName the pod can start without libcuda.so.1 (the README's Resource profiles section). Same in the e2e header. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/kubeai_e2e.sh | 2 +- docs/kubeai-backend.md | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 5430c807..92dfd7cb 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -3,7 +3,7 @@ # # Prereqs (once): a cluster + the KubeAI chart. On a single GPU host: # ./scripts/bootstrap_k3s.sh -# printf 'resourceProfiles:\n %s:\n limits:\n nvidia.com/gpu: "1"\n' \ +# printf 'resourceProfiles:\n %s:\n runtimeClassName: nvidia\n requests: {nvidia.com/gpu: "1"}\n limits: {nvidia.com/gpu: "1"}\n' \ # "${E2E_RESOURCE_PROFILE:-nvidia-gpu}" > /tmp/kubeai-values.yaml # ./scripts/install_kubeai.sh /tmp/kubeai-values.yaml kubeai # kubectl -n kubeai port-forward svc/kubeai 8000:80 & diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 8353bb23..492a8c8a 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -43,12 +43,20 @@ Where the two backends match and where they still differ, row by row: # 2. Resource profiles: name -> the requests/limits/nodeSelector that one # "GPU unit" means on your cluster. These names are what the catalog's -# `runtime.resource_profile` refers to. +# `runtime.resource_profile` refers to. Without the GPU request and +# runtimeClassName the pod can land on a GPU node and still start without +# libcuda.so.1. `infer-stack catalog suggest --backend kubeai` proposes +# one per GPU product once the device plugin runs. cat > kubeai-values.yaml <<'EOF' resourceProfiles: nvidia-gpu-rtx-4090: + runtimeClassName: nvidia + requests: + nvidia.com/gpu: "1" limits: nvidia.com/gpu: "1" + nodeSelector: + nvidia.com/gpu.product: NVIDIA-GeForce-RTX-4090 EOF # 3. Install the chart (HF_TOKEN, if exported, is passed to the chart secret): From 0afbbaa77d2fa32b9ca6f719d7a897372d454d39 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:29:23 -0400 Subject: [PATCH 053/141] UX: the TUI's "Clean up" is "Clear finished" The TUI's `x` button said "Clean up" and ran `gc --forget`, which only forgets released leases and stopped deployments. `infer-stack clean` releases every lease and tears everything down. Same word, opposite consequences; the button is "Clear finished" now, and the manual says how it differs from `clean`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 5 +++++ docs/source/manual/leasing-demo.md | 5 +++-- infer_stack/tui.py | 10 ++++++---- tests/test_tui.py | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 2c8f81be..b81ae4db 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -253,6 +253,11 @@ dry-run root. message when there is no GPU, and in the README), and `config init` points at `suggest`. Checked: init, suggest --simulator --apply, acquire, test, release, from nothing. +- [x] Consistency: the TUI's "Clean up" (`x`) runs `gc --forget`, which + only forgets finished rows, while `infer-stack clean` releases and + tears everything down. It is "Clear finished" now. The KubeAI setup + example's resource profile had no GPU request or runtime class, + which the README warns makes a pod start without CUDA. ### 10. [ ] Handover diff --git a/docs/source/manual/leasing-demo.md b/docs/source/manual/leasing-demo.md index c1799324..9005b5a9 100644 --- a/docs/source/manual/leasing-demo.md +++ b/docs/source/manual/leasing-demo.md @@ -175,8 +175,9 @@ something is still warming up vs truly live. > Each pane carries its own description and buttons; expanding a pane (or its > tab) is what triggers its polling, so hidden data is never fetched. > New here? The catalog buttons (or `g`) **Suggest** a set sized to your GPUs, -> and `m` / `n` open wizards to add a model / endpoint by hand. **Clean up** -> (`x`) forgets released/stopped entries. **Ctrl+click** a served endpoint +> and `m` / `n` open wizards to add a model / endpoint by hand. **Clear finished** +> (`x`) forgets released/stopped entries (`gc --forget`; unlike `infer-stack +> clean`, it stops nothing). **Ctrl+click** a served endpoint > (or `o`) opens it in Open WebUI. Resize panes by dragging the splitter bars > (or `[` `]` / `-` `+`). Opt-in extra: `pip install "infer-stack[tui]"`. diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 647a4d61..aed8b25d 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -841,7 +841,9 @@ class InferStackTUI(App): # Global on purpose: an error can happen while any tab is in front. ('l', 'show_app_log', 'TUI log'), # Forgets finished history in both tables, so it belongs to neither pane. - ('x', 'cleanup', 'Clean up'), + # Not "Clean up": `infer-stack clean` releases and tears everything + # down, and this only forgets finished rows (`gc --forget`). + ('x', 'cleanup', 'Clear finished'), ('tab', 'focus_next', 'Next pane'), ('q', 'quit', 'Quit'), # Pane-scoped actions: keys still work, but they live as buttons under @@ -2747,7 +2749,7 @@ def action_evict_all(self) -> None: self._do_evict_all() def action_cleanup(self) -> None: - self._status('cleaning up released/expired leases + stopped deployments…') + self._status('clearing finished rows: released/expired leases, stopped deployments…') self._cli(cli.command('gc', '--forget')) self._do_cleanup() @@ -3460,10 +3462,10 @@ def _do_evict_all(self) -> None: def _do_cleanup(self) -> None: try: n_leases, n_deployments = self.controller.prune() - msg = (f'cleaned up {n_leases} released/expired lease(s) + ' + msg = (f'cleared {n_leases} released/expired lease(s) + ' f'{n_deployments} stopped deployment(s)') except Exception as ex: # noqa: BLE001 - msg = f'cleanup failed: {_why(ex)}' + msg = f'clear finished failed: {_why(ex)}' self._after_mutation(msg) def _after_mutation(self, message: str) -> None: diff --git a/tests/test_tui.py b/tests/test_tui.py index 5cc88a85..b9b14c90 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -1279,7 +1279,7 @@ async def scenario(): _run(scenario) assert seen['buttons'] == [] - assert ('x', 'cleanup', 'Clean up') in seen['footer'] # tuples show in the footer + assert ('x', 'cleanup', 'Clear finished') in seen['footer'] # tuples show in the footer assert 'CLI: infer-stack gc --forget' in seen['applog'] assert 'CLI: infer-stack status' in seen['applog'] From 817d7712bfaa6ae315f299eb5d71a221e760dce3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:37:22 -0400 Subject: [PATCH 054/141] UX: `test` and `env` work on kubeai; readable TTLs; clean pipes Audit pass 2, the day-2 commands on both backends with a live model: - `infer-stack test` failed on kubeai with HTTP 401 and `env LITELLM_MASTER_KEY` read nothing: both read a hard-coded compose .env. They now ask the configured backend's gateway for its .env and base URL (_gateway_state), including the in-cluster gateway's NodePort. - `leases` printed a TTL as `ttl=@`; it says `1h59m` / `expired`. - `clean`'s dry run said `gpus=[]` where `leases` said `cpu`; converge logged "on GPU(s) (cpu)" for a deployment with no GPU. - `logs` into a pipe or file kept the engines' ANSI color codes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 15 +++++++ infer_stack/cli/commands_leasing.py | 70 ++++++++++++++++++++++------- infer_stack/cli/commands_runtime.py | 13 ++++-- infer_stack/leasing/compose.py | 3 +- tests/test_day2.py | 10 +++++ tests/test_leasing_kubeai.py | 13 ++++++ tests/test_leasing_profile.py | 2 +- 7 files changed, 106 insertions(+), 20 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index b81ae4db..1c138152 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -259,6 +259,21 @@ dry-run root. example's resource profile had no GPU request or runtime class, which the README warns makes a pod start without CUDA. +Audit pass 2 (2026-09-26): the day-2 commands swept on both backends with a +live model each (`leases status ps env doctor clean gc renew routes wait +test logs`), plus pass 1 again. + +- [x] `infer-stack test` failed on kubeai with HTTP 401, and `env + LITELLM_MASTER_KEY` read nothing there (and so did the TUI's curl): + both read a hard-coded compose `.env`. They now ask the configured + backend's gateway for its `.env` and base URL (the in-cluster + gateway's NodePort included). +- [x] `leases` showed a lease's TTL as `ttl=@1790450980`; now `ttl=1h59m` + (JSON keeps the timestamp). `clean`'s dry run said `gpus=[]` where + `leases` says `cpu`, and converge logged "on GPU(s) (cpu)". +- [x] `logs` into a pipe or file kept the engines' color codes; with color + off they are stripped. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index e63ec468..37d036a4 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1457,10 +1457,9 @@ def _list_only(orphans): print(f' release {le.id} owner={le.owner} ' f'{",".join(le.endpoints)}') for g in held: - gpus = assignments.get(g.id) print(f' tear down {g.id} {g.state}' f'{" running" if g.id in observed else ""}' - f' gpus={gpus if gpus is not None else "-"}' + f' gpus={_gpu_label(g.id, observed, assignments)}' f' {",".join(sorted(g.served))}') for c in found: print(f' remove {c.container_id[:12]} {c.service or "?"} ' @@ -1928,8 +1927,26 @@ def main(cls, argv=True, **kwargs): controller.release(outcome.lease.id) -def _lease_ttl(le) -> str: - return 'inf' if le.expires_at is None else f'@{le.expires_at:.0f}' +def _lease_ttl(le, now: float | None = None) -> str: + """Time left on a lease, for a person: ``1h59m``, ``12m``, ``expired``. + + >>> from types import SimpleNamespace + >>> [_lease_ttl(SimpleNamespace(expires_at=e), now=1000.0) + ... for e in (None, 8140.0, 1720.0, 1045.0, 900.0)] + ['inf', '1h59m', '12m', '45s', 'expired'] + """ + import time + + if le.expires_at is None: + return 'inf' + left = le.expires_at - (time.time() if now is None else now) + if left <= 0: + return 'expired' + if left >= 3600: + return f'{int(left // 3600)}h{int(left % 3600 // 60):02d}m' + if left >= 60: + return f'{int(left // 60)}m' + return f'{int(left)}s' def _placement_view(controller): @@ -2176,9 +2193,29 @@ def _postgres_initialised() -> bool: return path.is_dir() and any(path.iterdir()) -def _secret_env_path() -> Path: - """The managed compose secrets file (.env that docker compose auto-loads).""" - return data_root() / 'leasing' / 'compose' / '.env' +def _gateway_state(config) -> tuple[Path, str]: + """``(managed .env, base URL)`` of the configured backend's gateway. + + The compose stack's own; on kubeai the gateway's, on this host or in the + cluster (whose base URL is a node's NodePort). A backend with no gateway + falls back to the compose location and the default port. One place, so + `env`, `test` and a script built from `env` cannot disagree. + """ + from ..config import DEFAULT_PORTS + + try: + front = getattr(_make_backend(config), 'front_door', lambda: None)() + except Exception: # noqa: BLE001 - a lookup must not fail on the backend + front = None + if front is not None and getattr(front, 'litellm', False): + return front.gateway._env_path, f'{front.gateway._gateway_base()}/v1' + return (data_root() / 'leasing' / 'compose' / '.env', + f'http://127.0.0.1:{DEFAULT_PORTS["litellm"]}/v1') + + +def _secret_env_path(config=None) -> Path: + """The gateway's managed secrets file (see :func:`_gateway_state`).""" + return _gateway_state(config)[0] def _front_door(config) -> tuple[str, str | None]: @@ -2188,20 +2225,23 @@ def _front_door(config) -> tuple[str, str | None]: `test` is cheap and doesn't need GPU detection or a backend object. An explicit ``--base-url`` overrides the derived URL. """ - from ..config import DEFAULT_PORTS + from urllib.parse import urlparse + env_path, derived = _gateway_state(config) base_url = getattr(config, 'base_url', None) if not base_url: - port = int(getattr(config, 'port', None) or DEFAULT_PORTS['litellm']) - base_url = f'http://127.0.0.1:{port}/v1' + base_url = derived + port = getattr(config, 'port', None) + if port: + base_url = urlparse(derived)._replace( + netloc=f'{urlparse(derived).hostname}:{int(port)}').geturl() key = None - env_path = _secret_env_path() if env_path.exists(): key = parse_env_file(env_path).get('LITELLM_MASTER_KEY') return base_url.rstrip('/'), key -def _front_door_env(stored: dict[str, str]) -> dict[str, str]: +def _front_door_env(stored: dict[str, str], config=None) -> dict[str, str]: """The front-door values ``env`` answers without them being stored. A base URL is not a secret and has nothing to be read *out* of: it is @@ -2226,7 +2266,7 @@ def _front_door_env(stored: dict[str, str]) -> dict[str, str]: base_url = stored.get('OPENAI_BASE_URL') if not base_url: - base_url, _ = _front_door(None) + base_url, _ = _front_door(config) entries = {'OPENAI_BASE_URL': base_url} port = urlparse(base_url).port if port is not None: @@ -2425,7 +2465,7 @@ class EnvCLI(_PathOverridesMixin): def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) _apply_path_overrides(config) - env_path = _secret_env_path() + env_path = _secret_env_path(config) # Write: `env KEY=VALUE` if config.arg and '=' in config.arg: @@ -2475,7 +2515,7 @@ def main(cls, argv=True, **kwargs): # Read: `env KEY` / `env --export`. A stored value always beats the # derived one -- writing the key is how you override the front door. env = parse_env_file(env_path) if env_path.exists() else {} - derived = _front_door_env(env) + derived = _front_door_env(env, config) if config.arg: if config.arg in env: print(env[config.arg]) diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 8cbb880f..42d475cf 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -527,9 +527,16 @@ def main(cls, argv=True, **kwargs): def _colorize(lines, *, enabled: bool): - """Color each ``name | `` prefix, one color per name, like Compose did.""" + """Color each ``name | `` prefix, one color per name, like Compose did. + + Disabled (a pipe, a file, ``--no-color``), the engines' own color codes + go too: they are noise in a file and break a grep. + """ if not enabled: - yield from lines + from ..log_filter import _ANSI_ESCAPE_RE + + for line in lines: + yield _ANSI_ESCAPE_RE.sub('', line) return colors: dict[str, str] = {} for line in lines: @@ -629,7 +636,7 @@ def listing(): text = proc.stdout.decode('utf-8', 'replace') lines = [f'{inst.name} | {ln}\n' if prefix else f'{ln}\n' for ln in text.splitlines()] - for line in _colorize(lines, enabled=color and prefix): + for line in _colorize(lines, enabled=color): sys.stdout.write(line) status = status or proc.returncode return int(status) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index ff1f25bd..5d1ed8ae 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1749,7 +1749,8 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N self.last_displaced = list(plan.displaced) self.last_degraded = list(plan.degraded) for gid, gpus in sorted(plan.assignments.items()): - logger.info(' placed {} on GPU(s) {}', gid, gpus or '(cpu)') + logger.info(' placed {} {}', gid, + f'on GPU(s) {gpus}' if gpus else 'without a GPU') for err in plan.errors: logger.warning(' placement: {}', err) for note in plan.warnings: diff --git a/tests/test_day2.py b/tests/test_day2.py index 83be298d..0e787b21 100644 --- a/tests/test_day2.py +++ b/tests/test_day2.py @@ -252,3 +252,13 @@ def wait(self, timeout=None): break follower.terminate() assert lines == ['first\n', 'second\n'] + + +def test_logs_into_a_pipe_drop_the_engines_color_codes(monkeypatch, capsys): + import subprocess + + rt = _cli(monkeypatch, _Backend([POD]), {'grp-1': ['qwen']}) + monkeypatch.setattr(subprocess, 'run', lambda argv, **kw: SimpleNamespace( + stdout=b'\x1b[1;36m(APIServer pid=1)\x1b[0;0m INFO ready\n', returncode=0)) + assert rt.LogsCLI.main(argv=['qwen']) == 0 + assert capsys.readouterr().out == '(APIServer pid=1) INFO ready\n' diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index acbe6362..743f3af4 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -1121,3 +1121,16 @@ def test_a_render_refusal_is_not_a_capacity_problem(tmp_path): with pytest.raises(PlacementError) as ei: ctl.acquire('alice', [_req('qwen', profile=None)], wait=False) assert ei.value.capacity is False + + +def test_env_and_test_use_the_kubeai_gateways_key(tmp_path, monkeypatch): + """They read a hard-coded compose .env, so `test` got 401 on kubeai and + `env LITELLM_MASTER_KEY` printed nothing.""" + from infer_stack.cli import commands_leasing + + be, _ = make_front_door_backend(tmp_path) + monkeypatch.setattr(commands_leasing, '_make_backend', lambda config, **kw: be) + key = be.master_key() + env_path, base = commands_leasing._gateway_state(None) + assert env_path == be.gateway.gateway._env_path + assert commands_leasing._front_door(None) == (base.rstrip('/'), key) diff --git a/tests/test_leasing_profile.py b/tests/test_leasing_profile.py index 231fa82c..4e294100 100644 --- a/tests/test_leasing_profile.py +++ b/tests/test_leasing_profile.py @@ -570,7 +570,7 @@ def write(path, values): handle.close() monkeypatch.setattr(cl, 'write_env_file', write) - monkeypatch.setattr(cl, '_secret_env_path', lambda: tmp_path / '.env') + monkeypatch.setattr(cl, '_secret_env_path', lambda config=None: tmp_path / '.env') assert cl.EnvCLI.main(argv=['HF_TOKEN=x']) == 0 assert held == [True] From aff378e6c342182f75cd1d242316503e8c7b0e45 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:43:59 -0400 Subject: [PATCH 055/141] UX: kubeai narration and doctor say what they mean; an audit script - Every kubeai acquire logged "Converging 0 deployment(s): (none)" after "Converging 1 deployment(s) onto kubeai": the gateway-only Compose project narrating itself. It now says "Converging the gateway project". - `doctor` on kubeai called KubeAI's own API (the port-forward) "gateway", the word for infer-stack's LiteLLM gateway; it is "KubeAI's API" now. dev/ux_audit.sh runs the automated part of the audit on one backend: every help summary, common mistakes, and the day-2 commands with a model up, flagging tracebacks, compose wording on kubeai, raw timestamps, ANSI in piped output and stray "Write .env" lines. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- README.md | 2 +- dev/ux_audit.sh | 90 +++++++++++++++++++++++++++++ docs/backend-parity.md | 2 +- docs/kubeai-backend.md | 2 +- docs/queue.md | 10 ++++ infer_stack/backends/kubeai.py | 8 ++- infer_stack/cli/commands_runtime.py | 2 +- infer_stack/leasing/compose.py | 18 ++++-- tests/test_leasing_kubeai.py | 4 +- 9 files changed, 125 insertions(+), 13 deletions(-) create mode 100755 dev/ux_audit.sh diff --git a/README.md b/README.md index 251908b1..5c89183f 100644 --- a/README.md +++ b/README.md @@ -533,7 +533,7 @@ The short version: ./scripts/install_kubeai.sh kubeai-values.yaml # the chart, with your resourceProfiles kubectl -n kubeai port-forward svc/kubeai 8000:80 & infer-stack config set backend kubeai -infer-stack doctor # cluster -> CRD -> namespace -> gateway +infer-stack doctor # cluster -> CRD -> namespace -> KubeAI's API infer-stack acquire --ttl 2h --env-file lease.env --yes ``` diff --git a/dev/ux_audit.sh b/dev/ux_audit.sh new file mode 100755 index 00000000..144a4f3e --- /dev/null +++ b/dev/ux_audit.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# The automated part of the UX audit (docs/queue.md, item 9): every command's +# help, the examples in them, common mistakes, and the day-2 commands with a +# model up, on one backend. It prints what it ran and flags the shapes that +# have been bugs before: a traceback, compose/docker wording on kubeai, a raw +# timestamp, ANSI codes in piped output, stray "Write .env" lines. +# +# dev/ux_audit.sh compose # the simulator catalog, no GPU +# dev/ux_audit.sh kubeai # CPU vLLM on the dev cluster +# +# The TUI, the first run from the README and wording consistency stay manual: +# they need eyes (docs/queue.md says what to look at). Isolated config and +# data roots; everything is released and taken down at the end. +set -uo pipefail + +BACKEND="${1:-compose}" +HERE="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/infer-stack-ux.XXXXXX")" +export INFER_STACK_CONFIG_DIR="$WORK/config" INFER_STACK_DATA_DIR="$WORK/data" +mkdir -p "$WORK/config" +REPORT="$WORK/report.txt" +FLAGS=0 + +flag() { echo "!! $*" | tee -a "$REPORT"; FLAGS=$((FLAGS + 1)); } +run() { # run a command, keep its output, and check it for known bad shapes + local out rc + out=$(timeout 300 infer-stack "$@" 2>&1); rc=$? + { echo "\$ infer-stack $* (rc=$rc)"; echo "$out" | head -20; echo; } >> "$REPORT" + echo "$out" | grep -q 'Traceback' && flag "traceback: infer-stack $*" + echo "$out" | grep -q 'Write .env' && flag "stray 'Write .env': infer-stack $*" + echo "$out" | grep -qE 'ttl=@[0-9]' && flag "raw timestamp: infer-stack $*" + echo "$out" | grep -q $'\x1b\[' && flag "ANSI codes in piped output: infer-stack $*" + if [ "$BACKEND" = kubeai ]; then + echo "$out" | grep -v '^\s*\(INFO\|[0-9:]* INFO\)' \ + | grep -iqE '\bcompose (project|backend)\b|docker compose' \ + && flag "compose wording on kubeai: infer-stack $*" + fi + return 0 +} + +cleanup() { + infer-stack clean -f >/dev/null 2>&1 + infer-stack stack down >/dev/null 2>&1 + echo "report: $REPORT" + echo "flags: $FLAGS" +} +trap cleanup EXIT + +case "$BACKEND" in + compose) + cp "$HERE/dev/e2e_tests/catalog-mock.yaml" "$WORK/config/catalog.yaml" + ENDPOINT=mock-smol + ;; + kubeai) + cat > "$WORK/config/catalog.yaml" <&2; exit 2 ;; +esac +infer-stack config set backend "$BACKEND" >/dev/null + +echo '== help: every one-line summary is a sentence' +COLUMNS=250 infer-stack help tree 2>/dev/null | sed -E 's/^[│ ├└─]+//' \ + | awk '{$1=""; print substr($0,2)}' | grep -vE '[.)?]$|^$' \ + | while read -r line; do flag "summary ends mid-sentence: $line"; done + +echo '== mistakes: each names its cause, none is a traceback' +run acquire nope --yes +run release lease-nope --yes +run wait nope --timeout 5 +run logs nope +run evict grp-nope +run renew lease-nope --ttl 1h +run test "$ENDPOINT" --timeout 5 + +echo "== day-2, with $ENDPOINT up" +run acquire "$ENDPOINT" --yes --ttl 1h --timeout 600 --env-file "$WORK/lease.env" +for cmd in leases status ps env "env OPENAI_BASE_URL" doctor clean gc \ + "renew --env-file $WORK/lease.env --ttl 2h" "routes list" \ + "wait $ENDPOINT" "test $ENDPOINT" "logs $ENDPOINT --tail 3"; do + # shellcheck disable=SC2086 + run $cmd +done +run release --env-file "$WORK/lease.env" --yes diff --git a/docs/backend-parity.md b/docs/backend-parity.md index 31a0da56..ee60b760 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -26,7 +26,7 @@ cluster setup is [kubeai-backend.md](kubeai-backend.md). | tools | Docker with the NVIDIA runtime | a cluster (`scripts/bootstrap_k3s.sh` for one host, `scripts/join_agent.sh` for another), `kubectl`, `helm`, the KubeAI chart (`scripts/install_kubeai.sh`), the NVIDIA device plugin on GPU nodes | | information | none: GPUs are discovered with `nvidia-smi` | `resourceProfiles` in the chart values (what one GPU unit requests, and on which nodes; `catalog suggest --backend kubeai` proposes them from the nodes' GPU labels), and per endpoint `runtime.resource_profile`, a `min_vram_gib` that picks one by size, or a default `kubeai_resource_profile` | | settings | `backend compose` | `backend kubeai`; `kubeai_gateway cluster` for more than one workstation; `kubeai_namespace`, `kubeai_base_url`, `kubeai_gateway_upstream` when the defaults (namespace `kubeai`, a port-forward on 8000, the Service's cluster IP) do not hold | -| preflight | none needed | `infer-stack doctor`: cluster → CRD → namespace → gateway | +| preflight | none needed | `infer-stack doctor`: cluster → CRD → namespace → KubeAI's API | The catalog, `acquire … --env-file`, `release`, the TUI and the env file a card sources are the same on both. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 492a8c8a..93cad3b9 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -113,7 +113,7 @@ infer-stack catalog suggest --backend kubeai # catalog on stdout, profiles on ``` Verify the setup before the first acquire — `doctor` checks the chain in -dependency order (cluster reachable → CRD installed → namespace → gateway): +dependency order (cluster reachable → CRD installed → namespace → KubeAI's API): ```bash infer-stack doctor diff --git a/docs/queue.md b/docs/queue.md index 1c138152..0372f892 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -274,6 +274,16 @@ test logs`), plus pass 1 again. - [x] `logs` into a pipe or file kept the engines' color codes; with color off they are stripped. +Audit pass 3 (2026-09-26): `dev/ux_audit.sh compose` and `kubeai` (help, +mistakes, the day-2 sweep; 0 flags on both), and the reports read by eye. + +- [x] Every kubeai acquire logged "Converging 0 deployment(s): (none)" right + after "Converging 1 deployment(s) onto kubeai": the gateway-only + project narrating itself. It says "Converging the gateway project". +- [x] `doctor` on kubeai called KubeAI's own API "gateway", beside + infer-stack's LiteLLM gateway; it is "KubeAI's API" now, here and in + the docs. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index e766e95b..b7e24828 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -444,6 +444,8 @@ def __init__( # request names (the endpoint aliases) as on the compose backend. # Without it, clients talk to KubeAI directly under its Model names. self.gateway = gateway + if gateway is not None: + gateway.fronts_elsewhere = True # Builds the gateway for a placement ('host' or 'cluster'), so a # recovery profile can say where it runs (see use_profile). self.gateway_factory = gateway_factory @@ -1086,7 +1088,7 @@ def doctor(self) -> list[tuple[str, bool, str]]: Everything ``acquire`` needs, checked cheaply and in dependency order, so a fresh setup fails as a checklist instead of a mid-acquire traceback: cluster reachable -> KubeAI CRD installed -> namespace - exists -> gateway answering at ``base_url``. Never raises. + exists -> KubeAI's API answering at ``base_url``. Never raises. """ checks: list[tuple[str, bool, str]] = [] @@ -1135,5 +1137,7 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: f'`kubectl -n {self.namespace} port-forward svc/kubeai ' '8000:80` (or set kubeai_base_url)' ) - checks.append((f'gateway at {self.base_url}', ok, detail)) + # KubeAI's own API (what the port-forward reaches), not infer-stack's + # LiteLLM gateway, which starts with the first acquire. + checks.append((f"KubeAI's API at {self.base_url}", ok, detail)) return checks diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 42d475cf..34eb2f6e 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -813,7 +813,7 @@ class DoctorCLI(_PathOverridesMixin): """Preflight the configured backend: is everything acquire needs in place? Runs the backend's cheap dependency-ordered checks (for kubeai: cluster - reachable -> KubeAI CRD installed -> namespace exists -> gateway + reachable -> KubeAI CRD installed -> namespace exists -> KubeAI's API answering) and prints a checklist. Exits nonzero if any check fails, so scripts can gate on it. Backends without a preflight (null/compose) report that there is nothing to check. diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 5d1ed8ae..e5d33b1d 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1700,6 +1700,9 @@ def _merged_route_registry( #: dynamic routes, both pointing at its servers. Set before each render. upstream_rows: dict[str, dict[str, Any]] = {} upstream_routes: list[dict[str, Any]] = [] + #: Set by an owner whose engines run elsewhere (kubeai): this project is + #: only the front door. + fronts_elsewhere = False def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: """Route-registry rows for every endpoint of ``catalog``.""" @@ -1737,11 +1740,16 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N desired = list(desired) with self._converge_lock(): - logger.info( - 'Converging {} deployment(s): {}', - len(desired), - ', '.join(sorted(g.id for g in desired)) or '(none)', - ) + if self.fronts_elsewhere: + # A gateway-only project (kubeai's): its owner narrates the + # deployments, and "0 deployment(s)" read as a contradiction. + logger.info('Converging the gateway project ({})', self.project) + else: + logger.info( + 'Converging {} deployment(s): {}', + len(desired), + ', '.join(sorted(g.id for g in desired)) or '(none)', + ) docs = self._render_documents(desired, placement) plan, rendered, planned = docs['plan'], docs['rendered'], docs['planned'] fingerprints = docs['fingerprints'] diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 743f3af4..6da550ac 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -517,7 +517,7 @@ def test_doctor_all_green(tmp_path): names = [c[0] for c in checks] assert any('cluster' in n for n in names) assert any('CRD' in n for n in names) - assert any('gateway' in n for n in names) + assert any("KubeAI's API" in n for n in names) def test_doctor_stops_at_first_missing_dependency(tmp_path): @@ -539,7 +539,7 @@ def get(self, url, **kw): kubectl = FakeKubectl() be = KubeaiBackend(state_dir=tmp_path, run=kubectl, http=DownHttp()) checks = be.doctor() - gateway = [c for c in checks if c[0].startswith('gateway')][0] + gateway = [c for c in checks if c[0].startswith("KubeAI's API")][0] assert gateway[1] is False assert 'port-forward' in gateway[2] From 96fc44233e8c1a93873ee566aba2ef9df1d6482e Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:49:39 -0400 Subject: [PATCH 056/141] UX: kubeai's doctor checks the gateway's Docker too On kubeai the gateway on this host is a Compose project, but `doctor` checked only the cluster: with Docker stopped, preflight passed and the first acquire failed. The gateway project's own checks (Docker, compose, its images) now follow KubeAI's, prefixed "gateway:"; a gateway-only project skips the engine image and GPU checks, which are not its. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 9 +++++++++ infer_stack/backends/kubeai.py | 5 +++++ infer_stack/leasing/compose.py | 7 ++++++- tests/test_leasing_kubeai.py | 9 +++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 0372f892..46c4313a 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -284,6 +284,15 @@ mistakes, the day-2 sweep; 0 flags on both), and the reports read by eye. infer-stack's LiteLLM gateway; it is "KubeAI's API" now, here and in the docs. +Audit pass 4 (2026-09-26): `dev/ux_audit.sh` on both backends (0 flags; +the two reports differ only in backend facts), the reports read by eye. + +- [x] `doctor` on kubeai did not check Docker, though the gateway on this + host is a Compose project: a stopped daemon passed preflight and + failed the first acquire. The gateway project's checks (Docker, + compose, its images; no engine image or GPU check) now follow + KubeAI's. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index b7e24828..53cd481a 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -1140,4 +1140,9 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: # KubeAI's own API (what the port-forward reaches), not infer-stack's # LiteLLM gateway, which starts with the first acquire. checks.append((f"KubeAI's API at {self.base_url}", ok, detail)) + if self.gateway is not None and hasattr(self.gateway, 'doctor'): + # The gateway on this host is a Compose project: Docker and its + # images are part of what an acquire needs. + checks.extend((f'gateway: {name}', good, why) + for name, good, why in self.gateway.doctor()) return checks diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index e5d33b1d..0d2cb526 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1357,7 +1357,9 @@ def doctor(self) -> list[tuple[str, bool, str]]: # 3. Every image this configuration would actually bring up. Checking # the full PINNED_IMAGES set would fail on services that are off. - wanted: dict[str, str] = {'vllm': self.images['vllm']} + # A gateway-only project (kubeai's) runs no engine here. + wanted: dict[str, str] = ({} if self.fronts_elsewhere + else {'vllm': self.images['vllm']}) if self.litellm: wanted['litellm'] = self.images['litellm'] if self.dynamic_routing: @@ -1396,6 +1398,9 @@ def doctor(self) -> list[tuple[str, bool, str]]: # 4. GPUs. Last because a CPU-only stack is legitimate (mock endpoints, # the null backend), so this is informational rather than fatal. + # Not for a gateway-only project: its engines are elsewhere. + if self.fronts_elsewhere: + return checks try: gpus = self.inventory.get('gpus') or [] if gpus: diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 6da550ac..31d7b03f 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -1134,3 +1134,12 @@ def test_env_and_test_use_the_kubeai_gateways_key(tmp_path, monkeypatch): env_path, base = commands_leasing._gateway_state(None) assert env_path == be.gateway.gateway._env_path assert commands_leasing._front_door(None) == (base.rstrip('/'), key) + + +def test_doctor_checks_the_host_gateways_docker_too(tmp_path): + """The gateway on this host is a Compose project: without Docker the + first acquire failed after a clean preflight.""" + be, _ = make_front_door_backend(tmp_path) + names = [name for name, ok, _ in be.doctor()] + assert 'gateway: docker daemon reachable' in names + assert not any('image vllm' in n or 'GPUs visible' in n for n in names) From 9696fa090da276b5f55ca89ca9d9155b1f77019d Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 14:50:05 -0400 Subject: [PATCH 057/141] Docs: the manual describes the TUI's runtime pane and API tab as they are It described a "docker" pane with a Containers docker-ps view and an api pane; they are the runtime pane (logs, Instances, Apply/Down, either backend) and the API tab. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/source/manual/leasing-demo.md | 14 ++++++++------ infer_stack/tui.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/manual/leasing-demo.md b/docs/source/manual/leasing-demo.md index 9005b5a9..436bc185 100644 --- a/docs/source/manual/leasing-demo.md +++ b/docs/source/manual/leasing-demo.md @@ -168,12 +168,14 @@ something is still warming up vs truly live. > **Prefer a live dashboard?** `infer-stack tui` opens a multi-pane Textual UI > (run `infer-stack config init` once first): > a **catalog** pane to pick an endpoint and serve it (`s`/Enter), live -> **leases** + **deployments** tables, and three collapsible panes below: **docker** -> (a `logs -f` tail + a **Containers** `docker ps` view: status/uptime · created -> · container id · ports), **system** (`nvidia-smi` GPUs + host CPU/mem), and -> **api** (send a prompt to a *ready* model — only running models are listed). -> Each pane carries its own description and buttons; expanding a pane (or its -> tab) is what triggers its polling, so hidden data is never fetched. +> **leases** + **deployments** tables, and two collapsible panes below: +> **runtime** (`c`; the engines' logs, an **Instances** view of the +> containers or pods and what each serves, and **Apply** / **Down**, on +> either backend) and **system** (`nvidia-smi` GPUs + host CPU/mem). The +> **API** tab sends a prompt to a *ready* model (only running models are +> listed). Each pane carries its own description and buttons; expanding a +> pane (or its tab) is what triggers its polling, so hidden data is never +> fetched. > New here? The catalog buttons (or `g`) **Suggest** a set sized to your GPUs, > and `m` / `n` open wizards to add a model / endpoint by hand. **Clear finished** > (`x`) forgets released/stopped entries (`gc --forget`; unlike `infer-stack diff --git a/infer_stack/tui.py b/infer_stack/tui.py index aed8b25d..987492a1 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1477,7 +1477,7 @@ def _drag_sidebar(self, delta: int) -> None: self._apply_sizes() def _drag_logs(self, delta: int) -> None: - # Dragging the divider down (delta > 0) makes the docker pane shorter. + # Dragging the divider down (delta > 0) makes the runtime pane shorter. self._log_h = max(6, min(60, self._log_h - delta)) self._apply_sizes() From 9bd31548a9a2a4f08b962c85c36e227cfb1da7c4 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:01:44 -0400 Subject: [PATCH 058/141] ps and the TUI show instance start times in local time The runtimes report UTC; everything else on screen is local. The UX audit script refuses to run without infer-stack on PATH instead of passing hollow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/ux_audit.sh | 7 ++++++- docs/queue.md | 11 +++++++++++ infer_stack/cli/commands_runtime.py | 4 +++- infer_stack/leasing/instances.py | 29 +++++++++++++++++++++++++++++ infer_stack/tui.py | 4 +++- tests/test_tui.py | 4 +++- 6 files changed, 55 insertions(+), 4 deletions(-) diff --git a/dev/ux_audit.sh b/dev/ux_audit.sh index 144a4f3e..097dbb40 100755 --- a/dev/ux_audit.sh +++ b/dev/ux_audit.sh @@ -14,6 +14,8 @@ set -uo pipefail BACKEND="${1:-compose}" +# A missing command must not read as a clean pass. +command -v infer-stack >/dev/null || { echo 'infer-stack is not on PATH' >&2; exit 2; } HERE="$(cd "$(dirname "$0")/.." && pwd)" WORK="$(mktemp -d "${TMPDIR:-/tmp}/infer-stack-ux.XXXXXX")" export INFER_STACK_CONFIG_DIR="$WORK/config" INFER_STACK_DATA_DIR="$WORK/data" @@ -26,12 +28,15 @@ run() { # run a command, keep its output, and check it for known bad shapes local out rc out=$(timeout 300 infer-stack "$@" 2>&1); rc=$? { echo "\$ infer-stack $* (rc=$rc)"; echo "$out" | head -20; echo; } >> "$REPORT" + [ "$rc" -ge 124 ] && flag "rc=$rc (timeout or not run): infer-stack $*" echo "$out" | grep -q 'Traceback' && flag "traceback: infer-stack $*" echo "$out" | grep -q 'Write .env' && flag "stray 'Write .env': infer-stack $*" echo "$out" | grep -qE 'ttl=@[0-9]' && flag "raw timestamp: infer-stack $*" echo "$out" | grep -q $'\x1b\[' && flag "ANSI codes in piped output: infer-stack $*" if [ "$BACKEND" = kubeai ]; then - echo "$out" | grep -v '^\s*\(INFO\|[0-9:]* INFO\)' \ + # The host-side gateway really is a compose project: doctor's "gateway:" + # checks name docker compose because that gateway needs it. + echo "$out" | grep -v '^\s*\(INFO\|[0-9:]* INFO\)' | grep -v '^\[[a-z ]*\] gateway: ' \ | grep -iqE '\bcompose (project|backend)\b|docker compose' \ && flag "compose wording on kubeai: infer-stack $*" fi diff --git a/docs/queue.md b/docs/queue.md index 46c4313a..7254a998 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -293,6 +293,17 @@ the two reports differ only in backend facts), the reports read by eye. compose, its images; no engine image or GPU check) now follow KubeAI's. +Audit pass 5 (2026-09-26): `dev/ux_audit.sh` on both backends. + +- [x] `ps` (and the TUI's Instances table) printed the runtime's UTC start + time with no zone, 4 hours off every other time on screen here. Now + local time; `ps --json` keeps the runtime's UTC stamp. +- [x] The audit script itself: without `infer-stack` on PATH every command + returned 127 and the pass reported 0 flags. It now refuses to start, + and flags a timeout or a command that did not run. Doctor's `gateway:` + lines on kubeai no longer trip the Compose-wording check: that gateway + is a Compose project and needs it. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 34eb2f6e..01d7a26e 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -450,12 +450,14 @@ def _ps_rows(instances, served) -> list[dict[str, Any]]: def _print_ps(rows) -> None: + from ..leasing.instances import local_time + def cell(row): serves = ', '.join(row['serves']) or ('-' if row['deployment'] else '(front door)') gpus = ','.join(map(str, row['gpus'])) or '-' ident = row['id'][:12] if row['runtime'] == 'docker' else '-' return (row['name'], row['status'], serves, gpus, - (row['started'] or '-')[:19], ident, row['ports'] or '-') + local_time(row['started'] or '') or '-', ident, row['ports'] or '-') head = ('NAME', 'STATUS', 'SERVES', 'GPUS', 'STARTED', 'ID', 'PORTS') table = [head, *(cell(r) for r in rows)] diff --git a/infer_stack/leasing/instances.py b/infer_stack/leasing/instances.py index 6a9db195..92ee2d0e 100644 --- a/infer_stack/leasing/instances.py +++ b/infer_stack/leasing/instances.py @@ -60,6 +60,35 @@ def status(self) -> str: return f'{self.state} ({", ".join(bits)})' if bits else self.state +def local_time(started: str) -> str: + """A runtime's UTC start stamp as local ``YYYY-MM-DD HH:MM:SS``. + + Docker and Kubernetes both report UTC; everything else a person reads + here (log lines, lease TTLs) is local, so a bare UTC time reads as hours + off. Anything unparsable is shown as given, trimmed to the second. + + >>> import os, time + >>> os.environ['TZ'] = 'America/New_York'; time.tzset() + >>> local_time('2026-09-26T18:51:30Z') + '2026-09-26 14:51:30' + >>> local_time('2026-09-26T18:50:17.123456789Z') + '2026-09-26 14:50:17' + >>> local_time('') + '' + >>> local_time('soon') + 'soon' + >>> del os.environ['TZ']; time.tzset() + """ + import re + from datetime import datetime, timezone + + m = re.match(r'(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)', started or '') + if not m: + return (started or '')[:19] + stamp = datetime.fromisoformat(m.group(1)).replace(tzinfo=timezone.utc) + return stamp.astimezone().strftime('%Y-%m-%d %H:%M:%S') + + def from_residency(residency, *, runtime: str = DOCKER, namespace: str = '', container: str = '') -> list[Instance]: """Every instance in a :class:`~infer_stack.leasing.residency.Residency`.""" diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 987492a1..518eb310 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -2033,6 +2033,8 @@ def _instance_list(self): def _ps_rows(self, instances) -> list[dict[str, str]] | None: """Rows for the Instances table (same shape as ``infer-stack ps``).""" + from .leasing.instances import local_time + if instances is None: return None served = {g.id: sorted(g.served) for g in (self._last_deployments or [])} @@ -2041,7 +2043,7 @@ def _ps_rows(self, instances) -> list[dict[str, str]] | None: 'status': i.status, 'serves': (', '.join(served.get(i.deployment_id, [])) if i.deployment_id else '(front door)'), - 'started': (i.started or '')[:19].replace('T', ' '), + 'started': local_time(i.started), 'ports': i.ports, } for i in instances] diff --git a/tests/test_tui.py b/tests/test_tui.py index b9b14c90..e563e64b 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -935,7 +935,9 @@ async def scenario(): engine, gateway = app._ps_rows(instances) assert engine['serves'] == 'qwen' assert engine['status'] == 'restarting (CrashLoopBackOff, 2 restarts)' - assert engine['started'] == '2026-06-19 00:00:00' + # Shown in local time, like every other time a person reads here. + from infer_stack.leasing.instances import local_time + assert engine['started'] == local_time('2026-06-19T00:00:00Z') assert gateway['serves'] == '(front door)' assert gateway['ports'] == '14042->4000/tcp' assert app._ps_rows(None) is None # the runtime was unreadable From 4c772374adadef48b188b2098f15efe0f5ea32b0 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:11:37 -0400 Subject: [PATCH 059/141] TUI: one gateway URL authority, tab keys, a sidebar that uses the width Gateway.urls() derives the client and Open WebUI URLs once; the env file, env, test and the TUI read it through the backend's front door. The TUI built localhost: itself, wrong for a gateway in the cluster. Keys 1-5 and palette entries reach every top tab; the UI tab is now TUI settings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 17 +++++++ infer_stack/cli/commands_leasing.py | 11 ++-- infer_stack/leasing/gateway.py | 52 ++++++++++++++++--- infer_stack/tui.py | 78 +++++++++++++++++++++++------ tests/test_tui.py | 78 +++++++++++++++++++++++++---- 5 files changed, 200 insertions(+), 36 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 7254a998..3cc7df30 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -304,6 +304,23 @@ Audit pass 5 (2026-09-26): `dev/ux_audit.sh` on both backends. lines on kubeai no longer trip the Compose-wording check: that gateway is a Compose project and needs it. +Pass 5, the TUI by eye (compose, 80x24 and 200x50): + +- [x] At 200 columns the catalog sidebar stayed 38 wide and cut its gpu + column to "aut" beside an empty 160-column table. It now follows the + terminal (38 up to 64 columns) until resized by hand. +- [x] API, UI and Settings were reachable only by mouse or by tabbing into + the tab bar; the command palette found none of them. Keys 1-5 and + "Go to " palette entries now reach every top tab. +- [x] The API tab showed `http://localhost:14042/v1` while `env` and `test` + say `127.0.0.1`: the TUI derived the gateway and Open WebUI URLs from + the backend's ports itself, which is wrong for the in-cluster gateway + (a node's NodePort) and found no Open WebUI on kubeai. A duplicate + authority; now `Gateway.urls()` is the one derivation, and the env + file, `env`, `test` and the TUI all read it through the front door. +- [x] The tab named "UI" held the TUI's own refresh settings, beside "open + webui" everywhere else. Renamed "TUI settings", like "TUI log". + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 37d036a4..4e7d28b7 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2203,12 +2203,15 @@ def _gateway_state(config) -> tuple[Path, str]: """ from ..config import DEFAULT_PORTS + from ..leasing.gateway import front_door_urls + try: - front = getattr(_make_backend(config), 'front_door', lambda: None)() + backend = _make_backend(config) except Exception: # noqa: BLE001 - a lookup must not fail on the backend - front = None - if front is not None and getattr(front, 'litellm', False): - return front.gateway._env_path, f'{front.gateway._gateway_base()}/v1' + backend = None + base_url, _ = front_door_urls(backend) + if backend is not None and base_url is not None: + return backend.front_door().gateway._env_path, base_url return (data_root() / 'leasing' / 'compose' / '.env', f'http://127.0.0.1:{DEFAULT_PORTS["litellm"]}/v1') diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index a73dd7de..2d6c91ce 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -1232,6 +1232,25 @@ def merge_route_registry( ) return merged + def urls(self) -> tuple[str | None, str | None]: + """``(OpenAI base URL, Open WebUI URL)``; ``None`` for one that is off. + + The one derivation of where a client goes: the env-file descriptor, + ``env``, ``test`` and the TUI all read it (see :func:`front_door_urls`). + + >>> import tempfile + >>> gw = Gateway(tempfile.mkdtemp(), ports={'litellm': 14042, 'open_webui': 13000}, + ... litellm=True, ui=True) + >>> gw.urls() + ('http://127.0.0.1:14042/v1', 'http://127.0.0.1:13000') + >>> Gateway(tempfile.mkdtemp(), litellm=True, ui=False, + ... base_url='http://10.0.0.5:30442/').urls() + ('http://10.0.0.5:30442/v1', None) + """ + base = f'{self._gateway_base()}/v1' if self.litellm else None + ui = f'http://127.0.0.1:{self.ui_port}' if self.ui else None + return base, ui + def _gateway_base(self) -> str: where = self.base_url if where is None: @@ -1449,18 +1468,17 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: single base URL, but a managed Open WebUI (if on) is still a useful access point, so report just its URL rather than ``None``. """ - if not self.litellm: - if self.ui: - return {'ui_url': f'http://127.0.0.1:{self.ui_port}'} - return None + base_url, ui_url = self.urls() + if base_url is None: + return {'ui_url': ui_url} if ui_url else None info: dict[str, Any] = { - 'base_url': f'{self._gateway_base()}/v1', + 'base_url': base_url, 'api_key_env': API_KEY_ENV, 'api_key': self.master_key(), 'request_names': {ep: ep for ep in endpoints}, } - if self.ui: - info['ui_url'] = f'http://127.0.0.1:{self.ui_port}' + if ui_url: + info['ui_url'] = ui_url if self.reverse_proxy: # The unified front door: one origin, UI at / and the API at /v1. info['proxy_url'] = f'http://127.0.0.1:{self.reverse_proxy_port}' @@ -1474,3 +1492,23 @@ def merged_route_registry(self, incoming: dict[str, dict[str, Any]]) -> dict[str for w in warnings: logger.warning(' route registry: {}', w) return merged + + +def front_door_urls(backend) -> tuple[str | None, str | None]: + """``(OpenAI base URL, Open WebUI URL)`` of ``backend``'s front door. + + Whatever holds the gateway answers: the compose project itself, or on + kubeai the gateway on this host or in the cluster. ``(None, None)`` for a + backend with no gateway, or when asking fails (the cluster's node address + is read with kubectl): a URL lookup must not fail its caller. + + >>> from infer_stack.leasing import NullBackend + >>> front_door_urls(NullBackend()) + (None, None) + """ + try: + front = getattr(backend, 'front_door', lambda: None)() + gateway = getattr(front, 'gateway', None) + return gateway.urls() if gateway is not None else (None, None) + except Exception: # noqa: BLE001 - see the docstring + return None, None diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 518eb310..9e08ace3 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -35,7 +35,7 @@ from typing import Any, Callable from textual import events, work -from textual.app import App, ComposeResult +from textual.app import App, ComposeResult, SystemCommand from textual.binding import Binding from textual.coordinate import Coordinate from textual.containers import Horizontal, Vertical, VerticalScroll @@ -95,6 +95,9 @@ # candidate that is an actual sentinel object rather than a bool. #: Title of the tab that shows what the TUI itself did, and its errors. APP_LOG_TAB_TITLE = 'TUI log' +# The top-level tabs, in order: keys 1-5 and the command palette reach them. +TOP_TABS = (('Dashboard', 'tab-dashboard'), ('API', 'tab-api'), ('TUI settings', 'tab-ui'), + ('Settings', 'tab-settings'), (APP_LOG_TAB_TITLE, 'tab-applog')) #: The docker log pane shows at most this many lines (RichLog ``max_lines``). LOG_PANE_LINES = 2000 @@ -866,6 +869,9 @@ class InferStackTUI(App): Binding('minus', 'logs_shorter', 'logs -', show=False), Binding('plus', 'logs_taller', 'logs +', show=False), Binding('equals_sign', 'logs_taller', 'logs +', show=False), + # A focused Input takes digits first, so these never eat typing. + *(Binding(str(i), f"show_tab('{pane}')", label, show=False) + for i, (label, pane) in enumerate(TOP_TABS, 1)), ] def __init__( @@ -887,7 +893,7 @@ def __init__( self.controller = controller self.catalog = catalog self.interval = interval - # Two cadences (see the UI tab): the ledger is cheap in-memory state, so + # Two cadences (see the TUI settings tab): the ledger is cheap in-memory state, so # it drives the visible refresh; ``observe()``/``plan()`` shell out to # docker, so they run on a slower beat and their result is cached between # ledger ticks. Persisted UI prefs (tui_settings.yaml) override the @@ -961,6 +967,8 @@ def __init__( self._app_log_errors = 0 self._api_lines: list[str] = [] # mirror of the API output, for tests self._sidebar_w = 38 # resizable via [ ] or dragging #vsplit + # Follows the terminal's width (see _auto_sidebar) until resized by hand. + self._sidebar_set = False self._log_h = 16 # resizable via - + or dragging #hsplit self._models_h = 8 # resizable by dragging #csplit self._leases_h = 14 # resizable by dragging #tsplit @@ -991,7 +999,7 @@ def compose(self) -> ComposeResult: # Hidden at launch: built right after the first frame, not before. with TabPane('API', id='tab-api'): yield Lazy(_Section(self._compose_api, id='section-api')) - with TabPane('UI', id='tab-ui'): + with TabPane('TUI settings', id='tab-ui'): yield Lazy(_Section(self._compose_ui_settings, id='section-ui')) with TabPane('Settings', id='tab-settings'): yield Lazy(_Section(self._compose_settings, id='section-settings')) @@ -1453,8 +1461,23 @@ def _log_height(self) -> int: #: Below this many rows the pane descriptions give way to tables and logs. COMPACT_ROWS = 32 + @staticmethod + def _auto_sidebar(columns: int) -> int: + """The catalog sidebar's width before anyone drags it. + + 38 fits an 80-column terminal beside the tables; a wide terminal gives + the catalog's columns room instead of truncating them next to empty + space. + + >>> [InferStackTUI._auto_sidebar(c) for c in (80, 120, 160, 200, 300)] + [38, 40, 53, 64, 64] + """ + return max(38, min(64, columns // 3)) + def on_resize(self, event: events.Resize) -> None: self.set_class(event.size.height < self.COMPACT_ROWS, 'compact') + if not self._sidebar_set: + self._sidebar_w = self._auto_sidebar(event.size.width) try: self._apply_sizes() except Exception: # noqa: BLE001 - not mounted yet @@ -1473,6 +1496,7 @@ def _drag_sidebar(self, delta: int) -> None: # Allow the full width range (down to a sliver, up to nearly all of it), # not just the middle — clamp against the actual terminal width. hi = max(20, self.size.width - 12) + self._sidebar_set = True self._sidebar_w = max(10, min(hi, self._sidebar_w + delta)) self._apply_sizes() @@ -1771,7 +1795,7 @@ def _first_paint(self) -> None: def _apply_poll_settings(self) -> None: """Restart the refresh timer at the current ledger cadence and force the - next observe to run, so changes from the UI tab take effect at once.""" + next observe to run, so changes from the TUI settings tab take effect at once.""" if self._refresh_timer is not None: self._refresh_timer.stop() self._refresh_timer = self.set_interval( @@ -2301,6 +2325,25 @@ def _mark_app_log_tab(self) -> None: if self._app_log_errors else APP_LOG_TAB_TITLE ) + def action_show_tab(self, pane: str) -> None: + """Bring a top-level tab to the front (keys 1-5, the palette).""" + if pane == 'tab-applog': + self.action_show_app_log() + return + from textual.widgets._tabbed_content import ContentTabs + + top = self.query_one('#top', TabbedContent) + top.active = pane + # Focus the tab bar, as action_show_app_log focuses its log: a table + # still focused on the dashboard would pull the dashboard back. + top.query_one(ContentTabs).focus() + + def get_system_commands(self, screen): + yield from super().get_system_commands(screen) + for i, (label, pane) in enumerate(TOP_TABS, 1): + yield SystemCommand(f'Go to {label}', f'Show the {label} tab (key {i})', + lambda pane=pane: self.action_show_tab(pane)) + def action_show_app_log(self) -> None: """Jump to the TUI log (what the status line and the toast point at).""" self.query_one('#top', TabbedContent).active = 'tab-applog' @@ -2693,7 +2736,7 @@ def _on_apply_ui_settings(self) -> None: f'{self.observe_interval:g}s → {path}' ) except Exception as ex: # noqa: BLE001 - self._status(f'save UI settings failed: {ex}') + self._status(f'save TUI settings failed: {ex}') def _on_save_settings(self) -> None: from .paths import load_settings, save_settings @@ -2807,8 +2850,8 @@ def _ui_url(self, endpoint: str) -> str | None: return f'{base}/?models={endpoint}' if base else None def _openwebui_url(self) -> str | None: - port = getattr(self.controller.backend, 'ui_port', None) - return f'http://localhost:{port}' if port else None + from .leasing.gateway import front_door_urls + return front_door_urls(self.controller.backend)[1] def _served_endpoints(self) -> set[str]: try: @@ -3146,9 +3189,12 @@ def _do_suggest(self) -> None: # -- API tester -------------------------------------------------------- def _litellm(self) -> tuple[str | None, str | None]: + """``(OpenAI base URL ending in /v1, master key)``: what `env` prints.""" + from .leasing.gateway import front_door_urls + backend = self.controller.backend - port = getattr(backend, 'litellm_port', None) - if not port: + base, _ = front_door_urls(backend) + if not base: return None, None key = None mk = getattr(backend, 'master_key', None) @@ -3156,7 +3202,7 @@ def _litellm(self) -> tuple[str | None, str | None]: key = mk() if callable(mk) else None except Exception: # noqa: BLE001 key = None - return f'http://localhost:{port}', key + return base, key def _http_client(self) -> Any: if self._http is not None: @@ -3216,11 +3262,11 @@ def _api_chat(self, model: str, prompt: str) -> str: raise RuntimeError('no LiteLLM gateway (needs the compose backend)') headers = {'Authorization': f'Bearer {key}'} if key else {} if self._protocol_for(model) == 'completions': - url = f'{base}/v1/completions' + url = f'{base}/completions' body = {'model': model, 'prompt': prompt, 'max_tokens': 128, 'temperature': 0} else: - url = f'{base}/v1/chat/completions' + url = f'{base}/chat/completions' body = {'model': model, 'messages': [{'role': 'user', 'content': prompt}], 'max_tokens': 128, 'temperature': 0} @@ -3249,13 +3295,13 @@ def _curl_for(self, model: str, prompt: str, *, reveal_key: bool = False) -> str else: auth = '' if self._protocol_for(model) == 'completions': - path = '/v1/completions' + path = '/completions' body = _json.dumps({ 'model': model or '', 'prompt': prompt or 'hello', }) else: - path = '/v1/chat/completions' + path = '/chat/completions' body = _json.dumps({ 'model': model or '', 'messages': [{'role': 'user', 'content': prompt or 'hello'}], @@ -3270,7 +3316,7 @@ def _update_api_urls(self) -> None: ui = self._openwebui_url() parts = [] if base: - parts.append(f'gateway: {base}/v1') + parts.append(f'gateway: {base}') if ui: parts.append(f'open webui: {ui}') text = ' · '.join(parts) or '(acquire a model to get a gateway URL)' @@ -3364,7 +3410,7 @@ def _do_api_list(self) -> None: try: headers = {'Authorization': f'Bearer {key}'} if key else {} resp = self._http_client().get( - f'{base}/v1/models', headers=headers, timeout=30) + f'{base}/models', headers=headers, timeout=30) self._raise_for_body(resp) ids = [m.get('id') for m in (resp.json().get('data') or [])] self.call_from_thread( diff --git a/tests/test_tui.py b/tests/test_tui.py index e563e64b..2fa92c8b 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -31,6 +31,19 @@ def _ctx(): return controller, catalog +def _front(backend, litellm_port=14042, ui_port=None): + """Give a NullBackend a gateway front door at these ports.""" + import tempfile + + from infer_stack.leasing.gateway import Gateway + + ports = {'litellm': litellm_port or 0, 'open_webui': ui_port or 0} + gateway = Gateway(tempfile.mkdtemp(), ports=ports, + litellm=bool(litellm_port), ui=bool(ui_port)) + backend.front_door = lambda: type('Front', (), {'gateway': gateway})() + backend.master_key = lambda: 'sk-test' + + class _FakeProc: """Stands in for a `docker compose logs -f` process.""" @@ -1000,9 +1013,9 @@ async def scenario(): proc_factory=lambda svc: None) async with app.run_test() as pilot: await pilot.pause() - controller.backend.ui_port = 13000 + _front(controller.backend, None, 13000) assert app._ui_url('qwen-coder') == ( - 'http://localhost:13000/?models=qwen-coder' + 'http://127.0.0.1:13000/?models=qwen-coder' ) _run(scenario) @@ -1037,7 +1050,7 @@ async def scenario(): proc_factory=lambda svc: None, http=http) async with app.run_test() as pilot: await pilot.pause() - controller.backend.litellm_port = 14042 + _front(controller.backend, 14042) # only ready (running) models are offered; simulate one being ready app._sync_api_models(['qwen-coder']) assert app.query_one('#api-model', Select).value == 'qwen-coder' @@ -1078,7 +1091,7 @@ async def scenario(): proc_factory=lambda svc: None, http=_HTTP()) async with app.run_test() as pilot: await pilot.pause() - controller.backend.litellm_port = 14042 + _front(controller.backend, 14042) app._sync_api_models(['qwen-coder']) assert app.query_one('#api-model', Select).value == 'qwen-coder' app.action_api_send() @@ -1101,8 +1114,7 @@ async def scenario(): proc_factory=lambda svc: None) async with app.run_test() as pilot: await pilot.pause() - controller.backend.litellm_port = 14042 - controller.backend.ui_port = 13000 + _front(controller.backend, 14042, 13000) app._update_api_urls() await pilot.pause() text = str(app.query_one('#api-urls').render()) # must not raise @@ -1135,7 +1147,7 @@ async def scenario(): proc_factory=lambda svc: None, http=http) async with app.run_test() as pilot: await pilot.pause() - controller.backend.litellm_port = 14042 + _front(controller.backend, 14042) app._sync_api_models(['qwen-coder']) app._update_api_curl() curl = str(app.query_one('#api-curl').render()) @@ -1192,7 +1204,7 @@ async def scenario(): proc_factory=lambda svc: None, http=http) async with app.run_test() as pilot: await pilot.pause() - controller.backend.litellm_port = 14042 + _front(controller.backend, 14042) app._sync_api_models(['legacy-completions']) assert app.query_one('#api-model', Select).value == 'legacy-completions' # curl preview reflects the completions surface @@ -2236,7 +2248,7 @@ async def scenario(): proc_factory=lambda svc: None) async with app.run_test() as pilot: await pilot.pause() - app._litellm = lambda: ('http://localhost:14042', 'sk-secret-key') + app._litellm = lambda: ('http://127.0.0.1:14042/v1', 'sk-secret-key') app._sync_api_models(['qwen-coder']) app._update_api_curl() shown = str(app.query_one('#api-curl').render()) @@ -2275,3 +2287,51 @@ async def scenario(): assert '\x1b' not in text and '[1;36m' not in text _run(scenario) + + +def test_tui_sidebar_follows_terminal_width_until_resized_by_hand(): + # Pass 5 of the UX audit: at 200 columns the catalog sidebar stayed 38 + # wide and cut its gpu column to "aut" beside 160 columns of empty table. + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test(size=(200, 50)) as pilot: + await pilot.pause() + assert app.query_one('#sidebar').size.width == 64 + await pilot.press('right_square_bracket') + await pilot.pause() + assert app._sidebar_w == 68 + await pilot.resize_terminal(80, 24) + await pilot.pause() + assert app._sidebar_w == 68 # a hand-set width is kept + + _run(scenario) + + +def test_tui_number_keys_and_palette_reach_every_top_tab(): + # Pass 5 of the UX audit: API, UI and Settings were reachable only by + # mouse or by tabbing into the tab bar; the palette knew none of them. + from textual.widgets import TabbedContent + + from infer_stack.tui import TOP_TABS, InferStackTUI + + controller, catalog = _ctx() + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test(size=(80, 24)) as pilot: + await pilot.pause() + top = app.query_one('#top', TabbedContent) + for i, (_, pane) in reversed(list(enumerate(TOP_TABS, 1))): + await pilot.press(str(i)) + await pilot.pause() + assert top.active == pane + titles = [c.title for c in app.get_system_commands(app.screen)] + assert {f'Go to {label}' for label, _ in TOP_TABS} <= set(titles) + + _run(scenario) From 052fb36ae85ac77262aaa93928f0d510e59a7761 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:18:04 -0400 Subject: [PATCH 060/141] TUI: the runtime pane grows when the terminal does The resize handler sized the log pane from the app's size, which still held the old one, so a terminal enlarged after start kept one log line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 7 +++++++ infer_stack/tui.py | 11 ++++++----- tests/test_tui.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 3cc7df30..228d3bc8 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -321,6 +321,13 @@ Pass 5, the TUI by eye (compose, 80x24 and 200x50): - [x] The tab named "UI" held the TUI's own refresh settings, beside "open webui" everywhere else. Renamed "TUI settings", like "TUI log". +Pass 5, the TUI by eye on kubeai (80x24, then enlarged to 200x50): + +- [x] Enlarging the terminal left the runtime pane at its small-screen + height: one log line under a 26-row leases pane. The resize handler + read the app's size before it updated; it now uses the event's. + Instances, Control and Deployments read the same as on compose. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 9e08ace3..09409072 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1449,13 +1449,13 @@ def _draw_activity(self) -> None: # -- resizable panes --------------------------------------------------- - def _log_height(self) -> int: + def _log_height(self, rows: int | None = None) -> int: """The runtime pane's log height: the chosen one, capped near half the screen, so on a small terminal the lease and deployment tables keep rows when the pane opens (its tabs and picker take ~9 rows already).""" # The tabbed area holds the tab strip (2) and the source picker (3) # before any log line: 8 is those and three lines. - rows = self.size.height or 50 + rows = rows or self.size.height or 50 return max(8, min(self._log_h, rows // 2 - 4)) #: Below this many rows the pane descriptions give way to tables and logs. @@ -1479,13 +1479,14 @@ def on_resize(self, event: events.Resize) -> None: if not self._sidebar_set: self._sidebar_w = self._auto_sidebar(event.size.width) try: - self._apply_sizes() + # The event's size: the app's own still reads the old one here. + self._apply_sizes(event.size.height) except Exception: # noqa: BLE001 - not mounted yet pass - def _apply_sizes(self) -> None: + def _apply_sizes(self, rows: int | None = None) -> None: self.query_one('#sidebar').styles.width = self._sidebar_w - self.query_one('#docker-tabs').styles.height = self._log_height() + self.query_one('#docker-tabs').styles.height = self._log_height(rows) # Fixed heights only matter beside a divider; a tab fills its pane. if not self.TABBED_CATALOG: self.query_one('#models').styles.height = self._models_h diff --git a/tests/test_tui.py b/tests/test_tui.py index 2fa92c8b..96ad270d 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -2335,3 +2335,25 @@ async def scenario(): assert {f'Go to {label}' for label, _ in TOP_TABS} <= set(titles) _run(scenario) + + +def test_tui_log_pane_grows_when_the_terminal_does(): + # Pass 5 of the UX audit: started at 80x24 and enlarged to 200x50, the + # runtime pane kept its small-screen height (one log line) because the + # resize handler read the app's size before it updated. + from infer_stack.tui import InferStackTUI + + controller, catalog = _ctx() + + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, + proc_factory=lambda svc: None) + async with app.run_test(size=(80, 24)) as pilot: + await pilot.pause() + small = app.query_one('#docker-tabs').styles.height.value + await pilot.resize_terminal(200, 50) + await pilot.pause() + assert app.query_one('#docker-tabs').styles.height.value > small + assert app.query_one('#docker-tabs').styles.height.value == app._log_height(50) + + _run(scenario) From 91d36a242576dc061b528521219a9882156b7815 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:24:15 -0400 Subject: [PATCH 061/141] A released stop deployment is reclaimed, not missing leases warned NOT-RUNNING and clean offered a second teardown for a reclaim: stop deployment its release had already torn down: both read every IDLE row as meant to run. Controller.keeps_up is the one reading of the reclaim policy, used by admission, health and clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 12 ++++++++++++ infer_stack/cli/commands_leasing.py | 9 ++++++--- infer_stack/leasing/controller.py | 21 +++++++++++++++++++-- tests/test_cli_leasing.py | 18 ++++++++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 228d3bc8..29a43bc0 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -328,6 +328,18 @@ Pass 5, the TUI by eye on kubeai (80x24, then enlarged to 200x50): read the app's size before it updated; it now uses the event's. Instances, Control and Deployments read the same as on compose. +Pass 5, the first run from the README (no-GPU path, empty roots): config +init, `catalog suggest --simulator --apply`, acquire, test, the env exports +and a raw request all worked as written. + +- [x] After `release --all` tore down the simulator (`reclaim: stop`), + `leases` warned `NOT-RUNNING` and `clean` offered to tear it down + again: both read every IDLE row as meant to run. The ledger keeps a + released `stop` deployment IDLE by design. `Controller.keeps_up` is + now the one reading of the reclaim policy (admission, health, + `clean`); such a row is `reclaimed`, and `clean` says the stack is + already clean ("no active leases, no models up"). + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 4e7d28b7..d75bb979 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1420,9 +1420,12 @@ def main(cls, argv=True, **kwargs): controller = _open_controller(config, interactive=False) leases, deployments = controller.ledger.status(virtual_expiry=True) active = [le for le in leases if le.state == LeaseState.ACTIVE] - held = [g for g in deployments - if g.state in (DeploymentState.LIVE, DeploymentState.IDLE)] observed, assignments = _placement_view(controller) + # An idle `stop` deployment with nothing running was already torn + # down by its release: nothing is left to do but forget it (-f does). + held = [g for g in deployments + if g.state in (DeploymentState.LIVE, DeploymentState.IDLE) + and (controller.keeps_up(g) or g.id in observed)] # On any backend: one that selects its units by label (kubeai) has none. can_orphan = bool(config.orphans) @@ -1450,7 +1453,7 @@ def _list_only(orphans): print(json.dumps(plan, indent=2)) return 0 if not (active or held or found): - print('clean: already clean (no leases, no deployments, no orphans)') + print('clean: already clean (no active leases, no models up, no orphans)') return 0 print('clean: would release and tear down (dry run; -f to do it)') for le in active: diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 1cb9429a..43577c2c 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -518,6 +518,19 @@ def _render(self) -> ReconcileResult: # error. Backends that neither place nor inspect (dry-run, tests) get the # same surface from SimpleAdmission. + def keeps_up(self, deployment: Deployment) -> bool: + """Whether the reconciler keeps ``deployment`` running. + + LIVE always; IDLE only under ``keep-warm`` (the default policy). An + IDLE ``stop`` deployment is torn down by the release that idled it + and stays IDLE in the ledger, so a view that treats every IDLE row as + running reports a missing container that is exactly as intended. + """ + if deployment.state == DeploymentState.LIVE: + return True + return (deployment.state == DeploymentState.IDLE + and deployment.spec.get('reclaim', self.reclaim_default) == KEEP_WARM) + def _stored_state(self, lease_id: str): """A lease's state as stored (not virtually expired), or ``None``.""" lease = self.ledger.get_lease(lease_id) @@ -563,7 +576,7 @@ def _admission_view( # unique container). Never freshly placed; it blocks new # allocation until released (see _admit). elif deployment.state == DeploymentState.IDLE: - if deployment.spec.get('reclaim', self.reclaim_default) != KEEP_WARM: + if not self.keeps_up(deployment): continue resident = residency.resident(gid) if residency is not None else None if resident is not None and not resident.all_gpus: @@ -682,6 +695,8 @@ def observe_state(self) -> dict: neither started nor removed until its lease is released); * ``displaced``: an idle keep-warm model that yielded its GPUs; * ``unresolved``: LIVE from before allocations, with no unique container; + * ``reclaimed``: idle under ``stop``, and its container is gone, as + intended (see :meth:`keeps_up`); * ``running`` / ``not-running``: whether its single container serves. """ from .profile import profile_drift @@ -718,8 +733,10 @@ def observe_state(self) -> dict: and residency is not None and residency.resident(g.id) is None): condition = 'unresolved' + elif residency is not None and residency.resident(g.id): + condition = 'running' elif residency is not None: - condition = 'running' if residency.resident(g.id) else 'not-running' + condition = 'not-running' if self.keeps_up(g) else 'reclaimed' else: condition = None rows.append({'id': g.id, 'state': g.state, 'condition': condition, diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 9dcb0095..a6d846d4 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -1121,6 +1121,24 @@ def test_clean_force_releases_leases_and_evicts_keep_warm(env, capsys): assert all(d['state'] == 'stopped' for d in data['deployments']) + +def test_a_released_stop_deployment_is_neither_missing_nor_left_to_clean(env, capsys): + # UX audit pass 5, the README's first run: after `release --all` tore a + # `reclaim: stop` model down, `leases` warned NOT-RUNNING and `clean` + # offered to tear it down again. It is IDLE in the ledger by design. + from infer_stack.cli.commands_leasing import CleanCLI + + catalog = yaml.safe_load(open(env.cat)) + catalog['endpoints']['reranker']['reclaim'] = {'policy': 'stop'} + open(env.cat, 'w').write(yaml.safe_dump(catalog)) + AcquireCLI.main(argv=['reranker', *_base(env)]) + ReleaseCLI.main(argv=['--ledger', env.db, '--all']) + capsys.readouterr() + LeasesCLI.main(argv=['--ledger', env.db]) + assert 'NOT-RUNNING' not in capsys.readouterr().out + assert CleanCLI.main(argv=['--ledger', env.db]) == 0 + assert 'already clean' in capsys.readouterr().out + def test_clean_reports_an_already_clean_stack(env, capsys): from infer_stack.cli.commands_leasing import CleanCLI From 6495f5cc7208858c28c05e2b57a7ab69bdc1ab4c Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:25:00 -0400 Subject: [PATCH 062/141] release: a summary that reads as a sentence Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 14 ++++++++++++++ infer_stack/cli/commands_leasing.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 29a43bc0..61c1a600 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -340,6 +340,20 @@ and a raw request all worked as written. `clean`); such a row is `reclaimed`, and `clean` says the stack is already clean ("no active leases, no models up"). +Pass 5, consistency: product names (Open WebUI, LiteLLM, KubeAI, front +door, keep-warm) are spelled one way in user-facing text; the TUI's panes +and buttons use the CLI's words (leases, deployments, instances, apply, +down, `gc --forget` as "Clear finished"). + +- [x] `release`'s summary read "deployments idle/teardown per their reclaim + policy"; now "its models stay warm or stop, per their reclaim policy". +- [ ] Considered, not changed: argparse accepts abbreviated flags, so + `leases --all` fails as "ambiguous option: --all could match + --allowed_gpus". kwconf's `__allow_abbrev__ = False` would give + "unrecognized arguments" instead, and would break any script that + abbreviates a flag today. Backwards compatibility wins; revisit with a + deprecation warning if it bites. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index d75bb979..7191e048 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1107,7 +1107,7 @@ def _declined_exit() -> SystemExit: class ReleaseCLI(_ApprovalMixin): - """Release a lease; deployments idle/teardown per their reclaim policy. + """Release a lease; its models stay warm or stop, per their reclaim policy. On a terminal the resulting compose change is shown and confirmed before docker is touched (``--yes`` skips); declining leaves the lease released in From d88a50c357d009f3a849f4e60a1dd00bb950b58d Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:25:07 -0400 Subject: [PATCH 063/141] queue: the abbreviation finding is decided, not open Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 61c1a600..adf2752f 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -347,7 +347,7 @@ down, `gc --forget` as "Clear finished"). - [x] `release`'s summary read "deployments idle/teardown per their reclaim policy"; now "its models stay warm or stop, per their reclaim policy". -- [ ] Considered, not changed: argparse accepts abbreviated flags, so +- [x] Decided, no change: argparse accepts abbreviated flags, so `leases --all` fails as "ambiguous option: --all could match --allowed_gpus". kwconf's `__allow_abbrev__ = False` would give "unrecognized arguments" instead, and would break any script that From d4872aecb9e48fff82fb25e7b4297f5fa057442d Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:31:17 -0400 Subject: [PATCH 064/141] release/renew: a missing env-file is a message, not a traceback A cleanup trap after a failed acquire releases through an env-file that was never written. The UX audit removes its data root when done. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/ux_audit.sh | 13 ++++++++++++- docs/queue.md | 14 ++++++++++++++ infer_stack/cli/commands_leasing.py | 27 ++++++++++++++++++++++----- tests/test_cli_leasing.py | 13 +++++++++++++ 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/dev/ux_audit.sh b/dev/ux_audit.sh index 097dbb40..a2fbf72a 100755 --- a/dev/ux_audit.sh +++ b/dev/ux_audit.sh @@ -27,7 +27,15 @@ flag() { echo "!! $*" | tee -a "$REPORT"; FLAGS=$((FLAGS + 1)); } run() { # run a command, keep its output, and check it for known bad shapes local out rc out=$(timeout 300 infer-stack "$@" 2>&1); rc=$? - { echo "\$ infer-stack $* (rc=$rc)"; echo "$out" | head -20; echo; } >> "$REPORT" + { # the head, and the tail where a failure says why + echo "\$ infer-stack $* (rc=$rc)" + if [ "$(echo "$out" | wc -l)" -gt 26 ]; then + echo "$out" | head -18; echo ' [...]'; echo "$out" | tail -6 + else + echo "$out" + fi + echo + } >> "$REPORT" [ "$rc" -ge 124 ] && flag "rc=$rc (timeout or not run): infer-stack $*" echo "$out" | grep -q 'Traceback' && flag "traceback: infer-stack $*" echo "$out" | grep -q 'Write .env' && flag "stray 'Write .env': infer-stack $*" @@ -46,6 +54,9 @@ run() { # run a command, keep its output, and check it for known bad shapes cleanup() { infer-stack clean -f >/dev/null 2>&1 infer-stack stack down >/dev/null 2>&1 + # The data root holds Open WebUI's files and model weights, about 1 GB a + # run: kept, a day of passes put the dev cluster's node under disk pressure. + rm -rf "$WORK/data" echo "report: $REPORT" echo "flags: $FLAGS" } diff --git a/docs/queue.md b/docs/queue.md index adf2752f..5778a764 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -354,6 +354,20 @@ down, `gc --forget` as "Clear finished"). abbreviates a flag today. Backwards compatibility wins; revisit with a deprecation warning if it bites. +Audit pass 6 (2026-09-26): `dev/ux_audit.sh` on both backends. Compose 0 +flags; kubeai 2. + +- [x] `release --env-file lease.env` and `renew --env-file ...` on a file the + acquire never wrote (it failed and rolled back): a FileNotFoundError + traceback, which is what a cleanup trap hits. Now "no env-file at ...; + the acquire that writes it did not finish", and a file without a lease + id says so. +- [x] The acquire failed because the dev cluster's node was under disk + pressure and evicted KubeAI: ten audit data roots (0.9 GB each, Open + WebUI's files and the weights) were still in /tmp. The audit now + removes its data root and keeps the report, whose long outputs keep + their tail (where a failure says why). `doctor` named the dead API. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 7191e048..c3d01690 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -432,10 +432,27 @@ def _resolve(catalog, names, *, sharing=None): raise SystemExit(str(ex)) -def _resolve_lease(config) -> str | None: +def _resolve_lease(config, verb: str) -> str | None: + """The lease id given directly, or the one an ``--env-file`` records. + + A missing env-file is the usual cleanup-trap case: the acquire that would + have written it failed (and rolled back), so there is nothing of it to + release. Said so, not a traceback. + """ + from ..leasing.envfile import LEASE_ENV + sid = getattr(config, 'lease', None) - if not sid and getattr(config, 'env_file', None): - sid = read_lease_id(config.env_file) + env_file = getattr(config, 'env_file', None) + if not sid and env_file: + path = Path(env_file).expanduser() + if not path.is_file(): + raise SystemExit( + f'{verb}: no env-file at {path}; the acquire that writes it did ' + 'not finish, so it holds no lease (`infer-stack leases` lists ' + 'what is held)') + sid = read_lease_id(path) + if not sid: + raise SystemExit(f'{verb}: {path} names no lease ({LEASE_ENV})') return sid @@ -1147,7 +1164,7 @@ def main(cls, argv=True, **kwargs): raise SystemExit('release: --all takes no lease/--env-file') targets = None else: - sid = _resolve_lease(config) + sid = _resolve_lease(config, 'release') if not sid: raise SystemExit( 'release: give a lease id, --env-file, or --all' @@ -1819,7 +1836,7 @@ class RenewCLI(_LeasingCommonMixin): def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) - sid = _resolve_lease(config) + sid = _resolve_lease(config, 'renew') if not sid: raise SystemExit('renew: give a lease id or --env-file') # Through the controller: a renew can revive an idle deployment, which diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index a6d846d4..8f4b7a91 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -1156,3 +1156,16 @@ def test_clean_json_dry_run_is_pure_json(env, capsys): assert data['dry_run'] is True assert [le['endpoints'] for le in data['leases']] == [['qwen-coder']] assert data['deployments'][0]['served'] == ['qwen-coder'] + + +def test_release_and_renew_name_a_missing_env_file(env, tmp_path): + # UX audit pass 6: a cleanup trap after a failed acquire ran + # `release --env-file lease.env` on a file never written: a traceback. + missing = str(tmp_path / 'lease.env') + with pytest.raises(SystemExit, match='release: no env-file at .*did not finish'): + ReleaseCLI.main(argv=['--ledger', env.db, '--env-file', missing]) + with pytest.raises(SystemExit, match='renew: no env-file at'): + RenewCLI.main(argv=['--ledger', env.db, '--env-file', missing, '--ttl', '1h']) + (tmp_path / 'lease.env').write_text('export OPENAI_BASE_URL=x\n') + with pytest.raises(SystemExit, match='names no lease'): + ReleaseCLI.main(argv=['--ledger', env.db, '--env-file', missing]) From 1692fdbd535af43e59a72732decebdf5662cfcd6 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:48:42 -0400 Subject: [PATCH 065/141] dev: the second k3s node reclaims a fixed 1Gi under disk pressure On a shared, nearly full disk the default minimum reclaim (10%) kept the node tainted after the space came back. Journal and lesson for passes 5-6. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/journals/claude.md | 22 ++++++++++++++++++++++ dev/k3s_agent_container.sh | 7 ++++++- dev/lessons/lessons.md | 16 ++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/dev/journals/claude.md b/dev/journals/claude.md index c2441faa..f3b3bdae 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3290,3 +3290,25 @@ suite with the other branch forced on: the failures list exactly what to migrate. (2) A seam that wraps a runner must wrap only what it owns; a wrapper that replaces is a second authority. (3) Put every e2e check's output in a file before testing it. + +## 2026-09-26 — UX audit passes 5 and 6 + +**Did.** Pass 5 (both audit scripts, the TUI by eye on both backends at +80x24 and 200x50, the README's first run from empty roots, a naming grep) +found eight things, each fixed with a test: instance start times in UTC, +the TUI deriving the gateway URL itself (a duplicate authority; now +`Gateway.urls()`), top tabs unreachable from the keyboard, a sidebar and +log pane that ignored the terminal's size, and a released `reclaim: stop` +deployment reported as missing (now `Controller.keeps_up`). Pass 6 found a +traceback on `release --env-file` for a file never written. + +**Environment.** Pass 6's kubeai run failed because the dev cluster's node +went into disk pressure: my audit roots filled the shared disk. Deleting +them did not clear the taint (minimum reclaim, lessons.md). I wrote +`/etc/rancher/k3s/config.yaml` with a 1Gi minimum reclaim and restarted k3s; +remove the file to undo it. + +**Takeaways.** (1) An audit that passes when its command is missing is not +an audit: the script now refuses to start without `infer-stack` on PATH. +(2) Read a report whole: the one pass-5 finding I nearly missed was a line +my own `sed` range skipped. diff --git a/dev/k3s_agent_container.sh b/dev/k3s_agent_container.sh index 379594af..f5053431 100755 --- a/dev/k3s_agent_container.sh +++ b/dev/k3s_agent_container.sh @@ -18,6 +18,10 @@ set -euo pipefail ACTION="${1:-up}" NAME="${2:-k3s-agent-b}" CONTAINER="infer-stack-${NAME}" +# The node shares the host's disk. The kubelet's default minimum reclaim (10% +# above the 5% hard threshold) cannot be met on a disk other work keeps full, +# so one dip left the node tainted disk-pressure for good. +RECLAIM='eviction-minimum-reclaim=imagefs.available=1Gi,nodefs.available=1Gi' case "$ACTION" in up) @@ -28,7 +32,8 @@ case "$ACTION" in docker rm -f "$CONTAINER" >/dev/null 2>&1 || true docker run -d --name "$CONTAINER" --privileged --hostname "$NAME" \ -e K3S_URL="https://${server_ip}:6443" -e K3S_TOKEN="$token" \ - --tmpfs /run --tmpfs /var/run "$image" agent --node-name "$NAME" >/dev/null + --tmpfs /run --tmpfs /var/run "$image" agent --node-name "$NAME" \ + --kubelet-arg "$RECLAIM" >/dev/null for _ in $(seq 60); do kubectl get node "$NAME" >/dev/null 2>&1 && break sleep 2 diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index 31b2bbfb..cabd927c 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -125,3 +125,19 @@ evidence; prefer append-only; supersede incorrect entries with a new one. the pipeline and 0 for the file. - **Applies when:** an e2e or handover script checks a command's output with `grep -q` under `set -euo pipefail`. + +- **Lesson:** On a disk other work keeps nearly full, a k3s node that dips + under the 5% eviction threshold stays tainted `disk-pressure` after the + space comes back: the kubelet's default minimum reclaim adds 10%, so the + taint clears only at 15% free. Freeing your own files is not enough; set + `eviction-minimum-reclaim` to a fixed size (1Gi) on a dev cluster. +- **Evidence / MWE:** 2026-09-26 on the guest: ten UX-audit data roots + (0.9 GB each) pushed the disk over; after deleting them the node sat at + 12.8% free, still tainted, for 10 minutes, KubeAI's controller Pending. + `kubectl get --raw /api/v1/nodes//proxy/configz` showed + `evictionMinimumReclaim: 10%`. With `kubelet-arg: + eviction-minimum-reclaim=imagefs.available=1Gi,nodefs.available=1Gi` in + `/etc/rancher/k3s/config.yaml` and a k3s restart, the taint cleared. + `dev/k3s_agent_container.sh` passes the same to the second node. +- **Applies when:** a dev cluster shares its disk with anything else, and + whenever a test or audit keeps large artifacts in `/tmp`. From 548486ec78806a7455ad3db45292cbb9d772ec65 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 15:54:48 -0400 Subject: [PATCH 066/141] ps: GPUS says cpu or all, as leases does Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 8 ++++++++ infer_stack/cli/commands_runtime.py | 25 ++++++++++++++++++++++++- infer_stack/leasing/instances.py | 5 ++++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 5778a764..1227ea19 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -368,6 +368,14 @@ flags; kubeai 2. removes its data root and keeps the report, whose long outputs keep their tail (where a failure says why). `doctor` named the dead API. +Audit pass 7 (2026-09-26): `dev/ux_audit.sh` on both backends, 0 flags; +both reports read whole. + +- [x] For the same GPU-less mock, `leases` and `clean` said `gpus=cpu` and + `ps` said `-`. `ps` now uses the same words: `cpu` for a Docker engine + with no GPU, `all` for one that requests every GPU, `-` for a pod + (the cluster picks) or a front-door service. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 01d7a26e..430b4597 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -442,6 +442,7 @@ def _ps_rows(instances, served) -> list[dict[str, Any]]: 'status': inst.status, 'restarts': inst.restarts, 'gpus': list(inst.gpus), + 'all_gpus': inst.all_gpus, 'started': inst.started or None, 'ports': inst.ports or None, 'runtime': inst.runtime, @@ -449,12 +450,34 @@ def _ps_rows(instances, served) -> list[dict[str, Any]]: return rows +def _gpus_cell(row) -> str: + """``ps``'s GPUS cell, in the words ``leases`` uses for the same thing. + + A Docker engine with no device request runs on the CPU (``leases`` says + ``gpus=cpu``); one that requests every GPU says ``all``. A pod's GPUs are + the cluster's to pick, and a front-door service holds none: ``-``. + + >>> row = {'gpus': [], 'all_gpus': False, 'deployment': 'grp-1', 'runtime': 'docker'} + >>> _gpus_cell(row), _gpus_cell({**row, 'gpus': [0, 1]}), _gpus_cell({**row, 'all_gpus': True}) + ('cpu', '0,1', 'all') + >>> _gpus_cell({**row, 'deployment': None}), _gpus_cell({**row, 'runtime': 'kubernetes'}) + ('-', '-') + """ + if row['gpus']: + return ','.join(map(str, row['gpus'])) + if row.get('all_gpus'): + return 'all' + if row['deployment'] and row['runtime'] == 'docker': + return 'cpu' + return '-' + + def _print_ps(rows) -> None: from ..leasing.instances import local_time def cell(row): serves = ', '.join(row['serves']) or ('-' if row['deployment'] else '(front door)') - gpus = ','.join(map(str, row['gpus'])) or '-' + gpus = _gpus_cell(row) ident = row['id'][:12] if row['runtime'] == 'docker' else '-' return (row['name'], row['status'], serves, gpus, local_time(row['started'] or '') or '-', ident, row['ports'] or '-') diff --git a/infer_stack/leasing/instances.py b/infer_stack/leasing/instances.py index 92ee2d0e..013244c4 100644 --- a/infer_stack/leasing/instances.py +++ b/infer_stack/leasing/instances.py @@ -39,6 +39,8 @@ class Instance: #: ``healthy`` / ``starting`` / ``unhealthy``, when the runtime has a check. health: str = '' gpus: tuple[int, ...] = () + #: Docker only: a device request for every GPU, not a list of indices. + all_gpus: bool = False started: str = '' ports: str = '' runtime: str = DOCKER @@ -98,7 +100,8 @@ def from_residency(residency, *, runtime: str = DOCKER, namespace: str = '', out.append(Instance( name=name, id=c.container_id, deployment_id=c.deployment_id, state=c.state, restarts=c.restart_count, reason=c.reason, - health=c.health, gpus=tuple(c.gpus), started=c.started, ports=c.ports, + health=c.health, gpus=tuple(c.gpus), all_gpus=bool(c.all_gpus), + started=c.started, ports=c.ports, runtime=runtime, namespace=namespace, container=container, )) return sorted(out, key=lambda i: (not i.is_engine, i.name, i.id)) From ff38cd35c69ff2749e4f18b52ca8cc01eb332197 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 16:00:25 -0400 Subject: [PATCH 067/141] queue: the UX audit passes (pass 8 found nothing new) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 1227ea19..9db979b5 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -147,7 +147,7 @@ it for tensor parallelism. Adding it changes every vLLM service's fingerprint, so the next apply recreates running engines. Needs a TP=2 run on a GPU host to confirm the symptom first. **The operator's call.** -### 9. [ ] UX audit loop: do not stop without a passing audit +### 9. [x] UX audit loop: do not stop without a passing audit Polishing the UX can take hours; that is expected, and this item is the reason the queue exists. Loop: @@ -376,6 +376,11 @@ both reports read whole. with no GPU, `all` for one that requests every GPU, `-` for a pod (the cluster picks) or a front-door service. +Audit pass 8 (2026-09-26): **passed.** `dev/ux_audit.sh` on both backends +(0 flags, reports read whole, the only non-zero exits are the deliberate +mistakes); the README's no-GPU first run from empty roots; the TUI on both +backends at 80x24, enlarged to 200x50, every top tab by key. Nothing new. + ### 10. [ ] Handover Summarize for the operator: what was verified here, and the two handover From 7833668c74aad3851f54a392c528739b004c372a Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 16:11:34 -0400 Subject: [PATCH 068/141] queue: handover for the operator; the full kubeai e2e passes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 9db979b5..d4ec7ecf 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -381,7 +381,36 @@ Audit pass 8 (2026-09-26): **passed.** `dev/ux_audit.sh` on both backends mistakes); the README's no-GPU first run from empty roots; the TUI on both backends at 80x24, enlarged to 200x50, every top tab by key. Nothing new. -### 10. [ ] Handover +### 10. [x] Handover Summarize for the operator: what was verified here, and the two handover scripts (items 6 and 7) with what each run proves. + +**Verified here** (a VM with no GPU; k3s with a second node in a container): + +- Both backends through one admission path, day-2 commands and the TUI + through the backend's instances (items 1-5): the unit suite, the 21-row + parity table on both backends, and `dev/kubeai_e2e.sh` with every phase + (CPU vLLM: acquire, env-file client, gateway routes and Open WebUI, + release, refusal, `min_vram_gib` sizing on fake GPU labels, dynamic + routing, the gateway in the cluster, a Model on the second node). +- The UX audit (item 9): eight passes, the last with nothing new. + +**Needs GPUs or a second machine** (run on the operator's hosts): + +- `dev/handover/p4_gpu_labels.sh` on a GPU k3s node. Proves GPU Feature + Discovery's real labels feed sizing: `catalog suggest --backend kubeai` + proposes a profile per GPU product, an endpoint with only `min_vram_gib` + lands on the right one and answers, and `measure --record` reads vLLM's + memory profile from the pod log. Here those labels were faked. +- `dev/handover/p5_two_hosts.sh` on the first node once a second GPU + workstation has joined. Proves pod-to-pod traffic across the real + network: a Model on the second node's GPU answers through the in-cluster + gateway, the NodePort answers on the second node's own address (one env + file for cards on either machine), and `secrets rotate` holds with the + Model served. Here the second node shared the host's network. + +Both keep their own config and data roots and print PASS/FAIL per step. + +**Open decisions:** 8a (rewrite or delete the pre-leasing recipes) and 8b +(`/dev/shm` for vLLM, which recreates running engines). From 376784b63bdf190bff1424d3eff9dd3bd5d533f0 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 16:20:37 -0400 Subject: [PATCH 069/141] Decide /dev/shm and the pre-leasing recipes (queue 8a, 8b) runtime.shm_size sets a vLLM container's /dev/shm on Compose. Opt-in, so no running engine is recreated; a parallel engine without it warns once, and catalog suggest sets it on multi-GPU entries. KubeAI pods already mount one. The pre-leasing demos, recipes and examples are removed. Their 4 x 96 GB vLLM tuning lives on as three gated suggestion variants. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- .gitignore | 4 - CHANGELOG.md | 11 + docs/demos/ci_smoke_test.md | 203 -------------- docs/demos/ollama_direct_quickstart.md | 126 --------- docs/demos/quickstart.md | 227 ---------------- docs/demos/qwen_3090_profiles.md | 31 --- docs/demos/smollm2_gpu1_backend_switch.md | 132 --------- docs/persistent-caches-and-warm-restarts.md | 20 +- docs/queue.md | 49 ++-- examples/config.multi-model.yaml | 90 ------- examples/openwebui-tls-ldap/README.md | 63 ----- examples/single-node/README.md | 45 ---- examples/single-node/config.yaml | 84 ------ examples/single-node/models.yaml | 24 -- infer_stack/leasing/catalog.py | 27 +- infer_stack/leasing/compose.py | 28 ++ infer_stack/leasing/suggest.py | 3 + infer_stack/templates/suggestion-pool.yaml | 45 ++++ .../compose_pythia_inspect_mmlu_compat.md | 139 ---------- .../compose_pythia_qwen36_mixed_4x96GB.md | 193 -------------- recipies/compose_qwen35_122b_4x96GB.md | 222 ---------------- recipies/compose_qwen35_122b_fp8_4x96GB.md | 217 --------------- recipies/compose_qwen36_35b_a3b_4x96GB.md | 251 ------------------ tests/test_leasing_suggest.py | 20 ++ tests/test_shm_size.py | 56 ++++ 25 files changed, 234 insertions(+), 2076 deletions(-) delete mode 100644 docs/demos/ci_smoke_test.md delete mode 100644 docs/demos/ollama_direct_quickstart.md delete mode 100644 docs/demos/quickstart.md delete mode 100644 docs/demos/qwen_3090_profiles.md delete mode 100644 docs/demos/smollm2_gpu1_backend_switch.md delete mode 100644 examples/config.multi-model.yaml delete mode 100644 examples/openwebui-tls-ldap/README.md delete mode 100644 examples/single-node/README.md delete mode 100644 examples/single-node/config.yaml delete mode 100644 examples/single-node/models.yaml delete mode 100644 recipies/compose_pythia_inspect_mmlu_compat.md delete mode 100644 recipies/compose_pythia_qwen36_mixed_4x96GB.md delete mode 100644 recipies/compose_qwen35_122b_4x96GB.md delete mode 100644 recipies/compose_qwen35_122b_fp8_4x96GB.md delete mode 100644 recipies/compose_qwen36_35b_a3b_4x96GB.md create mode 100644 tests/test_shm_size.py diff --git a/.gitignore b/.gitignore index f7556690..943676a4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,6 @@ generated/* !generated/kubeai/ generated/kubeai/* !generated/kubeai/.gitignore -!examples/ -!examples/single-node/ -!examples/single-node/config.yaml -!examples/single-node/models.yaml # rsync target for e2e harness reports (developer artifacts, not source) e2e-report-*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 57289221..9f26b622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### `/dev/shm` for vLLM on Compose, and the old recipes as suggestions + +`runtime.shm_size` (e.g. `16g`) sets a vLLM container's `/dev/shm` on the +Compose backend; Docker's default is 64 MiB, and vLLM's workers share memory +when a model spans GPUs. It is opt-in, so no running engine is recreated; a +parallel engine without it logs a warning, and `catalog suggest` sets it on +multi-GPU entries. The pre-leasing `docs/demos/`, `recipies/` and `examples/` +are removed: their 4 x 96 GB vLLM tuning is now three suggestion variants +(`qwen3.5-122b-a10b-tp4-128k`, `qwen3.5-122b-a10b-fp8-tp4-262k`, +`qwen3.6-35b-a3b-tp2-262k`). + ### The gateway can run inside the cluster `infer-stack config set kubeai_gateway cluster` runs the LiteLLM gateway as a diff --git a/docs/demos/ci_smoke_test.md b/docs/demos/ci_smoke_test.md deleted file mode 100644 index 763c08e8..00000000 --- a/docs/demos/ci_smoke_test.md +++ /dev/null @@ -1,203 +0,0 @@ -# CI smoke test - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -Walks through the canonical `infer-stack` workflow end-to-end: pick a -profile, render the deployment, inspect the rendered artifacts, and -exercise a few read-only inspection commands. - -This file is **double-runnable**: read it as a tutorial, or replay -every code block in order with - -```text -pytest --codeblocks docs/demos/ci_smoke_test.md -``` - -to verify each step still works (the test suite does this in CI). Each -fenced ``bash`` block is a self-contained shell script — copy-paste any -one of them into a terminal and it runs on its own. - -All blocks share a single working directory so they build on each -other within one demo run: - -```text -$INFER_STACK_CONFIG_DIR = /tmp/infer-stack-demo/config -$INFER_STACK_DATA_DIR = /tmp/infer-stack-demo/data -``` - -Each block re-exports those variables at the top — that's required by -``pytest-codeblocks`` (each block runs as its own subprocess) and it -also makes the blocks usable as standalone shell snippets. - -## 1. Clean slate - -Wipe any previous demo state so this run starts fresh. - -If a previous run left containers running, use ``infer-stack purge`` so that -Docker-owned (root-written) directories are removed correctly: - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack purge --yes --delete-cache 2>/dev/null || true -``` - -Then recreate the empty directories: - -```bash -rm -rf /tmp/infer-stack-demo -mkdir -p /tmp/infer-stack-demo/config /tmp/infer-stack-demo/data -``` - -## 2. Set up a profile - -Pick the smallest vLLM test profile (``test-single-11gb`` runs SmolLM2 135M -on a single GPU) and write a config for it. This demo exercises the classic -`Open WebUI -> LiteLLM -> vLLM` path; use -``docs/demos/ollama_direct_quickstart.md`` for the no-LiteLLM Ollama path: - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack setup --backend compose --profile test-single-11gb -``` - -You should see ``Wrote …/config.yaml`` and a summary of the configured -backend / active profile. After this, ``config.yaml`` exists under -``$INFER_STACK_CONFIG_DIR``. A ``models.yaml`` file is only needed when you -add local ``vllm_models``, ``ollama_models``, or custom stack profiles. - -## 3. List the catalog - -What profiles are available? - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack list-profiles | head -20 -``` - -## 4. Inspect a single profile - -``describe-profile`` shows what a profile actually resolves to — -served model name, endpoint shape, transport details — without -rendering anything yet: - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack describe-profile test-single-11gb --format yaml --simulate-hardware 1x24 | head -30 -``` - -The ``--simulate-hardware`` flag pretends this host has the requested -number of GPUs, so describe-profile can plan placement even on a -machine without an actual GPU. Real hosts can omit it. - -## 5. Validate before rendering - -``validate`` runs the resolver and the policy checks; it exits -non-zero if the resolved deployment has errors. It writes ``plan.yaml`` -as a side effect so subsequent ``render`` calls see a consistent plan. - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack validate --simulate-hardware 1x24 -``` - -A clean run prints ``"ok": true`` with empty ``errors`` and -``warnings`` arrays. - -## 6. Render the compose stack - -``render`` writes the actual ``docker-compose.yml``, ``.env``, and -mounted-runtime files. ``--yes`` skips the interactive per-file -confirmation diff (which you want for scripted use). - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack render --yes --simulate-hardware 1x24 -``` - -You can confirm the artifacts landed where you expect: - -```bash -test -f /tmp/infer-stack-demo/data/generated/docker-compose.yml -test -f /tmp/infer-stack-demo/data/generated/.env -test -f /tmp/infer-stack-demo/data/generated/plan.yaml -``` - -## 7. Constrain placement to specific GPU indices - -If GPU 0 is already in use, you can ask the planner to consider only -other GPUs without editing any profile: - -```bash -export INFER_STACK_CONFIG_DIR=/tmp/infer-stack-demo/config -export INFER_STACK_DATA_DIR=/tmp/infer-stack-demo/data -infer-stack render --yes --simulate-hardware 4x24 --allowed-gpus 1 -grep -A1 device_ids /tmp/infer-stack-demo/data/generated/docker-compose.yml | head -3 -``` - -The rendered ``device_ids`` for the vLLM container now pins to GPU 1. -``INFER_STACK_ALLOWED_GPUS=1,3`` works as an env-var equivalent. - -## 8. Programmatic API - -Every subcommand is also importable as a Python class — useful for -notebooks, integration tests, and anything that wants the CLI's -semantics without spawning a subprocess: - -```python -import os, tempfile -os.environ["INFER_STACK_CONFIG_DIR"] = "/tmp/infer-stack-demo/config" -os.environ["INFER_STACK_DATA_DIR"] = "/tmp/infer-stack-demo/data" - -from infer_stack.cli import ListProfilesCLI, ValidateCLI - -# ListProfilesCLI prints to stdout; capture or just verify it runs. -ListProfilesCLI.main(argv=False) - -# ValidateCLI returns 0 on a clean plan, 2 on validation errors. -rv = ValidateCLI.main(argv=False, simulate_hardware="1x24") -assert rv == 0, f"validate returned {rv}" -``` - -## 9. Tear down - -Stop the stack and delete all Docker-written state (postgres data, caches, etc.): - -```bash -infer-stack purge --yes -``` - -Model weights in hf-cache and vllm-cache are preserved by default so a -subsequent run doesn't need to re-download them. Pass ``--delete-cache`` to -also wipe those directories. - -Afterwards the ``/tmp/infer-stack-demo`` directory can be removed normally: - -```bash -rm -rf /tmp/infer-stack-demo -``` - -## Where to go next - -- ``infer-stack up`` actually starts the compose stack (needs Docker - and real GPU access). Try it with one of the ``*-single`` profiles - on a workstation. -- ``infer-stack switch --apply`` swaps the active profile - on a running stack. -- The hardware-shape integration profiles are named ``test-*``: pick - ``test-single-11gb`` for a workstation card or ``test-multi-gpu`` - with ``--allowed-gpus 1,3`` to exercise tensor-parallel rendering. - - -## Readiness vs Docker health - -Use `infer-stack wait-ready` when a test must wait until the active model can -actually serve a request. Compose health can become true before a routed vLLM -model is ready through LiteLLM. `infer-stack smoke-test` runs this readiness -probe by default; pass `--no-wait` only when you intentionally want to test the -currently listening endpoint without waiting for model readiness. diff --git a/docs/demos/ollama_direct_quickstart.md b/docs/demos/ollama_direct_quickstart.md deleted file mode 100644 index fd790ad8..00000000 --- a/docs/demos/ollama_direct_quickstart.md +++ /dev/null @@ -1,126 +0,0 @@ -# Quickstart — direct Ollama on a dual GTX 1080 Ti host - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -This quickstart brings up the simplest local stack for Pascal-era GPUs: - -```text -Open WebUI -> Ollama -``` - -There is no LiteLLM router and no predeclared model registry. Ollama manages -its own model store under `state.ollama`, and models are pulled with the -Ollama CLI or through Open WebUI. - -## Prerequisites - -- Docker with the NVIDIA container runtime. -- `nvidia-smi` works on the host and from GPU-enabled containers. -- `infer-stack` installed in your Python environment. - -This path is useful for GTX 1080 Ti systems because Ollama supports Pascal -GPUs, while current official vLLM images require newer CUDA compute capability. - -## 1. Set up the profile - -```bash -infer-stack setup \ - --backend compose \ - --profile ollama-direct \ - --data-dir /data/service/docker/infer-stack -``` - -For a two-card 1080 Ti machine, the built-in profile pins Ollama to GPUs -`[0, 1]` and uses conservative daemon settings: - -```yaml -keep_alive: 2m -context_length: 4096 -num_parallel: 1 -max_loaded_models: 1 -max_queue: 8 -``` - -## 2. Render and start - -```bash -infer-stack render --yes --simulate-hardware 2x11 -infer-stack up -d -``` - -Expected rendered services: - -```text -postgres-open-webui -ollama -open-webui -``` - -Not expected in this profile: - -```text -postgres-litellm -litellm -vllm-* -``` - -Open WebUI is available at . Ollama is published on - because the direct profile sets `publish_port: true`. - -## 3. Pull a model - -Start with a small model before trying larger ones: - -```bash -infer-stack ollama-pull qwen3.5:4b -``` - -Other good initial candidates for 11 GiB cards: - -```bash -infer-stack ollama-pull qwen3.5:2b -infer-stack ollama-pull qwen3.5:9b-q4_K_M -infer-stack ollama-pull gemma4:e2b -``` - -Run a direct API check through the CLI smoke test. It uses Ollama's native API -for direct-Ollama profiles and reads rendered runtime settings automatically: - -```bash -infer-stack smoke-test --model qwen3.5:4b -``` - -## 4. Day-to-day operations - -```bash -infer-stack ps -infer-stack logs -f ollama -infer-stack down -infer-stack up -d -``` - -Inspect loaded models and where they live: - -```bash -infer-stack ollama-list -infer-stack ollama-ps -``` - -## 5. When to add LiteLLM - -Stay direct when the machine is just a local Ollama server or Home Assistant -backend. Switch to an Ollama gateway profile when you want a stable OpenAI -`/v1` route name that can later move to vLLM: - -```bash -infer-stack switch ollama-qwen3.5-4b-gateway --apply -``` - -That changes the shape to: - -```text -Open WebUI -> LiteLLM -> Ollama -``` - -For mixed Ollama+vLLM routing, use a profile such as `mixed-ollama-smollm` as -a template. diff --git a/docs/demos/quickstart.md b/docs/demos/quickstart.md deleted file mode 100644 index b9fe2169..00000000 --- a/docs/demos/quickstart.md +++ /dev/null @@ -1,227 +0,0 @@ -# Quickstart — vLLM + LiteLLM on an RTX 3090 workstation - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -This guide brings up the traditional vLLM-backed stack on a single-GPU -workstation where GPU 0 may be reserved for the desktop and GPU 1 is free for -inference. It starts with the tiniest vLLM model to prove the plumbing works, -then switches to progressively larger vLLM profiles. - -For the simpler Pascal/1080 Ti path that runs **Open WebUI -> Ollama** with no -LiteLLM and no predeclared model registry, use -[`ollama_direct_quickstart.md`](ollama_direct_quickstart.md). - -Rendered shape for this guide: - -```text -Open WebUI -> LiteLLM -> vLLM -``` - -Default ports for this profile family: - -- LiteLLM: -- Open WebUI: - -After setup, all state and generated artifacts live under -`/data/service/docker/infer-stack/`: - -```text -/data/service/docker/infer-stack/ - generated/ <- docker-compose.yml, .env, plan.yaml - hf-cache/ <- downloaded Hugging Face weights - vllm-cache/ <- compiled vLLM artifacts - open-webui/ <- Open WebUI state - postgres-open-webui/ <- Open WebUI database - postgres-litellm/ <- LiteLLM database, only when LiteLLM is enabled - runtime/ <- runtime bind-mount configs such as litellm_config.yaml - -~/.config/infer_stack/ - config.yaml <- active profile, backend, paths, ports - models.yaml <- optional custom vllm_models, ollama_models, profiles -``` - -## Prerequisites - -- Docker with the NVIDIA container runtime (`nvidia-smi` visible inside containers). -- `infer-stack` installed in your Python environment. -- A Hugging Face token for gated models, stored in the managed `.env`. - Compose never reads `HF_TOKEN` from your shell: - -```bash -infer-stack env HF_TOKEN=hf_... -``` - -The first profile in this guide, `gpt2-single`, is public and does not need -`HF_TOKEN`. - -## 1. First-time setup with GPT-2 - -Start with `gpt2-single`: GPT-2 124M, ~250 MB download, no auth, -completions-only. The goal is to validate Docker, GPU placement, LiteLLM, and -Open WebUI before committing to a large model. - -```bash -infer-stack setup \ - --backend compose \ - --profile gpt2-single \ - --data-dir /data/service/docker/infer-stack -``` - -Render and start: - -```bash -infer-stack render --yes -infer-stack up -d -``` - -vLLM downloads GPT-2 and compiles a small graph. Expect roughly 30-60 seconds -on first start and a few seconds on warm restart. - -Watch it come up: - -```bash -infer-stack ps -infer-stack logs -f -``` - -Once all containers are healthy, smoke-test the API: - -```bash -infer-stack smoke-test -``` - -Manual completion check: - -```bash -curl -s http://127.0.0.1:14042/v1/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(infer-stack env --key LITELLM_MASTER_KEY)" \ - -d '{ - "model": "gpt2", - "prompt": "Once upon a time, ", - "max_tokens": 32 - }' | python3 -m json.tool -``` - -Open WebUI is up at , but GPT-2 is a base model, so -switch to a chat-capable profile before using it interactively. - -## 2. Switch to SmolLM2 135M, chat-capable - -```bash -infer-stack switch smollm2-135m-single --apply -``` - -`switch --apply` updates `config.yaml`, re-renders the stack, removes orphaned -vLLM containers, and refreshes LiteLLM/Open WebUI in place. Cached weights stay -on disk. - -Smoke test again: - -```bash -infer-stack smoke-test -``` - -Then open . Open WebUI now has a chat-capable route. - -## 3. Switch to a workstation-sized vLLM model - -After the plumbing is proven, switch to a real workstation profile. The -`workstation-safe` profile uses first-fit placement so it avoids display GPUs -when the policy reserves them: - -```bash -infer-stack switch workstation-safe --apply -``` - -First start downloads the model and warms the vLLM cache; subsequent restarts -reuse `hf-cache/` and `vllm-cache/`. - -```bash -infer-stack smoke-test -``` - -If you want a profile that explicitly pins a runtime to a particular GPU, copy -one of the built-in profile definitions into `~/.config/infer_stack/models.yaml` -and change `providers.vllm.runtimes..placement.gpu_indices`. - -## Day-to-day workflow - -Stop without deleting state: - -```bash -infer-stack down -``` - -Start after a reboot: - -```bash -infer-stack up -d -``` - -Tail logs and inspect status: - -```bash -infer-stack logs -infer-stack ps -``` - -Run the smoke test: - -```bash -infer-stack smoke-test -``` - -Read the LiteLLM key from `.env`: - -```bash -infer-stack env --key LITELLM_MASTER_KEY -``` - -## Troubleshooting - -### Required host ports are already bound - -`infer-stack up` does a pre-flight check on enabled component ports. For this -profile family, the usual ports are 14042 for LiteLLM and 13000 for Open WebUI. -Direct Ollama profiles may also publish 11434. - -Find what is holding a port: - -```bash -ss -tlnp 'sport = :14042' -sudo lsof -nP -iTCP:14042 -sTCP:LISTEN -docker ps --filter publish=14042 -``` - -Common fixes: - -- Stop the old stack with `infer-stack down`. -- Stop a stale container, for example `docker stop litellm && docker rm litellm`. -- Change ports with setup flags such as - `infer-stack setup --litellm-port 14001 --open-webui-port 13001`, then render - and start again. - -### Smoke test errors - -- Could not connect: give the stack more time; check `infer-stack ps`. -- Connection closed before a response: the router is up but the upstream model - may still be loading; inspect `infer-stack logs vllm-*`. -- 401/403: the key in `.env` does not match the running LiteLLM container; - restart with `infer-stack down && infer-stack up -d`. -- 503: vLLM is still loading; inspect `infer-stack logs vllm-*`. - -### Wiping state - -Delete databases, Open WebUI state, and runtime configs while keeping model -caches: - -```bash -infer-stack purge --yes -``` - -Delete everything, including model caches: - -```bash -infer-stack purge --yes --delete-cache -``` diff --git a/docs/demos/qwen_3090_profiles.md b/docs/demos/qwen_3090_profiles.md deleted file mode 100644 index 6cb591ca..00000000 --- a/docs/demos/qwen_3090_profiles.md +++ /dev/null @@ -1,31 +0,0 @@ -# Qwen profiles for a 3090 workstation - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -These built-in profiles are intended for quick GPU-1 testing on a workstation-class -RTX 3090 host. - -## Qwen3.5 9B on GPU 1 - -```bash -infer-stack switch qwen3.5-9b-vllm-gpu1-3090 --apply --yes -infer-stack wait-ready --model qwen3.5-9b -infer-stack smoke-test --model qwen3.5-9b -``` - -This profile runs `Qwen/Qwen3.5-9B` through vLLM on physical GPU 1 and routes -Open WebUI through LiteLLM. It intentionally lowers context/runtime settings from -the catalog default to better fit a 24 GB 3090. - -## Qwen3.6 35B-A3B FP8 on GPU 1, experimental - -```bash -infer-stack switch qwen3.6-35b-a3b-vllm-gpu1-3090 --apply --yes -infer-stack wait-ready --model qwen3.6-35b-a3b -infer-stack smoke-test --model qwen3.6-35b-a3b -``` - -This profile uses `Qwen/Qwen3.6-35B-A3B-FP8` with a short context and low -concurrency. It is included as a best-effort experiment for the 3090 machine, but -Qwen's official vLLM recipe targets data-center GPUs for this checkpoint. Expect -OOM or unsupported-kernel failures on a 24 GB / non-FP8 GPU. diff --git a/docs/demos/smollm2_gpu1_backend_switch.md b/docs/demos/smollm2_gpu1_backend_switch.md deleted file mode 100644 index 53641c8f..00000000 --- a/docs/demos/smollm2_gpu1_backend_switch.md +++ /dev/null @@ -1,132 +0,0 @@ -# SmolLM2 backend switch test on GPU 1 - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -This is a copy/paste smoke path for a workstation where GPU 1 is available. -It uses only built-in profiles and `infer-stack` runtime commands. - -The two profiles are: - -- `smollm2-135m-vllm-gpu1`: Open WebUI -> LiteLLM -> vLLM on physical GPU 1. -- `smollm2-135m-ollama-gpu1`: Open WebUI -> Ollama on physical GPU 1. - -Switching profiles is non-destructive for state. `infer-stack switch --apply` -re-renders and runs Compose with `--remove-orphans`, but it does not delete -state directories or model caches. Containers whose definitions are unchanged -stay up. In this specific vLLM-to-Ollama switch, Open WebUI is recreated because -its provider changes from LiteLLM to Ollama; LiteLLM and vLLM are removed as -orphans. You can switch back to the vLLM profile later. - -`--apply` already runs the converge step. You only need a separate -`infer-stack up -d` after plain `switch` without `--apply`, or when you edited -rendered files/configuration by hand and want to reconcile them later. - -`infer-stack wait-ready` is the command to use after a switch when you care -about the model being fully servable, not merely the Docker container being -started or healthcheck-passing. It polls the same access surface that clients -use and requires a tiny generation/completion to succeed. - -## 1. vLLM on GPU 1 - -```bash -cd /home/joncrall/code/helm_audit/submodules/infer_stack - -infer-stack switch smollm2-135m-vllm-gpu1 --apply --yes -infer-stack ps -infer-stack wait-ready --model smollm2-135m -infer-stack smoke-test --model smollm2-135m -``` - -Open WebUI should be available at: - -```text -http://127.0.0.1:13000 -``` - -In this profile, Open WebUI talks to LiteLLM, and LiteLLM routes the -`smollm2-135m` model alias to the vLLM runtime. - -Useful checks: - -```bash -infer-stack logs --tail=120 litellm vllm-chat -infer-stack env LITELLM_MASTER_KEY -``` - -## 2. Switch to direct Ollama on GPU 1 - -```bash -infer-stack switch smollm2-135m-ollama-gpu1 --apply --yes -infer-stack ps -``` - -Expected services now: - -```text -postgres-open-webui -ollama -open-webui -``` - -LiteLLM and vLLM containers are removed as orphans, but their persistent state -and caches are not deleted. - -Pull the comparable Ollama model through the CLI wrapper: - -```bash -infer-stack ollama-pull smollm2:135m -infer-stack ollama-list -infer-stack wait-ready --model smollm2:135m -infer-stack smoke-test --model smollm2:135m -``` - -Open WebUI is still available at: - -```text -http://127.0.0.1:13000 -``` - -In this profile, Open WebUI talks directly to Ollama, so it should discover -`smollm2:135m` from Ollama after the pull. - -Useful checks: - -```bash -infer-stack ollama-ps -infer-stack logs --tail=120 ollama open-webui -``` - -## 3. Switch back to vLLM - -```bash -infer-stack switch smollm2-135m-vllm-gpu1 --apply --yes -infer-stack wait-ready --model smollm2-135m -infer-stack smoke-test --model smollm2-135m -``` - -## 4. Stop without deleting state - -```bash -infer-stack down -``` - -`down` does not remove volumes or state directories. Use `infer-stack purge` -only when you intentionally want to delete Docker-written state. - -## Diagnosing readiness during switches - -When switching vLLM profiles, the vLLM runtime process may be replaced even if -Open WebUI and LiteLLM stay up. During that window LiteLLM can list the new -model alias but return upstream connection errors until the vLLM process is -actually serving requests. - -Use: - -```bash -infer-stack diagnose --model smollm2-135m --generation -infer-stack diagnose --logs --tail 80 -``` - -The first command prints active profile, compose service state, LiteLLM route -state, and provider probes. The second adds recent logs for LiteLLM, Open -WebUI, vLLM, and Ollama services. diff --git a/docs/persistent-caches-and-warm-restarts.md b/docs/persistent-caches-and-warm-restarts.md index e5cd8f21..19336794 100644 --- a/docs/persistent-caches-and-warm-restarts.md +++ b/docs/persistent-caches-and-warm-restarts.md @@ -84,12 +84,20 @@ to come back healthy: the expensive work is not just downloading weights. ## Shared memory -vLLM uses shared memory for tensor-parallel communication and worker IPC, and -upstream vLLM's Docker guidance is `ipc: host` or a `shm_size` of a few GiB -for TP > 1. The rendered Compose currently sets neither, so vLLM containers get -Docker's default 64 MiB `/dev/shm`. If an endpoint, most likely a -tensor-parallel one, fails on shared memory, this is the first thing to -check. +vLLM's workers share memory when a model spans GPUs (tensor or pipeline +parallel), and Docker gives a container a 64 MiB `/dev/shm`. Upstream vLLM's +Docker guidance is `ipc: host` or a `shm_size` of a few GiB. On Compose, set +it per endpoint: + +```yaml +runtime: {tensor_parallel_size: 4, shm_size: 16g} +``` + +It is opt-in: an endpoint without it renders exactly as before, so the +upgrade recreates no running engine. A parallel engine without it logs one +warning naming the key, and `catalog suggest` sets it on the multi-GPU +entries it adds. KubeAI needs nothing: its vLLM pods mount a memory-backed +`/dev/shm` (checked on KubeAI v0.23.4). ## Minimal-restart workflow diff --git a/docs/queue.md b/docs/queue.md index d4ec7ecf..5aaa25e4 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -129,23 +129,37 @@ top-level `restart` / `stop` / `start` / `pull` (they live under `stack`). **Done when:** every command in the README and `docs/` runs as written against the current CLI, and `grep` finds none of those verbs. -### 8a. [ ] Decide: rewrite or delete the pre-leasing recipes +### 8a. [x] Decide: rewrite or delete the pre-leasing recipes *Why added (2026-09-26):* `docs/demos/`, `recipies/` and `examples/` (about -2,000 lines) are hardware recipes for the removed profile CLI; the Makefile -targets that used them are gone. Each now opens with a "Pre-leasing" banner -saying its commands no longer run. Rewriting them as catalog entries needs -the hardware they describe; deleting them loses tuning notes. **The -operator's call.** - -### 8b. [ ] Decide: `/dev/shm` for vLLM containers - -*Why added (2026-09-26):* the leasing renderer sets neither `ipc: host` nor -`shm_size`, so a vLLM container gets Docker's 64 MiB `/dev/shm`; the -pre-leasing template had `ipc: host`, and vLLM's own Docker instructions use -it for tensor parallelism. Adding it changes every vLLM service's -fingerprint, so the next apply recreates running engines. Needs a TP=2 run -on a GPU host to confirm the symptom first. **The operator's call.** +2,000 lines) were hardware recipes for the removed profile CLI. + +*Decided (2026-09-26): keep the tuning, delete the pages.* The only content +the current docs lacked was the 4 x 96 GB vLLM tuning. It is now three gated +variants in the suggestion pool, which `catalog suggest` offers on a host +with four GPUs of 80 GiB or more: `qwen3.5-122b-a10b-tp4-128k`, +`qwen3.5-122b-a10b-fp8-tp4-262k`, `qwen3.6-35b-a3b-tp2-262k` (TP, context, +batch limits, `--language-model-only`, the qwen3 reasoning parser). They are +marked as not re-run since leasing. The rest was covered already: the +completions-only Pythia recipes by the README's protocol modes, the TLS/LDAP +example by the README's statement that those settings are gone, and the +tutorials by the manual. The pages are deleted; git history keeps them. +Test: `test_a_4x96_host_gets_the_former_recipes_as_variants`. + +### 8b. [x] Decide: `/dev/shm` for vLLM containers + +*Why added (2026-09-26):* the renderer set neither `ipc: host` nor +`shm_size`, so a vLLM container got Docker's 64 MiB `/dev/shm`. + +*Decided (2026-09-26): opt-in, with a nudge.* `runtime.shm_size` (e.g. +`16g`) renders `shm_size` on the Compose service; unset, the service is +byte-identical to before, so no running engine is recreated by the upgrade. +A tensor- or pipeline-parallel engine without it logs one warning naming the +key, and `catalog suggest` sets `16g` on the multi-GPU entries it adds. +KubeAI needs nothing: its vLLM pods mount a memory-backed `/dev/shm` +(checked on the dev cluster, KubeAI v0.23.4). The TP=2 failure itself is +still unobserved here (no GPUs); the key is how an operator fixes it when +seen. Tests: `tests/test_shm_size.py`. ### 9. [x] UX audit loop: do not stop without a passing audit @@ -412,5 +426,6 @@ scripts (items 6 and 7) with what each run proves. Both keep their own config and data roots and print PASS/FAIL per step. -**Open decisions:** 8a (rewrite or delete the pre-leasing recipes) and 8b -(`/dev/shm` for vLLM, which recreates running engines). +**Decided without GPUs, worth confirming on them:** 8a's recipe variants +were not re-run since leasing, and 8b's `shm_size` fixes a TP failure not yet +observed here. diff --git a/examples/config.multi-model.yaml b/examples/config.multi-model.yaml deleted file mode 100644 index da0c81e3..00000000 --- a/examples/config.multi-model.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# Pre-leasing: a stack-profile config for the removed profile CLI; it is -# not read by infer-stack any more. See the README for the catalog. -name: aiq-multi-model -backend: compose -active_profile: qwen-mixed -catalog: - builtin_models: true - builtin_profiles: true - user_models_file: models.yaml -policy: - require_fit_validation: true - reserve_display_gpu: auto - forbid_reserved_gpu_use: false - require_homogeneous_multi_gpu_groups: true - minimum_vram_headroom_gib: 2 - allow_unsupported_render: false -runtime: - compose_cmd: docker compose - target_inventory: auto -ports: - litellm: 14042 - open_webui: 13000 - postgres: 15432 - ollama: 11434 -state: - hf_cache: ./state/hf-cache - vllm_cache: ./state/vllm-cache - ollama: ./state/ollama - open_webui: ./state/open-webui - postgres_open_webui: ./state/postgres-open-webui - postgres_litellm: ./state/postgres-litellm - runtime: ./state/runtime -profiles: - qwen-mixed: - description: Three vLLM runtimes behind one LiteLLM/Open WebUI route surface. - providers: - vllm: - runtimes: - chat: - model: qwen3.5-27b - placement: - strategy: exact - gpu_indices: [0] - fast: - model: qwen3.5-2b - placement: - strategy: exact - gpu_indices: [1] - big: - model: qwen3.5-122b - placement: - strategy: exact - gpu_indices: [2, 3] - topology: - tensor_parallel_size: 2 - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - qwen3.5-27b: - provider: vllm - runtime: chat - qwen3.5-2b: - provider: vllm - runtime: fast - qwen3.5-122b: - provider: vllm - runtime: big - - local-ollama-direct: - description: Single Ollama daemon with Open WebUI connected directly. - providers: - ollama: - enabled: true - gpu_indices: [0, 1] - publish_port: true - keep_alive: 2m - context_length: 4096 - gateways: - litellm: - enabled: false - frontends: - open_webui: - enabled: true - provider: ollama - routes: {} diff --git a/examples/openwebui-tls-ldap/README.md b/examples/openwebui-tls-ldap/README.md deleted file mode 100644 index 753f329c..00000000 --- a/examples/openwebui-tls-ldap/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Open WebUI with TLS reverse proxy and LDAP - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -This example runs the built-in `openwebui-tls-ldap` profile: an opt-in -Compose stack with - -- Ollama as the provider, -- Open WebUI connected directly to Ollama, -- Open WebUI LDAP login wiring, -- nginx as a TLS reverse proxy, and -- no public Ollama or Open WebUI host ports (only the proxy is exposed). - -The reverse proxy and LDAP settings are ordinary config fields. If the -typed nginx renderer is not enough, set -`frontends.reverse_proxy.config_path` to an existing nginx config and -infer-stack mounts that file instead of rendering `state.runtime/nginx.conf`. - -## Setup - -```bash -infer-stack setup --backend compose --profile openwebui-tls-ldap -``` - -Then edit `config.yaml` and/or the generated `.env` after the first render: - -```bash -infer-stack render --yes --simulate-hardware 2x24 -``` - -The generated `.env` contains LDAP placeholders such as `LDAP_HOST`, -`LDAP_PASSWD`, and `LDAP_SEARCH_BASE`. Fill those in before starting the -stack. - -## TLS certificates - -Cert/key/dhparam host paths live under `frontends.reverse_proxy.ssl`: - -```yaml -frontends: - reverse_proxy: - ssl: - certificate: ./certs/site.crt - certificate_key: ./certs/site.key - dhparam: ./dhparam.pem -``` - -> **Relative paths resolve against the generated directory.** These -> values are written verbatim into the generated `docker-compose.yml`, so -> Docker resolves them relative to that file's location — not your -> current working directory. Either place `certs/` and `dhparam.pem` -> alongside the generated compose file, or use absolute paths. -> `infer-stack render` prints a warning when a referenced cert or config -> file cannot be found. - -## Fully manual nginx config - -```yaml -frontends: - reverse_proxy: - enabled: true - config_path: /absolute/path/to/nginx.conf -``` diff --git a/examples/single-node/README.md b/examples/single-node/README.md deleted file mode 100644 index feb685ef..00000000 --- a/examples/single-node/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Single-node KubeAI example - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../../README.md). - -This example uses the stack-graph schema, but KubeAI currently renders only the -`providers.vllm.runtimes` section. Ollama, LiteLLM, and Open WebUI are disabled -in this example because they are Compose-only for now. - -Copy these files into the repo root: - -```bash -cp examples/single-node/config.yaml ./config.yaml -cp examples/single-node/models.yaml ./models.yaml -``` - -Then edit: - -- `resource_profiles.blackwell-96gb.node_selector.kubernetes.io/hostname` -- any Hugging Face model ids you want to use -- `cluster.ingress.host` if you want an Ingress on your LAN - -Typical flow: - -```bash -python manage.py render -bash scripts/bootstrap_k3s.sh -python manage.py deploy -python manage.py status -python manage.py smoke-test --base-url http://127.0.0.1:8000/openai/v1 -``` - -If you are not using ingress yet, port-forward first: - -```bash -kubectl -n kubeai port-forward svc/kubeai 8000:80 -``` - -Or, from repo root: - -```bash -make example-single-node -make bootstrap-k3s -python manage.py render -python manage.py deploy -``` diff --git a/examples/single-node/config.yaml b/examples/single-node/config.yaml deleted file mode 100644 index 93bac038..00000000 --- a/examples/single-node/config.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Pre-leasing: a stack-profile config for the removed profile CLI; it is -# not read by infer-stack any more. See the README for the catalog. -name: single-node-kubeai -backend: kubeai -active_profile: qwen-single-node -catalog: - builtin_models: false - builtin_profiles: false - user_models_file: models.yaml -policy: - require_fit_validation: true - reserve_display_gpu: auto - forbid_reserved_gpu_use: false - require_homogeneous_multi_gpu_groups: true - minimum_vram_headroom_gib: 4 - allow_unsupported_render: false -runtime: - compose_cmd: docker compose - target_inventory: auto -ports: - litellm: 14042 - open_webui: 13000 - postgres: 15432 - ollama: 11434 -state: - hf_cache: ./state/hf-cache - vllm_cache: ./state/vllm-cache - runtime: ./state/runtime -cluster: - namespace: kubeai - kubeai_release_name: kubeai - kubeai_chart: kubeai/kubeai - service_name: kubeai - ingress: - enabled: true - class_name: traefik - host: llm.local - path_prefix: / - tls_secret_name: "" -resource_profiles: - blackwell-96gb: - node_selector: - kubernetes.io/hostname: CHANGE-ME - requests: - nvidia.com/gpu: "1" - cpu: "6" - memory: 24Gi - limits: - nvidia.com/gpu: "1" - cpu: "6" - memory: 24Gi -profiles: - qwen-single-node: - description: Run one small Qwen model and one larger Qwen model on a single KubeAI node. - providers: - vllm: - runtimes: - qwen-small: - model: qwen-small - resource_profile: blackwell-96gb:1 - placement: - strategy: first_fit - gpu_count: 1 - topology: - tensor_parallel_size: 1 - min_replicas: 0 - max_replicas: 1 - qwen-large: - model: qwen-large - resource_profile: blackwell-96gb:4 - placement: - strategy: first_fit - gpu_count: 4 - topology: - tensor_parallel_size: 4 - min_replicas: 0 - max_replicas: 1 - gateways: - litellm: - enabled: false - frontends: - open_webui: - enabled: false - routes: {} diff --git a/examples/single-node/models.yaml b/examples/single-node/models.yaml deleted file mode 100644 index 7ec5f4b9..00000000 --- a/examples/single-node/models.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Pre-leasing: a stack-profile config for the removed profile CLI; it is -# not read by infer-stack any more. See the README for the catalog. -vllm_models: - qwen-small: - hf_model_id: Qwen/Qwen2.5-7B-Instruct - features: [TextGeneration] - defaults: - max_model_len: 32768 - gpu_memory_utilization: 0.9 - max_num_batched_tokens: 8192 - max_num_seqs: 16 - enable_prefix_caching: true - qwen-large: - hf_model_id: Qwen/Qwen2.5-32B-Instruct - features: [TextGeneration] - defaults: - max_model_len: 32768 - gpu_memory_utilization: 0.92 - max_num_batched_tokens: 4096 - max_num_seqs: 8 - enable_prefix_caching: true - -ollama_models: {} -profiles: {} diff --git a/infer_stack/leasing/catalog.py b/infer_stack/leasing/catalog.py index 912468d3..8b98d978 100644 --- a/infer_stack/leasing/catalog.py +++ b/infer_stack/leasing/catalog.py @@ -29,6 +29,10 @@ # endpoint (GPU count is appended automatically from tp*pp*dp); falls # back to `config set kubeai_resource_profile `. # runtime: {resource_profile: nvidia-gpu-rtx-4090, ...} + # compose backend only: the container's /dev/shm (Docker's default is + # 64 MiB; vLLM's workers share memory when a model spans GPUs). Not + # set, nothing is rendered; KubeAI mounts a memory-backed /dev/shm. + # runtime: {tensor_parallel_size: 4, shm_size: 16g} # An image with its own launcher (Compose only), described in data; # see infer_stack/leasing/launch.py for the fields and templates. # runtime: {image: ..., command: [single], @@ -284,10 +288,29 @@ def _placement_errors(ep: EndpointSpec) -> list[str]: def _runtime_errors(ep: EndpointSpec) -> list[str]: - """Validate the generic launch fields (see :mod:`.launch`).""" + """Validate the generic launch fields (see :mod:`.launch`) and ``shm_size``. + + >>> ep = EndpointSpec(name='e', engine='vllm', model='m', runtime={'shm_size': '16g'}) + >>> _runtime_errors(ep) + [] + >>> ep.runtime['shm_size'] = 'lots' + >>> _runtime_errors(ep)[0].split(':')[0] + "endpoint 'e'" + """ + import re + from .launch import launch_errors - return launch_errors(ep.name, ep.engine, ep.runtime) + errors = launch_errors(ep.name, ep.engine, ep.runtime) + shm = (ep.runtime or {}).get('shm_size') + if shm is not None: + if ep.engine != VLLM: + errors.append(f"endpoint '{ep.name}': runtime.shm_size is for vllm " + 'endpoints') + elif not re.fullmatch(r'\d+(\.\d+)?\s*[kmg]?b?', str(shm), re.IGNORECASE): + errors.append(f"endpoint '{ep.name}': runtime.shm_size must be a size " + f"such as 16g or 8gb, not {shm!r}") + return errors @dataclass diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 0d2cb526..7b4e805b 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -367,6 +367,8 @@ def vllm_service_dict(deployment: Deployment) -> dict[str, Any]: # vLLM's (see profile_runtime.simulator_args). Absent => a real engine. 'simulator': runtime.get('simulator') or None, 'extra_args': list(runtime.get('extra_args', []) or []), + # Compose only (a KubeAI pod mounts a memory-backed /dev/shm already). + 'shm_size': runtime.get('shm_size'), } @@ -397,6 +399,28 @@ def _serve_config_hash( return hashlib.sha256(material.encode("utf-8")).hexdigest()[:12] +_SHM_WARNED: set[str] = set() + + +def _warn_small_shm(deployment: Deployment, svc: dict[str, Any]) -> None: + """Once per deployment: a parallel vLLM engine on Docker's 64 MiB /dev/shm. + + vLLM's workers talk through shared memory when a model spans GPUs, and + Docker gives a container 64 MiB of it. Opt-in, not a default: setting it + on existing engines would change their rendered service, and recreate + them, for every catalog that works today. + """ + parallel = svc['tensor_parallel_size'] * svc['pipeline_parallel_size'] + if parallel <= 1 or deployment.id in _SHM_WARNED: + return + _SHM_WARNED.add(deployment.id) + from .._log import logger + logger.warning( + '{} spans {} GPUs on Docker\'s 64 MiB /dev/shm; if its workers fail on ' + 'shared memory, set runtime.shm_size (e.g. 16g) on the endpoint', + served_name(deployment), parallel) + + def _vllm_service( deployment: Deployment, gpus: list[int], @@ -497,6 +521,10 @@ def _vllm_service( service['ports'] = [f'{host_port}:8000'] if gpus: service['deploy'] = _gpu_reservation(gpus) + if svc.get('shm_size') and not simulated: + service['shm_size'] = str(svc['shm_size']) + elif not simulated: + _warn_small_shm(deployment, svc) if simulated: # A simulator downloads no weights and compiles no graphs, so the # caches above are dead weight -- and worse, the images ship diff --git a/infer_stack/leasing/suggest.py b/infer_stack/leasing/suggest.py index 428cd978..259cab0d 100644 --- a/infer_stack/leasing/suggest.py +++ b/infer_stack/leasing/suggest.py @@ -359,6 +359,9 @@ def derive_runtime( if model.preferred_gpu_count > 1: runtime['tensor_parallel_size'] = model.preferred_gpu_count + # Workers across GPUs talk through /dev/shm; Docker's default is + # 64 MiB (vLLM's Docker guidance: a few GiB, or ipc: host). + runtime['shm_size'] = '16g' if model.defaults.get('enable_prefix_caching'): runtime['enable_prefix_caching'] = True diff --git a/infer_stack/templates/suggestion-pool.yaml b/infer_stack/templates/suggestion-pool.yaml index 04fe905b..847d638b 100644 --- a/infer_stack/templates/suggestion-pool.yaml +++ b/infer_stack/templates/suggestion-pool.yaml @@ -350,6 +350,21 @@ vllm_models: max_model_len: 32768 gpu_memory_utilization: 0.9 enable_prefix_caching: true + endpoint_variants: + tp4-128k: + # From the pre-leasing recipe for a 4 x 96 GB host (removed 2026-09-26; + # git history has it): not re-run since the leasing rewrite. + preferred_gpu_count: 4 + min_vram_gib_per_replica: 80 + runtime: + tensor_parallel_size: 4 + shm_size: 16g + max_model_len: 131072 + gpu_memory_utilization: 0.95 + max_num_batched_tokens: 1024 + max_num_seqs: 1 + enable_prefix_caching: false + extra_args: [--language-model-only, --reasoning-parser, qwen3] qwen3.5-122b-a10b-fp8: hf_model_id: Qwen/Qwen3.5-122B-A10B-FP8 served_model_name: qwen3.5-122b-a10b-fp8 @@ -363,6 +378,21 @@ vllm_models: max_model_len: 32768 gpu_memory_utilization: 0.9 enable_prefix_caching: true + endpoint_variants: + tp4-262k: + # From the pre-leasing recipe for a 4 x 96 GB host (removed 2026-09-26; + # git history has it): not re-run since the leasing rewrite. + preferred_gpu_count: 4 + min_vram_gib_per_replica: 80 + runtime: + tensor_parallel_size: 4 + shm_size: 16g + max_model_len: 262144 + gpu_memory_utilization: 0.95 + max_num_batched_tokens: 1024 + max_num_seqs: 1 + enable_prefix_caching: false + extra_args: [--language-model-only, --reasoning-parser, qwen3] # -- Qwen3.6 (current gen, reasoning, MoE) ----------------------------- # The FP8 build squeezes onto one 24 GiB GPU; the bf16 build wants two. @@ -379,6 +409,21 @@ vllm_models: max_model_len: 32768 gpu_memory_utilization: 0.9 enable_prefix_caching: false + endpoint_variants: + tp2-262k: + # From the pre-leasing recipe for a 4 x 96 GB host (removed 2026-09-26; + # git history has it): not re-run since the leasing rewrite. + preferred_gpu_count: 2 + min_vram_gib_per_replica: 80 + runtime: + tensor_parallel_size: 2 + shm_size: 16g + max_model_len: 262144 + gpu_memory_utilization: 0.95 + max_num_batched_tokens: 8192 + max_num_seqs: 8 + enable_prefix_caching: false + extra_args: [--language-model-only, --reasoning-parser, qwen3] qwen3.6-35b-a3b-fp8: hf_model_id: Qwen/Qwen3.6-35B-A3B-FP8 served_model_name: qwen3.6-35b-a3b-fp8 diff --git a/recipies/compose_pythia_inspect_mmlu_compat.md b/recipies/compose_pythia_inspect_mmlu_compat.md deleted file mode 100644 index 8687a7bc..00000000 --- a/recipies/compose_pythia_inspect_mmlu_compat.md +++ /dev/null @@ -1,139 +0,0 @@ -# Compose recipe: Pythia with LiteLLM-only chat compatibility (Inspect / MMLU) - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). - -> Schema note: built-in profiles now resolve to stack-graph profiles. vLLM runtimes live under `providers.vllm.runtimes`, LiteLLM routes live under `routes`, and direct Ollama profiles can run without LiteLLM. - -This recipe uses the built-in `pythia-inspect-mmlu-compat` profile to -serve two Pythia base models as completions models, with a LiteLLM-only -adapter that flattens chat-shaped requests into a plain prompt before -forwarding to vLLM `/v1/completions`. - -| Service | Model | Protocol | LiteLLM upstream | -|---------------|-----------------------------|--------------|-------------------------------------------| -| `pythia-69b` | `EleutherAI/pythia-6.9b` | completions | `text-completion-openai/eleutherai/pythia-6.9b` | -| `pythia-28b` | `EleutherAI/pythia-2.8b-v0` | completions | `text-completion-openai/eleutherai/pythia-2.8b-v0` | - -**This is an adapter, not a chat tune.** Pythia is still a base model; -prompt formatting still matters scientifically. `/v1/completions` -remains the canonical benchmark path for HELM-style evaluations. The -`chat_compat` adapter exists only so chat-shaped clients (notably -InspectAI / Inspect Evals stock MMLU tasks) can talk to Pythia -through LiteLLM without errors. - -## What the adapter does - -For each Pythia model the rendered LiteLLM config sets: - -```yaml -litellm_params: - model: text-completion-openai/eleutherai/pythia-6.9b - api_base: http://vllm-pythia-69b:8000/v1 - api_key: os.environ/VLLM_BACKEND_API_KEY - initial_prompt_value: "" - roles: - system: { pre_message: "", post_message: "\n" } - user: { pre_message: "", post_message: "\n" } - assistant: { pre_message: "", post_message: "\n" } - final_prompt_value: "" -``` - -Result: when LiteLLM receives a `/v1/chat/completions` request with -multiple messages, it concatenates the `content` of each message -followed by `\n` — no role labels — and forwards a plain `/v1/completions` -call to vLLM. This matches: - -```jinja2 -{% for message in messages %}{{ message['content'] }} -{% endfor %} -``` - -The vLLM container is **not** restarted, **not** reconfigured, and -**no** `--chat-template` flag is rendered in Compose. The adapter lives -entirely at the LiteLLM layer. - -## 1. Setup, render - -```bash -cd /path/to/infer_stack -python manage.py setup --backend compose --profile pythia-inspect-mmlu-compat -python manage.py render -python manage.py up -d -``` - -## 2. Restart only LiteLLM after a re-render - -If you change the chat-compat metadata (or any other LiteLLM-only -config) and your Pythia containers are already running serving traffic, -restart only LiteLLM — leave the vLLM containers untouched: - -```bash -python manage.py render - -infer-stack restart litellm -``` - -`--no-deps` prevents Compose from touching the vLLM Pythia containers -or the Postgres containers. To confirm what's running first: - -```bash -infer-stack ps -``` - -Avoid `docker compose down`, `down -v`, or a profile-wide -`up --force-recreate` — those will disrupt the Pythia containers you -are trying to keep running. - -## 3. Where the rendered LiteLLM config lives - -On the host (writable, regenerated by `render`): - -```text -/data/service/docker/infer-stack/runtime/litellm_config.yaml -``` - -Inside the `litellm` container (read-only mount): - -```text -/app/config.yaml -``` - -If your `state.runtime` path is set elsewhere in `config.yaml`, the -host path follows that — see `python manage.py describe-profile -pythia-inspect-mmlu-compat --format yaml` and look at `state.runtime`. - -## 4. Smoke-test - -A chat-shaped call should now work end-to-end: - -```bash -eval "$(infer-stack env --export)" -curl -s http://127.0.0.1:14042/v1/chat/completions \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d '{"model":"eleutherai/pythia-6.9b", - "messages":[ - {"role":"system","content":"You are a helpful assistant."}, - {"role":"user","content":"The capital of France is"} - ], - "max_tokens":16}' -``` - -For HELM/Inspect runs that need exact prompt control, prefer -`/v1/completions` directly: - -```bash -curl -s http://127.0.0.1:14042/v1/completions \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d '{"model":"eleutherai/pythia-6.9b","prompt":"The capital of France is","max_tokens":16}' -``` - -## Summary - -- Two Pythia services, completions-only on the vLLM side. -- LiteLLM `flat_messages` adapter translates chat-shaped requests into - plain prompts (no role labels, newline-joined `content`). -- No `--chat-template` rendered in Compose. -- Re-rendering and restarting `litellm` alone is sufficient to roll out - adapter changes. diff --git a/recipies/compose_pythia_qwen36_mixed_4x96GB.md b/recipies/compose_pythia_qwen36_mixed_4x96GB.md deleted file mode 100644 index 464f67d0..00000000 --- a/recipies/compose_pythia_qwen36_mixed_4x96GB.md +++ /dev/null @@ -1,193 +0,0 @@ -# Compose recipe: Pythia 6.9B + Pythia 2.8B + Qwen3.6-35B-A3B on a 4x96GB host - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). - -> Schema note: built-in profiles now resolve to stack-graph profiles. vLLM runtimes live under `providers.vllm.runtimes`, LiteLLM routes live under `routes`, and direct Ollama profiles can run without LiteLLM. - -This recipe uses the built-in `pythia-qwen3.6-mixed-4x96` profile to serve -three models behind a single LiteLLM router on a machine with **4 x 96GB -GPUs**: - -| Service | Model | GPUs | Protocol | -|---------------|-----------------------------|--------|--------------| -| `qwen36-35b` | `Qwen/Qwen3.6-35B-A3B` | 0, 1 | chat (TP2, reasoning) | -| `pythia-69b` | `EleutherAI/pythia-6.9b` | 2 | completions | -| `pythia-28b` | `EleutherAI/pythia-2.8b-v0` | 3 | completions | - -The Qwen service has `--reasoning-parser qwen3` auto-rendered by the -profile (vLLM enables reasoning whenever a parser is set). The two Pythia services are routed -through LiteLLM as `text-completion-openai/...` so chat-shaped requests -from Open WebUI are translated into upstream `/v1/completions` calls. - ---- - -## 1. Setup, render, run - -```bash -cd /path/to/infer_stack -python manage.py setup --backend compose --profile pythia-qwen3.6-mixed-4x96 -python manage.py render -python manage.py up -d -``` - -If you already have the stack running on a different profile: - -```bash -python manage.py switch pythia-qwen3.6-mixed-4x96 --apply -``` - -`switch --apply` re-renders, brings the stack up convergently, and -recreates `litellm` + `open-webui` in place so the new alias list takes -effect. Postgres volumes are not touched. - -### LiteLLM-only chat compatibility for the Pythia services - -The two Pythia services in this profile carry -`chat_compat: { enabled: true, strategy: flat_messages }`. That makes -LiteLLM advertise `eleutherai/pythia-6.9b` and `eleutherai/pythia-2.8b-v0` -as chat-capable to clients that only speak `/v1/chat/completions` -(notably InspectAI / Inspect Evals stock MMLU tasks). LiteLLM flattens -the chat messages into a plain prompt — concatenated `content` joined -by `\n`, no role labels — and forwards a `/v1/completions` call to vLLM. - -This is **LiteLLM-only**: - -- vLLM is not restarted, no `--chat-template` is rendered. -- No second/duplicate Pythia container. -- Pythia stays `protocol_mode: completions`; HELM-style direct - `/v1/completions` clients still get exact-prompt control. -- The Qwen3.6 entry is unchanged — it remains a normal chat route. - -To roll out a LiteLLM config change (e.g. after editing chat-compat -fields) without disturbing any of the running vLLM containers: - -```bash -python manage.py render - -infer-stack restart litellm -``` - -`--no-deps` keeps `vllm-qwen36-35b`, `vllm-pythia-69b`, and -`vllm-pythia-28b` running. Confirm with: - -```bash -infer-stack ps -``` - -Avoid `docker compose down`, `down -v`, or a profile-wide -`up --force-recreate`. - -### Restarting only the Qwen container - -If you change Qwen settings (for example, the tool-call parser) and -you do not want to disturb the Pythia containers — they may be in -active use — re-render and recreate only the Qwen vLLM service: - -```bash -python manage.py render -infer-stack restart vllm-qwen36-35b -``` - -`--no-deps` prevents Compose from touching `postgres-litellm`, -`postgres-open-webui`, `litellm`, `open-webui`, or the Pythia vLLM -services. To confirm the rendered service name first: - -```bash -infer-stack ps -``` - -Avoid `docker compose down`, `down -v`, or a profile-wide -`up --force-recreate` for this — they will disrupt the Pythia -containers you are trying to keep running. - -### Tool calling - -Per the `Qwen/Qwen3.6-35B-A3B` model card, the recommended vLLM flags -for this model are: - -```text ---reasoning-parser qwen3 ---enable-auto-tool-choice ---tool-call-parser qwen3_coder ---language-model-only -``` - -(also: `vllm >= 0.19.0`, which the pinned image satisfies). - -The mixed profile renders all four automatically from the Qwen -service's `reasoning`, `tool_calling`, and `extra_args` metadata, so -clients like Kilo Code that send `tool_choice: "auto"` work out of the -box. - -If `qwen3_coder` parses tool calls poorly against your specific prompts, -`qwen3_xml` is a manual fallback worth testing — but it is not what the -model card recommends. To try it, change the `parser:` value in -`default-profiles.yaml` (or in a local stack profile under `models.yaml`) and rerun -the targeted Qwen restart above. - ---- - -## 2. Set your Hugging Face token (Qwen3.6 is gated) - -```bash -grep '^HF_TOKEN=' generated/.env -``` - -Edit `generated/.env` and set `HF_TOKEN=...`. Unknown `.env` keys -(yours and ours) are preserved across re-renders. - ---- - -## 3. Verify the routes - -```bash -eval "$(infer-stack env --export)" -curl -s http://127.0.0.1:14042/v1/models \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" | jq '.data[].id' -``` - -Expected aliases: - -- `qwen3.6-35b-a3b` -- `eleutherai/pythia-6.9b` -- `eleutherai/pythia-2.8b-v0` - -Smoke-test each (the smoke-test picks `/v1/chat/completions` or -`/v1/completions` based on the profile's protocol): - -```bash -python manage.py smoke-test --model qwen3.6-35b-a3b -python manage.py smoke-test --model eleutherai/pythia-6.9b -python manage.py smoke-test --model eleutherai/pythia-2.8b-v0 -``` - ---- - -## 4. Open WebUI - -Open `http://127.0.0.1:13000`. All three model aliases will appear. -Reasoning streams from the Qwen3.6 service and is displayed inline -when chat streaming is enabled. The two Pythia models will respond, -but remember they are base/completions models — prompt formatting -matters and chat-style multi-turn use will not produce great results -without explicit prompt strategies. - -For exact prompt control on the Pythia models, prefer direct -`/v1/completions` against LiteLLM: - -```bash -curl -s http://127.0.0.1:14042/v1/completions \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d '{"model":"eleutherai/pythia-6.9b","prompt":"The capital of France is","max_tokens":16}' -``` - ---- - -## Summary - -- Three vLLM containers, one shared LiteLLM router, separate Postgres - containers for Open WebUI and LiteLLM. -- Reasoning flags are emitted from profile metadata, not hand-written. -- Completions-only models are usable from chat clients via LiteLLM's - `text-completion-openai` provider, with no extra vLLM containers. diff --git a/recipies/compose_qwen35_122b_4x96GB.md b/recipies/compose_qwen35_122b_4x96GB.md deleted file mode 100644 index 9a0fcc6e..00000000 --- a/recipies/compose_qwen35_122b_4x96GB.md +++ /dev/null @@ -1,222 +0,0 @@ -# Compose recipe: Qwen3.5-122B-A10B on a 4x96GB host - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). - -This is the shortest working end-to-end example for serving **Qwen/Qwen3.5-122B-A10B** on a machine with **4 x 96GB GPUs** using the **Compose** backend. - -This profile uses: - -- one model instance -- all 4 GPUs -- tensor parallel size 4 -- text-only mode -- **131,072** token context -- Open WebUI on top of LiteLLM - ---- - -## Quick path: use the built-in profile - -Available out of the box as the built-in profile -`qwen3.5-122b-a10b-tp4-4x96`. To skip writing local YAML: - -```bash -infer-stack setup --backend compose --profile qwen3.5-122b-a10b-tp4-4x96 -infer-stack render --yes -infer-stack up -d -``` - -The rest of this recipe walks through the same deployment manually. - ---- - -## 1. Start from the repo root - -```bash -cd /path/to/infer_stack -``` - -Optional sanity check: - -```bash -nvidia-smi --query-gpu=index,name,memory.total --format=csv -``` - ---- - -## 2. Initialize Compose config - -```bash -python manage.py setup --backend compose -``` - ---- - -## 3. Write the local model and profile definitions - -Write provider-specific model definitions and a stack profile in `models.yaml`: - -```bash -cat > models.yaml <<'EOF' -vllm_models: - qwen3.5-122b-a10b-local: - hf_model_id: Qwen/Qwen3.5-122B-A10B - tokenizer_name: Qwen/Qwen3.5-122B-A10B - served_model_name: qwen3.5-122b-a10b-128k - family: qwen3.5 - modalities: [text] - memory_class_gib: 80 - min_vram_gib_per_replica: 80 - preferred_gpu_count: 4 - context_window: 131072 - defaults: - max_model_len: 131072 - gpu_memory_utilization: 0.95 - enable_prefix_caching: false - max_num_batched_tokens: 1024 - max_num_seqs: 1 - -profiles: - qwen3.5-122b-a10b-tp4-128k-local: - description: "Single Qwen3.5-122B-A10B vLLM runtime across all 4 GPUs at 128k context." - vllm: - enable_responses_api_store: false - logging_level: INFO - providers: - vllm: - runtimes: - qwen-122b: - model: qwen3.5-122b-a10b-local - served_model_name: qwen3.5-122b-a10b-128k - placement: - strategy: exact - gpu_indices: [0, 1, 2, 3] - topology: - tensor_parallel_size: 4 - runtime: - max_model_len: 131072 - gpu_memory_utilization: 0.95 - max_num_batched_tokens: 1024 - max_num_seqs: 1 - enable_prefix_caching: false - extra_args: - - --language-model-only - - --reasoning-parser - - qwen3 - - --enable-log-requests - - --max-log-len - - "4000" - - --enable-prompt-tokens-details - - --enable-force-include-usage - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - qwen3.5-122b-a10b-128k: - provider: vllm - runtime: qwen-122b -EOF -``` - ---- - -## 4. Switch to the local profile - -```bash -python manage.py switch qwen3.5-122b-a10b-tp4-128k-local -``` - ---- - -## 5. Render the stack - -```bash -python manage.py render -``` - -This creates: - -- `generated/docker-compose.yml` -- `generated/.env` - ---- - -## 6. Set your Hugging Face token - -Add your token to `generated/.env`: - -```bash -grep '^HF_TOKEN=' generated/.env -``` - -Then edit the file and set: - -```text -HF_TOKEN=your_token_here -``` - ---- - -## 7. Start the stack - -From the repo root: - -```bash -infer-stack up -d -``` - ---- - -## 8. Verify the backend - -Check that the model is exposed: - -```bash -infer-stack smoke-test -``` - -You should see: - -- `qwen3.5-122b-a10b-128k` - ---- - -## 9. Open the UI - -Open: - -```text -http://127.0.0.1:13000 -``` - -On first run, create the Open WebUI admin account. - -Then select the model: - -- `qwen3.5-122b-a10b-128k` - ---- - -## 10. Optional repo smoke test - -From the repo root: - -```bash -python manage.py smoke-test --model qwen3.5-122b-a10b-128k --prompt "Say hello in one sentence." -``` - ---- - -## Summary - -This profile serves: - -- `Qwen/Qwen3.5-122B-A10B` -- on 4 x 96GB GPUs -- with tensor parallel size 4 -- at 131,072 token context -- through both the direct backend and Open WebUI diff --git a/recipies/compose_qwen35_122b_fp8_4x96GB.md b/recipies/compose_qwen35_122b_fp8_4x96GB.md deleted file mode 100644 index b01417d5..00000000 --- a/recipies/compose_qwen35_122b_fp8_4x96GB.md +++ /dev/null @@ -1,217 +0,0 @@ -# Compose recipe: Qwen3.5-122B-A10B-FP8 on a 4x96GB host - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). - -This is the shortest working end-to-end example for serving **Qwen/Qwen3.5-122B-A10B-FP8** on a machine with **4 x 96GB GPUs** using the **Compose** backend. - -This profile uses: - -- one model instance -- all 4 GPUs -- tensor parallel size 4 -- text-only mode -- native **262,144** token context -- Open WebUI on top of LiteLLM - ---- - -## Quick path: use the built-in profile - -Available out of the box as the built-in profile -`qwen3.5-122b-a10b-fp8-tp4-4x96`. To skip writing local YAML: - -```bash -infer-stack setup --backend compose --profile qwen3.5-122b-a10b-fp8-tp4-4x96 -infer-stack render --yes -infer-stack up -d -``` - -The rest of this recipe walks through the same deployment manually. - ---- - -## 1. Start from the repo root - -```bash -cd /path/to/infer_stack -``` - -Optional sanity check: - -```bash -nvidia-smi --query-gpu=index,name,memory.total --format=csv -``` - ---- - -## 2. Initialize Compose config - -```bash -python manage.py setup --backend compose -``` - ---- - -## 3. Write the local model and profile definitions - -Write provider-specific model definitions and a stack profile in `models.yaml`: - -```bash -cat > models.yaml <<'EOF' -vllm_models: - qwen3.5-122b-a10b-fp8-local: - hf_model_id: Qwen/Qwen3.5-122B-A10B-FP8 - tokenizer_name: Qwen/Qwen3.5-122B-A10B-FP8 - served_model_name: qwen3.5-122b-a10b-fp8-262k - family: qwen3.5 - modalities: [text] - memory_class_gib: 80 - min_vram_gib_per_replica: 80 - preferred_gpu_count: 4 - context_window: 262144 - defaults: - max_model_len: 262144 - gpu_memory_utilization: 0.95 - enable_prefix_caching: false - max_num_batched_tokens: 1024 - max_num_seqs: 1 - -profiles: - qwen3.5-122b-a10b-fp8-tp4-262k-local: - description: "Single Qwen3.5-122B-A10B-FP8 vLLM runtime across all 4 GPUs at 262k context." - vllm: - enable_responses_api_store: false - logging_level: INFO - providers: - vllm: - runtimes: - qwen-122b-fp8: - model: qwen3.5-122b-a10b-fp8-local - served_model_name: qwen3.5-122b-a10b-fp8-262k - placement: - strategy: exact - gpu_indices: [0, 1, 2, 3] - topology: - tensor_parallel_size: 4 - runtime: - max_model_len: 262144 - gpu_memory_utilization: 0.95 - max_num_batched_tokens: 1024 - max_num_seqs: 1 - enable_prefix_caching: false - extra_args: - - --language-model-only - - --reasoning-parser - - qwen3 - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - qwen3.5-122b-a10b-fp8-262k: - provider: vllm - runtime: qwen-122b-fp8 -EOF -``` - ---- - -## 4. Switch to the local profile - -```bash -python manage.py switch qwen3.5-122b-a10b-fp8-tp4-262k-local -``` - ---- - -## 5. Render the stack - -```bash -python manage.py render -``` - -This creates: - -- `generated/docker-compose.yml` -- `generated/.env` - ---- - -## 6. Set your Hugging Face token - -Add your token to `generated/.env`: - -```bash -grep '^HF_TOKEN=' generated/.env -``` - -Then edit the file and set: - -```text -HF_TOKEN=your_token_here -``` - ---- - -## 7. Start the stack - -From the repo root: - -```bash -infer-stack up -d -``` - ---- - -## 8. Verify the backend - -Check that the model is exposed: - -```bash -infer-stack smoke-test -``` - -You should see: - -- `qwen3.5-122b-a10b-fp8-262k` - ---- - -## 9. Open the UI - -Open: - -```text -http://127.0.0.1:13000 -``` - -On first run, create the Open WebUI admin account. - -Then select the model: - -- `qwen3.5-122b-a10b-fp8-262k` - ---- - -## 10. Optional repo smoke test - -From the repo root: - -```bash -python manage.py smoke-test --model qwen3.5-122b-a10b-fp8-262k --prompt "Say hello in one sentence." -``` - ---- - -## Summary - -This profile serves: - -- `Qwen/Qwen3.5-122B-A10B-FP8` -- on 4 x 96GB GPUs -- with tensor parallel size 4 -- at 262,144 token context -- through both the direct backend and Open WebUI diff --git a/recipies/compose_qwen36_35b_a3b_4x96GB.md b/recipies/compose_qwen36_35b_a3b_4x96GB.md deleted file mode 100644 index c9cbd4a2..00000000 --- a/recipies/compose_qwen36_35b_a3b_4x96GB.md +++ /dev/null @@ -1,251 +0,0 @@ -# Compose recipe: Qwen3.6-35B-A3B on a 4x96GB host - -> **Pre-leasing.** This page uses the stack-profile CLI (`setup`, `up`, `deploy`, `switch`, `smoke-test`, `config.yaml` profiles), which was removed. Its commands no longer run. The model and vLLM settings may still inform a catalog entry; the current workflow is in the [README](../README.md). - -This is the shortest end-to-end example for serving **Qwen/Qwen3.6-35B-A3B** on a machine with **4 x 96GB GPUs** using the **Compose** backend. - -This profile uses: - -- two model instances -- GPUs 0 and 1 for one instance -- GPUs 2 and 3 for one instance -- tensor parallel size 2 per instance -- text-only mode -- native **262,144** token context -- the Qwen reasoning parser -- higher batching and concurrency for more users -- Open WebUI on top of LiteLLM - ---- - -## Quick path: use the built-in profile - -This deployment is available out of the box as the built-in profile -`qwen3.6-35b-a3b-dual-tp2-4x96`. To skip writing any local YAML: - -```bash -infer-stack setup --backend compose --profile qwen3.6-35b-a3b-dual-tp2-4x96 -infer-stack render --yes -infer-stack up -d -``` - -The rest of this recipe walks through the same deployment manually, so you -can see how the model and profile YAML map to the rendered Compose stack. - ---- - -## 1. Start from the repo root - -```bash -cd /path/to/infer_stack -``` - ---- - -## 2. Initialize Compose config - -```bash -python manage.py setup --backend compose -``` - ---- - -## 3. Write the local model and profile definitions - -Write provider-specific model definitions and a stack profile in `models.yaml`: - -```bash -cat > models.yaml <<'EOF' -vllm_models: - qwen3.6-35b-a3b-local: - hf_model_id: Qwen/Qwen3.6-35B-A3B - tokenizer_name: Qwen/Qwen3.6-35B-A3B - served_model_name: qwen3.6-35b-a3b - family: qwen3.6 - modalities: [text] - memory_class_gib: 80 - min_vram_gib_per_replica: 24 - preferred_gpu_count: 2 - context_window: 262144 - defaults: - max_model_len: 262144 - gpu_memory_utilization: 0.95 - enable_prefix_caching: false - max_num_batched_tokens: 8192 - max_num_seqs: 8 - -profiles: - qwen3.6-35b-a3b-dual-tp2-262k-local: - description: "Two Qwen3.6-35B-A3B vLLM runtimes across 4 GPUs at 262k context." - vllm: - enable_responses_api_store: false - logging_level: INFO - providers: - vllm: - runtimes: - qwen36-35b-gpu01: - model: qwen3.6-35b-a3b-local - served_model_name: qwen3.6-35b-a3b-gpu01 - placement: - strategy: exact - gpu_indices: [0, 1] - topology: - tensor_parallel_size: 2 - runtime: - max_model_len: 262144 - gpu_memory_utilization: 0.95 - max_num_batched_tokens: 8192 - max_num_seqs: 8 - enable_prefix_caching: false - extra_args: - - --language-model-only - - --reasoning-parser - - qwen3 - - qwen36-35b-gpu23: - model: qwen3.6-35b-a3b-local - served_model_name: qwen3.6-35b-a3b-gpu23 - placement: - strategy: exact - gpu_indices: [2, 3] - topology: - tensor_parallel_size: 2 - runtime: - max_model_len: 262144 - gpu_memory_utilization: 0.95 - max_num_batched_tokens: 8192 - max_num_seqs: 8 - enable_prefix_caching: false - extra_args: - - --language-model-only - - --reasoning-parser - - qwen3 - gateways: - litellm: - enabled: true - frontends: - open_webui: - enabled: true - provider: litellm - routes: - qwen3.6-35b-a3b-gpu01: - provider: vllm - runtime: qwen36-35b-gpu01 - qwen3.6-35b-a3b-gpu23: - provider: vllm - runtime: qwen36-35b-gpu23 -EOF -``` - ---- - -## 4. Switch to the local profile - -```bash -python manage.py switch qwen3.6-35b-a3b-dual-tp2-262k-local -``` - -Optional sanity check before rendering: - -```bash -python manage.py describe-profile qwen3.6-35b-a3b-dual-tp2-262k-local --format yaml -``` - ---- - -## 5. Render the stack - -```bash -python manage.py render -``` - -This creates: - -- `generated/docker-compose.yml` -- `generated/.env` - ---- - -## 6. Set your Hugging Face token - -Add your token to `generated/.env`: - -```bash -grep '^HF_TOKEN=' generated/.env -``` - -Then edit the file and set: - -```text -HF_TOKEN=your_token_here -``` - ---- - -## 7. Start the stack - -From the repo root: - -```bash -infer-stack up -d -``` - ---- - -## 8. Verify the backend - -Check that both model endpoints are exposed: - -```bash -infer-stack smoke-test -``` - -You should see: - -- `qwen3.6-35b-a3b-gpu01` -- `qwen3.6-35b-a3b-gpu23` - ---- - -## 9. Open the UI - -Open: - -```text -http://127.0.0.1:13000 -``` - -On first run, create the Open WebUI admin account. - -Then select one of the models: - -- `qwen3.6-35b-a3b-gpu01` -- `qwen3.6-35b-a3b-gpu23` - ---- - -## 10. Optional repo smoke tests - -From the repo root: - -```bash -python manage.py smoke-test --model qwen3.6-35b-a3b-gpu01 --prompt "Say hello in one sentence." - -python manage.py smoke-test --model qwen3.6-35b-a3b-gpu23 --prompt "Say hello in one sentence." -``` - ---- - -## Summary - -This profile serves: - -- `Qwen/Qwen3.6-35B-A3B` -- as two TP2 instances across all 4 x 96GB GPUs -- at 262,144 token context -- with `max_num_batched_tokens: 8192` -- with `max_num_seqs: 8` -- with `--reasoning-parser qwen3` -- through both the direct backend and Open WebUI - -Note: with the current repo templates, each backend instance is exposed as its own LiteLLM model name. diff --git a/tests/test_leasing_suggest.py b/tests/test_leasing_suggest.py index 7bf5654e..91f0a406 100644 --- a/tests/test_leasing_suggest.py +++ b/tests/test_leasing_suggest.py @@ -272,3 +272,23 @@ def test_display_gpu_reservation_shrinks_the_pool(): def test_empty_inventory_yields_empty_suggestion(): out = suggest_catalog({'gpu_count': 0, 'gpus': []}) assert out == {'models': {}, 'endpoints': {}} + + +def test_a_4x96_host_gets_the_former_recipes_as_variants(): + # Queue item 8a: the pre-leasing 4 x 96 GB recipes live on as gated + # variants, so `catalog suggest` offers their tuning where it applies. + inv = {'gpu_count': 4, 'gpus': [_gpu(i, 96) for i in range(4)]} + out = suggest_catalog(inv) + eps = out['endpoints'] + wanted = {'qwen3.5-122b-a10b-tp4-128k': (4, 131072), + 'qwen3.5-122b-a10b-fp8-tp4-262k': (4, 262144), + 'qwen3.6-35b-a3b-tp2-262k': (2, 262144)} + for name, (tp, ctx) in wanted.items(): + rt = eps[name]['runtime'] + assert (rt['tensor_parallel_size'], rt['max_model_len']) == (tp, ctx) + assert rt['shm_size'] == '16g' + assert rt['extra_args'] == ['--language-model-only', '--reasoning-parser', 'qwen3'] + Catalog.from_dict(out) # a valid catalog as suggested + + small = {'gpu_count': 2, 'gpus': [_gpu(i, 48) for i in range(2)]} + assert not set(wanted) & set(suggest_catalog(small)['endpoints']) diff --git a/tests/test_shm_size.py b/tests/test_shm_size.py new file mode 100644 index 00000000..8e75f93a --- /dev/null +++ b/tests/test_shm_size.py @@ -0,0 +1,56 @@ +"""``runtime.shm_size``: /dev/shm for a vLLM engine on Compose (queue item 8b). + +Opt-in: an endpoint that does not set it renders exactly as before, so no +running engine is recreated by the upgrade. A KubeAI pod already mounts a +memory-backed /dev/shm, so the key is Compose's alone. +""" + +import pytest + +from infer_stack.leasing.catalog import Catalog, CatalogError +from infer_stack.leasing.compose import _SHM_WARNED, _vllm_service +from infer_stack.leasing.models import Deployment, DeploymentState + +IMAGES = {'vllm': 'vllm/vllm-openai:latest'} + + +def _deployment(runtime, gid='dep'): + return Deployment( + gid, 'ck-' + gid, 'vllm', 'shared-compatible', {}, + {'engine': 'vllm', 'hf_model_id': 'org/m', 'served_model_name': 'm', + 'runtime': runtime, 'reclaim': 'stop'}, + {'m': {'served_model_name': 'm', 'protocol': 'chat'}}, + DeploymentState.LIVE, 0.0, 0.0, + ) + + +def test_shm_size_renders_when_set_and_is_absent_otherwise(): + svc = _vllm_service(_deployment({'tensor_parallel_size': 2, 'shm_size': '16g'}), + [0, 1], None, IMAGES, {}) + assert svc['shm_size'] == '16g' + plain = _vllm_service(_deployment({}), [0], None, IMAGES, {}) + assert 'shm_size' not in plain # today's catalogs render unchanged + assert not any('shm' in arg for arg in plain['command']) + + +def test_a_parallel_engine_without_it_is_warned_once(monkeypatch): + from infer_stack import _log + + seen = [] + monkeypatch.setattr(_log.logger, 'warning', lambda msg, *a: seen.append(msg.format(*a))) + _SHM_WARNED.discard('tp2') + for _ in range(3): + _vllm_service(_deployment({'tensor_parallel_size': 2}, 'tp2'), [0, 1], None, IMAGES, {}) + _vllm_service(_deployment({}, 'tp1'), [0], None, IMAGES, {}) + assert len(seen) == 1 and 'runtime.shm_size' in seen[0] and '2 GPUs' in seen[0] + + +def test_the_catalog_checks_shm_size(): + def catalog(shm): + return {'models': {'m': {'source': 'hf://org/m'}}, + 'endpoints': {'e': {'engine': 'vllm', 'model': 'm', + 'runtime': {'shm_size': shm}}}} + + Catalog.from_dict(catalog('16g')) + with pytest.raises(CatalogError, match='shm_size must be a size'): + Catalog.from_dict(catalog('plenty')) From 1f58cc64cae3e77463305a977a17ba0877d0a1c5 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 17:35:50 -0400 Subject: [PATCH 070/141] queue: reopen with review part 1 (replica residency, backend protocol, cross-feature tests) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 16 +++++-- docs/queue.md | 62 +++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 4c85f48f..e89b423e 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,8 +1,11 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **all phases done** 2026-09-26; P4 and P5 end in -a handover run on real hardware (`dev/handover/`). Execution order: [../queue.md](../queue.md). +`dev/backend-unification` · phases P1-P6 landed 2026-09-26 (P4 and P5 end in +a handover run on real hardware, `dev/handover/`) · **reopened 2026-09-27** by +an outside review: residency treats KubeAI replicas as duplicates, the +backend protocol is not the one the backends implement, and the parity suite +misses cross-feature invariants (queue items 11-13). Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the @@ -39,7 +42,14 @@ it is not a sibling with its own habits. ## Phases -### P0. One contract, one liveness view, one diagnosis — done +### P0. One contract, one liveness view, one diagnosis — landed, reopened + +*Reopened 2026-09-27:* the contract `Controller` is typed against still +inherits the pre-admission `realize/teardown` protocol and carries Compose +mechanics (`run`, `_load_sidecar`, `network`, `adopted`); and the liveness +view equates "more than one unit" with a conflict, which is wrong for +KubeAI replicas. Queue items 11 and 12. + The 2026-09-24 audit landed: an e2e on k3s (K0); the LiteLLM gateway fronting both backends, so the alias and key are the same everywhere (K1); diff --git a/docs/queue.md b/docs/queue.md index 5aaa25e4..f762e78c 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -429,3 +429,65 @@ Both keep their own config and data roots and print PASS/FAIL per step. **Decided without GPUs, worth confirming on them:** 8a's recipe variants were not re-run since leasing, and 8b's `shm_size` fixes a TP failure not yet observed here. + +## Reopened 2026-09-27: outside review, part 1 + +Found by a review of the finished campaign; a part 2 (publication and +recovery, placement and profiles, destructive operations, gateway authority, +controller decomposition, test quality) follows and may reopen more. Scope +here: these three and their direct tests and docs, nothing broader. + +### 11. [ ] Residency: replicas are not duplicates + +*Why added:* `Residency.ambiguous()` means "more than one unit" and +`resident()` returns a unit only when there is exactly one. On Compose two +containers for one deployment is a conflict and must fail closed. On KubeAI +`runtime.min_replicas: 2` is a supported, documented setting, and its two +pods made the deployment `ambiguous` and not resident: releasing the last +lease of a keep-warm replicated Model pruned it, `leases` showed it +AMBIGUOUS, and `_pinned_endpoints` stopped protecting its definition. + +**Do:** model residency at the deployment level (resident, conflicted, +units, warm units) and keep "one physical unit" only where GPU adoption on +Compose needs it. Compose keeps failing closed on duplicate containers; +"could not inspect" stays distinct from "nothing running"; KubeAI gains no +host-GPU accounting; no KubeAI branch in the controller. + +**Done when:** tests show a two-replica KubeAI Model is resident and not +conflicted; releasing its last keep-warm lease keeps it desired; `gc` +keeps it; `leases` shows it running; its idle definition stays pinned; a +rollout snapshot (old pod leaving, new pod up) prunes nothing; two Compose +containers for one deployment still fail closed. + +### 12. [ ] One backend protocol, the one the controller uses + +*Why added:* `Controller` takes a `Backend` (the old `realize/teardown/ +observe` protocol) and casts it to `AdmissionBackend`, which inherits that +protocol and adds Compose internals: `run` (a docker command; on KubeAI the +same name runs kubectl), `_load_sidecar()` whose schema the controller +reads, `network`, `adopted`, probed with `hasattr`. KubeAI keeps a `realize` +that is deliberately `pass`, and would be unsafe if it did anything. + +**Do:** type `Controller` against the protocol it uses, with no cast. Move +`realize/teardown` to a small protocol behind `SimpleAdmission`. Replace the +Compose internals with semantic capabilities (orphan removal, stable +network, legacy adoption) whose mechanics stay inside Compose. No new +`hasattr` probes. + +**Done when:** no production backend has a method that exists only to +satisfy a protocol; the controller builds no `docker` command and reads no +sidecar; `ty` passes without the cast. + +### 13. [ ] Parity tests: cross-feature invariants + +*Why added:* the parity suite's KubeAI fake ran one pod per Model, so +"replicas render" and "release keeps keep-warm" each passed while their +combination was broken. Rows of the matrix are not independent. + +**Do:** a few poison cases, not a matrix: the lifecycle with replica count +above one, and a rollout snapshot. Record the rule in the roadmap: a +backend-specific feature needs one test combining it with the common +lifecycle. + +**Done when:** those tests are in `tests/test_parity.py` (or beside it) and +fail against the code before item 11. From 4555fcd095ad536f18f061c3dec5c985d91b5eff Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 17:51:33 -0400 Subject: [PATCH 071/141] queue: add review part 2 (items 14-22); roadmap status is review hardening Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 11 +- docs/queue.md | 231 ++++++++++++++++++++---- 2 files changed, 206 insertions(+), 36 deletions(-) diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index e89b423e..31454cd0 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,11 +1,12 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · phases P1-P6 landed 2026-09-26 (P4 and P5 end in -a handover run on real hardware, `dev/handover/`) · **reopened 2026-09-27** by -an outside review: residency treats KubeAI replicas as duplicates, the -backend protocol is not the one the backends implement, and the parity suite -misses cross-feature invariants (queue items 11-13). Execution order: [../queue.md](../queue.md). +`dev/backend-unification` · **features complete, review hardening +outstanding**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover +run on real hardware, `dev/handover/`); a two-part outside review on +2026-09-27 found replica residency, the backend protocol, scheduler reclaim, +publication phases, secret rotation, route ordering, coalescing and route +seeding to fix (queue items 11-22). Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the diff --git a/docs/queue.md b/docs/queue.md index f762e78c..bd9a4d44 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -430,53 +430,71 @@ Both keep their own config and data roots and print PASS/FAIL per step. were not re-run since leasing, and 8b's `shm_size` fixes a TP failure not yet observed here. -## Reopened 2026-09-27: outside review, part 1 - -Found by a review of the finished campaign; a part 2 (publication and -recovery, placement and profiles, destructive operations, gateway authority, -controller decomposition, test quality) follows and may reopen more. Scope -here: these three and their direct tests and docs, nothing broader. +## Reopened 2026-09-27: outside review + +Found by a two-part review of the finished campaign. Part 1: items 11-13. +Part 2: items 14-22, in the order the review recommends; item 11 also +takes part 2's finding on replicated startup diagnosis. The campaign's +completion rule changes with them: not "every *same* row has a test" but +"every shared lifecycle invariant holds when combined with the +backend-specific features that stress it" (item 13 and the poison cases in +each item). Item 9's UX audit is not the closing gate for these: a terminal +audit cannot show transaction safety under partial failure. + +Rules for this section: correctness before cleanup; no broad CLI polish; a +poison test that fails on the old code with every fix; no new `hasattr` +probes; no second KubeAI lifecycle path; backwards compatibility for cards, +catalogs and CLI invocations that work today. ### 11. [ ] Residency: replicas are not duplicates -*Why added:* `Residency.ambiguous()` means "more than one unit" and -`resident()` returns a unit only when there is exactly one. On Compose two +*Why added:* `Residency.ambiguous()` meant "more than one unit" and +`resident()` returned a unit only when there was exactly one. On Compose two containers for one deployment is a conflict and must fail closed. On KubeAI `runtime.min_replicas: 2` is a supported, documented setting, and its two pods made the deployment `ambiguous` and not resident: releasing the last lease of a keep-warm replicated Model pruned it, `leases` showed it -AMBIGUOUS, and `_pinned_endpoints` stopped protecting its definition. +AMBIGUOUS, and `_pinned_endpoints` stopped protecting its definition. Part 2 +adds: `diagnose_startup()` returns nothing unless there is exactly one unit, +so a replicated Model whose every pod crash-loops waits out the timeout +instead of failing fast. **Do:** model residency at the deployment level (resident, conflicted, units, warm units) and keep "one physical unit" only where GPU adoption on -Compose needs it. Compose keeps failing closed on duplicate containers; -"could not inspect" stays distinct from "nothing running"; KubeAI gains no -host-GPU accounting; no KubeAI branch in the controller. +Compose needs it. Audit every "the one resident" caller: keep-warm +admission, `_pinned_endpoints`, status, served-model rows, TUI/runtime +views, readiness, logs, startup diagnosis. Diagnosis of a replica set: fatal +when every unit is fatal, nothing while one is healthy. Compose keeps +failing closed on duplicate containers; "could not inspect" stays distinct +from "nothing running"; KubeAI gains no host-GPU accounting. **Done when:** tests show a two-replica KubeAI Model is resident and not -conflicted; releasing its last keep-warm lease keeps it desired; `gc` -keeps it; `leases` shows it running; its idle definition stays pinned; a -rollout snapshot (old pod leaving, new pod up) prunes nothing; two Compose -containers for one deployment still fail closed. +conflicted; releasing its last keep-warm lease keeps it desired; `gc` keeps +it; `leases` and `status` show it running; its idle definition stays pinned; +a rollout snapshot (old pod leaving, new pod up) prunes nothing; every +replica crash-looping fails fast with the engine's error, one healthy +replica does not; two Compose containers for one deployment still fail +closed; the k3s e2e has a two-replica keep-warm phase and passes. ### 12. [ ] One backend protocol, the one the controller uses -*Why added:* `Controller` takes a `Backend` (the old `realize/teardown/ -observe` protocol) and casts it to `AdmissionBackend`, which inherits that -protocol and adds Compose internals: `run` (a docker command; on KubeAI the +*Why added:* `Controller` took a `Backend` (the old `realize/teardown/ +observe` protocol) and cast it to `AdmissionBackend`, which inherited that +protocol and added Compose internals: `run` (a docker command; on KubeAI the same name runs kubectl), `_load_sidecar()` whose schema the controller -reads, `network`, `adopted`, probed with `hasattr`. KubeAI keeps a `realize` -that is deliberately `pass`, and would be unsafe if it did anything. +read, `network`, `adopted`, probed with `hasattr`. KubeAI kept a `realize` +that was deliberately `pass`, and would be unsafe if it did anything. **Do:** type `Controller` against the protocol it uses, with no cast. Move `realize/teardown` to a small protocol behind `SimpleAdmission`. Replace the -Compose internals with semantic capabilities (orphan removal, stable -network, legacy adoption) whose mechanics stay inside Compose. No new -`hasattr` probes. +Compose internals with semantic capabilities (stable network, legacy +adoption, orphan removal, placement notes) whose mechanics stay inside +Compose. No new `hasattr` probes. **Done when:** no production backend has a method that exists only to satisfy a protocol; the controller builds no `docker` command and reads no -sidecar; `ty` passes without the cast. +sidecar; `ty` checks each backend against the protocol, and fails when one +drifts. ### 13. [ ] Parity tests: cross-feature invariants @@ -484,10 +502,161 @@ sidecar; `ty` passes without the cast. "replicas render" and "release keeps keep-warm" each passed while their combination was broken. Rows of the matrix are not independent. -**Do:** a few poison cases, not a matrix: the lifecycle with replica count -above one, and a rollout snapshot. Record the rule in the roadmap: a -backend-specific feature needs one test combining it with the common -lifecycle. +**Do:** a small poison suite, not a matrix: replica cardinality (multi-pod +keep-warm, overlapping pods in a rollout, replicated crash-loop), and the +cases items 14-19 add. The fake keeps "one Model, one pod" as its default +and gains a replica and a scheduler mode. Record the completion rule in the +roadmap. + +**Done when:** the suite is in `tests/test_parity.py` (or beside it), each +case fails on the code before its fix, and the roadmap states the rule. + +### 14. [ ] Scheduler-aware reclaim, not "Unschedulable means evict" + +*Why added:* KubeAI's readiness sets `needs_room` whenever a pod waits with +reason `Unschedulable`, and `wait_ready` answers by evicting the +longest-idle deployment every 30 s. `Unschedulable` also covers an +impossible node selector, an untolerated taint, affinity, a missing +profile, and pressure no idle Model relieves; and on several nodes an idle +Model on node A frees nothing for a pod that fits only node B. A permanently +impossible request can empty the warm set one Model at a time. + +**Do:** replace the boolean with a structured signal, or a backend +operation that names reclaim candidates for a blocked deployment. Read the +scheduler's condition message, not only its reason; evict only for a +capacity shortage; only a resident idle victim in a compatible scheduling +domain (its node could host the blocked pod, by profile or node selector) +is a candidate. Keep: a leased deployment is never a victim; the policy +"leased demand outranks idle keep-warm" stays in the controller. + +**Done when:** poison tests: an impossible node selector evicts nothing; an +untolerated taint evicts nothing; with two profiles on two nodes the idle +Model on the wrong node is not chosen; real reclaimable pressure evicts a +compatible idle Model and the leased one proceeds. + +### 15. [ ] Publication phases: the approved digest outlives a partial apply + +*Why added:* `_apply_pending()` clears the approved-render digest right +after `apply()` returns, before checking it returned `False`. Compose +returns `False` when the runtime changed but dynamic routes did not verify: +the publication stays pending, but the record of which render was approved +is gone, so a renderer that changes before the retry (an upgrade) applies +an unapproved render. + +**Do:** keep the digest until the publication completes, unless an explicit +re-approval replaces it. Make the apply result say what happened (runtime +applied, routes verified, complete) instead of `True / False / raise`, and +only as far as the phases that differ today. + +**Done when:** a test: preview approves D1, the first apply changes the +runtime and returns partial; the publication is pending with D1; the +renderer changes to D2; an ordinary retry refuses on the digest mismatch; +`infer-stack apply` re-approves and applies. + +### 16. [ ] Secret rotation is a transaction + +*Why added:* `rotate_gateway_key()` writes the new key into `.env`, then +publishes, and restores the old key only on `ConvergeAborted`. A strict +residency or render failure before apply leaves `.env` on the new key while +LiteLLM runs the old one, so clients are handed a key that does not work. +The tests' fake gateway reads `.env` per request, which hides it. + +**Do:** restore the old key on any failure before apply begins; after apply +may have begun, keep a recoverable pending publication instead (do not +revert the file blindly). Uses item 15's phase information. + +**Done when:** a gateway fake that captures its key when it (re)starts; +tests for a declined approval, a render failure before apply (both: old +file, old running key), and a failure after the gateway restarted (pending, +converges on retry). + +### 17. [ ] Dynamic routes never point at a torn-down upstream + +*Why added:* Compose `apply()` removes departing engines, then reconciles +routes; KubeAI deletes stale Models, then applies the gateway. If route +removal fails, the publication is pending but LiteLLM still routes to a +dead upstream, which with several dedicated deployments behind one alias +fails some requests. Replacing a drifted route deletes before it adds, so a +failed add leaves a gap. + +**Do:** order a removal as: remove and verify the routes that would dangle, +tear down the departing upstreams, converge the desired ones, add and +verify new routes. Make the replace-route gap an explicit, reported state. + +**Done when:** failure injected into route deletion while releasing one of +two dedicated deployments for one alias leaves the departing upstream +running (the invariant, not just `publication_pending`), on Compose and on +KubeAI with the host gateway. + +### 18. [ ] Coalescing prefers a LIVE deployment over reviving an IDLE one + +*Why added (older than this campaign):* `plan_acquire()` takes the first +compatible deployment by creation time among LIVE and IDLE, so an older +IDLE deployment is revived even when a LIVE one already satisfies the +request. 8k, then 32k, then 4k on one GPU: the 4k request revives the idle +8k deployment and queues, although the live 32k one serves it now. + +**Do:** rank adequate candidates LIVE, then resident IDLE, then other IDLE, +then create; the ledger stays the authority on eligibility and takes a +residency hint from the controller. + +**Done when:** the 8k/32k/4k sequence on a one-slot backend coalesces the +4k request onto the live 32k deployment: nothing revived, no placement, no +queue, no runtime change. + +### 19. [ ] `routes seed` fails closed on a redefinition; one route API + +*Why added:* the registry merge is documented as additive, but a same-name +row with a different definition wins with a warning, and `routes seed` +skips confirmation because "seeding is additive". That silently redirects a +public alias. `routes prune` in the CLI reaches into +`controller._admission_view`, `gateway._load_route_registry`, +`backend._converge_lock`, `_atomic_write` and `gateway._registry_file`. + +**Do:** a public route-registry operation that plans a seed or prune +(added, unchanged, updated, conflicted) and commits it under the normal +locks; conflicts refuse by default, an explicit override shows the change +and asks. The CLI presents and confirms only. + +**Done when:** tests: seed A; seed identical A is a no-op; seed a +conflicting A refuses and leaves the registry unchanged; the override +reports A updated. No private names from the CLI. + +### 20. [ ] Preview does not write secrets + +*Why added:* rendering calls `master_key()`, `db_password()`, +`webui_secret()`, which generate and write missing secrets, so a preview +that is then refused has still changed `.env`. + +**Do:** initialize secrets before a pure render, or commit generated +secrets with the publication. If kept, document the narrower guarantee. + +**Done when:** a refused acquire on a fresh data root leaves no `.env`, or +the docs say exactly what a preview may write. + +### 21. [ ] Controller decomposition, where the authorities now show it + +*Why added:* the controller holds admission, residency interpretation, +placement, publication markers, recovery snapshots, network migration, +secret rotation, adoption, orphans, reclaim, leases and rollback. Split +only along state machines (publication/recovery, admission/reclaim, +profile snapshots, backend capabilities), and only after 14-20. + +**Done when:** each extracted part needs no private cross-layer call, or +the item records why nothing was extracted. + +### 22. [ ] Docs and full verification + +*Why added:* the parity matrix, roadmap and `known-limitations.md` describe +the semantics these items change; `known-limitations.md` still says KubeAI +has no strict residency. + +**Do:** update them from the resulting semantics; set the roadmap status to +"features complete, review hardening" until this section is done; run the +full suite, `ty`, flake8, and the full k3s e2e. + +**Done when:** those pass and the docs match the code. -**Done when:** those tests are in `tests/test_parity.py` (or beside it) and -fail against the code before item 11. +Also observed while working on these: `test_an_edit_made_outside_the_tui_appears_on_the_next_refresh` +failed once in a full-suite run and passed alone three times; timing +around the catalog reload. Looked at under item 22 if it recurs. From 35be92c7463d52af8a52606dd31273a812e2c36b Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 17:55:03 -0400 Subject: [PATCH 072/141] Residency asks about deployments; one backend protocol; replica poison tests Residency: several pods for one KubeAI Model are replicas (the snapshot says so), several Docker containers for one deployment are a conflict. Callers ask is_resident / is_conflicted / resident_gpus; only GPU adoption asks for the unique unit. A keep-warm two-replica Model now survives its release and gc, shows running, stays pinned, and a replica set crash-looping fails fast. Controller takes a ServingBackend, with no cast. realize/teardown are a Realizer protocol behind SimpleAdmission; KubeAI's no-op realize is gone. Compose-only work (stable network, legacy adoption, orphan removal) is a HostRuntime capability; the controller builds no docker command and reads no sidecar. ty checks each backend against the protocol. Parity suite: the KubeAI fake runs minReplicas pods; poison cases for replicated keep-warm, a rollout snapshot, replicated crash-loop, and Compose duplicates. e2e: a two-replica keep-warm phase. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/kubeai_e2e.sh | 42 ++++ infer_stack/backends/kubeai.py | 26 +-- infer_stack/cli/commands_leasing.py | 8 +- infer_stack/cli/commands_runtime.py | 17 +- infer_stack/leasing/__init__.py | 11 +- infer_stack/leasing/backend.py | 231 ++++++++++++------- infer_stack/leasing/compose.py | 37 ++- infer_stack/leasing/controller.py | 154 +++++++------ infer_stack/leasing/diagnosis.py | 51 ++-- infer_stack/leasing/residency.py | 95 ++++++-- tests/test_leasing_kubeai.py | 4 +- tests/test_leasing_residency.py | 28 +-- tests/test_leasing_serialised_publication.py | 2 +- tests/test_parity.py | 116 +++++++++- 14 files changed, 570 insertions(+), 252 deletions(-) diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 92dfd7cb..9c3942ef 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -31,6 +31,8 @@ # on two nodes with fake GPU labels. Needs # dev/k3s_agent_container.sh up (E2E_SIZED_NODE names # the node; default k3s-agent-b). +# E2E_REPLICAS 1 (default): a keep-warm Model with two replicas +# stays up after its last release and a gc. # E2E_CLUSTER_GATEWAY 1 (default): finish with the gateway in the cluster # (kubeai_gateway cluster): a NodePort, doctor, # secrets rotate, stack down. @@ -174,6 +176,46 @@ remaining=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ [ "$remaining" = 0 ] || { echo '!! a Model was applied for it' >&2; exit 1; } echo ' refused at admission: no lease, no Model' +if [ "${E2E_REPLICAS:-1}" = 1 ]; then + # Two pods for one Model are its replicas, not two claimants: releasing the + # last lease of a keep-warm Model must leave it up (queue item 11). + echo '== a keep-warm Model with two replicas stays up after its release' + cat >> "$WORK/config/catalog.yaml" <&2; exit 1; } + run_is release --env-file "$WORK/pair.env" --yes + run_is gc + models=$(kubectl -n "$NAMESPACE" get models.kubeai.org \ + -l infer-stack/managed=true -o name | wc -l) + [ "$models" = 1 ] || { echo '!! the idle replicated Model was pruned' >&2; exit 1; } + run_is leases > "$WORK/pair.log" 2>&1 + if grep -q 'AMBIGUOUS\|NOT-RUNNING' "$WORK/pair.log"; then + cat "$WORK/pair.log" >&2; echo '!! replicas reported as a problem' >&2; exit 1 + fi + grep -q 'state=idle running=running' "$WORK/pair.log" \ + || { cat "$WORK/pair.log" >&2; echo '!! the idle Model is not shown running' >&2; exit 1; } + echo ' released and gc-ed: still desired, both replicas up, shown running' + run_is evict --all --yes + for _ in $(seq 60); do + [ -z "$(kubectl -n "$NAMESPACE" get models.kubeai.org -l infer-stack/managed=true -o name)" ] && break + sleep 2 + done +fi + if [ "${E2E_MAKE_ROOM:-0}" = 1 ]; then # Needs a profile only one Model fits at a time (`cpu-half` in # dev/e2e_tests/kubeai-cpu-values.yaml): E2E_ROOM_PROFILE=cpu-half. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 53cd481a..2fd74f25 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -951,7 +951,7 @@ def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: from ..leasing.residency import ResidencyUnknown try: - pods = self.residency().containers(deployment.id) + pods = self.residency().units(deployment.id) except ResidencyUnknown: return '' parts: list[str] = [] @@ -971,17 +971,19 @@ def startup_failure(self, deployment: Deployment) -> str | None: from ..leasing.residency import ResidencyUnknown try: - pods = self.residency().containers(deployment.id) + residency = self.residency() except ResidencyUnknown: return None # cannot read the cluster: say nothing - return diagnose_startup(pods, lambda: self.deployment_logs(deployment, tail=200)) + return diagnose_startup(residency.units(deployment.id), + lambda: self.deployment_logs(deployment, tail=200), + replicated=residency.replicated) def _waiting_reason(self, deployment: Deployment) -> str: """The pod's own reason for not running yet (``ImagePullBackOff``...).""" from ..leasing.residency import ResidencyUnknown try: - pods = self.residency().containers(deployment.id) + pods = self.residency().units(deployment.id) except ResidencyUnknown: return '' reasons = sorted({p.reason for p in pods if p.reason and p.state != 'running'}) @@ -1056,20 +1058,8 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: }, } - # -- realize/teardown (Protocol completeness; converge path supersedes) --- - - def realize(self, deployment: Deployment) -> None: # pragma: no cover - # The controller always drives converge-style backends through - # converge()/apply(); realize exists only so the Backend Protocol is - # satisfied. Deliberately NOT converge([deployment]) — a one-element - # desired set would prune every other managed Model. - pass - - def teardown(self, deployment: Deployment) -> None: - name = model_name(deployment, unique=self.dynamic_routing) - self._kubectl( - ['delete', 'models.kubeai.org', name, '--ignore-not-found'] - ) + #: No containers of this host to manage: the cluster owns network and pods. + host_runtime = None def down(self) -> None: """Delete every infer-stack-managed Model (explicit stop), and the gateway.""" diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index c3d01690..c320d68b 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1238,7 +1238,7 @@ class EvictCLI(_ApprovalMixin): overriding keep-warm. Target by served endpoint alias or deployment id, or ``--all`` for every idle deployment. (Live models — those with an active lease — are never evicted; release them first.) On a terminal the teardown is shown - and confirmed before docker is touched (``--yes`` skips). + and confirmed before anything is torn down (``--yes`` skips). """ __command__ = 'evict' @@ -1996,9 +1996,9 @@ def _placement_view(controller): try: residency = backend.residency() for g in deployments: - c = residency.resident(g.id) - if g.id not in assignments and c is not None: - assignments[g.id] = list(c.gpus) + gpus = residency.resident_gpus(g.id) + if g.id not in assignments and gpus is not None: + assignments[g.id] = gpus except Exception: # noqa: BLE001 pass return observed, assignments diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 430b4597..bad89e0c 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -172,13 +172,16 @@ def _served_models(deployments, backend=None, *, for d in live: if residency is None: health = 'unverified' - elif residency.resident(d.id) is not None: - # Warm; a health check that has not passed yet means still loading. - health = ('starting' if residency.resident(d.id).health == 'starting' - else 'up') - elif residency.containers(d.id): - # There, but not warm: starting, crashed, or ambiguous. - health = residency.containers(d.id)[0].state + elif residency.is_resident(d.id): + # Warm; up once any unit (a replica) has passed its health check, + # still loading while none has. + warm = residency.warm_units(d.id) + health = 'starting' if all(c.health == 'starting' for c in warm) else 'up' + elif residency.is_conflicted(d.id): + health = 'conflicted' # several containers claim it + elif residency.units(d.id): + # There, but not warm: starting or crashed (every replica's state). + health = '/'.join(sorted({c.state for c in residency.units(d.id)})) elif pending: # Recorded, not applied yet: an apply is running, or a failed one # left the change for `infer-stack apply`. diff --git a/infer_stack/leasing/__init__.py b/infer_stack/leasing/__init__.py index 21bb9d55..c23f9917 100644 --- a/infer_stack/leasing/__init__.py +++ b/infer_stack/leasing/__init__.py @@ -9,7 +9,14 @@ from __future__ import annotations -from .backend import Backend, MemoryBackend, NullBackend, Readiness +from .backend import ( + Backend, + HostRuntime, + MemoryBackend, + NullBackend, + Readiness, + ServingBackend, +) from .catalog import ( Catalog, CatalogError, @@ -61,6 +68,7 @@ 'AcquireOutcome', 'AcquireResult', 'Backend', + 'HostRuntime', 'Catalog', 'CatalogError', 'ComposeBackend', @@ -77,6 +85,7 @@ 'MemoryBackend', 'ModelSpec', 'NullBackend', + 'ServingBackend', 'RESERVED_ENDPOINT', 'RESERVED_ENGINE', 'Readiness', diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index d7023a9f..3ca6a3fc 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -1,14 +1,18 @@ """Backend protocol: the seam between the ledger and real serving. -The ledger decides *what should be running* (desired deployment deployments); a -backend makes it so. The :class:`Controller` reconciles between them through the -four methods below. Keeping this surface tiny is deliberate — it is the only -thing a new backend (Compose, KubeAI, ...) must implement, and it is where the -redesign draws the line between "infer-stack coordinates" and "the backend / -KubeAI / k8s schedules". - -All four methods MUST be idempotent: the reconciler may ``realize`` a deployment that -is already up, or ``teardown`` one that is already gone. +The ledger decides *what should be running* (the desired deployments); a +backend makes it so. :class:`ServingBackend` is everything the +:class:`Controller` asks of one, and all of it has a meaning on every +backend: render a desired set, apply it, say what is resident, probe +readiness, preview an admission. Compose, KubeAI, and the in-process +backends (through :class:`SimpleAdmission`) implement it. + +What only a backend that runs containers on this host can do (a stable +address network, adopting containers from before ownership labels, removing +unmanaged containers) is :class:`HostRuntime`, reached through +``backend.host_runtime``, which is ``None`` everywhere else. The controller +asks for the capability by meaning; the ``docker`` commands and the sidecar +stay inside Compose. """ from __future__ import annotations @@ -101,61 +105,101 @@ def allocates_gpus(backend) -> bool: return bool(getattr(backend, 'allocates_gpus', True)) -@runtime_checkable -class Backend(Protocol): - """What the :class:`Controller` needs from a serving backend. +class HostRuntime(Protocol): + """Capabilities of a backend that runs containers on this host (Compose). - Implementations: :class:`MemoryBackend` (here, for tests/dry-runs), and - later ``ComposeBackend`` / ``KubeAIBackend``. + ``backend.host_runtime`` is one of these, or ``None`` for a backend with + nothing of the kind (KubeAI: the cluster owns the network and the pods; + the in-process backends). Each method is a meaning; how it is done + (``docker network``, ``docker rm``, the render sidecar) is the + implementation's business. """ - def realize(self, deployment: Deployment) -> None: - """Ensure a deployment for ``deployment`` exists and is converging.""" + def network_table(self) -> dict[str, Any] | None: + """The stable-address table renders use, or ``None`` before a migration.""" ... - def teardown(self, deployment: Deployment) -> None: - """Ensure ``deployment``'s deployment is stopped/removed.""" + def configure_network(self, table: dict[str, Any] | None, *, + on_addresses: Callable[[dict[str, str]], None] | None = None, + ) -> None: + """Render with this table (``{'subnet', 'addresses'}``); ``on_addresses`` + persists addresses an approved render allocates.""" ... - def observe(self) -> set[str]: - """Return the set of deployment ids currently realized in the backend.""" + def subnet_clashes(self, subnet: str) -> list[str]: + """Why ``subnet`` cannot be used here: overlapping networks or routes.""" ... - def probe_ready( - self, deployment: Deployment, endpoint: str - ) -> Readiness: - """Report whether one served ``endpoint`` of ``deployment`` is ready.""" + def rendered_services(self) -> dict[str, tuple[str, str]]: + """``{service: (fingerprint, deployment id or '')}`` of the last render.""" + ... + + def set_adopted(self, table: dict[str, dict[str, str]]) -> None: + """Containers from before ownership labels that count as managed.""" + ... + + def remove_containers(self, container_ids: list[str]) -> None: + """Remove these containers now (orphans the operator agreed to remove).""" ... @runtime_checkable -class ConvergeBackend(Backend, Protocol): - """The optional converge-style surface the controller prefers. - - A backend exposing these drives the render/apply split: the controller - calls ``converge(desired, apply=False)`` inside the render lock (fast, - writes on-disk state only) and then ``apply()`` in the same lock hold - (slow, bounded; see :meth:`Controller._apply_pending`). After a - converge the controller reads the three ``last_*`` attributes: - - * ``last_unplaced`` — desired deployment ids the render/placement could not - deliver; an acquire whose deployment lands here fails loudly - (:class:`PlacementError`) and rolls its lease back. - * ``last_errors`` — per-deployment reasons, each prefixed with the - deployment id. - * ``last_assignments`` — deployment id -> GPU indices (empty for backends - where the cluster schedules). - - A backend with only ``realize``/``teardown`` gets this surface from - :class:`SimpleAdmission`. +class ServingBackend(Protocol): + """What the :class:`Controller` needs from a serving backend. + + Every acquire goes through admission: a strict :meth:`residency`, an + in-memory :meth:`preview`, then :meth:`converge` inside the render lock + (fast, writes the backend's state only) and :meth:`apply` in the same + lock hold (slow, bounded; see ``Controller._apply_pending``). After a + converge the controller reads the ``last_*`` attributes: + + * ``last_unplaced``: desired deployment ids the render could not deliver; + an acquire whose deployment lands here fails (:class:`PlacementError`) + and rolls its lease back; + * ``last_errors``: per-deployment reasons, each prefixed with the id; + * ``last_assignments``: deployment id -> GPU indices (empty where the + cluster schedules). + + Compose and KubeAI implement it directly; :class:`SimpleAdmission` + supplies it for the in-process backends. """ last_unplaced: set[str] last_errors: list[str] last_assignments: dict[str, list[int]] + last_preview_digest: str | None + + @property + def host_runtime(self) -> HostRuntime | None: + """This host's container capabilities, or ``None`` (see :class:`HostRuntime`).""" + ... + + def observe(self) -> set[str]: + """Deployment ids the backend has rendered and brought up (lenient).""" + ... + + def residency(self) -> Any: + """A strict :class:`~infer_stack.leasing.residency.Residency`, or raise.""" + ... + + def instances(self) -> list[Any]: + """Every running unit, as :class:`~infer_stack.leasing.instances.Instance`. + + Built from :meth:`residency`, so it raises when that does. + """ + ... + + def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: + """Whether one served ``endpoint`` of ``deployment`` answers.""" + ... + + def preview(self, desired: list[Deployment], placement: Any = None, *, + approve: bool = False) -> Any: + """Place and render ``desired`` without writing; ``(plan, rendered)``.""" + ... def converge(self, desired: list[Deployment], *, apply: bool = True, - placement: Any = None): + placement: Any = None) -> Any: """Render the desired set to backend state; optionally apply it.""" ... @@ -168,48 +212,36 @@ def apply(self) -> bool | None: """ ... + def placement_notes(self) -> dict[str, list[str]]: + """What the last render recorded about placement, for health views: + ``degraded`` (committed GPUs no longer valid) and ``displaced`` (idle + residents that yielded their GPUs). Empty where nothing is placed.""" + ... -@runtime_checkable -class AdmissionBackend(ConvergeBackend, Protocol): - """The surface the controller drives: every acquire goes through admission. - - Both real backends have it: strict residency and an in-memory - ``preview``; :class:`SimpleAdmission` supplies it for backends that - neither place nor inspect (the dry-run and test backends). Compose also takes the stable-address network and the - container-adoption table from the controller (``network``, - ``on_addresses``, ``adopted``); those are Compose-only, and the - controller hands them over only to a backend that declares them. The - controller reaches these through ``Controller._admitting``, only after - the capability check, so the type checker sees one named capability - instead of attributes a minimal :class:`Backend` does not have. - """ - network: dict[str, Any] | None - on_addresses: Callable[[dict[str, str]], None] | None - adopted: dict[str, Any] - last_preview_digest: str | None - def residency(self) -> Any: - """A strict :class:`~infer_stack.leasing.residency.Residency`, or raise.""" - ... +#: The name ``infer_stack.leasing`` has always exported for the controller's +#: backend protocol. +Backend = ServingBackend - def instances(self) -> list[Any]: - """Every running unit, as :class:`~infer_stack.leasing.instances.Instance`. +class Realizer(Protocol): + """The per-deployment primitives :class:`SimpleAdmission` converges with. - Built from :meth:`residency`, so it raises when that does. - """ - ... + Only the in-process backends implement these; a real backend converges a + whole desired set at once, where realizing one deployment alone would + prune the others. + """ - def preview(self, desired: list[Deployment], placement: Any = None, *, - approve: bool = False) -> Any: - """Place and render ``desired`` without writing; ``(plan, rendered)``.""" + def realize(self, deployment: Deployment) -> None: + """Ensure ``deployment`` exists (idempotent).""" ... - def run(self, args: list[str], **kwargs: Any) -> str: - """Run a runtime command (``docker ...``) and return its stdout.""" + def teardown(self, deployment: Deployment) -> None: + """Ensure ``deployment`` is gone (idempotent).""" ... - def _load_sidecar(self) -> dict: + def observe(self) -> set[str]: + """The deployment ids realized now.""" ... @@ -281,6 +313,11 @@ def _save_sidecar(self, data: dict) -> None: self._atomic_write(self._state_file, json.dumps(data, indent=2)) + def placement_notes(self) -> dict[str, list[str]]: + """``degraded`` / ``displaced`` as the last render recorded them.""" + sidecar = self._load_sidecar() + return {key: list(sidecar.get(key) or ()) for key in ('degraded', 'displaced')} + #: Digest of files an admission preview already had approved. _preapproved: str | None = None #: Digest of the files the last render produced (approved-digest guard). @@ -354,19 +391,24 @@ class RenderPreview: class SimpleAdmission: - """The admission surface for a backend that neither places nor inspects. + """:class:`ServingBackend` for a backend that neither places nor inspects. For in-process backends (the dry-run and test backends): they allocate no GPUs, their :meth:`residency` is what :meth:`observe` reports (each - deployment one warm instance), and converge/apply is ``realize`` / - ``teardown`` over the rendered set. A subclass that emulates capacity - overrides :meth:`plan`; one that emulates render failures overrides - :meth:`refuse`. Everything else goes through the one admission path the - real backends use. + deployment one warm instance), and converge/apply is the subclass's + :class:`Realizer` primitives over the rendered set. A subclass that + emulates capacity overrides :meth:`plan`; one that emulates render + failures overrides :meth:`refuse`. Everything else goes through the one + admission path the real backends use. """ allocates_gpus = False last_preview_digest: str | None = None + #: No containers on this host to manage. + host_runtime: HostRuntime | None = None + + def placement_notes(self) -> dict[str, list[str]]: + return {} def plan(self, desired: list[Deployment], placement: Any = None): """Which of ``desired`` fit; here, all of them, on no GPU.""" @@ -539,3 +581,28 @@ def probe_ready( self, deployment: Deployment, endpoint: str ) -> Readiness: return Readiness(True, 'dry-run') + + +def _conforms() -> None: # pragma: no cover - read by the type checker only + """Each backend the controller drives is a :class:`ServingBackend`. + + Never called: ``ty`` checks these returns, so a backend that drifts from + the protocol fails the type check rather than a run. + """ + from ..backends.kubeai import KubeaiBackend + from .compose import ComposeBackend + + def compose(backend: ComposeBackend) -> ServingBackend: + return backend + + def kubeai(backend: KubeaiBackend) -> ServingBackend: + return backend + + def memory(backend: MemoryBackend) -> ServingBackend: + return backend + + def null(backend: NullBackend) -> ServingBackend: + return backend + + def host(backend: ComposeBackend) -> HostRuntime: + return backend diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 7b4e805b..eea1e203 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1688,8 +1688,9 @@ def startup_failure(self, deployment: Deployment) -> str | None: except ResidencyUnknown: return None # cannot read Docker: say nothing return diagnose_startup( - residency.containers(deployment.id), + residency.units(deployment.id), lambda: self.deployment_logs(deployment, tail=200), + replicated=residency.replicated, ) def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: @@ -1911,6 +1912,40 @@ def apply(self) -> bool: #: {service, fingerprint}. Set by the controller from the ledger. adopted: dict[str, dict[str, str]] = {} + # -- HostRuntime: what only containers on this host have ----------------- + + @property + def host_runtime(self) -> ComposeBackend: + """This backend is its own :class:`~infer_stack.leasing.backend.HostRuntime`.""" + return self + + def network_table(self) -> dict[str, Any] | None: + return self.network + + def configure_network(self, table, *, on_addresses=None) -> None: + self.network = table + if on_addresses is not None: + self.on_addresses = on_addresses + + def subnet_clashes(self, subnet: str) -> list[str]: + from .network import overlapping_subnets + + return overlapping_subnets(subnet, self.run) + + def rendered_services(self) -> dict[str, tuple[str, str]]: + sidecar = self._load_sidecar() + fingerprints = sidecar.get('fingerprints') or {} + services = sidecar.get('services') or {} + return {name: (fp, str(services.get(name) or '')) + for name, fp in fingerprints.items()} + + def set_adopted(self, table) -> None: + self.adopted = dict(table) + + def remove_containers(self, container_ids: list[str]) -> None: + if container_ids: + self.run(['docker', 'rm', '-f', *container_ids]) + def _wait_until(self, predicate, *, deadline_s: float, what: str, interval: float = 1.0): """Poll strict residency until ``predicate(snapshot)``; abort at the deadline.""" deadline = self._clock() + deadline_s diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 43577c2c..170ae37c 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -28,9 +28,9 @@ from collections.abc import Set as AbstractSet from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Iterable, TypeVar, cast +from typing import Callable, Iterable, TypeVar -from .backend import AdmissionBackend, Backend +from .backend import HostRuntime, ServingBackend from .ledger import Ledger from .models import Deployment, DeploymentState, EndpointRequest, Lease, LeaseState @@ -79,7 +79,7 @@ class ReconcileResult: placement_errors: list[str] = field(default_factory=list) # deployment id -> GPU indices it is on / slated for (placement backends only). assignments: dict[str, list[int]] = field(default_factory=dict) - # False when reconcile only rendered the on-disk state (no docker up/down). + # False when reconcile only rendered the on-disk state (nothing brought up or taken down). applied: bool = True # True when a publication marker is still set after this operation: the # desired state was staged (--no-apply, render) or its apply did not fully @@ -168,7 +168,7 @@ class Controller: def __init__( self, ledger: Ledger, - backend: Backend, + backend: ServingBackend, *, clock: Callable[[], float] | None = None, sleep: Callable[[float], None] = time.sleep, @@ -468,14 +468,15 @@ def _render(self) -> ReconcileResult: # Residency decides which idle keep-warm deployments are candidates at # all; unknown residency fails the render (the change stays pending) # rather than guessing which warm models exist. - residency = self._admitting.residency() + residency = self.backend.residency() self._backfill_allocations(residency) - self._prepare_network() + host = self.backend.host_runtime + self._prepare_network(host) desired, placement = self._admission_view(residency) - if hasattr(self.backend, 'adopted'): - self._admitting.adopted = self._prune_adopted(residency) + if host is not None: + host.set_adopted(self._prune_adopted(residency)) before = set(self.backend.observe()) - self._admitting.converge(desired, apply=False, placement=placement) + self.backend.converge(desired, apply=False, placement=placement) after = set(self.backend.observe()) rec = ReconcileResult( realized=sorted(after - before), @@ -487,8 +488,8 @@ def _render(self) -> ReconcileResult: displaced=list(getattr(self.backend, 'last_displaced', ()) or ()), degraded=list(getattr(self.backend, 'last_degraded', ()) or ()), ) - if hasattr(self.backend, 'adopted'): - self._adopt_existing(residency) + if host is not None: + self._adopt_existing(residency, host) return rec # -- serialised publication -------------------------------------------- @@ -536,15 +537,6 @@ def _stored_state(self, lease_id: str): lease = self.ledger.get_lease(lease_id) return None if lease is None else lease.state - @property - def _admitting(self) -> AdmissionBackend: - """``self.backend`` as the admission surface. - - Every backend the controller drives has it (the real ones, and - :class:`~infer_stack.leasing.backend.SimpleAdmission` for the rest). - """ - return cast(AdmissionBackend, self.backend) - def _admission_view( self, residency, *, overlay=None, virtual_expiry: bool = False ): @@ -578,30 +570,33 @@ def _admission_view( elif deployment.state == DeploymentState.IDLE: if not self.keeps_up(deployment): continue - resident = residency.resident(gid) if residency is not None else None - if resident is not None and not resident.all_gpus: - hints[gid] = list(resident.gpus) + # Resident, with its GPUs known: every warm unit counts (a + # KubeAI Model's replicas), a conflict or an unmappable + # reservation does not. [] where the cluster places. + gpus = residency.resident_gpus(gid) if residency is not None else None + if gpus is not None: + hints[gid] = gpus optional.append(deployment) inputs = PlacementInputs( required_ids=set(required), hard=hard, optional_hints=hints, ) return [*required.values(), *optional], inputs - def _prepare_network(self) -> None: - """Give the backend the stable-address table, once a network is migrated. + def _prepare_network(self, host: HostRuntime | None) -> None: + """Give the host runtime the stable-address table, once a network is migrated. - Compose only: a backend without a ``network`` attribute has no - host network to stamp addresses on. + Only a backend with containers on this host has a network to stamp + addresses on; for any other there is nothing to do. """ - if not hasattr(self.backend, 'network'): + if host is None: return config = self.ledger.network_config() if config is None: - self._admitting.network = None + host.configure_network(None) return - self._admitting.network = { - 'subnet': config['subnet'], 'addresses': self.ledger.service_addresses()} - self._admitting.on_addresses = self.ledger.add_service_addresses + host.configure_network( + {'subnet': config['subnet'], 'addresses': self.ledger.service_addresses()}, + on_addresses=self.ledger.add_service_addresses) def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResult: """``infer-stack network migrate``: move the project to stable addresses. @@ -613,10 +608,14 @@ def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResul """ import ipaddress - from .network import overlapping_subnets from .profile import ProfileMismatch subnet = str(ipaddress.ip_network(subnet)) + host = self.backend.host_runtime + if host is None: + raise ProfileMismatch( + 'network migrate moves containers on this host to fixed addresses; ' + 'this backend runs none (the compose backend does)') with self._global_lock(): leases, _ = self.ledger.status(virtual_expiry=True) active = [le.id for le in leases if le.state == LeaseState.ACTIVE] @@ -625,7 +624,7 @@ def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResul f'network migrate recreates every container; {len(active)} lease(s) ' 'are active (release them, or pass --force)' ) - clash = overlapping_subnets(subnet, self._admitting.run) + clash = host.subnet_clashes(subnet) if clash: raise ProfileMismatch(f'subnet {subnet} overlaps: {"; ".join(clash)}') current = self.ledger.network_config() @@ -633,19 +632,19 @@ def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResul # Preview the migrated render and take approval BEFORE any write: # a declined migration must leave the subnet and addresses as they were. self._sync_profile(create=True) - residency = self._admitting.residency() - saved = self._admitting.network - self._admitting.network = { + residency = self.backend.residency() + saved = host.network_table() + host.configure_network({ 'subnet': subnet, 'addresses': {} if reset else self.ledger.service_addresses(), - } + }) try: desired, inputs = self._admission_view( residency, virtual_expiry=True ) - self._admitting.preview(desired, inputs, approve=True) + self.backend.preview(desired, inputs, approve=True) finally: - self._admitting.network = saved + host.configure_network(saved) self.ledger.store.migrate_network( subnet=subnet, reset_addresses=reset, approved_digest=getattr(self.backend, 'last_preview_digest', None), @@ -703,27 +702,24 @@ def observe_state(self) -> dict: from .residency import ResidencyUnknown leases, deployments = self.ledger.status(virtual_expiry=True) - sidecar = {} - loader = getattr(self.backend, '_load_sidecar', None) - if loader is not None: - try: - sidecar = loader() or {} - except Exception: # noqa: BLE001 - a view must always render - sidecar = {} + try: + notes = self.backend.placement_notes() or {} + except Exception: # noqa: BLE001 - a view must always render + notes = {} residency, residency_error = None, None try: - residency = self._admitting.residency() + residency = self.backend.residency() except ResidencyUnknown as ex: residency_error = str(ex) - degraded = set(sidecar.get('degraded') or ()) - displaced = set(sidecar.get('displaced') or ()) + degraded = set(notes.get('degraded') or ()) + displaced = set(notes.get('displaced') or ()) rows = [] for g in deployments: if g.state not in (DeploymentState.LIVE, DeploymentState.IDLE): continue if residency is None and residency_error is not None: condition = 'unknown' - elif residency is not None and residency.ambiguous(g.id): + elif residency is not None and residency.is_conflicted(g.id): condition = 'ambiguous' elif g.id in degraded and g.state == DeploymentState.LIVE: condition = 'degraded' @@ -731,9 +727,10 @@ def observe_state(self) -> dict: condition = 'displaced' elif (g.state == DeploymentState.LIVE and g.assigned_gpus is None and residency is not None - and residency.resident(g.id) is None): + and self._gpu_units(g) > 0 + and residency.unique_unit(g.id) is None): condition = 'unresolved' - elif residency is not None and residency.resident(g.id): + elif residency is not None and residency.is_resident(g.id): condition = 'running' elif residency is not None: condition = 'not-running' if self.keeps_up(g) else 'reclaimed' @@ -774,7 +771,7 @@ def _prune_adopted(self, residency) -> dict: self.ledger.set_adopted_containers(kept) return kept - def _adopt_existing(self, residency) -> None: + def _adopt_existing(self, residency, host: HostRuntime) -> None: """One-time migration: adopt project containers from before ownership labels. Runs after the first render on a ledger. A container @@ -788,8 +785,9 @@ def _adopt_existing(self, residency) -> None: """ if self.ledger.adopted_containers() is not None: return - fingerprints = self._admitting._load_sidecar().get('fingerprints') or {} - services = self._admitting._load_sidecar().get('services') or {} + rendered = host.rendered_services() + fingerprints = {name: fp for name, (fp, _) in rendered.items()} + services = {name: gid for name, (_, gid) in rendered.items()} _, deployments = self.ledger.status() by_id = {g.id: g for g in deployments} adopted = {} @@ -803,13 +801,13 @@ def _adopt_existing(self, residency) -> None: live_here = (deployment.state == DeploymentState.LIVE and list(c.gpus) == list(deployment.assigned_gpus or [])) resident = (deployment.state == DeploymentState.IDLE - and residency.resident(c.deployment_id) is not None) + and residency.is_resident(c.deployment_id)) if not (live_here or resident): continue adopted[c.container_id] = { 'service': c.service, 'fingerprint': fingerprints[c.service]} self.ledger.set_adopted_containers(adopted) - self._admitting.adopted = adopted + host.set_adopted(adopted) def remove_orphans(self, confirm: Callable[[list], bool]) -> list: """``gc --orphans``: remove the project's unmanaged containers, with consent. @@ -818,14 +816,17 @@ def remove_orphans(self, confirm: Callable[[list], bool]) -> list: neither labelled nor adopted, and removes exactly those if ``confirm`` (shown the list) returns True. Returns the removed containers. """ + host = self.backend.host_runtime + if host is None: + return [] # nothing of this host's to be an orphan with self._global_lock(): - residency = self._admitting.residency() + residency = self.backend.residency() adopted = self.ledger.adopted_containers() or {} orphans = [c for c in residency.all_containers() if not c.labelled and c.container_id not in adopted] if not orphans or not confirm(orphans): return [] - self._admitting.run(['docker', 'rm', '-f', *[c.container_id for c in orphans]]) + host.remove_containers([c.container_id for c in orphans]) return orphans def _gpu_units(self, deployment) -> int: @@ -851,7 +852,8 @@ def _backfill_allocations(self, residency) -> list[str]: if self._gpu_units(deployment) == 0: self.ledger.set_allocation(deployment.id, []) continue - resident = residency.resident(deployment.id) + # One reservation to adopt: a unique unit, not a replica set. + resident = residency.unique_unit(deployment.id) if resident is not None and not resident.all_gpus and resident.gpus: self.ledger.set_allocation(deployment.id, list(resident.gpus)) else: @@ -874,7 +876,7 @@ def _admit(self, overlay, residency): continue if self._gpu_units(deployment) == 0: continue - resident = residency.resident(gid) if ( + resident = residency.unique_unit(gid) if ( residency is not None and gid in overlay.revived) else None if resident is not None and not resident.all_gpus and resident.gpus: adopted[gid] = list(resident.gpus) # IDLE->LIVE keeps its GPUs @@ -898,9 +900,9 @@ def _admit(self, overlay, residency): ] for gid, gpus in adopted.items(): overlay.deployments[gid].assigned_gpus = gpus - self._prepare_network() + self._prepare_network(self.backend.host_runtime) desired, inputs = self._admission_view(residency, overlay=overlay) - plan, rendered = self._admitting.preview(desired, inputs) + plan, rendered = self.backend.preview(desired, inputs) from .backend import allocates_gpus reasons = [] @@ -941,7 +943,7 @@ def _admit(self, overlay, residency): # after the commit produces the same files and does not ask again. # Its digest goes into the pending marker, so a recovery after a # crash (and perhaps an upgrade) cannot apply something else. - self._admitting.preview(desired, inputs, approve=True) + self.backend.preview(desired, inputs, approve=True) self._admission_digest = getattr(self.backend, 'last_preview_digest', None) for gid in adopted: overlay.deployments[gid].assigned_gpus = None # committed with the lease @@ -1003,7 +1005,7 @@ def _pinned_endpoints(self, residency=None) -> set[str]: if deployment.state == DeploymentState.LIVE: pinned.update(deployment.served) elif deployment.state == DeploymentState.IDLE and residency is not None: - if residency.resident(deployment.id) is not None: + if residency.is_resident(deployment.id): pinned.update(deployment.served) return pinned @@ -1208,7 +1210,7 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: marker = self.ledger.publication_pending() if marker is None: return rec - apply_fn = self._admitting.apply + apply_fn = self.backend.apply if not marker['apply_requested']: rec.publication_pending = True # staged; never applied here return rec @@ -1423,10 +1425,10 @@ def _never_ran(self, deployment_ids: list[str]) -> list[str]: from .residency import ResidencyUnknown try: - snap = self._admitting.residency() + snap = self.backend.residency() except ResidencyUnknown: return [] - return [gid for gid in deployment_ids if not snap.containers(gid)] + return [gid for gid in deployment_ids if not snap.units(gid)] def _rollback_acquire(self, lease_id: str, *, apply: bool) -> ReconcileResult | None: """Roll a failed acquire back and publish the result, under the lock. @@ -1577,7 +1579,7 @@ def _acquire_by_admission( self.ledger.sweep() self._publish() try: - residency = self._admitting.residency() + residency = self.backend.residency() self._residency_error = None except ResidencyUnknown as ex: residency = None @@ -1795,7 +1797,7 @@ def publish_profile(self, profile: dict) -> ReconcileResult: f'lease(s) ({", ".join(active[:3])}); release them first' ) try: - snap = self._admitting.residency() + snap = self.backend.residency() except ResidencyUnknown as ex: raise ProfileMismatch( f'config publish cannot confirm the stack is quiescent: {ex}' @@ -1814,18 +1816,18 @@ def publish_profile(self, profile: dict) -> ReconcileResult: previous = self._applied_profile use(profile) try: - residency = self._admitting.residency() - self._prepare_network() + residency = self.backend.residency() + self._prepare_network(self.backend.host_runtime) desired, inputs = self._admission_view( residency, virtual_expiry=True ) - self._admitting.preview(desired, inputs, approve=True) + self.backend.preview(desired, inputs, approve=True) except BaseException: if previous is not None: use(previous) raise self.ledger.store.publish_profile( - profile, approved_digest=self._admitting.last_preview_digest) + profile, approved_digest=self.backend.last_preview_digest) self._profile_error = None self._applied_profile = profile self._invocation_profile = profile @@ -1867,7 +1869,7 @@ def renew(self, lease_id: str, *, ttl_seconds: float | None) -> RenewOutcome: from .residency import ResidencyUnknown try: - residency = self._admitting.residency() + residency = self.backend.residency() self._residency_error = None except ResidencyUnknown as ex: residency = None diff --git a/infer_stack/leasing/diagnosis.py b/infer_stack/leasing/diagnosis.py index 7a7b8ac7..fe69daa8 100644 --- a/infer_stack/leasing/diagnosis.py +++ b/infer_stack/leasing/diagnosis.py @@ -101,9 +101,15 @@ def _engine_error_summary(logs: str) -> str: return f'{summary}; likely cause: {hint}' if hint else summary +def _crashed(instance) -> bool: + """The unit has crashed at least once, or exited non-zero for good.""" + exited_for_good = (instance.state in {'exited', 'dead'} + and (instance.exit_code or 0) != 0) + return exited_for_good or instance.restart_count >= 1 -def diagnose_startup(instances: Sequence, read_logs: Callable[[], str]) -> str | None: +def diagnose_startup(instances: Sequence, read_logs: Callable[[], str], *, + replicated: bool = False) -> str | None: """Diagnose an engine that cannot start, or ``None`` if it may still load. A restart policy makes an engine that exits immediately restart forever, @@ -120,23 +126,36 @@ def diagnose_startup(instances: Sequence, read_logs: Callable[[], str]) -> str | * an unrecognised crash keeps the blunt budget of :data:`CRASH_LOOP_RESTARTS` restarts. - ``instances`` must be exactly one (absent or ambiguous: ``None``). The - returned string carries the engine's last words, because the cause is in - its log and nowhere else. + ``instances`` are the deployment's units. Without ``replicated`` (Docker) + there must be exactly one: none is "not created yet", several are a + conflict, and both say nothing. With it (a Kubernetes Model's replicas, + :attr:`Residency.replicated`) the set is diagnosed as one: fatal only + when every unit has crashed, since one unit still up or loading may + serve; the budget is the least-restarted unit's. The returned string + carries the engine's last words, because the cause is in its log and + nowhere else. + + >>> from infer_stack.leasing.residency import Container + >>> crash = 'error: unrecognized arguments: --nope' + >>> bad = Container('p1', 'grp', 'restarting', restart_count=1, exit_code=1) + >>> up = Container('p2', 'grp', 'running') + >>> diagnose_startup([bad, bad], lambda: crash, replicated=True)[:25] + 'engine is not starting (r' + >>> diagnose_startup([bad, up], lambda: crash, replicated=True) is None + True + >>> diagnose_startup([bad, bad], lambda: crash) is None # a Docker conflict + True """ - if len(instances) != 1: - return None # absent (not created yet) or ambiguous - instance = instances[0] - exited_for_good = ( - instance.state in {'exited', 'dead'} - and (instance.exit_code or 0) != 0 - ) - crashed = exited_for_good or instance.restart_count >= 1 - if not crashed: - return None # created, starting, or healthy + if not instances or (len(instances) > 1 and not replicated): + return None # not created yet, or conflicting units + if not all(_crashed(i) for i in instances): + return None # something is created, starting, or healthy + instance = min(instances, key=lambda i: i.restart_count) + exited_for_good = (instance.state in {'exited', 'dead'} + and (instance.exit_code or 0) != 0) logs = read_logs() verdict = classify_engine_log(logs) - if verdict == 'transient' and instance.will_be_restarted: + if verdict == 'transient' and any(i.will_be_restarted for i in instances): return None # a retry is coming, and may work if verdict == 'transient': return (f'engine is not starting (exited with code ' @@ -147,4 +166,6 @@ def diagnose_startup(instances: Sequence, read_logs: Callable[[], str]) -> str | return None # unrecognised: keep the restart budget why = (f'restarted {instance.restart_count} time(s)' if instance.restart_count else f'exited with code {instance.exit_code}') + if len(instances) > 1: + why += f', all {len(instances)} replicas' return f'engine is not starting ({why}){_engine_error_summary(logs)}' diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index 51ccbabf..692e13e1 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -16,9 +16,12 @@ * Docker cannot be read, or returns something unparseable → :class:`ResidencyUnknown` is raised. A snapshot is never silently empty. -* One deployment has more than one container → the deployment is *ambiguous*. - All containers are kept (never collapsed to one), :meth:`Residency.resident` - returns ``None`` for it, and callers must fail closed. +* One deployment has more than one container on Docker → the deployment is + *conflicted*: two authorities claim it. All containers are kept (never + collapsed to one), it is not resident, and callers must fail closed. On + Kubernetes several pods for one deployment are its replicas (or a rollout + in progress), so the same shape is ordinary residency there: the snapshot + says which rule applies (:attr:`Residency.replicated`). * A container's GPUs cannot be mapped to physical indices (a count-based reservation, or device UUIDs rather than indices) → it is treated as occupying *every* GPU, so no GPU is ever handed over on a guess. @@ -32,12 +35,12 @@ ... 'HostConfig': {'DeviceRequests': [{'Count': 0, 'DeviceIDs': ['1', '2']}]}}, ... ]) >>> res = residency_from_inspect(raw, project='infer-stack') - >>> res.resident('grp-a').gpus - (1, 2) + >>> res.is_resident('grp-a'), res.unique_unit('grp-a').gpus + (True, (1, 2)) >>> [c.container_id for c in res.occupants(2)] ['c1'] - >>> res.resident('grp-missing') is None - True + >>> res.is_resident('grp-missing'), res.unique_unit('grp-missing') + (False, None) """ from __future__ import annotations @@ -158,36 +161,81 @@ def will_be_restarted(self) -> bool: @dataclass(frozen=True) class Residency: - """A strict snapshot of deployment containers, keyed by deployment id. - - Every container matching a deployment is kept. Nothing is collapsed, so a - duplicate is visible rather than silently lost. + """A strict snapshot of deployment units, keyed by deployment id. + + A unit is a Docker container or a Kubernetes pod. Every unit matching a + deployment is kept; nothing is collapsed, so a duplicate is visible + rather than silently lost. + + The questions are asked of a deployment, not of a unit. Whether several + units for one deployment are normal depends on the runtime, and the + snapshot carries that rule (:attr:`replicated`), so no caller branches on + the backend: + + >>> pod = lambda name, state='running': Container(name, 'grp-a', state) + >>> pods = Residency({'grp-a': (pod('p1'), pod('p2'))}, replicated=True) + >>> pods.is_resident('grp-a'), pods.is_conflicted('grp-a'), pods.unique_unit('grp-a') + (True, False, None) + >>> docker = Residency({'grp-a': (pod('c1'), pod('c2'))}) + >>> docker.is_resident('grp-a'), docker.is_conflicted('grp-a') + (False, True) + >>> rollout = Residency({'grp-a': (pod('old', 'removing'), pod('new'))}, replicated=True) + >>> rollout.is_resident('grp-a'), [c.container_id for c in rollout.warm_units('grp-a')] + (True, ['new']) """ by_deployment: dict[str, tuple[Container, ...]] = field(default_factory=dict) - #: Project containers that belong to no deployment (infrastructure, or + #: Project units that belong to no deployment (infrastructure, or #: containers without infer-stack labels at all). others: tuple[Container, ...] = () + #: Whether one deployment may run as several units. True on Kubernetes + #: (replicas; old and new pods during a rollout); False on Docker, where a + #: second container for one deployment is a conflicting authority. + replicated: bool = False - def containers(self, deployment_id: str) -> tuple[Container, ...]: - """Every container carrying this deployment's label, in any state.""" + def units(self, deployment_id: str) -> tuple[Container, ...]: + """Every unit carrying this deployment's label, in any state.""" return self.by_deployment.get(deployment_id, ()) - def ambiguous(self, deployment_id: str) -> bool: - """More than one container claims this deployment; fail closed.""" - return len(self.containers(deployment_id)) > 1 + def warm_units(self, deployment_id: str) -> tuple[Container, ...]: + """The units that hold, or will reclaim on their own, a warm model.""" + return tuple(c for c in self.units(deployment_id) if c.warm) + + def is_conflicted(self, deployment_id: str) -> bool: + """Several units claim a deployment that may have only one; fail closed.""" + return not self.replicated and len(self.units(deployment_id)) > 1 - def resident(self, deployment_id: str) -> Container | None: - """The deployment's single warm container, or ``None``. + def is_resident(self, deployment_id: str) -> bool: + """The deployment is running: at least one warm unit, and no conflict.""" + return not self.is_conflicted(deployment_id) and bool(self.warm_units(deployment_id)) - ``None`` covers "no container", "one container that is not warm", and - "ambiguous". Use :meth:`ambiguous` to tell the last apart. + def unique_unit(self, deployment_id: str) -> Container | None: + """The deployment's one warm unit, or ``None``. + + For decisions that recover one physical fact from one unit: which + GPUs a deployment's reservation holds. ``None`` for no unit, a unit + that is not warm, a conflict, and several replicas alike: none of + those names one reservation, so GPU adoption fails closed. """ - found = self.containers(deployment_id) + found = self.units(deployment_id) if len(found) != 1 or not found[0].warm: return None return found[0] + def resident_gpus(self, deployment_id: str) -> list[int] | None: + """The physical GPUs a resident deployment holds, or ``None`` if unknown. + + ``[]`` for a resident deployment holding none (a CPU engine, or a pod: + the cluster owns placement). ``None`` when it is not resident, or a + unit's reservation could not be mapped to indices. + """ + if not self.is_resident(deployment_id): + return None + warm = self.warm_units(deployment_id) + if any(c.all_gpus for c in warm): + return None + return sorted({g for c in warm for g in c.gpus}) + def occupants(self, gpu: int) -> tuple[Container, ...]: """Every container, in any state, whose reservation includes ``gpu``.""" return tuple(c for c in self.all_containers() if c.occupies(gpu)) @@ -331,7 +379,7 @@ def residency_from_pods(raw: str) -> Residency: ... 'state': {'waiting': {'reason': 'CrashLoopBackOff'}}, ... 'lastState': {'terminated': {'exitCode': 1, 'reason': 'Error'}}, ... 'restartCount': 3}]}}]}) - >>> c = residency_from_pods(raw).containers('grp-a')[0] + >>> c = residency_from_pods(raw).units('grp-a')[0] >>> (c.state, c.reason, c.restart_count, c.exit_code, c.warm) ('restarting', 'CrashLoopBackOff', 3, 1, True) """ @@ -393,4 +441,5 @@ def residency_from_pods(raw: str) -> Residency: return Residency( by_deployment={gid: tuple(found) for gid, found in grouped.items()}, others=tuple(others), + replicated=True, ) diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 31d7b03f..fe305b55 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -733,7 +733,7 @@ def test_residency_reads_pods_and_never_guesses(tmp_path): be, kubectl = make_pod_backend(tmp_path) kubectl.pods = [_pod('model-qwen-1', 'grp-a', ready=True)] - pod = be.residency().resident('grp-a') + pod = be.residency().unique_unit('grp-a') assert (pod.state, pod.health, pod.labelled) == ('running', 'healthy', True) kubectl.fail_pods = True with pytest.raises(ResidencyUnknown): @@ -744,7 +744,7 @@ def test_an_unschedulable_pod_says_why(tmp_path): be, kubectl = make_pod_backend(tmp_path) kubectl.pods = [_pod('model-big-1', 'grp-b', statuses=False, conditions=[ {'type': 'PodScheduled', 'status': 'False', 'reason': 'Unschedulable'}])] - (pod,) = be.residency().containers('grp-b') + (pod,) = be.residency().units('grp-b') assert (pod.state, pod.reason, pod.warm) == ('created', 'Unschedulable', False) diff --git a/tests/test_leasing_residency.py b/tests/test_leasing_residency.py index 8e75f3d5..41bd4943 100644 --- a/tests/test_leasing_residency.py +++ b/tests/test_leasing_residency.py @@ -59,7 +59,7 @@ def snap(*entries): @pytest.mark.parametrize('state', sorted(WARM_STATES)) def test_warm_states_are_resident(state): res = snap(container('c1', 'grp-a', state=state)) - assert res.resident('grp-a').container_id == 'c1' + assert res.unique_unit('grp-a').container_id == 'c1' @pytest.mark.parametrize('state', ['created', 'exited', 'removing', 'dead']) @@ -67,7 +67,7 @@ def test_non_warm_states_are_not_resident_but_still_occupy_their_gpu(state): # Not warm cache -- but a created/exited container still carries a device # request that `up` could start, so nothing may be started on top of it. res = snap(container('c1', 'grp-a', state=state, device_ids=('3',))) - assert res.resident('grp-a') is None + assert res.unique_unit('grp-a') is None assert [c.container_id for c in res.occupants(3)] == ['c1'] @@ -76,14 +76,14 @@ def test_non_warm_states_are_not_resident_but_still_occupy_their_gpu(state): def test_gpus_come_from_device_request_indices(): res = snap(container('c1', 'grp-a', device_ids=('2', '1'))) - c = res.resident('grp-a') + c = res.unique_unit('grp-a') assert c.gpus == (1, 2) and c.all_gpus is False assert {g for g in range(4) if res.occupants(g)} == {1, 2} def test_container_without_a_gpu_reservation_occupies_nothing(): res = snap(container('c1', 'grp-a', device_ids=())) # DeviceRequests: None - assert res.resident('grp-a').gpus == () + assert res.unique_unit('grp-a').gpus == () assert all(res.occupants(g) == () for g in range(4)) @@ -95,7 +95,7 @@ def test_container_without_a_gpu_reservation_occupies_nothing(): def test_unmappable_reservation_is_treated_as_every_gpu(requests): # Never guess which GPU a container holds: fail closed for every GPU. res = snap(container('c1', 'grp-a', requests=requests)) - assert res.resident('grp-a').all_gpus is True + assert res.unique_unit('grp-a').all_gpus is True assert all( [c.container_id for c in res.occupants(g)] == ['c1'] for g in range(8) ) @@ -126,18 +126,18 @@ def test_duplicate_deployment_containers_are_kept_and_ambiguous(): container('c2', 'grp-a', device_ids=('1',)), container('c1', 'grp-a', state='exited', device_ids=('0',)), ) - assert res.ambiguous('grp-a') is True - assert res.resident('grp-a') is None - assert [c.container_id for c in res.containers('grp-a')] == ['c1', 'c2'] + assert res.is_conflicted('grp-a') is True + assert res.unique_unit('grp-a') is None + assert [c.container_id for c in res.units('grp-a')] == ['c1', 'c2'] assert [c.container_id for c in res.occupants(0)] == ['c1'] assert [c.container_id for c in res.occupants(1)] == ['c2'] def test_absent_deployment_is_not_resident_and_not_ambiguous(): res = snap(container('c1', 'grp-a')) - assert res.resident('grp-b') is None - assert res.ambiguous('grp-b') is False - assert res.containers('grp-b') == () + assert res.unique_unit('grp-b') is None + assert res.is_conflicted('grp-b') is False + assert res.units('grp-b') == () @pytest.mark.parametrize('raw', ['not json', '{"Id": "c1"}', '[1, 2]', @@ -189,7 +189,7 @@ def test_residency_lists_the_whole_project_in_every_state(tmp_path): assert ps[:4] == ['docker', 'ps', '-a', '--no-trunc'] assert f'label={COMPOSE_PROJECT_LABEL}={PROJECT}' in ps assert docker.calls[1] == ['docker', 'inspect', 'c1'] - assert res.resident('grp-a').gpus == (1,) + assert res.unique_unit('grp-a').gpus == (1,) def test_residency_with_no_containers_is_empty_and_skips_inspect(tmp_path): @@ -259,10 +259,10 @@ def run(args): be = ComposeBackend(state_dir=tmp_path, inventory=simulate_inventory('4x80'), run=run, project=project) res = be.residency() - a, b = res.containers('grp-aaa'), res.containers('grp-bbb') + a, b = res.units('grp-aaa'), res.units('grp-bbb') assert len(a) == 1 and a[0].gpus == (1, 2) and a[0].state == 'created' assert len(b) == 1 and b[0].gpus == () - assert res.resident('grp-aaa') is None # created is not warm + assert res.unique_unit('grp-aaa') is None # created is not warm assert [c.deployment_id for c in res.occupants(2)] == ['grp-aaa'] finally: subprocess.run([*base, 'down'], capture_output=True, timeout=300) diff --git a/tests/test_leasing_serialised_publication.py b/tests/test_leasing_serialised_publication.py index 945b8a4c..f056b5ec 100644 --- a/tests/test_leasing_serialised_publication.py +++ b/tests/test_leasing_serialised_publication.py @@ -626,7 +626,7 @@ def test_rollback_keeps_a_deployment_that_has_a_container(tmp_path): def any_id(backend): class Everything(Residency): - def containers(self, deployment_id): + def units(self, deployment_id): return (Container('c1', deployment_id, 'exited'),) return Everything({}) diff --git a/tests/test_parity.py b/tests/test_parity.py index 34087a31..47be1dad 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -113,13 +113,19 @@ def _compose(tmp_path, **gateway_kw) -> Stack: class _Kubectl: - """kubectl over applied Model CRs; each Model runs one pod.""" + """kubectl over applied Model CRs; each Model runs ``minReplicas`` pods. + + ``extra_pods`` adds pods a test injects, such as the old pod of a + rollout still terminating beside its replacement. + """ def __init__(self): from test_leasing_kubeai import FakeKubectl self.inner = FakeKubectl() self.crash: dict[str, str] = {} + self.crash_one: set[str] = set() + self.extra_pods: list[dict] = [] @property def applied(self): @@ -132,13 +138,17 @@ def __call__(self, args: list[str]) -> str: pods = [] for name, doc in self.inner.applied.items(): gid = doc['metadata']['labels']['infer-stack/deployment'] - if gid in self.crash: - pods.append(_pod(f'model-{name}-0', gid, restarts=3, - state={'waiting': {'reason': 'CrashLoopBackOff'}}, - last={'exitCode': 1, 'reason': 'Error'})) - else: - pods.append(_pod(f'model-{name}-0', gid, ready=True)) - return json.dumps({'items': pods}) + replicas = int((doc.get('spec') or {}).get('minReplicas') or 1) + # `crash` marks every replica crash-looping; `crash_one`, one. + bad = replicas if gid in self.crash else int(gid in self.crash_one) + for i in range(replicas): + if i < bad: + pods.append(_pod(f'model-{name}-{i}', gid, restarts=3, + state={'waiting': {'reason': 'CrashLoopBackOff'}}, + last={'exitCode': 1, 'reason': 'Error'})) + else: + pods.append(_pod(f'model-{name}-{i}', gid, ready=True)) + return json.dumps({'items': [*pods, *self.extra_pods]}) return self.inner(args) @@ -447,3 +457,93 @@ async def scenario(): assert target == [engine['name']] asyncio.run(scenario()) + + +# -- cross-feature invariants ----------------------------------------------------- +# +# Rows of the matrix are not independent: a backend-specific feature must +# leave the common lifecycle around it unchanged. Each case below combines +# one such feature with acquire / release / gc / status / pinning. A few +# poison cases, not a matrix (roadmap, "Testing"). + +REPLICATED = Catalog.from_dict({**CATALOG, 'endpoints': { + **CATALOG['endpoints'], + 'pair': {'engine': 'vllm', 'model': 'tiny', 'reclaim': {'policy': 'keep-warm'}, + 'runtime': {'max_model_len': 1024, 'min_replicas': 2, 'max_replicas': 2}}, +}}) + + +def _replicated(tmp_path): + """A KubeAI stack serving `pair`, a keep-warm Model with two replicas.""" + stack = _kubeai(tmp_path) + stack.backend.catalog = REPLICATED + out = stack.ctl.acquire('alice', REPLICATED.resolve_names(['pair']), wait=False) + gid = out.deployments[0].id + assert len(stack.backend.residency().warm_units(gid)) == 2 + return stack, out, gid + + +def _condition(stack, gid): + rows = stack.ctl.observe_state()['deployments'] + return next(r['condition'] for r in rows if r['id'] == gid) + + +def _desired(stack, gid): + return any(doc['metadata']['labels']['infer-stack/deployment'] == gid + for doc in stack.runtime.applied.values()) + + +def test_replicas_leave_the_keep_warm_lifecycle_unchanged(tmp_path): + from infer_stack.cli.commands_runtime import _served_models + + stack, out, gid = _replicated(tmp_path) + assert _condition(stack, gid) == 'running' # replicas, not a conflict + _, deployments = stack.ctl.ledger.status() + assert [row[3] for row in _served_models(deployments, stack.backend)] == ['up'] + stack.ctl.release(out.lease.id) + assert _desired(stack, gid) and gid in stack.running() # keep-warm stays + assert _condition(stack, gid) == 'running' + # The idle Model still pins the definition it is running. + assert 'pair' in stack.ctl._pinned_endpoints(stack.backend.residency()) + stack.ctl.gc() + assert _desired(stack, gid) and gid in stack.running() # plain gc leaves it + stack.ctl.evict(None) + assert not _desired(stack, gid) + + +def test_a_rollout_snapshot_prunes_nothing(tmp_path): + from test_leasing_kubeai import _pod + + stack, out, gid = _replicated(tmp_path) + # The old pod of a rollout, terminating beside the new ones. + old = _pod('model-pair-old', gid, ready=True) + old['metadata']['deletionTimestamp'] = '2026-09-27T00:00:00Z' + stack.runtime.extra_pods.append(old) + stack.ctl.release(out.lease.id) # re-renders from this snapshot + assert _desired(stack, gid) and _condition(stack, gid) == 'running' + stack.ctl.gc() + assert _desired(stack, gid) + + +def test_two_compose_containers_for_one_deployment_still_fail_closed(tmp_path): + stack = _compose(tmp_path) + gid = stack.acquire('warm').deployments[0].id + first = next(c for c in stack.runtime.containers.values() + if c['labels'].get(DEPLOYMENT_LABEL) == gid) + stack.runtime.add_container(first['service'], labels=first['labels'], + device_ids=first['device_ids']) + residency = stack.backend.residency() + assert residency.is_conflicted(gid) and not residency.is_resident(gid) + assert residency.unique_unit(gid) is None and residency.resident_gpus(gid) is None + assert _condition(stack, gid) == 'ambiguous' + + +def test_every_replica_crash_looping_fails_fast_and_one_healthy_does_not(tmp_path): + stack, out, gid = _replicated(tmp_path) + deployment = stack.ctl.ledger.get_deployment(gid) + stack.backend.deployment_logs = lambda deployment, tail=400: CRASH_LOG + stack.runtime.crash_one.add(gid) # one replica still serves + assert stack.backend.startup_failure(deployment) is None + stack.runtime.crash[gid] = CRASH_LOG # now every replica loops + why = stack.backend.startup_failure(deployment) + assert why and 'all 2 replicas' in why and 'trust_remote_code' in why From ee7bd7f7723031a989f0c72681993a4ac4e9d0f8 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:09:37 -0400 Subject: [PATCH 073/141] Reclaim only for capacity, only a victim that could help Unschedulable no longer means "evict the longest-idle model". KubeAI's probe asks for room only when the scheduler says a node is short of a resource; reclaim_candidates names idle Models on nodes the blocked pod could use that hold that resource. The controller keeps the policy: idle only, longest idle first, one per cooldown. An impossible selector or an untolerated taint evicts nothing. Also: the TUI catalog test counts only catalog refusals (a background worker could refuse something else meanwhile). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/backend-parity.md | 2 +- docs/queue.md | 37 ++++++-- infer_stack/backends/kubeai.py | 144 +++++++++++++++++++++++++++++- infer_stack/leasing/backend.py | 22 ++++- infer_stack/leasing/compose.py | 6 ++ infer_stack/leasing/controller.py | 43 +++++---- infer_stack/leasing/residency.py | 14 ++- tests/test_leasing_make_room.py | 18 ++-- tests/test_parity.py | 120 +++++++++++++++++++++++++ tests/test_tui.py | 5 +- 10 files changed, 375 insertions(+), 36 deletions(-) diff --git a/docs/backend-parity.md b/docs/backend-parity.md index ee60b760..b2fbeed5 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -75,7 +75,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | `--queue` | waits for a free GPU | ≈ admitted at once, the cluster is the queue; an unrenderable endpoint fails at once | | `config publish` | pure preview, then commit | same | | `--no-apply` / `apply` / `render` | same | same | -| unleased keep-warm yields to leased demand | at placement | ≈ during the wait: `needs_room` evicts the longest-idle, one per 30 s | +| unleased keep-warm yields to leased demand | at placement | ≈ during the wait, one per 30 s: only when the scheduler reports a capacity shortage, and only an idle Model on a node the pod could use, holding the resource that is short (longest idle first) | | crash-loop fail-fast, the engine's error quoted | same | same (pods, `kubectl logs --previous`) | | image-pull progress | yes | n/a: the kubelet pulls; `ImagePullBackOff` is a reported wait reason | | strict `residency()` for decisions, lenient `observe()` for reports | same | same | diff --git a/docs/queue.md b/docs/queue.md index bd9a4d44..b4715847 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -446,7 +446,7 @@ poison test that fails on the old code with every fix; no new `hasattr` probes; no second KubeAI lifecycle path; backwards compatibility for cards, catalogs and CLI invocations that work today. -### 11. [ ] Residency: replicas are not duplicates +### 11. [x] Residency: replicas are not duplicates *Why added:* `Residency.ambiguous()` meant "more than one unit" and `resident()` returned a unit only when there was exactly one. On Compose two @@ -476,7 +476,14 @@ replica crash-looping fails fast with the engine's error, one healthy replica does not; two Compose containers for one deployment still fail closed; the k3s e2e has a two-replica keep-warm phase and passes. -### 12. [ ] One backend protocol, the one the controller uses +*Done 2026-09-27 (35be92c):* `Residency.replicated` (true for pods) with +`is_resident`, `is_conflicted`, `warm_units`, `resident_gpus`, and +`unique_unit` kept for GPU adoption only; every caller audited; +`diagnose_startup(..., replicated=)` fails fast only when every replica has +crashed. Unit and parity tests; the k3s e2e's two-replica keep-warm phase +passed (released and gc-ed: still desired, both replicas up, shown running). + +### 12. [x] One backend protocol, the one the controller uses *Why added:* `Controller` took a `Backend` (the old `realize/teardown/ observe` protocol) and cast it to `AdmissionBackend`, which inherited that @@ -496,6 +503,15 @@ satisfy a protocol; the controller builds no `docker` command and reads no sidecar; `ty` checks each backend against the protocol, and fails when one drifts. +*Done 2026-09-27 (35be92c):* `ServingBackend` is the controller's protocol +(no cast); `Realizer` holds `realize/teardown` for `SimpleAdmission` only; +KubeAI's no-op methods are gone; `HostRuntime` (Compose: +`network_table/configure_network`, `subnet_clashes`, `rendered_services`, +`set_adopted`, `remove_containers`) behind `backend.host_runtime`; +`placement_notes()` replaces the sidecar read. `backend._conforms()` makes +`ty` check each backend against the protocol (verified: removing KubeAI's +`host_runtime` fails the check). `Backend` stays exported as an alias. + ### 13. [ ] Parity tests: cross-feature invariants *Why added:* the parity suite's KubeAI fake ran one pod per Model, so @@ -511,7 +527,7 @@ roadmap. **Done when:** the suite is in `tests/test_parity.py` (or beside it), each case fails on the code before its fix, and the roadmap states the rule. -### 14. [ ] Scheduler-aware reclaim, not "Unschedulable means evict" +### 14. [x] Scheduler-aware reclaim, not "Unschedulable means evict" *Why added:* KubeAI's readiness sets `needs_room` whenever a pod waits with reason `Unschedulable`, and `wait_ready` answers by evicting the @@ -534,6 +550,15 @@ untolerated taint evicts nothing; with two profiles on two nodes the idle Model on the wrong node is not chosen; real reclaimable pressure evicts a compatible idle Model and the leased one proceeds. +*Done 2026-09-27:* KubeAI's probe sets `needs_room` only when the +scheduler's message names an `Insufficient` resource (the message is now in +residency); `ServingBackend.reclaim_candidates(blocked, idle)` names idle +Models with a pod on a node the blocked pod could use (node selector, +taints; required affinity is not evaluated, so nothing is evicted) that +request the short resource. Compose returns none, the in-process backends +all. `_make_room` keeps the policy. Four parity poison tests; three fail on +the old code. + ### 15. [ ] Publication phases: the approved digest outlives a partial apply *Why added:* `_apply_pending()` clears the approved-render digest right @@ -658,5 +683,7 @@ full suite, `ty`, flake8, and the full k3s e2e. **Done when:** those pass and the docs match the code. Also observed while working on these: `test_an_edit_made_outside_the_tui_appears_on_the_next_refresh` -failed once in a full-suite run and passed alone three times; timing -around the catalog reload. Looked at under item 22 if it recurs. +failed in two of five full-suite runs and never alone. It counted every +`_refuse` call after two refreshes, and a refresh's background worker can +refuse something unrelated meanwhile; it now counts catalog-reload +refusals only (the likely cause, not reproduced on demand). diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 2fd74f25..ae01c386 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -396,6 +396,88 @@ def kubectl_complaint(stderr: str) -> str: + +def insufficient_resources(message: str) -> set[str]: + """The resources a scheduler message says some node is short of. + + ``Unschedulable`` covers a node selector nothing matches, a taint the pod + does not tolerate, affinity, and plain capacity alike. Only capacity can + be relieved by stopping another Model, and the message says which it is, + node by node. + + >>> sorted(insufficient_resources( + ... '0/3 nodes are available: 1 Insufficient nvidia.com/gpu, 1 Insufficient ' + ... "memory, 1 node(s) didn't match Pod's node affinity/selector.")) + ['memory', 'nvidia.com/gpu'] + >>> insufficient_resources("0/2 nodes are available: 2 node(s) had untolerated " + ... "taint {gpu: reserved}.") + set() + """ + import re + + return set(re.findall(r'Insufficient ([A-Za-z0-9./_-]+?)[,.]?(?:\s|$)', message or '')) + + +def _tolerates(tolerations: list[dict], taint: dict) -> bool: + """Whether a pod's tolerations admit one node taint (Kubernetes' rule).""" + if taint.get('effect') not in ('NoSchedule', 'NoExecute'): + return True # PreferNoSchedule never blocks + for t in tolerations: + if t.get('effect') and t.get('effect') != taint.get('effect'): + continue + if t.get('operator') == 'Exists': + if not t.get('key') or t.get('key') == taint.get('key'): + return True + elif t.get('key') == taint.get('key') and t.get('value') == taint.get('value'): + return True + return False + + +def eligible_nodes(pod_spec: dict, nodes: list[dict]) -> set[str] | None: + """Nodes a pod could run on if they had room, or ``None`` if not known. + + A node qualifies when it is schedulable, carries every label of the + pod's ``nodeSelector``, and every blocking taint is tolerated. Required + node affinity is not evaluated here: ``None`` then, and the caller frees + nothing on a guess. + + >>> spec = {'nodeSelector': {'gpu': 'a100'}, 'tolerations': []} + >>> nodes = [{'metadata': {'name': 'a', 'labels': {'gpu': 'a100'}}, 'spec': {}}, + ... {'metadata': {'name': 'b', 'labels': {'gpu': 'l4'}}, 'spec': {}}, + ... {'metadata': {'name': 'c', 'labels': {'gpu': 'a100'}}, + ... 'spec': {'taints': [{'key': 'x', 'effect': 'NoSchedule'}]}}] + >>> eligible_nodes(spec, nodes) + {'a'} + """ + required = (((pod_spec.get('affinity') or {}).get('nodeAffinity') or {}) + .get('requiredDuringSchedulingIgnoredDuringExecution')) + if required: + return None + selector = pod_spec.get('nodeSelector') or {} + tolerations = pod_spec.get('tolerations') or [] + out = set() + for node in nodes: + meta, spec = node.get('metadata') or {}, node.get('spec') or {} + labels = meta.get('labels') or {} + if spec.get('unschedulable'): + continue + if any(labels.get(k) != v for k, v in selector.items()): + continue + if all(_tolerates(tolerations, t) for t in spec.get('taints') or []): + out.add(str(meta.get('name') or '')) + return out + + +def _requests(pod: dict) -> set[str]: + """The resource names a pod's containers request or limit.""" + names: set[str] = set() + for c in (pod.get('spec') or {}).get('containers') or []: + res = c.get('resources') or {} + names.update((res.get('requests') or {}).keys()) + names.update((res.get('limits') or {}).keys()) + return names + + class KubeaiBackend(ConvergeScaffold): """Cluster KubeAI backend (converge-style). @@ -978,6 +1060,63 @@ def startup_failure(self, deployment: Deployment) -> str | None: lambda: self.deployment_logs(deployment, tail=200), replicated=residency.replicated) + def _capacity_shortage(self, deployment: Deployment) -> set[str]: + """Resources the scheduler says this Model's pending pods lack.""" + from ..leasing.residency import ResidencyUnknown + + try: + pods = self.residency().units(deployment.id) + except ResidencyUnknown: + return set() + return {r for p in pods if p.reason == 'Unschedulable' + for r in insufficient_resources(p.message)} + + def reclaim_candidates(self, blocked: Deployment, + idle: list[Deployment]) -> list[str]: + """Which idle Models could free room for ``blocked``, in ``idle``'s order. + + One whose pod runs on a node ``blocked`` could be scheduled on, and + requests a resource the scheduler says is short. Nothing when the + blocked pod is not short of capacity, when its node constraints + cannot be evaluated, or when the cluster cannot be read: an eviction + that cannot help only loses a warm model. + """ + try: + pods = json.loads(self._kubectl( + ['get', 'pods', '-l', f'{MANAGED_LABEL}=true', '-o', 'json']) or '{}') + nodes = json.loads(self._kubectl(['get', 'nodes', '-o', 'json']) or '{}') + except Exception: # noqa: BLE001 - unknown: evict nothing + return [] + items = pods.get('items') or [] + + def gid(pod): + return ((pod.get('metadata') or {}).get('labels') or {}).get(DEPLOYMENT_LABEL) + + short: set[str] = set() + where: set[str] | None = set() + for pod in items: + if gid(pod) != blocked.id: + continue + for cond in (pod.get('status') or {}).get('conditions') or []: + if cond.get('reason') == 'Unschedulable' and cond.get('status') == 'False': + found = insufficient_resources(str(cond.get('message') or '')) + if found: + short |= found + nodes_here = eligible_nodes(pod.get('spec') or {}, + nodes.get('items') or []) + if nodes_here is None: + where = None + elif where is not None: + where |= nodes_here + if not short or not where: + return [] + wanted = {g.id for g in idle} + useful = {gid(pod) for pod in items + if gid(pod) in wanted + and (pod.get('spec') or {}).get('nodeName') in where + and _requests(pod) & short} + return [g.id for g in idle if g.id in useful] + def _waiting_reason(self, deployment: Deployment) -> str: """The pod's own reason for not running yet (``ImagePullBackOff``...).""" from ..leasing.residency import ResidencyUnknown @@ -1034,8 +1173,11 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: if failure is not None: return Readiness(False, failure, fatal=True) waiting = self._waiting_reason(deployment) + # Room helps only when the scheduler says some node is short of a + # resource; a selector, taint or affinity no node satisfies stays + # unschedulable whatever is stopped (reclaim_candidates picks whom). return Readiness(False, f'{reason} (pod: {waiting})' if waiting else reason, - needs_room='Unschedulable' in waiting) + needs_room=bool(self._capacity_shortage(deployment))) def access(self, endpoints: list[str]) -> dict[str, Any] | None: """Where a client reaches these endpoints (env-file descriptor). diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 3ca6a3fc..1a502f60 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -37,9 +37,10 @@ class Readiness: ready: bool detail: str = '' fatal: bool = False - #: The endpoint is waiting for resources held by others (a Kubernetes pod - #: the scheduler cannot place). The controller then frees room by - #: evicting idle keep-warm deployments, which no lease holds. + #: The endpoint is waiting for capacity held by others: the scheduler says + #: a node is short of a resource, not that no node could ever host it. + #: The controller then evicts an idle keep-warm deployment the backend + #: names in ``reclaim_candidates`` (none: nothing is evicted). needs_room: bool = False @@ -218,6 +219,16 @@ def placement_notes(self) -> dict[str, list[str]]: residents that yielded their GPUs). Empty where nothing is placed.""" ... + def reclaim_candidates(self, blocked: Deployment, + idle: list[Deployment]) -> list[str]: + """Which of ``idle`` could free room ``blocked`` is waiting for. + + Asked only when a probe said ``needs_room``. The backend knows the + scheduling domain (which nodes, which resources); the controller + keeps the policy (only idle deployments, longest idle first, one at + a time). Return in ``idle``'s order; empty means evicting would not help. + """ + ... #: The name ``infer_stack.leasing`` has always exported for the controller's @@ -410,6 +421,11 @@ class SimpleAdmission: def placement_notes(self) -> dict[str, list[str]]: return {} + def reclaim_candidates(self, blocked: Deployment, + idle: list[Deployment]) -> list[str]: + """One scheduling domain in-process: any idle deployment frees room.""" + return [g.id for g in idle] + def plan(self, desired: list[Deployment], placement: Any = None): """Which of ``desired`` fit; here, all of them, on no GPU.""" from .placement import GpuPlan diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index eea1e203..554cacea 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1946,6 +1946,12 @@ def remove_containers(self, container_ids: list[str]) -> None: if container_ids: self.run(['docker', 'rm', '-f', *container_ids]) + def reclaim_candidates(self, blocked: Deployment, + idle: list[Deployment]) -> list[str]: + """None: admission already moves idle models aside when it places, so + a Compose probe never reports ``needs_room``.""" + return [] + def _wait_until(self, predicate, *, deadline_s: float, what: str, interval: float = 1.0): """Poll strict residency until ``predicate(snapshot)``; abort at the deadline.""" deadline = self._clock() + deadline_s diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 170ae37c..cc0c7562 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1362,13 +1362,14 @@ def wait_ready( while True: pending = [] failures = [] - needs_room = False + blocked: dict[str, Deployment] = {} for (g, ep) in pairs: probe = self.backend.probe_ready(g, ep) if probe.ready: continue pending.append((g, ep)) - needs_room = needs_room or probe.needs_room + if probe.needs_room: + blocked[g.id] = g if probe.fatal: failures.append((g.id, ep, probe.detail)) if not pending: @@ -1385,22 +1386,24 @@ def wait_ready( pending=[(g.id, ep) for g, ep in pending], ) # After the deadline check: a wait that has given up evicts nothing. - if needs_room and self.clock() - last_room >= ROOM_COOLDOWN_S: - # A leased model is waiting on resources: a model without a - # lease is always a candidate to give them up. - if self._make_room(): + if blocked and self.clock() - last_room >= ROOM_COOLDOWN_S: + # A leased model is waiting on capacity: an idle model the + # backend says could free it gives it up. + if self._make_room(list(blocked.values())): last_room = self.clock() self.sleep(interval) pairs = pending - def _make_room(self) -> str | None: - """Evict the longest-idle deployment for leased demand; its id, or ``None``. + def _make_room(self, blocked: list[Deployment]) -> str | None: + """Evict one idle deployment that could free room for ``blocked``. - Idle means no lease holds it (a keep-warm model left resident). One at - a time: the runtime cannot say how much room is needed, and every - warm model kept is a load avoided. Compose backends never ask, since - their admission already moves idle models aside; this serves backends - where the runtime schedules (KubeAI). + The policy is here: only idle deployments (no lease holds them, so a + leased one is never a victim), the longest idle first, one at a time, + since the runtime cannot say how much room is needed and every warm + model kept is a load avoided. Whether a victim could help at all is + the backend's to say (:meth:`ServingBackend.reclaim_candidates`: the + same node pool and the resource that is short); when none could, + nothing is evicted. Returns the victim's id, or ``None``. """ from .._log import logger @@ -1409,11 +1412,17 @@ def _make_room(self) -> str | None: key=lambda g: (g.updated_at, g.id)) if not idle: return None - victim = idle[0].id + useful: set[str] = set() + for g in blocked: + useful.update(self.backend.reclaim_candidates(g, idle)) + victims = [g for g in idle if g.id in useful] + if not victims: + return None + victim = victims[0] logger.info('making room for leased demand: evicting idle keep-warm {} ({})', - victim, ', '.join(sorted(idle[0].served))) - self.evict([victim]) - return victim + victim.id, ', '.join(sorted(victim.served))) + self.evict([victim.id]) + return victim.id def _never_ran(self, deployment_ids: list[str]) -> list[str]: """Which of these deployments definitely have no container at all. diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index 692e13e1..71889dda 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -130,6 +130,9 @@ class Container: #: ``CrashLoopBackOff``, ``ImagePullBackOff``, ``OOMKilled`` (Kubernetes). #: Empty when there is nothing to say, or the runtime does not say. reason: str = '' + #: The runtime's own words for ``reason``, when it gives any: a + #: scheduler's ``0/2 nodes are available: 2 Insufficient nvidia.com/gpu``. + message: str = '' #: When the current run started (the runtime's timestamp), for display. started: str = '' #: Published ports, ``host->container/proto`` joined by ``, ``; display only. @@ -412,11 +415,15 @@ def residency_from_pods(raw: str) -> Residency: else: state = 'created' ended = current.get('terminated') or last + message = '' if not waiting and not statuses: # Not started at all: the pod's own condition says why, e.g. the - # scheduler found no node with the resources ("Unschedulable"). - waiting = next((str(c.get('reason') or '') for c in status.get('conditions') or [] - if c.get('status') == 'False' and c.get('reason')), '') + # scheduler found no node with the resources ("Unschedulable"), + # and its message says which constraint each node failed. + failed = next((c for c in status.get('conditions') or [] + if c.get('status') == 'False' and c.get('reason')), {}) + waiting = str(failed.get('reason') or '') + message = str(failed.get('message') or '') ready = any(c.get('type') == 'Ready' and c.get('status') == 'True' for c in status.get('conditions') or []) container = Container( @@ -431,6 +438,7 @@ def residency_from_pods(raw: str) -> Residency: # A Deployment's pods are always restarted by the kubelet. restart_policy='always', reason=waiting or str(ended.get('reason') or ''), + message=message, started=str((current.get('running') or {}).get('startedAt') or status.get('startTime') or ''), ) diff --git a/tests/test_leasing_make_room.py b/tests/test_leasing_make_room.py index ec6aeb05..5f9e400b 100644 --- a/tests/test_leasing_make_room.py +++ b/tests/test_leasing_make_room.py @@ -3,8 +3,10 @@ The rule (decided 2026-09-24): an idle keep-warm deployment -- resident, but held by no lease -- is always a candidate for eviction when a leased model needs a resource. Compose admission enforces it when placing. Where the -runtime schedules instead (KubeAI), the probe says `needs_room` and the wait -evicts idle models, the longest idle first, one per cooldown. +runtime schedules instead (KubeAI), the probe says `needs_room` only when +the scheduler reports a capacity shortage, and the wait evicts, one per +cooldown and the longest idle first, an idle model the backend says could +free that capacity (tests/test_parity.py has the scheduler cases). """ from __future__ import annotations @@ -104,9 +106,15 @@ def test_an_unschedulable_kubeai_pod_asks_for_room(tmp_path): be, kubectl = make_pod_backend(tmp_path) dep = vllm('grp-big', served='big') be.converge([dep]) - kubectl.pods = [_pod('model-big-1', 'grp-big', statuses=False, conditions=[ - {'type': 'PodScheduled', 'status': 'False', 'reason': 'Unschedulable'}])] - be.http.post = lambda url, **kw: FakeHttp._Resp(503, {'detail': 'not ready'}) + def pending(message): + kubectl.pods = [_pod('model-big-1', 'grp-big', statuses=False, conditions=[ + {'type': 'PodScheduled', 'status': 'False', 'reason': 'Unschedulable', + 'message': message}])] + be.http.post = lambda url, **kw: FakeHttp._Resp(503, {'detail': 'not ready'}) + pending('0/1 nodes are available: 1 Insufficient nvidia.com/gpu.') probe = be.probe_ready(dep, 'grp-big') assert probe.needs_room and not probe.fatal and not probe.ready + # Unschedulable for a reason room cannot fix: no room is asked for. + pending("0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector.") + assert not be.probe_ready(dep, 'grp-big').needs_room diff --git a/tests/test_parity.py b/tests/test_parity.py index 47be1dad..a971373b 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -126,6 +126,22 @@ def __init__(self): self.crash: dict[str, str] = {} self.crash_one: set[str] = set() self.extra_pods: list[dict] = [] + # Scheduler mode: nodes, where each Model's pods run, what they + # request, and Models whose pods the scheduler cannot place + # ({gid: {'message', 'spec', 'until_gone'}}: pending until the Model + # of deployment `until_gone` is deleted, when it names one). + self.nodes: list[dict] = [] + self.node_of: dict[str, str] = {} + self.pending: dict[str, dict] = {} + + def is_pending(self, gid: str) -> bool: + entry = self.pending.get(gid) + if entry is None: + return False + gone = entry.get('until_gone') + return not gone or any( + d['metadata']['labels']['infer-stack/deployment'] == gone + for d in self.inner.applied.values()) @property def applied(self): @@ -134,6 +150,8 @@ def applied(self): def __call__(self, args: list[str]) -> str: from test_leasing_kubeai import _pod + if len(args) > 4 and args[3] == 'get' and args[4] == 'nodes': + return json.dumps({'items': self.nodes}) if len(args) > 4 and args[3] == 'get' and args[4] == 'pods': pods = [] for name, doc in self.inner.applied.items(): @@ -141,6 +159,14 @@ def __call__(self, args: list[str]) -> str: replicas = int((doc.get('spec') or {}).get('minReplicas') or 1) # `crash` marks every replica crash-looping; `crash_one`, one. bad = replicas if gid in self.crash else int(gid in self.crash_one) + if self.is_pending(gid): + entry = self.pending[gid] + pod = _pod(f'model-{name}-0', gid, statuses=False, conditions=[ + {'type': 'PodScheduled', 'status': 'False', + 'reason': 'Unschedulable', 'message': entry['message']}]) + pod['spec'] = dict(entry.get('spec') or {}) + pods.append(pod) + continue for i in range(replicas): if i < bad: pods.append(_pod(f'model-{name}-{i}', gid, restarts=3, @@ -148,6 +174,12 @@ def __call__(self, args: list[str]) -> str: last={'exitCode': 1, 'reason': 'Error'})) else: pods.append(_pod(f'model-{name}-{i}', gid, ready=True)) + for pod in pods: + gid = pod['metadata']['labels']['infer-stack/deployment'] + if not self.is_pending(gid): + pod['spec'] = {'nodeName': self.node_of.get(gid, 'node-a'), + 'containers': [{'resources': { + 'requests': {'nvidia.com/gpu': '1'}}}]} return json.dumps({'items': [*pods, *self.extra_pods]}) return self.inner(args) @@ -547,3 +579,91 @@ def test_every_replica_crash_looping_fails_fast_and_one_healthy_does_not(tmp_pat stack.runtime.crash[gid] = CRASH_LOG # now every replica loops why = stack.backend.startup_failure(deployment) assert why and 'all 2 replicas' in why and 'trust_remote_code' in why + + +# Scheduler pressure: `Unschedulable` is not "evict an idle model". Only a +# capacity shortage, and only an idle Model on a node the blocked pod could +# use, holding the resource that is short. + +SCHED = Catalog.from_dict({**CATALOG, 'endpoints': { + **CATALOG['endpoints'], + 'warm-a': {'engine': 'vllm', 'model': 'tiny', 'reclaim': {'policy': 'keep-warm'}, + 'runtime': {'max_model_len': 1024}}, + 'warm-b': {'engine': 'vllm', 'model': 'big', 'reclaim': {'policy': 'keep-warm'}, + 'runtime': {'max_model_len': 1024}}, + 'blocked': {'engine': 'vllm', 'model': 'tiny', 'reclaim': {'policy': 'stop'}, + 'runtime': {'max_model_len': 4096}}, +}}) +NODES = [{'metadata': {'name': 'node-a', 'labels': {'gpu': 'a100'}}, 'spec': {}}, + {'metadata': {'name': 'node-b', 'labels': {'gpu': 'l4'}}, 'spec': {}}] + + +def _two_idle_models(tmp_path): + """warm-a idle on node-a (the longest idle), warm-b idle on node-b.""" + import time + + stack = _kubeai(tmp_path) + stack.backend.catalog = SCHED + stack.runtime.nodes = NODES + ids = {} + for name, node in (('warm-a', 'node-a'), ('warm-b', 'node-b')): + out = stack.ctl.acquire('alice', SCHED.resolve_names([name]), wait=False) + ids[name] = out.deployments[0].id + stack.runtime.node_of[ids[name]] = node + stack.ctl.release(out.lease.id) + time.sleep(0.01) # distinct idle times + stack.ctl.sleep = lambda seconds: time.sleep(0.005) + return stack, ids + + +def _wait_blocked(stack, message, spec, until_gone=None): + """Acquire `blocked`, whose pod the scheduler cannot place; wait briefly.""" + out = stack.ctl.acquire('alice', SCHED.resolve_names(['blocked']), wait=False) + gid = out.deployments[0].id + stack.runtime.pending[gid] = {'message': message, 'spec': spec, + 'until_gone': until_gone} + post = stack.backend.http.post + stack.backend.http.post = lambda url, **kw: ( + stack.backend.http._Resp(503, {'detail': 'pending'}) + if stack.runtime.is_pending(gid) and (kw.get('json') or {}).get('model') == 'blocked' + else post(url, **kw)) + return stack.ctl.wait_ready(out.deployments, timeout=0.3, interval=0.01) + + +def _idle(stack, ids): + return {name for name, gid in ids.items() + if stack.ctl.ledger.get_deployment(gid).state == 'idle'} + + +def test_an_impossible_node_selector_evicts_nothing(tmp_path): + stack, ids = _two_idle_models(tmp_path) + result = _wait_blocked(stack, "0/2 nodes are available: 2 node(s) didn't match " + "Pod's node affinity/selector.", {'nodeSelector': {'gpu': 'h100'}}) + assert not result.ready + assert _idle(stack, ids) == {'warm-a', 'warm-b'} + + +def test_an_untolerated_taint_evicts_nothing(tmp_path): + stack, ids = _two_idle_models(tmp_path) + _wait_blocked(stack, '0/2 nodes are available: 2 node(s) had untolerated taint ' + '{dedicated: training}.', {}) + assert _idle(stack, ids) == {'warm-a', 'warm-b'} + + +def test_only_an_idle_model_on_a_usable_node_is_evicted(tmp_path): + stack, ids = _two_idle_models(tmp_path) + # Short of GPUs on the l4 node, the only one the pod may use: warm-a is + # the longest idle, but it runs on the a100 node and frees nothing useful. + _wait_blocked(stack, "0/2 nodes are available: 1 Insufficient nvidia.com/gpu, " + "1 node(s) didn't match Pod's node affinity/selector.", + {'nodeSelector': {'gpu': 'l4'}}) + assert _idle(stack, ids) == {'warm-a'} + + +def test_reclaimable_pressure_evicts_a_compatible_idle_model_and_the_lease_proceeds(tmp_path): + stack, ids = _two_idle_models(tmp_path) + result = _wait_blocked(stack, '0/2 nodes are available: 1 Insufficient nvidia.com/gpu, ' + "1 node(s) didn't match Pod's node affinity/selector.", + {'nodeSelector': {'gpu': 'a100'}}, until_gone=ids['warm-a']) + assert _idle(stack, ids) == {'warm-b'} + assert result.ready diff --git a/tests/test_tui.py b/tests/test_tui.py index 96ad270d..0228d5a0 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -2207,7 +2207,10 @@ async def scenario(): catalog_path.write_text('endpoints: [not, a, mapping\n') app.action_refresh() app.action_refresh() - assert len(refused) == 1 and 'catalog reload failed' in refused[0] + # Only the catalog's refusals: a refresh's background worker may + # report something else meanwhile (it did, under a loaded suite). + reloads = [m for m in refused if 'catalog reload failed' in m] + assert len(reloads) == 1 assert 'qwen-extra' in app._endpoint_names # the last good one stays _run(scenario) From 2face9898805c63982a52d5af6762ab24c8c3119 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:16:47 -0400 Subject: [PATCH 074/141] Keep the approved render until its publication completes apply() says how far it got (ApplyResult: runtime, routes). The approved digest and the pending marker are cleared only when both are done, so a render that drifts before the retry of a partial apply is refused, not applied. KubeAI's apply now returns its host gateway's result instead of dropping it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 10 +++++- infer_stack/backends/kubeai.py | 12 ++++--- infer_stack/leasing/backend.py | 49 ++++++++++++++++++++++++--- infer_stack/leasing/compose.py | 35 +++++++++++-------- infer_stack/leasing/controller.py | 29 ++++++++++------ tests/test_leasing_admission.py | 34 +++++++++++++++++++ tests/test_leasing_dynamic_routing.py | 10 +++--- tests/test_leasing_kubeai.py | 12 +++++++ 8 files changed, 151 insertions(+), 40 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index b4715847..8f0fe831 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -559,7 +559,7 @@ request the short resource. Compose returns none, the in-process backends all. `_make_room` keeps the policy. Four parity poison tests; three fail on the old code. -### 15. [ ] Publication phases: the approved digest outlives a partial apply +### 15. [x] Publication phases: the approved digest outlives a partial apply *Why added:* `_apply_pending()` clears the approved-render digest right after `apply()` returns, before checking it returned `False`. Compose @@ -578,6 +578,14 @@ runtime and returns partial; the publication is pending with D1; the renderer changes to D2; an ordinary retry refuses on the digest mismatch; `infer-stack apply` re-approves and applies. +*Done 2026-09-27:* `apply()` returns an `ApplyResult` (`runtime`, `routes`, +`detail`; `None`/`True`/`False` still map onto it). The controller clears +the approved digest and the marker only on a complete result. Found on the +way: KubeAI's `apply` dropped its host gateway's result, so unverified +routes behind KubeAI cleared the marker; it now returns both. Tests: the +partial-apply/renderer-drift case (fails on the old controller: the digest +was gone) and the KubeAI propagation. + ### 16. [ ] Secret rotation is a transaction *Why added:* `rotate_gateway_key()` writes the new key into `.env`, then diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index ae01c386..7faf001f 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -37,7 +37,7 @@ import yaml -from ..leasing.backend import ConvergeScaffold, Readiness +from ..leasing.backend import ApplyResult, ConvergeScaffold, Readiness from ..leasing.compose import dns_slug, vllm_service_dict from ..leasing.models import Deployment from ..probe import openai_ready @@ -951,18 +951,19 @@ def use_profile(self, profile: dict) -> None: self.gateway = self.gateway_factory(wanted) self.gateway.use_profile(front) - def apply(self) -> None: - """Converge the cluster to the last render: apply + prune. + def apply(self) -> ApplyResult: + """Converge the cluster to the last render: apply + prune, then the gateway. Reads the on-disk manifest last written by :meth:`converge` (render) and applies it — it does NOT re-render (compose parity: the controller coalesces applies under its own lock/generation). Then deletes any infer-stack-managed Model the render no longer contains. Idempotent. + The result includes the gateway's (its routes, with dynamic routing). """ from .._log import logger if not self.models_file.exists(): - return + return ApplyResult() text = self.models_file.read_text() wanted = set(self._load_sidecar().get('models') or {}) if text.strip(): @@ -990,7 +991,8 @@ def apply(self) -> None: ['delete', 'models.kubeai.org', name, '--ignore-not-found'] ) if self.gateway is not None: - self.gateway.apply() + return ApplyResult() & ApplyResult.of(self.gateway.apply()) + return ApplyResult() def residency(self): """The managed Models' pods, strictly: raises rather than guess. diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 1a502f60..09142efa 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -44,6 +44,45 @@ class Readiness: needs_room: bool = False +@dataclass(frozen=True) +class ApplyResult: + """How far an apply got: the runtime, then the gateway's routes. + + A publication is complete only when both hold. ``runtime`` false: the + runtime was not brought to the render (an unreadable render, a render + from before fingerprints). ``routes`` false: the runtime reached the + render but the gateway's routes (dynamic routing) were not verified. In + either case the publication stays pending, and so does the approval of + the render it is applying. Failures raise instead; this is for the + outcomes that are not errors. + + >>> ApplyResult.of(None).complete, ApplyResult.of(False).runtime + (True, False) + >>> ApplyResult(routes=False).complete + False + """ + + runtime: bool = True + routes: bool = True + detail: str = '' + + @property + def complete(self) -> bool: + return self.runtime and self.routes + + @classmethod + def of(cls, value: ApplyResult | bool | None) -> ApplyResult: + """An apply's return value as a result; ``None``/``True`` is complete.""" + if isinstance(value, ApplyResult): + return value + return cls() if value is not False else cls(runtime=False) + + def __and__(self, other: ApplyResult) -> ApplyResult: + """Both parts of one apply (KubeAI's Models, then its gateway).""" + return ApplyResult(self.runtime and other.runtime, self.routes and other.routes, + '; '.join(d for d in (self.detail, other.detail) if d)) + + class BackendTimeout(RuntimeError): """A backend command exceeded its time bound and was killed. @@ -204,12 +243,14 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, """Render the desired set to backend state; optionally apply it.""" ... - def apply(self) -> bool | None: + def apply(self) -> ApplyResult | bool | None: """Converge reality to the last render (idempotent, slow half). - Return ``False`` if the apply did not fully take effect (the controller - keeps the change pending and retries); ``True`` or ``None`` otherwise. - Raise on backend failure, which also leaves the change pending. + Return an :class:`ApplyResult` saying how far it got; ``None`` or + ``True`` means complete, ``False`` means the runtime was not reached. + The controller keeps the change pending, and its approval, until a + complete result. Raise on backend failure, which also leaves the + change pending. """ ... diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 554cacea..eba82dbe 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -51,7 +51,7 @@ from ..env_utils import parse_env_file from ..probe import openai_ready from ..profile_runtime import simulator_args, vllm_args -from .backend import ConvergeScaffold, Readiness +from .backend import ApplyResult, ConvergeScaffold, Readiness from .gateway import ( LITELLM_CONFIG_FILENAME, LITELLM_SERVICE, @@ -1850,41 +1850,43 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N self.apply() return plan - def apply(self) -> bool: - """Bring the already-rendered compose project up; return whether it fully succeeded. + def apply(self) -> ApplyResult: + """Bring the already-rendered compose project up; say how far it got. Reads the on-disk compose file last written by :meth:`converge` (render) and applies it -- it does NOT re-render. The controller serialises render and apply under its host-wide lock, so the file cannot change underneath this call. Idempotent: a no-op when reality already matches the file. - Returns ``False`` when the apply did not fully take effect, so the - controller keeps the change pending and retries it: + An incomplete :class:`~infer_stack.leasing.backend.ApplyResult` keeps + the change pending, and the controller retries it: - * the rendered file cannot be read; - * in dynamic-routing mode, the gateway's routes could not be reconciled - and verified within budget. The budget is short when the gateway was - already running (steady state), and long when this apply is bringing - it up (bootstrap: it waits on Postgres health and runs DB migrations). + * ``runtime`` false: the rendered file cannot be read, or predates + fingerprints; + * ``routes`` false: in dynamic-routing mode, the gateway's routes could + not be reconciled and verified within budget. The budget is short + when the gateway was already running (steady state), and long when + this apply is bringing it up (bootstrap: it waits on Postgres health + and runs DB migrations). Docker failures and timeouts raise, and leave the change pending too. """ from .._log import logger if not self.compose_file.exists(): - return True + return ApplyResult() try: doc = yaml.safe_load(self.compose_file.read_text()) or {} except Exception: # noqa: BLE001 - reported as "not applied", never raised logger.warning('apply: rendered compose file is unreadable; change stays pending') - return False + return ApplyResult(runtime=False, detail='the rendered compose file is unreadable') services = doc.get('services') or {} sidecar = self._load_sidecar() fingerprints = sidecar.get('fingerprints') if fingerprints is None or set(fingerprints) != set(services): # A render from before fingerprints: re-render (any mutation) first. logger.warning('apply: the render predates fingerprints; re-render, then apply') - return False + return ApplyResult(runtime=False, detail='the render predates fingerprints') # A service that runs as a user needs its bind-mount sources made by # us, as that user: Docker would make a missing one as root. for svc in services.values(): @@ -1902,11 +1904,14 @@ def apply(self) -> bool: networks=doc.get('networks') or {}, ) if dynamic and 'litellm' in services: - return self.gateway._reconcile_routes( + verified = self.gateway._reconcile_routes( deadline_s=ROUTE_RECONCILE_STEADY_S if 'litellm' in outcome.kept_services else ROUTE_RECONCILE_BOOTSTRAP_S, ) - return True + if not verified: + return ApplyResult(routes=False, + detail='the gateway routes were not verified') + return ApplyResult() #: Service-level ownership adopted at migration: container id -> #: {service, fingerprint}. Set by the controller from the ledger. diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index cc0c7562..6aaf5089 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import Callable, Iterable, TypeVar -from .backend import HostRuntime, ServingBackend +from .backend import ApplyResult, HostRuntime, ServingBackend from .ledger import Ledger from .models import Deployment, DeploymentState, EndpointRequest, Lease, LeaseState @@ -85,6 +85,8 @@ class ReconcileResult: # desired state was staged (--no-apply, render) or its apply did not fully # succeed. The next applying operation, or `infer-stack apply`, publishes it. publication_pending: bool = False + # Whether the runtime reached the render (routes may still be unverified). + runtime_applied: bool = False # Admission mode: idle keep-warm residents that yielded their GPUs, and # LIVE deployments whose committed allocation is no longer valid. displaced: list[str] = field(default_factory=list) @@ -198,6 +200,9 @@ def __init__( # Set only by apply_now(): the operator explicitly re-approves a render # that differs from an earlier approved digest. self._explicit_apply = False + # Set when an apply is invoked: whether a failed publication may have + # changed the runtime (secret rotation reads it; see rotate_gateway_key). + self._apply_began = False self._admission_digest: str | None = None #: Whether the last refused admission was for lack of GPUs. self._admission_capacity = False @@ -1228,8 +1233,9 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: if marker['interrupted']: self._wait_for_settled_runtime() before = set(self.backend.observe()) + self._apply_began = True try: - ok = apply_fn() + outcome = ApplyResult.of(apply_fn()) except BaseException as ex: from .compose import ApplyAborted @@ -1240,21 +1246,24 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: apply_requested=True, interrupted=not isinstance(ex, ApplyAborted)) rec.publication_pending = True raise - if approved: - # The approved render reached Docker; a retry for routes (or any - # later change) renders from newer state and needs no re-approval. - self.ledger.store.clear_approved_digest() after = set(self.backend.observe()) rec.realized = sorted(set(rec.realized) | (after - before)) rec.torn_down = sorted(set(rec.torn_down) | (before - after)) rec.applied = True - if ok is False: + rec.runtime_applied = outcome.runtime + if not outcome.complete: + # The publication is not done, so neither is the approval of its + # render: a retry re-renders, and a render that drifted (an + # upgrade in between) must be approved again, not applied. rec.publication_pending = True logger.warning( - 'apply did not fully take effect; the change stays pending and ' - 'the next acquire/release or `infer-stack apply` retries it' - ) + 'apply did not fully take effect ({}); the change stays pending and ' + 'the next acquire/release or `infer-stack apply` retries it', + outcome.detail or ('routes not verified' if outcome.runtime + else 'runtime not reached')) return rec + if approved: + self.ledger.store.clear_approved_digest() self.ledger.clear_publication_pending(marker['version']) rec.publication_pending = False return rec diff --git a/tests/test_leasing_admission.py b/tests/test_leasing_admission.py index 2ebd2494..d3097b95 100644 --- a/tests/test_leasing_admission.py +++ b/tests/test_leasing_admission.py @@ -466,3 +466,37 @@ def flaky(): assert ledger.publication_pending()['approved_digest'] is None acquire(ctl, 'two') # an ordinary publisher proceeds assert ledger.publication_pending() is None + + +def test_a_partial_apply_keeps_the_approval_so_a_drifted_render_is_refused(tmp_path): + """Queue item 15: the approved digest outlives an apply that did not finish. + + The first apply reaches the runtime but its routes do not verify, so the + publication stays pending. The renderer then changes (an upgrade): an + ordinary retry must refuse the unapproved render, and `infer-stack apply` + is the deliberate re-approval. + """ + from infer_stack.leasing.backend import ApplyResult, ConvergeScaffold + from infer_stack.leasing.profile import ProfileMismatch + + ledger, ctl, docker = make(tmp_path) + real = ctl.backend.apply + + def partial(): + real() # the runtime changes... + return ApplyResult(routes=False) # ...its routes do not verify + + ctl.backend.apply = partial + acquire(ctl, 'one') + approved = ledger.publication_pending()['approved_digest'] + assert approved and approved == ctl.backend.last_planned_digest + assert docker.containers # the runtime did change + + ctl.backend.apply = real + original = ConvergeScaffold._planned_digest + ctl.backend._planned_digest = lambda planned: 'v2-' + original(planned) # an upgrade + with pytest.raises(ProfileMismatch, match='approved'): + ctl.gc() # an ordinary retry + assert ledger.publication_pending()['approved_digest'] == approved + ctl.apply_now() # the explicit re-approval + assert ledger.publication_pending() is None diff --git a/tests/test_leasing_dynamic_routing.py b/tests/test_leasing_dynamic_routing.py index 4000dd3b..32740dca 100644 --- a/tests/test_leasing_dynamic_routing.py +++ b/tests/test_leasing_dynamic_routing.py @@ -507,13 +507,13 @@ def test_apply_returns_false_when_routes_do_not_verify(tmp_path): time = FakeTime() be = _timed_backend(tmp_path, UnreachableGateway(time), time) be.converge([dep('grp-aaaaaa', served='smol')], apply=False) - assert be.apply() is False + assert not be.apply().routes def test_apply_returns_true_once_routes_verify(tmp_path): be = make_backend(tmp_path, RecordingGateway()) be.converge([dep('grp-aaaaaa', served='smol')], apply=False) - assert be.apply() is True + assert be.apply().complete def test_apply_uses_the_short_deadline_when_the_gateway_was_already_up(tmp_path): @@ -526,11 +526,11 @@ def test_apply_uses_the_short_deadline_when_the_gateway_was_already_up(tmp_path) be = _timed_backend(tmp_path, UnreachableGateway(time), time) be.converge([dep('grp-aaaaaa', served='smol')], apply=False) - assert be.apply() is False # bootstrap: gateway was not running + assert not be.apply().routes # bootstrap: gateway was not running assert ROUTE_RECONCILE_STEADY_S < time.now <= ROUTE_RECONCILE_BOOTSTRAP_S time.now = 0.0 - assert be.apply() is False # steady: the first up started it + assert not be.apply().routes # steady: the first up started it assert time.now <= ROUTE_RECONCILE_STEADY_S @@ -538,4 +538,4 @@ def test_apply_returns_false_for_an_unreadable_render(tmp_path): be = make_backend(tmp_path, RecordingGateway()) be.compose_file.parent.mkdir(parents=True, exist_ok=True) be.compose_file.write_text('services: [unclosed\n') - assert be.apply() is False + assert not be.apply().runtime # unreadable: runtime not reached diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index fe305b55..28edbbbc 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -1143,3 +1143,15 @@ def test_doctor_checks_the_host_gateways_docker_too(tmp_path): names = [name for name, ok, _ in be.doctor()] assert 'gateway: docker daemon reachable' in names assert not any('image vllm' in n or 'GPUs visible' in n for n in names) + + +def test_the_host_gateways_unverified_routes_leave_the_apply_incomplete(tmp_path): + """KubeAI's apply used to drop its gateway's result: a publication whose + routes did not verify was cleared as complete (queue item 15).""" + from infer_stack.leasing.backend import ApplyResult + + be, _ = make_gateway_backend(tmp_path) + be.converge([vllm('grp-a', served='a')], apply=False) + be.gateway.apply = lambda: ApplyResult(routes=False, detail='routes not verified') + outcome = be.apply() + assert outcome.runtime and not outcome.routes and not outcome.complete From 8e4d9ed1b5c36d849e8e84ca60d113580d1572a2 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:20:02 -0400 Subject: [PATCH 075/141] Secret rotation restores the old key on any failure before apply A running LiteLLM keeps the key it started with, so a rotation whose publication fails before its apply begins (declined, residency unreadable, render refused) puts the old key back in the .env. After the apply began, the new key stays and the pending publication converges the gateway to it. Tests use a gateway fake that captures its key at (re)start. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 10 +++- infer_stack/leasing/controller.py | 18 +++++-- tests/test_leasing_secrets.py | 81 +++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 6 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 8f0fe831..075269d5 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -586,7 +586,7 @@ routes behind KubeAI cleared the marker; it now returns both. Tests: the partial-apply/renderer-drift case (fails on the old controller: the digest was gone) and the KubeAI propagation. -### 16. [ ] Secret rotation is a transaction +### 16. [x] Secret rotation is a transaction *Why added:* `rotate_gateway_key()` writes the new key into `.env`, then publishes, and restores the old key only on `ConvergeAborted`. A strict @@ -603,6 +603,14 @@ tests for a declined approval, a render failure before apply (both: old file, old running key), and a failure after the gateway restarted (pending, converges on retry). +*Done 2026-09-27:* the controller records when an apply begins; a rotation +whose publication fails before that restores the old key (any failure, not +only a decline), and after it keeps the new key with the publication +pending and interrupted. Tests use a gateway that accepts the key its +container started with: a residency failure before apply (fails on the old +code), a decline, and a failure after apply began that converges on +`infer-stack apply`. + ### 17. [ ] Dynamic routes never point at a torn-down upstream *Why added:* Compose `apply()` removes departing engines, then reconciles diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 6aaf5089..d9163750 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -662,10 +662,16 @@ def rotate_gateway_key(self, *, force: bool = False) -> ReconcileResult: Refused while any lease is ACTIVE unless ``force``: its holder authenticates with the old key, and the gateway restarts. The new key is written, then published like any desired-state change, so the - gateway (and Open WebUI) are recreated with it. A declined apply puts - the old key back. + gateway (and Open WebUI) are recreated with it. + + A running LiteLLM keeps the key it started with. So when the + publication fails before its apply began (declined, residency + unreadable, a render refused) nothing was recreated, and the old key + goes back into the managed ``.env``: otherwise clients would be handed + a key the gateway rejects. Once the apply began, the gateway may + already run the new key; the file keeps it, and the pending + publication converges the runtime to it on the next apply. """ - from .backend import ConvergeAborted from .profile import ProfileMismatch rotate = getattr(self.backend, 'rotate_master_key', None) @@ -681,10 +687,12 @@ def rotate_gateway_key(self, *, force: bool = False) -> ReconcileResult: ) self._mark_pending(apply=True) replaced = rotate() + self._apply_began = False try: return self._publish() - except ConvergeAborted: - getattr(self.backend, 'restore_env')(replaced) + except BaseException: + if not self._apply_began: + getattr(self.backend, 'restore_env')(replaced) raise def observe_state(self) -> dict: diff --git a/tests/test_leasing_secrets.py b/tests/test_leasing_secrets.py index f5a730c0..0c95f3d5 100644 --- a/tests/test_leasing_secrets.py +++ b/tests/test_leasing_secrets.py @@ -147,3 +147,84 @@ def test_an_unreachable_gateway_is_not_a_rejection(tmp_path): ledger, ctl = make(tmp_path) ctl.backend.http.down = True assert ctl.backend.gateway_accepts('sk-anything', wait=5.0) is None + + +# -- rotation is a transaction (queue item 16) -------------------------------------- + + +class StartedGateway(Gateway): + """Accepts the key its container STARTED with, as a real LiteLLM does. + + The key is read from the .env when the litellm container is (re)created, + not per request: a key written to the file afterwards is not the one the + gateway checks. + """ + + def __init__(self, env_path, docker): + super().__init__(env_path) + self.docker = docker + self.container = None + self.key = None + + def get(self, url, headers=None, **kw): + current = next((cid for cid, c in self.docker.containers.items() + if c['service'] == 'litellm'), None) + if current != self.container: # (re)created: read the key now + self.container = current + self.key = parse_env_file(self.env_path).get(API_KEY_ENV) + if headers and headers.get('Authorization') == f'Bearer {self.key}': + return FakeResp(200, {'data': []}) + return FakeResp(400, {'error': 'Authentication Error'}) + + +def make_started(tmp_path): + ledger, ctl = make(tmp_path) + ctl.backend.http = StartedGateway(ctl.backend.gateway._env_path, ctl.backend.run) + ctl.backend.gateway.http = ctl.backend.http + ctl.release(acquire(ctl, 'one').lease.id) + return ledger, ctl + + +def test_a_render_failure_before_apply_keeps_file_and_gateway_on_the_old_key(tmp_path): + from infer_stack.leasing.residency import ResidencyUnknown + + ledger, ctl = make_started(tmp_path) + old = env(ctl)[API_KEY_ENV] + assert ctl.backend.gateway_accepts(old) + real = ctl.backend.residency + ctl.backend.residency = lambda: (_ for _ in ()).throw(ResidencyUnknown('docker down')) + with pytest.raises(ResidencyUnknown): + ctl.rotate_gateway_key() + assert env(ctl)[API_KEY_ENV] == old # the file says what the gateway runs + ctl.backend.residency = real + assert ctl.backend.gateway_accepts(old) + + +def test_a_declined_rotation_keeps_file_and_gateway_on_the_old_key(tmp_path): + from infer_stack.leasing.backend import ConvergeAborted + + ledger, ctl = make_started(tmp_path) + old = env(ctl)[API_KEY_ENV] + ctl.backend._approve_changes = lambda planned: (_ for _ in ()).throw(ConvergeAborted('no')) + with pytest.raises(ConvergeAborted): + ctl.rotate_gateway_key() + assert env(ctl)[API_KEY_ENV] == old and ctl.backend.gateway_accepts(old) + + +def test_a_failure_after_apply_began_keeps_the_new_key_and_converges(tmp_path): + from infer_stack.leasing.backend import BackendTimeout + + ledger, ctl = make_started(tmp_path) + old = env(ctl)[API_KEY_ENV] + real = ctl.backend.apply + ctl.backend.apply = lambda: (_ for _ in ()).throw(BackendTimeout('up timed out')) + ctl.backend.settle_snapshot = lambda: () + with pytest.raises(BackendTimeout): + ctl.rotate_gateway_key() + new = env(ctl)[API_KEY_ENV] + assert new != old # not reverted blindly + assert ledger.publication_pending()['apply_requested'] + ctl.backend.apply = real + ctl.apply_now() # the pending publication converges + assert ctl.backend.gateway_accepts(new) and not ctl.backend.gateway_accepts(old) + assert ledger.publication_pending() is None From b1e5a41b1837cb1e382f8777fad9676ddda5b763 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:24:30 -0400 Subject: [PATCH 076/141] Retire routes before their upstreams go With dynamic routing, an apply first deletes and verifies the gateway routes the render drops, and only then removes engines (Compose) or stale Models (KubeAI with a host gateway). If the gateway cannot delete them, nothing is torn down and the publication stays pending, so no route is left pointing at a stopped upstream. A replacement route removed but not re-added is logged by name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 12 ++++- infer_stack/backends/kubeai.py | 6 ++- infer_stack/backends/kubeai_gateway.py | 5 ++ infer_stack/leasing/compose.py | 26 +++++++++ infer_stack/leasing/gateway.py | 26 +++++++-- tests/test_leasing_dynamic_routing.py | 74 ++++++++++++++++++++++++++ 6 files changed, 144 insertions(+), 5 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 075269d5..cc86eed0 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -611,7 +611,7 @@ container started with: a residency failure before apply (fails on the old code), a decline, and a failure after apply began that converges on `infer-stack apply`. -### 17. [ ] Dynamic routes never point at a torn-down upstream +### 17. [x] Dynamic routes never point at a torn-down upstream *Why added:* Compose `apply()` removes departing engines, then reconciles routes; KubeAI deletes stale Models, then applies the gateway. If route @@ -629,6 +629,16 @@ two dedicated deployments for one alias leaves the departing upstream running (the invariant, not just `publication_pending`), on Compose and on KubeAI with the host gateway. +*Done 2026-09-27:* an apply's first phase, `retire_routes()`, deletes and +verifies the routes the render drops (only with dynamic routing and a +running gateway); if that fails the apply returns "runtime not reached" and +tears nothing down. KubeAI retires before pruning stale Models; its +in-cluster gateway has static routes to one upstream and retires nothing. +A replacement whose add fails after its delete is logged by route. Tests on +Compose and on KubeAI with a host gateway (both fail on the old code): the +departing engine or Model keeps running while its route stays, and goes +once the gateway recovers. + ### 18. [ ] Coalescing prefers a LIVE deployment over reviving an IDLE one *Why added (older than this campaign):* `plan_acquire()` takes the first diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 7faf001f..4d94bd74 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -981,10 +981,14 @@ def apply(self) -> ApplyResult: 'reachable? See scripts/install_kubeai.sh and ' 'docs/kubeai-backend.md.' ) from ex - # Prune: managed Models on the cluster that the render dropped. + # Prune: managed Models on the cluster that the render dropped. Their + # gateway routes go first, so none is left pointing at a deleted Model. stale = [ name for name in self._cluster_models() if name not in wanted ] + if stale and self.gateway is not None and not self.gateway.retire_routes(): + return ApplyResult(runtime=False, routes=False, detail=( + 'routes to departing Models could not be removed; nothing was deleted')) for name in sorted(stale): logger.info('kubectl delete model {}', name) self._kubectl( diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index 95d24b54..80415f8e 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -217,6 +217,11 @@ def apply(self) -> None: self._kubectl(['apply', '-f', str(self.manifests_file)]) self._kubectl(['rollout', 'status', f'deployment/{NAME}', '--timeout=300s']) + def retire_routes(self) -> bool: + """Nothing to retire: this gateway's routes are static and point at + KubeAI's one upstream, which no Model's removal takes away.""" + return True + def down(self) -> None: self._kubectl(['delete', 'deployment,service,configmap,secret', NAME, '--ignore-not-found']) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index eba82dbe..cac3bb36 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1897,6 +1897,10 @@ def apply(self) -> ApplyResult: if source.startswith('/') and not Path(source).exists(): Path(source).mkdir(parents=True, exist_ok=True) dynamic = bool(self.litellm and self.dynamic_routing) + if dynamic and 'litellm' in services and not self.retire_routes(): + return ApplyResult(runtime=False, routes=False, detail=( + 'routes to departing upstreams could not be removed; ' + 'nothing was torn down')) outcome = self.selective_apply( services, fingerprints, degraded=set(sidecar.get('degraded') or ()), @@ -1957,6 +1961,28 @@ def reclaim_candidates(self, blocked: Deployment, a Compose probe never reports ``needs_room``.""" return [] + def retire_routes(self) -> bool: + """Remove the gateway routes the last render dropped, before their + upstreams go; whether none of them is left. + + Only dynamic routing publishes a route per upstream, and only a + running gateway serves one: otherwise there is nothing to retire (a + gateway that starts later reconciles its stored routes itself). The + first phase of :meth:`apply`, and of KubeAI's apply for its host + gateway, so an upstream is never torn down under a route that still + points at it. + """ + from .gateway import ROUTE_RECONCILE_STEADY_S + + if not (self.litellm and self.dynamic_routing): + return True + running = any(c.service == 'litellm' and c.warm + for c in self.residency().all_containers()) + if not running: + return True + return self.gateway._reconcile_routes( + deadline_s=ROUTE_RECONCILE_STEADY_S, retire_only=True) + def _wait_until(self, predicate, *, deadline_s: float, what: str, interval: float = 1.0): """Poll strict residency until ``predicate(snapshot)``; abort at the deadline.""" deadline = self._clock() + deadline_s diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 2d6c91ce..9d973a7a 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -1274,6 +1274,7 @@ def _desired_routes(self) -> list[dict[str, Any]]: def _reconcile_routes( self, *, deadline_s: float = ROUTE_RECONCILE_BOOTSTRAP_S, delay: float = 2.0, + retire_only: bool = False, ) -> bool: """Make the live gateway's managed routes match the rendered route set. @@ -1302,6 +1303,17 @@ def _reconcile_routes( managed route set equals the desired set; any failure is logged and returns ``False`` rather than raising, so the caller decides whether an unverified route set blocks anything. + + ``retire_only`` is the first phase of an apply that tears upstreams + down: delete only the managed routes the render no longer has (no + adds, no replacements), and verify they are gone. It runs before the + runtime removes anything, so a route is never left pointing at an + upstream this apply stopped. + + A replacement (same id, different semantics) is a delete then an add, + since ``/model/new`` does not update on every LiteLLM release. If the + add fails after the delete, the route is missing until the next apply: + that gap is logged by name, and the result is ``False``. """ from .._log import logger @@ -1322,11 +1334,12 @@ def _reconcile_routes( '{:g}s; leaving it for the next apply', deadline_s, ) return False - mismatched = sorted( + mismatched = [] if retire_only else sorted( rid for rid in desired.keys() & current.keys() if desired_semantics[rid] != current[rid] ) - to_add_ids = sorted((desired.keys() - current.keys()) | set(mismatched)) + to_add_ids = [] if retire_only else sorted( + (desired.keys() - current.keys()) | set(mismatched)) to_delete = sorted((current.keys() - desired.keys()) | set(mismatched)) to_add = [desired[rid] for rid in to_add_ids] if not (to_add or to_delete): @@ -1346,9 +1359,16 @@ def _reconcile_routes( deadline=deadline, ) for route in to_add: - ok &= self._post_route( + added = self._post_route( '/model/new', route, route.get('model_name'), deadline=deadline, ) + rid = route['model_info']['id'] + if not added and rid in mismatched: + logger.warning( + 'dynamic routing: route {} ({}) was removed for replacement ' + 'and not re-added; it is missing until the next apply', + rid, route.get('model_name')) + ok &= added logger.info( 'dynamic routing: +{} route(s), -{} route(s), ~{} replacement(s) ' '(now {} desired)', diff --git a/tests/test_leasing_dynamic_routing.py b/tests/test_leasing_dynamic_routing.py index 32740dca..3f8ef952 100644 --- a/tests/test_leasing_dynamic_routing.py +++ b/tests/test_leasing_dynamic_routing.py @@ -539,3 +539,77 @@ def test_apply_returns_false_for_an_unreadable_render(tmp_path): be.compose_file.parent.mkdir(parents=True, exist_ok=True) be.compose_file.write_text('services: [unclosed\n') assert not be.apply().runtime # unreadable: runtime not reached + + +# -- a route never outlives its upstream (queue item 17) ------------------------ + + +class NoDeleteGateway(RecordingGateway): + """The admin API refuses to delete routes (a LiteLLM or DB failure).""" + + refuse = True + + def post(self, url, **kw): + if self.refuse and url.endswith('/model/delete'): + return FakeResp(500, {'error': 'db unavailable'}) + return super().post(url, **kw) + + +def test_a_departing_upstream_outlives_a_route_that_could_not_be_removed(tmp_path): + """Two dedicated deployments behind one alias; one is released while the + gateway cannot delete routes. Its engine must keep running until its route + is gone: an engine stopped under a live route fails some of the alias's + requests, and nothing but the next apply would repair it.""" + time = FakeTime() + gw = NoDeleteGateway() + be = _timed_backend(tmp_path, gw, time) + a, b = dep('grp-aaaaaa', served='smol', t=0), dep('grp-bbbbbb', served='smol', t=1) + be.converge([a, b], apply=False) + assert be.apply().complete + b_service = vllm_service_name(b, unique=True) + assert b_service in be.run.running and len(_managed(gw)) == 2 + + be.converge([a], apply=False) # b released + outcome = be.apply() + assert not outcome.runtime and not outcome.complete + assert b_service in be.run.running # not torn down under its route + assert _route_id(b.id, 'smol') in _managed(gw) + + gw.refuse = False # the gateway recovers + assert be.apply().complete + assert b_service not in be.run.running + assert _managed(gw) == {_route_id(a.id, 'smol')} + + +def test_kubeai_keeps_a_departing_model_while_its_route_stays(tmp_path): + """The same invariant behind KubeAI's host gateway: a stale Model is not + deleted while the gateway still routes to it.""" + from infer_stack.backends.kubeai import KubeaiBackend + from test_leasing_kubeai import FakeKubectl + + time = FakeTime() + gw = NoDeleteGateway() + kubectl = FakeKubectl() + gateway = ComposeBackend( + state_dir=tmp_path / 'gateway', inventory={'gpu_count': 0, 'gpus': []}, + run=FakeDocker(), http=gw, project='infer-stack-gateway', images=IMAGES, + ports=PORTS, state=STATE, litellm=True, ui=False, dynamic_routing=True, + sleep=time.sleep, clock=time.clock, + ) + be = KubeaiBackend(state_dir=tmp_path / 'kubeai', run=kubectl, http=gw, + gateway=gateway, gateway_upstream='http://10.43.0.9/openai/v1', + default_resource_profile='gpu') + a, b = dep('grp-aaaaaa', served='smol', t=0), dep('grp-bbbbbb', served='smol', t=1) + be.converge([a, b], apply=False) + assert be.apply().complete and len(kubectl.applied) == 2 + managed_before = _managed(gw) + assert len(managed_before) == 2 + + be.converge([a], apply=False) # b released + assert not be.apply().complete + assert len(kubectl.applied) == 2 # b's Model is still there + assert _managed(gw) == managed_before + + gw.refuse = False + assert be.apply().complete + assert len(kubectl.applied) == 1 and len(_managed(gw)) == 1 From 2473e952e3d6f100b5957955ca9cb4062e6afd86 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:28:01 -0400 Subject: [PATCH 077/141] Coalescing prefers what costs least: LIVE, then resident IDLE plan_acquire took the oldest adequate deployment, so a request both a live deployment and an older idle one could serve revived the idle one (and on one GPU queued behind the live one). Candidates are now ranked LIVE, being created in this plan, resident IDLE, other IDLE; the ledger still decides eligibility and takes residency as a hint. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 9 ++++++- infer_stack/leasing/controller.py | 4 ++- infer_stack/leasing/ledger.py | 27 +++++++++++++++++--- tests/test_leasing_controller.py | 42 +++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index cc86eed0..01d530fe 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -639,7 +639,7 @@ Compose and on KubeAI with a host gateway (both fail on the old code): the departing engine or Model keeps running while its route stays, and goes once the gateway recovers. -### 18. [ ] Coalescing prefers a LIVE deployment over reviving an IDLE one +### 18. [x] Coalescing prefers a LIVE deployment over reviving an IDLE one *Why added (older than this campaign):* `plan_acquire()` takes the first compatible deployment by creation time among LIVE and IDLE, so an older @@ -655,6 +655,13 @@ residency hint from the controller. 4k request onto the live 32k deployment: nothing revived, no placement, no queue, no runtime change. +*Done 2026-09-27:* `Ledger.plan_acquire(requests, resident=)` ranks +adequate candidates LIVE, then created in this plan, then resident IDLE, +then other IDLE, creation order within each; the controller passes +`residency.is_resident`. Tests: the 8k/32k/4k sequence coalesces onto the +live 32k deployment with nothing revived or started, and a resident idle +deployment beats an older one that is gone (both fail on the old code). + ### 19. [ ] `routes seed` fails closed on a redefinition; one route API *Why added:* the registry merge is documented as additive, but a same-name diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index d9163750..0032e4dc 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1619,7 +1619,9 @@ def _acquire_by_admission( validate = getattr(self.backend, 'validate_requests', None) if validate is not None: validate(requests) - overlay = self.ledger.plan_acquire(requests) + overlay = self.ledger.plan_acquire( + requests, + resident=residency.is_resident if residency is not None else None) allocations, reasons = self._admit(overlay, residency) except BaseException: if candidate_profile is not None: diff --git a/infer_stack/leasing/ledger.py b/infer_stack/leasing/ledger.py index 09e878bc..290093b3 100644 --- a/infer_stack/leasing/ledger.py +++ b/infer_stack/leasing/ledger.py @@ -149,10 +149,26 @@ def _default_id(prefix: str) -> str: # -- public API -------------------------------------------------------- - def plan_acquire(self, requests: list[EndpointRequest]) -> AcquireOverlay: - """Decide how ``requests`` would coalesce, without writing anything.""" + def plan_acquire(self, requests: list[EndpointRequest], *, + resident: Callable[[str], bool] | None = None) -> AcquireOverlay: + """Decide how ``requests`` would coalesce, without writing anything. + + Among the shared deployments whose capacity covers a request, the one + that costs least to use wins: one already LIVE (nothing to place or + start), then one this plan is already creating, then an IDLE one that + is still ``resident`` (the controller's hint from residency: a revival + with nothing to start), then any other IDLE one; creation order within + each. Eligibility stays here; ``resident`` only orders. + """ import copy + def cost(deployment: Deployment, pending: bool) -> int: + if deployment.state == DeploymentState.LIVE and not pending: + return 0 + if pending: + return 1 + return 2 if resident is not None and resident(deployment.id) else 3 + now = self.clock() state_version = self.store.admission_state_version() view: dict[str, Deployment] = {} @@ -171,8 +187,11 @@ def plan_acquire(self, requests: list[EndpointRequest]) -> AcquireOverlay: g for g in created.values() if g.compat_key == req.compat_key and g.sharing == Sharing.SHARED ] - for candidate in [*stored, *pending]: - current = view.get(candidate.id) or copy.deepcopy(candidate) + candidates = [(view.get(g.id) or copy.deepcopy(g), False) for g in stored] + candidates += [(g, True) for g in pending] + ranked = sorted(enumerate(candidates), + key=lambda item: (cost(*item[1]), item[0])) + for _, (current, _) in ranked: if capacity_satisfies(current.capacity, req.capacity): chosen = current break diff --git a/tests/test_leasing_controller.py b/tests/test_leasing_controller.py index da275819..27cf8fd0 100644 --- a/tests/test_leasing_controller.py +++ b/tests/test_leasing_controller.py @@ -231,3 +231,45 @@ def test_controller_with_catalog(): assert backend.realize_calls == [out.deployments[0].id] assert set(out.deployments[0].served) == {'e1', 'e2'} assert out.wait.ready is True + + +# -- coalescing prefers what costs least to use (queue item 18) -------------------- + + +def test_a_request_coalesces_onto_a_live_deployment_not_an_older_idle_one(): + """8k, released; then 32k (a new deployment, the 8k one cannot serve it); + then 4k, which both could serve. The live 32k deployment serves it now; the + older idle 8k one is not revived (on one GPU that revival would queue).""" + ctl, backend, _ = make_controller(ready=True) + ledger = ctl.ledger + small = ctl.acquire('a', [vreq('m', max_model_len=8192)]) + ctl.release(small.lease.id) + small_id = small.deployments[0].id + big = ctl.acquire('b', [vreq('m', max_model_len=32768)]) + big_id = big.deployments[0].id + assert big_id != small_id + starts = list(backend.realize_calls) + + out = ctl.acquire('c', [vreq('m', max_model_len=4096)]) + + assert out.deployments[0].id == big_id + assert ledger.get_deployment(small_id).state == DeploymentState.IDLE # not revived + assert backend.realize_calls == starts # nothing started + + +def test_a_resident_idle_deployment_beats_an_older_one_that_is_gone(): + ctl, backend, _ = make_controller(ready=True) + ledger = ctl.ledger + old = ctl.acquire('a', [vreq('m', max_model_len=8192)]) + ctl.release(old.lease.id) + new = ctl.acquire('b', [vreq('m', max_model_len=16384, sharing=Sharing.DEDICATED)]) + ctl.release(new.lease.id) + old_id, new_id = old.deployments[0].id, new.deployments[0].id + # Make the newer one shareable, and the older one no longer resident. + ledger.store._conn.execute("UPDATE deployments SET sharing = ? WHERE id = ?", + (Sharing.SHARED, new_id)) + backend.realized.pop(old_id) + + out = ctl.acquire('c', [vreq('m', max_model_len=4096)]) + + assert out.deployments[0].id == new_id From b2d003ab33f9bde2dca45d5c0f12bd2fdcf797ed Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:33:58 -0400 Subject: [PATCH 078/141] routes seed refuses to redefine an alias; one route-registry API Seeding a new alias is additive; seeding a different definition of an existing one redirects its clients, so it is refused and nothing is written unless --replace is given. Seed and prune are planned and committed by the controller (rechecked under the lock, published like any change); the CLI presents and confirms, and touches no private registry, lock or file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 15 ++- infer_stack/cli/commands_leasing.py | 145 +++++++++++----------------- infer_stack/leasing/backend.py | 24 +++++ infer_stack/leasing/controller.py | 84 ++++++++++++++++ infer_stack/leasing/gateway.py | 34 +++++-- infer_stack/leasing/routes.py | 72 ++++++++++++++ tests/test_cli_leasing.py | 39 ++++++++ 7 files changed, 317 insertions(+), 96 deletions(-) create mode 100644 infer_stack/leasing/routes.py diff --git a/docs/queue.md b/docs/queue.md index 01d530fe..9324ce04 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -662,7 +662,7 @@ then other IDLE, creation order within each; the controller passes live 32k deployment with nothing revived or started, and a resident idle deployment beats an older one that is gone (both fail on the old code). -### 19. [ ] `routes seed` fails closed on a redefinition; one route API +### 19. [x] `routes seed` fails closed on a redefinition; one route API *Why added:* the registry merge is documented as additive, but a same-name row with a different definition wins with a warning, and `routes seed` @@ -680,6 +680,19 @@ and asks. The CLI presents and confirms only. conflicting A refuses and leaves the registry unchanged; the override reports A updated. No private names from the CLI. +*Done 2026-09-27:* `leasing/routes.py` plans a seed (added, unchanged, +conflicted) or a prune (dropped); `Controller.plan_route_seed/prune` and +`commit_route_seed/prune` recheck under the lock and publish through +`publish_change`; the gateway exposes `route_registry()`, +`route_entries()`, `replace_route_entries()`. `routes seed` refuses a +redefinition and leaves the registry unchanged unless `--replace` (which +lists each change and asks on a terminal); its JSON says added, unchanged, +updated, conflicted. The CLI uses no private name. The render's own merge +keeps "incoming wins" (an edited catalog updates its route) and its +docstring now says so. Also fixed: under `--json` the conflict list went to +stdout and broke the JSON. Test: seed, identical seed, refused conflict, +`--replace` (fails on the old code). + ### 20. [ ] Preview does not write secrets *Why added:* rendering calls `master_key()`, `db_password()`, diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index c320d68b..6f0707ce 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2569,19 +2569,15 @@ def main(cls, argv=True, **kwargs): # --------------------------------------------------------------------------- -def _require_compose_backend(controller): - """What holds the gateway's route registry (its front door). - - The stack itself on the compose backend; on kubeai the gateway, on this - host or in the cluster. A SystemExit for a backend with no gateway (null, - or ``litellm false``).""" - project = getattr(controller.backend, 'front_door', lambda: None)() - if project is None or not getattr(project, 'litellm', False): - raise SystemExit( - 'the `routes` commands need a LiteLLM gateway (the compose or kubeai ' - 'backend, with `litellm` on)' - ) - return project +def _routes_or_exit(call): + """Run a route-registry operation; its refusals become CLI errors.""" + from ..leasing.profile import ProfileMismatch + from ..leasing.routes import RouteConflict + + try: + return call() + except (ProfileMismatch, RouteConflict) as ex: + raise SystemExit(f'routes: {ex}') def _live_endpoints(controller) -> set[str]: @@ -2616,8 +2612,7 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) - backend = _require_compose_backend(controller) - registry = backend.gateway._load_route_registry() + registry = _routes_or_exit(controller.route_registry) entries = registry.get('entries', {}) live = _live_endpoints(controller) @@ -2681,70 +2676,30 @@ class RoutesPruneCLI(_ApprovalMixin): @classmethod def main(cls, argv=True, **kwargs): - from ..diff_prompt import confirm_writes from ..leasing.backend import ConvergeAborted - from ..leasing.gateway import LITELLM_REGISTRY_VERSION, _dump_route_registry config = cls.cli(argv=argv, data=kwargs) # interactive=False so reconcile auto-applies the compose diff; the # meaningful gate (which routes get dropped) is confirmed here instead. controller = _open_controller(config, interactive=False) - backend = _require_compose_backend(controller) - - def prune_plan() -> tuple[dict, dict, list[str]]: - # The desired set exactly as the next render sees it, and the rows - # that render merges: the catalog's and the live deployments'. - # Any: _require_compose_backend above confirmed a gateway, and - # both backends that have one supply route rows. - engines: Any = controller.backend - desired, inputs = controller._admission_view(engines.residency()) - keep = dict(engines.route_rows(desired, inputs)) - current = backend.gateway._load_route_registry().get('entries', {}) - return current, keep, sorted(set(current) - set(keep)) - - # Preview outside the lock (the prompt must not hold it); the change - # itself is recomputed under the lock below. - _, keep, dropped = prune_plan() - if not dropped: + # Planned outside the lock (the prompt must not hold it); the commit + # rechecks under the lock and drops only what is still unneeded. + plan = _routes_or_exit(controller.plan_route_prune) + if not plan.dropped: print('routes prune: nothing to drop (registry already minimal)') return 0 - - pruned = {'version': LITELLM_REGISTRY_VERSION, 'entries': keep} skip_prompt = bool(config.yes) or not sys.stdout.isatty() if not skip_prompt: print('routes prune will DROP these routes:') - for name in dropped: + for name in plan.dropped: print(f' - {name}') - ok = confirm_writes( - {backend.gateway._registry_file: _dump_route_registry(pruned)}, - assume_yes=False, - title='infer-stack routes prune', - ) - if not ok: + if input('drop them? [y/N] ').strip().lower() not in ('y', 'yes'): raise SystemExit('aborted: registry not pruned') - - confirmed = set(dropped) - - def change(): - # Under the lock: drop only routes that were confirmed AND are still - # unneeded now; anything that became needed meanwhile is kept. (No - # sweep: an expired-but-unswept deployment only keeps its routes.) - current, _, still = prune_plan() - drop = sorted(confirmed & set(still)) - entries = {k: v for k, v in current.items() if k not in drop} - with backend._converge_lock(): - backend._atomic_write( - backend.gateway._registry_file, - _dump_route_registry( - {'version': LITELLM_REGISTRY_VERSION, 'entries': entries}), - ) - return drop, sorted(entries) - try: - (dropped, kept), rec = controller.publish_change(change) + dropped, rec = _routes_or_exit(lambda: controller.commit_route_prune(plan)) except ConvergeAborted: raise SystemExit('aborted: compose changes not applied') - + kept = sorted(_routes_or_exit(controller.route_registry).get('entries', {})) if config.json: print(json.dumps({'dropped': dropped, 'kept': kept, 'publication_pending': rec.publication_pending}, indent=2)) @@ -2766,10 +2721,19 @@ class RoutesSeedCLI(_ApprovalMixin): Unlike a normal converge (which only ever merges the *invoking* process's own catalog), this folds in sibling catalogs you name explicitly. + + Adding an alias is additive; redefining one already in the registry + redirects its clients, so that is refused and nothing is written unless + ``--replace`` is given (which shows each change and asks on a terminal). """ __command__ = 'seed' + replace = kw.Value( + False, isflag=True, + help='Redefine aliases the registry already routes elsewhere.', + ) + catalogs = kw.Value( None, nargs='+', position=1, type=str, help='One or more catalog.yaml files whose endpoints to merge into the ' @@ -2785,50 +2749,53 @@ def main(cls, argv=True, **kwargs): paths = _collect_names(config.catalogs) if not paths: raise SystemExit('routes seed: name at least one catalog.yaml file') - # interactive=False: the reconcile auto-applies (seeding is additive, so - # there is no destructive gate to confirm). - controller = _open_controller(config, interactive=False) - backend = _require_compose_backend(controller) - # Any: a gateway exists (checked above), and its backend supplies rows. - engines: Any = controller.backend - - incoming: dict = {} + catalogs = [] for raw in paths: path = Path(raw).expanduser() if not path.exists(): raise SystemExit(f'catalog not found: {path}') try: - cat = Catalog.load(path) + catalogs.append(Catalog.load(path)) except CatalogError as ex: raise SystemExit(f'invalid catalog {path}: {ex}') - # The engine backend's rows for it: compose upstreams on compose, - # the cluster's Models on kubeai. - incoming.update(engines.catalog_route_rows(cat)) - if not incoming: + # interactive=False: the reconcile auto-applies; the gate that matters + # (a redefined alias) is decided here. + controller = _open_controller(config, interactive=False) + plan = _routes_or_exit(lambda: controller.plan_route_seed(catalogs)) + if not plan.incoming: raise SystemExit( 'routes seed: the named catalog(s) resolved no routable endpoints' ) - - def change(): - before = set(backend.gateway._load_route_registry().get('entries', {})) - backend.merge_route_registry(incoming) - return sorted(set(incoming) - before) - + if plan.conflicted: + # stderr under --json, which must stay one JSON document on stdout. + out = sys.stderr if config.json else sys.stdout + print(f'routes seed: {len(plan.conflicted)} alias(es) would be redefined:', + file=out) + for name, (old, new) in sorted(plan.conflicted.items()): + print(f' ~ {name}: {old} -> {new}', file=out) + if not config.replace: + raise SystemExit('routes seed: refused, the registry is unchanged ' + '(pass --replace to redefine them)') + if not (bool(config.yes) or not sys.stdout.isatty()): + if input('redefine them? [y/N] ').strip().lower() not in ('y', 'yes'): + raise SystemExit('aborted: registry unchanged') try: - added, rec = controller.publish_change(change) + done, rec = _routes_or_exit( + lambda: controller.commit_route_seed(plan, replace=bool(config.replace))) except ConvergeAborted: raise SystemExit('aborted: compose changes not applied') - + updated = sorted(done.conflicted) if config.replace else [] if config.json: print(json.dumps( - {'merged': sorted(incoming), 'added': added, + {'added': sorted(done.added), 'unchanged': done.unchanged, + 'updated': updated, 'conflicted': sorted(done.conflicted), 'publication_pending': rec.publication_pending}, indent=2 )) else: - print( - f'routes seed: merged {len(incoming)} route(s) ' - f'({len(added)} new): {", ".join(added) or "(all already present)"}' - ) + print(f'routes seed: {len(done.added)} added, {len(done.unchanged)} ' + f'unchanged, {len(updated)} updated' + + (f': {", ".join(sorted(done.added) + updated)}' + if done.added or updated else '')) return 3 if rec.publication_pending else 0 diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 09142efa..217e9a3f 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -260,6 +260,20 @@ def placement_notes(self) -> dict[str, list[str]]: residents that yielded their GPUs). Empty where nothing is placed.""" ... + def front_door(self) -> Any: + """What holds the LiteLLM gateway (its keys and route registry), or + ``None`` for a backend without one.""" + ... + + def route_rows(self, desired: list[Deployment], placement: Any = None + ) -> dict[str, dict[str, Any]]: + """The gateway route rows a render of ``desired`` merges in.""" + ... + + def catalog_route_rows(self, catalog: Any) -> dict[str, dict[str, Any]]: + """The route rows every endpoint of ``catalog`` would get.""" + ... + def reclaim_candidates(self, blocked: Deployment, idle: list[Deployment]) -> list[str]: """Which of ``idle`` could free room ``blocked`` is waiting for. @@ -467,6 +481,16 @@ def reclaim_candidates(self, blocked: Deployment, """One scheduling domain in-process: any idle deployment frees room.""" return [g.id for g in idle] + def front_door(self) -> Any: + return None # no gateway in-process + + def route_rows(self, desired: list[Deployment], placement: Any = None + ) -> dict[str, dict[str, Any]]: + return {} + + def catalog_route_rows(self, catalog: Any) -> dict[str, dict[str, Any]]: + return {} + def plan(self, desired: list[Deployment], placement: Any = None): """Which of ``desired`` fit; here, all of them, on no GPU.""" from .placement import GpuPlan diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 0032e4dc..51ee3430 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -31,6 +31,7 @@ from typing import Callable, Iterable, TypeVar from .backend import ApplyResult, HostRuntime, ServingBackend +from .routes import RoutePlan from .ledger import Ledger from .models import Deployment, DeploymentState, EndpointRequest, Lease, LeaseState @@ -1777,6 +1778,89 @@ def release_leases( rec = self._publish() return ReleaseLeasesOutcome(released, missing, idled, list(evicted), rec) + # -- the gateway's route registry (routes seed / prune) ---------------- + + def _route_gateway(self): + """The front door's gateway, or a ProfileMismatch saying there is none.""" + from .profile import ProfileMismatch + + front = self.backend.front_door() + if front is None or not getattr(front, 'litellm', False): + raise ProfileMismatch( + 'the `routes` commands need a LiteLLM gateway (the compose or ' + 'kubeai backend, with `litellm` on)') + return front.gateway + + def route_registry(self) -> dict: + """The route registry as stored (``routes list``).""" + return self._route_gateway().route_registry() + + def plan_route_seed(self, catalogs: Iterable) -> RoutePlan: + """What seeding these catalogs' routes would add, keep, or redefine.""" + from .routes import plan_seed + + gateway = self._route_gateway() + incoming: dict[str, dict] = {} + for catalog in catalogs: + incoming.update(self.backend.catalog_route_rows(catalog)) + return plan_seed(gateway.route_entries(), incoming) + + def commit_route_seed(self, plan: RoutePlan, *, replace: bool = False + ) -> tuple[RoutePlan, ReconcileResult]: + """Add the plan's routes and publish; redefine conflicts only with + ``replace``. Rechecked under the lock: a conflict that appeared since + the plan refuses too (:class:`~infer_stack.leasing.routes.RouteConflict`) + and nothing is written.""" + from .routes import RouteConflict, plan_seed + + gateway = self._route_gateway() + fresh = plan_seed(gateway.route_entries(), plan.incoming) + if fresh.conflicted and not replace: + raise RouteConflict(sorted(fresh.conflicted)) + + def change(): + now = plan_seed(gateway.route_entries(), plan.incoming) + if now.conflicted and not replace: + raise RouteConflict(sorted(now.conflicted)) + entries = gateway.route_entries() + entries.update(now.added) + if replace: + entries.update({name: new for name, (_, new) in now.conflicted.items()}) + if entries != gateway.route_entries(): + gateway.replace_route_entries(entries) + return now + + return self.publish_change(change) + + def plan_route_prune(self) -> RoutePlan: + """Which routes a prune drops: all but the catalog's and the live ones, + exactly as the next render would keep them.""" + from .routes import plan_prune + + gateway = self._route_gateway() + desired, inputs = self._admission_view(self.backend.residency()) + return plan_prune(gateway.route_entries(), self.backend.route_rows(desired, inputs)) + + def commit_route_prune(self, plan: RoutePlan) -> tuple[list[str], ReconcileResult]: + """Drop the plan's routes that are still unneeded, and publish. A route + that became needed since the plan is kept.""" + from .routes import plan_prune + + gateway = self._route_gateway() + confirmed = set(plan.dropped) + + def change(): + desired, inputs = self._admission_view(self.backend.residency()) + current = gateway.route_entries() + still = plan_prune(current, self.backend.route_rows(desired, inputs)).dropped + drop = sorted(confirmed & set(still)) + if drop: + gateway.replace_route_entries( + {k: v for k, v in current.items() if k not in drop}) + return drop + + return self.publish_change(change) + def publish_change(self, change: Callable[[], _T]) -> tuple[_T, ReconcileResult]: """Run ``change`` and publish, as one serialised desired-state mutation. diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 9d973a7a..6b4b06e4 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -310,12 +310,14 @@ def upstream_route(deployment_id: str, endpoint: str, served: str, def _merge_route_registry( existing: dict[str, Any], incoming: dict[str, dict[str, Any]] ) -> tuple[dict[str, Any], list[str]]: - """Merge ``incoming`` semantic rows into ``existing`` (append-only). - - Idempotent (merging identical rows is a no-op) and additive (never removes a - row). On a conflict — same key, different row — *incoming wins* and a warning - naming both definitions is emitted; the changed definition changes the - rendered bytes, which is the one justified recreate. The existing ``version`` + """Merge ``incoming`` semantic rows into ``existing`` (a render's merge). + + Idempotent (merging identical rows is a no-op) and never removes a row. On + a conflict (same key, different row) *incoming wins* with a warning: this + is how a render folds in the invoking catalog, so an endpoint whose + catalog entry was edited gets its new route, the one justified recreate. + Seeding other catalogs is not this: ``routes seed`` refuses a redefinition + (:mod:`infer_stack.leasing.routes`). The existing ``version`` is preserved (an unknown version merged under is not silently rewritten to the current schema; see :meth:`ComposeBackend._load_route_registry`).""" version = LITELLM_REGISTRY_VERSION @@ -1208,6 +1210,26 @@ def _save_route_registry(self, merged: dict[str, Any]) -> None: logger.info(' route registry: updated route(s): {}', ', '.join(updated)) self._atomic_write(self._registry_file, _dump_route_registry(merged)) + def route_registry(self) -> dict[str, Any]: + """The registry as stored: ``{'version': ..., 'entries': {...}}``.""" + registry = self._load_route_registry() + return registry if isinstance(registry, dict) else {} + + def route_entries(self) -> dict[str, dict[str, Any]]: + """The registry's entries: public alias -> route row.""" + entries = self.route_registry().get('entries') + return dict(entries) if isinstance(entries, dict) else {} + + def replace_route_entries(self, entries: dict[str, dict[str, Any]]) -> None: + """Write the registry's entries (under the converge flock, atomically); + its schema version is kept.""" + with self._converge_lock(): + existing = self._load_route_registry() + version = (existing.get('version', LITELLM_REGISTRY_VERSION) + if isinstance(existing, dict) else LITELLM_REGISTRY_VERSION) + self._atomic_write(self._registry_file, _dump_route_registry( + {'version': version, 'entries': dict(entries)})) + def merge_route_registry( self, incoming: dict[str, dict[str, Any]] ) -> dict[str, Any]: diff --git a/infer_stack/leasing/routes.py b/infer_stack/leasing/routes.py new file mode 100644 index 00000000..a8658dd9 --- /dev/null +++ b/infer_stack/leasing/routes.py @@ -0,0 +1,72 @@ +"""Planning changes to the gateway's route registry (``routes seed`` / ``prune``). + +The registry maps a public alias to the upstream it routes to. Adding an +alias is additive; changing where an existing alias points redirects every +client that uses it, so a seed that would redefine one refuses by default +and replaces it only when asked. A plan says which is which before anything +is written; :class:`~infer_stack.leasing.controller.Controller` commits it +through the normal publication. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +class RouteConflict(ValueError): + """A seed would redefine existing aliases; ``names`` are the conflicts.""" + + def __init__(self, names: list[str]): + self.names = list(names) + super().__init__( + f'routes seed would redefine {len(self.names)} existing route(s): ' + f'{", ".join(self.names)}; pass --replace to redefine them') + + +@dataclass +class RoutePlan: + """What a seed or prune would do to the registry's entries. + + ``added``: new aliases; ``unchanged``: identical rows already there; + ``conflicted``: aliases whose incoming row differs (``{name: (current, + incoming)}``); ``dropped``: aliases a prune removes. ``incoming`` keeps a + seed's rows so the commit can recheck them under the lock. + """ + + added: dict[str, dict[str, Any]] = field(default_factory=dict) + unchanged: list[str] = field(default_factory=list) + conflicted: dict[str, tuple[dict[str, Any], dict[str, Any]]] = field(default_factory=dict) + dropped: list[str] = field(default_factory=list) + incoming: dict[str, dict[str, Any]] = field(default_factory=dict) + + +def plan_seed(current: dict[str, dict[str, Any]], + incoming: dict[str, dict[str, Any]]) -> RoutePlan: + """Classify ``incoming`` rows against the registry's ``current`` entries. + + >>> plan = plan_seed({'a': {'up': 1}, 'b': {'up': 2}}, + ... {'a': {'up': 1}, 'b': {'up': 9}, 'c': {'up': 3}}) + >>> sorted(plan.added), plan.unchanged, sorted(plan.conflicted) + (['c'], ['a'], ['b']) + """ + plan = RoutePlan(incoming=dict(incoming)) + for name in sorted(incoming): + row = incoming[name] + if name not in current: + plan.added[name] = row + elif current[name] == row: + plan.unchanged.append(name) + else: + plan.conflicted[name] = (current[name], row) + return plan + + +def plan_prune(current: dict[str, dict[str, Any]], + keep: dict[str, dict[str, Any]]) -> RoutePlan: + """Which of ``current`` a prune drops: everything not in ``keep``. + + >>> plan_prune({'a': {}, 'b': {}}, {'a': {}}).dropped + ['b'] + """ + return RoutePlan(dropped=sorted(set(current) - set(keep))) diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 8f4b7a91..90aa82d5 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -1169,3 +1169,42 @@ def test_release_and_renew_name_a_missing_env_file(env, tmp_path): (tmp_path / 'lease.env').write_text('export OPENAI_BASE_URL=x\n') with pytest.raises(SystemExit, match='names no lease'): ReleaseCLI.main(argv=['--ledger', env.db, '--env-file', missing]) + + +def test_routes_seed_adds_but_refuses_to_redefine_without_replace(tmp_path, monkeypatch, capsys): + """Queue item 19: seeding a new alias is additive; seeding a different + definition of an existing alias redirects its clients, so it refuses by + default and leaves the registry as it was.""" + from infer_stack.cli.commands_leasing import RoutesSeedCLI + + state = tmp_path / 'state' + state.mkdir() + db = str(tmp_path / 'ledger.db') + _patch_backend(monkeypatch, state) + first = tmp_path / 'a.yaml' + first.write_text(yaml.safe_dump(_one_endpoint_catalog('alpha'))) + other = _one_endpoint_catalog('alpha') + other['endpoints']['alpha']['served_name'] = 'alpha-v2' # alpha, routed elsewhere + second = tmp_path / 'b.yaml' + second.write_text(yaml.safe_dump(other)) + + def seed(*args): + capsys.readouterr() + rc = RoutesSeedCLI.main(argv=['--ledger', db, *args, '--json', '--yes']) + return rc, json.loads(capsys.readouterr().out) + + rc, out = seed(str(first)) + assert rc == 0 and out['added'] == ['alpha'] + registry = (state / 'litellm_registry.json').read_text() + + rc, out = seed(str(first)) # identical: a no-op + assert out['added'] == [] and out['unchanged'] == ['alpha'] + assert (state / 'litellm_registry.json').read_text() == registry + + with pytest.raises(SystemExit, match='refused'): + RoutesSeedCLI.main(argv=['--ledger', db, str(second), '--yes']) + assert (state / 'litellm_registry.json').read_text() == registry # unchanged + + rc, out = seed(str(second), '--replace') + assert rc == 0 and out['updated'] == ['alpha'] + assert (state / 'litellm_registry.json').read_text() != registry From 9fa01162a107ba6462471a095fb698eb89bfce2c Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:38:09 -0400 Subject: [PATCH 079/141] A preview writes no secrets Admission previews the render, and the render names the gateway's secrets, which used to be generated and written on first use: a refused acquire left a new .env. A preview now stages missing secrets in memory; the commit's render writes those same values, and fingerprints read the staged values too, so the approved digest still matches. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 11 ++++- infer_stack/backends/kubeai_gateway.py | 3 +- infer_stack/leasing/compose.py | 19 +++++--- infer_stack/leasing/gateway.py | 62 ++++++++++++++++++++------ tests/test_leasing_admission.py | 24 ++++++++++ 5 files changed, 96 insertions(+), 23 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 9324ce04..c74c49ec 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -693,7 +693,7 @@ docstring now says so. Also fixed: under `--json` the conflict list went to stdout and broke the JSON. Test: seed, identical seed, refused conflict, `--replace` (fails on the old code). -### 20. [ ] Preview does not write secrets +### 20. [x] Preview does not write secrets *Why added:* rendering calls `master_key()`, `db_password()`, `webui_secret()`, which generate and write missing secrets, so a preview @@ -705,6 +705,15 @@ secrets with the publication. If kept, document the narrower guarantee. **Done when:** a refused acquire on a fresh data root leaves no `.env`, or the docs say exactly what a preview may write. +*Done 2026-09-27:* the gateway's secrets go through one `_managed_secret`; +inside `staging_secrets()` (Compose `preview`, the in-cluster gateway's +`preview`) a missing secret is generated in memory and not written, and the +next writing call persists that same value. Fingerprints read +`managed_env()` (file plus staged), so the commit's render matches the +preview's digest. Test: a refused acquire on a fresh root leaves no key in +`.env` (fails on the old code: it wrote the master key and the Open WebUI +secret), and the next admitted acquire needs no second approval. + ### 21. [ ] Controller decomposition, where the authorities now show it *Why added:* the controller holds admission, residency interpretation, diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index 80415f8e..ab8a6184 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -183,7 +183,8 @@ def _render_documents(self) -> tuple[dict[Path, str], dict[str, Any]]: def preview(self, desired=(), placement=None, *, approve: bool = False): """Render without writing; with ``approve``, show the diff now.""" - planned, _ = self._render_documents() + with self.gateway.staging_secrets(): + planned, _ = self._render_documents() self._preview_approval(planned, approve=approve) return None, None diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index cac3bb36..d79605d3 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -252,6 +252,7 @@ class RenderedCompose: def stamp_fingerprints( compose: dict[str, Any], *, files: dict[Path, str], env_file: Path | None = None, + env_values: dict[str, str] | None = None, ) -> dict[str, str]: """Label every service with a behavioural fingerprint; return service -> fingerprint. @@ -263,7 +264,9 @@ def stamp_fingerprints( recreate everything on every apply.) ``files`` holds content about to be written (path -> text); a mounted file - that is not in it is read from disk if it exists. + that is not in it is read from disk if it exists. ``env_values``, when + given, are the managed ``.env``'s effective values (a preview's staged + secrets included) and stand in for reading ``env_file``. Example: >>> doc = {'services': {'a': {'image': 'x', 'labels': {}}}} @@ -277,11 +280,12 @@ def stamp_fingerprints( import re by_path = {str(Path(p)): text for p, text in files.items()} - env_values: dict[str, str] = {} - if env_file is not None and Path(env_file).exists(): - from ..env_utils import parse_env_file + if env_values is None: + env_values = {} + if env_file is not None and Path(env_file).exists(): + from ..env_utils import parse_env_file - env_values = parse_env_file(Path(env_file)) + env_values = parse_env_file(Path(env_file)) out: dict[str, str] = {} for name, svc in (compose.get('services') or {}).items(): labels = dict(svc.get('labels') or {}) @@ -1487,7 +1491,8 @@ def preview(self, desired: list[Deployment], placement=None, *, approve: bool = :class:`ConvergeAborted` on decline); the render that follows the commit then does not ask again as long as it produces the same files. """ - docs = self._render_documents(list(desired), placement) + with self.gateway.staging_secrets(): # a preview writes nothing + docs = self._render_documents(list(desired), placement) self._preview_approval(docs['planned'], approve=approve) return docs['plan'], docs['rendered'] @@ -1549,7 +1554,7 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A if rendered.litellm_routes is not None: planned[self.gateway._routes_file] = json.dumps(rendered.litellm_routes, indent=2) fingerprints = stamp_fingerprints( - rendered.compose, files=planned, env_file=self.gateway._env_path, + rendered.compose, files=planned, env_values=self.gateway.managed_env(), ) planned[self.compose_file] = yaml.safe_dump(rendered.compose, sort_keys=False) return {'plan': plan, 'rendered': rendered, 'planned': planned, diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 6b4b06e4..d463b33b 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -1052,6 +1052,51 @@ def _routes_file(self) -> Path: def _registry_file(self) -> Path: return self.state_dir / LITELLM_REGISTRY_FILENAME + #: Secrets a preview generated and has not written (see staging_secrets). + _staged: dict[str, str] | None = None + _staging_depth: int = 0 + + def staging_secrets(self): + """While active, a missing managed secret is generated in memory, not + written: a preview (admission, the idle-host feasibility check) leaves + the ``.env`` as it was, even when the admission is then refused. The + next writing call persists the same value, so the commit's render + matches what the preview approved.""" + import contextlib + + @contextlib.contextmanager + def staging(): + self._staging_depth += 1 + try: + yield + finally: + self._staging_depth -= 1 + + return staging() + + def managed_env(self) -> dict[str, str]: + """The managed ``.env`` as a render sees it: the file, plus secrets a + preview has staged and not yet written (so a preview's fingerprints + match the commit's render).""" + values = parse_env_file(self._env_path) if self._env_path.exists() else {} + return {**(self._staged or {}), **values} + + def _managed_secret(self, name: str, *, prefix: str = '') -> str: + """A secret from the managed ``.env``, made on first use (see + :meth:`staging_secrets` for when "made" is not yet "written").""" + existing = parse_env_file(self._env_path) + if existing.get(name): + return existing[name] + if self._staged is None: + self._staged = {} + value = self._staged.get(name) or ensure_secret({}, name, prefix=prefix) + if self._staging_depth: + self._staged[name] = value + else: + write_env_file(self._env_path, {name: value}) + self._staged.pop(name, None) + return value + def master_key(self) -> str: """The managed LiteLLM master key. @@ -1061,11 +1106,7 @@ def master_key(self) -> str: — it is baked into the LiteLLM service, used by the readiness probe, and shipped in the env-file descriptor (``infer-stack env KEY`` prints it). """ - existing = parse_env_file(self._env_path) - key = ensure_secret(existing, API_KEY_ENV, prefix='sk-') - if key != existing.get(API_KEY_ENV): - write_env_file(self._env_path, {API_KEY_ENV: key}) - return key + return self._managed_secret(API_KEY_ENV, prefix='sk-') def rotate_master_key(self) -> dict[str, str | None]: """Write a fresh master key to the ``.env``; return the values it replaced. @@ -1113,11 +1154,7 @@ def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: def webui_secret(self) -> str: """Open WebUI's managed session key (the .env), made on first use.""" - existing = parse_env_file(self._env_path) - key = ensure_secret(existing, WEBUI_SECRET_ENV) - if key != existing.get(WEBUI_SECRET_ENV): - write_env_file(self._env_path, {WEBUI_SECRET_ENV: key}) - return key + return self._managed_secret(WEBUI_SECRET_ENV) def db_password(self) -> str: """The managed Postgres password for LiteLLM's model store. @@ -1129,10 +1166,7 @@ def db_password(self) -> str: Only used when ``dynamic_routing`` is on. ``token_urlsafe`` output is safe inside the ``postgresql://`` URL (no ``@ : /`` characters). """ - existing = parse_env_file(self._env_path) - pw = ensure_secret(existing, DB_PASSWORD_ENV) - if pw != existing.get(DB_PASSWORD_ENV): - write_env_file(self._env_path, {DB_PASSWORD_ENV: pw}) + pw = self._managed_secret(DB_PASSWORD_ENV) return pw def _load_route_registry(self) -> dict[str, Any]: diff --git a/tests/test_leasing_admission.py b/tests/test_leasing_admission.py index d3097b95..eb70b67d 100644 --- a/tests/test_leasing_admission.py +++ b/tests/test_leasing_admission.py @@ -500,3 +500,27 @@ def partial(): assert ledger.publication_pending()['approved_digest'] == approved ctl.apply_now() # the explicit re-approval assert ledger.publication_pending() is None + + +def test_a_refused_acquire_writes_no_secret(tmp_path): + """Queue item 20: admission previews the render, and the render names the + gateway's secrets; a preview must not create them. A refused acquire on a + fresh data root leaves no .env; the next admitted one writes the key its + preview staged, so the commit's render matches what was approved.""" + from infer_stack.leasing.backend import PlacementError + from infer_stack.env_utils import parse_env_file + + state = tmp_path / 'state' + backend = ComposeBackend(state_dir=state, inventory=simulate_inventory('1x80'), + run=FakeDocker(), http=FakeHttp(state), images=IMAGES, + ports=PORTS, state=STATE, catalog=CAT, litellm=True, ui=True) + ledger = Ledger(SqliteStore(str(tmp_path / 'ledger.db')), clock=Clock()) + ctl = Controller(ledger, backend) + env = backend.gateway._env_path + with pytest.raises(PlacementError): + acquire(ctl, 'big') # two GPUs on a one-GPU host + assert not env.exists() or 'LITELLM_MASTER_KEY' not in parse_env_file(env) + + acquire(ctl, 'one') + assert parse_env_file(env)['LITELLM_MASTER_KEY'] == backend.master_key() + assert ledger.publication_pending() is None # no second approval needed From 7e531627c9f632ec6ff853472a1e0ca0b42525df Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:46:25 -0400 Subject: [PATCH 080/141] Public seams instead of private reaches: lock, profile, marker The CLI takes controller.publication_lock() and invocation_profile() instead of its private members; the controller writes the publication marker's approval through Ledger, not its store. Queue item 21 records why no class was extracted: a split would not remove a cross-layer call yet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 19 ++++++++++++++++++- infer_stack/cli/commands_leasing.py | 9 ++++----- infer_stack/leasing/controller.py | 19 +++++++++++++++---- infer_stack/leasing/ledger.py | 16 ++++++++++++++++ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index c74c49ec..a204d54d 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -714,7 +714,7 @@ preview's digest. Test: a refused acquire on a fresh root leaves no key in `.env` (fails on the old code: it wrote the master key and the Open WebUI secret), and the next admitted acquire needs no second approval. -### 21. [ ] Controller decomposition, where the authorities now show it +### 21. [x] Controller decomposition, where the authorities now show it *Why added:* the controller holds admission, residency interpretation, placement, publication markers, recovery snapshots, network migration, @@ -725,6 +725,23 @@ profile snapshots, backend capabilities), and only after 14-20. **Done when:** each extracted part needs no private cross-layer call, or the item records why nothing was extracted. +*Decided 2026-09-27: no class extracted; the cross-layer calls removed +instead.* After items 11-20 the controller calls no private backend member +and builds no runtime command; the CLI's two private reaches +(`controller._global_lock`, `_invocation_profile`) are now +`publication_lock()` and `invocation_profile()`; the publication marker's +approval is written through `Ledger` (`clear_approved_digest`, +`publish_profile`, `migrate_network`), not `ledger.store`. A split would not +remove a call today: the publication coordinator needs `_render` and +`_admission_view`, and admission needs the profile snapshot, so each part +would call back into the others. The boundaries the review names +(publication/recovery, admission/reclaim, profile snapshots, backend +capabilities) are where to cut when one of them grows. Left as it was: 15 +optional-hook `getattr(self.backend, ...)` reads that predate this section +(`use_profile`, `render_profile`, `settle_snapshot`, the `last_*` +attributes, ...). They could become protocol members with in-process +defaults, as `front_door` and `route_rows` did here. + ### 22. [ ] Docs and full verification *Why added:* the parity matrix, roadmap and `known-limitations.md` describe diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 6f0707ce..c7e6c0b3 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2502,7 +2502,7 @@ def main(cls, argv=True, **kwargs): # each other's keys. ledger = Ledger(SqliteStore(str(default_ledger_path()))) controller = Controller(ledger, NullBackend()) - with controller._global_lock(): + with controller.publication_lock(): if key in _SERVICE_ENV_KEYS: leases, _ = ledger.status(virtual_expiry=True) active = [le.id for le in leases if le.state == LeaseState.ACTIVE] @@ -2837,12 +2837,11 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config, interactive=True) - render_profile = getattr(controller.backend, 'render_profile', None) - if render_profile is None: - raise SystemExit('config publish: this backend has no render profile') # The profile this invocation resolves to, before the backend was # switched to the published one (see Controller._sync_profile). - profile = controller._invocation_profile or render_profile() + profile = controller.invocation_profile() + if profile is None: + raise SystemExit('config publish: this backend has no render profile') paths = _collect_names(config.catalogs) if paths: sources = [] diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 51ee3430..fd493eb1 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -380,6 +380,17 @@ def _lock_path_reason(self, path: Path) -> str: except OSError as exc: return f'inspection failed: {exc}' + def publication_lock(self): + """Hold while changing state a render reads outside the ledger (the + managed ``.env``): the same host-wide lock every publication holds.""" + return self._global_lock() + + def invocation_profile(self) -> dict | None: + """The recovery profile this invocation resolves to (its settings and + catalogs), before any switch to a stored one; ``None`` without one.""" + render = getattr(self.backend, 'render_profile', None) + return self._invocation_profile or (render() if render is not None else None) + @contextlib.contextmanager def _global_lock(self): """Serialize desired-state publication, single-writer. @@ -651,7 +662,7 @@ def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResul self.backend.preview(desired, inputs, approve=True) finally: host.configure_network(saved) - self.ledger.store.migrate_network( + self.ledger.migrate_network( subnet=subnet, reset_addresses=reset, approved_digest=getattr(self.backend, 'last_preview_digest', None), ) @@ -1272,7 +1283,7 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: else 'runtime not reached')) return rec if approved: - self.ledger.store.clear_approved_digest() + self.ledger.clear_approved_digest() self.ledger.clear_publication_pending(marker['version']) rec.publication_pending = False return rec @@ -1483,7 +1494,7 @@ def _rollback_acquire(self, lease_id: str, *, apply: bool) -> ReconcileResult | self._mark_pending(apply=apply) # The approved admission is being compensated away: its digest no # longer describes the pending desired state. - self.ledger.store.clear_approved_digest() + self.ledger.clear_approved_digest() rel = self.ledger.release(lease_id) if rel.idled_deployment_ids: never_ran = self._never_ran(list(rel.idled_deployment_ids)) @@ -1938,7 +1949,7 @@ def publish_profile(self, profile: dict) -> ReconcileResult: if previous is not None: use(previous) raise - self.ledger.store.publish_profile( + self.ledger.publish_profile( profile, approved_digest=self.backend.last_preview_digest) self._profile_error = None self._applied_profile = profile diff --git a/infer_stack/leasing/ledger.py b/infer_stack/leasing/ledger.py index 290093b3..a684848e 100644 --- a/infer_stack/leasing/ledger.py +++ b/infer_stack/leasing/ledger.py @@ -222,6 +222,22 @@ def cost(deployment: Deployment, pending: bool) -> int: state_version=state_version, ) + # -- the publication marker's approval (see Controller._apply_pending) ------ + + def clear_approved_digest(self) -> None: + """The approved render was applied, or deliberately abandoned.""" + self.store.clear_approved_digest() + + def publish_profile(self, profile: dict, *, approved_digest: str | None) -> None: + """Write the recovery profile and its pending, approved marker at once.""" + self.store.publish_profile(profile, approved_digest=approved_digest) + + def migrate_network(self, *, subnet: str, reset_addresses: bool, + approved_digest: str | None) -> None: + """Switch the stable-address subnet and mark it pending, at once.""" + self.store.migrate_network(subnet=subnet, reset_addresses=reset_addresses, + approved_digest=approved_digest) + def acquire( self, owner: str, From e9e1cf9232d66bc0c55c8a644386b39e6f8375c6 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:47:44 -0400 Subject: [PATCH 081/141] Docs follow the hardened semantics; the replacement-gap poison case Parity matrix rows for replicas, residency conflicts, reclaim, apply results, route ordering, seed conflicts and preview purity; the roadmap's principle that parity is an invariant under backend features, and its authorities table; known-limitations no longer says KubeAI lacks strict residency. CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 21 +++++++++++++++++ docs/backend-parity.md | 13 +++++++---- docs/planning/backend-parity-roadmap.md | 15 ++++++++++++- docs/planning/known-limitations.md | 24 ++++++++++---------- docs/queue.md | 16 ++++++++++++- tests/test_leasing_dynamic_routing.py | 30 +++++++++++++++++++++++++ 6 files changed, 101 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f26b622..b59a9ca8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### Hardening from a review of the backend-parity work + +- A KubeAI Model with several replicas (`min_replicas`) is resident like one + pod: releasing its last keep-warm lease no longer prunes it, `leases` shows + it running, and it fails fast when every replica crash-loops. +- On KubeAI, a pod the scheduler cannot place evicts an idle Model only when + the scheduler reports a capacity shortage, and only one on a node the pod + could use; an impossible node selector or a taint evicts nothing. +- An apply that reached the runtime but not its routes keeps the publication + and its approved render pending, so a render that changes before the retry + (an upgrade) must be approved again. KubeAI's apply reports its host + gateway's result instead of dropping it. +- `secrets rotate` restores the old key when the publication fails before + its apply began. +- With dynamic routing, routes are removed and verified before their + upstreams are stopped. +- An acquire coalesces onto an adequate live deployment before reviving an + older idle one. +- `routes seed` refuses to redefine an existing alias unless `--replace`. +- A preview (admission) no longer writes the gateway's secrets. + ### `/dev/shm` for vLLM on Compose, and the old recipes as suggestions `runtime.shm_size` (e.g. `16g`) sets a vLLM container's `/dev/shm` on the diff --git a/docs/backend-parity.md b/docs/backend-parity.md index b2fbeed5..ef136b3a 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -62,7 +62,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | one `OPENAI_BASE_URL`, the managed key, the alias as model name | same | same through the gateway; with `litellm false`, Model names and no key | -| `secrets rotate` | same | same | +| `secrets rotate` | same; a failure before the apply began restores the old key, after it the pending publication converges to the new one | same | | env file (`INFER_STACK_*`, `OPENAI_*`) | same | same | | readiness is a real generation through the front door | same | same | @@ -74,11 +74,14 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | admission: previewed in memory, then the lease committed with its allocation; a refused acquire writes nothing | yes | same; the allocation is empty (the cluster places), so admission decides renderability, and a pod the cluster cannot place is a wait reason | | `--queue` | waits for a free GPU | ≈ admitted at once, the cluster is the queue; an unrenderable endpoint fails at once | | `config publish` | pure preview, then commit | same | +| a preview (admission, `config publish`) writes nothing, not even a first secret | same | same | +| an apply reports how far it got (runtime, routes); the publication and its approved render stay pending until both | same | same, the host gateway's routes included | | `--no-apply` / `apply` / `render` | same | same | | unleased keep-warm yields to leased demand | at placement | ≈ during the wait, one per 30 s: only when the scheduler reports a capacity shortage, and only an idle Model on a node the pod could use, holding the resource that is short (longest idle first) | | crash-loop fail-fast, the engine's error quoted | same | same (pods, `kubectl logs --previous`) | | image-pull progress | yes | n/a: the kubelet pulls; `ImagePullBackOff` is a reported wait reason | -| strict `residency()` for decisions, lenient `observe()` for reports | same | same | +| strict `residency()` for decisions, lenient `observe()` for reports | same: two containers for one deployment are a conflict, and decisions fail closed | same: several pods for one Model are its replicas (or a rollout), resident while any is warm | +| coalescing: an adequate LIVE deployment before a resident IDLE one before any other IDLE one | same | same | | recovery after an interrupted apply (settle check) | yes | ≈ `kubectl apply` is idempotent; nothing is left half-created | ### Placement @@ -101,7 +104,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | served-name rules | same | same (`leasing/naming.py`) | | `runtime.command`, `runtime.mounts` (custom launchers) | yes | boundary: a stock vLLM Model has no place for them; the render refuses loudly | | ollama endpoints (`runtime_hosts`) | yes | boundary: daemon-shaped, not model-shaped | -| `min_replicas` / `max_replicas` | n/a | pass-through, default 1/1 | +| `min_replicas` / `max_replicas` | n/a | pass-through, default 1/1; a replicated Model is kept warm, pinned, shown running and diagnosed as one (fatal when every replica crash-loops) | | weight cache | the host HF cache, mounted | whatever the chart provides; cache profiles are not rendered | ### Gateway @@ -109,8 +112,9 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | static superset routes, no blip on model churn | same | same | -| `routes` inspect / seed / prune | yes | same; a route points at the cluster under the Model's name | +| `routes` inspect / seed / prune | yes; `seed` refuses to redefine an alias without `--replace` | same; a route points at the cluster under the Model's name | | `dynamic_routing` (admin API + Postgres; distinct upstreams for same-model `--dedicated`) | yes | same: each deployment is its own Model (`-`) | +| a departing upstream outlives its route: routes are removed and verified before engines or Models go | yes | same with the host gateway; the in-cluster gateway's static routes have nothing to remove | | Open WebUI (`ui`), reverse proxy | yes | same, in the gateway's project on this host | | the gateway's changes approved with the acquire's, before the lease commits | yes | same | | `network migrate` / `network check` | yes | n/a: Service addresses are stable | @@ -135,6 +139,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. |---|---|---| | unit suite with fake runtimes | yes | yes | | parity suite (`tests/test_parity.py`: each *same* row, both backends) | yes | yes | +| cross-feature poison cases (a backend feature combined with the shared lifecycle: replicas, rollouts, scheduler pressure, partial applies, route failures, coalescing, route conflicts) | yes | yes | | end to end | `dev/e2e_tests/run.sh` (tiers; `--gpu` for serving) | `dev/kubeai_e2e.sh` against a real cluster; k3s with CPU vLLM needs no GPU | ## Deviations that stay diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 31454cd0..d653f2ac 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -30,6 +30,14 @@ it is not a sibling with its own habits. ## Principles +- **Parity is an invariant, not a row.** A row reading *same* for a verb or + an output shape is not enough: every shared lifecycle invariant must hold + when combined with the backend-specific features that stress it (replicas, + scheduler pressure, partial applies, route failures). Each such feature + gets at least one test combining it with the common lifecycle; the + 2026-09-27 review found a replicated keep-warm Model pruned on release + although both of those rows passed alone. + - **Superset, not sibling.** KubeAI gains what Compose has. Where a Compose feature has no cluster meaning (GPU indices, `network migrate`), the matrix says *n/a* and the CLI says why; it does not grow a second design. @@ -282,7 +290,12 @@ worse. A blocker is fixed whatever its size. | KubeAI's route rows | written into the registry before any approval, and only for live Models (so every new Model recreated the gateway) | **fixed** (P3): render inputs, persisted after approval; the catalog's rows too, so no blip | | a route's upstream | derived twice, in the render and in `routes list` (which showed `?` for a cluster route) | **fixed** (P3): `registry_route_entry` | | a deployment's Model name | derived in five places | **fixed** (P3): `model_name()` | -| "is this the compose backend" in the CLI | `isinstance(…, ComposeBackend)` for `routes`, `gc --orphans`, `clean`, `network migrate`, `secrets rotate` | **fixed** (P3): the capability each needs (`compose_project()`, residency labels, `network`, `litellm`) | +| "is this the compose backend" in the CLI | `isinstance(…, ComposeBackend)` for `routes`, `gc --orphans`, `clean`, `network migrate`, `secrets rotate` | **fixed** (P3): the capability each needs (`compose_project()`, residency labels, `network`, `litellm`); since item 12, `backend.host_runtime` | +| how many units a deployment may have | "more than one is ambiguous", written into the shared residency | **fixed** (item 11): `Residency.replicated`, set by the runtime that built it; only GPU adoption asks for a unique unit | +| the controller's backend contract | the old `realize/teardown` protocol, cast to one carrying Compose internals, probed with `hasattr` | **fixed** (item 12): `ServingBackend`, `HostRuntime` for Compose-only work, `Realizer` behind `SimpleAdmission`; `ty` checks each backend | +| whether an idle model can free room | the probe's reason (`Unschedulable`) alone | **fixed** (item 14): the scheduler's message says capacity, the backend's `reclaim_candidates` says which victim could help | +| how far an apply got | `True`/`False`/raise, and the approval cleared before looking | **fixed** (item 15): `ApplyResult`; the approval lives until the publication completes | +| a route registry change | the CLI wrote the registry file under the backend's lock itself | **fixed** (item 19): the controller plans and commits it; the gateway owns the file | ## Not in scope diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index 66fd77e0..0a73a9c6 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -132,13 +132,12 @@ kinds. ### A queued acquire holds nothing while it waits (current, by design) -With the Compose backend an acquire is admitted atomically: its lease and its -GPU allocations are committed together, or nothing is. A queued (`--queue`) -acquire that has not been admitted yet has written nothing, so it holds no GPU -and has no place in line (see "Admission is first-come" below). - -Backends without strict residency (KubeAI, the null backend) keep the earlier -behaviour: the lease is committed first and placed by the render. +On every backend an acquire is admitted atomically: its lease and its +allocations are committed together, or nothing is (on KubeAI, where the +cluster schedules, the allocation is empty and admission decides only +renderability). A queued (`--queue`) acquire that has not been admitted yet +has written nothing, so it holds no GPU and has no place in line (see +"Admission is first-come" below). ### Admission is first-come, not fair (design boundary) @@ -283,11 +282,12 @@ starved new leases. has no single running container (for example a GPU reservation, which runs nothing), it stays *unresolved*: it keeps working, but no new GPU is allocated to anyone until its lease is released. -- **Other backends.** On backends without strict residency (KubeAI, null), - idle keep-warm deployments stay in the desired set, as before. A failed - acquire whose rollback cannot read residency can then leave a leaseless idle - candidate that the next apply starts (workaround: `infer-stack evict - `). +- **Other backends.** KubeAI reads strict residency from pods: an idle + keep-warm Model stays in the desired set while it is resident (any of its + replicas warm), and on a scheduling failure the wait evicts only an idle + Model that could free the capacity that is short (see + [../backend-parity.md](../backend-parity.md)). The null backend reports + nothing running. ### Known fault: the gateway can route a model's traffic to another container (until `network migrate`) diff --git a/docs/queue.md b/docs/queue.md index a204d54d..2c5e1783 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -512,7 +512,7 @@ KubeAI's no-op methods are gone; `HostRuntime` (Compose: `ty` check each backend against the protocol (verified: removing KubeAI's `host_runtime` fails the check). `Backend` stays exported as an alias. -### 13. [ ] Parity tests: cross-feature invariants +### 13. [x] Parity tests: cross-feature invariants *Why added:* the parity suite's KubeAI fake ran one pod per Model, so "replicas render" and "release keeps keep-warm" each passed while their @@ -527,6 +527,20 @@ roadmap. **Done when:** the suite is in `tests/test_parity.py` (or beside it), each case fails on the code before its fix, and the roadmap states the rule. +*Done 2026-09-27:* the KubeAI fake runs `minReplicas` pods, injects rollout +pods, and has a scheduler mode (nodes, pending pods with a message). Poison +cases: replicated keep-warm, rollout snapshot, replicated crash-loop, +Compose duplicates (`tests/test_parity.py`); scheduler selector, taint, +wrong-node victim, reclaimable pressure (`tests/test_parity.py`); partial +apply with renderer drift and the KubeAI gateway result +(`test_leasing_admission.py`, `test_leasing_kubeai.py`); rotation at both +boundaries (`test_leasing_secrets.py`); route removal failure on Compose and +KubeAI, and a replacement gap (`test_leasing_dynamic_routing.py`); +LIVE-over-IDLE coalescing (`test_leasing_controller.py`); seed conflicts +(`test_cli_leasing.py`). Each failed on the code before its fix, except the +two positive controls (reclaimable pressure, the replacement-gap report). +The rule is in the roadmap's Principles. + ### 14. [x] Scheduler-aware reclaim, not "Unschedulable means evict" *Why added:* KubeAI's readiness sets `needs_room` whenever a pod waits with diff --git a/tests/test_leasing_dynamic_routing.py b/tests/test_leasing_dynamic_routing.py index 3f8ef952..3a74c335 100644 --- a/tests/test_leasing_dynamic_routing.py +++ b/tests/test_leasing_dynamic_routing.py @@ -613,3 +613,33 @@ def test_kubeai_keeps_a_departing_model_while_its_route_stays(tmp_path): gw.refuse = False assert be.apply().complete assert len(kubectl.applied) == 1 and len(_managed(gw)) == 1 + + +def test_a_replacement_whose_add_fails_is_reported_as_a_gap(tmp_path, monkeypatch): + """A drifted route is replaced by delete-then-add; if the add fails, the + alias has no route until the next apply. That is not hidden: the result is + unverified and the gap is logged by route id.""" + from infer_stack import _log + + class NoAddGateway(RecordingGateway): + refuse_add = False + + def post(self, url, **kw): + if self.refuse_add and url.endswith('/model/new'): + return FakeResp(500, {'error': 'db unavailable'}) + return super().post(url, **kw) + + time = FakeTime() + gw = NoAddGateway() + be = _timed_backend(tmp_path, gw, time) + a = dep('grp-aaaaaa', served='smol', t=0) + be.converge([a], apply=True) + rid = _route_id(a.id, 'smol') + gw.models[rid]['litellm_params']['model'] = 'ollama/wrong-tag' # drift + gw.refuse_add = True + seen = [] + monkeypatch.setattr(_log.logger, 'warning', lambda msg, *a, **k: seen.append(msg.format(*a))) + + assert be.gateway._reconcile_routes(deadline_s=10.0) is False + assert rid not in gw.models # the gap + assert any(rid in m and 'missing until the next apply' in m for m in seen) From de7dea81a3812af3b54c3643c703a4d9d6f96928 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:47:57 -0400 Subject: [PATCH 082/141] queue: the replacement-gap test fails on the old code too Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 2c5e1783..ede821df 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -537,8 +537,8 @@ apply with renderer drift and the KubeAI gateway result boundaries (`test_leasing_secrets.py`); route removal failure on Compose and KubeAI, and a replacement gap (`test_leasing_dynamic_routing.py`); LIVE-over-IDLE coalescing (`test_leasing_controller.py`); seed conflicts -(`test_cli_leasing.py`). Each failed on the code before its fix, except the -two positive controls (reclaimable pressure, the replacement-gap report). +(`test_cli_leasing.py`). Each failed on the code before its fix, except +the positive control for reclaimable pressure, which passes on both. The rule is in the roadmap's Principles. ### 14. [x] Scheduler-aware reclaim, not "Unschedulable means evict" From 2d8785b52feb897e92e1c40e7c313f57ef206514 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:48:27 -0400 Subject: [PATCH 083/141] dev: journal and a lesson for the review hardening Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/journals/claude.md | 35 +++++++++++++++++++++++++++++++++++ dev/lessons/lessons.md | 12 ++++++++++++ 2 files changed, 47 insertions(+) diff --git a/dev/journals/claude.md b/dev/journals/claude.md index f3b3bdae..118f278e 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3312,3 +3312,38 @@ remove the file to undo it. an audit: the script now refuses to start without `infer-stack` on PATH. (2) Read a report whole: the one pass-5 finding I nearly missed was a line my own `sed` range skipped. + +## 2026-09-26 18:30:00 -0400 — review hardening of the backend-parity campaign + +**Intent.** An outside review, in two parts, of the finished queue: fix what +it confirmed (replica residency, the backend protocol, cross-feature tests; +then scheduler reclaim, approval lifetime, secret rotation, route ordering, +coalescing, route seeding, preview purity, decomposition, docs), queueing +the work first. Model: Claude Opus 5.5 (claude-opus-5-5), Claude Code. + +**Did.** Queue items 11-22, each with a poison test that fails on the old +code. The common thread the review named held: several shared concepts +carried less than the controller needed. `Residency` now says whether +several units are replicas or a conflict; `Readiness.needs_room` is only +set for a capacity shortage, and the backend names useful victims; +`apply()` returns how far it got; routes retire before upstreams go. The +protocol the controller takes is the one the backends implement, checked by +`ty` (a deliberate drift fails it). + +**Found on the way.** KubeAI's `apply` dropped its host gateway's result, +so unverified routes behind KubeAI cleared the marker; `routes seed --json` +printed its conflict list into the JSON; a TUI test counted unrelated +refusals. + +**Uncertain.** The scheduler parsing reads Kubernetes' English message +("Insufficient "); a scheduler that words it differently evicts +nothing, which is the safe failure. Required node affinity is not +evaluated, also safe. The replica diagnosis quotes the concatenated pod +logs; with mixed failures it names the first fatal line it finds. + +**Takeaways.** (1) A "same" row is a claim about one path; the bug lived +where two rows met, so a backend feature needs one test with the common +lifecycle. (2) When a boolean crosses a boundary, ask what the caller does +with the cases it merges; `True/False` and `needs_room` each merged cases +that needed opposite actions. (3) Run long e2e passes from a frozen copy +(lessons.md). diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index cabd927c..3116e64d 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -141,3 +141,15 @@ evidence; prefer append-only; supersede incorrect entries with a new one. `dev/k3s_agent_container.sh` passes the same to the second node. - **Applies when:** a dev cluster shares its disk with anything else, and whenever a test or audit keeps large artifacts in `/tmp`. + +- **Lesson:** An e2e that runs `infer-stack` from an editable install imports + the live checkout, so editing the tree while it runs can break it midway. + Run long e2e passes from a copy: `cp -r infer_stack dev/... $SNAP` and + `PYTHONPATH=$SNAP` (it precedes the editable install's path entry). +- **Evidence / MWE:** 2026-09-27: a `dev/kubeai_e2e.sh` run failed at its + routes step with `ImportError: cannot import name 'Backend'` while the + backend protocol was being renamed, and its cleanup failed the same way, + leaving a Model and the gateway containers behind. The rerun from a copy + printed the copy's `infer_stack.__file__` and passed while edits continued. +- **Applies when:** a long e2e or audit runs in tmux while work continues in + the same checkout. From b1040f1de8fc4f04b6b0a83e2a818b7278f81295 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 18:48:43 -0400 Subject: [PATCH 084/141] docs: date the review hardening 2026-09-26, the day it happened Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/lessons/lessons.md | 2 +- docs/planning/backend-parity-roadmap.md | 6 +++--- docs/queue.md | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index 3116e64d..f96d8327 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -146,7 +146,7 @@ evidence; prefer append-only; supersede incorrect entries with a new one. the live checkout, so editing the tree while it runs can break it midway. Run long e2e passes from a copy: `cp -r infer_stack dev/... $SNAP` and `PYTHONPATH=$SNAP` (it precedes the editable install's path entry). -- **Evidence / MWE:** 2026-09-27: a `dev/kubeai_e2e.sh` run failed at its +- **Evidence / MWE:** 2026-09-26: a `dev/kubeai_e2e.sh` run failed at its routes step with `ImportError: cannot import name 'Backend'` while the backend protocol was being renamed, and its cleanup failed the same way, leaving a Model and the gateway containers behind. The rerun from a copy diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index d653f2ac..14f172af 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -4,7 +4,7 @@ `dev/backend-unification` · **features complete, review hardening outstanding**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover run on real hardware, `dev/handover/`); a two-part outside review on -2026-09-27 found replica residency, the backend protocol, scheduler reclaim, +2026-09-26 found replica residency, the backend protocol, scheduler reclaim, publication phases, secret rotation, route ordering, coalescing and route seeding to fix (queue items 11-22). Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). @@ -35,7 +35,7 @@ it is not a sibling with its own habits. when combined with the backend-specific features that stress it (replicas, scheduler pressure, partial applies, route failures). Each such feature gets at least one test combining it with the common lifecycle; the - 2026-09-27 review found a replicated keep-warm Model pruned on release + 2026-09-26 review found a replicated keep-warm Model pruned on release although both of those rows passed alone. - **Superset, not sibling.** KubeAI gains what Compose has. Where a Compose @@ -53,7 +53,7 @@ it is not a sibling with its own habits. ### P0. One contract, one liveness view, one diagnosis — landed, reopened -*Reopened 2026-09-27:* the contract `Controller` is typed against still +*Reopened 2026-09-26:* the contract `Controller` is typed against still inherits the pre-admission `realize/teardown` protocol and carries Compose mechanics (`run`, `_load_sidecar`, `network`, `adopted`); and the liveness view equates "more than one unit" with a conflict, which is wrong for diff --git a/docs/queue.md b/docs/queue.md index ede821df..769a2f63 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -430,7 +430,7 @@ Both keep their own config and data roots and print PASS/FAIL per step. were not re-run since leasing, and 8b's `shm_size` fixes a TP failure not yet observed here. -## Reopened 2026-09-27: outside review +## Reopened 2026-09-26: outside review Found by a two-part review of the finished campaign. Part 1: items 11-13. Part 2: items 14-22, in the order the review recommends; item 11 also @@ -476,7 +476,7 @@ replica crash-looping fails fast with the engine's error, one healthy replica does not; two Compose containers for one deployment still fail closed; the k3s e2e has a two-replica keep-warm phase and passes. -*Done 2026-09-27 (35be92c):* `Residency.replicated` (true for pods) with +*Done 2026-09-26 (35be92c):* `Residency.replicated` (true for pods) with `is_resident`, `is_conflicted`, `warm_units`, `resident_gpus`, and `unique_unit` kept for GPU adoption only; every caller audited; `diagnose_startup(..., replicated=)` fails fast only when every replica has @@ -503,7 +503,7 @@ satisfy a protocol; the controller builds no `docker` command and reads no sidecar; `ty` checks each backend against the protocol, and fails when one drifts. -*Done 2026-09-27 (35be92c):* `ServingBackend` is the controller's protocol +*Done 2026-09-26 (35be92c):* `ServingBackend` is the controller's protocol (no cast); `Realizer` holds `realize/teardown` for `SimpleAdmission` only; KubeAI's no-op methods are gone; `HostRuntime` (Compose: `network_table/configure_network`, `subnet_clashes`, `rendered_services`, @@ -527,7 +527,7 @@ roadmap. **Done when:** the suite is in `tests/test_parity.py` (or beside it), each case fails on the code before its fix, and the roadmap states the rule. -*Done 2026-09-27:* the KubeAI fake runs `minReplicas` pods, injects rollout +*Done 2026-09-26:* the KubeAI fake runs `minReplicas` pods, injects rollout pods, and has a scheduler mode (nodes, pending pods with a message). Poison cases: replicated keep-warm, rollout snapshot, replicated crash-loop, Compose duplicates (`tests/test_parity.py`); scheduler selector, taint, @@ -564,7 +564,7 @@ untolerated taint evicts nothing; with two profiles on two nodes the idle Model on the wrong node is not chosen; real reclaimable pressure evicts a compatible idle Model and the leased one proceeds. -*Done 2026-09-27:* KubeAI's probe sets `needs_room` only when the +*Done 2026-09-26:* KubeAI's probe sets `needs_room` only when the scheduler's message names an `Insufficient` resource (the message is now in residency); `ServingBackend.reclaim_candidates(blocked, idle)` names idle Models with a pod on a node the blocked pod could use (node selector, @@ -592,7 +592,7 @@ runtime and returns partial; the publication is pending with D1; the renderer changes to D2; an ordinary retry refuses on the digest mismatch; `infer-stack apply` re-approves and applies. -*Done 2026-09-27:* `apply()` returns an `ApplyResult` (`runtime`, `routes`, +*Done 2026-09-26:* `apply()` returns an `ApplyResult` (`runtime`, `routes`, `detail`; `None`/`True`/`False` still map onto it). The controller clears the approved digest and the marker only on a complete result. Found on the way: KubeAI's `apply` dropped its host gateway's result, so unverified @@ -617,7 +617,7 @@ tests for a declined approval, a render failure before apply (both: old file, old running key), and a failure after the gateway restarted (pending, converges on retry). -*Done 2026-09-27:* the controller records when an apply begins; a rotation +*Done 2026-09-26:* the controller records when an apply begins; a rotation whose publication fails before that restores the old key (any failure, not only a decline), and after it keeps the new key with the publication pending and interrupted. Tests use a gateway that accepts the key its @@ -643,7 +643,7 @@ two dedicated deployments for one alias leaves the departing upstream running (the invariant, not just `publication_pending`), on Compose and on KubeAI with the host gateway. -*Done 2026-09-27:* an apply's first phase, `retire_routes()`, deletes and +*Done 2026-09-26:* an apply's first phase, `retire_routes()`, deletes and verifies the routes the render drops (only with dynamic routing and a running gateway); if that fails the apply returns "runtime not reached" and tears nothing down. KubeAI retires before pruning stale Models; its @@ -669,7 +669,7 @@ residency hint from the controller. 4k request onto the live 32k deployment: nothing revived, no placement, no queue, no runtime change. -*Done 2026-09-27:* `Ledger.plan_acquire(requests, resident=)` ranks +*Done 2026-09-26:* `Ledger.plan_acquire(requests, resident=)` ranks adequate candidates LIVE, then created in this plan, then resident IDLE, then other IDLE, creation order within each; the controller passes `residency.is_resident`. Tests: the 8k/32k/4k sequence coalesces onto the @@ -694,7 +694,7 @@ and asks. The CLI presents and confirms only. conflicting A refuses and leaves the registry unchanged; the override reports A updated. No private names from the CLI. -*Done 2026-09-27:* `leasing/routes.py` plans a seed (added, unchanged, +*Done 2026-09-26:* `leasing/routes.py` plans a seed (added, unchanged, conflicted) or a prune (dropped); `Controller.plan_route_seed/prune` and `commit_route_seed/prune` recheck under the lock and publish through `publish_change`; the gateway exposes `route_registry()`, @@ -719,7 +719,7 @@ secrets with the publication. If kept, document the narrower guarantee. **Done when:** a refused acquire on a fresh data root leaves no `.env`, or the docs say exactly what a preview may write. -*Done 2026-09-27:* the gateway's secrets go through one `_managed_secret`; +*Done 2026-09-26:* the gateway's secrets go through one `_managed_secret`; inside `staging_secrets()` (Compose `preview`, the in-cluster gateway's `preview`) a missing secret is generated in memory and not written, and the next writing call persists that same value. Fingerprints read @@ -739,7 +739,7 @@ profile snapshots, backend capabilities), and only after 14-20. **Done when:** each extracted part needs no private cross-layer call, or the item records why nothing was extracted. -*Decided 2026-09-27: no class extracted; the cross-layer calls removed +*Decided 2026-09-26: no class extracted; the cross-layer calls removed instead.* After items 11-20 the controller calls no private backend member and builds no runtime command; the CLI's two private reaches (`controller._global_lock`, `_invocation_profile`) are now From b1f5f62590b62e33af419095037149810da42118 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 19:03:36 -0400 Subject: [PATCH 085/141] queue: items 11-22 done; roadmap status: review hardening done Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 15 ++++++++------- docs/queue.md | 10 +++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 14f172af..7ae6790c 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,12 +1,13 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **features complete, review hardening -outstanding**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover -run on real hardware, `dev/handover/`); a two-part outside review on -2026-09-26 found replica residency, the backend protocol, scheduler reclaim, -publication phases, secret rotation, route ordering, coalescing and route -seeding to fix (queue items 11-22). Execution order: [../queue.md](../queue.md). +`dev/backend-unification` · **features complete, review hardening done**: +phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover run on real +hardware, `dev/handover/`); a two-part outside review the same day found +replica residency, the backend protocol, scheduler reclaim, publication +phases, secret rotation, route ordering, coalescing and route seeding to +fix, done as queue items 11-22 (verified on k3s; item 21 records why the +controller was not split). Execution order: [../queue.md](../queue.md). **Current state:** [../backend-parity.md](../backend-parity.md). **Origin:** the scale-up run needs more than one workstation, and the KubeAI backend had drifted from Compose for three months before the @@ -51,7 +52,7 @@ it is not a sibling with its own habits. ## Phases -### P0. One contract, one liveness view, one diagnosis — landed, reopened +### P0. One contract, one liveness view, one diagnosis — done (reopened and closed 2026-09-26) *Reopened 2026-09-26:* the contract `Controller` is typed against still inherits the pre-admission `realize/teardown` protocol and carries Compose diff --git a/docs/queue.md b/docs/queue.md index 769a2f63..5bfb23e1 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -756,7 +756,7 @@ optional-hook `getattr(self.backend, ...)` reads that predate this section attributes, ...). They could become protocol members with in-process defaults, as `front_door` and `route_rows` did here. -### 22. [ ] Docs and full verification +### 22. [x] Docs and full verification *Why added:* the parity matrix, roadmap and `known-limitations.md` describe the semantics these items change; `known-limitations.md` still says KubeAI @@ -768,6 +768,14 @@ full suite, `ty`, flake8, and the full k3s e2e. **Done when:** those pass and the docs match the code. +*Done 2026-09-26:* the parity matrix, the roadmap (principle, authorities, +status) and `known-limitations.md` describe the new semantics; CHANGELOG +entry. Verified: the full suite, `ty` and flake8 pass; the full k3s e2e +(`E2E_SIZED=1 E2E_REMOTE_NODE=1`, dynamic routing and the in-cluster +gateway on by default, the new replica phase) passed from a frozen copy; +`dev/ux_audit.sh` reports 0 flags on both backends, and the only non-zero +exits are its deliberate mistakes. + Also observed while working on these: `test_an_edit_made_outside_the_tui_appears_on_the_next_refresh` failed in two of five full-suite runs and never alone. It counted every `_refuse` call after two refreshes, and a refresh's background worker can From 0da932b92415a515bf49ef7feb83d58e955890b3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 19:54:20 -0400 Subject: [PATCH 086/141] queue: reopen 11, 14, 15, 16, 21 and add 23 from the re-review Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 2 +- docs/queue.md | 62 +++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 7ae6790c..efa6daab 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,7 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **features complete, review hardening done**: +`dev/backend-unification` · **features complete; review hardening done, re-review items open (queue 11, 14, 15, 16, 21, 23)**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover run on real hardware, `dev/handover/`); a two-part outside review the same day found replica residency, the backend protocol, scheduler reclaim, publication diff --git a/docs/queue.md b/docs/queue.md index 5bfb23e1..bdd392b8 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -446,7 +446,7 @@ poison test that fails on the old code with every fix; no new `hasattr` probes; no second KubeAI lifecycle path; backwards compatibility for cards, catalogs and CLI invocations that work today. -### 11. [x] Residency: replicas are not duplicates +### 11. [ ] Residency: replicas are not duplicates (reopened by the re-review) *Why added:* `Residency.ambiguous()` meant "more than one unit" and `resident()` returned a unit only when there was exactly one. On Compose two @@ -541,7 +541,7 @@ LIVE-over-IDLE coalescing (`test_leasing_controller.py`); seed conflicts the positive control for reclaimable pressure, which passes on both. The rule is in the roadmap's Principles. -### 14. [x] Scheduler-aware reclaim, not "Unschedulable means evict" +### 14. [ ] Scheduler-aware reclaim, not "Unschedulable means evict" (reopened by the re-review) *Why added:* KubeAI's readiness sets `needs_room` whenever a pod waits with reason `Unschedulable`, and `wait_ready` answers by evicting the @@ -573,7 +573,7 @@ request the short resource. Compose returns none, the in-process backends all. `_make_room` keeps the policy. Four parity poison tests; three fail on the old code. -### 15. [x] Publication phases: the approved digest outlives a partial apply +### 15. [ ] Publication phases: the approved digest outlives a partial apply (reopened by the re-review) *Why added:* `_apply_pending()` clears the approved-render digest right after `apply()` returns, before checking it returned `False`. Compose @@ -600,7 +600,7 @@ routes behind KubeAI cleared the marker; it now returns both. Tests: the partial-apply/renderer-drift case (fails on the old controller: the digest was gone) and the KubeAI propagation. -### 16. [x] Secret rotation is a transaction +### 16. [ ] Secret rotation is a transaction (reopened by the re-review) *Why added:* `rotate_gateway_key()` writes the new key into `.env`, then publishes, and restores the old key only on `ConvergeAborted`. A strict @@ -728,7 +728,7 @@ preview's digest. Test: a refused acquire on a fresh root leaves no key in `.env` (fails on the old code: it wrote the master key and the Open WebUI secret), and the next admitted acquire needs no second approval. -### 21. [x] Controller decomposition, where the authorities now show it +### 21. [ ] Controller decomposition, where the authorities now show it (reopened by the re-review) *Why added:* the controller holds admission, residency interpretation, placement, publication markers, recovery snapshots, network migration, @@ -781,3 +781,55 @@ failed in two of five full-suite runs and never alone. It counted every `_refuse` call after two refreshes, and a refresh's background worker can refuse something unrelated meanwhile; it now counts catalog-reload refusals only (the likely cause, not reproduced on demand). + +## Re-review (2026-09-26): second-order cases + +The re-review closed most of items 11-22 and reopened five, for states the +new abstractions exposed but did not carry through. Order: 21's settlement +bug first (a safety bug), then 16, 15, 11, 14, 23, then the rest of 21. + +- **21, settlement.** `_wait_for_settled_runtime` reads + `getattr(backend, 'settle_snapshot')`; KubeAI has none, but its host + gateway is a Compose runtime whose Docker work can outlive a killed client. + After a `BackendTimeout` in the gateway's apply, the next apply starts + another Compose operation unsettled. **Do:** a required + `settle_snapshot() -> object | None`; KubeAI delegates to a host gateway. + **Done when:** a KubeAI backend whose host gateway times out mid-apply + makes the next apply settle first (or raise `RuntimeUnsettled`). +- **16.** An apply that returns `ApplyResult(runtime=False)` cleanly leaves + `.env` on the new key and the gateway on the old one: the boundary was + "apply was called", not "the runtime changed". **Done when:** a backend + returning `runtime=False` without recreating the gateway leaves no key in + `.env` the gateway rejects. +- **15.** `infer-stack apply` accepting D2 over an approved D1 does not + record D2, so a partial D2 apply leaves D1 approved and every retry asks + again. **Done when:** after an explicit D2 apply that is partial, the + marker holds D2 and an ordinary retry of D2 proceeds; a D3 still refuses. +- **11, diagnosis.** A replica set is judged from the concatenated log, so + one fatal and one transient replica read as fatal. **Done when:** replicas + are classified one by one; fatal only when each is fatal; one transient + replica with retries left suppresses it. +- **11, health.** `status` equated residency with serving: every replica in + CrashLoopBackOff is "resident" (restarting is warm) and showed `up`. + **Done when:** one deployment health summary (up / starting / restarting / + conflicted) feeds `status` and `observe_state`; all replicas looping is not + up, one healthy is up, one starting plus one looping is not up. +- **14.** `eligible_nodes()` models the default scheduler; a pod with + `schedulerName` (the catalog's `scheduler_name`) or required pod + (anti-)affinity may be filtered differently. **Done when:** those evict + nothing (test through `scheduler_name`). +- **23 (new), route seed race.** A conflict found under the lock raises after + `publish_change` set the marker, so a clean refusal leaves a publication + pending. **Done when:** a race-time conflict leaves the marker as it was. + (The stale `--replace` display is noted, not fixed.) +- **21, optional hooks.** Classified by the re-review: the `last_*` + attributes and a new `last_planned_digest` and `allocates_gpus` are + required state read directly; `plan_on_idle_host` and `validate_requests` + become common operations with in-process defaults; `last_displaced` / + `last_degraded` go (`placement_notes()` is the authority); + `render_profile` / `use_profile` become a nullable recovery-profile + capability; `rotate_master_key` / `restore_env` / `litellm` belong on a + typed `front_door() -> FrontDoor | None`. Rule: a `getattr` default may + serve display, never a correctness decision. + +### 23. [ ] A race-time `routes seed` conflict leaves no marker From 4b8783ba175916c32f83fead7db6436f1409a53b Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 19:57:22 -0400 Subject: [PATCH 087/141] KubeAI settles its host gateway after an interrupted apply settle_snapshot() is a required backend operation: Compose samples its containers, KubeAI delegates to a host gateway (a Compose project whose Docker work can outlive a killed client) and returns None for an in-cluster one, the in-process backends None. The controller calls it directly; before this a KubeAI publication interrupted in the gateway's apply started the next Compose operation unsettled. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/backends/kubeai.py | 9 +++++++ infer_stack/backends/kubeai_gateway.py | 4 +++ infer_stack/leasing/backend.py | 16 ++++++++++++ infer_stack/leasing/controller.py | 12 +++++---- tests/test_parity.py | 35 ++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 4d94bd74..69f1a327 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -1209,6 +1209,15 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: #: No containers of this host to manage: the cluster owns network and pods. host_runtime = None + def settle_snapshot(self): + """The host gateway's Compose containers, or ``None``. + + ``kubectl apply`` is declarative server-side and leaves nothing half + done, but a gateway on this host is a Compose project whose Docker + work can outlive a killed client, exactly as on the compose backend. + """ + return self.gateway.settle_snapshot() if self.gateway is not None else None + def down(self) -> None: """Delete every infer-stack-managed Model (explicit stop), and the gateway.""" for name in sorted(self._cluster_models()): diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index ab8a6184..5a0d0d13 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -218,6 +218,10 @@ def apply(self) -> None: self._kubectl(['apply', '-f', str(self.manifests_file)]) self._kubectl(['rollout', 'status', f'deployment/{NAME}', '--timeout=300s']) + def settle_snapshot(self) -> None: + """Nothing local: this gateway is Kubernetes objects, applied declaratively.""" + return None + def retire_routes(self) -> bool: """Nothing to retire: this gateway's routes are static and point at KubeAI's one upstream, which no Model's removal takes away.""" diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 217e9a3f..5389c496 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -254,6 +254,19 @@ def apply(self) -> ApplyResult | bool | None: """ ... + def settle_snapshot(self) -> Any: + """A sample of runtime work that can outlive a killed client, or ``None``. + + After an interrupted apply the controller samples this until two + consecutive samples are equal (``Controller._wait_for_settled_runtime``). + ``None`` means nothing of the kind to wait for: an in-process backend, + or a cluster whose apply is declarative server-side. A backend that + drives a local runtime anywhere (KubeAI's host gateway is a Compose + project) must return its sample, not ``None``. Raises when the runtime + cannot be read. + """ + ... + def placement_notes(self) -> dict[str, list[str]]: """What the last render recorded about placement, for health views: ``degraded`` (committed GPUs no longer valid) and ``displaced`` (idle @@ -476,6 +489,9 @@ class SimpleAdmission: def placement_notes(self) -> dict[str, list[str]]: return {} + def settle_snapshot(self) -> None: + return None # in-process: nothing outlives a call + def reclaim_candidates(self, blocked: Deployment, idle: list[Deployment]) -> list[str]: """One scheduling domain in-process: any idle deployment frees room.""" diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index fd493eb1..713328b0 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1295,14 +1295,14 @@ def _wait_for_settled_runtime(self) -> None: cannot be read, a container still ``removing``, or a runtime that keeps changing past :data:`SETTLE_DEADLINE_S` raises :class:`~infer_stack.leasing.backend.RuntimeUnsettled` and leaves the - change pending. Backends without ``settle_snapshot`` (KubeAI's apply is - declarative server-side) skip the check. + change pending. A backend whose ``settle_snapshot`` returns ``None`` + has nothing local to wait for (an in-process backend, or KubeAI with + its gateway in the cluster); KubeAI with a host gateway returns that + gateway's Compose containers and is waited on like Compose. """ from .backend import RuntimeUnsettled - snapshot = getattr(self.backend, 'settle_snapshot', None) - if snapshot is None: - return + snapshot = self.backend.settle_snapshot deadline = self.clock() + SETTLE_DEADLINE_S previous = None while True: @@ -1313,6 +1313,8 @@ def _wait_for_settled_runtime(self) -> None: f'cannot read the runtime after an interrupted apply: {ex}; ' 'the change stays pending' ) from ex + if current is None: + return # nothing local that could still be running busy = any(state == 'removing' for _, state in current) if previous is not None and current == previous and not busy: return diff --git a/tests/test_parity.py b/tests/test_parity.py index a971373b..7c109512 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -667,3 +667,38 @@ def test_reclaimable_pressure_evicts_a_compatible_idle_model_and_the_lease_proce {'nodeSelector': {'gpu': 'a100'}}, until_gone=ids['warm-a']) assert _idle(stack, ids) == {'warm-b'} assert result.ready + + +# Interrupted applies: KubeAI's host gateway is a Compose runtime, so its +# Docker work can outlive a killed client exactly as on Compose (re-review 1). + + +def test_kubeai_settles_its_host_gateway_after_an_interrupted_apply(tmp_path): + from infer_stack.leasing.backend import BackendTimeout, RuntimeUnsettled + + stack = _kubeai(tmp_path) + clock = {'now': 0.0} + stack.ctl.clock = lambda: clock['now'] + stack.ctl.sleep = lambda s: clock.__setitem__('now', clock['now'] + s) + real = stack.front.apply + calls = [] + + def timed_out(): + calls.append('apply') + raise BackendTimeout('docker compose up timed out') + + stack.front.apply = timed_out + with pytest.raises(BackendTimeout): + stack.acquire('one') + assert stack.ctl.ledger.publication_pending()['interrupted'] + + samples = iter(range(1000)) # Docker still changing + stack.front.settle_snapshot = lambda: ((f'c{next(samples)}', 'running'),) + stack.front.apply = real + with pytest.raises(RuntimeUnsettled): + stack.ctl.apply_now() + assert calls == ['apply'] # no second Compose operation + + stack.front.settle_snapshot = lambda: (('c1', 'running'),) # settled + stack.ctl.apply_now() + assert stack.ctl.ledger.publication_pending() is None From 4476573ec57235463140dc4fb4e16c9406dd23e8 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:00:11 -0400 Subject: [PATCH 088/141] Rotation keeps the old key when the apply cleanly misses the runtime An apply that returns ApplyResult(runtime=False) did not recreate the gateway, so the rotation restores the old key and refuses with the apply's reason, instead of leaving .env on a key the running gateway rejects. The boundary is now whether the runtime changed, not whether apply was called. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/leasing/controller.py | 21 ++++++++++++++++++--- tests/test_leasing_secrets.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 713328b0..aa3da250 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -86,8 +86,10 @@ class ReconcileResult: # desired state was staged (--no-apply, render) or its apply did not fully # succeed. The next applying operation, or `infer-stack apply`, publishes it. publication_pending: bool = False - # Whether the runtime reached the render (routes may still be unverified). + # Whether the runtime reached the render (routes may still be unverified), + # and the apply's own words when it did not fully take effect. runtime_applied: bool = False + apply_detail: str = '' # Admission mode: idle keep-warm residents that yielded their GPUs, and # LIVE deployments whose committed allocation is no longer valid. displaced: list[str] = field(default_factory=list) @@ -682,7 +684,12 @@ def rotate_gateway_key(self, *, force: bool = False) -> ReconcileResult: goes back into the managed ``.env``: otherwise clients would be handed a key the gateway rejects. Once the apply began, the gateway may already run the new key; the file keeps it, and the pending - publication converges the runtime to it on the next apply. + publication converges the runtime to it on the next apply. An apply + that returns having not reached the runtime (``ApplyResult.runtime`` + false: an unreadable render, routes that could not be retired) did + not recreate the gateway either, so the old key goes back too and the + rotation is refused with the apply's reason: the file must never + name a key the running gateway rejects. """ from .profile import ProfileMismatch @@ -701,11 +708,18 @@ def rotate_gateway_key(self, *, force: bool = False) -> ReconcileResult: replaced = rotate() self._apply_began = False try: - return self._publish() + rec = self._publish() except BaseException: if not self._apply_began: getattr(self.backend, 'restore_env')(replaced) raise + if rec.applied and not rec.runtime_applied: + getattr(self.backend, 'restore_env')(replaced) + raise ProfileMismatch( + 'the key was not changed: the gateway was not recreated (' + + (rec.apply_detail or 'the runtime was not reached') + + '); retry once `infer-stack apply` succeeds') + return rec def observe_state(self) -> dict: """A read-only health view for ``leases`` / ``status`` (plan step P10). @@ -1271,6 +1285,7 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: rec.torn_down = sorted(set(rec.torn_down) | (before - after)) rec.applied = True rec.runtime_applied = outcome.runtime + rec.apply_detail = outcome.detail if not outcome.complete: # The publication is not done, so neither is the approval of its # render: a retry re-renders, and a render that drifted (an diff --git a/tests/test_leasing_secrets.py b/tests/test_leasing_secrets.py index 0c95f3d5..e136b844 100644 --- a/tests/test_leasing_secrets.py +++ b/tests/test_leasing_secrets.py @@ -228,3 +228,17 @@ def test_a_failure_after_apply_began_keeps_the_new_key_and_converges(tmp_path): ctl.apply_now() # the pending publication converges assert ctl.backend.gateway_accepts(new) and not ctl.backend.gateway_accepts(old) assert ledger.publication_pending() is None + + +def test_an_apply_that_cleanly_misses_the_runtime_keeps_the_old_key(tmp_path): + """Re-review 2: `ApplyResult(runtime=False)` returned without an + exception; the gateway was not recreated, so .env must not move ahead.""" + from infer_stack.leasing.backend import ApplyResult + + ledger, ctl = make_started(tmp_path) + old = env(ctl)[API_KEY_ENV] + ctl.backend.apply = lambda: ApplyResult(runtime=False, detail='render unreadable') + with pytest.raises(ProfileMismatch, match='key was not changed.*render unreadable'): + ctl.rotate_gateway_key() + assert env(ctl)[API_KEY_ENV] == old + assert ctl.backend.gateway_accepts(env(ctl)[API_KEY_ENV]) # what clients get works From 768d92489f36e611007fd8ef126c0f46f9d8854c Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:02:54 -0400 Subject: [PATCH 089/141] An explicit re-approval is durable across its own partial apply infer-stack apply accepting a render that differs from the approved one now records it as the approved digest (in place, same marker version) before applying. A partial apply of it leaves it approved, so an ordinary retry proceeds; a render that changes again still needs approval. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/leasing/controller.py | 23 +++++++++++++-------- infer_stack/leasing/ledger.py | 5 +++++ infer_stack/leasing/store.py | 9 +++++++-- tests/test_leasing_admission.py | 33 +++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index aa3da250..b3e90299 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1255,15 +1255,22 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: return rec approved = marker.get('approved_digest') rendered = getattr(self.backend, 'last_planned_digest', None) - if approved and rendered and rendered != approved and not self._explicit_apply: - from .profile import ProfileMismatch + if approved and rendered and rendered != approved: + if not self._explicit_apply: + from .profile import ProfileMismatch - rec.publication_pending = True - raise ProfileMismatch( - 'the rendered state differs from what was approved at publication ' - '(e.g. infer-stack was upgraded in between); review and approve it ' - 'with `infer-stack apply`' - ) + rec.publication_pending = True + raise ProfileMismatch( + 'the rendered state differs from what was approved at publication ' + '(e.g. infer-stack was upgraded in between); review and approve it ' + 'with `infer-stack apply`' + ) + # `infer-stack apply` approved this render: record it before + # applying, so a partial apply leaves THIS render approved and an + # ordinary retry of it proceeds (a later, different one still + # needs approval). + self.ledger.reapprove_render(rendered) + approved = rendered if marker['interrupted']: self._wait_for_settled_runtime() before = set(self.backend.observe()) diff --git a/infer_stack/leasing/ledger.py b/infer_stack/leasing/ledger.py index a684848e..56c33c5f 100644 --- a/infer_stack/leasing/ledger.py +++ b/infer_stack/leasing/ledger.py @@ -228,6 +228,11 @@ def clear_approved_digest(self) -> None: """The approved render was applied, or deliberately abandoned.""" self.store.clear_approved_digest() + def reapprove_render(self, digest: str) -> None: + """An operator approved this render over the one the marker records + (``infer-stack apply``); it is now the approved one, durably.""" + self.store.set_approved_digest(digest) + def publish_profile(self, profile: dict, *, approved_digest: str | None) -> None: """Write the recovery profile and its pending, approved marker at once.""" self.store.publish_profile(profile, approved_digest=approved_digest) diff --git a/infer_stack/leasing/store.py b/infer_stack/leasing/store.py index a0c393a1..586b0bfa 100644 --- a/infer_stack/leasing/store.py +++ b/infer_stack/leasing/store.py @@ -435,11 +435,16 @@ def _write_marker(self, *, apply_requested, interrupted=False, def clear_approved_digest(self) -> None: """The approved render was applied, or deliberately abandoned (a rollback); the digest no longer describes the pending state.""" + self.set_approved_digest(None) + + def set_approved_digest(self, digest: str | None) -> None: + """Replace the pending marker's approved digest in place (same + version: the pending change is the same one, only its approval moved).""" with self.transaction(): current = self._read_publication_pending() - if current is None or current.get('approved_digest') is None: + if current is None or current.get('approved_digest') == digest: return - current['approved_digest'] = None + current['approved_digest'] = digest self._conn.execute( "UPDATE meta SET value = ? WHERE key = 'publication_pending'", (json.dumps(current, sort_keys=True),), diff --git a/tests/test_leasing_admission.py b/tests/test_leasing_admission.py index eb70b67d..a2c75c00 100644 --- a/tests/test_leasing_admission.py +++ b/tests/test_leasing_admission.py @@ -524,3 +524,36 @@ def test_a_refused_acquire_writes_no_secret(tmp_path): acquire(ctl, 'one') assert parse_env_file(env)['LITELLM_MASTER_KEY'] == backend.master_key() assert ledger.publication_pending() is None # no second approval needed + + +def test_an_explicit_reapproval_survives_its_own_partial_apply(tmp_path): + """Re-review 4: `infer-stack apply` approving D2 over D1 must be durable, + so a partial D2 apply leaves D2 approved; D3 still needs approval.""" + from infer_stack.leasing.backend import ApplyResult, ConvergeScaffold + from infer_stack.leasing.profile import ProfileMismatch + + ledger, ctl, docker = make(tmp_path) + real = ctl.backend.apply + + def partial(): + real() + return ApplyResult(routes=False) + + ctl.backend.apply = partial + acquire(ctl, 'one') # D1 approved, partial + original = ConvergeScaffold._planned_digest + ctl.backend._planned_digest = lambda planned: 'v2-' + original(planned) # D2 + with pytest.raises(ProfileMismatch, match='approved'): + ctl.gc() + ctl.apply_now() # approves D2; partial again + d2 = ctl.backend.last_planned_digest + assert ledger.publication_pending()['approved_digest'] == d2 + ctl.backend.apply = real + ctl.gc() # an ordinary retry of D2 + assert ledger.publication_pending() is None + + ctl.backend.apply = partial # D3 needs its own approval + acquire(ctl, 'two') + ctl.backend._planned_digest = lambda planned: 'v3-' + original(planned) + with pytest.raises(ProfileMismatch, match='approved'): + ctl.gc() From b8b3542c589973ea70f631b20c0cefb9b6493373 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:07:22 -0400 Subject: [PATCH 090/141] Replicas: judged one by one, and serving is not residency Startup diagnosis classifies each replica from its own log (KubeAI's _pod_logs); a replica set is fatal only when every replica is, and never from a combined log. deployment_health() says whether a deployment serves (up / starting / restarting / conflicted), separate from whether it is resident; status and observe_state use it, so replicas that all crash-loop are 'restarting', not 'up', and leases flags them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/backends/kubeai.py | 22 +++++--- infer_stack/cli/commands_leasing.py | 1 + infer_stack/cli/commands_runtime.py | 14 ++--- infer_stack/leasing/controller.py | 11 +++- infer_stack/leasing/diagnosis.py | 86 ++++++++++++++++++----------- infer_stack/leasing/residency.py | 46 +++++++++++++++ tests/test_parity.py | 28 +++++++++- 7 files changed, 152 insertions(+), 56 deletions(-) diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 69f1a327..60700690 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -1042,15 +1042,18 @@ def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: pods = self.residency().units(deployment.id) except ResidencyUnknown: return '' + return '\n'.join(t for t in (self._pod_logs(pod, tail) for pod in pods) if t) + + def _pod_logs(self, pod, tail: int = 400) -> str: + """One pod's log: its current run, then its previous one (fail-open).""" parts: list[str] = [] - for pod in pods: - runs = [[]] + ([['--previous']] if pod.restart_count else []) - for extra in runs: - try: - parts.append(self._kubectl(['logs', pod.container_id, '--tail', - str(tail), *extra])) - except Exception: # noqa: BLE001 - gone, or no previous run - pass + runs = [[]] + ([['--previous']] if pod.restart_count else []) + for extra in runs: + try: + parts.append(self._kubectl(['logs', pod.container_id, '--tail', + str(tail), *extra])) + except Exception: # noqa: BLE001 - gone, or no previous run + pass return '\n'.join(p for p in parts if p) def startup_failure(self, deployment: Deployment) -> str | None: @@ -1064,7 +1067,8 @@ def startup_failure(self, deployment: Deployment) -> str | None: return None # cannot read the cluster: say nothing return diagnose_startup(residency.units(deployment.id), lambda: self.deployment_logs(deployment, tail=200), - replicated=residency.replicated) + replicated=residency.replicated, + unit_logs=lambda pod: self._pod_logs(pod, 200)) def _capacity_shortage(self, deployment: Deployment) -> set[str]: """Resources the scheduler says this Model's pending pods lack.""" diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index c7e6c0b3..f8140897 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2189,6 +2189,7 @@ def _print_health(health: dict) -> None: lines.append(f'residency UNKNOWN: {health["residency_error"]}') for row in health.get('deployments') or []: if row['condition'] in {'unknown', 'ambiguous', 'degraded', 'displaced', + 'restarting', 'unresolved', 'not-running'}: lines.append(f'{row["condition"].upper()}: {row["id"]} ({row["state"]})') for orphan in health.get('orphans') or []: diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index bad89e0c..5ca4e5aa 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -156,7 +156,7 @@ def _served_models(deployments, backend=None, *, right now" without cross-reading `leases` against `ps`. """ from ..leasing import DeploymentState - from ..leasing.residency import ResidencyUnknown + from ..leasing.residency import ResidencyUnknown, deployment_health live = [d for d in deployments if d.state == DeploymentState.LIVE] if not live: @@ -172,16 +172,10 @@ def _served_models(deployments, backend=None, *, for d in live: if residency is None: health = 'unverified' - elif residency.is_resident(d.id): - # Warm; up once any unit (a replica) has passed its health check, - # still loading while none has. - warm = residency.warm_units(d.id) - health = 'starting' if all(c.health == 'starting' for c in warm) else 'up' - elif residency.is_conflicted(d.id): - health = 'conflicted' # several containers claim it elif residency.units(d.id): - # There, but not warm: starting or crashed (every replica's state). - health = '/'.join(sorted({c.state for c in residency.units(d.id)})) + # Serving, not residency: a crash-looping replica is resident + # but not up (see deployment_health). + health = deployment_health(residency, d.id) or '-' elif pending: # Recorded, not applied yet: an apply is running, or a failed one # left the change for `infer-stack apply`. diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index b3e90299..15b50ed6 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -735,10 +735,13 @@ def observe_state(self) -> dict: * ``unresolved``: LIVE from before allocations, with no unique container; * ``reclaimed``: idle under ``stop``, and its container is gone, as intended (see :meth:`keeps_up`); - * ``running`` / ``not-running``: whether its single container serves. + * ``restarting``: resident, but every unit is crash-looping; + * ``running`` / ``not-running``: whether it is resident (any replica + warm), from :func:`~infer_stack.leasing.residency.deployment_health` + for whether it serves. """ from .profile import profile_drift - from .residency import ResidencyUnknown + from .residency import ResidencyUnknown, deployment_health leases, deployments = self.ledger.status(virtual_expiry=True) try: @@ -770,7 +773,9 @@ def observe_state(self) -> dict: and residency.unique_unit(g.id) is None): condition = 'unresolved' elif residency is not None and residency.is_resident(g.id): - condition = 'running' + condition = ('restarting' + if deployment_health(residency, g.id) == 'restarting' + else 'running') elif residency is not None: condition = 'not-running' if self.keeps_up(g) else 'reclaimed' else: diff --git a/infer_stack/leasing/diagnosis.py b/infer_stack/leasing/diagnosis.py index fe69daa8..44a0a093 100644 --- a/infer_stack/leasing/diagnosis.py +++ b/infer_stack/leasing/diagnosis.py @@ -9,7 +9,7 @@ from __future__ import annotations -from typing import Callable, Sequence +from typing import Any, Callable, Sequence #: Restarts after which the runtime's own bookkeeping says an engine is #: looping, not loading. Two is already conclusive: a model that loads does not exit. @@ -108,8 +108,30 @@ def _crashed(instance) -> bool: return exited_for_good or instance.restart_count >= 1 +def _unit_verdict(instance, logs: str, replicas: int = 1) -> str | None: + """Why this one unit can never start, from its own log, or ``None``.""" + exited_for_good = (instance.state in {'exited', 'dead'} + and (instance.exit_code or 0) != 0) + verdict = classify_engine_log(logs) + if verdict == 'transient' and instance.will_be_restarted: + return None # a retry is coming, and may work + if verdict == 'transient': + return (f'engine is not starting (exited with code ' + f'{instance.exit_code} and will not be restarted)' + f'{_engine_error_summary(logs)}') + looping = instance.restart_count >= CRASH_LOOP_RESTARTS + if verdict != 'fatal' and not (exited_for_good or looping): + return None # unrecognised: keep the restart budget + why = (f'restarted {instance.restart_count} time(s)' if instance.restart_count + else f'exited with code {instance.exit_code}') + if replicas > 1: + why += f', all {replicas} replicas' + return f'engine is not starting ({why}){_engine_error_summary(logs)}' + + def diagnose_startup(instances: Sequence, read_logs: Callable[[], str], *, - replicated: bool = False) -> str | None: + replicated: bool = False, + unit_logs: Callable[[Any], str] | None = None) -> str | None: """Diagnose an engine that cannot start, or ``None`` if it may still load. A restart policy makes an engine that exits immediately restart forever, @@ -129,43 +151,43 @@ def diagnose_startup(instances: Sequence, read_logs: Callable[[], str], *, ``instances`` are the deployment's units. Without ``replicated`` (Docker) there must be exactly one: none is "not created yet", several are a conflict, and both say nothing. With it (a Kubernetes Model's replicas, - :attr:`Residency.replicated`) the set is diagnosed as one: fatal only - when every unit has crashed, since one unit still up or loading may - serve; the budget is the least-restarted unit's. The returned string - carries the engine's last words, because the cause is in its log and - nowhere else. + :attr:`Residency.replicated`) the set is fatal only when EVERY replica is, + each judged from its own log (``unit_logs``): one still up, loading, or + failing transiently with a retry to come may yet serve. Without per-unit + logs a replica set is never declared fatal, since a combined log cannot + tell whose error is whose. The returned string carries the engine's last + words, because the cause is in its log and nowhere else. >>> from infer_stack.leasing.residency import Container - >>> crash = 'error: unrecognized arguments: --nope' - >>> bad = Container('p1', 'grp', 'restarting', restart_count=1, exit_code=1) - >>> up = Container('p2', 'grp', 'running') - >>> diagnose_startup([bad, bad], lambda: crash, replicated=True)[:25] + >>> fatal = 'error: unrecognized arguments: --nope' + >>> hub = 'requests.exceptions.ConnectionError: Max retries exceeded' + >>> a = Container('p1', 'grp', 'restarting', restart_count=1, exit_code=1, + ... restart_policy='always') + >>> b = Container('p2', 'grp', 'restarting', restart_count=1, exit_code=1, + ... restart_policy='always') + >>> up = Container('p3', 'grp', 'running') + >>> logs = {'p1': fatal, 'p2': fatal, 'p3': ''} + >>> diagnose_startup([a, b], lambda: '', replicated=True, + ... unit_logs=lambda u: logs[u.container_id])[:25] 'engine is not starting (r' - >>> diagnose_startup([bad, up], lambda: crash, replicated=True) is None + >>> logs['p2'] = hub # one replica may still recover + >>> diagnose_startup([a, b], lambda: '', replicated=True, + ... unit_logs=lambda u: logs[u.container_id]) is None True - >>> diagnose_startup([bad, bad], lambda: crash) is None # a Docker conflict + >>> diagnose_startup([a, up], lambda: fatal, replicated=True) is None + True + >>> diagnose_startup([a, b], lambda: fatal) is None # a Docker conflict True """ if not instances or (len(instances) > 1 and not replicated): return None # not created yet, or conflicting units if not all(_crashed(i) for i in instances): return None # something is created, starting, or healthy - instance = min(instances, key=lambda i: i.restart_count) - exited_for_good = (instance.state in {'exited', 'dead'} - and (instance.exit_code or 0) != 0) - logs = read_logs() - verdict = classify_engine_log(logs) - if verdict == 'transient' and any(i.will_be_restarted for i in instances): - return None # a retry is coming, and may work - if verdict == 'transient': - return (f'engine is not starting (exited with code ' - f'{instance.exit_code} and will not be restarted)' - f'{_engine_error_summary(logs)}') - looping = instance.restart_count >= CRASH_LOOP_RESTARTS - if verdict != 'fatal' and not (exited_for_good or looping): - return None # unrecognised: keep the restart budget - why = (f'restarted {instance.restart_count} time(s)' if instance.restart_count - else f'exited with code {instance.exit_code}') - if len(instances) > 1: - why += f', all {len(instances)} replicas' - return f'engine is not starting ({why}){_engine_error_summary(logs)}' + if len(instances) == 1: + return _unit_verdict(instances[0], read_logs()) + if unit_logs is None: + return None # whose error is whose cannot be told + verdicts = [_unit_verdict(i, unit_logs(i), len(instances)) for i in instances] + if not all(verdicts): + return None # some replica may still come up + return verdicts[0] diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index 71889dda..8b43fb63 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -250,6 +250,52 @@ def all_containers(self) -> tuple[Container, ...]: ) +def deployment_health(residency: Residency, deployment_id: str) -> str | None: + """Whether a deployment serves, as opposed to whether it is resident. + + Residency is physical: a crash-looping unit is warm (it holds, and will + reclaim, its resources). Serving is not: a deployment is ``up`` only when + some unit runs and has not failed or not yet passed its health check. A + Docker container with no healthcheck reports an empty health, which is + up when running; a Kubernetes pod reports ``healthy`` when Ready. + + * ``up``: some unit running, not ``starting`` or ``unhealthy``; + * ``starting``: none up, some running but not ready yet, or created; + * ``restarting``: every warm unit crash-looping (restarting); + * ``conflicted``: units that may not coexist (see :meth:`Residency.is_conflicted`); + * otherwise the units' states (``exited``, ``removing`` ...), or ``None`` + when the deployment has no unit. + + >>> def unit(name, state='running', health=''): + ... return Container(name, 'grp', state, health=health) + >>> def health(*units): + ... return deployment_health(Residency({'grp': units}, replicated=True), 'grp') + >>> health(unit('a', 'restarting'), unit('b', 'restarting')) + 'restarting' + >>> health(unit('a', health='healthy'), unit('b', 'restarting')) + 'up' + >>> health(unit('a', health='starting'), unit('b', 'restarting')) + 'starting' + >>> health(unit('c')) # Docker, no healthcheck, running + 'up' + """ + units = residency.units(deployment_id) + if not units: + return None + if residency.is_conflicted(deployment_id): + return 'conflicted' + if any(u.state == 'running' and u.health not in ('starting', 'unhealthy') + for u in units): + return 'up' + if any(u.state == 'created' or (u.state == 'running' and u.health == 'starting') + for u in units): + return 'starting' + warm = residency.warm_units(deployment_id) + if warm and all(u.state == 'restarting' for u in warm): + return 'restarting' + return '/'.join(sorted({u.state for u in units})) + + def _gpus_from_device_requests(requests: Any) -> tuple[tuple[int, ...], bool]: """Map Docker ``HostConfig.DeviceRequests`` to ``(indices, all_gpus)``. diff --git a/tests/test_parity.py b/tests/test_parity.py index 7c109512..574609d3 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -573,10 +573,15 @@ def test_two_compose_containers_for_one_deployment_still_fail_closed(tmp_path): def test_every_replica_crash_looping_fails_fast_and_one_healthy_does_not(tmp_path): stack, out, gid = _replicated(tmp_path) deployment = stack.ctl.ledger.get_deployment(gid) - stack.backend.deployment_logs = lambda deployment, tail=400: CRASH_LOG + logs = {} + stack.backend._pod_logs = lambda pod, tail=400: logs.get(pod.container_id, '') stack.runtime.crash_one.add(gid) # one replica still serves + logs['model-pair-0'] = CRASH_LOG assert stack.backend.startup_failure(deployment) is None - stack.runtime.crash[gid] = CRASH_LOG # now every replica loops + stack.runtime.crash[gid] = CRASH_LOG # now every replica loops... + logs['model-pair-1'] = 'ConnectionError: Max retries exceeded' # ...one transiently + assert stack.backend.startup_failure(deployment) is None + logs['model-pair-1'] = CRASH_LOG # both fatal: fail fast why = stack.backend.startup_failure(deployment) assert why and 'all 2 replicas' in why and 'trust_remote_code' in why @@ -702,3 +707,22 @@ def timed_out(): stack.front.settle_snapshot = lambda: (('c1', 'running'),) # settled stack.ctl.apply_now() assert stack.ctl.ledger.publication_pending() is None + + +def test_status_reports_serving_not_residency_for_replicas(tmp_path): + """Re-review 6: crash-looping units are resident (they hold resources) but + do not serve. Status must say so, and `leases` health must flag it.""" + from infer_stack.cli.commands_runtime import _served_models + + stack, out, gid = _replicated(tmp_path) + + def health(): + _, deployments = stack.ctl.ledger.status() + return [row[3] for row in _served_models(deployments, stack.backend)] + + stack.runtime.crash_one.add(gid) # one healthy replica + assert health() == ['up'] + stack.runtime.crash[gid] = CRASH_LOG # every replica looping + assert health() == ['restarting'] + assert _condition(stack, gid) == 'restarting' + assert stack.backend.residency().is_resident(gid) # still resident, as it should be From 55c1f963e6578211f0330454b6abb59ed858f1f1 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:10:23 -0400 Subject: [PATCH 091/141] Reclaim fails closed on scheduling it does not model A pod handed to another scheduler (a resource profile's scheduler_name), with required pod affinity or anti-affinity, or with a hard topology spread constraint may be filtered in ways infer-stack cannot evaluate, so no idle Model is evicted for it. Tested through scheduler_name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/backends/kubeai.py | 27 +++++++++++++++++++++------ tests/test_parity.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 60700690..747add48 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -437,9 +437,13 @@ def eligible_nodes(pod_spec: dict, nodes: list[dict]) -> set[str] | None: """Nodes a pod could run on if they had room, or ``None`` if not known. A node qualifies when it is schedulable, carries every label of the - pod's ``nodeSelector``, and every blocking taint is tolerated. Required - node affinity is not evaluated here: ``None`` then, and the caller frees - nothing on a guess. + pod's ``nodeSelector``, and every blocking taint is tolerated: the + default scheduler's filters for those fields. Anything that filters + differently makes the answer unknown (``None``), and the caller frees + nothing on a guess: another scheduler (``schedulerName``, which a + resource profile's ``scheduler_name`` sets), required node affinity, + required pod affinity or anti-affinity, and a topology spread + constraint that forbids scheduling. >>> spec = {'nodeSelector': {'gpu': 'a100'}, 'tolerations': []} >>> nodes = [{'metadata': {'name': 'a', 'labels': {'gpu': 'a100'}}, 'spec': {}}, @@ -448,10 +452,21 @@ def eligible_nodes(pod_spec: dict, nodes: list[dict]) -> set[str] | None: ... 'spec': {'taints': [{'key': 'x', 'effect': 'NoSchedule'}]}}] >>> eligible_nodes(spec, nodes) {'a'} + >>> eligible_nodes({**spec, 'schedulerName': 'volcano'}, nodes) is None + True + >>> eligible_nodes({**spec, 'affinity': {'podAntiAffinity': { + ... 'requiredDuringSchedulingIgnoredDuringExecution': [{}]}}}, nodes) is None + True """ - required = (((pod_spec.get('affinity') or {}).get('nodeAffinity') or {}) - .get('requiredDuringSchedulingIgnoredDuringExecution')) - if required: + if pod_spec.get('schedulerName') not in (None, '', 'default-scheduler'): + return None # its filters are not these + affinity = pod_spec.get('affinity') or {} + hard = 'requiredDuringSchedulingIgnoredDuringExecution' + if any((affinity.get(kind) or {}).get(hard) + for kind in ('nodeAffinity', 'podAffinity', 'podAntiAffinity')): + return None + if any(c.get('whenUnsatisfiable', 'DoNotSchedule') == 'DoNotSchedule' + for c in pod_spec.get('topologySpreadConstraints') or []): return None selector = pod_spec.get('nodeSelector') or {} tolerations = pod_spec.get('tolerations') or [] diff --git a/tests/test_parity.py b/tests/test_parity.py index 574609d3..0511aeb5 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -726,3 +726,31 @@ def health(): assert health() == ['restarting'] assert _condition(stack, gid) == 'restarting' assert stack.backend.residency().is_resident(gid) # still resident, as it should be + + +def test_a_profile_with_its_own_scheduler_evicts_nothing(tmp_path): + """Re-review 3: a resource profile's `scheduler_name` hands the pod to a + scheduler whose filters infer-stack does not model, so it cannot tell + that stopping an idle Model would help, even on a capacity complaint.""" + from infer_stack.config import resource_profiles_to_kubeai_values + + values = resource_profiles_to_kubeai_values( + {'batch': {'scheduler_name': 'volcano', 'node_selector': {'gpu': 'a100'}}}) + profile = values['resourceProfiles']['batch'] + spec = {'schedulerName': profile['schedulerName'], + 'nodeSelector': profile['nodeSelector']} + stack, ids = _two_idle_models(tmp_path) + _wait_blocked(stack, '0/2 nodes are available: 1 Insufficient nvidia.com/gpu, ' + "1 node(s) didn't match Pod's node affinity/selector.", spec) + assert _idle(stack, ids) == {'warm-a', 'warm-b'} + + +def test_required_pod_anti_affinity_evicts_nothing(tmp_path): + stack, ids = _two_idle_models(tmp_path) + spec = {'nodeSelector': {'gpu': 'a100'}, 'affinity': {'podAntiAffinity': { + 'requiredDuringSchedulingIgnoredDuringExecution': [ + {'topologyKey': 'kubernetes.io/hostname', + 'labelSelector': {'matchLabels': {'app': 'model'}}}]}}} + _wait_blocked(stack, '0/2 nodes are available: 1 Insufficient nvidia.com/gpu, ' + "1 node(s) didn't match Pod's node affinity/selector.", spec) + assert _idle(stack, ids) == {'warm-a', 'warm-b'} From 30a62bb1ec18b0513f80f469499bf6d36c19091a Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:13:07 -0400 Subject: [PATCH 092/141] A race-time routes seed conflict leaves no pending publication publish_change takes a preflight that runs under the lock before the marker is set; route seed rechecks for conflicts there, so a redefinition another process made after the plan refuses without leaving a publication pending. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/leasing/controller.py | 18 +++++++++++++++--- tests/test_cli_leasing.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 15b50ed6..576cffd3 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1858,10 +1858,15 @@ def commit_route_seed(self, plan: RoutePlan, *, replace: bool = False if fresh.conflicted and not replace: raise RouteConflict(sorted(fresh.conflicted)) - def change(): + def preflight(): + # Under the lock, before the marker: a conflict another process + # made since the plan refuses without leaving a publication pending. now = plan_seed(gateway.route_entries(), plan.incoming) if now.conflicted and not replace: raise RouteConflict(sorted(now.conflicted)) + + def change(): + now = plan_seed(gateway.route_entries(), plan.incoming) entries = gateway.route_entries() entries.update(now.added) if replace: @@ -1870,7 +1875,7 @@ def change(): gateway.replace_route_entries(entries) return now - return self.publish_change(change) + return self.publish_change(change, preflight=preflight) def plan_route_prune(self) -> RoutePlan: """Which routes a prune drops: all but the catalog's and the live ones, @@ -1901,15 +1906,22 @@ def change(): return self.publish_change(change) - def publish_change(self, change: Callable[[], _T]) -> tuple[_T, ReconcileResult]: + def publish_change(self, change: Callable[[], _T], *, + preflight: Callable[[], None] | None = None + ) -> tuple[_T, ReconcileResult]: """Run ``change`` and publish, as one serialised desired-state mutation. For changes to backend state that the render reads (the route registry, via ``routes seed`` / ``routes prune``). ``change`` runs under the lock after the marker is set, so a crash leaves it pending like any other mutation. Returns ``(change's result, reconcile result)``. + + ``preflight`` runs under the lock before the marker is set: a check + that refuses there (raises) leaves no publication pending. """ with self._global_lock(): + if preflight is not None: + preflight() self._mark_pending(apply=True) result = change() rec = self._publish() diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 90aa82d5..fcf378b8 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -1208,3 +1208,33 @@ def seed(*args): rc, out = seed(str(second), '--replace') assert rc == 0 and out['updated'] == ['alpha'] assert (state / 'litellm_registry.json').read_text() != registry + + +def test_a_seed_conflict_that_appears_after_the_plan_leaves_no_marker(tmp_path, monkeypatch): + """Re-review 7: another process redefines the alias between the plan and + the commit; the commit refuses, writes nothing, and leaves no pending + publication behind.""" + from infer_stack.cli import commands_leasing as cl + from infer_stack.leasing import Catalog + from infer_stack.leasing.routes import RouteConflict + + state = tmp_path / 'state' + state.mkdir() + db = str(tmp_path / 'ledger.db') + _patch_backend(monkeypatch, state) + config = cl.RoutesSeedCLI.cli(argv=['--ledger', db, str(tmp_path / 'x.yaml')]) + controller = cl._open_controller(config, interactive=False) + plan = controller.plan_route_seed([Catalog.from_dict(_one_endpoint_catalog('alpha'))]) + gateway = controller.backend.front_door().gateway + real_lock = controller._global_lock + + def racing_lock(): + # The other process wins the lock first and redefines alpha. + gateway.replace_route_entries({'alpha': {'engine': 'vllm', 'served': 'elsewhere'}}) + return real_lock() + + controller._global_lock = racing_lock + with pytest.raises(RouteConflict): + controller.commit_route_seed(plan) + assert controller.ledger.publication_pending() is None + assert gateway.route_entries() == {'alpha': {'engine': 'vllm', 'served': 'elsewhere'}} From 122f1b4dcec3cdb2ee8a1c4e51c1bce2fe5dac8c Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:15:36 -0400 Subject: [PATCH 093/141] tests: the publish_change spy passes preflight through The previous commit went in with this test failing: the chain that ran the suite did not gate the commit on it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- tests/test_cli_leasing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index fcf378b8..5222cec4 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -1042,11 +1042,11 @@ def test_routes_seed_and_prune_publish_through_the_controller(tmp_path, monkeypa calls = [] real = Controller.publish_change - def spy(self, change): + def spy(self, change, **kw): calls.append(self._flock_depth) # The registry must not be written before the controller runs the change. before = (state / 'litellm_registry.json').exists() - result = real(self, change) + result = real(self, change, **kw) calls.append(before) return result From 2f9adc4f90c4aff80401081255f7329788f901b1 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:22:09 -0400 Subject: [PATCH 094/141] Backend hooks: required state, common operations, typed capabilities The controller reads no optional backend hook with getattr any more: - required state: last_unplaced/errors/assignments, last_preview_digest, last_planned_digest (the approval guard) and allocates_gpus (Compose now says True; absence no longer implies host GPU ownership); - common operations with in-process defaults: plan_on_idle_host, validate_requests; - gone: last_displaced/last_degraded (placement_notes() is the authority); - nullable capabilities: recovery_profile (render/use a recovery snapshot) and a typed front_door() -> FrontDoor (litellm, keys, rotation, restore). ty checks each backend and front door against these. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/backends/kubeai.py | 5 ++ infer_stack/leasing/backend.py | 99 +++++++++++++++++++++++--- infer_stack/leasing/compose.py | 12 ++-- infer_stack/leasing/controller.py | 71 +++++++++--------- tests/test_leasing_controller_queue.py | 7 +- 5 files changed, 142 insertions(+), 52 deletions(-) diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 747add48..bb1a9052 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -1228,6 +1228,11 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: #: No containers of this host to manage: the cluster owns network and pods. host_runtime = None + @property + def recovery_profile(self) -> KubeaiBackend: + """This backend is its own :class:`~infer_stack.leasing.backend.RecoveryProfile`.""" + return self + def settle_snapshot(self): """The host gateway's Compose containers, or ``None``. diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 5389c496..fa4dabfe 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -142,7 +142,7 @@ def allocates_gpus(backend) -> bool: admission commits an empty allocation, and no GPU is ever "unresolved". The one place the controller and the CLI ask this. """ - return bool(getattr(backend, 'allocates_gpus', True)) + return bool(backend.allocates_gpus) class HostRuntime(Protocol): @@ -183,6 +183,48 @@ def remove_containers(self, container_ids: list[str]) -> None: ... +class FrontDoor(Protocol): + """The LiteLLM gateway as the controller and the CLI address it. + + ``backend.front_door()`` returns one, or ``None`` for a backend without a + gateway: the Compose project itself, KubeAI's gateway on this host or in + the cluster. Keys, the route registry, and "does it accept this key" live + here, not as copies on every serving backend. + """ + + #: Whether LiteLLM is on (a front door can be Open WebUI alone). + litellm: bool + #: The gateway's state: settings, managed keys, the route registry. + gateway: Any + + def master_key(self) -> str: + ... + + def rotate_master_key(self) -> dict[str, str | None]: + """Write a fresh key; return what it replaced (for :meth:`restore_env`).""" + ... + + def restore_env(self, values: dict[str, str | None]) -> None: + ... + + def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: + ... + + +class RecoveryProfile(Protocol): + """A backend's recovery snapshot: what it would render, and adopting one. + + ``backend.recovery_profile`` is one of these, or ``None`` (the in-process + backends have no settings to snapshot). + """ + + def render_profile(self) -> dict[str, Any]: + ... + + def use_profile(self, profile: dict[str, Any]) -> None: + ... + + @runtime_checkable class ServingBackend(Protocol): """What the :class:`Controller` needs from a serving backend. @@ -208,6 +250,25 @@ class ServingBackend(Protocol): last_errors: list[str] last_assignments: dict[str, list[int]] last_preview_digest: str | None + #: The digest of the last render's files: the approval guard compares it + #: with the approved one, so every backend must keep it current. + last_planned_digest: str | None + #: Whether this backend owns host GPU indices (Compose) or the cluster + #: does (KubeAI). Required: a wrong default would claim host GPUs. + allocates_gpus: bool + + @property + def recovery_profile(self) -> RecoveryProfile | None: + """This backend's recovery snapshot, or ``None`` (see :class:`RecoveryProfile`).""" + ... + + def plan_on_idle_host(self, desired: list[Deployment]) -> Any: + """Placement for ``desired`` alone, as if nothing else ran (a plan).""" + ... + + def validate_requests(self, requests: list[Any]) -> None: + """Refuse requests this backend can never serve (raise); else nothing.""" + ... @property def host_runtime(self) -> HostRuntime | None: @@ -273,7 +334,7 @@ def placement_notes(self) -> dict[str, list[str]]: residents that yielded their GPUs). Empty where nothing is placed.""" ... - def front_door(self) -> Any: + def front_door(self) -> FrontDoor | None: """What holds the LiteLLM gateway (its keys and route registry), or ``None`` for a backend without one.""" ... @@ -483,8 +544,17 @@ class SimpleAdmission: allocates_gpus = False last_preview_digest: str | None = None - #: No containers on this host to manage. + last_planned_digest: str | None = None + #: No containers on this host to manage, no settings to snapshot. host_runtime: HostRuntime | None = None + recovery_profile: RecoveryProfile | None = None + + def plan_on_idle_host(self, desired: list[Deployment]): + """Nothing else competes in-process: the ordinary plan.""" + return self.plan(desired) + + def validate_requests(self, requests: list[Any]) -> None: + """Every request is servable in-process.""" def placement_notes(self) -> dict[str, list[str]]: return {} @@ -497,7 +567,7 @@ def reclaim_candidates(self, blocked: Deployment, """One scheduling domain in-process: any idle deployment frees room.""" return [g.id for g in idle] - def front_door(self) -> Any: + def front_door(self) -> FrontDoor | None: return None # no gateway in-process def route_rows(self, desired: list[Deployment], placement: Any = None @@ -575,10 +645,10 @@ def apply(self) -> None: if gid not in self.observe(): self.realize(deployment) - # Set by converge (the controller reads them with a default before then). - last_unplaced: set[str] - last_errors: list[str] - last_assignments: dict[str, list[int]] + # Replaced (never mutated) by each converge. + last_unplaced: set[str] = set() + last_errors: list[str] = [] + last_assignments: dict[str, list[int]] = {} def observe(self) -> set[str]: # pragma: no cover - provided by the backend raise NotImplementedError @@ -687,6 +757,7 @@ def _conforms() -> None: # pragma: no cover - read by the type checker only the protocol fails the type check rather than a run. """ from ..backends.kubeai import KubeaiBackend + from ..backends.kubeai_gateway import ClusterGateway from .compose import ComposeBackend def compose(backend: ComposeBackend) -> ServingBackend: @@ -703,3 +774,15 @@ def null(backend: NullBackend) -> ServingBackend: def host(backend: ComposeBackend) -> HostRuntime: return backend + + def compose_front(backend: ComposeBackend) -> FrontDoor: + return backend + + def cluster_front(backend: ClusterGateway) -> FrontDoor: + return backend + + def compose_profile(backend: ComposeBackend) -> RecoveryProfile: + return backend + + def kubeai_profile(backend: KubeaiBackend) -> RecoveryProfile: + return backend diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index d79605d3..3171b0c9 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1141,8 +1141,6 @@ def __init__( self.last_errors: list[str] = [] self.last_unplaced: set[str] = set() # desired deployment ids placement skipped self.last_assignments: dict[str, list[int]] = {} # deployment id -> GPU ids - self.last_displaced: list[str] = [] # optional residents that yielded - self.last_degraded: list[str] = [] # invalid committed allocations self._pulled: set[str] = set() # (deployment:tag) pulled this process # VRAM facts (docs/planning/vram-aware-placement.md Phase 3): the # measured-requirement overlay + a per-process cache of weight-bytes @@ -1793,8 +1791,6 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N plan, rendered, planned = docs['plan'], docs['rendered'], docs['planned'] fingerprints = docs['fingerprints'] self.last_assignments = dict(plan.assignments) - self.last_displaced = list(plan.displaced) - self.last_degraded = list(plan.degraded) for gid, gpus in sorted(plan.assignments.items()): logger.info(' placed {} {}', gid, f'on GPU(s) {gpus}' if gpus else 'without a GPU') @@ -1926,6 +1922,14 @@ def apply(self) -> ApplyResult: #: {service, fingerprint}. Set by the controller from the ledger. adopted: dict[str, dict[str, str]] = {} + #: This backend places on this host's GPUs and records them. + allocates_gpus = True + + @property + def recovery_profile(self) -> ComposeBackend: + """This backend is its own :class:`~infer_stack.leasing.backend.RecoveryProfile`.""" + return self + # -- HostRuntime: what only containers on this host have ----------------- @property diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 576cffd3..6042f146 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -390,8 +390,9 @@ def publication_lock(self): def invocation_profile(self) -> dict | None: """The recovery profile this invocation resolves to (its settings and catalogs), before any switch to a stored one; ``None`` without one.""" - render = getattr(self.backend, 'render_profile', None) - return self._invocation_profile or (render() if render is not None else None) + recovery = self.backend.recovery_profile + return self._invocation_profile or ( + recovery.render_profile() if recovery is not None else None) @contextlib.contextmanager def _global_lock(self): @@ -446,14 +447,10 @@ def _infeasible_alone(self, deployments, requested: set) -> dict: """Which of this lease's deployments cannot be placed even on an idle host. Returns ``{deployment_id: reason}``, empty when the whole set fits. - Backends without an idle-host planner (null, kubeai) return empty -- - the check is an optimisation over waiting, never a new failure mode. + The check is an optimisation over waiting, never a new failure mode. """ - plan_alone = getattr(self.backend, 'plan_on_idle_host', None) - if plan_alone is None: - return {} try: - plan = plan_alone(list(deployments)) + plan = self.backend.plan_on_idle_host(list(deployments)) except Exception as ex: # noqa: BLE001 -- never fail an acquire on the check # Waiting (the old behavior) is still correct, so a broken check # must not break acquire. But it must not be silent either: a @@ -497,15 +494,17 @@ def _render(self) -> ReconcileResult: before = set(self.backend.observe()) self.backend.converge(desired, apply=False, placement=placement) after = set(self.backend.observe()) + notes = self.backend.placement_notes() rec = ReconcileResult( realized=sorted(after - before), torn_down=sorted(before - after), - unplaced=sorted(getattr(self.backend, 'last_unplaced', ()) or ()), - placement_errors=list(getattr(self.backend, 'last_errors', ()) or ()), - assignments=dict(getattr(self.backend, 'last_assignments', {}) or {}), + unplaced=sorted(self.backend.last_unplaced), + placement_errors=list(self.backend.last_errors), + assignments=dict(self.backend.last_assignments), applied=False, - displaced=list(getattr(self.backend, 'last_displaced', ()) or ()), - degraded=list(getattr(self.backend, 'last_degraded', ()) or ()), + # The render's own record, the one authority for these facts. + displaced=list(notes.get('displaced') or ()), + degraded=list(notes.get('degraded') or ()), ) if host is not None: self._adopt_existing(residency, host) @@ -666,7 +665,7 @@ def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResul host.configure_network(saved) self.ledger.migrate_network( subnet=subnet, reset_addresses=reset, - approved_digest=getattr(self.backend, 'last_preview_digest', None), + approved_digest=self.backend.last_preview_digest, ) return self._publish() @@ -693,8 +692,8 @@ def rotate_gateway_key(self, *, force: bool = False) -> ReconcileResult: """ from .profile import ProfileMismatch - rotate = getattr(self.backend, 'rotate_master_key', None) - if rotate is None or not getattr(self.backend, 'litellm', False): + front = self.backend.front_door() + if front is None or not front.litellm: raise ProfileMismatch('no LiteLLM gateway to rotate a key for') with self._global_lock(): leases, _ = self.ledger.status(virtual_expiry=True) @@ -705,16 +704,16 @@ def rotate_gateway_key(self, *, force: bool = False) -> ReconcileResult: '(release them, or pass --force)' ) self._mark_pending(apply=True) - replaced = rotate() + replaced = front.rotate_master_key() self._apply_began = False try: rec = self._publish() except BaseException: if not self._apply_began: - getattr(self.backend, 'restore_env')(replaced) + front.restore_env(replaced) raise if rec.applied and not rec.runtime_applied: - getattr(self.backend, 'restore_env')(replaced) + front.restore_env(replaced) raise ProfileMismatch( 'the key was not changed: the gateway was not recreated (' + (rec.apply_detail or 'the runtime was not reached') @@ -988,7 +987,7 @@ def _admit(self, overlay, residency): # Its digest goes into the pending marker, so a recovery after a # crash (and perhaps an upgrade) cannot apply something else. self.backend.preview(desired, inputs, approve=True) - self._admission_digest = getattr(self.backend, 'last_preview_digest', None) + self._admission_digest = self.backend.last_preview_digest for gid in adopted: overlay.deployments[gid].assigned_gpus = None # committed with the lease return allocations, reasons @@ -1028,10 +1027,10 @@ def set_invocation_catalog(self, catalog) -> None: """ from .profile import catalog_sources - render = getattr(self.backend, 'render_profile', None) - if render is None: + recovery = self.backend.recovery_profile + if recovery is None: return - base = dict(self._invocation_profile or render()) + base = dict(self._invocation_profile or recovery.render_profile()) base['catalogs'] = catalog_sources(catalog) self._invocation_profile = base self._profile_drift_warned = False @@ -1151,9 +1150,9 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: return candidate if candidate != stored else None def _use_profile_candidate(self, profile: dict) -> None: - use = getattr(self.backend, 'use_profile', None) - if use is not None: - use(profile) + recovery = self.backend.recovery_profile + if recovery is not None: + recovery.use_profile(profile) def _commit_profile_candidate(self, profile: dict) -> None: """Persist an already-previewed acquire profile without publishing alone. @@ -1185,11 +1184,10 @@ def _sync_profile(self, *, create: bool) -> None: from .._log import logger from .profile import profile_drift - render = getattr(self.backend, 'render_profile', None) - use = getattr(self.backend, 'use_profile', None) - read = getattr(self.ledger, 'profile', None) - if render is None or use is None or read is None: + recovery = self.backend.recovery_profile + if recovery is None: return + render, use, read = recovery.render_profile, recovery.use_profile, self.ledger.profile if create and self._profile_error is not None: raise self._profile_error stored = read() @@ -1259,7 +1257,7 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: rec.publication_pending = True # staged; never applied here return rec approved = marker.get('approved_digest') - rendered = getattr(self.backend, 'last_planned_digest', None) + rendered = self.backend.last_planned_digest if approved and rendered and rendered != approved: if not self._explicit_apply: from .profile import ProfileMismatch @@ -1657,9 +1655,7 @@ def _acquire_by_admission( if candidate_profile is not None: self._use_profile_candidate(candidate_profile) try: - validate = getattr(self.backend, 'validate_requests', None) - if validate is not None: - validate(requests) + self.backend.validate_requests(requests) overlay = self.ledger.plan_acquire( requests, resident=residency.is_resident if residency is not None else None) @@ -1825,7 +1821,7 @@ def _route_gateway(self): from .profile import ProfileMismatch front = self.backend.front_door() - if front is None or not getattr(front, 'litellm', False): + if front is None or not front.litellm: raise ProfileMismatch( 'the `routes` commands need a LiteLLM gateway (the compose or ' 'kubeai backend, with `litellm` on)') @@ -1940,9 +1936,10 @@ def publish_profile(self, profile: dict) -> ReconcileResult: from .profile import ProfileMismatch from .residency import ResidencyUnknown - use = getattr(self.backend, 'use_profile', None) - if use is None: + recovery = self.backend.recovery_profile + if recovery is None: raise ProfileMismatch('this backend has no publishable profile') + use = recovery.use_profile stored = self.ledger.profile() if stored is not None and stored.get('backend') != profile.get('backend'): # The quiescence check below can only see the NEW backend's diff --git a/tests/test_leasing_controller_queue.py b/tests/test_leasing_controller_queue.py index 2f05038f..6db265c7 100644 --- a/tests/test_leasing_controller_queue.py +++ b/tests/test_leasing_controller_queue.py @@ -321,10 +321,11 @@ def _sleep(dt): assert backend.idle_plan_calls == 1 # consulted once, said "keep waiting" -def test_backend_without_an_idle_planner_is_unaffected(): - """null/kubeai have no plan_on_idle_host: the check is skipped, not failed.""" +def test_an_in_process_idle_planner_is_the_ordinary_plan(): + """Every backend has plan_on_idle_host (it is part of the protocol); the + in-process one is its ordinary plan, so a lease that fits alone is + feasible, as when the check used to be skipped.""" ctl, backend, _, _ = _make(budget=1, sleep=_advance_only) - assert not hasattr(backend, 'plan_on_idle_host') a = ctl.acquire('alice', [vreq('A')]) assert ctl._infeasible_alone(a.deployments, {g.id for g in a.deployments}) == {} From abcd03d7ce8e3484d87658981b43e1777aa80264 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:22:42 -0400 Subject: [PATCH 095/141] queue: re-review items closed; roadmap and parity rows follow Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/backend-parity.md | 8 +++--- docs/planning/backend-parity-roadmap.md | 2 +- docs/queue.md | 38 +++++++++++++++++++++---- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/docs/backend-parity.md b/docs/backend-parity.md index ef136b3a..79006962 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -62,7 +62,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | one `OPENAI_BASE_URL`, the managed key, the alias as model name | same | same through the gateway; with `litellm false`, Model names and no key | -| `secrets rotate` | same; a failure before the apply began restores the old key, after it the pending publication converges to the new one | same | +| `secrets rotate` | same; a failure before the apply began, or an apply that did not reach the runtime, restores the old key; after the runtime changed, the pending publication converges to the new one | same | | env file (`INFER_STACK_*`, `OPENAI_*`) | same | same | | readiness is a real generation through the front door | same | same | @@ -77,12 +77,12 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | a preview (admission, `config publish`) writes nothing, not even a first secret | same | same | | an apply reports how far it got (runtime, routes); the publication and its approved render stay pending until both | same | same, the host gateway's routes included | | `--no-apply` / `apply` / `render` | same | same | -| unleased keep-warm yields to leased demand | at placement | ≈ during the wait, one per 30 s: only when the scheduler reports a capacity shortage, and only an idle Model on a node the pod could use, holding the resource that is short (longest idle first) | +| unleased keep-warm yields to leased demand | at placement | ≈ during the wait, one per 30 s: only when the scheduler reports a capacity shortage, and only an idle Model on a node the pod could use, holding the resource that is short (longest idle first); nothing for a pod on another scheduler or with pod (anti-)affinity or a hard spread | | crash-loop fail-fast, the engine's error quoted | same | same (pods, `kubectl logs --previous`) | | image-pull progress | yes | n/a: the kubelet pulls; `ImagePullBackOff` is a reported wait reason | | strict `residency()` for decisions, lenient `observe()` for reports | same: two containers for one deployment are a conflict, and decisions fail closed | same: several pods for one Model are its replicas (or a rollout), resident while any is warm | | coalescing: an adequate LIVE deployment before a resident IDLE one before any other IDLE one | same | same | -| recovery after an interrupted apply (settle check) | yes | ≈ `kubectl apply` is idempotent; nothing is left half-created | +| recovery after an interrupted apply (settle check) | yes | same for the host gateway (a Compose project); `kubectl apply` itself is idempotent | ### Placement @@ -125,7 +125,7 @@ a different mechanism. **gap**: missing on one side and on the roadmap. | | Compose | KubeAI | |---|---|---| | `doctor` | nothing to check | four checks | -| `ps`, `logs [-f] `, `status` health | containers | same shape; pods, and the gateway's containers | +| `ps`, `logs [-f] `, `status` health | containers | same shape; pods, and the gateway's containers; health is serving (up / starting / restarting), not residency | | `stack up` (= `apply`), `stack down` | same | same | | `stack compose …`, `restart` / `pull` / `start` / `stop` | the stack's Compose project | the gateway's Compose project (the engines are pods) | | TUI: leases, deployments, catalog editing, acquire / release / evict, API tab, settings | same | same | diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index efa6daab..c494eee8 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,7 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **features complete; review hardening done, re-review items open (queue 11, 14, 15, 16, 21, 23)**: +`dev/backend-unification` · **features complete; review hardening and re-review done (queue 11-23)**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover run on real hardware, `dev/handover/`); a two-part outside review the same day found replica residency, the backend protocol, scheduler reclaim, publication diff --git a/docs/queue.md b/docs/queue.md index bdd392b8..0921fffe 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -446,7 +446,7 @@ poison test that fails on the old code with every fix; no new `hasattr` probes; no second KubeAI lifecycle path; backwards compatibility for cards, catalogs and CLI invocations that work today. -### 11. [ ] Residency: replicas are not duplicates (reopened by the re-review) +### 11. [x] Residency: replicas are not duplicates (reopened by the re-review, closed) *Why added:* `Residency.ambiguous()` meant "more than one unit" and `resident()` returned a unit only when there was exactly one. On Compose two @@ -541,7 +541,7 @@ LIVE-over-IDLE coalescing (`test_leasing_controller.py`); seed conflicts the positive control for reclaimable pressure, which passes on both. The rule is in the roadmap's Principles. -### 14. [ ] Scheduler-aware reclaim, not "Unschedulable means evict" (reopened by the re-review) +### 14. [x] Scheduler-aware reclaim, not "Unschedulable means evict" (reopened by the re-review, closed) *Why added:* KubeAI's readiness sets `needs_room` whenever a pod waits with reason `Unschedulable`, and `wait_ready` answers by evicting the @@ -573,7 +573,7 @@ request the short resource. Compose returns none, the in-process backends all. `_make_room` keeps the policy. Four parity poison tests; three fail on the old code. -### 15. [ ] Publication phases: the approved digest outlives a partial apply (reopened by the re-review) +### 15. [x] Publication phases: the approved digest outlives a partial apply (reopened by the re-review, closed) *Why added:* `_apply_pending()` clears the approved-render digest right after `apply()` returns, before checking it returned `False`. Compose @@ -600,7 +600,7 @@ routes behind KubeAI cleared the marker; it now returns both. Tests: the partial-apply/renderer-drift case (fails on the old controller: the digest was gone) and the KubeAI propagation. -### 16. [ ] Secret rotation is a transaction (reopened by the re-review) +### 16. [x] Secret rotation is a transaction (reopened by the re-review, closed) *Why added:* `rotate_gateway_key()` writes the new key into `.env`, then publishes, and restores the old key only on `ConvergeAborted`. A strict @@ -728,7 +728,7 @@ preview's digest. Test: a refused acquire on a fresh root leaves no key in `.env` (fails on the old code: it wrote the master key and the Open WebUI secret), and the next admitted acquire needs no second approval. -### 21. [ ] Controller decomposition, where the authorities now show it (reopened by the re-review) +### 21. [x] Controller decomposition, where the authorities now show it (reopened by the re-review, closed) *Why added:* the controller holds admission, residency interpretation, placement, publication markers, recovery snapshots, network migration, @@ -832,4 +832,30 @@ bug first (a safety bug), then 16, 15, 11, 14, 23, then the rest of 21. typed `front_door() -> FrontDoor | None`. Rule: a `getattr` default may serve display, never a correctness decision. -### 23. [ ] A race-time `routes seed` conflict leaves no marker +### 23. [x] A race-time `routes seed` conflict leaves no marker + +*Done 2026-09-26 (30a62bb, 122f1b4):* `publish_change(..., preflight=)` runs +a check under the lock before the marker; route seed rechecks there. Test +injects the other process's write when the lock is taken (the old code left +the marker). The stale `--replace` display is not fixed. + +### Re-review closure (2026-09-26) + +Each with a test that fails on the code before it: + +- **21, settlement (4b8783b):** `settle_snapshot()` is required; KubeAI + delegates to its host gateway, `None` for an in-cluster one and in-process. +- **16 (4476573):** an apply returning `runtime=False` restores the old key + and refuses with the apply's reason (`ReconcileResult.apply_detail`). +- **15 (768d924):** `infer-stack apply` over a changed render records it as + approved (in place, same marker version) before applying. +- **11 (b8b3542):** replicas classified from their own logs (`_pod_logs`), + fatal only when each is; `deployment_health()` (up / starting / restarting + / conflicted) feeds `status` and `observe_state`, which flags RESTARTING. +- **14 (55c1f96):** `eligible_nodes()` returns unknown (evict nothing) for a + non-default `schedulerName`, required pod (anti-)affinity, or a hard + topology spread; tested through a profile's `scheduler_name`. +- **21, hooks (2f9adc4):** as classified above. No `getattr(self.backend, + ...)` remains in the controller; `ty` checks `FrontDoor` and + `RecoveryProfile` conformance, and removing Compose's `allocates_gpus` + fails it. From fde855e75df1e62d58c23ed5a1f68278221f39a1 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 20:23:05 -0400 Subject: [PATCH 096/141] dev: journal for the re-review pass Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/journals/claude.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/dev/journals/claude.md b/dev/journals/claude.md index 118f278e..0b006ec6 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3347,3 +3347,32 @@ lifecycle. (2) When a boolean crosses a boundary, ask what the caller does with the cases it merges; `True/False` and `needs_room` each merged cases that needed opposite actions. (3) Run long e2e passes from a frozen copy (lessons.md). + +## 2026-09-26 20:23:05 -0400 — re-review: second-order cases + +**Intent.** The re-review closed most of items 11-22 and reopened five, plus +a route-seed race, for states the richer abstractions exposed but did not +carry through; and it classified the optional backend hooks instead of +promoting them all. Model: Claude Opus 5.5 (claude-opus-5-5), Claude Code. + +**Did.** In the order the reviewer asked, each with a test failing on the +code before it: KubeAI settles its host gateway after an interrupted apply +(the safety bug: `settle_snapshot` was an optional probe KubeAI lacked); +rotation keys on "the runtime changed", not "apply was called"; an explicit +re-approval is durable; replicas are judged from their own logs and +`deployment_health` separates serving from residency; reclaim fails closed +on schedulers and constraints it does not model; a race-time seed conflict +leaves no marker; the hooks are required state, common operations, or two +typed nullable capabilities (`recovery_profile`, `FrontDoor`). + +**Mistake.** One commit went in with a failing test: the chain ran pytest +and then committed regardless. Fixed forward in the next commit; later +commits gate on pytest's exit code. + +**Takeaways.** (1) An optional structural probe is a silent default: absent +meant "nothing to settle" and "owns host GPUs", both wrong for a backend +that simply had not implemented them. Required members make the checker +catch it. (2) Physical state and serving state are different questions; +once residency became physical, every reader that meant "is it serving" +needed its own summary. (3) A race test must inject the other writer at the +lock, not before the call; otherwise an earlier unlocked check passes it. From 4d54d60b7c4ffecc2546c4aac774a11a1bf4f037 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 21:33:44 -0400 Subject: [PATCH 097/141] queue: final closure items 24-27 (rotation x dynamic routing first) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 2 +- docs/queue.md | 38 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index c494eee8..ba484b53 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,7 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **features complete; review hardening and re-review done (queue 11-23)**: +`dev/backend-unification` · **features complete; review hardening done except queue 24-27 (final closure pass)**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover run on real hardware, `dev/handover/`); a two-part outside review the same day found replica residency, the backend protocol, scheduler reclaim, publication diff --git a/docs/queue.md b/docs/queue.md index 0921fffe..be100888 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -859,3 +859,41 @@ Each with a test that fails on the code before it: ...)` remains in the controller; `ty` checks `FrontDoor` and `RecoveryProfile` conformance, and removing Compose's `allocates_gpus` fails it. + +## Final closure pass (2026-09-26) + +A third review found one cross-feature blocker and three small items. + +### 24. [ ] `secrets rotate` under a running dynamic-routing gateway + +*Why added:* rotation writes K1 to `.env` before publishing; the dynamic +apply first retires routes on the still-running gateway, whose admin API +accepts only K0, but authenticates with `.env`'s K1. Retirement cannot +list routes, the apply returns `runtime=False`, and rotation restores K0 +and refuses: with dynamic routing on (Compose, or KubeAI's host gateway) +rotation can never complete. The e2e rotated only after switching dynamic +routing off, and the unit fake ran without it. + +**Do:** before the gateway is recreated, admin calls use the credential the +running gateway holds; after, the desired one. Keep route retirement. +**Done when:** with dynamic routing and a gateway started on K0, rotation +succeeds: `.env` is K1, the gateway accepts K1 and rejects K0, routes +verify, no marker; on Compose and KubeAI's host gateway; and the e2e rotates +once while the host dynamic gateway is the front door. + +### 25. [ ] `SimpleAdmission` keeps `last_planned_digest`; the guard fails closed + +*Why added:* the in-process backends never set `last_planned_digest`, and +the approval guard skips when it is empty, so an approved digest was not +checked there. **Done when:** converge sets it from the same digest as +preview, and an approved digest with no rendered digest refuses. + +### 26. [ ] Say what `leases` health reports + +*Why added:* `observe_state` takes only `restarting` from +`deployment_health`, while the docs say the summary "feeds" it. **Done +when:** code and docs agree. + +### 27. [ ] Record the `routes seed --replace` confirmation race as deferred + +**Done when:** it is in `known-limitations.md`. From 45e0e1c1697f2f15737232f341a9a7340f6554c6 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 21:38:15 -0400 Subject: [PATCH 098/141] Rotation completes under a running dynamic-routing gateway A rotation writes the new key to .env before the gateway is recreated, and a dynamic apply's first phase retires routes on the gateway still running the old key. That phase now authenticates with the key the running container holds (read from its environment, so a crash mid-rotation recovers too); the recreated gateway runs the new key and the routes verify with it. Before, retirement failed, the apply missed the runtime, and the rotation restored the old key every time (Compose and KubeAI's host gateway). The e2e now rotates inside the dynamic-routing phase. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/kubeai_e2e.sh | 18 ++++++ infer_stack/leasing/compose.py | 25 ++++++-- infer_stack/leasing/gateway.py | 26 ++++++++- tests/fake_docker_state.py | 29 +++++++++- tests/test_leasing_secrets.py | 103 +++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+), 6 deletions(-) diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 9c3942ef..14a13709 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -338,7 +338,25 @@ if [ "${GATEWAY:-1}" = 1 ] && [ "${E2E_DYNAMIC:-1}" = 1 ]; then else echo "!! no generation for $ALIAS under dynamic routing" >&2; exit 1 fi + # Rotation while this host's dynamic-routing gateway is the front door: + # the apply first retires routes on the gateway still running the OLD key, + # then recreates it with the new one (queue item 24). run_is release --env-file "$WORK/dyn1.env" --yes + run_is secrets rotate --force > "$WORK/dyn-rotate.log" 2>&1 \ + || { cat "$WORK/dyn-rotate.log" >&2; echo '!! rotation under dynamic routing failed' >&2; exit 1; } + grep -q 'new key accepted, old key rejected' "$WORK/dyn-rotate.log" \ + || { cat "$WORK/dyn-rotate.log" >&2; exit 1; } + new_key=$(run_is env LITELLM_MASTER_KEY) + [ "$new_key" != "$OPENAI_API_KEY" ] || { echo '!! the key did not change' >&2; exit 1; } + if curl -sS --fail-with-body "$OPENAI_BASE_URL/chat/completions" \ + -H "Authorization: Bearer $new_key" -H 'Content-Type: application/json' \ + -d "{\"model\": \"$ALIAS\", \"max_tokens\": 4, + \"messages\": [{\"role\": \"user\", \"content\": \"say ok\"}]}" \ + | grep -q 'choices'; then + echo ' secrets rotate under dynamic routing: the alias answers with the new key' + else + echo "!! no generation for $ALIAS with the rotated key" >&2; exit 1 + fi run_is release --env-file "$WORK/dyn2.env" --yes fi diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 3171b0c9..4dc56240 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -53,6 +53,7 @@ from ..profile_runtime import simulator_args, vllm_args from .backend import ApplyResult, ConvergeScaffold, Readiness from .gateway import ( + API_KEY_ENV, LITELLM_CONFIG_FILENAME, LITELLM_SERVICE, NGINX_CONFIG_FILENAME, @@ -1985,12 +1986,28 @@ def retire_routes(self) -> bool: if not (self.litellm and self.dynamic_routing): return True - running = any(c.service == 'litellm' and c.warm - for c in self.residency().all_containers()) + running = [c for c in self.residency().all_containers() + if c.service == 'litellm' and c.warm] if not running: return True - return self.gateway._reconcile_routes( - deadline_s=ROUTE_RECONCILE_STEADY_S, retire_only=True) + # This phase talks to the gateway as it runs NOW, which is not yet + # recreated: after a rotation it holds the old key, not the .env's. + with self.gateway.live_credential(self._container_env(running[0], API_KEY_ENV)): + return self.gateway._reconcile_routes( + deadline_s=ROUTE_RECONCILE_STEADY_S, retire_only=True) + + def _container_env(self, container, name: str) -> str | None: + """One variable of a running container's environment, as Docker + started it; ``None`` when it cannot be read.""" + try: + info = json.loads(self.run(['docker', 'inspect', container.container_id]) or '[]') + except Exception: # noqa: BLE001 - unknown: the caller keeps the managed value + return None + prefix = f'{name}=' + for item in ((info[0] if info else {}).get('Config') or {}).get('Env') or []: + if str(item).startswith(prefix): + return str(item)[len(prefix):] + return None def _wait_until(self, predicate, *, deadline_s: float, what: str, interval: float = 1.0): """Poll strict residency until ``predicate(snapshot)``; abort at the deadline.""" diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index d463b33b..c78dc966 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -1317,8 +1317,32 @@ def _gateway_base(self) -> str: base = where() return base.rstrip('/') + #: The key the RUNNING gateway holds, while it differs from the managed + #: one (see :meth:`live_credential`); ``None``: the managed key. + _live_key: str | None = None + + def live_credential(self, key: str | None): + """While active, admin calls authenticate with ``key``. + + A rotation writes the new key to the ``.env`` before the gateway is + recreated with it, and an apply's first phase (retiring routes) talks + to the gateway still running on the old one. ``None`` (the running + key is unknown) keeps the managed key. + """ + import contextlib + + @contextlib.contextmanager + def scope(): + previous, self._live_key = self._live_key, key + try: + yield + finally: + self._live_key = previous + + return scope() + def _auth_headers(self) -> dict[str, str]: - return {'Authorization': f'Bearer {self.master_key()}'} + return {'Authorization': f'Bearer {self._live_key or self.master_key()}'} def _desired_routes(self) -> list[dict[str, Any]]: """The rendered desired route set (litellm_routes.json), or empty.""" diff --git a/tests/fake_docker_state.py b/tests/fake_docker_state.py index 04145c1b..ca5366cc 100644 --- a/tests/fake_docker_state.py +++ b/tests/fake_docker_state.py @@ -73,7 +73,9 @@ def __call__(self, args, **_): state['Health'] = {'Status': c['health']} out.append({'Id': cid, 'State': state, 'RestartCount': c.get('restart_count', 0), - 'Config': {'Labels': c['labels']}, + 'Config': {'Labels': c['labels'], + 'Env': [f'{k}={v}' for k, v in + sorted((c.get('env') or {}).items())]}, 'HostConfig': {'DeviceRequests': requests, 'RestartPolicy': { 'Name': c.get('restart_policy', ''), @@ -118,6 +120,8 @@ def __call__(self, args, **_): self.compose_file = args[args.index('-f') + 1] if '-p' in args: self.project = args[args.index('-p') + 1] + if '--env-file' in args: + self.env_file = args[args.index('--env-file') + 1] if 'up' in args: services = self._services() for name, spec in (self._doc().get('networks') or {}).items(): @@ -169,6 +173,29 @@ def _up_service(self, name, svc): self.containers[cid]['restart_policy'] = str(svc.get('restart') or '') if svc.get('healthcheck'): self.containers[cid]['health'] = self.initial_health + self.containers[cid]['env'] = self._interpolated(svc.get('environment') or {}) + + def _interpolated(self, environment) -> dict: + """The service's environment as compose sets it: ${VAR} and + ${VAR:-default} from the --env-file, as the container starts.""" + import re + + values = {} + env_file = getattr(self, 'env_file', None) + if env_file and Path(env_file).exists(): + for line in Path(env_file).read_text().splitlines(): + if '=' in line and not line.lstrip().startswith('#'): + key, _, value = line.partition('=') + values[key.strip()] = value.strip().strip('"').strip("'") + if isinstance(environment, list): + environment = dict(item.split('=', 1) for item in environment if '=' in item) + + def sub(match): + name, _, default = match.group(1).partition(':-') + return values.get(name, default) + + return {str(k): re.sub(r'\$\{([^}]+)\}', sub, str(v)) + for k, v in environment.items()} def answer_residency(args, *, compose_file, running, project): # pragma: no cover - legacy diff --git a/tests/test_leasing_secrets.py b/tests/test_leasing_secrets.py index e136b844..a6cbda50 100644 --- a/tests/test_leasing_secrets.py +++ b/tests/test_leasing_secrets.py @@ -242,3 +242,106 @@ def test_an_apply_that_cleanly_misses_the_runtime_keeps_the_old_key(tmp_path): ctl.rotate_gateway_key() assert env(ctl)[API_KEY_ENV] == old assert ctl.backend.gateway_accepts(env(ctl)[API_KEY_ENV]) # what clients get works + + +# -- rotation under dynamic routing (queue item 24) --------------------------------- + + +class AdminGateway(StartedGateway): + """A dynamic-routing LiteLLM: the admin API, answering only the key its + container started with (read from the container's environment).""" + + def __init__(self, env_path, docker): + super().__init__(env_path, docker) + self.routes: dict[str, dict] = {} + + def _started_key(self): + current = next((c for c in self.docker.containers.values() + if c['service'] == 'litellm'), None) + return (current or {}).get('env', {}).get(API_KEY_ENV) + + def _authorized(self, headers): + return bool(headers) and headers.get('Authorization') == f'Bearer {self._started_key()}' + + def get(self, url, headers=None, **kw): + if url.endswith('/v1/model/info'): + if not self._authorized(headers): + return FakeResp(400, {'error': 'Authentication Error'}) + return FakeResp(200, {'data': list(self.routes.values())}) + if headers and headers.get('Authorization') == f'Bearer {self._started_key()}': + return FakeResp(200, {'data': []}) + return FakeResp(400, {'error': 'Authentication Error'}) + + def post(self, url, headers=None, **kw): + if not self._authorized(headers): + return FakeResp(400, {'error': 'Authentication Error'}) + body = kw.get('json') or {} + if url.endswith('/model/new'): + self.routes[body['model_info']['id']] = body + elif url.endswith('/model/delete'): + self.routes.pop(body['id'], None) + return FakeResp(200, {}) + + +def test_rotation_completes_under_a_running_dynamic_routing_gateway(tmp_path): + """The running gateway holds K0; rotation writes K1 first. The apply's + route-retirement phase must still talk to the gateway with K0, the + recreated gateway then runs K1, and the routes verify with K1.""" + state = tmp_path / 'state' + clock = Clock() + docker = FakeDocker() + backend = ComposeBackend( + state_dir=state, inventory=simulate_inventory('2x80'), run=docker, + http=None, images={**IMAGES, 'postgres': 'pg:test'}, ports=PORTS, state=STATE, + catalog=CAT, litellm=True, ui=False, dynamic_routing=True, + sleep=clock.sleep, clock=clock, + ) + backend.http = backend.gateway.http = AdminGateway(backend.gateway._env_path, docker) + ledger = Ledger(SqliteStore(str(tmp_path / 'ledger.db')), clock=clock) + ctl = Controller(ledger, backend, clock=clock, sleep=clock.sleep) + ctl.release(acquire(ctl, 'one').lease.id) # a keep-warm model, routed + old = env(ctl)[API_KEY_ENV] + assert backend.gateway_accepts(old) and backend.http.routes + + rec = ctl.rotate_gateway_key() + + new = env(ctl)[API_KEY_ENV] + assert new != old and not rec.publication_pending + assert backend.gateway_accepts(new) and not backend.gateway_accepts(old) + assert backend.http.routes # verified with the new key + assert ledger.publication_pending() is None + + +def test_rotation_completes_behind_kubeai_with_a_dynamic_host_gateway(tmp_path): + """The same interaction on KubeAI: its host gateway is the same Compose + machinery, so route retirement must use the running gateway's key.""" + from infer_stack.backends.kubeai import KubeaiBackend + from test_leasing_kubeai import FakeKubectl + + clock = Clock() + docker = FakeDocker() + gateway = ComposeBackend( + state_dir=tmp_path / 'gateway', inventory={'gpu_count': 0, 'gpus': []}, + run=docker, http=None, project='infer-stack-gateway', + images={**IMAGES, 'postgres': 'pg:test'}, ports=PORTS, state=STATE, + litellm=True, ui=False, dynamic_routing=True, sleep=clock.sleep, clock=clock, + ) + gateway.http = gateway.gateway.http = AdminGateway(gateway.gateway._env_path, docker) + backend = KubeaiBackend(state_dir=tmp_path / 'kubeai', run=FakeKubectl(), + http=gateway.http, gateway=gateway, + gateway_upstream='http://10.43.0.9/openai/v1', + default_resource_profile='gpu') + backend.catalog = CAT + ledger = Ledger(SqliteStore(str(tmp_path / 'ledger.db')), clock=clock) + ctl = Controller(ledger, backend, clock=clock, sleep=clock.sleep) + ctl.release(acquire(ctl, 'one').lease.id) + env_path = gateway.gateway._env_path + old = parse_env_file(env_path)[API_KEY_ENV] + assert gateway.gateway_accepts(old) + + ctl.rotate_gateway_key() + + new = parse_env_file(env_path)[API_KEY_ENV] + assert new != old + assert gateway.gateway_accepts(new) and not gateway.gateway_accepts(old) + assert ledger.publication_pending() is None From 3abb2ffd1f7f7679853abedd016ac04a1c708d87 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 21:43:54 -0400 Subject: [PATCH 099/141] The in-process backends keep last_planned_digest; the guard fails closed SimpleAdmission's converge records the digest from the same representation as its preview, so the approval guard is exercised in-process too; the guard refuses an approved render when the backend cannot name the one it is about to apply, instead of skipping. Test backends that override converge keep the same contract. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- infer_stack/leasing/backend.py | 14 +++++++-- infer_stack/leasing/controller.py | 7 +++-- tests/test_leasing_controller.py | 32 ++++++++++++++++++++ tests/test_leasing_controller_lock.py | 8 +++++ tests/test_leasing_serialised_publication.py | 16 ++++++++++ 5 files changed, 72 insertions(+), 5 deletions(-) diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index fa4dabfe..4d592136 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -608,12 +608,19 @@ def _preview(self, desired: list[Deployment], placement: Any = None): refused = self.refuse(desired) return plan, RenderPreview(set(refused), [f'{g}: {why}' for g, why in refused.items()]) + @staticmethod + def _digest(plan, rendered) -> str: + """What an in-process render amounts to: who is placed where. One + representation for the preview and the converge, so the approval + guard compares like with like.""" + return repr(sorted( + (gid, list(gpus)) for gid, gpus in plan.assignments.items() + if gid not in rendered.unrenderable)) + def preview(self, desired: list[Deployment], placement: Any = None, *, approve: bool = False): plan, rendered = self._preview(desired, placement) - self.last_preview_digest = repr(sorted( - (gid, list(gpus)) for gid, gpus in plan.assignments.items() - if gid not in rendered.unrenderable)) + self.last_preview_digest = self._digest(plan, rendered) return plan, rendered def converge(self, desired: list[Deployment], *, apply: bool = True, @@ -621,6 +628,7 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, """Record the renderable, placed part of ``desired``; ``apply`` realizes it.""" desired = list(desired) plan, rendered = self._preview(desired, placement) + self.last_planned_digest = self._digest(plan, rendered) self.last_errors = list(plan.errors) + list(rendered.errors) self.last_unplaced = { g.id for g in desired diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 6042f146..a49dc1f5 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1258,7 +1258,9 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: return rec approved = marker.get('approved_digest') rendered = self.backend.last_planned_digest - if approved and rendered and rendered != approved: + # Fail closed: an approved render and no digest for this one is not + # proof they are the same. + if approved and rendered != approved: if not self._explicit_apply: from .profile import ProfileMismatch @@ -1272,7 +1274,8 @@ def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: # applying, so a partial apply leaves THIS render approved and an # ordinary retry of it proceeds (a later, different one still # needs approval). - self.ledger.reapprove_render(rendered) + if rendered is not None: + self.ledger.reapprove_render(rendered) approved = rendered if marker['interrupted']: self._wait_for_settled_runtime() diff --git a/tests/test_leasing_controller.py b/tests/test_leasing_controller.py index 27cf8fd0..608fcb0a 100644 --- a/tests/test_leasing_controller.py +++ b/tests/test_leasing_controller.py @@ -273,3 +273,35 @@ def test_a_resident_idle_deployment_beats_an_older_one_that_is_gone(): out = ctl.acquire('c', [vreq('m', max_model_len=4096)]) assert out.deployments[0].id == new_id + + +# -- the approval guard in-process (queue item 25) --------------------------------- + + +def test_in_process_converge_records_the_digest_its_preview_approved(): + ctl, backend, _ = make_controller(ready=True) + ctl.acquire('a', [vreq('m')]) + assert backend.last_planned_digest is not None + assert backend.last_planned_digest == backend.last_preview_digest + + +def test_an_approved_render_with_no_rendered_digest_is_refused(): + import pytest + + from infer_stack.leasing.profile import ProfileMismatch + + ctl, backend, _ = make_controller(ready=True) + ctl.acquire('a', [vreq('m')]) + ctl.ledger.mark_publication_pending(apply_requested=True, approved_digest='d1') + real = type(backend).converge + + def forgetful(self, desired, **kw): + real(self, desired, **kw) + self.last_planned_digest = None # a backend that lost track + + type(backend).converge = forgetful + try: + with pytest.raises(ProfileMismatch, match='approved'): + ctl.gc() + finally: + type(backend).converge = real diff --git a/tests/test_leasing_controller_lock.py b/tests/test_leasing_controller_lock.py index 74d10cb0..95790fd4 100644 --- a/tests/test_leasing_controller_lock.py +++ b/tests/test_leasing_controller_lock.py @@ -45,6 +45,10 @@ def __init__(self) -> None: self.last_assignments: dict = {} def converge(self, desired, *, apply: bool = True, placement=None) -> None: + + # The protocol's digest contract, as SimpleAdmission keeps it. + + self.last_planned_digest = self._digest(*self._preview(list(desired), placement)) with self._guard: self.active += 1 self.max_active = max(self.max_active, self.active) @@ -186,6 +190,10 @@ def __init__(self, shared: dict, guard: threading.Lock) -> None: self._ids: set = set() def converge(self, desired, *, apply: bool = True, placement=None) -> None: + + # The protocol's digest contract, as SimpleAdmission keeps it. + + self.last_planned_digest = self._digest(*self._preview(list(desired), placement)) with self.guard: self.shared['active'] += 1 self.shared['max'] = max(self.shared['max'], self.shared['active']) diff --git a/tests/test_leasing_serialised_publication.py b/tests/test_leasing_serialised_publication.py index f056b5ec..b88b5d72 100644 --- a/tests/test_leasing_serialised_publication.py +++ b/tests/test_leasing_serialised_publication.py @@ -64,6 +64,10 @@ def __init__(self, shared: dict, guard: threading.Lock, apply_sleep: float = 0.0 self.last_assignments: dict[str, list[int]] = {} def converge(self, desired, *, apply: bool = True, placement=None) -> None: + + # The protocol's digest contract, as SimpleAdmission keeps it. + + self.last_planned_digest = self._digest(*self._preview(list(desired), placement)) ids = {g.id for g in desired} with self.guard: self.shared['rendered'] = set(ids) @@ -446,6 +450,10 @@ def refuse(self, desired): return {g.id: 'no room' for g in desired if 'big' in g.served} def converge(self, desired, *, apply: bool = True, placement=None) -> None: + + # The protocol's digest contract, as SimpleAdmission keeps it. + + self.last_planned_digest = self._digest(*self._preview(list(desired), placement)) refused = self.refuse(desired) self.last_unplaced = list(refused) super().converge([g for g in desired if g.id not in refused], apply=apply) @@ -463,6 +471,10 @@ def refuse(self, desired): return {} def converge(self, desired, *, apply: bool = True, placement=None) -> None: + + # The protocol's digest contract, as SimpleAdmission keeps it. + + self.last_planned_digest = self._digest(*self._preview(list(desired), placement)) refused = [g.id for g in desired if 'big' in g.served] SharedStackBackend.converge( self, [g for g in desired if g.id not in refused], apply=apply) @@ -554,6 +566,10 @@ def __init__(self, *args, **kwargs): self.declined = False def converge(self, desired, *, apply: bool = True, placement=None) -> None: + + # The protocol's digest contract, as SimpleAdmission keeps it. + + self.last_planned_digest = self._digest(*self._preview(list(desired), placement)) self.converge_calls += 1 if desired and not self.declined: self.declined = True From cfe6512c0f95748bc30d729d95721a56307d0236 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 21:44:22 -0400 Subject: [PATCH 100/141] Docs say what leases health reports; the seed --replace race is recorded leases flags only a crash loop from the serving summary (a starting deployment reads running there; status shows starting): the docstrings and the queue now say that instead of "feeds". The --replace confirmation race is a deferred entry in known-limitations.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/known-limitations.md | 10 ++++++++++ docs/queue.md | 14 +++++++++++--- infer_stack/leasing/controller.py | 9 ++++++--- infer_stack/leasing/residency.py | 3 +++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index 0a73a9c6..0c8bfecb 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -310,3 +310,13 @@ migrated keep Docker's dynamic addresses. - after a misroute, stop traffic for about five minutes before retrying; - setting `AIOHTTP_TTL_DNS_CACHE` low on the gateway narrows the window but does not close it. + +### `routes seed --replace` confirms against a snapshot (current, deferred) + +`routes seed --replace` shows each redefinition (`A0 -> A1`) and asks, then +commits under the publication lock. If another process changes the alias +to `A2` in between, the commit writes `A1` over `A2` without showing `A2`: +the replace is "replace whatever is there", not compare-and-swap. A +conflict that appears without `--replace` still refuses, and leaves no +publication pending. A strict version would carry the expected old row in +the plan and ask again on a mismatch under the lock. diff --git a/docs/queue.md b/docs/queue.md index be100888..d84c02c5 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -851,7 +851,8 @@ Each with a test that fails on the code before it: approved (in place, same marker version) before applying. - **11 (b8b3542):** replicas classified from their own logs (`_pod_logs`), fatal only when each is; `deployment_health()` (up / starting / restarting - / conflicted) feeds `status` and `observe_state`, which flags RESTARTING. + / conflicted) is what `status` shows; `observe_state` takes only + `restarting` from it and `leases` flags RESTARTING (item 26). - **14 (55c1f96):** `eligible_nodes()` returns unknown (evict nothing) for a non-default `schedulerName`, required pod (anti-)affinity, or a hard topology spread; tested through a profile's `scheduler_name`. @@ -888,12 +889,19 @@ the approval guard skips when it is empty, so an approved digest was not checked there. **Done when:** converge sets it from the same digest as preview, and an approved digest with no rendered digest refuses. -### 26. [ ] Say what `leases` health reports +### 26. [x] Say what `leases` health reports *Why added:* `observe_state` takes only `restarting` from `deployment_health`, while the docs say the summary "feeds" it. **Done when:** code and docs agree. -### 27. [ ] Record the `routes seed --replace` confirmation race as deferred +*Done 2026-09-26:* the behaviour stays: `leases` flags a crash loop, the +state that needs an operator; a deployment still starting reads `running` +there, and `status` shows `starting`. The docstrings of `observe_state` and +`deployment_health` and the item 11 note now say so. + +### 27. [x] Record the `routes seed --replace` confirmation race as deferred **Done when:** it is in `known-limitations.md`. + +*Done 2026-09-26:* recorded there as current, deferred. diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index a49dc1f5..85897adf 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -734,10 +734,13 @@ def observe_state(self) -> dict: * ``unresolved``: LIVE from before allocations, with no unique container; * ``reclaimed``: idle under ``stop``, and its container is gone, as intended (see :meth:`keeps_up`); - * ``restarting``: resident, but every unit is crash-looping; + * ``restarting``: resident, but every unit is crash-looping (the one + serving state taken from + :func:`~infer_stack.leasing.residency.deployment_health`: it needs + an operator, and ``leases`` flags it); * ``running`` / ``not-running``: whether it is resident (any replica - warm), from :func:`~infer_stack.leasing.residency.deployment_health` - for whether it serves. + warm). A deployment still starting is ``running`` here, since + startup needs no one; ``status`` shows ``starting``. """ from .profile import profile_drift from .residency import ResidencyUnknown, deployment_health diff --git a/infer_stack/leasing/residency.py b/infer_stack/leasing/residency.py index 8b43fb63..c7adf0ef 100644 --- a/infer_stack/leasing/residency.py +++ b/infer_stack/leasing/residency.py @@ -263,6 +263,9 @@ def deployment_health(residency: Residency, deployment_id: str) -> str | None: * ``starting``: none up, some running but not ready yet, or created; * ``restarting``: every warm unit crash-looping (restarting); * ``conflicted``: units that may not coexist (see :meth:`Residency.is_conflicted`); + + ``status`` shows it as is; ``leases`` (``Controller.observe_state``) + takes only ``restarting`` from it, the state that needs an operator. * otherwise the units' states (``exited``, ``removing`` ...), or ``None`` when the deployment has no unit. From 5337815df115abf152c202261a96840256994646 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 22:11:19 -0400 Subject: [PATCH 101/141] secrets rotate waits for the old key's rejection behind a rollout The in-cluster gateway's rollout completes while the old pod is still terminating and behind the NodePort Service for a moment, so one early acceptance of the old key failed the rotation's check (seen once in the final e2e). The check now polls for the rejection, bounded at 60 s. Queue items 24 and 25 recorded as done. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 18 ++++++++++++++++-- infer_stack/cli/commands_leasing.py | 20 +++++++++++++++++++- tests/test_leasing_secrets.py | 20 ++++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index d84c02c5..c62c567e 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -865,7 +865,7 @@ Each with a test that fails on the code before it: A third review found one cross-feature blocker and three small items. -### 24. [ ] `secrets rotate` under a running dynamic-routing gateway +### 24. [x] `secrets rotate` under a running dynamic-routing gateway *Why added:* rotation writes K1 to `.env` before publishing; the dynamic apply first retires routes on the still-running gateway, whose admin API @@ -882,13 +882,27 @@ succeeds: `.env` is K1, the gateway accepts K1 and rejects K0, routes verify, no marker; on Compose and KubeAI's host gateway; and the e2e rotates once while the host dynamic gateway is the front door. -### 25. [ ] `SimpleAdmission` keeps `last_planned_digest`; the guard fails closed +*Done 2026-09-26 (45e0e1c):* route retirement (the phase before the gateway +is recreated) authenticates with the key the running LiteLLM container +holds, read from its environment by `docker inspect`, through +`Gateway.live_credential()`; everything after recreation uses the `.env` +key. No new durable state, so a crash mid-rotation recovers the same way. +Unit tests on Compose and on KubeAI with a host gateway use a fake whose +admin API answers only the key its container started with (both fail on +the old code); the e2e rotates inside the dynamic-routing phase with one +dedicated lease live, and the alias answers with the new key. + +### 25. [x] `SimpleAdmission` keeps `last_planned_digest`; the guard fails closed *Why added:* the in-process backends never set `last_planned_digest`, and the approval guard skips when it is empty, so an approved digest was not checked there. **Done when:** converge sets it from the same digest as preview, and an approved digest with no rendered digest refuses. +*Done 2026-09-26 (3abb2ff):* `SimpleAdmission._digest(plan, rendered)` for +both; the guard is `approved and rendered != approved`. Six test backends +that override `converge` had not kept the contract; they do now. + ### 26. [x] Say what `leases` health reports *Why added:* `observe_state` takes only `restarting` from diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index f8140897..451e8ed8 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2954,6 +2954,24 @@ def main(cls, argv=True, **kwargs): return 4 if any(r['status'] == 'routing-fault' for r in result.values()) else 0 +def _old_key_rejected(backend, key, *, within: float = 60.0) -> bool: + """Whether the gateway comes to reject ``key`` within ``within`` seconds. + + A replaced gateway can still answer for a moment: a Kubernetes rollout is + complete while the old pod is terminating and still behind the Service, + so one early "accepted" is not a failure. Polled, bounded. + """ + import time + + deadline = time.monotonic() + within + while True: + if backend.gateway_accepts(key) is False: + return True + if time.monotonic() >= deadline: + return False + time.sleep(2.0) + + class SecretsRotateCLI(_ApprovalMixin): """Replace the LiteLLM master key and restart the gateway with it. @@ -2995,7 +3013,7 @@ def main(cls, argv=True, **kwargs): print(' gateway not running: it will use the new key when it starts') elif not accepted: raise SystemExit('secrets rotate: the gateway rejects the new key') - elif backend.gateway_accepts(old) is not False: + elif not _old_key_rejected(backend, old): raise SystemExit('secrets rotate: could not confirm the gateway ' 'rejects the OLD key') else: diff --git a/tests/test_leasing_secrets.py b/tests/test_leasing_secrets.py index a6cbda50..46dd7f72 100644 --- a/tests/test_leasing_secrets.py +++ b/tests/test_leasing_secrets.py @@ -345,3 +345,23 @@ def test_rotation_completes_behind_kubeai_with_a_dynamic_host_gateway(tmp_path): assert new != old assert gateway.gateway_accepts(new) and not gateway.gateway_accepts(old) assert ledger.publication_pending() is None + + +def test_the_old_key_may_linger_briefly_behind_a_rollout(monkeypatch): + """A rollout completes while the old pod still answers for a moment; the + rotation's check waits for the rejection instead of failing on it.""" + import time + + from infer_stack.cli.commands_leasing import _old_key_rejected + + monkeypatch.setattr(time, 'sleep', lambda s: None) + + class Lingering: + def __init__(self, answers): + self.answers = list(answers) + + def gateway_accepts(self, key, wait=0.0): + return self.answers.pop(0) if len(self.answers) > 1 else self.answers[0] + + assert _old_key_rejected(Lingering([True, True, False]), 'k') + assert not _old_key_rejected(Lingering([True]), 'k', within=0.0) From e8e9d42bc5d351d6f96144017f2ad643cecaf7fd Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 22:12:09 -0400 Subject: [PATCH 102/141] queue: campaign 2 (external endpoints, access above leasing), queued behind 11-27 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) diff --git a/docs/queue.md b/docs/queue.md index c62c567e..0e284945 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -919,3 +919,197 @@ there, and `status` shows `starting`. The docstrings of `observe_state` and **Done when:** it is in `known-limitations.md`. *Done 2026-09-26:* recorded there as current, deferred. + +--- + +# Campaign 2: external endpoints, and access above leasing + +**Start only after items 11-27 are committed, green (suite, `ty`, flake8, +full k3s e2e, UX audits) and the docs coherent. Do not interleave.** A +separately reviewable campaign, requested 2026-09-26. + +**Goal.** First-class catalog endpoints whose model server already exists +outside infer-stack, used as the occasion to remove conceptual duplication +in endpoint -> lease -> route -> access. User stories: (1) a workflow asks +for `qwen`; today infer-stack leases a local vLLM for it, tomorrow `qwen` +points at a running OpenAI-compatible server, and the workflow does not +change; (2) register a hardcoded OpenAI-compatible server as an endpoint +without a fake lease. + +**Target data flow** (fewer transformations than today): +catalog definition -> resolved endpoint meaning -> (managed: +`EndpointRequest` -> ledger) / (external: target) -> semantic gateway +route(s) -> gateway publication -> access descriptor. + +**Stop rule.** If the diff ends with more conceptual branches than today, +stop and reconsider the abstraction. Do not bolt `external` onto every +`if engine == ...`. If the code contradicts a conclusion below, stop and +document it instead of forcing it. + +## Conclusions to preserve + +- **Endpoint identity is not lease identity.** An external endpoint creates + no `Lease`, `Deployment`, demand, GPU allocation, TTL or reclaim state. + The ledger holds only what infer-stack owns. +- **"External" is not an engine.** No `engine: external`. Ownership is its + own axis: `Endpoint(alias, protocol, target: ManagedTarget(vllm|ollama) | + ExternalOpenAITarget(api_base, upstream model, api_key_env?))`. Existing + managed YAML is unchanged. Additive YAML: + `endpoints: {qwen-remote: {external: {api_base: ..., model: ..., + api_key_env: ...}, protocol: chat}}`. An external endpoint rejects + `runtime`, `placement`, `sharing`, `reclaim`, `host` (unless one proves a + meaning) and needs no `models:` entry. +- **The catalog is the authoring authority.** Routes are derived + publication state; `routes seed` stays for multi-runbook pre-seeding, but + registering an external endpoint is a catalog operation, protected by the + profile/catalog conflict machinery: identical definitions coexist, + different ones conflict, a pinned managed endpoint cannot be silently + turned external, and once it is unpinned the change is allowed. +- **Access through the front door only (first version).** One base URL, + the alias as model name, mixed bundles work, upstream credentials stay + behind the gateway. No implicit direct-to-external fallback. An external + endpoint without a LiteLLM front door fails clearly ("external catalog + targets currently require the front door"). +- **Externally owned means not controlled.** No startup, restart, + keep-warm, reclaim, placement for external targets. Publishing the route + is enough for `access` (no generation probe); an optional reachability + probe may report, never own. Views say `external`, not stopped / idle / + unplaced. + +## Work items (in order, small commits) + +### 28. [ ] Read, then write the design down first +Read catalog, profile, gateway, controller and access code together. Write +the data flow and invariants into a planning doc, and name the concepts +that become unnecessary (e.g. `UPSTREAM_ROUTE`, unreachable +`render_front_door` branches, lease-bound descriptors, route-derived +catalog equality). Gate for the rest. + +### 29. [ ] `ResolvedEndpoint`: endpoint meaning, not a lease request +One normalized object (alias, protocol, target) with a canonical semantic +key used for catalog-union conflicts, profile drift, and "did this endpoint +change". A managed target derives an `EndpointRequest`; an external one has +none. `CatalogUnion` / profile code stop importing gateway or Compose route +conversion (cleanup A). Transitional names may stay for compatibility. + +### 30. [ ] The external target in the catalog and its CLI +YAML `external:` block, validation of illegal mixtures, round-trip. +`catalog endpoint add` gains mutually exclusive external options (e.g. +`--external-api-base`, `--external-model`, `--external-api-key-env`), never +an `--engine` choice; reject `--engine vllm` + external, `--gpu`/`--reclaim` ++ external. `catalog show` makes ownership obvious. + +### 31. [ ] One terminology +Endpoint alias (what users request through the front door), upstream model +(what the target server expects as `model`), deployment id (one managed +realization). Stop calling `served_model_name` a "public name"; keep old +YAML spellings as compatibility aliases; all new code uses the three terms. + +### 32. [ ] One semantic `GatewayRoute` +Alias, upstream adapter (OpenAI-compatible / Ollama), upstream model, +api_base, optional credential-env name, optional managed dynamic route id. +Produced by Compose (managed service name), KubeAI (cluster gateway), +external targets (fixed URL), Ollama (its adapter); one renderer to +LiteLLM's model entry; `routes list` uses the same object. Remove +`UPSTREAM_ROUTE` if unneeded; persist a versioned serialization of the +route in the registry if it removes branches. Same route type, different +publication mechanisms (static registry vs dynamic reconciliation), not one +storage for everything. Delete `render_front_door` branches that only tests +reach (cleanup B), never a supported mode. + +### 33. [ ] External routes in static and dynamic routing +Dynamic mode: an external route is standing desired state with a +deterministic managed id from endpoint identity plus non-secret target +semantics (independent of any deployment). It survives zero leases, +releases, other acquires, gc, a dedicated deployment going away. Route +retirement (item 17) stays intact; infer-stack never tears down an external +upstream, only its route. + +### 34. [ ] External credentials by env reference +`api_key_env` only; never a literal key in the catalog, registry, rendered +route or logs. Render as a LiteLLM env reference; the gateway container +receives the variable from the managed `.env` (`infer-stack env KEY=...`). +Fail before apply if a declared key has no value; never generate provider +keys. Dynamic routing: LiteLLM redacts credentials when listing, so do not +read keys back; the route identity/fingerprint includes the env NAME (a +changed reference forces replacement), never the value; a changed value +triggers what the gateway needs to see it, without leaking it. + +### 35. [ ] Publication without a ledger mutation +External-only access may change nothing in the ledger but still needs: +check/incorporate the invocation catalog/profile, render gateway state, +persist routes, apply under the publication lock with the pending / +approved-render machinery. Factor it so managed acquire and external-only +access reach the same publication coordinator; no second ad-hoc gateway +apply path; no private gateway writes from the CLI. + +### 36. [ ] Access above leasing +An access result: requested endpoints, base URL, credential info, +alias -> request-model mapping, optional real lease, optional GPU +reservation metadata (external-only: `lease=None`; mixed: one real lease +for the managed subset; the lease never lists external names). The env +format omits `INFER_STACK_LEASE_ID` when there is no lease (no sentinel); +managed-only descriptors stay byte-identical where practical. A high-level +`prepare_access`/`acquire_access` partitions managed/external, leases only +managed, ensures publication, waits for managed runtime as today, returns +one result. `infer-stack access ... [--env-file]` for workflows without +`run`; `run` uses it. `acquire` stays lease-only: for external-only it +says "'qwen' is externally provided and does not require a lease; use +`infer-stack access qwen` or `infer-stack run ...`"; mixed bundles go +through access. `release` of external-only access has nothing to release. +Access is a typed capability, not a `getattr(backend, 'access')` hook +(cleanup D). + +### 37. [ ] Views: `external` +Status, TUI and catalog views show external targets as `external`. + +### 38. [ ] `FrontDoor` naming collision (cleanup C) +If both the controller capability and the rendered artifact are still +called `FrontDoor`, rename (e.g. `FrontDoorControl`, `RenderedFrontDoor`). + +### 39. [ ] Tests (not only parser tests) +1 external parses and round-trips semantically; 2 invalid mixtures fail +clearly; 3 no lease/deployment for external; 4 external-only access +publishes a route with zero ledger demand; 5 mixed bundle creates only +managed demand; 6 `run` works external-only and mixed; 7 managed-only `run` +unchanged; 8 external needs a usable front door; 9 static routing keeps +external routes across unrelated acquire/release/gc; 10 same for dynamic; +11 two catalogs, same alias and target, union cleanly; 12 same alias, +different target, conflict; 13 an active managed `qwen` cannot be silently +redefined external; 14 once unpinned the same edit succeeds; 15 external -> +managed likewise; 16 upstream key referenced by name, never serialized as +value; 17 missing required key fails before apply; 18 a changed key +reference forces route reconciliation; 19 external-only descriptor has no +fake lease id; 20 releasing a mixed access result affects only its real +lease; 21 `routes list` renders the same target as the gateway renderer; +22 external routes recover after a process restart from durable state. +Parity: Compose and KubeAI-with-gateway; on KubeAI an external route must +not be rewritten to point at the cluster gateway. +Mixed bundle acceptance (`access pair` / `run --endpoint pair`, pair = +managed `local-model` + external `remote-model`): one real lease for +`local-model`, none for `remote-model`, wait for the local one, external +route published, one base URL, both aliases in the descriptor, only the +managed lease released at the end. + +### 40. [ ] Real e2e with a local fake OpenAI server +No internet provider. Phases: external only; managed only; mixed bundle; +managed alias -> external after release; external under dynamic routing if +practical. Run from a frozen copy. + +### 41. [ ] Docs, journal, verification +Design doc: Endpoint (public contract) / Fulfillment (managed runtime or +external upstream) / Lease (only when infer-stack owns runtime), with the +workflow (`external:` in the catalog; `infer-stack access qwen --env-file +./qwen.env`; `infer-stack env REMOTE_QWEN_API_KEY=...`) and that switching +back to managed changes neither alias nor workflow. Update architecture +docs, not only CLI help. Journal: fake lease vs access above leasing; +`engine: external` vs target ownership; registry as authoring vs derived +state; direct remote vs stable front door. Full suite, `ty`, flake8, UX +audits, the relevant real e2e. + +## Non-goals +Arbitrary providers; Anthropic/Bedrock/Azure schemas; direct multi-base-URL +descriptors; health ownership or lifecycle of external services; fake +leases; a service mesh; rewriting existing catalog entries. The first +external target is an already-running OpenAI-compatible upstream; the +target union allows more later. From 79a96e63ccc2f99eb13e435c8cf582dc8e7438de Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sat, 26 Sep 2026 22:24:51 -0400 Subject: [PATCH 103/141] roadmap: review hardening closed (queue 11-27) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/backend-parity-roadmap.md | 2 +- docs/queue.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index ba484b53..384412e7 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -1,7 +1,7 @@ # Backend parity roadmap: KubeAI as a superset of Compose **Status:** proposed 2026-09-25 · **P0 done** 2026-09-24 on -`dev/backend-unification` · **features complete; review hardening done except queue 24-27 (final closure pass)**: +`dev/backend-unification` · **features complete; review hardening, re-review and final closure done (queue 11-27)**: phases P1-P6 landed 2026-09-26 (P4 and P5 end in a handover run on real hardware, `dev/handover/`); a two-part outside review the same day found replica residency, the backend protocol, scheduler reclaim, publication diff --git a/docs/queue.md b/docs/queue.md index 0e284945..fe8b35b8 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -863,7 +863,10 @@ Each with a test that fails on the code before it: ## Final closure pass (2026-09-26) -A third review found one cross-feature blocker and three small items. +A third review found one cross-feature blocker and three small items. All +done; verified by the suite, `ty`, flake8, both UX audits and the full k3s +e2e (which also exposed, and 5337815 fixed, a race in `secrets rotate`'s +old-key check behind an in-cluster rollout). ### 24. [x] `secrets rotate` under a running dynamic-routing gateway From 0082538ff9ee13b4236055c6b0f027fc127a6abc Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 00:14:47 -0400 Subject: [PATCH 104/141] queue: log everything still open outside the numbered items GPU handover runs, confirming 8a/8b on GPUs, root-owned vLLM caches, the in-cluster gateway's deferred features, seed --replace CAS, CLI capability probes that still decide behaviour, the TUI flake, the commit-history squash, and the guest VM's k3s setting. known-limitations: multi-node placement no longer says scheduling across hosts is unsupported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/known-limitations.md | 6 +++-- docs/queue.md | 41 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index 0c8bfecb..cea54690 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -165,8 +165,10 @@ asks for it. ### Multi-node placement (current) -Placement is per host. Spanning one deployment across machines, or scheduling -across several hosts, is not supported. +The compose backend places on one host. The kubeai backend schedules across +the cluster's nodes (verified with a second node in a container; a run +across two real machines is `dev/handover/p5_two_hosts.sh`). One deployment +spanning machines (tensor parallel across nodes) is supported on neither. ### Forged ownership labels are outside the threat model (design boundary) diff --git a/docs/queue.md b/docs/queue.md index fe8b35b8..a8ccbb30 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -923,6 +923,47 @@ there, and `status` shows `starting`. The docstrings of `observe_state` and *Done 2026-09-26:* recorded there as current, deferred. + +## Still open outside the numbered items (logged 2026-09-27) + +Everything known to be open that is not a numbered item above or in +campaign 2. None blocks campaign 2. + +- [ ] **GPU handover runs (operator).** `dev/handover/p4_gpu_labels.sh` on a + GPU k3s node (GFD labels feed sizing, `measure --record`) and + `dev/handover/p5_two_hosts.sh` across two real machines (pod-to-pod over + the real network, the NodePort on the second node, rotation with a Model + served). Verified here only with fake labels and a node in a container. +- [ ] **Confirm 8a and 8b on GPUs.** The three 4 x 96 GB suggestion variants + were not re-run since leasing; `runtime.shm_size` fixes a TP > 1 failure + never observed here. Serve one variant and a TP=2 endpoint with and + without `shm_size`. +- [ ] **vLLM caches written as root.** The Open WebUI fix (seed item under 9) + noted that engine caches under the data root are the same shape (vLLM runs + as root, so an operator's `rm -rf` of a data root fails); not looked at. +- [ ] **In-cluster gateway: deferred features (P5).** Dynamic routing and + Open WebUI with `kubeai_gateway cluster` (they need Postgres and a UI in the + cluster), and an Ingress (`kubeai_gateway_url` accepts one; none tested). +- [ ] **`routes seed --replace` compare-and-swap.** Deferred; see + `known-limitations.md`. Carry the expected old row in the plan and ask + again on a mismatch under the lock. +- [ ] **CLI capability probes that decide behaviour.** The controller has no + optional-hook `getattr` left, but the CLI still branches on + `getattr(backend, ...)` for `down`, `doctor`, `compose_project`, + `rendered_file`/`compose_file`, `deployment_logs` and `access` (the last is + campaign 2's cleanup D). By the re-review's rule a `getattr` default may + serve display, never a decision: make the deciding ones protocol members + or typed capabilities. +- [ ] **TUI catalog-reload flake.** The fix counts only catalog refusals; the + cause (a background worker refusing meanwhile) was inferred, not + reproduced. Reopen if it recurs. +- [ ] **Commit history before merge (maintainer's call).** `30a62bb` has one + failing test, fixed by `122f1b4`; squash them if every commit must be green. +- [ ] **Guest VM state to remember.** `/etc/rancher/k3s/config.yaml` sets a + 1Gi eviction minimum reclaim for the dev cluster (undo: remove it, restart + k3s); the second node is recreated by `dev/k3s_agent_container.sh` with the + same setting. + --- # Campaign 2: external endpoints, and access above leasing From 7e0e0b3b3cceffedf3156b11d43c747774f4849c Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:08:01 -0400 Subject: [PATCH 105/141] queue: campaign 2 design review folded in (item 28 decisions first) Published-endpoint lifetime and route ownership, one access transaction, credential lifecycle on both gateway placements, stable route identity, front-door readiness and failure semantics are decided in item 28 before ResolvedEndpoint; items 29, 33-37 and the non-goals updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 89 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index a8ccbb30..d0449393 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1029,6 +1029,57 @@ that become unnecessary (e.g. `UPSTREAM_ROUTE`, unreachable `render_front_door` branches, lease-bound descriptors, route-derived catalog equality). Gate for the rest. +*Added by the campaign-2 design review (2026-09-27): decide these in the +doc before item 29.* + +1. **Published endpoint lifetime.** What keeps an external endpoint + registered with zero leases; how it is explicitly removed; an unrelated + runbook or a global-setting change must not silently unregister it. + Today the recovery profile replaces its catalog set wholesale when + quiescent, and the route registry is append-only: two authorities that + external endpoints would make visibly disagree. Proposed: separate the + published catalog set's lifetime from the recovery settings' (may stay in + one persisted profile); quiescence permits changing frozen settings but + does not prune unrelated published definitions; an explicit + prune/unpublish removes them. +2. **Route ownership.** Catalog-derived routes (exactly from the published + catalogs), manual/seeded routes (if still needed, owned explicitly), and + dynamic deployment routes (runtime-derived) are three owners, not one + alias -> row map. Decide whether catalog rows stay in the persistent + registry at all; if the registry carries nothing the published catalogs + do not, remove the duplicate instead of teaching it external targets. + `routes seed FILE...` may become "merge these catalogs into the + published union". `CatalogUnion` importing + `_registry_incoming_from_catalog` is the symptom. +3. **One access transaction.** Mixed managed + external access is one + plan, one preview, one approval digest, one commit (profile/catalog, + lease, marker), one render/apply; not "acquire, then publish routes". + `acquire()` stays the lease-specific public operation over the same + transaction machinery. +4. **Credential lifecycle.** Host Compose gateway: the LiteLLM service's + environment names each referenced variable (`KEY: ${KEY}`), so the + existing fingerprint recreates it on a value change without the value in + any route. In-cluster gateway: referenced variables go into its Secret + (`envFrom`), the pod-template hash covers their values, never in a + manifest or diff. `infer-stack env KEY=new`: choose publish-on-write or + stage-and-say-so ("gateway uses it after `infer-stack apply`"), never + silent. Reject reserved names (`LITELLM_MASTER_KEY`, salt, DB password, + Open WebUI secret, `HF_TOKEN`) and invalid variable syntax as + `api_key_env`. +5. **Stable route identity.** An external route's id is its logical owner + (e.g. `external-route:`); api_base / model / credential name are + its content, compared by the semantic route to decide replacement. + Managed dedicated routes keep `deployment id + alias`. +6. **Front-door readiness.** External upstream readiness is not + infer-stack's; its own gateway's is. `access` returns only after the + front door answers with the expected master key (no generation to the + upstream). Poison test: LiteLLM starts slowly. +7. **Failure semantics.** A managed member of a mixed bundle that fails and + rolls back leaves the external definition and route published; a process + that dies after publication leaves the route (a committed lease follows + normal TTL/recovery; no access handle); removing an external endpoint + from the catalog unpublishes it by the rule decided in 1. + ### 29. [ ] `ResolvedEndpoint`: endpoint meaning, not a lease request One normalized object (alias, protocol, target) with a canonical semantic key used for catalog-union conflicts, profile drift, and "did this endpoint @@ -1036,6 +1087,16 @@ change". A managed target derives an `EndpointRequest`; an external one has none. `CatalogUnion` / profile code stop importing gateway or Compose route conversion (cleanup A). Transitional names may stay for compatibility. +*Review:* the managed target's key must cover everything behaviourally +relevant, resolved (model source/revision, quantization/dtype, engine, +upstream served name, launch/runtime structural knobs, capacity, meaningful +placement, default sharing, reclaim policy, protocol), never the raw +`models:` key: two catalogs naming one source differently stay equal. A +request-time `--dedicated` is not part of the key. API: make +`Catalog.resolve_endpoint(name)` return the `ResolvedEndpoint`, with +`resolved.to_request(sharing_override=...)` (or `resolve_requests`) for the +ledger; no parallel `resolve_endpoint_meaning` / `resolve_access_endpoint`. + ### 30. [ ] The external target in the catalog and its CLI YAML `external:` block, validation of illegal mixtures, round-trip. `catalog endpoint add` gains mutually exclusive external options (e.g. @@ -1062,9 +1123,9 @@ storage for everything. Delete `render_front_door` branches that only tests reach (cleanup B), never a supported mode. ### 33. [ ] External routes in static and dynamic routing -Dynamic mode: an external route is standing desired state with a -deterministic managed id from endpoint identity plus non-secret target -semantics (independent of any deployment). It survives zero leases, +Dynamic mode: an external route is standing desired state with a stable +managed id from its logical owner, the endpoint alias (decision 28.5), not +from target semantics; the semantic route decides whether it is replaced. It survives zero leases, releases, other acquires, gc, a dedicated deployment going away. Route retirement (item 17) stays intact; infer-stack never tears down an external upstream, only its route. @@ -1074,12 +1135,17 @@ upstream, only its route. route or logs. Render as a LiteLLM env reference; the gateway container receives the variable from the managed `.env` (`infer-stack env KEY=...`). Fail before apply if a declared key has no value; never generate provider -keys. Dynamic routing: LiteLLM redacts credentials when listing, so do not +keys. Per decision 28.4: host Compose service environment and fingerprint; +in-cluster gateway Secret and pod-template hash; the chosen `env KEY=new` +semantics; reserved-name validation. Real test against the pinned LiteLLM: +add a model through the admin API with `api_key = os.environ/TEST_KEY`, +request, change `TEST_KEY`, recreate/roll as designed, request again. Test +both gateway placements. Dynamic routing: LiteLLM redacts credentials when listing, so do not read keys back; the route identity/fingerprint includes the env NAME (a changed reference forces replacement), never the value; a changed value triggers what the gateway needs to see it, without leaking it. -### 35. [ ] Publication without a ledger mutation +### 35. [ ] Publication without a ledger mutation (one transaction, 28.3) External-only access may change nothing in the ledger but still needs: check/incorporate the invocation catalog/profile, render gateway state, persist routes, apply under the publication lock with the pending / @@ -1088,6 +1154,14 @@ access reach the same publication coordinator; no second ad-hoc gateway apply path; no private gateway writes from the CLI. ### 36. [ ] Access above leasing +*Review:* split the front door's part from the endpoints': a typed +`FrontDoorControl.connection_info()` (base URL, front-door credential, +optional UI URL), plus the resolved endpoints / routes (with LiteLLM the +request name is the alias), make an `AccessResult`; the untyped +`backend.access(endpoints) -> dict` goes (renamed, not a second "access"). +Mixed access is one transaction (28.3); `access` waits for front-door +readiness (28.6); failure semantics per 28.7. + An access result: requested endpoints, base URL, credential info, alias -> request-model mapping, optional real lease, optional GPU reservation metadata (external-only: `lease=None`; mixed: one real lease @@ -1106,6 +1180,9 @@ Access is a typed capability, not a `getattr(backend, 'access')` hook ### 37. [ ] Views: `external` Status, TUI and catalog views show external targets as `external`. +*Review:* no pseudo-deployment rows: `leases` stays leases and +deployments. Show external endpoints in an endpoint/routing section +(`status`, the TUI's catalog or routes view, `catalog show`). ### 38. [ ] `FrontDoor` naming collision (cleanup C) If both the controller capability and the rendered artifact are still @@ -1152,6 +1229,8 @@ state; direct remote vs stable front door. Full suite, `ty`, flake8, UX audits, the relevant real e2e. ## Non-goals +Direct external access (a second access mode where the alias stops being +the model name) is a possible later optimization, not campaign 2. Arbitrary providers; Anthropic/Bedrock/Azure schemas; direct multi-base-URL descriptors; health ownership or lifecycle of external services; fake leases; a service mesh; rewriting existing catalog entries. The first From 58df8394a2d6a7d25441c0a627e68e492c0ebed2 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:12:32 -0400 Subject: [PATCH 106/141] paths flags state directories another user (root) wrote vLLM and Ollama run as root in their containers, so their caches under the data root are root's and removing a data root fails. infer-stack paths now lists every state directory, marks one holding another user's files foreign-owned, and prints the docker command that takes it back. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 11 ++++-- infer_stack/cli/commands_meta.py | 57 ++++++++++++++++++++++++++++++++ tests/test_cli_meta.py | 19 +++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index d0449393..f00423ef 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -938,9 +938,16 @@ campaign 2. None blocks campaign 2. were not re-run since leasing; `runtime.shm_size` fixes a TP > 1 failure never observed here. Serve one variant and a TP=2 endpoint with and without `shm_size`. -- [ ] **vLLM caches written as root.** The Open WebUI fix (seed item under 9) +- [x] **vLLM caches written as root.** The Open WebUI fix (seed item under 9) noted that engine caches under the data root are the same shape (vLLM runs - as root, so an operator's `rm -rf` of a data root fails); not looked at. + as root, so an operator's `rm -rf` of a data root fails). + *Done 2026-09-27 (VM part):* `infer-stack paths` now lists every state + directory (its docstring promised the caches; it showed only the data + root), marks one holding another user's files `foreign-owned`, and prints + the `docker run --rm -v DIR:/d busybox chown -R uid:gid /d` that takes it + back (verified: a root-owned model directory blocked `rm -rf`; after the + printed command it did not). Running vLLM itself as the owner would change + a GPU runtime this VM cannot verify; not done. - [ ] **In-cluster gateway: deferred features (P5).** Dynamic routing and Open WebUI with `kubeai_gateway cluster` (they need Postgres and a UI in the cluster), and an Ingress (`kubeai_gateway_url` accepts one; none tested). diff --git a/infer_stack/cli/commands_meta.py b/infer_stack/cli/commands_meta.py index 7fc3a50a..94f559db 100644 --- a/infer_stack/cli/commands_meta.py +++ b/infer_stack/cli/commands_meta.py @@ -134,6 +134,49 @@ def _entry(label: str, path: Path, *, kind: str) -> dict[str, str]: } +def _foreign_owner(path: Path, *, limit: int = 5000) -> int | None: + """The uid of the first file under ``path`` another user owns, or ``None``. + + Engines run as root in their containers (vLLM, Ollama), so the caches + they fill are root's, and removing a data root then fails with + "permission denied". Bounded: at most ``limit`` entries are looked at. + + >>> import tempfile + >>> _foreign_owner(Path(tempfile.mkdtemp())) is None + True + """ + import os + + me = os.getuid() + seen = 0 + try: + for root, dirs, files in os.walk(path): + for name in (*dirs, *files): + seen += 1 + try: + owner = os.lstat(os.path.join(root, name)).st_uid + except OSError: + continue + if owner != me: + return owner + if seen >= limit: + return None + except OSError: + return None + return None + + +def _state_entry(label: str, path: Path) -> dict[str, str]: + """A state directory, flagged when another user (root) owns files in it.""" + entry = _entry(label, path, kind='dir') + if entry['status'] == 'exists': + owner = _foreign_owner(path) + if owner is not None: + entry['status'] = 'foreign-owned' + entry['owner'] = str(owner) + return entry + + def _status_style(status: str) -> str: if status == 'exists': return 'green' @@ -240,8 +283,12 @@ def main(cls, argv=True, **kwargs): kind='file'), ] if target in {'all', 'data'}: + from ..config import default_state_paths + groups['data'] = [ _entry('data_root', data_root(), kind='dir'), + *(_state_entry(name, Path(path)) + for name, path in default_state_paths().items()), ] if target in {'all', 'leasing'}: compose_dir = data_root() / 'leasing' / 'compose' @@ -279,6 +326,16 @@ def main(cls, argv=True, **kwargs): _render_rich(groups, console) else: _render_plain(groups) + foreign = [e for entries in groups.values() for e in entries + if e['status'] == 'foreign-owned'] + if foreign: + import os + + print('\nforeign-owned: an engine container (running as root) wrote ' + 'these, so deleting them as you fails; to take them back:') + for e in foreign: + print(f" docker run --rm -v {e['path']}:/d busybox " + f"chown -R {os.getuid()}:{os.getgid()} /d") return 0 diff --git a/tests/test_cli_meta.py b/tests/test_cli_meta.py index 08030a34..fa292a5c 100644 --- a/tests/test_cli_meta.py +++ b/tests/test_cli_meta.py @@ -153,3 +153,22 @@ def test_day2_compose_verbs_target_the_backends_project(tmp_path: Path, monkeypa assert base == [ 'docker', 'compose', '-p', 'infer-stack', '-f', str(compose_file) ] + + +def test_paths_flags_state_written_by_another_user(tmp_path, monkeypatch, capsys): + """An engine running as root fills the caches with root's files, and a + data root then cannot be removed; `paths` says which and how to fix it.""" + import os + + from infer_stack.cli.commands_meta import ConfigPathsCLI + + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + monkeypatch.setenv('INFER_STACK_CONFIG_DIR', str(tmp_path / 'cfg')) + (tmp_path / 'hf-cache').mkdir() + (tmp_path / 'hf-cache' / 'weights').write_text('x') + real = os.getuid() + monkeypatch.setattr(os, 'getuid', lambda: real + 1) # the files are "root's" + assert ConfigPathsCLI.main(argv=['data']) == 0 + out = capsys.readouterr().out + assert 'hf_cache (dir, foreign-owned)' in out + assert f'chown -R {real + 1}:' in out and 'hf-cache:/d busybox' in out From fd1104145a24a9b53d1769b2b8208ec63df75fc1 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:13:43 -0400 Subject: [PATCH 107/141] Campaign 2 design: published endpoints, derived routes, one access transaction Queue item 28: planning/external-endpoints.md decides the published endpoint lifetime (the profile's catalogs are the durable union; quiescence no longer replaces them), route ownership (derived at render; the registry becomes read-only legacy input), one access transaction, the credential lifecycle on both gateway placements, stable route identity, front-door readiness, failure semantics, and lists what goes away. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/external-endpoints.md | 188 ++++++++++++++++++++++++++++ docs/queue.md | 11 +- 2 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 docs/planning/external-endpoints.md diff --git a/docs/planning/external-endpoints.md b/docs/planning/external-endpoints.md new file mode 100644 index 00000000..faaf8bbe --- /dev/null +++ b/docs/planning/external-endpoints.md @@ -0,0 +1,188 @@ +# External endpoints, and access above leasing (design) + +**Status:** design (queue item 28), 2026-09-27. Implementation: queue items +29-41 in [../queue.md](../queue.md). Parent: the backend-parity campaign +([backend-parity-roadmap.md](backend-parity-roadmap.md)). + +An endpoint is the public name a workflow asks for (`qwen`). Today every +endpoint is a model infer-stack runs, so asking for one means taking a +lease. This change lets an endpoint name a model server that already runs +elsewhere, and uses that to separate three things that are currently one: + +| Layer | What it is | Owner | +|---|---|---| +| Endpoint | the public contract: alias, protocol | the catalog | +| Fulfillment | a managed runtime (vLLM, Ollama) or an external OpenAI-compatible upstream | the catalog's target | +| Lease | protection of a runtime infer-stack runs | the ledger, only for managed targets | + +## The chain, after the change + +```text +catalog definitions (the invocation's, merged into the published union) + | + v +ResolvedEndpoint(alias, protocol, target) one semantic key per endpoint + |-- ManagedTarget -> EndpointRequest -> ledger (lease, deployment) + '-- ExternalTarget -> nothing in the ledger + | + v +GatewayRoute(alias, kind, model, api_base, key_env, route_id?) + |-- static: derived from the published union (+ live deployments) + '-- dynamic: the same routes, reconciled through LiteLLM's admin API + | + v +front-door publication (render, approve, apply: one transaction) + | + v +AccessResult(connection info, alias -> request name, optional lease) +``` + +Each arrow has one implementation; each box one owner. + +## Decisions + +### 1. Published endpoint lifetime + +Today two things persist endpoint meaning: the recovery profile's +`catalogs` (replaced wholesale when an acquire finds the stack quiescent, +`Controller._acquire_profile_candidate`) and the route registry (append-only +rows derived from those catalogs, `_merge_route_registry`). They agree only +because routes outlive the catalog that made them. An external endpoint has +no lease and no residency to pin it, so it would be kept by whichever copy +happened to remain. + +**Decision.** The profile's `catalogs` is the published catalog union, the +one durable store of endpoint definitions, with its own lifetime: + +- every acquire and access merges the invocation's catalogs into it (the + existing merge path: identical definitions coexist; a different + definition of an unpinned endpoint replaces it; of a pinned one, refuses); +- quiescence permits replacing the frozen settings (backend, ports, images, + gateway placement) as today, but no longer replaces the catalog set: an + unrelated runbook or a settings change does not unregister anything; +- `routes prune` (renamed in help to what it now does: unpublish) removes + published definitions the invocation's catalogs do not have and no live + deployment uses; it is the explicit removal, confirmed as today. + +Visible behaviour for managed endpoints does not change: their routes +already outlived catalog replacement through the append-only registry. + +### 2. Route ownership: derive, do not store + +Three owners, never mixed in one map: + +| Routes | Derived from | Stored | +|---|---|---| +| catalog routes | the published union | no: derived at render | +| deployment routes (static) | live deployments whose endpoint no published catalog defines | no: derived at render | +| dynamic routes | placed deployments, and external targets | reconciled into LiteLLM's DB (as today) | +| legacy rows | an existing `litellm_registry.json` from before this change (seeded siblings, a migrated `litellm_config.yaml`) | read-only compatibility; `routes prune` drops them | + +The registry file stops being written with catalog-derived rows: once the +union is durable, it carries nothing the union does not. Existing files are +still read, so an upgrade loses no route; `routes list` marks their rows as +legacy. `routes seed FILE...` becomes "merge these catalogs into the +published union" (same user-visible effect: the gateway serves their +aliases; conflicts refuse unless `--replace`). `CatalogUnion` then decides +conflicts from endpoint semantics alone, and the import of +`_registry_incoming_from_catalog` goes. + +### 3. One access transaction + +`Controller.acquire` already runs one admission transaction: sync the +profile, preview (placement and render), commit the profile candidate, the +marker and the lease together, render, apply, wait. Access is the same +transaction with the managed request set possibly empty and the external +definitions carried by the profile candidate. One preview, one approval +digest, one marker, one render and apply, for managed-only, external-only and +mixed alike. `acquire` stays the public lease operation over it and refuses +an external-only request with a pointer to `access`. + +### 4. Credential lifecycle + +- **Reference, never value.** `external.api_key_env: NAME`; the route renders + `api_key: os.environ/NAME`; the catalog, the published profile, routes, + fingerprints and logs hold the name only. +- **Host gateway (Compose, KubeAI's host gateway).** The LiteLLM service's + environment lists each referenced name (`NAME: ${NAME}`), so the existing + per-service fingerprint (which hashes the values of the variables a stanza + references) recreates LiteLLM when a value changes. +- **In-cluster gateway.** Referenced names go into its Secret; the key-hash + annotation covers every value in it, so a change rolls the pods; values + never appear in the manifest or a diff. +- **`infer-stack env NAME=value`** stays a file write. When NAME is referenced + by a published external endpoint it says so: "the gateway uses it after + `infer-stack apply`". No silent staleness, and `env` does not become a + publication command. +- **Validation.** `NAME` must be a valid environment name and not one of + infer-stack's own (`LITELLM_MASTER_KEY`, `LITELLM_SALT_KEY`, + `LITELLM_DB_PASSWORD`, `WEBUI_SECRET_KEY`, `HF_TOKEN`): a catalog must not + be able to send the gateway's own credentials to a server it names. A + referenced name with no value fails before apply; provider keys are never + generated. + +### 5. Stable route identity + +A dynamic route's managed id is its logical owner: `(deployment id, alias)` +for a managed deployment (unchanged: one alias can have several dedicated +deployments), `('external', alias)` for an external target. api_base, model +and key name are content; the semantic route compares content to decide a +replacement. Redefining `qwen` from server A to B replaces one route. + +### 6. Front-door readiness + +An external upstream's readiness is not infer-stack's; its own gateway's is. +Managed access already waits for a generation through the front door. +Access with external members also waits until the front door accepts the +master key (`gateway_accepts`), bounded, and fails if it never does. No +request is sent to the external upstream. + +### 7. Failure semantics + +- A managed member that fails and rolls back leaves the external + definitions published (they are catalog state, not tied to the call). +- A process that dies after publication leaves the routes; a committed lease + follows TTL and recovery as today. There is no access handle to clean up. +- Removing an external endpoint from the published union (`routes prune`) + removes its route on the next publication. + +### Also settled + +- **Front door only.** An external endpoint without a LiteLLM front door + fails clearly. Direct access is a later option, not this change. +- **Views.** `leases` stays leases and deployments. External endpoints show + in an endpoint/routing section of `status`, in `catalog show`, `routes + list` and the TUI's catalog, as `external`. +- **Names.** Endpoint alias (what clients request), upstream model (what the + target server expects), deployment id. `public_name`/`served_name` in YAML + stay as spellings of the upstream model name; new code uses the three + terms. + +## What goes away + +- catalog-derived and deployment-derived rows in `litellm_registry.json` + (the file becomes read-only legacy input); +- `CatalogUnion`'s route-row comparison and its import of + `_registry_incoming_from_catalog`; +- the `upstream` pseudo-engine (`UPSTREAM_ROUTE`): a KubeAI or external route + is a `GatewayRoute` of kind `openai` with an api_base; +- the registry row shapes `{engine, served, host}` and their second + interpreter in `routes list`: one `GatewayRoute` and one renderer; +- `render_front_door`'s catalog-superset and legacy placed-deployment + branches, reached only by tests (the tests move to the route planner); +- `backend.access(endpoints) -> dict`: the front door gives connection info, + endpoints give request names, an `AccessResult` combines them; +- a descriptor that needs a lease (`INFER_STACK_LEASE_ID` is written only + when there is one); +- two unrelated types named `FrontDoor` (the capability becomes + `FrontDoorControl`, the rendered artifact `RenderedFrontDoor`). + +## Alternatives rejected + +- **A fake lease for an external endpoint** keeps today's API shape but puts + demand, TTL and reclaim state in the ledger for something with no runtime. +- **`engine: external`** mixes how a runtime is realized with who owns it. +- **Authoring external endpoints as route rows** makes the registry a second + catalog, outside the conflict and pinning rules that protect meaning. +- **Direct access to an external server** gives external-only requests a + second access mode where the alias stops being the model name. diff --git a/docs/queue.md b/docs/queue.md index f00423ef..6bb4d0f9 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1029,7 +1029,7 @@ document it instead of forcing it. ## Work items (in order, small commits) -### 28. [ ] Read, then write the design down first +### 28. [x] Read, then write the design down first Read catalog, profile, gateway, controller and access code together. Write the data flow and invariants into a planning doc, and name the concepts that become unnecessary (e.g. `UPSTREAM_ROUTE`, unreachable @@ -1087,6 +1087,15 @@ doc before item 29.* normal TTL/recovery; no access handle); removing an external endpoint from the catalog unpublishes it by the rule decided in 1. +*Done 2026-09-27:* [planning/external-endpoints.md](planning/external-endpoints.md) +decides all seven. The chosen rules: the profile's `catalogs` is the durable +published union (merged on every acquire/access; quiescence replaces +settings, not catalogs; `routes prune` unpublishes); routes are derived at +render, the registry becomes read-only legacy input; access is the acquire +transaction with a possibly empty managed set; credentials by reference +with `env` saying when an apply is needed; external route id = +`('external', alias)`; access waits for the front door's key. + ### 29. [ ] `ResolvedEndpoint`: endpoint meaning, not a lease request One normalized object (alias, protocol, target) with a canonical semantic key used for catalog-union conflicts, profile drift, and "did this endpoint From bc390ad2aa1752d9884180ec77bd82fc42ec8224 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:18:19 -0400 Subject: [PATCH 108/141] ResolvedEndpoint: what an endpoint means, above its lease request Catalog.resolve_endpoint returns a ResolvedEndpoint (alias, protocol, target); a managed target derives the ledger's EndpointRequest through to_request(sharing_override=), and the semantic key comes from the resolved request, never the catalog's model keys. resolve() and resolve_requests() expand bundles (resolve_names stays as an alias). CatalogUnion decides conflicts from endpoint meaning alone and no longer imports route derivation. Managed ResolvedEndpoints keep the EndpointRequest fields for callers from before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 12 ++- infer_stack/cli/commands_leasing.py | 4 +- infer_stack/leasing/catalog.py | 56 ++++++---- infer_stack/leasing/compose.py | 2 +- infer_stack/leasing/endpoints.py | 153 ++++++++++++++++++++++++++++ infer_stack/leasing/gateway.py | 4 +- infer_stack/leasing/profile.py | 43 ++++---- infer_stack/tui.py | 2 +- tests/test_leasing_profile.py | 15 +++ 9 files changed, 244 insertions(+), 47 deletions(-) create mode 100644 infer_stack/leasing/endpoints.py diff --git a/docs/queue.md b/docs/queue.md index 6bb4d0f9..557994c1 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1096,7 +1096,7 @@ transaction with a possibly empty managed set; credentials by reference with `env` saying when an apply is needed; external route id = `('external', alias)`; access waits for the front door's key. -### 29. [ ] `ResolvedEndpoint`: endpoint meaning, not a lease request +### 29. [x] `ResolvedEndpoint`: endpoint meaning, not a lease request One normalized object (alias, protocol, target) with a canonical semantic key used for catalog-union conflicts, profile drift, and "did this endpoint change". A managed target derives an `EndpointRequest`; an external one has @@ -1113,6 +1113,16 @@ request-time `--dedicated` is not part of the key. API: make `resolved.to_request(sharing_override=...)` (or `resolve_requests`) for the ledger; no parallel `resolve_endpoint_meaning` / `resolve_access_endpoint`. +*Done 2026-09-27:* `leasing/endpoints.py`: `ResolvedEndpoint(alias, +protocol, target)`, `ManagedTarget(request)`, `ExternalTarget(api_base, +model, api_key_env)`, `semantic_key()` from the resolved request. +`Catalog.resolve_endpoint` returns it; `resolve(names)` gives meanings, +`resolve_requests(names, sharing=)` ledger requests (`resolve_names` kept +as its alias). `CatalogUnion` compares semantic keys and no longer imports +`_registry_incoming_from_catalog` (cleanup A). A managed `ResolvedEndpoint` +still answers the `EndpointRequest` fields: eval_audit reads them off +`resolve_endpoint` (its test failures are the same with and without this). + ### 30. [ ] The external target in the catalog and its CLI YAML `external:` block, validation of illegal mixtures, round-trip. `catalog endpoint add` gains mutually exclusive external options (e.g. diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 451e8ed8..b0e87a01 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -427,7 +427,7 @@ def _requests_catalog(controller, config): def _resolve(catalog, names, *, sharing=None): try: - return catalog.resolve_names(names, sharing=sharing) + return catalog.resolve_requests(names, sharing=sharing) except CatalogError as ex: raise SystemExit(str(ex)) @@ -1656,7 +1656,7 @@ def main(cls, argv=True, **kwargs): ) catalog = _requests_catalog(controller, config) name = config.endpoint - request = catalog.resolve_names([name])[0] + request = catalog.resolve_requests([name])[0] if request.engine != 'vllm': raise SystemExit( f'measure supports vllm endpoints only ' diff --git a/infer_stack/leasing/catalog.py b/infer_stack/leasing/catalog.py index 8b98d978..21ef64f2 100644 --- a/infer_stack/leasing/catalog.py +++ b/infer_stack/leasing/catalog.py @@ -77,6 +77,7 @@ import yaml from .launch import launch_identity, translate_legacy +from .endpoints import ManagedTarget, ResolvedEndpoint from .models import ( EndpointRequest, Sharing, @@ -496,40 +497,51 @@ def _unknown_endpoint_error(self, name: str) -> CatalogError: def resolve_endpoint( self, name: str, *, sharing: str | None = None - ) -> EndpointRequest: - """Resolve one endpoint name into a ledger :class:`EndpointRequest`. + ) -> ResolvedEndpoint: + """What endpoint ``name`` means: a :class:`ResolvedEndpoint`. - ``sharing`` overrides the catalog's declared policy (e.g. the CLI - ``--dedicated`` flag) when given. + A managed target carries the catalog's request; ``to_request`` gives + the ledger's. ``sharing`` (from before ``to_request`` took the + override) is applied to that request when given. """ if name not in self.endpoints: raise self._unknown_endpoint_error(name) ep = self.endpoints[name] share = sharing or ep.sharing if ep.engine == VLLM: - return self._resolve_vllm(ep, share) - if ep.engine == OLLAMA: - return self._resolve_ollama(ep, share) - raise CatalogError( - f"endpoint '{name}' has unknown engine '{ep.engine}'" - ) - - def resolve_names( - self, names: list[str], *, sharing: str | None = None - ) -> list[EndpointRequest]: - """Expand a mix of endpoint and bundle names into requests. + request = self._resolve_vllm(ep, share) + elif ep.engine == OLLAMA: + request = self._resolve_ollama(ep, share) + else: + raise CatalogError( + f"endpoint '{name}' has unknown engine '{ep.engine}'" + ) + return ResolvedEndpoint(ep.name, ep.protocol, ManagedTarget(request)) - Bundles expand to their member endpoints; duplicates (e.g. an endpoint - named directly and also via a bundle) are de-duplicated, preserving - order. - """ + def expand(self, names: list[str]) -> list[str]: + """Endpoint names for a mix of endpoint and bundle names: bundles + expand to their members, duplicates go, order stays.""" ordered: list[str] = [] for name in names: - members = self.bundles.get(name, [name]) - for member in members: + for member in self.bundles.get(name, [name]): if member not in ordered: ordered.append(member) - return [self.resolve_endpoint(n, sharing=sharing) for n in ordered] + return ordered + + def resolve(self, names: list[str]) -> list[ResolvedEndpoint]: + """The meanings of a mix of endpoint and bundle names.""" + return [self.resolve_endpoint(n) for n in self.expand(names)] + + def resolve_requests( + self, names: list[str], *, sharing: str | None = None + ) -> list[EndpointRequest]: + """Ledger requests for a mix of endpoint and bundle names + (``sharing`` overrides the catalog's, e.g. ``--dedicated``).""" + return [self.resolve_endpoint(n).to_request(sharing_override=sharing) + for n in self.expand(names)] + + #: The name from before :meth:`resolve_requests`; kept for its callers. + resolve_names = resolve_requests def _resolve_vllm( self, ep: EndpointSpec, sharing: str diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 4dc56240..a93a04ed 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -152,7 +152,7 @@ def profile_images(profile: dict[str, Any]) -> list[str]: union = CatalogUnion.from_sources(sources) for name in union.endpoints: try: - req = union.resolve_endpoint(name) + req = union.resolve_endpoint(name).to_request() except Exception: # noqa: BLE001 - an unresolvable endpoint serves nothing continue if req.engine == 'vllm': # same sources the render uses diff --git a/infer_stack/leasing/endpoints.py b/infer_stack/leasing/endpoints.py new file mode 100644 index 00000000..2fb87666 --- /dev/null +++ b/infer_stack/leasing/endpoints.py @@ -0,0 +1,153 @@ +"""What a catalog endpoint means: an alias, a protocol, and a target. + +An endpoint is the public name a client asks for. Its *target* says who +fulfils it: + +* :class:`ManagedTarget` -- a runtime infer-stack realizes (vLLM, Ollama), + protected by a lease; it derives the ledger's :class:`EndpointRequest`; +* :class:`ExternalTarget` -- an OpenAI-compatible server that already runs + elsewhere; nothing in the ledger. + +:class:`ResolvedEndpoint` is the one normalized meaning of an endpoint. +Catalog-union conflicts, profile drift and "did this endpoint change" all +compare its :meth:`~ResolvedEndpoint.semantic_key`, which is built from the +resolved request (model source, revision, runtime knobs...), never from the +catalog's own model keys, so two catalogs that name one model differently +still agree. It knows nothing of leases, routes or backends. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +from dataclasses import dataclass +from typing import Any + +from .models import EndpointRequest + + +@dataclass(frozen=True) +class ManagedTarget: + """A runtime infer-stack realizes. ``request`` is the catalog's own + request (its declared sharing); :meth:`ResolvedEndpoint.to_request` applies + a request-time override.""" + + request: EndpointRequest + + kind = 'managed' + + +@dataclass(frozen=True) +class ExternalTarget: + """An OpenAI-compatible server infer-stack does not run. + + ``model`` is what that server expects as ``model``; ``api_key_env`` names + the variable holding its key (never the key). + """ + + api_base: str + model: str + api_key_env: str | None = None + + kind = 'external' + + +@dataclass(frozen=True) +class ResolvedEndpoint: + """One endpoint's meaning: ``alias`` (what clients request), ``protocol`` + (``chat`` / ``completions``) and ``target``. + + Callers from before this type read ``EndpointRequest`` fields off + ``Catalog.resolve_endpoint(name)`` (``engine``, ``served``, ``spec``, + ``compat_key``...); for a managed target those still work, delegated to + its request. New code uses :meth:`to_request`. + + Example: + >>> from infer_stack.leasing.models import vllm_structural + >>> req = EndpointRequest('qwen', 'vllm', vllm_structural(model_ref='hf://org/m'), + ... served={'served_model_name': 'qwen', 'protocol': 'chat'}) + >>> local = ResolvedEndpoint('qwen', 'chat', ManagedTarget(req)) + >>> remote = ResolvedEndpoint('qwen', 'chat', ExternalTarget('http://box/v1', 'Qwen/Q')) + >>> local.managed, remote.managed, local.engine + (True, False, 'vllm') + >>> local.semantic_key() == remote.semantic_key() + False + >>> remote.to_request() + Traceback (most recent call last): + ... + ValueError: 'qwen' is externally provided; it has no lease request + """ + + alias: str + protocol: str + target: ManagedTarget | ExternalTarget + + @property + def managed(self) -> bool: + return isinstance(self.target, ManagedTarget) + + def to_request(self, *, sharing_override: str | None = None) -> EndpointRequest: + """The ledger request for a managed target (``--dedicated`` overrides + the catalog's sharing); an external target has none.""" + if not isinstance(self.target, ManagedTarget): + raise ValueError(f'{self.alias!r} is externally provided; it has no lease request') + request = self.target.request + if sharing_override and sharing_override != request.sharing: + request = dataclasses.replace(request, sharing=sharing_override) + return request + + def semantic(self) -> dict[str, Any]: + """The endpoint's meaning as plain data (what :meth:`semantic_key` hashes).""" + if isinstance(self.target, ManagedTarget): + target: dict[str, Any] = {'kind': 'managed', + 'request': dataclasses.asdict(self.target.request)} + else: + target = {'kind': 'external', **dataclasses.asdict(self.target)} + return {'alias': self.alias, 'protocol': self.protocol, 'target': target} + + def semantic_key(self) -> str: + """A stable digest of :meth:`semantic`: equal iff the endpoint means the same.""" + text = json.dumps(self.semantic(), sort_keys=True, separators=(',', ':'), default=str) + return hashlib.sha256(text.encode('utf-8')).hexdigest() + + # -- EndpointRequest fields, for callers from before this type ------------- + + def _request(self) -> EndpointRequest: + return self.to_request() + + @property + def endpoint(self) -> str: + return self.alias + + @property + def engine(self) -> str: + return self._request().engine + + @property + def structural(self) -> dict[str, Any]: + return self._request().structural + + @property + def capacity(self) -> dict[str, Any]: + return self._request().capacity + + @property + def sharing(self) -> str: + return self._request().sharing + + @property + def spec(self) -> dict[str, Any]: + return self._request().spec + + @property + def served(self) -> dict[str, Any]: + return self._request().served + + @property + def host(self) -> str | None: + return self._request().host + + @property + def compat_key(self) -> str: + return self._request().compat_key diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index c78dc966..a1eebdc3 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -162,7 +162,7 @@ def _litellm_model_list_from_catalog(catalog: Any) -> list[dict[str, Any]]: entries: list[dict[str, Any]] = [] for name in sorted(getattr(catalog, 'endpoints', {})): try: - req = catalog.resolve_endpoint(name) + req = catalog.resolve_endpoint(name).to_request() except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway continue if req.engine == 'vllm': @@ -201,7 +201,7 @@ def _registry_incoming_from_catalog(catalog: Any) -> dict[str, dict[str, Any]]: incoming: dict[str, dict[str, Any]] = {} for name in sorted(getattr(catalog, 'endpoints', {})): try: - req = catalog.resolve_endpoint(name) + req = catalog.resolve_endpoint(name).to_request() except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway continue if req.engine == 'vllm': diff --git a/infer_stack/leasing/profile.py b/infer_stack/leasing/profile.py index 5eadf197..c837580d 100644 --- a/infer_stack/leasing/profile.py +++ b/infer_stack/leasing/profile.py @@ -76,18 +76,20 @@ def canonical_digest(data: Any) -> str: return hashlib.sha256(text.encode('utf-8')).hexdigest() -def _request_key(catalog: Catalog, name: str, sharing: str | None = None) -> Any: +def _endpoint_key(catalog: Catalog, name: str) -> str: + """What ``name`` means in ``catalog`` (its semantic key), or why it cannot.""" try: - return dataclasses.asdict(catalog.resolve_endpoint(name, sharing=sharing)) + return catalog.resolve_endpoint(name).semantic_key() except CatalogError as ex: - return {'unresolvable': str(ex)} + return f'unresolvable: {ex}' class CatalogUnion: """Several catalogs presented as one, for resolution and route rendering. Implements the part of :class:`Catalog` the leasing paths use: - ``endpoints``, ``bundles``, ``resolve_endpoint`` and ``resolve_names``. + ``endpoints``, ``bundles``, ``resolve_endpoint``, ``resolve`` and + ``resolve_requests``. """ def __init__(self, sources: list[dict[str, Any]], catalogs: list[Catalog]): @@ -96,13 +98,11 @@ def __init__(self, sources: list[dict[str, Any]], catalogs: list[Catalog]): self._owner: dict[str, Catalog] = {} self.endpoints: dict[str, Any] = {} self.bundles: dict[str, list[str]] = {} - from .compose import _registry_incoming_from_catalog - - routes: dict[str, Any] = {} + # Conflicts are endpoint meaning only: routes are derived from it. for cat in catalogs: for name, spec in cat.endpoints.items(): if name in self._owner: - if _request_key(self._owner[name], name) != _request_key(cat, name): + if _endpoint_key(self._owner[name], name) != _endpoint_key(cat, name): raise CatalogConflict( f'endpoint {name!r} is defined differently in two ' 'published catalogs', [name] @@ -117,13 +117,6 @@ def __init__(self, sources: list[dict[str, Any]], catalogs: list[Catalog]): [name] ) self.bundles[name] = list(members) - for alias, row in _registry_incoming_from_catalog(cat).items(): - if alias in routes and routes[alias] != row: - raise CatalogConflict( - f'route {alias!r} is defined differently in two published catalogs', - [alias] - ) - routes[alias] = row clash = sorted(set(self.bundles) & set(self.endpoints)) if clash: raise CatalogConflict( @@ -148,20 +141,34 @@ def resolve_endpoint(self, name: str, *, sharing: str | None = None): raise CatalogError(f"unknown endpoint '{name}'") return self._owner[name].resolve_endpoint(name, sharing=sharing) - def resolve_names(self, names: list[str], *, sharing: str | None = None): + def expand(self, names: list[str]) -> list[str]: ordered: list[str] = [] for name in names: for member in self.bundles.get(name, [name]): if member not in ordered: ordered.append(member) - return [self.resolve_endpoint(n, sharing=sharing) for n in ordered] + return ordered + + def resolve(self, names: list[str]): + return [self.resolve_endpoint(n) for n in self.expand(names)] + + def resolve_requests(self, names: list[str], *, sharing: str | None = None): + return [self.resolve_endpoint(n).to_request(sharing_override=sharing) + for n in self.expand(names)] + + resolve_names = resolve_requests def request_matches(self, request) -> bool: """Whether ``request`` is exactly what this union resolves its name to.""" cat = self._owner.get(request.endpoint) if cat is None: return False - return _request_key(cat, request.endpoint, request.sharing) == dataclasses.asdict(request) + try: + mine = cat.resolve_endpoint(request.endpoint).to_request( + sharing_override=request.sharing) + except (CatalogError, ValueError): + return False + return dataclasses.asdict(mine) == dataclasses.asdict(request) def _merged_view(self) -> Catalog: view = Catalog() diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 09409072..1762a244 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -3439,7 +3439,7 @@ def _do_api_test_all(self, models: list[str]) -> None: @work(thread=True, exclusive=True, group='mutate') def _do_acquire(self, name: str) -> None: try: - requests = self.catalog.resolve_names([name]) + requests = self.catalog.resolve_requests([name]) outcome = self.controller.acquire( 'manual', requests, ttl_seconds=None, wait=False, apply=True ) diff --git a/tests/test_leasing_profile.py b/tests/test_leasing_profile.py index 4e294100..a4ec239d 100644 --- a/tests/test_leasing_profile.py +++ b/tests/test_leasing_profile.py @@ -621,3 +621,18 @@ def test_an_idle_but_resident_keep_warm_definition_stays_frozen(tmp_path): _, ctl3 = controller(tmp_path, catalog=changed, docker=docker) ctl3.acquire('z', changed.resolve_names(['alpha']), wait=False) assert ctl3.backend.catalog.resolve_endpoint('alpha').capacity['max_model_len'] == 1024 + + +def test_catalogs_naming_one_model_differently_agree_on_its_endpoint(): + """Endpoint meaning is the resolved request, not the catalog's model key + (campaign 2, item 29): `alias-a` and `alias-b` for one source agree.""" + from infer_stack.leasing.profile import CatalogUnion + + def cat(model_key): + return {'models': {model_key: {'source': 'hf://org/m'}}, + 'endpoints': {'qwen': {'engine': 'vllm', 'model': model_key}}} + + union = CatalogUnion.from_sources([cat('alias-a'), cat('alias-b')]) + assert list(union.endpoints) == ['qwen'] + a, b = (Catalog.from_dict(cat(k)).resolve_endpoint('qwen') for k in ('alias-a', 'alias-b')) + assert a.semantic_key() == b.semantic_key() From 131dd2692aa51ddc89e9a4bfdf4a91a51a6ce902 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:22:54 -0400 Subject: [PATCH 109/141] External endpoints in the catalog: an external: target, not an engine An endpoint may name a server that already runs (external: api_base, model, optional api_key_env). It needs no models: entry; runtime, placement, sharing, reclaim, host, engine and served names are refused beside it; the key env must be a variable name and not one of infer-stack's own secrets. catalog endpoint add takes --external-api-base/--external-model/ --external-api-key-env and refuses runtime options with them. Asking for a lease on an external endpoint (acquire, a mixed bundle) points at access. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 9 ++- infer_stack/cli/commands_catalog.py | 71 ++++++++++++++++++++- infer_stack/leasing/catalog.py | 32 +++++++++- infer_stack/leasing/endpoints.py | 59 +++++++++++++++++- infer_stack/leasing/profile.py | 7 ++- tests/test_external_endpoints.py | 96 +++++++++++++++++++++++++++++ 6 files changed, 264 insertions(+), 10 deletions(-) create mode 100644 tests/test_external_endpoints.py diff --git a/docs/queue.md b/docs/queue.md index 557994c1..25742434 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1123,13 +1123,20 @@ as its alias). `CatalogUnion` compares semantic keys and no longer imports still answers the `EndpointRequest` fields: eval_audit reads them off `resolve_endpoint` (its test failures are the same with and without this). -### 30. [ ] The external target in the catalog and its CLI +### 30. [x] The external target in the catalog and its CLI YAML `external:` block, validation of illegal mixtures, round-trip. `catalog endpoint add` gains mutually exclusive external options (e.g. `--external-api-base`, `--external-model`, `--external-api-key-env`), never an `--engine` choice; reject `--engine vllm` + external, `--gpu`/`--reclaim` + external. `catalog show` makes ownership obvious. +*Done 2026-09-27:* `EndpointSpec.external`; `endpoints.external_errors` +(managed-only keys, URL, model, env-name syntax, reserved names); +`catalog endpoint add --external-*`; `resolve_requests` on an external +member raises "... does not require a lease; use `infer-stack access +NAME`". `catalog show` prints the `external:` block. Tests 1-2 and the CLI in +`tests/test_external_endpoints.py`. + ### 31. [ ] One terminology Endpoint alias (what users request through the front door), upstream model (what the target server expects as `model`), deployment id (one managed diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index 24e90062..fe6d08f9 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -648,6 +648,13 @@ class EndpointAddCLI(_CatalogCommon): model just gets the next index instead of colliding. Give an explicit ``NAME`` when you want a stable alias decoupled from the model (e.g. ``chat`` you can re-point). + + ``--external-api-base`` and ``--external-model`` instead register a server + that already runs elsewhere (OpenAI-compatible): infer-stack routes the + alias to it through the gateway and runs nothing, so none of the runtime + options apply. ``--external-api-key-env`` names the variable holding its + key (set it with ``infer-stack env NAME=...``); the key itself is never + written to the catalog. """ __command__ = 'add' @@ -655,7 +662,20 @@ class EndpointAddCLI(_CatalogCommon): None, position=1, type=str, help='Endpoint alias (default: {model}-N, auto-incrementing).', ) - engine = kw.Value('vllm', type=str, choices=['vllm', 'ollama']) + engine = kw.Value(None, type=str, choices=['vllm', 'ollama'], + help='How infer-stack runs it (default: vllm).') + external_api_base = kw.Value( + None, type=str, + help='An OpenAI-compatible server that already runs, e.g. ' + 'http://box:8000/v1 (with --external-model; runs nothing).', + ) + external_model = kw.Value( + None, type=str, help='The model name that external server expects.', + ) + external_api_key_env = kw.Value( + None, type=str, + help="Variable holding the external server's key (its name, not the key).", + ) model = kw.Value(None, type=str, help='Model name (vllm) or tag (ollama).') host = kw.Value(None, type=str, help='Runtime host (ollama).') public_name = kw.Value( @@ -706,6 +726,8 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) path = _catalog_path(config) data = _load_raw(path) + if config.external_api_base or config.external_model or config.external_api_key_env: + return cls._add_external(config, path, data) if config.name: name = config.name # The guard moved below, after the entry exists to compare against. @@ -720,7 +742,7 @@ def main(cls, argv=True, **kwargs): name = _next_indexed_name( data['endpoints'], _slug_alias(config.model) ) - entry: dict[str, Any] = {'engine': config.engine} + entry: dict[str, Any] = {'engine': config.engine or 'vllm'} if config.model: entry['model'] = config.model if config.host: @@ -767,6 +789,51 @@ def main(cls, argv=True, **kwargs): return 0 + @classmethod + def _add_external(cls, config, path, data) -> int: + """``endpoint add NAME --external-api-base URL --external-model M``.""" + from ..leasing.endpoints import external_errors + + if not config.name: + raise SystemExit('endpoint add: an external endpoint needs a NAME ' + '(the alias clients will request)') + managed = { + '--engine': config.engine, '--model': config.model, '--host': config.host, + '--public-name': config.public_name, '--reclaim': config.reclaim, + '--min-vram-gib': config.min_vram_gib, '--gpu': config.gpu or None, + '--max-model-len': config.max_model_len, '--gpu-mem': config.gpu_mem, + '--tensor-parallel': config.tensor_parallel, + '--extra-args': config.extra_args, '--runtime': config.runtime or None, + } + given = [flag for flag, value in managed.items() if value is not None] + if given: + raise SystemExit( + f'endpoint add: {", ".join(given)} ' + f'{"describe" if len(given) > 1 else "describes"} a runtime ' + 'infer-stack runs; an external endpoint (--external-*) runs nothing') + ext: dict[str, Any] = {'api_base': config.external_api_base, + 'model': config.external_model} + if config.external_api_key_env: + ext['api_key_env'] = config.external_api_key_env + entry: dict[str, Any] = {'external': ext} + if config.protocol: + entry['protocol'] = config.protocol + problems = external_errors(config.name, entry, ext) + if problems: + raise SystemExit('endpoint add: ' + '; '.join(problems)) + action = _exists_guard(data, 'endpoints', config.name, config.force, entry) + if action == 'unchanged': + if not config.dry_run: + print(f"endpoint '{config.name}' already up to date") + return 0 + data['endpoints'][config.name] = entry + _save_raw(path, data, dry_run=config.dry_run) + if not config.dry_run: + print(f"added external endpoint '{config.name}' -> " + f"{config.external_model} at {config.external_api_base}") + return 0 + + class EndpointListCLI(_PathOverridesMixin): """List endpoint names.""" __command__ = 'list' diff --git a/infer_stack/leasing/catalog.py b/infer_stack/leasing/catalog.py index 21ef64f2..4c6f9768 100644 --- a/infer_stack/leasing/catalog.py +++ b/infer_stack/leasing/catalog.py @@ -77,7 +77,7 @@ import yaml from .launch import launch_identity, translate_legacy -from .endpoints import ManagedTarget, ResolvedEndpoint +from .endpoints import ExternalTarget, ManagedTarget, ResolvedEndpoint, external_errors from .models import ( EndpointRequest, Sharing, @@ -183,6 +183,11 @@ class EndpointSpec: # errors()). ``min_vram_gib`` is the portable eligibility declaration; # ``gpu_indices`` is an optional exact local pin from the TUI/CLI. placement: dict[str, Any] = field(default_factory=dict) + #: An OpenAI-compatible server that already runs (``external:``), or + #: ``None`` for a runtime infer-stack realizes. Ownership, not an engine. + external: ExternalTarget | None = None + #: The endpoint's mapping as written, for validating an external one. + raw: dict[str, Any] = field(default_factory=dict, compare=False, repr=False) def _parse_sharing(value: Any) -> str: @@ -368,6 +373,19 @@ def from_dict(cls, data: dict[str, Any] | None) -> Catalog: endpoints = {} for name, spec in (data.get('endpoints') or {}).items(): spec = spec or {} + if 'external' in spec: + ext = spec.get('external') + endpoints[name] = EndpointSpec( + name=name, engine='', model='', + protocol=_parse_protocol(spec.get('protocol')), + external=(ExternalTarget( + api_base=str(ext.get('api_base') or ''), + model=str(ext.get('model') or ''), + api_key_env=ext.get('api_key_env')) + if isinstance(ext, dict) else None), + raw=dict(spec), + ) + continue endpoints[name] = EndpointSpec( name=name, engine=spec.get('engine', VLLM), @@ -412,6 +430,9 @@ def validate(self) -> None: def errors(self) -> list[str]: errors: list[str] = [] for ep in self.endpoints.values(): + if 'external' in ep.raw: + errors.extend(external_errors(ep.name, ep.raw, ep.raw.get('external'))) + continue if ep.engine == VLLM: if not ep.model: errors.append( @@ -507,6 +528,8 @@ def resolve_endpoint( if name not in self.endpoints: raise self._unknown_endpoint_error(name) ep = self.endpoints[name] + if ep.external is not None: + return ResolvedEndpoint(ep.name, ep.protocol, ep.external) share = sharing or ep.sharing if ep.engine == VLLM: request = self._resolve_vllm(ep, share) @@ -537,8 +560,11 @@ def resolve_requests( ) -> list[EndpointRequest]: """Ledger requests for a mix of endpoint and bundle names (``sharing`` overrides the catalog's, e.g. ``--dedicated``).""" - return [self.resolve_endpoint(n).to_request(sharing_override=sharing) - for n in self.expand(names)] + try: + return [self.resolve_endpoint(n).to_request(sharing_override=sharing) + for n in self.expand(names)] + except ValueError as ex: # an external member: no lease request + raise CatalogError(str(ex)) from ex #: The name from before :meth:`resolve_requests`; kept for its callers. resolve_names = resolve_requests diff --git a/infer_stack/leasing/endpoints.py b/infer_stack/leasing/endpoints.py index 2fb87666..b2cceedf 100644 --- a/infer_stack/leasing/endpoints.py +++ b/infer_stack/leasing/endpoints.py @@ -38,6 +38,59 @@ class ManagedTarget: kind = 'managed' +#: Variables infer-stack itself keeps in the gateway's .env. An external +#: target may not name one as its key: the gateway would send infer-stack's +#: own credential to a server the catalog names. +RESERVED_KEY_ENVS = frozenset({ + 'LITELLM_MASTER_KEY', 'LITELLM_SALT_KEY', 'LITELLM_DB_PASSWORD', + 'WEBUI_SECRET_KEY', 'HF_TOKEN', +}) + +#: Endpoint keys that describe a runtime infer-stack realizes, so they mean +#: nothing beside ``external:``. +MANAGED_ONLY_KEYS = ('engine', 'model', 'host', 'runtime', 'placement', 'sharing', + 'reclaim', 'served_name', 'public_name') + + +def external_errors(name: str, raw: dict[str, Any], external: Any) -> list[str]: + """What is wrong with endpoint ``name`` whose ``external:`` is ``external`` + (``raw`` is the endpoint's whole mapping). + + >>> external_errors('q', {'external': {}, 'runtime': {}}, {'api_base': 'box', 'model': ''}) + ["endpoint 'q' is external: 'runtime' describes a runtime infer-stack runs and does not apply", "endpoint 'q': external.api_base must be an http(s) URL, not 'box'", "endpoint 'q': external.model is required (the name the upstream server expects)"] + >>> external_errors('q', {}, {'api_base': 'http://b/v1', 'model': 'm', 'api_key_env': 'HF_TOKEN'}) + ["endpoint 'q': external.api_key_env may not be HF_TOKEN, one of infer-stack's own secrets"] + """ + import re + + errors = [f"endpoint {name!r} is external: {key!r} describes a runtime " + 'infer-stack runs and does not apply' + for key in MANAGED_ONLY_KEYS if key in raw] + if not isinstance(external, dict): + return errors + [f"endpoint {name!r}: 'external' must be a mapping " + '(api_base, model, optional api_key_env)'] + unknown = sorted(set(external) - {'api_base', 'model', 'api_key_env'}) + if unknown: + errors.append(f"endpoint {name!r}: unknown external key(s) {unknown} " + '(api_base, model, api_key_env)') + base = external.get('api_base') + if not isinstance(base, str) or not re.match(r'https?://[^/\s]+', base): + errors.append(f"endpoint {name!r}: external.api_base must be an http(s) URL, " + f'not {base!r}') + if not external.get('model') or not isinstance(external.get('model'), str): + errors.append(f"endpoint {name!r}: external.model is required (the name the " + 'upstream server expects)') + key_env = external.get('api_key_env') + if key_env is not None: + if not isinstance(key_env, str) or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', key_env): + errors.append(f"endpoint {name!r}: external.api_key_env must be an " + f'environment variable name, not {key_env!r}') + elif key_env in RESERVED_KEY_ENVS: + errors.append(f"endpoint {name!r}: external.api_key_env may not be " + f"{key_env}, one of infer-stack's own secrets") + return errors + + @dataclass(frozen=True) class ExternalTarget: """An OpenAI-compatible server infer-stack does not run. @@ -76,7 +129,7 @@ class ResolvedEndpoint: >>> remote.to_request() Traceback (most recent call last): ... - ValueError: 'qwen' is externally provided; it has no lease request + ValueError: 'qwen' is externally provided and does not require a lease; use `infer-stack access qwen` or `infer-stack run ...` """ alias: str @@ -91,7 +144,9 @@ def to_request(self, *, sharing_override: str | None = None) -> EndpointRequest: """The ledger request for a managed target (``--dedicated`` overrides the catalog's sharing); an external target has none.""" if not isinstance(self.target, ManagedTarget): - raise ValueError(f'{self.alias!r} is externally provided; it has no lease request') + raise ValueError( + f'{self.alias!r} is externally provided and does not require a lease; ' + f'use `infer-stack access {self.alias}` or `infer-stack run ...`') request = self.target.request if sharing_override and sharing_override != request.sharing: request = dataclasses.replace(request, sharing=sharing_override) diff --git a/infer_stack/leasing/profile.py b/infer_stack/leasing/profile.py index c837580d..70776b42 100644 --- a/infer_stack/leasing/profile.py +++ b/infer_stack/leasing/profile.py @@ -153,8 +153,11 @@ def resolve(self, names: list[str]): return [self.resolve_endpoint(n) for n in self.expand(names)] def resolve_requests(self, names: list[str], *, sharing: str | None = None): - return [self.resolve_endpoint(n).to_request(sharing_override=sharing) - for n in self.expand(names)] + try: + return [self.resolve_endpoint(n).to_request(sharing_override=sharing) + for n in self.expand(names)] + except ValueError as ex: # an external member: no lease request + raise CatalogError(str(ex)) from ex resolve_names = resolve_requests diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py new file mode 100644 index 00000000..5591ac8b --- /dev/null +++ b/tests/test_external_endpoints.py @@ -0,0 +1,96 @@ +"""External endpoints: an alias fulfilled by a server infer-stack does not run. + +Campaign 2 (docs/planning/external-endpoints.md). An external endpoint is a +catalog definition with an ``external:`` target; it has no lease, deployment +or model entry, and it is reached through the front door like any other. +""" + +from __future__ import annotations + +import pytest +import yaml + +from infer_stack.leasing import Catalog +from infer_stack.leasing.catalog import CatalogError +from infer_stack.leasing.endpoints import ExternalTarget + +REMOTE = {'external': {'api_base': 'http://box:8000/v1', 'model': 'Qwen/Qwen3-32B', + 'api_key_env': 'REMOTE_QWEN_KEY'}, 'protocol': 'chat'} + + +def catalog(**endpoints): + return {'models': {'m': {'source': 'hf://org/m'}}, + 'endpoints': {'local': {'engine': 'vllm', 'model': 'm'}, **endpoints}, + 'bundles': {'pair': ['local', 'remote']} if 'remote' in endpoints else {}} + + +# -- 1. parses and round-trips semantically -------------------------------------- + + +def test_an_external_endpoint_parses_and_round_trips(): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + resolved = cat.resolve_endpoint('remote') + assert not resolved.managed + assert resolved.target == ExternalTarget('http://box:8000/v1', 'Qwen/Qwen3-32B', + 'REMOTE_QWEN_KEY') + again = Catalog.from_dict(yaml.safe_load(yaml.safe_dump(cat.source))) + assert again.resolve_endpoint('remote').semantic_key() == resolved.semantic_key() + assert 'remote' not in cat.models # no managed model artifact + + +# -- 2. invalid mixtures fail clearly ---------------------------------------------- + + +@pytest.mark.parametrize('extra, needle', [ + ({'runtime': {'max_model_len': 8}}, "'runtime' describes a runtime"), + ({'engine': 'vllm'}, "'engine' describes a runtime"), + ({'reclaim': 'stop'}, "'reclaim' describes a runtime"), + ({'placement': {'gpu_indices': [0]}}, "'placement' describes a runtime"), + ({'host': 'h'}, "'host' describes a runtime"), +]) +def test_managed_only_keys_beside_external_are_refused(extra, needle): + with pytest.raises(CatalogError, match=needle): + Catalog.from_dict(catalog(remote={**REMOTE, **extra})) + + +@pytest.mark.parametrize('external, needle', [ + ({'api_base': 'box:8000', 'model': 'm'}, 'must be an http'), + ({'api_base': 'http://box/v1'}, 'external.model is required'), + ({'api_base': 'http://box/v1', 'model': 'm', 'api_key_env': 'not a name'}, + 'environment variable name'), + ({'api_base': 'http://box/v1', 'model': 'm', 'api_key_env': 'LITELLM_MASTER_KEY'}, + "infer-stack's own secrets"), + ({'api_base': 'http://box/v1', 'model': 'm', 'api_key': 'sk-literal'}, + 'unknown external key'), +]) +def test_bad_external_blocks_are_refused(external, needle): + with pytest.raises(CatalogError, match=needle): + Catalog.from_dict(catalog(remote={'external': external})) + + +# -- acquire stays lease-only ------------------------------------------------------ + + +def test_asking_for_a_lease_on_an_external_endpoint_points_at_access(): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + with pytest.raises(CatalogError, match='does not require a lease.*infer-stack access remote'): + cat.resolve_requests(['remote']) + with pytest.raises(CatalogError, match='does not require a lease'): + cat.resolve_requests(['pair']) # a mixed bundle goes through access too + assert [r.endpoint for r in cat.resolve_requests(['local'])] == ['local'] + + +def test_the_cli_adds_an_external_endpoint_and_refuses_runtime_options(tmp_path, capsys): + from infer_stack.cli.commands_catalog import EndpointAddCLI + + path = tmp_path / 'catalog.yaml' + path.write_text('models: {}\nendpoints: {}\n') + common = ['--catalog', str(path), 'qwen', '--external-api-base', 'http://box/v1', + '--external-model', 'Q/Q'] + assert EndpointAddCLI.main(argv=[*common, '--external-api-key-env', 'QKEY']) == 0 + entry = yaml.safe_load(path.read_text())['endpoints']['qwen'] + assert entry == {'external': {'api_base': 'http://box/v1', 'model': 'Q/Q', + 'api_key_env': 'QKEY'}} + for flags in (['--engine', 'vllm'], ['--gpu', '0'], ['--reclaim', 'stop']): + with pytest.raises(SystemExit, match='runs nothing'): + EndpointAddCLI.main(argv=[*common, '--force', *flags]) From cbfc1da73e776ca5b94357db9e82345cf0c1c346 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:23:22 -0400 Subject: [PATCH 110/141] One terminology: endpoint alias, upstream model, deployment id Defined once in leasing/endpoints.py; the public name is the alias, and the served name (public_name in older YAML) is the upstream model name, which --public-name's help had called the public name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 9 ++++++++- infer_stack/cli/commands_catalog.py | 6 +++++- infer_stack/leasing/catalog.py | 3 +++ infer_stack/leasing/endpoints.py | 12 ++++++++++++ infer_stack/leasing/naming.py | 8 +++++--- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 25742434..8f2081c5 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1137,12 +1137,19 @@ member raises "... does not require a lease; use `infer-stack access NAME`". `catalog show` prints the `external:` block. Tests 1-2 and the CLI in `tests/test_external_endpoints.py`. -### 31. [ ] One terminology +### 31. [x] One terminology Endpoint alias (what users request through the front door), upstream model (what the target server expects as `model`), deployment id (one managed realization). Stop calling `served_model_name` a "public name"; keep old YAML spellings as compatibility aliases; all new code uses the three terms. +*Done 2026-09-27:* the three terms are defined once, in +`leasing/endpoints.py`'s docstring (the naming module and +`EndpointSpec.served_name` point at it); `--public-name`'s help no longer +calls the upstream model the public name. YAML `public_name`/`served_name` +unchanged. New code (endpoints, routes, access) uses alias / upstream model / +deployment id. + ### 32. [ ] One semantic `GatewayRoute` Alias, upstream adapter (OpenAI-compatible / Ollama), upstream model, api_base, optional credential-env name, optional managed dynamic route id. diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index fe6d08f9..79c460f4 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -679,7 +679,11 @@ class EndpointAddCLI(_CatalogCommon): model = kw.Value(None, type=str, help='Model name (vllm) or tag (ollama).') host = kw.Value(None, type=str, help='Runtime host (ollama).') public_name = kw.Value( - None, type=str, help='Served/public name (for coalescing aliases).' + None, type=str, + help='Upstream model name the engine serves (--served-model-name; ' + 'default: the alias). Endpoints with the same upstream name and ' + 'model coalesce onto one deployment. Not the public name: clients ' + 'request the alias.', ) reclaim = kw.Value( None, type=str, choices=['keep-warm', 'stop', 'scale-to-zero'], diff --git a/infer_stack/leasing/catalog.py b/infer_stack/leasing/catalog.py index 4c6f9768..bbeb4c7b 100644 --- a/infer_stack/leasing/catalog.py +++ b/infer_stack/leasing/catalog.py @@ -174,6 +174,9 @@ class EndpointSpec: runtime: dict[str, Any] = field(default_factory=dict) sharing: str = Sharing.SHARED reclaim: str = DEFAULT_RECLAIM + #: The upstream model name the engine serves (``--served-model-name``); + #: YAML ``served_name`` or its older spelling ``public_name``. Defaults to + #: the endpoint alias, which is the public name. served_name: str | None = None # OpenAI surface the readiness probe (and clients) should use: 'chat' hits # /chat/completions, 'completions' hits /completions. A completions-only diff --git a/infer_stack/leasing/endpoints.py b/infer_stack/leasing/endpoints.py index b2cceedf..4479758e 100644 --- a/infer_stack/leasing/endpoints.py +++ b/infer_stack/leasing/endpoints.py @@ -14,6 +14,18 @@ resolved request (model source, revision, runtime knobs...), never from the catalog's own model keys, so two catalogs that name one model differently still agree. It knows nothing of leases, routes or backends. + +Three names, and only these (new code uses them; older spellings stay +readable): + +* **endpoint alias** -- what a client requests through the front door, the + catalog's endpoint name (``ResolvedEndpoint.alias``). It is the public name. +* **upstream model** -- what the serving process expects as ``model``: a + managed vLLM's ``--served-model-name`` (catalog ``served_name``, also spelled + ``public_name``; defaults to the alias), an Ollama tag, or + ``ExternalTarget.model``. Not public: clients never send it. +* **deployment id** -- one managed realization of an endpoint, when there is + one (external targets have none). """ from __future__ import annotations diff --git a/infer_stack/leasing/naming.py b/infer_stack/leasing/naming.py index c89515f8..415d9a8b 100644 --- a/infer_stack/leasing/naming.py +++ b/infer_stack/leasing/naming.py @@ -1,9 +1,11 @@ """The names infer-stack gives the things it runs, in one place. A compose service, a Kubernetes object, and the upstream host a gateway -route points at are all derived from the served model name (or an Ollama -host). The engine side and the gateway side must agree on these exactly, so -both import them from here. +route points at are all derived from the upstream model name (the served +model name; or an Ollama host). The engine side and the gateway side must +agree on these exactly, so both import them from here. The terms (endpoint +alias, upstream model, deployment id) are defined in +:mod:`infer_stack.leasing.endpoints`. """ from __future__ import annotations From 862c178a02e374a05e374f004dce79bdcdb8acdb Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:30:41 -0400 Subject: [PATCH 111/141] Published endpoint definitions outlive quiescence The recovery profile's catalogs are the published endpoint union. Every acquire merges the invocation's catalogs into it: a changed definition replaces an unpinned published one, a pinned one refuses, everything else stays. Quiescence adopts new settings but no longer replaces the catalogs, and a live conflict drops only the conflicting names instead of every unpinned definition. An external endpoint has no lease to keep it published, so only an explicit prune may unpublish it (campaign 2, decision 1). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/adr/0001-user-config-is-authoritative.md | 17 ++-- docs/planning/known-limitations.md | 5 +- docs/queue.md | 7 ++ infer_stack/leasing/controller.py | 59 ++++++------ infer_stack/leasing/profile.py | 89 +++++++++++++++++-- tests/test_leasing_profile.py | 47 +++++++++- 6 files changed, 176 insertions(+), 48 deletions(-) diff --git a/docs/adr/0001-user-config-is-authoritative.md b/docs/adr/0001-user-config-is-authoritative.md index f2774dbb..1690e95a 100644 --- a/docs/adr/0001-user-config-is-authoritative.md +++ b/docs/adr/0001-user-config-is-authoritative.md @@ -37,10 +37,17 @@ On acquire, under the publication lock: 1. Resolve the requested endpoint from the user's current catalog. 2. Compare current user settings/catalog with the recovery snapshot. 3. If the stack is quiescent (no active lease and no managed deployment - container), replace the snapshot wholesale with current user config. + container), adopt the current user settings. 4. If workloads are resident, keep global render settings frozen for that - leasing epoch, but merge compatible catalog additions into the snapshot. - Existing endpoint/bundle/route definitions may not be redefined live. + leasing epoch. + In both cases merge the current catalog into the snapshot's published + endpoint definitions: a definition it changes replaces the published one + unless a resident workload runs it (then the acquire refuses); every other + published definition stays. *Amended 2026-09-27 + ([external-endpoints](../planning/external-endpoints.md), decision 1): + quiescence used to replace the catalogs wholesale too, but an external + endpoint has no lease to keep it published, so only an explicit + `routes prune` unpublishes.* 5. Preview/admit/render using that exact candidate snapshot, then persist the snapshot before committing the acquire's desired-state mutation. @@ -80,8 +87,8 @@ published definitions are retained and incoming definitions are appended only if the semantic `CatalogUnion` has no conflicts. Thus an unrelated catalog edit cannot silently change an existing deployment's recovery render. -A wholesale snapshot replacement is allowed only at the same quiescence -boundary used by explicit publication. This preserves the original motivation +Replacing the settings is allowed only at the same quiescence boundary used +by explicit publication; `config publish` still replaces the whole snapshot. This preserves the original motivation for the frozen profile while removing it from the normal UX. ## Consequences diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index cea54690..a951b5a2 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -105,8 +105,9 @@ For crash-safe recovery, leasing still persists a frozen render snapshot. The controller advances it automatically on acquire under the publication lock: - **Quiescent stack.** With no active lease and no managed deployment - container, the next acquire adopts the current user settings and catalog - wholesale. + container, the next acquire adopts the current user settings. The catalog + is merged, never replaced: published endpoint definitions stay until + `routes prune` (an external endpoint has no lease to keep it). - **Compatible catalog additions while live.** New endpoint/bundle/route definitions can be merged into the active snapshot without changing any definition already frozen for resident workloads. This is the normal diff --git a/docs/queue.md b/docs/queue.md index 8f2081c5..171098f9 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1151,6 +1151,13 @@ unchanged. New code (endpoints, routes, access) uses alias / upstream model / deployment id. ### 32. [ ] One semantic `GatewayRoute` + +*Step 1 done 2026-09-27 (decision 1):* `adopt_catalog_sources` merges the +invocation's catalogs into the published union on every acquire: a changed +definition replaces an unpinned published one, a pinned one refuses, +everything else stays; quiescence adopts settings only. The live-epoch path +used to drop every unpinned definition on any conflict; it now drops only the +conflicting names. ADR 0001, known-limitations and `profile.py` updated. Alias, upstream adapter (OpenAI-compatible / Ollama), upstream model, api_base, optional credential-env name, optional managed dynamic route id. Produced by Compose (managed service name), KubeAI (cluster gateway), diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 85897adf..ec311b84 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1080,18 +1080,21 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: User configuration remains authoritative; the persisted profile is an internal crash-recovery snapshot, not a third configuration surface. - * Quiescent stack: adopt the invocation profile wholesale. - * Live epoch: retain frozen global render settings, but append compatible - catalog snapshots so newly suggested/added endpoints can be acquired - side by side. Semantic conflicts fail closed. + * Settings (backend, ports, images, gateway placement): adopted from + the invocation when the stack is quiescent, frozen while workloads + are resident. + * Catalogs, the published endpoint definitions: always merged + (:func:`~infer_stack.leasing.profile.adopt_catalog_sources`): the + invocation's definitions replace published ones nothing runs; one a + resident workload runs refuses; unrelated published definitions, + external endpoints included, stay. Quiescence does not unpublish. """ from .._log import logger from .profile import ( CatalogConflict, ProfileMismatch, - merge_catalog_sources, + adopt_catalog_sources, profile_drift, - prune_catalog_sources, ) stored = self.ledger.profile() @@ -1110,33 +1113,25 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: # because the stack happens to be quiescent. if not drift: return None - if self._profile_quiescent(residency): - return invocation - - candidate = dict(stored) + quiescent = self._profile_quiescent(residency) + candidate = dict(invocation) if quiescent else dict(stored) + # Only definitions a resident workload is actually running have to stay + # frozen; redefining anything else (the normal case while iterating with + # `catalog endpoint add --force`) replaces it. Nothing else unpublishes. + pinned = set() if quiescent else self._pinned_endpoints(residency) try: - candidate['catalogs'] = merge_catalog_sources( - stored.get('catalogs') or [], invocation.get('catalogs') or [] - ) - except CatalogConflict: - # Only definitions a resident workload is actually running have to - # stay frozen. Redefining anything else -- the normal case while - # iterating with `catalog endpoint add --force` -- drops the stale - # definition instead of refusing every acquire on the host. - pinned = self._pinned_endpoints(residency) - try: - candidate['catalogs'] = merge_catalog_sources( - prune_catalog_sources(stored.get('catalogs') or [], pinned), - invocation.get('catalogs') or [], - ) - except CatalogConflict as pinned_ex: - blocked = sorted(set(pinned_ex.names) & pinned) or sorted(pinned_ex.names) - raise ProfileMismatch( - f'the current catalog redefines {", ".join(repr(b) for b in blocked)}, ' - 'which a resident deployment is running. Release or evict it ' - f'(`infer-stack evict {blocked[0]}`), then retry; definitions ' - 'nothing is running are updated automatically' - ) from pinned_ex + candidate['catalogs'] = adopt_catalog_sources( + stored.get('catalogs') or [], invocation.get('catalogs') or [], pinned) + except CatalogConflict as pinned_ex: + blocked = sorted(set(pinned_ex.names) & pinned) or sorted(pinned_ex.names) + raise ProfileMismatch( + f'the current catalog redefines {", ".join(repr(b) for b in blocked)}, ' + 'which a resident deployment is running. Release or evict it ' + f'(`infer-stack evict {blocked[0]}`), then retry; definitions ' + 'nothing is running are updated automatically' + ) from pinned_ex + if quiescent: + return candidate if candidate != stored else None deferred = [ key for key in drift if key != 'catalogs' diff --git a/infer_stack/leasing/profile.py b/infer_stack/leasing/profile.py index 70776b42..865d2d6e 100644 --- a/infer_stack/leasing/profile.py +++ b/infer_stack/leasing/profile.py @@ -9,9 +9,9 @@ The first controller mutation against a ledger with no profile freezes that invocation's resolved settings. Later operations render from the stored copy. For the ordinary ``config init -> catalog suggest --apply -> acquire`` workflow, -acquire advances this snapshot automatically: compatible catalog additions are -merged into a live epoch, and a quiescent stack adopts the current invocation -profile wholesale. ``infer-stack config publish`` remains an advanced explicit +acquire advances this snapshot automatically: the invocation's catalogs are +merged into the published endpoint definitions (:func:`adopt_catalog_sources`), +and a quiescent stack adopts the invocation's settings. ``infer-stack config publish`` remains an advanced explicit pre-seed/preview operation; it is not a required third configuration step. Not in the profile: @@ -229,10 +229,10 @@ def merge_catalog_sources( resolves differently, :class:`CatalogConflict` is raised before anything is written. - The snapshots are intentionally retained rather than replacing the old - source while workloads are resident. Once the stack is quiescent the next - acquire replaces the profile wholesale with the current user configuration, - compacting this history back to the authoritative source(s). + The snapshots are the published endpoint definitions (see + :func:`adopt_catalog_sources` for how an acquire updates them); an + external endpoint has no lease to pin it, so they are never dropped merely + because the stack is quiescent. """ out: list[dict[str, Any]] = [] seen: set[str] = set() @@ -247,6 +247,81 @@ def merge_catalog_sources( return out +def drop_catalog_names( + sources: list[dict[str, Any]], names: set[str] +) -> list[dict[str, Any]]: + """``sources`` without the endpoints and bundles named ``names``; a source + left with neither goes.""" + out: list[dict[str, Any]] = [] + for source in sources or []: + endpoints = {n: s for n, s in (source.get('endpoints') or {}).items() + if n not in names} + bundles = {n: m for n, m in (source.get('bundles') or {}).items() + if n not in names} + if not endpoints and not bundles: + continue + kept = dict(source) + kept['endpoints'] = endpoints + kept['bundles'] = bundles + out.append(kept) + return out + + +def adopt_catalog_sources( + published: list[dict[str, Any]], incoming: list[dict[str, Any]], + pinned: set[str], +) -> list[dict[str, Any]]: + """The published endpoint definitions after an acquire or access. + + The published union is where endpoint definitions live between runs + (docs/planning/external-endpoints.md, decision 1). ``incoming`` (the + invocation's catalogs) is merged in: a definition it redefines replaces the + published one, unless a resident workload runs it (``pinned``: then + :class:`CatalogConflict`); every other published definition stays, however + unrelated. A published snapshot also sheds definitions ``incoming`` repeats, + so editing a catalog over time does not pile up copies. + + Example: + >>> ep = lambda src: {'engine': 'vllm', 'model': 'm'} + >>> cat = lambda **eps: {'models': {'m': {'source': 'hf://o/m'}}, 'endpoints': eps} + >>> old = cat(a={'engine': 'vllm', 'model': 'm'}, + ... x={'external': {'api_base': 'http://h/v1', 'model': 'q'}}) + >>> new = cat(a={'engine': 'vllm', 'model': 'm', 'runtime': {'max_model_len': 8}}) + >>> out = adopt_catalog_sources([old], [new], pinned=set()) + >>> sorted(CatalogUnion.from_sources(out).endpoints) # x survives + ['a', 'x'] + >>> adopt_catalog_sources([old], [new], pinned={'a'}) + Traceback (most recent call last): + ... + infer_stack.leasing.profile.CatalogConflict: endpoint 'a' is defined differently in two published catalogs + """ + base = list(published or []) + want = [dict(s or {}) for s in incoming or []] + mine = CatalogUnion.from_sources(want) if want else None + if mine is not None: + # Definitions the invocation repeats identically live in its snapshot. + same = set() + for source in base: + for name in (source.get('endpoints') or {}): + if name in mine.endpoints: + try: + other = Catalog.from_dict(source).resolve_endpoint(name) + if other.semantic_key() == mine.resolve_endpoint(name).semantic_key(): + same.add(name) + except CatalogError: + pass + base = drop_catalog_names(base, same) + for _ in range(64): # each round drops at least one name + try: + return merge_catalog_sources(base, want) + except CatalogConflict as ex: + names: set[str] = {str(n) for n in ex.names} + if not names or names & pinned: + raise + base = drop_catalog_names(base, names) + raise CatalogConflict('catalog union did not settle', []) + + def prune_catalog_sources( sources: list[dict[str, Any]], keep: set[str] ) -> list[dict[str, Any]]: diff --git a/tests/test_leasing_profile.py b/tests/test_leasing_profile.py index a4ec239d..413ba56d 100644 --- a/tests/test_leasing_profile.py +++ b/tests/test_leasing_profile.py @@ -86,7 +86,9 @@ def test_drift_is_warned_once(tmp_path): assert len(warnings) == 1 and 'ui' in warnings[0] -def test_acquire_auto_adopts_current_catalog_when_quiescent(tmp_path): +def test_a_quiescent_acquire_publishes_its_catalog_beside_the_others(tmp_path): + """Quiescence lets settings change; it does not unpublish endpoints + (campaign 2 decision 1: an external endpoint has no lease to keep it).""" a = Catalog.from_dict(cat('alpha')) b = Catalog.from_dict(cat('beta')) ledger, ctl = controller(tmp_path, catalog=a) @@ -94,7 +96,7 @@ def test_acquire_auto_adopts_current_catalog_when_quiescent(tmp_path): _, ctl2 = controller(tmp_path, catalog=b) out = ctl2.acquire('x', b.resolve_names(['beta']), wait=False) assert out.lease.endpoints == ['beta'] - assert set(ctl2.backend.catalog.endpoints) == {'beta'} + assert set(ctl2.backend.catalog.endpoints) == {'alpha', 'beta'} assert ledger.publication_pending() is None @@ -636,3 +638,44 @@ def cat(model_key): assert list(union.endpoints) == ['qwen'] a, b = (Catalog.from_dict(cat(k)).resolve_endpoint('qwen') for k in ('alias-a', 'alias-b')) assert a.semantic_key() == b.semantic_key() + + +# -- the published endpoint union (campaign 2, decision 1) --------------------------- + + +def test_quiescence_adopts_new_settings_but_keeps_published_endpoints(tmp_path): + a = Catalog.from_dict(cat('alpha')) + b = Catalog.from_dict(cat('beta')) + ledger, ctl = controller(tmp_path, catalog=a, ui=True) + ctl.gc() # freezes ui=True, {alpha} + _, ctl2 = controller(tmp_path, catalog=b, ui=False) + ctl2.acquire('x', b.resolve_names(['beta']), wait=False) + assert ledger.profile()['ui'] is False # settings: the invocation's + assert set(ctl2.backend.catalog.endpoints) == {'alpha', 'beta'} + + +def test_a_live_redefinition_drops_only_the_redefined_name(tmp_path): + """A conflict used to drop every unpinned published definition; an + external endpoint published by another runbook would have gone with it.""" + def beta(**runtime): + return {'engine': 'vllm', 'model': 'm', 'reclaim': 'stop', + **({'runtime': runtime} if runtime else {})} + + other = Catalog.from_dict({ + 'models': {'m': {'source': 'hf://org/beta'}}, + 'endpoints': {'beta': beta(), + 'remote': {'external': {'api_base': 'http://box/v1', 'model': 'q'}}}, + }) + edited = Catalog.from_dict({'models': {'m': {'source': 'hf://org/beta'}}, + 'endpoints': {'beta': beta(max_model_len=1024)}}) + a = Catalog.from_dict(cat('alpha')) + ledger, ctl = controller(tmp_path, catalog=a) + ctl.acquire('a', a.resolve_names(['alpha']), wait=False) # alpha live, pinned + _, ctl2 = controller(tmp_path, catalog=other) + lease = ctl2.acquire('b', other.resolve_names(['beta']), wait=False).lease + ctl2.release(lease.id) # beta stopped: unpinned + _, ctl3 = controller(tmp_path, catalog=edited) + ctl3.acquire('c', edited.resolve_names(['beta']), wait=False) # redefines beta + published = ctl3.backend.catalog + assert {'alpha', 'beta', 'remote'} <= set(published.endpoints) + assert published.resolve_endpoint('beta').to_request().capacity == {'max_model_len': 1024} From e697ccd4506c7570d74bd9e64dfa58765a888bf4 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 11:57:34 -0400 Subject: [PATCH 112/141] One GatewayRoute; gateway routes derived from the published catalogs A GatewayRoute (alias, kind, upstream model, api_base, key name, managed id) has one renderer. front_door_routes derives the static table (registry < catalog < deployment < upstream) or the dynamic set for the Compose gateway, KubeAI's host gateway and the in-cluster gateway alike; render_front_door takes routes, and its catalog-superset and legacy branches and the upstream pseudo-engine are gone. The published catalog union is the one store of endpoint definitions: the route registry keeps only routes of deployments no catalog defines, remembered past release so releasing one does not recreate the gateway. Rows an older version wrote are still routed. routes seed merges catalogs into the union (a redefinition refuses unless --replace, never one a resident deployment runs), routes prune unpublishes, routes list shows each route's origin. External endpoints route to their own server on every backend, with their key by name; a dynamic external route's id comes from its alias. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 23 + dev/journals/claude.md | 34 ++ dev/leasing-followups.md | 4 +- dev/lessons/lessons.md | 13 + docs/litellm-gateway-routing.md | 163 +++--- docs/planning/backend-parity-roadmap.md | 2 +- docs/planning/external-endpoints.md | 24 +- docs/queue.md | 19 +- infer_stack/backends/kubeai.py | 77 +-- infer_stack/backends/kubeai_gateway.py | 36 +- infer_stack/cli/commands_leasing.py | 157 +++--- infer_stack/leasing/backend.py | 23 +- infer_stack/leasing/compose.py | 145 +++-- infer_stack/leasing/controller.py | 183 ++++-- infer_stack/leasing/gateway.py | 708 ++++++++---------------- infer_stack/leasing/naming.py | 3 +- infer_stack/leasing/routes.py | 154 +++++- tests/test_cli_leasing.py | 28 +- tests/test_external_endpoints.py | 134 +++++ tests/test_leasing_compose.py | 16 +- tests/test_leasing_dynamic_routing.py | 7 +- tests/test_leasing_route_registry.py | 424 ++++++-------- 22 files changed, 1277 insertions(+), 1100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b59a9ca8..3536f731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### External endpoints; gateway routes derived from the published catalogs + +- A catalog endpoint can name an OpenAI-compatible server infer-stack does + not run: `external: {api_base, model, api_key_env}` (or `catalog endpoint + add NAME --external-api-base ... --external-model ...`). It has no model, + lease or deployment; the gateway routes its alias to that server, sending + the key named by `api_key_env` (the name only is stored). Runtime options + beside `external:` are refused, and `api_key_env` may not name one of + infer-stack's own secrets. +- Published endpoint definitions outlive quiescence: an acquire merges its + catalogs into the published union instead of replacing it, so an unrelated + runbook no longer unpublishes another's endpoints. `routes prune` is the + explicit removal. +- Gateway routes are derived at every render from the published union and + the placed deployments. `litellm_registry.json` now keeps only the routes + of deployments no catalog defines (remembered past release, as before); + rows an older version wrote are still routed. `routes seed` publishes + catalogs, `routes prune` unpublishes, and `routes list` shows each route's + origin. A state dir from before the registry is no longer seeded from its + `litellm_config.yaml`. +- With dynamic routing, an external endpoint's route has a stable id from its + alias, so pointing it at another server replaces one route. + ### Hardening from a review of the backend-parity work - A KubeAI Model with several replicas (`min_replicas`) is resident like one diff --git a/dev/journals/claude.md b/dev/journals/claude.md index 0b006ec6..470c77ac 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3376,3 +3376,37 @@ catch it. (2) Physical state and serving state are different questions; once residency became physical, every reader that meant "is it serving" needed its own summary. (3) A race test must inject the other writer at the lock, not before the call; otherwise an earlier unlocked check passes it. + +## 2026-09-27 — campaign 2, item 32: one GatewayRoute, routes derived at render + +**Intent.** Replace the route plumbing (registry rows of three shapes, four +`render_front_door` strategies, `UPSTREAM_ROUTE`, KubeAI's parallel +`upstream_rows`) with one route type, and make the published catalog union +the only store of endpoint definitions (design decision 2). Model: Claude +Opus 5.5 (claude-opus-5-5), Claude Code. + +**Did.** `GatewayRoute` with one renderer; `front_door_routes` derives the +static table or the dynamic set for all three gateways; `routes seed` / +`prune` publish into and unpublish from the union; external endpoints route +to their own servers on every backend, with a dynamic id from their alias. + +**Changed my mind.** The design said deployment routes would be derived only +while live. The first full run failed +`test_converge_to_empty_keeps_the_front_door`: releasing an ad-hoc +deployment then recreated the gateway, a blip for every client. The registry +keeps that one owner (routes of aliases no catalog defines), and the design +doc says so. Also dropped the seed-from-`litellm_config.yaml` migration: +with catalog rows no longer written, reseeding from our own rendered config +would have made every route permanent. + +**Mistakes.** `Gateway.remember` took the converge flock inside a converge +and hung the suite (lesson added). A `pkill -f pytest` pattern matched my +own shell and killed the edit in the same command; the edit had to be +reapplied. + +**Takeaways.** (1) A "derive, do not store" rule needs checking against +every property the store was providing: the registry was also what kept the +gateway byte-stable across an ad-hoc release. (2) Tests that called +`render_compose` with deployments that serve no catalog endpoint were +testing a branch production never took; making the deployments serve the +catalog's aliases made them test the real property. diff --git a/dev/leasing-followups.md b/dev/leasing-followups.md index 9d6b1095..add2e805 100644 --- a/dev/leasing-followups.md +++ b/dev/leasing-followups.md @@ -133,8 +133,8 @@ revived and LiteLLM's admin API (`/model/new`, `/model/delete`, `STORE_MODEL_IN_DB` + `DATABASE_URL`) updates routes on a **running** gateway — zero blip, no restart. Off by default; enable with `config set dynamic_routing true` / `--dynamic-routing`. See `docs/litellm-gateway-routing.md` and -`infer_stack/leasing/compose.py` (`_litellm_routes`, -`ComposeBackend._reconcile_routes`); tests in +`infer_stack/leasing/gateway.py` (`deployment_routes`, +`Gateway._reconcile_routes`); tests in `tests/test_leasing_dynamic_routing.py`. It is strictly nicer than #1 (no blip, no advertising of undeployed models) and diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index f96d8327..e5a1c392 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -153,3 +153,16 @@ evidence; prefer append-only; supersede incorrect entries with a new one. printed the copy's `infer_stack.__file__` and passed while edits continued. - **Applies when:** a long e2e or audit runs in tmux while work continues in the same checkout. + +- **Lesson:** Code that runs inside `converge` must not take the backend's + converge flock again. `flock` locks belong to an open file description, so a + second open of the same lock file in the same process waits on itself for + ever; nothing times out and pytest just hangs. Helpers called from a + converge assume the caller holds the lock (and say so); only entry points + called from outside take it. +- **Evidence / MWE:** 2026-09-27: `Gateway.remember` opened with + `with self._converge_lock():` and was called from `ComposeBackend.converge`, + which already held it. `tests/test_leasing_compose.py` hung with no output + until killed; removing the inner lock made it finish in 4 s. +- **Applies when:** adding a gateway/backend helper that writes state under + the state dir and may be reached from a render or converge. diff --git a/docs/litellm-gateway-routing.md b/docs/litellm-gateway-routing.md index 2e2b1bf0..dd6c8133 100644 --- a/docs/litellm-gateway-routing.md +++ b/docs/litellm-gateway-routing.md @@ -4,83 +4,86 @@ This explains how the leasing stack exposes models through the LiteLLM gateway, why the current design avoids restarting the gateway, and the direction we intend to take it. -## How it works today: static superset route table, backed by a route registry +## How it works today: a static superset route table, derived at render The gateway is rendered with a **static superset route table**: one LiteLLM -`model_list` entry per endpoint, each pointed at the endpoint's deterministic -upstream host (`vllm_service_name_for` / `ollama_service_name_for`). - -The set of routes comes from an **append-only route registry** persisted in the -shared state dir (`/litellm_registry.json`). On every converge the -backend merges two sources into the registry and renders `model_list` from the -**whole registry**: - -1. the **invoking catalog's** endpoints (when a catalog is discoverable), and -2. every **placed deployment in the full `desired` set** — which spans *all* - runbooks sharing the stack, via the shared ledger. - -See `ComposeBackend._update_route_registry`, -`_litellm_model_list_from_registry`, and the `route_registry is not None` branch -of `render_compose`. The registry stores *semantic* inputs (served name / -engine / host), never rendered LiteLLM entries, so a renderer change propagates -to old rows automatically. +`model_list` entry per endpoint alias, each pointed at the endpoint's +deterministic upstream host (`vllm_service_name_for` / +`ollama_service_name_for`), or, for an external endpoint, at its own server. + +The table is **derived at every render**, never stored, from these layers +(a later one wins for the same alias; `gateway.front_door_routes`): + +1. the **route registry** (`/litellm_registry.json`): routes of + deployments no published catalog defines, remembered past their release, + plus rows an older infer-stack wrote for catalog endpoints; +2. the **published catalog union**: every endpoint every runbook on this + stack has published (kept in the recovery profile, merged on every + acquire; see docs/planning/external-endpoints.md, decision 1); +3. every **placed deployment in the full `desired` set**, which spans all + runbooks via the shared ledger; +4. routes another backend supplies (a KubeAI cluster's Models). + +Each is a `GatewayRoute` (`infer_stack/leasing/routes.py`), rendered by one +function, `GatewayRoute.entry()`. The registry stores *semantic* inputs +(served name / engine / host), never rendered LiteLLM entries. Two key properties: -- **The gateway config depends only on accumulated shared state, not on which - runbook invoked the converge or which deployments are currently placed.** - Acquiring or releasing a model does not change the rendered - `litellm_config.yaml`, so its `config_hash` label is stable and - `docker compose up -d` leaves the gateway container untouched — **no blip.** A - route whose upstream is not up yet simply cools down until the container - appears; `router_settings` make that warmup self-healing. -- **A cross-catalog converge can no longer strip another runbook's live - routes.** Because the render is the union of everything ever merged (not just - the invoking catalog), converging runbook B's disjoint catalog does not drop - runbook A's still-live routes. This is the fix for the "olmo healthy, gateway - 400 `Invalid model name`" incident: another runbook's converge used to - re-render the shared gateway from *its* catalog alone and evict olmo's routes, - even though the olmo container was healthy. Routes also **persist past - release** — a released endpoint stays routable/testable until explicitly - pruned. - -Once every catalog has been merged once, all converges render byte-identical -configs, so the gateway is never recreated regardless of how the runbooks -interleave. The only residual (justified) recreations are the first-ever -appearance of a new endpoint or a genuinely changed endpoint definition. - -### Managing the registry: `infer-stack routes` - -- `infer-stack routes list` — print the registry (alias, engine, served/tag, - upstream, and whether a live deployment currently backs each route). -- `infer-stack routes seed [...]` — merge sibling runbooks' - catalogs into the registry and converge once. The operational key to - blip-free concurrency: **seed all overlapping runbooks' catalogs once while - idle**, and then no converge from any of them ever recreates the gateway (the - registry is already the full union). Works before `stack up`. -- `infer-stack routes prune` — the explicit "forget stale endpoints" verb: - rewrite the registry to *invoking catalog ∪ live deployments* and converge - (one accepted recreate). It prints the exact drop list and asks for - confirmation first (`--yes` / non-terminal skips), because a prune from the - wrong `INFER_STACK_CONFIG_DIR` would silently drop every other runbook's - routes. Automatic pruning is deliberately excluded — any catalog-keyed rule - would reintroduce the cross-catalog alternation churn the registry exists to - avoid. - -Both `routes` and the automatic first-converge merge are safe under -concurrency: the read-merge-write runs under the per-state-dir converge flock -the converge already holds. - -### Seeding on upgrade - -A state dir upgraded in place (a rendered `litellm_config.yaml` but no registry -yet) seeds the registry from that config on the first converge: `openai/` -inverts exactly to a vLLM row, so no route is lost. Ollama rows are skipped with -a warning (their host survives only as a non-invertible slug in `api_base`) and -re-enter the registry at the next converge that has them in its catalog or live -set. A corrupt/garbage registry is fail-open (rebuilt from seed + catalog); an -*unknown* schema version with a parseable `entries` map is rendered as-is -without rewrite, so a binary rollback never discards the accumulated union. +- **The gateway config depends only on shared state, not on which runbook + invoked the converge or which deployments are currently placed.** Acquiring + or releasing a model does not change the rendered `litellm_config.yaml`, so + its `config_hash` label is stable and `docker compose up -d` leaves the + gateway container untouched: **no blip.** A route whose upstream is not up + yet cools down until the container appears; `router_settings` make that + warmup self-healing. An ad-hoc deployment (one no catalog defines) is + remembered in the registry after an approved render, so releasing it does + not change the config either. +- **A cross-catalog converge cannot strip another runbook's routes.** The + published union holds every runbook's catalogs, so converging runbook B's + disjoint catalog does not drop runbook A's routes. This is the fix for the + "olmo healthy, gateway 400 `Invalid model name`" incident: another runbook's + converge used to re-render the shared gateway from *its* catalog alone and + evict olmo's routes. Routes **persist past release**: a released endpoint + stays routable until explicitly pruned. + +The only gateway recreations are the first appearance of an endpoint (or an +ad-hoc deployment) and a genuinely changed endpoint definition. + +### Managing routes: `infer-stack routes` + +- `infer-stack routes list`: every route the next render has (alias, origin, + upstream model, upstream, and whether a live deployment backs it). Origins + are `catalog`, `external`, `deployment`, `upstream` (KubeAI) and `registry`. +- `infer-stack routes seed [...]`: publish sibling runbooks' + catalogs (merge them into the published union) and publish once. The + operational key to blip-free concurrency: **seed all overlapping runbooks' + catalogs once while idle**, and no converge from any of them recreates the + gateway afterwards. A seed that would redefine a published alias refuses + unless `--replace`, and never redefines one a resident deployment runs. +- `infer-stack routes prune`: the explicit "unpublish" verb. It drops every + published endpoint and registry row the invoking catalog does not define and + no deployment serves, then publishes (one accepted recreate). It prints the + exact drop list and asks first (`--yes` / non-terminal skips), because a + prune from the wrong `INFER_STACK_CONFIG_DIR` would drop every other + runbook's endpoints. Nothing prunes automatically: any catalog-keyed rule + would reintroduce the cross-catalog alternation churn. + +Both commit under the host-wide publication lock, like any desired-state +change. + +### Upgrading + +The published union and the registry are read as they are. A registry an +older infer-stack wrote holds catalog rows too; they stay routed (lowest +precedence, so a published definition wins) until `routes prune` drops the +ones nothing uses, and the file is not rewritten for them. A corrupt registry +is ignored; an *unknown* schema version with a parseable `entries` map is +rendered as-is and never rewritten, so a binary rollback loses nothing. A +state dir from before the registry existed (a `litellm_config.yaml` and no +registry) is no longer seeded from that config: its runbooks' catalogs +publish their endpoints again at their next acquire, or at once with +`routes seed`. LiteLLM reads `--config /etc/litellm/config.yaml` **once at startup** and does not watch the file, so this static-config approach is what lets us avoid @@ -99,17 +102,12 @@ recreated are first bring-up and an actual **catalog** change. `dev/leasing-followups.md` ("Upstream LiteLLM: a quiet, generation-level readiness probe"). Do not globally silence LiteLLM logs to hide it. - An endpoint that is **not in the catalog** (ad-hoc / interactive `acquire`) - is now routed too: every placed deployment in the `desired` set is merged into - the registry (item 2 above), so a non-catalog acquire becomes — and, via the - registry's persistence, stays — routable. (Before the registry, the legacy - `_litellm_model_list` fallback routed only placed deployments and - rewrote-and-recreated the gateway per change; that path now survives only in - `render_compose` for direct callers/tests, never from `ComposeBackend`.) + is routed too: every placed deployment is a route layer, and the registry + remembers it past release, so it stays routable without a recreate. - **Same-model `--dedicated` collapses.** Because the upstream host is derived from the served name alone (`vllm-`), N dedicated deployments of one model render to ONE compose service — one container, one GPU — defeating - multi-GPU dedicated use. Both this and the non-catalog limitation above are - fixed by the opt-in **dynamic routing** mode below. + multi-GPU dedicated use. The opt-in **dynamic routing** mode below fixes it. ## Dynamic routing: LiteLLM admin API + Postgres (implemented, opt-in) @@ -158,8 +156,9 @@ gateway `base_url`; nothing about the env-file descriptor changes. ### Why this is robust (the implementation honors these) -- **Deterministic ids ⇒ pure set-diff reconcile.** `_route_id(deployment, endpoint)` - is stable, so the same route is added once and never churned, and a dropped +- **Deterministic ids ⇒ pure set-diff reconcile.** `_route_id(owner, alias)` + (the owner is the deployment id, or `external` for an external endpoint, so + redefining an external endpoint's server replaces one route) is stable, so the same route is added once and never churned, and a dropped route is deleted by exactly its id. Reconcile is idempotent (a redundant apply is a no-op), which is what makes the controller's *coalesced* apply correct for routes too. diff --git a/docs/planning/backend-parity-roadmap.md b/docs/planning/backend-parity-roadmap.md index 384412e7..0d207b65 100644 --- a/docs/planning/backend-parity-roadmap.md +++ b/docs/planning/backend-parity-roadmap.md @@ -289,7 +289,7 @@ worse. A blocker is fixed whatever its size. | how the TUI runs a runtime command | it replaced the backend's runner with Docker's, whose allowlisted environment has no `KUBECONFIG`: every kubectl call from the TUI failed | **fixed** (P2): only `docker` commands are wrapped | | the KubeAI gateway's approval | the gateway project asked its own diff approval at render, after the lease committed | **fixed** (P3): previewed and approved with the acquire's | | KubeAI's route rows | written into the registry before any approval, and only for live Models (so every new Model recreated the gateway) | **fixed** (P3): render inputs, persisted after approval; the catalog's rows too, so no blip | -| a route's upstream | derived twice, in the render and in `routes list` (which showed `?` for a cluster route) | **fixed** (P3): `registry_route_entry` | +| a route's upstream | derived twice, in the render and in `routes list` (which showed `?` for a cluster route) | **fixed** (P3); since campaign 2 one `GatewayRoute` and one renderer (`GatewayRoute.entry`) | | a deployment's Model name | derived in five places | **fixed** (P3): `model_name()` | | "is this the compose backend" in the CLI | `isinstance(…, ComposeBackend)` for `routes`, `gc --orphans`, `clean`, `network migrate`, `secrets rotate` | **fixed** (P3): the capability each needs (`compose_project()`, residency labels, `network`, `litellm`); since item 12, `backend.host_runtime` | | how many units a deployment may have | "more than one is ambiguous", written into the shared residency | **fixed** (item 11): `Residency.replicated`, set by the runtime that built it; only GPU adoption asks for a unique unit | diff --git a/docs/planning/external-endpoints.md b/docs/planning/external-endpoints.md index faaf8bbe..94b09019 100644 --- a/docs/planning/external-endpoints.md +++ b/docs/planning/external-endpoints.md @@ -74,14 +74,18 @@ Three owners, never mixed in one map: | Routes | Derived from | Stored | |---|---|---| | catalog routes | the published union | no: derived at render | -| deployment routes (static) | live deployments whose endpoint no published catalog defines | no: derived at render | +| deployment routes (static) | placed deployments | remembered in `litellm_registry.json` past release, only for an alias no published catalog defines | | dynamic routes | placed deployments, and external targets | reconciled into LiteLLM's DB (as today) | -| legacy rows | an existing `litellm_registry.json` from before this change (seeded siblings, a migrated `litellm_config.yaml`) | read-only compatibility; `routes prune` drops them | +| older registry rows | a `litellm_registry.json` an older version wrote (catalog rows, seeded siblings) | read, lowest precedence; `routes prune` drops the ones nothing uses | The registry file stops being written with catalog-derived rows: once the -union is durable, it carries nothing the union does not. Existing files are -still read, so an upgrade loses no route; `routes list` marks their rows as -legacy. `routes seed FILE...` becomes "merge these catalogs into the +union is durable, it carries nothing the union does not. It keeps one owner: +ad-hoc deployments (no catalog defines them), whose route must outlive their +release or releasing one recreates the gateway (a blip for every client). +*Revised in implementation:* the first draft derived those only while live; +`test_converge_to_empty_keeps_the_front_door` showed the blip. Existing files +are still read, so an upgrade loses no route; `routes list` marks their rows +`registry`. `routes seed FILE...` becomes "merge these catalogs into the published union" (same user-visible effect: the gateway serves their aliases; conflicts refuse unless `--replace`). `CatalogUnion` then decides conflicts from endpoint semantics alone, and the import of @@ -160,16 +164,18 @@ request is sent to the external upstream. ## What goes away -- catalog-derived and deployment-derived rows in `litellm_registry.json` - (the file becomes read-only legacy input); +- catalog-derived rows in `litellm_registry.json` (it keeps only ad-hoc + deployments' routes); - `CatalogUnion`'s route-row comparison and its import of `_registry_incoming_from_catalog`; - the `upstream` pseudo-engine (`UPSTREAM_ROUTE`): a KubeAI or external route is a `GatewayRoute` of kind `openai` with an api_base; - the registry row shapes `{engine, served, host}` and their second interpreter in `routes list`: one `GatewayRoute` and one renderer; -- `render_front_door`'s catalog-superset and legacy placed-deployment - branches, reached only by tests (the tests move to the route planner); +- `render_front_door`'s four route strategies: it takes a route table, or + dynamic routes, from `front_door_routes`; +- the seed-from-`litellm_config.yaml` upgrade path (it would make every + rendered route permanent once the registry stopped being written); - `backend.access(endpoints) -> dict`: the front door gives connection info, endpoints give request names, an `AccessResult` combines them; - a descriptor that needs a lease (`INFER_STACK_LEASE_ID` is written only diff --git a/docs/queue.md b/docs/queue.md index 171098f9..247080c1 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1150,7 +1150,21 @@ calls the upstream model the public name. YAML `public_name`/`served_name` unchanged. New code (endpoints, routes, access) uses alias / upstream model / deployment id. -### 32. [ ] One semantic `GatewayRoute` +### 32. [x] One semantic `GatewayRoute` + +*Done 2026-09-27.* `GatewayRoute` (`leasing/routes.py`) with one renderer, +`entry()`; `front_door_routes` derives the static table (registry < catalog +< deployment < upstream) or the dynamic set (one per managed id) for Compose, +KubeAI's host gateway and the in-cluster gateway alike. `render_front_door` +takes routes; its catalog-superset and legacy branches and +`UPSTREAM_ROUTE` are gone (older registry rows spelled `upstream` still +read). The registry stores only ad-hoc deployments' routes (decision 2, +revised: see the design doc). Backend protocol: `routes()` / +`catalog_routes()` replace `route_rows()` / `catalog_route_rows()`. `routes +seed` merges into the published union (refuses conflicts unless +`--replace`, never a pinned one), `routes prune` unpublishes, `routes list` +shows origins. External routes (item 33's static half, and the dynamic id) +came with it: they route to their own server on every backend. *Step 1 done 2026-09-27 (decision 1):* `adopt_catalog_sources` merges the invocation's catalogs into the published union on every acquire: a changed @@ -1170,6 +1184,9 @@ storage for everything. Delete `render_front_door` branches that only tests reach (cleanup B), never a supported mode. ### 33. [ ] External routes in static and dynamic routing +*Partly done with 32:* static and dynamic routes, the `('external', alias)` +id, the key name in `model_info` for reconcile. Left: survival tests (gc, +release, dedicated going away) and retirement through a real reconcile. Dynamic mode: an external route is standing desired state with a stable managed id from its logical owner, the endpoint alias (decision 28.5), not from target semantics; the semantic route decides whether it is replaced. It survives zero leases, diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index bb1a9052..cc72398f 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -652,42 +652,52 @@ def dynamic_routing(self) -> bool: """The gateway manages routes live, so each deployment is its own Model.""" return bool(self.gateway is not None and getattr(self.gateway, 'dynamic_routing', False)) - def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: - """Registry rows sending each vLLM catalog endpoint to its Model. + def catalog_routes(self, catalog) -> list: + """The routes every endpoint of ``catalog`` gets: each vLLM endpoint to + its Model through KubeAI, each external one to its own server. The static superset, as on compose: the gateway's config then stays byte-stable as models come and go, so it is never recreated for one. """ - from ..leasing.gateway import UPSTREAM_ROUTE, _registry_incoming_from_catalog + from ..leasing.gateway import catalog_routes + from ..leasing.routes import GatewayRoute if catalog is None or self.gateway is None: - return {} + return [] base = self._upstream_url() - return { - alias: {'engine': UPSTREAM_ROUTE, - 'served': model_name_for(row.get('served') or alias), - 'api_base': base} - for alias, row in _registry_incoming_from_catalog(catalog).items() - if row.get('engine') == 'vllm' - } - def route_rows(self, desired: list[Deployment], placement=None) -> dict[str, dict[str, Any]]: - """The registry rows a render of ``desired`` merges (``routes prune`` - keeps exactly these): the catalog's, and every rendered Model's.""" - return self._front_door_inputs(desired, self._render_documents(desired)[1])[0] + def managed(alias, request): + if request.engine != 'vllm': + return None + served = request.served.get('served_model_name') or alias + return GatewayRoute(alias, 'openai', model_name_for(served), base) + + return catalog_routes(catalog, managed) + + def routes(self, desired: list[Deployment], placement=None) -> list: + """The routes a render of ``desired`` gives the gateway (``routes list``): + its whole table, the route registry included.""" + if self.gateway is None: + return [] + self._set_front_door(desired, self._render_documents(desired)[1]) + if getattr(self.gateway, 'in_cluster', False): + return self.gateway.static_routes() + return self.gateway.routes([]) def _front_door_inputs(self, desired, rendered: RenderedModels): - """``(registry rows, dynamic routes)`` for the gateway, from a render. + """``(static routes, dynamic routes)`` the gateway adds, from a render. - Static routing: one row per alias, to the Model serving it. Dynamic - routing: one route per (deployment, endpoint), each to its own Model, - so same-model ``--dedicated`` Models share the alias and LiteLLM - balances across them. + Static routing: one route per alias, to the Model serving it, over the + catalog's. Dynamic routing: one route per (deployment, endpoint), each + to its own Model, so same-model ``--dedicated`` Models share the alias + and LiteLLM balances across them. External endpoints route to their + own servers either way, never through KubeAI. """ - from ..leasing.gateway import UPSTREAM_ROUTE, upstream_route + from ..leasing.gateway import catalog_routes, upstream_route + from ..leasing.routes import GatewayRoute, route_table if self.gateway is None: - return {}, [] + return [], [] base = self._upstream_url() if self.dynamic_routing: by_id = {g.id: g for g in desired} @@ -696,19 +706,22 @@ def _front_door_inputs(self, desired, rendered: RenderedModels): for name, gid in sorted(rendered.models.items()) for endpoint in sorted(by_id[gid].served) ] - return {}, routes - rows = self.catalog_route_rows(self.catalog) - rows.update({ - alias: {'engine': UPSTREAM_ROUTE, 'served': name, 'api_base': base} - for alias, name in rendered.request_names.items() - }) - return rows, [] + return [], routes + catalog_routes(self.catalog, dynamic=True) + catalog = self.catalog_routes(self.catalog) + defined = {r.alias for r in catalog} + # A Model serving a catalog alias is that endpoint's route; one serving + # an alias no catalog defines is the cluster's own (the gateway + # remembers those past release, as it does ad-hoc deployments). + models = [GatewayRoute(alias, 'openai', name, base, + origin='catalog' if alias in defined else 'upstream') + for alias, name in rendered.request_names.items()] + return route_table(catalog, models), [] def _set_front_door(self, desired, rendered: RenderedModels) -> None: """Hand the gateway its inputs for the next render or preview.""" - rows, routes = self._front_door_inputs(desired, rendered) - self.gateway.upstream_rows = rows - self.gateway.upstream_routes = routes + static, dynamic = self._front_door_inputs(desired, rendered) + self.gateway.extra_routes = static + self.gateway.extra_dynamic_routes = dynamic # -- converge-style surface ------------------------------------------------ diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index 5a0d0d13..5e0ab3be 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -29,10 +29,12 @@ from ..config import PINNED_IMAGES from ..leasing.backend import ConvergeScaffold +from ..leasing.routes import GatewayRoute, route_table from ..leasing.gateway import ( API_KEY_ENV, LITELLM_CONTAINER_PORT, Gateway, + remembered_rows, render_front_door, ) @@ -83,8 +85,8 @@ def __init__( base_url=self.base_url) self.catalog = None #: Render inputs from the kubeai backend (see ComposeBackend's). - self.upstream_rows: dict[str, dict[str, Any]] = {} - self.upstream_routes: list[dict[str, Any]] = [] + self.extra_routes: list[GatewayRoute] = [] + self.extra_dynamic_routes: list[GatewayRoute] = [] self._node_address: str | None = None # -- where it is --------------------------------------------------------- @@ -154,24 +156,24 @@ def gateway_accepts(self, key: str, *, wait: float = 0.0): def access(self, endpoints: list[str]) -> dict[str, Any] | None: return self.gateway.access(endpoints) - def merge_route_registry(self, incoming): - return self.gateway.merge_route_registry(incoming) + def catalog_routes(self, catalog) -> list[GatewayRoute]: + return [] # the kubeai backend supplies its own routes - def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: - return {} # the kubeai backend supplies its own rows + def static_routes(self) -> list[GatewayRoute]: + """The static route table: the registry under the backend's routes.""" + return route_table(self.gateway.registry_routes(), self.extra_routes) # -- render / apply ----------------------------------------------------- - def _render_documents(self) -> tuple[dict[Path, str], dict[str, Any]]: - """``(planned files, merged registry)`` in memory: the one render.""" - registry = self.gateway.merged_route_registry(dict(self.upstream_rows)) + def _render_documents(self) -> dict[Path, str]: + """The planned files, in memory: the one render.""" front = render_front_door( - [], {}, engine_services=[], vllm_v1_urls=[], ollama_native_urls=[], + self.static_routes(), vllm_v1_urls=[], ollama_native_urls=[], images=self.images, state={}, litellm=True, litellm_port=LITELLM_CONTAINER_PORT, litellm_master_key=None, litellm_salt_key=False, ui=False, ui_port=0, reverse_proxy=False, reverse_proxy_port=0, reverse_proxy_config=None, aux_dir=self.state_dir, - catalog=None, route_registry=registry, dynamic_routing=False, + dynamic_routing=False, ) config = front.litellm_config or '' key_hash = hashlib.sha256(self.master_key().encode()).hexdigest()[:12] @@ -179,22 +181,24 @@ def _render_documents(self) -> tuple[dict[Path, str], dict[str, Any]]: namespace=self.namespace, image=self.images['litellm'], config=config, key_hash=key_hash, node_port=self.node_port) text = '---\n'.join(yaml.safe_dump(d, sort_keys=False) for d in docs) - return {self.manifests_file: text}, registry + return {self.manifests_file: text} def preview(self, desired=(), placement=None, *, approve: bool = False): """Render without writing; with ``approve``, show the diff now.""" with self.gateway.staging_secrets(): - planned, _ = self._render_documents() + planned = self._render_documents() self._preview_approval(planned, approve=approve) return None, None def converge(self, desired=(), *, apply: bool = True, placement=None): - """Render the gateway's objects (and persist the registry after approval).""" + """Render the gateway's objects, and write them after approval.""" with self._converge_lock(): - planned, registry = self._render_documents() + planned = self._render_documents() self.last_planned_digest = self._planned_digest(planned) self._approve_changes(planned) - self.gateway._save_route_registry(registry) + self.gateway.remember(remembered_rows( + [], {}, defined=set(getattr(self.catalog, 'endpoints', None) or {}), + extra=self.extra_routes)) for path, text in planned.items(): self._atomic_write(path, text) if apply: diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index b0e87a01..4af0aef6 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2571,13 +2571,13 @@ def main(cls, argv=True, **kwargs): def _routes_or_exit(call): - """Run a route-registry operation; its refusals become CLI errors.""" + """Run a routes operation; its refusals become CLI errors.""" from ..leasing.profile import ProfileMismatch from ..leasing.routes import RouteConflict try: return call() - except (ProfileMismatch, RouteConflict) as ex: + except (ProfileMismatch, RouteConflict, CatalogError) as ex: raise SystemExit(f'routes: {ex}') @@ -2592,15 +2592,33 @@ def _live_endpoints(controller) -> set[str]: } +def _route_engine(route) -> str: + """The ``engine`` column ``routes list --json`` has always had: ``vllm`` + or ``ollama`` for an engine this project runs, ``upstream`` for a + server it does not (a KubeAI cluster), ``external`` for an external + endpoint.""" + from ..leasing.naming import VLLM_CONTAINER_PORT, vllm_service_name_for + + if route.kind == 'ollama': + return 'ollama' + if route.origin in ('external', 'upstream'): + return route.origin + ours = f'http://{vllm_service_name_for(route.model)}:{VLLM_CONTAINER_PORT}/v1' + return 'vllm' if route.api_base == ours or route.origin == 'deployment' else 'upstream' + + class RoutesListCLI(_LeasingCommonMixin): - """Print the accumulated LiteLLM route registry (static-superset mode). - - One row per persisted route: its alias, engine, served-model/tag, the - upstream it routes to (a compose service, or a KubeAI cluster's gateway - under the Model's name), and whether a live deployment is - currently backing it. Routes with no live backer still list (that is the - point — a released endpoint stays routable/testable); their upstream simply - errors until something serves it. + """Print the gateway's routes, and where each comes from. + + One row per alias: where it routes (a compose service, a KubeAI cluster's + gateway under the Model's name, or an external server), its origin -- + ``catalog`` or ``external`` (a published catalog endpoint), + ``deployment`` (a placed deployment no published catalog defines), + ``upstream`` (KubeAI) or ``legacy`` (a row of a route registry from + before routes were derived; ``routes prune`` drops those) -- and whether + a live deployment backs it. A catalog route with no live backer still + lists: a released endpoint stays routable, and its upstream errors until + something serves it. With dynamic routing, one row per managed route. """ __command__ = 'list' @@ -2609,65 +2627,50 @@ class RoutesListCLI(_LeasingCommonMixin): @classmethod def main(cls, argv=True, **kwargs): - from ..leasing.gateway import registry_route_entry - config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) - registry = _routes_or_exit(controller.route_registry) - entries = registry.get('entries', {}) + routes = _routes_or_exit(controller.route_view) live = _live_endpoints(controller) - rows = [] - for name in sorted(entries): - row = entries[name] - engine = row.get('engine') - entry = registry_route_entry(name, row) # what the gateway renders - if entry is None: - target, upstream = '?', '?' - else: - params = entry['litellm_params'] - target = params['model'].split('/', 1)[-1] - upstream = params['api_base'] - rows.append({ - 'name': name, - 'engine': engine, - 'target': target, - 'upstream': upstream, - 'live': name in live, - }) + rows = [{ + 'name': r.alias, + 'engine': _route_engine(r), + 'origin': r.origin, + 'target': r.model, + 'upstream': r.api_base, + 'live': r.alias in live, + } for r in routes] if config.json: - print(json.dumps( - {'version': registry.get('version'), 'routes': rows}, indent=2 - )) + print(json.dumps({'routes': rows}, indent=2)) return 0 if not rows: - print('route registry is empty (no converge has run yet)') + print('no routes (nothing is published or placed yet)') return 0 - print(f'{len(rows)} route(s) in the registry:') + print(f'{len(rows)} route(s):') for r in rows: flag = 'live' if r['live'] else ' -' print( - f' [{flag}] {r["name"]:<28} {r["engine"]:<7} ' + f' [{flag}] {r["name"]:<28} {r["origin"]:<10} ' f'{r["target"]:<24} -> {r["upstream"]}' ) return 0 class RoutesPruneCLI(_ApprovalMixin): - """Forget stale routes: keep only the invoking catalog's and the live ones. + """Unpublish stale endpoints: keep only the invoking catalog's and the live ones. - Rewrites the registry to *invoking catalog ∪ live*, then converges (one - accepted gateway recreate). + Drops every published catalog endpoint (and legacy registry row) that the + invoking catalog does not define and no deployment serves, then + publishes: their routes go on the next render. - The registry is append-only by design (that is what keeps the gateway config - byte-stable), so pruning is the explicit, operator-driven "forget" verb — - automatic pruning is deliberately excluded because any catalog-keyed rule - reintroduces the cross-catalog alternation churn this whole mechanism exists - to avoid. + Published endpoints outlive the runbook that published them (that is what + keeps another runbook's routes, and an external endpoint with no lease, + in place), so this is the explicit, operator-driven removal; nothing + prunes automatically. - A prune run from the WRONG ``INFER_STACK_CONFIG_DIR`` would silently drop - every other runbook's routes, so the exact drop list is shown and confirmed + A prune run from the WRONG ``INFER_STACK_CONFIG_DIR`` would drop every + other runbook's endpoints, so the exact drop list is shown and confirmed first (``--yes`` / a non-terminal skips the prompt). """ @@ -2687,20 +2690,20 @@ def main(cls, argv=True, **kwargs): # rechecks under the lock and drops only what is still unneeded. plan = _routes_or_exit(controller.plan_route_prune) if not plan.dropped: - print('routes prune: nothing to drop (registry already minimal)') + print('routes prune: nothing to drop') return 0 skip_prompt = bool(config.yes) or not sys.stdout.isatty() if not skip_prompt: - print('routes prune will DROP these routes:') + print('routes prune will UNPUBLISH these endpoints:') for name in plan.dropped: print(f' - {name}') if input('drop them? [y/N] ').strip().lower() not in ('y', 'yes'): - raise SystemExit('aborted: registry not pruned') + raise SystemExit('aborted: nothing unpublished') try: dropped, rec = _routes_or_exit(lambda: controller.commit_route_prune(plan)) except ConvergeAborted: raise SystemExit('aborted: compose changes not applied') - kept = sorted(_routes_or_exit(controller.route_registry).get('entries', {})) + kept = sorted(r.alias for r in _routes_or_exit(controller.route_view)) if config.json: print(json.dumps({'dropped': dropped, 'kept': kept, 'publication_pending': rec.publication_pending}, indent=2)) @@ -2711,34 +2714,32 @@ def main(cls, argv=True, **kwargs): class RoutesSeedCLI(_ApprovalMixin): - """Merge extra catalog files' routes into the registry, then converge. - - The operational key to blip-free concurrency: seed *all* the overlapping - runbooks' catalogs once while the stack is idle, and then no converge from - any of them ever recreates the gateway (the registry is already the full - union). Works before ``stack up`` too — the follow-up reconcile brings the - standing gateway up with the complete route table, which is exactly what - pre-seeding is for. - - Unlike a normal converge (which only ever merges the *invoking* process's own - catalog), this folds in sibling catalogs you name explicitly. - - Adding an alias is additive; redefining one already in the registry - redirects its clients, so that is refused and nothing is written unless - ``--replace`` is given (which shows each change and asks on a terminal). + """Publish other catalog files' endpoints, so the gateway routes them. + + Merges the named catalogs into the published catalog union (the one + store of endpoint definitions, shared by every runbook on this stack), + then publishes. The operational key to blip-free concurrency: seed *all* + the overlapping runbooks' catalogs once while the stack is idle, and no + converge from any of them recreates the gateway afterwards. Works before + ``stack up`` too. + + Adding an alias is additive; redefining one already published (or in a + legacy registry) redirects its clients, so that is refused and nothing is + written unless ``--replace`` is given (which shows each change and asks + on a terminal). An endpoint a resident deployment runs is never + redefined. """ __command__ = 'seed' replace = kw.Value( False, isflag=True, - help='Redefine aliases the registry already routes elsewhere.', + help='Redefine aliases already published with another meaning.', ) catalogs = kw.Value( None, nargs='+', position=1, type=str, - help='One or more catalog.yaml files whose endpoints to merge into the ' - 'route registry.', + help='One or more catalog.yaml files whose endpoints to publish.', ) json = kw.Value(False, isflag=True) @@ -2765,7 +2766,7 @@ def main(cls, argv=True, **kwargs): plan = _routes_or_exit(lambda: controller.plan_route_seed(catalogs)) if not plan.incoming: raise SystemExit( - 'routes seed: the named catalog(s) resolved no routable endpoints' + 'routes seed: the named catalog(s) define no endpoints' ) if plan.conflicted: # stderr under --json, which must stay one JSON document on stdout. @@ -2775,11 +2776,11 @@ def main(cls, argv=True, **kwargs): for name, (old, new) in sorted(plan.conflicted.items()): print(f' ~ {name}: {old} -> {new}', file=out) if not config.replace: - raise SystemExit('routes seed: refused, the registry is unchanged ' + raise SystemExit('routes seed: refused, nothing was published ' '(pass --replace to redefine them)') if not (bool(config.yes) or not sys.stdout.isatty()): if input('redefine them? [y/N] ').strip().lower() not in ('y', 'yes'): - raise SystemExit('aborted: registry unchanged') + raise SystemExit('aborted: nothing published') try: done, rec = _routes_or_exit( lambda: controller.commit_route_seed(plan, replace=bool(config.replace))) @@ -3043,12 +3044,12 @@ class NetworkModalCLI(kw.ModalCLI): class RoutesModalCLI(kw.ModalCLI): - """Inspect + manage the LiteLLM route registry (static-superset mode). + """Inspect the LiteLLM gateway's routes; publish and unpublish endpoints. - The registry accumulates every catalog's and every live deployment's routes - across the runbooks that share one stack, so a cross-catalog converge cannot - strip another's live routes and the gateway config stays byte-stable. See - docs/litellm-gateway-routing.md. + Routes are derived from the published catalog union (every runbook's + endpoints on this stack) and the placed deployments, so a cross-catalog + converge cannot strip another's routes and the gateway config stays + byte-stable. See docs/litellm-gateway-routing.md. """ __command__ = 'routes' diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 4d592136..9094669a 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -22,6 +22,7 @@ from typing import Any, Callable, Protocol, runtime_checkable from .models import Deployment +from .routes import GatewayRoute @dataclass @@ -335,17 +336,17 @@ def placement_notes(self) -> dict[str, list[str]]: ... def front_door(self) -> FrontDoor | None: - """What holds the LiteLLM gateway (its keys and route registry), or + """What holds the LiteLLM gateway (its keys and routes), or ``None`` for a backend without one.""" ... - def route_rows(self, desired: list[Deployment], placement: Any = None - ) -> dict[str, dict[str, Any]]: - """The gateway route rows a render of ``desired`` merges in.""" + def routes(self, desired: list[Deployment], placement: Any = None + ) -> list[GatewayRoute]: + """The gateway routes a render of ``desired`` produces.""" ... - def catalog_route_rows(self, catalog: Any) -> dict[str, dict[str, Any]]: - """The route rows every endpoint of ``catalog`` would get.""" + def catalog_routes(self, catalog: Any) -> list[GatewayRoute]: + """The routes every endpoint of ``catalog`` would get here.""" ... def reclaim_candidates(self, blocked: Deployment, @@ -570,12 +571,12 @@ def reclaim_candidates(self, blocked: Deployment, def front_door(self) -> FrontDoor | None: return None # no gateway in-process - def route_rows(self, desired: list[Deployment], placement: Any = None - ) -> dict[str, dict[str, Any]]: - return {} + def routes(self, desired: list[Deployment], placement: Any = None + ) -> list[GatewayRoute]: + return [] - def catalog_route_rows(self, catalog: Any) -> dict[str, dict[str, Any]]: - return {} + def catalog_routes(self, catalog: Any) -> list[GatewayRoute]: + return [] def plan(self, desired: list[Deployment], placement: Any = None): """Which of ``desired`` fit; here, all of them, on no GPU.""" diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index a93a04ed..7288bb1a 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -15,17 +15,15 @@ That is what makes the endpoint descriptor's ``base_url`` correct (the backend supplies it via :meth:`ComposeBackend.access`). -In static-superset mode the gateway's ``model_list`` is rendered from an -**append-only route registry** (``litellm_registry.json`` in the shared state -dir): every converge merges the invoking catalog plus every live deployment -(across all runbooks sharing the stack) into the registry and renders from the -whole thing. That makes the render a function of accumulated shared state — not -of which runbook invoked the converge — so a cross-catalog converge can no -longer strip another's live routes and, once every catalog has merged once, the -config is byte-stable (the gateway is never recreated). See -:meth:`ComposeBackend._update_route_registry` and -:func:`_litellm_model_list_from_registry`; ``infer-stack routes`` inspects/seeds/ -prunes it; ``docs/litellm-gateway-routing.md`` has the full story. +The gateway's routes are derived at every render, never stored: one per +endpoint of the published catalog union (every runbook's catalogs, kept in +the recovery profile) and one per placed deployment +(:func:`~infer_stack.leasing.gateway.front_door_routes`). The render is then a +function of shared state, not of which runbook invoked it, so a cross-catalog +converge cannot strip another's routes and the config is byte-stable as +models come and go (the gateway is not recreated). ``infer-stack routes`` +lists, seeds and prunes them; ``docs/litellm-gateway-routing.md`` has the +full story. Docker and HTTP are invoked through injected seams (``run`` / ``http_get``), so all logic here is unit-testable without docker or a network. The real @@ -52,6 +50,7 @@ from ..probe import openai_ready from ..profile_runtime import simulator_args, vllm_args from .backend import ApplyResult, ConvergeScaffold, Readiness +from .routes import GatewayRoute from .gateway import ( API_KEY_ENV, LITELLM_CONFIG_FILENAME, @@ -61,8 +60,9 @@ ROUTE_RECONCILE_STEADY_S, SALT_KEY_ENV, Gateway, - _registry_incoming_from_catalog, - _registry_incoming_from_deployments, + catalog_routes, + front_door_routes, + remembered_rows, render_front_door, ) from .launch import env_string, fill, translate_legacy @@ -609,9 +609,9 @@ def render_compose( aux_dir: str | Path | None = None, project: str = LEASING_PROJECT, catalog: Any = None, - route_registry: dict[str, Any] | None = None, dynamic_routing: bool = False, - upstream_routes: list[dict[str, Any]] | None = None, + routes: list[GatewayRoute] | None = None, + dynamic_routes: list[GatewayRoute] | None = None, ui_run_as: str | None = None, ) -> RenderedCompose: """Render a compose project for the placed deployments. @@ -621,6 +621,10 @@ def render_compose( alias is reachable at one ``base_url``. When ``ui`` is also set, a managed Open WebUI is rendered in front of that gateway. + The gateway's ``routes`` (static) or ``dynamic_routes`` are the caller's + (:func:`~infer_stack.leasing.gateway.front_door_routes`); left out, they + are derived from ``catalog`` and the placed deployments alone. + The project name is baked into the file as a top-level ``name:`` so a plain ``docker compose -f docker-compose.yml up`` (infer-stack not involved) lands in the *same* project — same container names, same network — as @@ -689,9 +693,11 @@ def render_compose( ollama_native_urls.append(f'http://{name}:{OLLAMA_CONTAINER_PORT}') service_map[name] = deployment.id + if routes is None and dynamic_routes is None: + routes, dynamic_routes = front_door_routes( + deployments, assignments, catalog=catalog, dynamic=dynamic_routing) front = render_front_door( - deployments, assignments, - engine_services=list(service_map), + routes or [], vllm_v1_urls=vllm_v1_urls, ollama_native_urls=ollama_native_urls, images=images, state=state, litellm=litellm, litellm_port=litellm_port, @@ -699,8 +705,7 @@ def render_compose( ui=ui, ui_port=ui_port, reverse_proxy=reverse_proxy, reverse_proxy_port=reverse_proxy_port, reverse_proxy_config=reverse_proxy_config, aux_dir=aux_dir, - catalog=catalog, route_registry=route_registry, - dynamic_routing=dynamic_routing, upstream_routes=upstream_routes, + dynamic_routing=dynamic_routing, dynamic_routes=dynamic_routes, ui_run_as=ui_run_as, ) services.update(front.services) @@ -1184,7 +1189,7 @@ def compose_project(self): return self def front_door(self): - """What holds the gateway's keys and route registry: this project.""" + """What holds the gateway's keys and routes: this project.""" return self def compose_argv(self) -> list[str]: @@ -1296,9 +1301,6 @@ def restore_env(self, values: dict[str, str | None]) -> None: def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: return self.gateway.gateway_accepts(key, wait=wait) - def merge_route_registry(self, incoming: dict[str, dict[str, Any]]) -> dict[str, Any]: - return self.gateway.merge_route_registry(incoming) - def access(self, endpoints: list[str]) -> dict[str, Any] | None: """Where a client reaches these endpoints: the front door.""" return self.gateway.access(endpoints) @@ -1513,13 +1515,7 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A self._ui_root_noted = True logger.info('Open WebUI runs as root: {}', why) - route_registry = None - if self.litellm and not self.dynamic_routing: - # Unconditional in static-superset mode: `self.catalog` may be None; - # the incoming set is then deployments-only, and the render still - # comes from the accumulated registry, so a catalog-less converge - # cannot strip routes or blip. - route_registry = self._merged_route_registry(desired, plan.assignments) + routes, dynamic_routes = self._front_door_routes(desired, plan.assignments) rendered = render_compose( desired, plan.assignments, images=self.images, ports=self.ports, state=self.state, litellm=self.litellm, litellm_port=self.litellm_port, @@ -1529,8 +1525,8 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A reverse_proxy_port=self.reverse_proxy_port, reverse_proxy_config=self.reverse_proxy_config, aux_dir=self.state_dir, project=self.project, catalog=self.catalog, - route_registry=route_registry, dynamic_routing=self.dynamic_routing, - upstream_routes=self.upstream_routes, ui_run_as=ui_run_as, + dynamic_routing=self.dynamic_routing, routes=routes, + dynamic_routes=dynamic_routes, ui_run_as=ui_run_as, ) addresses = None if self.network is not None: @@ -1557,8 +1553,8 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A ) planned[self.compose_file] = yaml.safe_dump(rendered.compose, sort_keys=False) return {'plan': plan, 'rendered': rendered, 'planned': planned, - 'fingerprints': fingerprints, 'route_registry': route_registry, - 'addresses': addresses} + 'fingerprints': fingerprints, 'addresses': addresses, + 'remember': self._remembered_rows(desired, plan.assignments)} #: Stable addressing (plan step P7), set by the controller once #: `network migrate` has run: ``{'subnet': ..., 'addresses': {service: ip}}``. @@ -1719,49 +1715,49 @@ def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: except Exception: return '' - def _merged_route_registry( + def _front_door_routes( self, desired: list[Deployment], assignments: dict[str, list[int]] - ) -> dict[str, Any]: - """The route registry merged with this backend's rows, in memory. - - The rows are this backend's to supply: every catalog endpoint, and every - placed deployment (``desired`` spans all runbooks via the shared ledger, - so a live cross-runbook deployment stays routable, and past release). - """ - incoming = self.catalog_route_rows(self.catalog) - incoming.update(_registry_incoming_from_deployments(desired, assignments)) - incoming.update(self.upstream_rows) - return self.gateway.merged_route_registry(incoming) - - #: Render inputs from the owner of engines this project does not run (the - #: kubeai backend, whose gateway this is): static route-registry rows and - #: dynamic routes, both pointing at its servers. Set before each render. - upstream_rows: dict[str, dict[str, Any]] = {} - upstream_routes: list[dict[str, Any]] = [] + ) -> tuple[list[GatewayRoute], list[GatewayRoute]]: + """``(static, dynamic)`` routes of a render: the published catalog + union's endpoints and every placed deployment (``desired`` spans all + runbooks via the shared ledger), over the route registry, under what + the owner of engines elsewhere supplies.""" + if not self.litellm: + return [], [] + return front_door_routes( + desired, assignments, catalog=self.catalog, dynamic=self.dynamic_routing, + registry=[] if self.dynamic_routing else self.gateway.registry_routes(), + extra=self.extra_routes, extra_dynamic=self.extra_dynamic_routes) + + def _remembered_rows(self, desired, assignments) -> dict[str, dict[str, Any]]: + """The registry rows an approved static render adds: its routed + deployments' (and the owner's Models') aliases no published catalog + defines.""" + if not self.litellm or self.dynamic_routing: + return {} + defined = set(getattr(self.catalog, 'endpoints', None) or {}) + return remembered_rows(desired, assignments, defined=defined, + extra=self.extra_routes) + + #: Routes from the owner of engines this project does not run (the kubeai + #: backend, whose gateway this is): static and dynamic, both pointing at + #: its servers. Set before each render. + extra_routes: list[GatewayRoute] = [] + extra_dynamic_routes: list[GatewayRoute] = [] #: Set by an owner whose engines run elsewhere (kubeai): this project is #: only the front door. fronts_elsewhere = False - def catalog_route_rows(self, catalog) -> dict[str, dict[str, Any]]: - """Route-registry rows for every endpoint of ``catalog``.""" - return {} if catalog is None else _registry_incoming_from_catalog(catalog) - - def route_rows(self, desired: list[Deployment], placement=None) -> dict[str, dict[str, Any]]: - """The rows a render of ``desired`` merges: catalog, placed deployments, - and the owner's upstream rows (``routes prune`` keeps exactly these).""" - assignments = self.plan(desired, placement).assignments - rows = self.catalog_route_rows(self.catalog) - rows.update(_registry_incoming_from_deployments(desired, assignments)) - rows.update(self.upstream_rows) - return rows + def catalog_routes(self, catalog) -> list[GatewayRoute]: + """The routes every endpoint of ``catalog`` gets here.""" + return catalog_routes(catalog) - def _update_route_registry( - self, desired: list[Deployment], assignments: dict[str, list[int]] - ) -> dict[str, Any]: - """Merge and persist the route registry; return it (kept for callers).""" - merged = self._merged_route_registry(desired, assignments) - self.gateway._save_route_registry(merged) - return merged + def routes(self, desired: list[Deployment], placement=None) -> list[GatewayRoute]: + """The routes a render of ``desired`` gives the gateway (``routes list``): + the static table, or the dynamic routes.""" + static, dynamic = self._front_door_routes( + desired, self.plan(desired, placement).assignments) + return dynamic if self.dynamic_routing else static def converge(self, desired: list[Deployment], *, apply: bool = True, placement=None): """Place + render the desired union, then optionally apply it. @@ -1815,12 +1811,11 @@ def converge(self, desired: list[Deployment], *, apply: bool = True, placement=N self.last_planned_digest = self._planned_digest(planned) self._approve_changes(planned) # may raise ConvergeAborted - # Only after approval: persist new addresses and the merged route - # registry, then the files. + # Only after approval: persist new addresses and remembered + # routes, then the files. if docs['addresses'] is not None and self.on_addresses is not None: self.on_addresses(docs['addresses']) - if docs['route_registry'] is not None: - self.gateway._save_route_registry(docs['route_registry']) + self.gateway.remember(docs['remember']) for path, text in planned.items(): if path != self.compose_file: self._atomic_write(path, text) diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index ec311b84..e45e2bba 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1828,77 +1828,181 @@ def _route_gateway(self): 'kubeai backend, with `litellm` on)') return front.gateway - def route_registry(self) -> dict: - """The route registry as stored (``routes list``).""" - return self._route_gateway().route_registry() + def route_view(self) -> list: + """The routes the gateway serves now (``routes list``): derived from the + published catalogs, the placed deployments and any legacy registry, + exactly as the next render derives them.""" + from .residency import ResidencyUnknown + + self._route_gateway() + try: + residency = self.backend.residency() + except ResidencyUnknown: + residency = None + desired, inputs = self._admission_view(residency, virtual_expiry=True) + return self.backend.routes(desired, inputs) + + def _published_sources(self) -> list[dict]: + """The published catalog union's sources (the stored profile's, else + the ones this invocation would freeze).""" + profile = self.ledger.profile() or self.invocation_profile() or {} + return list(profile.get('catalogs') or []) + + def _meanings(self, union) -> dict: + """``{alias: Meaning}`` for every endpoint of ``union``.""" + from .catalog import CatalogError + from .routes import Meaning + + if union is None: + return {} + routes = {r.alias: r for r in self.backend.catalog_routes(union)} + meanings = {} + for name in sorted(union.endpoints): + try: + key = union.resolve_endpoint(name).semantic_key() + except CatalogError: + continue + meanings[name] = Meaning(routes.get(name), key) + return meanings + + def _published_meanings(self) -> dict: + """What every published alias means: route registry rows, then the + published union's definitions over them.""" + from .profile import CatalogUnion + from .routes import Meaning + + meanings = {r.alias: Meaning(r) for r in self._route_gateway().registry_routes()} + sources = self._published_sources() + meanings.update(self._meanings(CatalogUnion.from_sources(sources) if sources else None)) + return meanings + + def _store_catalogs(self, catalogs: list[dict]) -> None: + """Replace the published union (caller holds the lock, marker set).""" + stored = self.ledger.profile() or self.invocation_profile() + if stored is None: + return + profile = {**stored, 'catalogs': catalogs} + if profile != stored: + self._commit_profile_candidate(profile) + self._use_profile_candidate(profile) def plan_route_seed(self, catalogs: Iterable) -> RoutePlan: - """What seeding these catalogs' routes would add, keep, or redefine.""" + """What publishing these catalogs would add, keep, or redefine. + Raises :class:`~infer_stack.leasing.profile.CatalogConflict` if they + disagree among themselves.""" + from .profile import CatalogUnion, catalog_sources from .routes import plan_seed - gateway = self._route_gateway() - incoming: dict[str, dict] = {} - for catalog in catalogs: - incoming.update(self.backend.catalog_route_rows(catalog)) - return plan_seed(gateway.route_entries(), incoming) + self._route_gateway() + sources = [s for c in catalogs for s in catalog_sources(c)] + incoming = self._meanings(CatalogUnion.from_sources(sources) if sources else None) + plan = plan_seed(self._published_meanings(), incoming) + plan.sources = sources + return plan def commit_route_seed(self, plan: RoutePlan, *, replace: bool = False ) -> tuple[RoutePlan, ReconcileResult]: - """Add the plan's routes and publish; redefine conflicts only with - ``replace``. Rechecked under the lock: a conflict that appeared since - the plan refuses too (:class:`~infer_stack.leasing.routes.RouteConflict`) - and nothing is written.""" + """Merge the plan's catalogs into the published union and publish; + redefine conflicts only with ``replace``, and never one a resident + workload runs. Rechecked under the lock: a conflict that appeared + since the plan refuses too + (:class:`~infer_stack.leasing.routes.RouteConflict`) and nothing is + written.""" + from .profile import CatalogConflict, ProfileMismatch, adopt_catalog_sources + from .residency import ResidencyUnknown from .routes import RouteConflict, plan_seed gateway = self._route_gateway() - fresh = plan_seed(gateway.route_entries(), plan.incoming) + fresh = plan_seed(self._published_meanings(), plan.incoming) if fresh.conflicted and not replace: raise RouteConflict(sorted(fresh.conflicted)) def preflight(): # Under the lock, before the marker: a conflict another process # made since the plan refuses without leaving a publication pending. - now = plan_seed(gateway.route_entries(), plan.incoming) + now = plan_seed(self._published_meanings(), plan.incoming) if now.conflicted and not replace: raise RouteConflict(sorted(now.conflicted)) def change(): - now = plan_seed(gateway.route_entries(), plan.incoming) - entries = gateway.route_entries() - entries.update(now.added) - if replace: - entries.update({name: new for name, (_, new) in now.conflicted.items()}) - if entries != gateway.route_entries(): - gateway.replace_route_entries(entries) + now = plan_seed(self._published_meanings(), plan.incoming) + try: + residency = self.backend.residency() + except ResidencyUnknown: + residency = None + pinned = self._pinned_endpoints(residency) + try: + catalogs = adopt_catalog_sources( + self._published_sources(), plan.sources, pinned) + except CatalogConflict as ex: + blocked = sorted(set(ex.names) & pinned) or sorted(ex.names) + raise ProfileMismatch( + f'routes seed would redefine {", ".join(map(repr, blocked))}, ' + 'which a resident deployment is running; evict it first ' + f'(`infer-stack evict {blocked[0]}`)') from ex + self._store_catalogs(catalogs) + # A published definition supersedes a registry row of the same name. + legacy = gateway.route_entries() + kept = {k: v for k, v in legacy.items() if k not in plan.incoming} + if kept != legacy: + gateway.replace_route_entries(kept) return now return self.publish_change(change, preflight=preflight) - def plan_route_prune(self) -> RoutePlan: - """Which routes a prune drops: all but the catalog's and the live ones, - exactly as the next render would keep them.""" + def _prune_keep(self) -> set[str]: + """Aliases a prune keeps: the invocation's catalogs', and every one a + deployment the next render places (or a resident one) serves.""" + from .profile import CatalogUnion + from .residency import ResidencyUnknown + + mine = (self.invocation_profile() or {}).get('catalogs') or [] + keep = set(CatalogUnion.from_sources(mine).endpoints) if mine else set() + try: + residency = self.backend.residency() + except ResidencyUnknown: + residency = None + desired, _ = self._admission_view(residency, virtual_expiry=True) + keep.update(ep for g in desired for ep in g.served) + return keep | self._pinned_endpoints(residency) + + def _prunable(self) -> list[str]: + from .profile import CatalogUnion from .routes import plan_prune - gateway = self._route_gateway() - desired, inputs = self._admission_view(self.backend.residency()) - return plan_prune(gateway.route_entries(), self.backend.route_rows(desired, inputs)) + sources = self._published_sources() + current = set(self._route_gateway().route_entries()) + if sources: + current |= set(CatalogUnion.from_sources(sources).endpoints) + return plan_prune(current, self._prune_keep()).dropped + + def plan_route_prune(self) -> RoutePlan: + """Which aliases a prune unpublishes: every published definition and + registry row but the invocation's catalogs' and the ones deployments + serve.""" + return RoutePlan(dropped=self._prunable()) def commit_route_prune(self, plan: RoutePlan) -> tuple[list[str], ReconcileResult]: - """Drop the plan's routes that are still unneeded, and publish. A route - that became needed since the plan is kept.""" - from .routes import plan_prune + """Unpublish the plan's aliases that are still unneeded, and publish. + One that became needed since the plan is kept.""" + from .profile import drop_catalog_names gateway = self._route_gateway() confirmed = set(plan.dropped) def change(): - desired, inputs = self._admission_view(self.backend.residency()) - current = gateway.route_entries() - still = plan_prune(current, self.backend.route_rows(desired, inputs)).dropped - drop = sorted(confirmed & set(still)) + drop = sorted(confirmed & set(self._prunable())) if drop: - gateway.replace_route_entries( - {k: v for k, v in current.items() if k not in drop}) + legacy = gateway.route_entries() + kept = {k: v for k, v in legacy.items() if k not in drop} + if kept != legacy: + gateway.replace_route_entries(kept) + sources = self._published_sources() + # A bundle naming an unpublished endpoint goes with it. + bundles = {b for src in sources + for b, members in (src.get('bundles') or {}).items() + if set(members or []) & set(drop)} + self._store_catalogs(drop_catalog_names(sources, set(drop) | bundles)) return drop return self.publish_change(change) @@ -1908,8 +2012,9 @@ def publish_change(self, change: Callable[[], _T], *, ) -> tuple[_T, ReconcileResult]: """Run ``change`` and publish, as one serialised desired-state mutation. - For changes to backend state that the render reads (the route registry, - via ``routes seed`` / ``routes prune``). ``change`` runs under the lock + For changes to state the render reads outside the ledger (the published + catalogs and the legacy route registry, via ``routes seed`` / ``routes + prune``). ``change`` runs under the lock after the marker is set, so a crash leaves it pending like any other mutation. Returns ``(change's result, reconcile result)``. diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index a1eebdc3..5f9b2dff 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -2,10 +2,10 @@ proxy beside it. Engines (vLLM, Ollama) are the backend's business; everything a client -talks to is here. A backend hands the gateway where its models are (route -rows), and the gateway renders its Compose services and config, keeps the -route registry, reconciles dynamic routes through LiteLLM's admin API, and -manages the master key. The compose backend runs it beside its engines; the +talks to is here. A backend hands the gateway where its models are +(:class:`~infer_stack.leasing.routes.GatewayRoute` s), and the gateway renders +its Compose services and config, reconciles dynamic routes through LiteLLM's +admin API, and manages the master key. The compose backend runs it beside its engines; the kubeai backend runs it alone, in front of a cluster. """ @@ -16,7 +16,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence import yaml @@ -24,6 +24,7 @@ from ..config import DEFAULT_PORTS from ..env_utils import ensure_secret, parse_env_file, write_env_file from .backend import ConvergeScaffold +from .endpoints import ExternalTarget from .models import Deployment, served_name from .naming import ( OLLAMA_CONTAINER_PORT, @@ -34,6 +35,7 @@ vllm_service_name_for, ) from .residency import ENGINE_LABEL +from .routes import GatewayRoute, route_table LITELLM_CONTAINER_PORT = 4000 LITELLM_CONFIG_FILENAME = 'litellm_config.yaml' @@ -50,19 +52,10 @@ # model store, instead of a static config file. See render_compose + # ComposeBackend._reconcile_routes and docs/litellm-gateway-routing.md. LITELLM_ROUTES_FILENAME = 'litellm_routes.json' # rendered desired route set -# Append-only route registry for static-superset mode: accumulates the semantic -# route inputs (served name / engine / host) of every catalog *and* every live -# deployment ever merged, across all runbooks sharing this state dir. The gateway -# `model_list` is rendered from the whole registry, so a converge under one -# runbook's catalog can no longer strip another's still-live routes, and once -# every catalog has been merged once the rendered config is byte-stable (the -# gateway is never recreated). See docs/litellm-gateway-routing.md and -# ComposeBackend._update_route_registry. +# The route registry: routes of deployments no published catalog defines, +# kept past release (see remembered_rows and docs/litellm-gateway-routing.md). LITELLM_REGISTRY_FILENAME = 'litellm_registry.json' LITELLM_REGISTRY_VERSION = 1 -# A route-registry row for a server this project does not run: -# ``{'engine': UPSTREAM_ROUTE, 'served': , 'api_base': }``. -UPSTREAM_ROUTE = 'upstream' POSTGRES_SERVICE = 'postgres-litellm' POSTGRES_CONTAINER_PORT = 5432 POSTGRES_DB_NAME = 'litellm' @@ -84,363 +77,227 @@ ROUTE_RECONCILE_STEADY_S = 20.0 -def _vllm_route_entry( - model_name: str, served: str, api_base: str -) -> dict[str, Any]: - """One LiteLLM ``model_list`` entry routing ``model_name`` to a vLLM upstream. - - Shared by every render path (legacy per-deployment, catalog-superset, and - the route registry) so a registry-rendered entry can never drift from what - the catalog/deployment paths produce for the same endpoint.""" - return { - 'model_name': model_name, - 'litellm_params': { - 'model': f'openai/{served}', - 'api_base': api_base, - 'api_key': 'EMPTY', - }, - } - - -def _ollama_route_entry( - model_name: str, tag: str, api_base: str -) -> dict[str, Any]: - """One LiteLLM ``model_list`` entry routing ``model_name`` to an Ollama tag - (see :func:`_vllm_route_entry` for why this is factored out).""" - return { - 'model_name': model_name, - 'litellm_params': { - 'model': f'ollama/{tag}', - 'api_base': api_base, - }, - } +def compose_catalog_route(alias: str, request: Any) -> GatewayRoute | None: + """Where a Compose front door sends a managed catalog endpoint: the + engine service its served name determines (the same one every + deployment of it gets without ``--dedicated``).""" + if request.engine == 'vllm': + served = request.served.get('served_model_name') or alias + return GatewayRoute(alias, 'openai', served, + f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1') + if request.engine == 'ollama': + host = request.spec.get('host') or request.host + tag = request.served.get('model') or alias + return GatewayRoute(alias, 'ollama', tag, + f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}') + return None -def _litellm_model_list( - deployments: list[Deployment], assignments: dict[str, list[int]] -) -> list[dict[str, Any]]: - """One LiteLLM ``model_list`` entry per served endpoint alias.""" - entries: list[dict[str, Any]] = [] +def catalog_routes( + catalog: Any, + managed: Callable[[str, Any], GatewayRoute | None] = compose_catalog_route, + *, dynamic: bool = False, +) -> list[GatewayRoute]: + """One route per endpoint of ``catalog``, sorted by alias. + + An external target routes to its own server, whatever the backend; + ``managed`` maps a managed endpoint's request to where this backend runs + it (or ``None``). With ``dynamic`` only the external routes, each with + its stable managed id: dynamic routing routes managed endpoints per + deployment (:func:`deployment_routes`). An endpoint that does not + resolve is skipped: a bad endpoint must not break the gateway. + """ + routes: list[GatewayRoute] = [] + for alias in sorted(getattr(catalog, 'endpoints', None) or {}): + try: + resolved = catalog.resolve_endpoint(alias) + target = resolved.target + if isinstance(target, ExternalTarget): + routes.append(GatewayRoute( + alias, 'openai', target.model, target.api_base, + key_env=target.api_key_env, origin='external', + route_id=_route_id('external', alias) if dynamic else None)) + continue + if dynamic: + continue + route = managed(alias, resolved.to_request()) + except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway + continue + if route is not None: + routes.append(route) + return routes + + +def deployment_routes( + deployments: list[Deployment], assignments: dict[str, list[int]], + *, dynamic: bool = False, +) -> list[GatewayRoute]: + """One route per (placed deployment, served endpoint alias). + + Static (the default): to the engine service its served name determines, + so a catalog endpoint acquired live routes exactly as its catalog route + does and live-vs-released never moves the rendered bytes. ``dynamic``: to + the deployment's **own** service, with a managed id per (deployment, + alias) (:func:`_route_id`), so same-model ``--dedicated`` deployments + share the alias and LiteLLM balances across them. Only ``vllm`` and + ``ollama`` render a service, so only they route. + """ + routes: list[GatewayRoute] = [] for deployment in sorted(deployments, key=lambda g: (g.created_at, g.id)): if deployment.id not in assignments: continue if deployment.engine == 'vllm': served = served_name(deployment) - api_base = f'http://{vllm_service_name(deployment)}:8000/v1' + service = (vllm_service_name(deployment, unique=True) if dynamic + else vllm_service_name_for(served)) + api_base = f'http://{service}:{VLLM_CONTAINER_PORT}/v1' for endpoint in sorted(deployment.served): - entries.append(_vllm_route_entry(endpoint, served, api_base)) + routes.append(GatewayRoute( + endpoint, 'openai', served, api_base, origin='deployment', + route_id=_route_id(deployment.id, endpoint) if dynamic else None)) elif deployment.engine == 'ollama': - api_base = f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' + host = deployment.spec.get('host') or deployment.id + service = (ollama_service_name(deployment) if dynamic + else ollama_service_name_for(host)) + api_base = f'http://{service}:{OLLAMA_CONTAINER_PORT}' for endpoint, payload in sorted(deployment.served.items()): - tag = payload.get('model', endpoint) - entries.append(_ollama_route_entry(endpoint, tag, api_base)) - return entries - - -def _litellm_model_list_from_catalog(catalog: Any) -> list[dict[str, Any]]: - """A *static superset* ``model_list``: one route per catalog endpoint. - - Unlike :func:`_litellm_model_list` (which routes only the currently-placed - deployments), this routes *every* catalog endpoint to its deterministic - upstream host (:func:`vllm_service_name_for` / :func:`ollama_service_name_for`). - The resulting config therefore depends only on the catalog, not on which - models happen to be up — so acquiring/releasing a model leaves the gateway's - config (and its container) untouched (no blip). A route whose upstream is not - currently running simply errors/cools-down until it comes up; the - ``router_settings`` below make that warmup self-healing. ``/v1/models`` lists - the whole catalog (some upstreams down) rather than only the live set. - - This static-superset path is the default. Its one limitation — it cannot give - same-model ``--dedicated`` deployments distinct upstreams, and cannot route - non-catalog acquires without a config change — is addressed by the opt-in - *dynamic routing* mode (``dynamic_routing=True``), which manages routes live - via LiteLLM's admin API against a Postgres model store (see - :func:`_litellm_routes`, :meth:`ComposeBackend._reconcile_routes`, and - ``docs/litellm-gateway-routing.md``). The two are mutually exclusive per - converge; this function is used only when dynamic routing is off. + routes.append(GatewayRoute( + endpoint, 'ollama', payload.get('model', endpoint), api_base, + origin='deployment', + route_id=_route_id(deployment.id, endpoint) if dynamic else None)) + return routes + + +def front_door_routes( + deployments: list[Deployment], assignments: dict[str, list[int]], *, + catalog: Any = None, dynamic: bool = False, + registry: Sequence[GatewayRoute] = (), + extra: Sequence[GatewayRoute] = (), + extra_dynamic: Sequence[GatewayRoute] = (), +) -> tuple[list[GatewayRoute], list[GatewayRoute]]: + """``(static route table, dynamic routes)`` of a front door: one is empty. + + Static: ``registry`` < the catalog's endpoints < placed deployments < + ``extra`` (routes another backend supplies), one per alias. Dynamic: a + route per (placed deployment, alias), per external endpoint, and each of + ``extra_dynamic``, one per managed id. """ - entries: list[dict[str, Any]] = [] - for name in sorted(getattr(catalog, 'endpoints', {})): - try: - req = catalog.resolve_endpoint(name).to_request() - except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway - continue - if req.engine == 'vllm': - served = req.served.get('served_model_name') or name - api_base = ( - f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' - ) - entries.append(_vllm_route_entry(name, served, api_base)) - elif req.engine == 'ollama': - host = req.spec.get('host') or req.host - tag = req.served.get('model') or name - api_base = ( - f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' - ) - entries.append(_ollama_route_entry(name, tag, api_base)) - return entries + if dynamic: + by_id: dict[str | None, GatewayRoute] = {} + for route in [*deployment_routes(deployments, assignments, dynamic=True), + *catalog_routes(catalog, dynamic=True), *extra_dynamic]: + by_id[route.route_id] = route + return [], list(by_id.values()) + return route_table(registry, catalog_routes(catalog), + deployment_routes(deployments, assignments), extra), [] -# -- Route registry (static-superset persistence) -------------------------- +# -- Route registry -------------------------------------------------------- # -# The registry stores *semantic* route inputs (served name / engine / host), -# never rendered LiteLLM entries — render derives entries through the same -# helpers the catalog/deployment paths use (:func:`_litellm_model_list_from_registry`), -# so a future renderer change propagates to old registry rows automatically. -# All functions here are pure; the backend owns the file I/O and locking. - - -def _registry_incoming_from_catalog(catalog: Any) -> dict[str, dict[str, Any]]: - """Semantic route rows for every resolvable endpoint of ``catalog``. - - Mirrors :func:`_litellm_model_list_from_catalog`'s iteration (unresolvable - endpoints skipped) but emits registry rows keyed by endpoint name. A vLLM - row carries only ``served`` (the upstream host is re-derived at render via - :func:`vllm_service_name_for`); an Ollama row carries ``model`` (tag) + - ``host``.""" - incoming: dict[str, dict[str, Any]] = {} - for name in sorted(getattr(catalog, 'endpoints', {})): - try: - req = catalog.resolve_endpoint(name).to_request() - except Exception: # noqa: BLE001 - a bad endpoint must not break the gateway - continue - if req.engine == 'vllm': - served = req.served.get('served_model_name') or name - incoming[name] = {'engine': 'vllm', 'served': served} - elif req.engine == 'ollama': - host = req.spec.get('host') or req.host - tag = req.served.get('model') or name - incoming[name] = {'engine': 'ollama', 'model': tag, 'host': host} - return incoming - - -def _registry_incoming_from_deployments( - deployments: list[Deployment], assignments: dict[str, list[int]] +# ``litellm_registry.json`` keeps the routes of deployments no published +# catalog defines, past their release: an ad-hoc acquire's alias stays +# routable and releasing it does not recreate the gateway. Catalog endpoints +# are not stored here; they are derived from the published union at every +# render. A registry from before that also holds catalog rows; they are read +# like the rest (lowest precedence, so a published definition wins), and +# ``routes prune`` drops what nothing serves. Rows are semantic inputs +# (served name / engine / host), never rendered entries. + + +def remembered_rows( + deployments: list[Deployment], assignments: dict[str, list[int]], *, + defined: set[str], extra: Sequence[GatewayRoute] = (), ) -> dict[str, dict[str, Any]]: - """Semantic route rows for every *placed* deployment in ``assignments``. - - ``deployments`` is the full ``desired`` set (which spans all runbooks via - the shared ledger), so this keeps non-catalog / dedicated acquires routable - and — because the registry persists — routable past release. One row per key - of ``deployment.served`` (a coalesced deployment can back several endpoint - aliases). Only ``vllm``/``ollama`` engines contribute; ``RESERVED_ENGINE`` - and unknown engines render no service, so they contribute no row — exactly - as :func:`render_compose`'s service loop skips them. - - The vLLM ``served`` uses the same fallback chain as :func:`vllm_service_name` - (``spec['served_model_name'] or sorted(served)[0] or id``), so a - catalog-listed endpoint acquired live reduces to the identical row a catalog - merge produces — live-vs-released status never moves the rendered bytes.""" - incoming: dict[str, dict[str, Any]] = {} + """Registry rows for every placed deployment's alias, and every ``extra`` + route of a Model another backend runs, that ``defined`` (the published + catalog aliases) does not cover. + + A vLLM row carries ``served`` (the host is re-derived from it), an Ollama + row the tag and ``host``, an upstream row its ``api_base``: each renders + back (:func:`registry_route`) to the route the deployment has now. + """ + rows: dict[str, dict[str, Any]] = {} for deployment in deployments: if deployment.id not in assignments: continue if deployment.engine == 'vllm': served = served_name(deployment) for endpoint in sorted(deployment.served): - incoming[endpoint] = {'engine': 'vllm', 'served': served} + rows[endpoint] = {'engine': 'vllm', 'served': served} elif deployment.engine == 'ollama': host = deployment.spec.get('host') or deployment.id for endpoint, payload in sorted(deployment.served.items()): - tag = payload.get('model', endpoint) - incoming[endpoint] = { - 'engine': 'ollama', - 'model': tag, - 'host': host, - } - return incoming - - -def _litellm_model_list_from_registry( - registry: dict[str, Any] -) -> list[dict[str, Any]]: - """Render the gateway ``model_list`` from the whole accumulated registry. - - Iterates ``sorted(entries)`` (determinism, §8) and derives each upstream - ``api_base`` through the live naming helpers, so the registry never becomes - a rendered-config parse surface.""" - entries: list[dict[str, Any]] = [] - rows = registry.get('entries', {}) if isinstance(registry, dict) else {} - for name in sorted(rows): - entry = registry_route_entry(name, rows[name]) - if entry is not None: - entries.append(entry) - return entries - - -def registry_route_entry(name: str, row: Any) -> dict[str, Any] | None: - """The LiteLLM entry one registry row renders to, or ``None`` if it cannot. - - The one derivation of a row's upstream: the render uses it, and so does - ``routes list``. Upstreams come from the live naming helpers, so the - registry never becomes a rendered-config parse surface. - """ + rows[endpoint] = {'engine': 'ollama', + 'model': payload.get('model', endpoint), 'host': host} + for route in extra: + if route.origin == 'upstream' and route.kind == 'openai': + rows[route.alias] = {'engine': 'upstream', 'served': route.model, + 'api_base': route.api_base} + return {alias: row for alias, row in rows.items() if alias not in defined} + + +def registry_route(name: str, row: Any) -> GatewayRoute | None: + """The route one registry row renders to, or ``None`` if it cannot.""" if not isinstance(row, dict): return None engine = row.get('engine') if engine == 'vllm': served = row.get('served') or name - api_base = f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1' - return _vllm_route_entry(name, served, api_base) + return GatewayRoute( + name, 'openai', served, + f'http://{vllm_service_name_for(served)}:{VLLM_CONTAINER_PORT}/v1', + origin='registry') if engine == 'ollama': - tag = row.get('model') or name host = row.get('host') or name - api_base = f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}' - return _ollama_route_entry(name, tag, api_base) - if engine == UPSTREAM_ROUTE and row.get('api_base'): - # An OpenAI-compatible server this project does not run (a KubeAI - # cluster's gateway): the row carries its address and the name it - # serves the model under. - return _vllm_route_entry(name, row.get('served') or name, str(row['api_base'])) + return GatewayRoute( + name, 'ollama', row.get('model') or name, + f'http://{ollama_service_name_for(host)}:{OLLAMA_CONTAINER_PORT}', + origin='registry') + if engine == 'upstream' and row.get('api_base'): + # A server this project does not run (a KubeAI cluster's gateway). + return GatewayRoute(name, 'openai', row.get('served') or name, + str(row['api_base']), origin='registry') return None -def upstream_route(deployment_id: str, endpoint: str, served: str, - api_base: str) -> dict[str, Any]: - """A dynamic route to a server this project does not run (a KubeAI Model). +def registry_routes(registry: dict[str, Any] | None) -> list[GatewayRoute]: + """The routes a registry's rows render to, sorted by alias.""" + rows = (registry or {}).get('entries') if isinstance(registry, dict) else None + if not isinstance(rows, dict): + return [] + return [r for r in (registry_route(name, rows[name]) for name in sorted(rows)) + if r is not None] - The same entry shape, and the same deterministic id, as a Compose engine's - dynamic route (:func:`_litellm_routes`). - """ - entry = _vllm_route_entry(endpoint, served, api_base) - entry['model_info'] = {'id': _route_id(deployment_id, endpoint)} - return entry - - -def _merge_route_registry( - existing: dict[str, Any], incoming: dict[str, dict[str, Any]] -) -> tuple[dict[str, Any], list[str]]: - """Merge ``incoming`` semantic rows into ``existing`` (a render's merge). - - Idempotent (merging identical rows is a no-op) and never removes a row. On - a conflict (same key, different row) *incoming wins* with a warning: this - is how a render folds in the invoking catalog, so an endpoint whose - catalog entry was edited gets its new route, the one justified recreate. - Seeding other catalogs is not this: ``routes seed`` refuses a redefinition - (:mod:`infer_stack.leasing.routes`). The existing ``version`` - is preserved (an unknown version merged under is not silently rewritten to - the current schema; see :meth:`ComposeBackend._load_route_registry`).""" - version = LITELLM_REGISTRY_VERSION - entries: dict[str, dict[str, Any]] = {} - if isinstance(existing, dict): - version = existing.get('version', LITELLM_REGISTRY_VERSION) - prior = existing.get('entries') - if isinstance(prior, dict): - entries = {k: v for k, v in prior.items()} - warnings: list[str] = [] - for name in sorted(incoming): - row = incoming[name] - if name in entries and entries[name] != row: - warnings.append( - f"route {name!r} redefined: {entries[name]} -> {row} " - '(incoming wins; gateway will be recreated once)' - ) - entries[name] = row - return {'version': version, 'entries': entries}, warnings - - -def _seed_registry_from_litellm_config( - config_text: str, -) -> tuple[dict[str, Any], list[str]]: - """One-shot upgrade seed: recover registry rows from a rendered - ``litellm_config.yaml`` so the first post-upgrade converge does not strip - the other runbooks' routes. - - Single-format, migration-time only (no cross-version promise): ``openai/`` - inverts *exactly* to ``{engine: vllm, served}``. Ollama rows are skipped with - a warning — the host survives only as a non-invertible ``dns_slug`` inside - ``api_base`` — and re-enter the registry at the next converge that has them - in its catalog or live set. Anything else unparseable is likewise skipped.""" - entries: dict[str, dict[str, Any]] = {} - warnings: list[str] = [] - try: - data = yaml.safe_load(config_text) or {} - except Exception: # noqa: BLE001 - a torn file must not brick seeding - return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}}, [ - 'seed: litellm_config.yaml is unparseable; starting an empty registry' - ] - for entry in data.get('model_list', []) or []: - name = entry.get('model_name') - model = (entry.get('litellm_params') or {}).get('model', '') - if not name: - continue - if isinstance(model, str) and model.startswith('openai/'): - entries[name] = {'engine': 'vllm', 'served': model[len('openai/'):]} - elif isinstance(model, str) and model.startswith('ollama/'): - warnings.append( - f"seed: skipping Ollama route {name!r} (host not recoverable " - 'from the rendered api_base; it re-enters at its next converge)' - ) - else: - warnings.append(f'seed: skipping unparseable route {name!r}') - return {'version': LITELLM_REGISTRY_VERSION, 'entries': entries}, warnings + +def upstream_route(deployment_id: str, endpoint: str, served: str, + api_base: str) -> GatewayRoute: + """A dynamic route to a server this project does not run (a KubeAI Model), + with the same deterministic id as a Compose engine's dynamic route.""" + return GatewayRoute(endpoint, 'openai', served, api_base, origin='upstream', + route_id=_route_id(deployment_id, endpoint)) def _dump_route_registry(registry: dict[str, Any]) -> str: - """Canonical, byte-stable serialization (§3): sorted keys + trailing - newline. A nondeterministic dump would manufacture phantom hash changes.""" + """Canonical, byte-stable serialization: sorted keys + trailing newline.""" return json.dumps(registry, sort_keys=True, indent=2) + '\n' -def _route_id(deployment_id: str, endpoint: str) -> str: - """Deterministic LiteLLM model id for one (deployment, endpoint) route. +def _route_id(owner: str, endpoint: str) -> str: + """Deterministic LiteLLM model id for one logical route. - Stable across converges, so route reconcile (:meth:`ComposeBackend. - _reconcile_routes`) can identify one logical route across renders. A route - whose id disappears is deleted by exactly this id; a route whose id remains - but whose observable routing semantics drifted is replaced under the same id. - The ``isr-`` prefix marks it - infer-stack-managed so reconcile never deletes a model someone added by hand. + ``owner`` is the deployment id for a managed route (one alias can have + several dedicated deployments), or ``'external'`` for an external + target's. Stable across converges, so route reconcile + (:meth:`Gateway._reconcile_routes`) identifies one logical route across + renders: a route whose id disappears is deleted by exactly this id; one + whose id remains but whose semantics drifted is replaced under it. The + ``isr-`` prefix marks it infer-stack-managed so reconcile never deletes a + model someone added by hand. """ - digest = hashlib.sha256(f'{deployment_id}|{endpoint}'.encode()).hexdigest() + digest = hashlib.sha256(f'{owner}|{endpoint}'.encode()).hexdigest() return f'{ROUTE_ID_PREFIX}{digest[:32]}' -def _litellm_routes( - deployments: list[Deployment], assignments: dict[str, list[int]] -) -> list[dict[str, Any]]: - """Desired LiteLLM route set for the *live* deployments (dynamic routing). - - One entry per (placed deployment, served endpoint), addressing the - deployment's **own** unique upstream service (:func:`vllm_service_name` with - ``unique=True``). Several dedicated deployments of the same model therefore - yield several entries that share one public ``model_name`` but point at - distinct upstreams — LiteLLM load-balances the alias across them, so each - runs on its own GPU while clients still ask for the single name. Each entry - carries a deterministic ``model_info.id`` (:func:`_route_id`) so applying the - set via the admin API is an idempotent diff, not fire-and-forget calls. - """ - entries: list[dict[str, Any]] = [] - for deployment in sorted(deployments, key=lambda g: (g.created_at, g.id)): - if deployment.id not in assignments: - continue - if deployment.engine == 'vllm': - served = served_name(deployment) - api_base = ( - f'http://{vllm_service_name(deployment, unique=True)}' - f':{VLLM_CONTAINER_PORT}/v1' - ) - for endpoint in sorted(deployment.served): - entries.append(upstream_route(deployment.id, endpoint, served, api_base)) - elif deployment.engine == 'ollama': - api_base = ( - f'http://{ollama_service_name(deployment)}:{OLLAMA_CONTAINER_PORT}' - ) - for endpoint, payload in sorted(deployment.served.items()): - tag = payload.get('model', endpoint) - entries.append( - { - 'model_name': endpoint, - 'litellm_params': { - 'model': f'ollama/{tag}', - 'api_base': api_base, - }, - 'model_info': {'id': _route_id(deployment.id, endpoint)}, - } - ) - return entries - - CONFIG_HASH_LABEL = 'infer-stack.config-hash' @@ -809,10 +666,8 @@ class FrontDoor: def render_front_door( - deployments: list[Deployment], - assignments: dict[str, list[int]], + routes: list[GatewayRoute], *, - engine_services: list[str], vllm_v1_urls: list[str], ollama_native_urls: list[str], images: dict[str, str], @@ -827,19 +682,17 @@ def render_front_door( reverse_proxy_port: int, reverse_proxy_config: str | None, aux_dir: str | Path | None, - catalog: Any, - route_registry: dict[str, Any] | None, dynamic_routing: bool, - upstream_routes: list[dict[str, Any]] | None = None, + dynamic_routes: list[GatewayRoute] | None = None, ui_run_as: str | None = None, ) -> FrontDoor: """Render the gateway, its database, Open WebUI and the reverse proxy. - The engines are the caller's: it passes the services it rendered - (``engine_services``, only for the legacy per-model ``depends_on``) and - the in-network URLs a UI with no gateway can talk to directly. - ``upstream_routes`` are dynamic routes to servers this project does not - run (see :func:`upstream_route`). + ``routes`` is the static route table (one per alias, + :func:`~infer_stack.leasing.routes.route_table`); with + ``dynamic_routing`` the config has none and ``dynamic_routes`` are + reconciled through the admin API instead. The engines are the caller's: + it passes the in-network URLs a UI with no gateway can talk to directly. """ services: dict[str, Any] = {} litellm_config = None @@ -850,40 +703,22 @@ def render_front_door( # WebUI picker) up instead of tearing the whole stack down; only an explicit # `stack down` removes it. With no models the model_list is simply empty. if litellm: - # Three route-table strategies, in order of preference: + # Two route-table strategies: # * DYNAMIC ROUTING: the rendered config is a STATIC base (empty # model_list); the real routes live in Postgres and are applied to the # running gateway via the admin API (see _reconcile_routes). The config # hash never changes as models come/go, so the gateway is never - # recreated — no blip, and per-deployment routing works (so same-model + # recreated -- no blip, and per-deployment routing works (so same-model # --dedicated deployments each get their own upstream). - # * ROUTE REGISTRY (static-superset default from ComposeBackend): render - # from the whole accumulated registry (every catalog + live deployment - # ever merged, across all runbooks). Byte-stable once seeded, so the - # gateway is never recreated and a cross-catalog converge can no longer - # strip another runbook's routes. The backend loads/merges/writes the - # registry and passes the merged dict in; this function stays pure. - # * STATIC SUPERSET (catalog): one route per catalog endpoint to a - # deterministic host; config depends only on the catalog, so the - # gateway is not recreated as models come/go (no blip) but same-model - # dedicated collapses to one upstream. Unreachable from ComposeBackend - # once the registry is wired; kept for direct callers/tests. - # * LEGACY (no catalog): route only the placed deployments; churns the - # config (and recreates the gateway) on every model change. + # * STATIC: the model_list is the route table the caller derived from + # the published catalogs, the placed deployments and any legacy + # registry. It depends on endpoint definitions, not on which models + # are up, so the gateway is not recreated as models come and go. if dynamic_routing: entries: list[dict[str, Any]] = [] - litellm_routes = (_litellm_routes(deployments, assignments) - + list(upstream_routes or [])) - litellm_depends: list[str] = [] - elif route_registry is not None: - entries = _litellm_model_list_from_registry(route_registry) - litellm_depends = [] # no per-model depends_on -> no churn - elif catalog is not None: - entries = _litellm_model_list_from_catalog(catalog) - litellm_depends = [] # no per-model depends_on -> no churn + litellm_routes = [r.entry() for r in dynamic_routes or []] else: - entries = _litellm_model_list(deployments, assignments) - litellm_depends = list(engine_services) + entries = [r.entry() for r in routes] litellm_config = yaml.safe_dump( { 'model_list': entries, @@ -911,7 +746,7 @@ def render_front_door( if dynamic_routing: services[POSTGRES_SERVICE] = _postgres_service(images, state) services[LITELLM_SERVICE] = _litellm_service( - litellm_depends, + [], litellm_port, images, str(aux_dir or '.'), @@ -1170,80 +1005,29 @@ def db_password(self) -> str: return pw def _load_route_registry(self) -> dict[str, Any]: - """Read the route registry, tolerantly (fail-open — a broken registry - must never block a converge). - - Missing file → seed from the live ``litellm_config.yaml`` if present - (upgrade migration, §6), else an empty registry. An *unknown* schema - version whose ``entries`` still parses as a name→row map is preserved - as-is (render what's understood, warn, do NOT rewrite) rather than - reseeded, so a binary rollback doesn't discard the accumulated union. - Only a structurally unusable file (not a map / garbage JSON) falls back - to seeding.""" + """Read the route registry, tolerantly (fail-open: a broken + registry must never block a converge). Missing or structurally + unusable -> empty. An unknown schema version whose ``entries`` parses + is read as-is, rendering what is understood.""" from .._log import logger + empty = {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} if not self._registry_file.exists(): - config = self.state_dir / LITELLM_CONFIG_FILENAME - if config.exists(): - seeded, warnings = _seed_registry_from_litellm_config( - config.read_text() - ) - for w in warnings: - logger.warning(' route registry: {}', w) - logger.info( - ' route registry: seeded {} vLLM route(s) from {}', - len(seeded['entries']), config.name, - ) - return seeded - return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} + return empty try: data = json.loads(self._registry_file.read_text()) except (OSError, json.JSONDecodeError) as exc: - logger.warning( - ' route registry: {} is unreadable ({}); rebuilding from seed', - self._registry_file.name, exc, - ) - data = None - if not isinstance(data, dict) or not isinstance( - data.get('entries'), dict - ): - config = self.state_dir / LITELLM_CONFIG_FILENAME - if config.exists(): - seeded, warnings = _seed_registry_from_litellm_config( - config.read_text() - ) - for w in warnings: - logger.warning(' route registry: {}', w) - return seeded - return {'version': LITELLM_REGISTRY_VERSION, 'entries': {}} - version = data.get('version') - if version != LITELLM_REGISTRY_VERSION: - logger.warning( - ' route registry: unknown schema version {!r} in {} — ' - 'rendering as-is without rewrite (fields this renderer does ' - 'not understand are ignored)', - version, self._registry_file.name, - ) + logger.warning(' route registry: {} is unreadable ({}); ignoring it', + self._registry_file.name, exc) + return empty + if not isinstance(data, dict) or not isinstance(data.get('entries'), dict): + return empty + if data.get('version') != LITELLM_REGISTRY_VERSION: + logger.warning(' route registry: unknown schema version {!r} in {}; ' + 'rendering the rows it understands, never rewriting it', + data.get('version'), self._registry_file.name) return data - def _save_route_registry(self, merged: dict[str, Any]) -> None: - """Persist a merged registry if it changed (under the converge flock).""" - from .._log import logger - - existing = self._load_route_registry() - if merged == existing: - return - prior = existing.get('entries', {}) if isinstance(existing, dict) else {} - added = sorted(set(merged['entries']) - set(prior)) - updated = sorted( - k for k in merged['entries'] if k in prior and merged['entries'][k] != prior[k] - ) - if added: - logger.info(' route registry: +{} route(s): {}', len(added), ', '.join(added)) - if updated: - logger.info(' route registry: updated route(s): {}', ', '.join(updated)) - self._atomic_write(self._registry_file, _dump_route_registry(merged)) - def route_registry(self) -> dict[str, Any]: """The registry as stored: ``{'version': ..., 'entries': {...}}``.""" registry = self._load_route_registry() @@ -1254,6 +1038,31 @@ def route_entries(self) -> dict[str, dict[str, Any]]: entries = self.route_registry().get('entries') return dict(entries) if isinstance(entries, dict) else {} + def registry_routes(self) -> list[GatewayRoute]: + """The registry's routes, the lowest-precedence route layer.""" + return registry_routes(self._load_route_registry()) + + def remember(self, rows: dict[str, dict[str, Any]]) -> None: + """Add or update ``rows`` in the registry. Called after an approved + render, by a caller already holding the converge flock (taking it + again would block on itself); nothing is ever removed here.""" + from .._log import logger + + if not rows: + return + existing = self._load_route_registry() + if existing.get('version', LITELLM_REGISTRY_VERSION) != LITELLM_REGISTRY_VERSION: + return # a newer binary's file: read, never rewrite + entries = dict(existing.get('entries') or {}) + changed = sorted(k for k, v in rows.items() if entries.get(k) != v) + if not changed: + return + entries.update(rows) + logger.info(' route registry: remembered {}', ', '.join(changed)) + self._atomic_write(self._registry_file, _dump_route_registry( + {'version': existing.get('version', LITELLM_REGISTRY_VERSION), + 'entries': entries})) + def replace_route_entries(self, entries: dict[str, dict[str, Any]]) -> None: """Write the registry's entries (under the converge flock, atomically); its schema version is kept.""" @@ -1264,30 +1073,6 @@ def replace_route_entries(self, entries: dict[str, dict[str, Any]]) -> None: self._atomic_write(self._registry_file, _dump_route_registry( {'version': version, 'entries': dict(entries)})) - def merge_route_registry( - self, incoming: dict[str, dict[str, Any]] - ) -> dict[str, Any]: - """Public write path for out-of-converge registry seeds (``routes seed``). - - Takes the converge flock, read-merge-writes the registry, and returns the - merged dict. ``converge`` only ever merges the invoking process's own - catalog, so a standalone caller (seeding a *sibling* runbook's catalog) - needs this to fold extra rows in before the follow-up ``reconcile`` - renders+applies. The flock here and the one the subsequent converge takes - are sequential acquisitions, not nested — no reentrancy concern.""" - from .._log import logger - - with self._converge_lock(): - existing = self._load_route_registry() - merged, warnings = _merge_route_registry(existing, incoming) - for w in warnings: - logger.warning(' route registry: {}', w) - if merged != existing: - self._atomic_write( - self._registry_file, _dump_route_registry(merged) - ) - return merged - def urls(self) -> tuple[str | None, str | None]: """``(OpenAI base URL, Open WebUI URL)``; ``None`` for one that is off. @@ -1468,7 +1253,8 @@ def _route_semantics(route: dict[str, Any]) -> dict[str, Any]: LiteLLM's model-info response contains additional database/runtime fields and may redact credentials. The public alias, upstream model, and upstream base URL are the routing semantics infer-stack can both set and - reliably observe. A matching managed id with different values here is + reliably observe; so is the name of an external route's key variable, + kept in ``model_info`` because LiteLLM redacts the key itself. A matching managed id with different values here is drift and is replaced, not accepted as healthy. """ params = route.get('litellm_params') or {} @@ -1476,6 +1262,7 @@ def _route_semantics(route: dict[str, Any]) -> dict[str, Any]: 'model_name': route.get('model_name'), 'model': params.get('model'), 'api_base': params.get('api_base'), + 'key_env': (route.get('model_info') or {}).get('infer_stack_key_env'), } def _list_managed_routes( @@ -1584,15 +1371,6 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: info['proxy_url'] = f'http://127.0.0.1:{self.reverse_proxy_port}' return info - def merged_route_registry(self, incoming: dict[str, dict[str, Any]]) -> dict[str, Any]: - """The registry with ``incoming`` rows merged in, in memory (no write).""" - from .._log import logger - - merged, warnings = _merge_route_registry(self._load_route_registry(), incoming) - for w in warnings: - logger.warning(' route registry: {}', w) - return merged - def front_door_urls(backend) -> tuple[str | None, str | None]: """``(OpenAI base URL, Open WebUI URL)`` of ``backend``'s front door. diff --git a/infer_stack/leasing/naming.py b/infer_stack/leasing/naming.py index 415d9a8b..1a8389b5 100644 --- a/infer_stack/leasing/naming.py +++ b/infer_stack/leasing/naming.py @@ -38,7 +38,8 @@ def vllm_service_name_for(served: str) -> str: stability is what lets the LiteLLM gateway carry a *static* route table (one per catalog endpoint) whose upstream hosts match the containers when they come up — so adding/removing models does not rewrite the gateway's config and - the gateway is never recreated (no "blip"); see :func:`_litellm_model_list`. + the gateway is never recreated (no "blip"); see + :func:`~infer_stack.leasing.gateway.front_door_routes`. """ return f'vllm-{_dns_slug(served)}' diff --git a/infer_stack/leasing/routes.py b/infer_stack/leasing/routes.py index a8658dd9..ee1f78cd 100644 --- a/infer_stack/leasing/routes.py +++ b/infer_stack/leasing/routes.py @@ -1,17 +1,98 @@ -"""Planning changes to the gateway's route registry (``routes seed`` / ``prune``). - -The registry maps a public alias to the upstream it routes to. Adding an -alias is additive; changing where an existing alias points redirects every -client that uses it, so a seed that would redefine one refuses by default -and replaces it only when asked. A plan says which is which before anything -is written; :class:`~infer_stack.leasing.controller.Controller` commits it -through the normal publication. +"""Gateway routes, and planning ``routes seed`` / ``prune``. + +A :class:`GatewayRoute` is one alias the gateway serves and where it sends +it; :meth:`GatewayRoute.entry` is the one rendering of it as a LiteLLM +``model_list`` entry or admin-API route. Routes are derived at render +(docs/planning/external-endpoints.md, decision 2), one layer per owner, in +increasing precedence: + +* ``registry`` -- rows ``litellm_registry.json`` keeps: routes of + deployments no published catalog defines, remembered past their release + so the gateway is not recreated for one, and rows from before routes were + derived; ``routes prune`` drops those nothing serves; +* ``catalog`` / ``external`` -- every endpoint of the published catalog union; +* ``deployment`` -- every placed deployment; +* ``upstream`` -- routes another backend supplies (KubeAI's Models). + +``routes seed`` merges catalogs into the published union and ``routes +prune`` unpublishes. Adding an alias is additive; changing what an existing +one means redirects every client that uses it, so a seed that would redefine +one refuses by default and replaces it only when asked. A plan says which is +which before anything is written; :class:`~infer_stack.leasing.controller. +Controller` commits it through the normal publication. """ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any +from typing import Any, Iterable + +#: Where a route came from, lowest precedence first (see the module doc). +ORIGINS = ('registry', 'catalog', 'external', 'deployment', 'upstream') + + +@dataclass(frozen=True) +class GatewayRoute: + """One alias the gateway serves, and the upstream it sends it to. + + ``kind`` is ``openai`` (any OpenAI-compatible server: vLLM, a KubeAI + Model, an external target) or ``ollama``; ``model`` is the upstream + model, what that server expects as ``model``. ``key_env`` names the + variable holding the upstream's key (never the key); without one an + ``openai`` route sends ``EMPTY``. ``route_id`` is a dynamic route's + managed id. ``origin`` says which owner produced it and is not part of + what it means. + + Example: + >>> GatewayRoute('qwen', 'openai', 'Qwen/Q', 'http://box/v1', key_env='BOX_KEY').entry() + {'model_name': 'qwen', 'litellm_params': {'model': 'openai/Qwen/Q', 'api_base': 'http://box/v1', 'api_key': 'os.environ/BOX_KEY'}} + >>> GatewayRoute('tiny', 'ollama', 'tinyllama', 'http://ollama:11434', route_id='isr-1').entry() + {'model_name': 'tiny', 'litellm_params': {'model': 'ollama/tinyllama', 'api_base': 'http://ollama:11434'}, 'model_info': {'id': 'isr-1'}} + """ + + alias: str + kind: str + model: str + api_base: str + key_env: str | None = None + route_id: str | None = None + origin: str = field(default='catalog', compare=False) + + def entry(self) -> dict[str, Any]: + """The LiteLLM entry: a static ``model_list`` item, or with a + ``route_id`` the body of an admin-API ``/model/new``.""" + params: dict[str, Any] = {'model': f'{self.kind}/{self.model}', + 'api_base': self.api_base} + if self.kind == 'openai': + params['api_key'] = f'os.environ/{self.key_env}' if self.key_env else 'EMPTY' + entry: dict[str, Any] = {'model_name': self.alias, 'litellm_params': params} + if self.route_id: + entry['model_info'] = {'id': self.route_id} + if self.key_env: + # The key's name is part of what the route means, and LiteLLM + # redacts the key itself, so reconcile compares the name here. + entry['model_info']['infer_stack_key_env'] = self.key_env + return entry + + def describe(self) -> str: + """``kind/model -> api_base`` (and the key's name), for people.""" + key = f' (key ${self.key_env})' if self.key_env else '' + return f'{self.kind}/{self.model} -> {self.api_base}{key}' + + +def route_table(*layers: Iterable[GatewayRoute]) -> list[GatewayRoute]: + """One route per alias, a later layer winning, sorted by alias. + + >>> a = GatewayRoute('a', 'openai', 'm', 'http://old/v1', origin='registry') + >>> b = GatewayRoute('a', 'openai', 'm', 'http://new/v1') + >>> [r.api_base for r in route_table([a], [b])] + ['http://new/v1'] + """ + table: dict[str, GatewayRoute] = {} + for layer in layers: + for route in layer: + table[route.alias] = route + return [table[alias] for alias in sorted(table)] class RouteConflict(ValueError): @@ -24,26 +105,54 @@ def __init__(self, names: list[str]): f'{", ".join(self.names)}; pass --replace to redefine them') +@dataclass(frozen=True, eq=False) +class Meaning: + """What an alias means, for a seed: the endpoint definition's semantic + key (``None`` for a registry row, which has only a route) and the route it + renders (``None`` where this gateway cannot route it). + + Two meanings are the same when both keys match or, for a registry row, + when the routes do. + """ + + route: GatewayRoute | None + key: str | None = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Meaning): + return NotImplemented + if self.key is not None and other.key is not None: + return self.key == other.key + return self.route == other.route + + __hash__ = None # type: ignore[assignment] + + def __str__(self) -> str: + text = self.route.describe() if self.route is not None else '(not routed here)' + return text if self.key is not None else f'{text} [registry]' + + @dataclass class RoutePlan: - """What a seed or prune would do to the registry's entries. + """What a seed or prune would do. - ``added``: new aliases; ``unchanged``: identical rows already there; - ``conflicted``: aliases whose incoming row differs (``{name: (current, - incoming)}``); ``dropped``: aliases a prune removes. ``incoming`` keeps a - seed's rows so the commit can recheck them under the lock. + ``added``: aliases the seed publishes; ``unchanged``: aliases already + published with the same meaning; ``conflicted``: aliases whose incoming + meaning differs (``{name: (current, incoming)}``); ``dropped``: aliases a + prune unpublishes. ``incoming`` keeps a seed's meanings and ``sources`` + its catalogs, so the commit can recheck them under the lock. """ - added: dict[str, dict[str, Any]] = field(default_factory=dict) + added: dict[str, Any] = field(default_factory=dict) unchanged: list[str] = field(default_factory=list) - conflicted: dict[str, tuple[dict[str, Any], dict[str, Any]]] = field(default_factory=dict) + conflicted: dict[str, tuple[Any, Any]] = field(default_factory=dict) dropped: list[str] = field(default_factory=list) - incoming: dict[str, dict[str, Any]] = field(default_factory=dict) + incoming: dict[str, Any] = field(default_factory=dict) + sources: list[dict[str, Any]] = field(default_factory=list) -def plan_seed(current: dict[str, dict[str, Any]], - incoming: dict[str, dict[str, Any]]) -> RoutePlan: - """Classify ``incoming`` rows against the registry's ``current`` entries. +def plan_seed(current: dict[str, Any], incoming: dict[str, Any]) -> RoutePlan: + """Classify ``incoming`` meanings against ``current`` ones. >>> plan = plan_seed({'a': {'up': 1}, 'b': {'up': 2}}, ... {'a': {'up': 1}, 'b': {'up': 9}, 'c': {'up': 3}}) @@ -62,9 +171,8 @@ def plan_seed(current: dict[str, dict[str, Any]], return plan -def plan_prune(current: dict[str, dict[str, Any]], - keep: dict[str, dict[str, Any]]) -> RoutePlan: - """Which of ``current`` a prune drops: everything not in ``keep``. +def plan_prune(current: Iterable[str], keep: Iterable[str]) -> RoutePlan: + """Which of the ``current`` aliases a prune drops: all not in ``keep``. >>> plan_prune({'a': {}, 'b': {}}, {'a': {}}).dropped ['b'] diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 5222cec4..c5d48f5a 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -895,7 +895,7 @@ def _patch_backend(monkeypatch, state_dir, catalog=None): def test_routes_seed_then_list(tmp_path, monkeypatch, capsys): - """seed two sibling catalogs into the registry, then list shows both.""" + """seed two sibling catalogs (publish them), then list shows both.""" from infer_stack.cli.commands_leasing import RoutesListCLI, RoutesSeedCLI state = tmp_path / 'state' @@ -918,9 +918,19 @@ def test_routes_seed_then_list(tmp_path, monkeypatch, capsys): assert rc == 0 out = json.loads(capsys.readouterr().out) assert sorted(r['name'] for r in out['routes']) == ['alpha', 'beta'] - # Seeding then rendering left the gateway config listing both, unchanged on a - # re-list (no converge on list): the registry file is byte-stable. - assert (state / 'litellm_registry.json').exists() + assert {r['origin'] for r in out['routes']} == {'catalog'} + # Seeding published the catalogs; the registry stores none of their routes. + assert sorted(_published(db)) == ['alpha', 'beta'] + assert not (state / 'litellm_registry.json').exists() + + +def _published(db): + """The published catalog union's endpoints: ``{alias: spec}``.""" + from infer_stack.leasing import Ledger, SqliteStore + + profile = Ledger(SqliteStore(db)).profile() or {} + return {name: spec for source in profile.get('catalogs') or [] + for name, spec in (source.get('endpoints') or {}).items()} def test_routes_seed_requires_compose_backend(tmp_path, capsys): @@ -1174,7 +1184,7 @@ def test_release_and_renew_name_a_missing_env_file(env, tmp_path): def test_routes_seed_adds_but_refuses_to_redefine_without_replace(tmp_path, monkeypatch, capsys): """Queue item 19: seeding a new alias is additive; seeding a different definition of an existing alias redirects its clients, so it refuses by - default and leaves the registry as it was.""" + default and leaves the published catalogs as they were.""" from infer_stack.cli.commands_leasing import RoutesSeedCLI state = tmp_path / 'state' @@ -1195,19 +1205,19 @@ def seed(*args): rc, out = seed(str(first)) assert rc == 0 and out['added'] == ['alpha'] - registry = (state / 'litellm_registry.json').read_text() + published = _published(db) rc, out = seed(str(first)) # identical: a no-op assert out['added'] == [] and out['unchanged'] == ['alpha'] - assert (state / 'litellm_registry.json').read_text() == registry + assert _published(db) == published with pytest.raises(SystemExit, match='refused'): RoutesSeedCLI.main(argv=['--ledger', db, str(second), '--yes']) - assert (state / 'litellm_registry.json').read_text() == registry # unchanged + assert _published(db) == published # unchanged rc, out = seed(str(second), '--replace') assert rc == 0 and out['updated'] == ['alpha'] - assert (state / 'litellm_registry.json').read_text() != registry + assert _published(db)['alpha']['served_name'] == 'alpha-v2' def test_a_seed_conflict_that_appears_after_the_plan_leaves_no_marker(tmp_path, monkeypatch): diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 5591ac8b..362727b5 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -94,3 +94,137 @@ def test_the_cli_adds_an_external_endpoint_and_refuses_runtime_options(tmp_path, for flags in (['--engine', 'vllm'], ['--gpu', '0'], ['--reclaim', 'stop']): with pytest.raises(SystemExit, match='runs nothing'): EndpointAddCLI.main(argv=[*common, '--force', *flags]) + + +# -- routes: one GatewayRoute per alias, whoever runs it ---------------------------- + + +def _compose(tmp_path, cat, **kw): + from test_leasing_profile import backend + + return backend(tmp_path / 'state', catalog=cat, **kw) + + +def test_an_external_endpoint_routes_to_its_own_server_with_its_key_by_name(tmp_path): + be = _compose(tmp_path, Catalog.from_dict(catalog(remote=REMOTE))) + be.converge([], apply=False) + config = yaml.safe_load((be.state_dir / 'litellm_config.yaml').read_text()) + routes = {e['model_name']: e['litellm_params'] for e in config['model_list']} + assert routes['remote'] == {'model': 'openai/Qwen/Qwen3-32B', + 'api_base': 'http://box:8000/v1', + 'api_key': 'os.environ/REMOTE_QWEN_KEY'} + assert routes['local']['api_key'] == 'EMPTY' # managed entries unchanged + assert not (be.state_dir / 'litellm_registry.json').exists() + + +def test_a_dynamic_external_route_has_a_stable_id_from_its_alias(tmp_path): + import json + + from infer_stack.leasing.gateway import _route_id + + moved = {**REMOTE, 'external': {**REMOTE['external'], 'api_base': 'http://other:8000/v1'}} + ids = [] + for target in (REMOTE, moved): + be = _compose(tmp_path, Catalog.from_dict(catalog(remote=target)), dynamic_routing=True) + be.converge([], apply=False) + (route,) = json.loads((be.state_dir / 'litellm_routes.json').read_text()) + ids.append(route['model_info']['id']) + assert route['model_info']['infer_stack_key_env'] == 'REMOTE_QWEN_KEY' + # Redefining the server replaces one route under one id. + assert ids[0] == ids[1] == _route_id('external', 'remote') + + +def test_route_semantics_cover_the_key_name(): + from infer_stack.leasing.gateway import Gateway + from infer_stack.leasing.routes import GatewayRoute + + a = GatewayRoute('r', 'openai', 'm', 'http://b/v1', key_env='K1', route_id='isr-x').entry() + b = GatewayRoute('r', 'openai', 'm', 'http://b/v1', key_env='K2', route_id='isr-x').entry() + assert Gateway._route_semantics(a) != Gateway._route_semantics(b) + + +def test_kubeai_routes_an_external_endpoint_directly_not_through_the_cluster(tmp_path): + from test_leasing_kubeai import UPSTREAM, make_front_door_backend + + be, _ = make_front_door_backend(tmp_path) + be.catalog = Catalog.from_dict(catalog(remote=REMOTE)) + routes = {r.alias: r for r in be.routes([])} + assert routes['remote'].api_base == 'http://box:8000/v1' + assert routes['remote'].origin == 'external' + assert routes['local'].api_base == UPSTREAM + + +# -- publication: routes seed publishes, routes prune unpublishes ------------------- + + +def _ctl(tmp_path, cat, docker=None): + from test_leasing_profile import controller + + return controller(tmp_path, catalog=cat, docker=docker) + + +def _published(ledger): + return sorted(n for s in (ledger.profile() or {}).get('catalogs') or [] + for n in (s.get('endpoints') or {})) + + +def test_seed_publishes_an_external_endpoint_and_prune_unpublishes_it(tmp_path): + mine = Catalog.from_dict(catalog()) + ledger, ctl = _ctl(tmp_path, mine) + other = Catalog.from_dict({'endpoints': {'remote': REMOTE}}) + plan = ctl.plan_route_seed([other]) + assert list(plan.added) == ['remote'] + ctl.commit_route_seed(plan) + assert _published(ledger) == ['local', 'remote'] + view = {r.alias: r.origin for r in ctl.route_view()} + assert view == {'local': 'catalog', 'remote': 'external'} + + plan = ctl.plan_route_prune() # the invocation knows only 'local' + assert plan.dropped == ['remote'] + dropped, _ = ctl.commit_route_prune(plan) + assert dropped == ['remote'] + assert _published(ledger) == ['local'] + assert [r.alias for r in ctl.route_view()] == ['local'] + + +def test_prune_keeps_what_a_live_deployment_serves(tmp_path): + from test_leasing_compose import FakeDocker + + docker = FakeDocker() + both = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _ctl(tmp_path, both, docker) + ctl.acquire('w', both.resolve_names(['local']), wait=False) + _, other = _ctl(tmp_path, Catalog.from_dict({'endpoints': {'remote': REMOTE}}), docker) + assert other.plan_route_prune().dropped == [] # local is live, remote is mine + + +def test_seed_never_redefines_an_endpoint_a_resident_deployment_runs(tmp_path): + from infer_stack.leasing.profile import ProfileMismatch + from test_leasing_compose import FakeDocker + + docker = FakeDocker() + mine = Catalog.from_dict(catalog()) + _, ctl = _ctl(tmp_path, mine, docker) + ctl.acquire('w', mine.resolve_names(['local']), wait=False) + changed = catalog() + changed['endpoints']['local']['runtime'] = {'max_model_len': 8} + plan = ctl.plan_route_seed([Catalog.from_dict(changed)]) + assert list(plan.conflicted) == ['local'] + with pytest.raises(ProfileMismatch, match='resident deployment'): + ctl.commit_route_seed(plan, replace=True) + + +def test_kubeai_catalog_routes_are_not_remembered_but_ad_hoc_models_are(tmp_path): + import json + + from test_leasing_kubeai import make_front_door_backend, vllm + + be, _ = make_front_door_backend(tmp_path) + be.catalog = Catalog.from_dict(catalog()) + local = vllm('grp-l', served='local') + local.served = {'local': {'served_model_name': 'local', 'protocol': 'chat'}} + adhoc = vllm('grp-a', served='Org/Adhoc') + adhoc.served = {'adhoc': {'served_model_name': 'Org/Adhoc', 'protocol': 'chat'}} + be.converge([local, adhoc], apply=False) + registry = json.loads((tmp_path / 'gateway' / 'litellm_registry.json').read_text()) + assert set(registry['entries']) == {'adhoc'} diff --git a/tests/test_leasing_compose.py b/tests/test_leasing_compose.py index 8231dc74..9afc109f 100644 --- a/tests/test_leasing_compose.py +++ b/tests/test_leasing_compose.py @@ -812,16 +812,18 @@ def _two_endpoint_catalog(): def test_litellm_superset_config_is_invariant_across_model_set(tmp_path): """The no-blip property at the rendering level: with a catalog, the LiteLLM config + service spec do NOT change when the live model set changes, so - `docker compose up` would not recreate the gateway (no blip).""" + `docker compose up` would not recreate the gateway (no blip). The + deployments serve catalog endpoints, as every acquire under a catalog + does.""" cat = _two_endpoint_catalog() # alpha live, then beta live -- two different desired sets, same catalog. rc_a = render_compose( - [vllm('grp-a', served='alpha')], {'grp-a': [0]}, + [vllm('alpha', served='alpha')], {'alpha': [0]}, images=IMAGES, ports=PORTS, state=STATE, litellm=True, litellm_port=14042, aux_dir=tmp_path, catalog=cat, ) rc_b = render_compose( - [vllm('grp-b', served='beta')], {'grp-b': [0]}, + [vllm('beta', served='beta')], {'beta': [0]}, images=IMAGES, ports=PORTS, state=STATE, litellm=True, litellm_port=14042, aux_dir=tmp_path, catalog=cat, ) @@ -883,14 +885,14 @@ def test_untouched_model_stable_when_another_swaps_gpu(tmp_path): """ cat = _three_endpoint_catalog() rc1 = render_compose( - [vllm('grp-c', served='cee'), vllm('grp-a', served='alpha')], - {'grp-c': [2], 'grp-a': [0]}, + [vllm('cee', served='cee'), vllm('alpha', served='alpha')], + {'cee': [2], 'alpha': [0]}, images=IMAGES, ports=PORTS, state=STATE, litellm=True, litellm_port=14042, aux_dir=tmp_path, catalog=cat, ) rc2 = render_compose( - [vllm('grp-c', served='cee'), vllm('grp-b', served='beta')], - {'grp-c': [2], 'grp-b': [0]}, + [vllm('cee', served='cee'), vllm('beta', served='beta')], + {'cee': [2], 'beta': [0]}, images=IMAGES, ports=PORTS, state=STATE, litellm=True, litellm_port=14042, aux_dir=tmp_path, catalog=cat, ) diff --git a/tests/test_leasing_dynamic_routing.py b/tests/test_leasing_dynamic_routing.py index 3a74c335..bd34c0d0 100644 --- a/tests/test_leasing_dynamic_routing.py +++ b/tests/test_leasing_dynamic_routing.py @@ -27,11 +27,16 @@ from infer_stack.leasing.gateway import ( POSTGRES_SERVICE, ROUTE_ID_PREFIX, - _litellm_routes, + deployment_routes, _route_id, ) from infer_stack.leasing.models import Deployment, DeploymentState + +def _litellm_routes(deployments, assignments): + """The rendered dynamic route entries of the placed deployments.""" + return [r.entry() for r in deployment_routes(deployments, assignments, dynamic=True)] + STATE = {'hf_cache': '/cache/hf', 'ollama': '/cache/ollama'} IMAGES = { 'vllm': 'vllm/vllm-openai:test', diff --git a/tests/test_leasing_route_registry.py b/tests/test_leasing_route_registry.py index b29d8574..c8b38f82 100644 --- a/tests/test_leasing_route_registry.py +++ b/tests/test_leasing_route_registry.py @@ -1,15 +1,17 @@ -"""Tests for the append-only LiteLLM route registry (static-superset mode). - -The registry accumulates the semantic route inputs (served name / engine / host) -of every catalog *and* live deployment ever merged across the runbooks that share -one state dir, and the gateway ``model_list`` is rendered from the whole registry. -The headline property under test: once every catalog has been merged once, a -converge under any runbook renders a byte-identical gateway config, so the gateway -is never recreated and no cross-catalog converge can strip another's live routes. - -Driven by the same stateful fake-docker seam as ``test_leasing_compose.py``; no -real docker, GPUs, or network. Converges run ``apply=False`` so only the render -half (which owns the registry read-merge-write) executes — deterministic and fast. +"""Gateway routes: derived at render, one layer per owner. + +The gateway's static ``model_list`` is derived at every render from the +published catalog union (every runbook's endpoints) and the placed +deployments, over the route registry, which now keeps only the routes of +deployments no published catalog defines (remembered past release, so +releasing one does not recreate the gateway) plus any rows from before. +The headline property: the rendered config depends on endpoint definitions, +not on which models are up or which runbook converged, so the gateway is not +recreated as models come and go. + +Driven by the same stateful fake-docker seam as ``test_leasing_compose.py``; +no real docker, GPUs, or network. Converges run ``apply=False``: only the +render half (which reads the registry and remembers rows) executes. """ from __future__ import annotations @@ -20,23 +22,25 @@ import yaml from infer_stack.hardware import simulate_inventory -from infer_stack.leasing import Catalog, ComposeBackend, render_compose +from infer_stack.leasing import Catalog, ComposeBackend from infer_stack.leasing.gateway import ( CONFIG_HASH_LABEL, LITELLM_CONFIG_FILENAME, LITELLM_REGISTRY_FILENAME, LITELLM_ROUTES_FILENAME, - _litellm_model_list, - _litellm_model_list_from_registry, - _merge_route_registry, - _registry_incoming_from_catalog, - _registry_incoming_from_deployments, + catalog_routes, + deployment_routes, + front_door_routes, + registry_routes, + remembered_rows, ) from infer_stack.leasing.models import ( RESERVED_ENGINE, Deployment, DeploymentState, ) +from infer_stack.leasing.profile import CatalogUnion +from infer_stack.leasing.routes import GatewayRoute, route_table STATE = {'hf_cache': '/cache/hf', 'ollama': '/cache/ollama'} IMAGES = { @@ -72,13 +76,27 @@ def vllm_ep(endpoint, *, served=None, t=0.0): ) -def _catalog(endpoint, *, model='m'): - return Catalog.from_dict( - { - 'models': {model: {'source': f'hf://org/{model}'}}, - 'endpoints': {endpoint: {'engine': 'vllm', 'model': model}}, - } - ) +def _catalog_dict(endpoint, *, model='m', served=None): + spec = {'engine': 'vllm', 'model': model} + if served: + spec['served_name'] = served + return {'models': {model: {'source': f'hf://org/{model}'}}, + 'endpoints': {endpoint: spec}} + + +def _catalog(endpoint, *, model='m', served=None): + return Catalog.from_dict(_catalog_dict(endpoint, model=model, served=served)) + + +def _write_registry(tmp_path, entries, *, version=1): + path = tmp_path / LITELLM_REGISTRY_FILENAME + path.write_text(json.dumps({'version': version, 'entries': entries}, + sort_keys=True, indent=2) + '\n') + return path + + +def _litellm_service(tmp_path): + return yaml.safe_load((tmp_path / 'docker-compose.yml').read_text())['services']['litellm'] class FakeDocker: @@ -135,293 +153,203 @@ def capture_warnings(): logger.disable('infer_stack') -# -- 1. merge idempotence -------------------------------------------------- +# -- 1. precedence: registry < catalog < deployment < upstream ------------- -def test_merge_idempotent_byte_identical(tmp_path): - cat = _catalog('alpha') - be = make_backend(tmp_path, catalog=cat) - be.converge([vllm_ep('alpha')], apply=False) - first = (tmp_path / LITELLM_REGISTRY_FILENAME).read_bytes() - be.converge([vllm_ep('alpha')], apply=False) - second = (tmp_path / LITELLM_REGISTRY_FILENAME).read_bytes() - assert first == second # merging the same inputs twice is a no-op +def test_route_table_precedence(): + old = GatewayRoute('a', 'openai', 'old', 'http://old/v1', origin='registry') + cat = GatewayRoute('a', 'openai', 'cat', 'http://cat/v1') + dep = GatewayRoute('a', 'openai', 'dep', 'http://dep/v1', origin='deployment') + up = GatewayRoute('a', 'openai', 'up', 'http://up/v1', origin='upstream') + assert route_table([old], [cat], [dep], [up]) == [up] + assert route_table([up], [old]) == [old] # later layer wins + assert [r.alias for r in route_table([cat], [old.__class__('b', 'openai', 'x', 'y')])] == ['a', 'b'] -def test_merge_incoming_wins_on_conflict_pure(): - existing = {'version': 1, 'entries': {'a': {'engine': 'vllm', 'served': 'a'}}} - merged, warnings = _merge_route_registry( - existing, {'a': {'engine': 'vllm', 'served': 'a2'}} - ) - assert merged['entries']['a'] == {'engine': 'vllm', 'served': 'a2'} - assert warnings and 'redefined' in warnings[0] - - -def test_merge_additive_never_removes_pure(): - existing = {'version': 1, 'entries': {'a': {'engine': 'vllm', 'served': 'a'}}} - merged, _ = _merge_route_registry( - existing, {'b': {'engine': 'vllm', 'served': 'b'}} - ) - assert set(merged['entries']) == {'a', 'b'} - - -# -- 2. hash stability across catalog alternation (the headline property) --- - - -def test_hash_stable_across_alternation(tmp_path): - """Converge catalog A, then B, then A again on one shared state dir. Renders - 2 and 3 must produce a byte-identical ``litellm_config.yaml`` AND an equal - litellm service dict / CONFIG_HASH_LABEL — the label is what drives recreate. - """ - be_a = make_backend(tmp_path, catalog=_catalog('alpha')) - be_b = make_backend(tmp_path, catalog=_catalog('beta')) - - be_a.converge([vllm_ep('alpha')], apply=False) # render 1: {alpha} - be_b.converge([vllm_ep('beta')], apply=False) # render 2: {alpha, beta} - cfg_2 = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() - svc_2 = yaml.safe_load( - (tmp_path / 'docker-compose.yml').read_text() - )['services']['litellm'] - - be_a.converge([vllm_ep('alpha')], apply=False) # render 3: still {alpha, beta} - cfg_3 = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() - svc_3 = yaml.safe_load( - (tmp_path / 'docker-compose.yml').read_text() - )['services']['litellm'] - - assert cfg_2 == cfg_3 # gateway config byte-stable across the alternation - assert svc_2 == svc_3 - assert ( - svc_2['labels'][CONFIG_HASH_LABEL] == svc_3['labels'][CONFIG_HASH_LABEL] - ) +def test_catalog_endpoints_are_not_stored(tmp_path): + """The published union is the one store of endpoint definitions: a render + derives their routes, and the registry keeps none of them.""" + be = make_backend(tmp_path, catalog=_catalog('alpha')) + be.converge([vllm_ep('alpha')], apply=False) + assert _aliases(tmp_path) == ['alpha'] + assert not (tmp_path / LITELLM_REGISTRY_FILENAME).exists() -# -- 3. union correctness -------------------------------------------------- +# -- 2. byte stability (the headline property) ------------------------------ -def test_union_routes_both_catalogs_sorted(tmp_path): - be_a = make_backend(tmp_path, catalog=_catalog('alpha')) - be_b = make_backend(tmp_path, catalog=_catalog('beta')) - be_a.converge([vllm_ep('alpha')], apply=False) - be_b.converge([vllm_ep('beta')], apply=False) +def test_config_is_byte_stable_as_models_come_and_go(tmp_path): + union = CatalogUnion.from_sources([_catalog_dict('alpha'), _catalog_dict('beta')]) + be = make_backend(tmp_path, catalog=union) + be.converge([vllm_ep('alpha')], apply=False) + cfg_1 = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() + svc_1 = _litellm_service(tmp_path) + be.converge([vllm_ep('beta')], apply=False) + be.converge([], apply=False) + assert (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() == cfg_1 + assert _litellm_service(tmp_path)['labels'][CONFIG_HASH_LABEL] == \ + svc_1['labels'][CONFIG_HASH_LABEL] assert _aliases(tmp_path) == ['alpha', 'beta'] -# -- 4. live non-catalog deployment persists across a foreign converge ------ - - -def test_live_non_catalog_deployment_stays_routed(tmp_path): - """A deployment absent from the invoking catalog is merged from the desired - set and remains routed on a later converge under a different catalog.""" - be_a = make_backend(tmp_path, catalog=_catalog('alpha')) - # 'extra' is not in catalog A — it enters the registry via the desired set. - be_a.converge([vllm_ep('alpha'), vllm_ep('extra')], apply=False) - assert 'extra' in _aliases(tmp_path) +def test_hash_stable_across_runbook_alternation(tmp_path): + """Runbook A, then B, then A again on one ledger: the published union holds + both catalogs, so renders 2 and 3 are byte-identical.""" + from test_leasing_profile import cat, controller - be_b = make_backend(tmp_path, catalog=_catalog('beta')) - be_b.converge([vllm_ep('beta')], apply=False) # 'extra' no longer live - # Persisted past its live window and past a foreign converge. - assert _aliases(tmp_path) == ['alpha', 'beta', 'extra'] + a, b = Catalog.from_dict(cat('alpha')), Catalog.from_dict(cat('beta')) + _, ctl_a = controller(tmp_path, catalog=a) + ctl_a.acquire('a', a.resolve_names(['alpha']), wait=False) + _, ctl_b = controller(tmp_path, catalog=b) + ctl_b.acquire('b', b.resolve_names(['beta']), wait=False) + config = tmp_path / 'state' / LITELLM_CONFIG_FILENAME + cfg_2 = config.read_bytes() + _, ctl_a2 = controller(tmp_path, catalog=a) + ctl_a2.acquire('a2', a.resolve_names(['alpha']), wait=False) + assert config.read_bytes() == cfg_2 + assert sorted(e['model_name'] for e in yaml.safe_load(cfg_2)['model_list']) == \ + ['alpha', 'beta'] -# -- 5. conflict: incoming wins, warning, bytes change --------------------- +# -- 3. a non-catalog deployment is remembered past its release ------------- -def test_conflict_incoming_wins_and_changes_bytes(tmp_path): +def test_live_non_catalog_deployment_stays_routed(tmp_path): + """A deployment no published catalog defines is remembered in the registry, + so it stays routed after release and releasing it does not recreate the + gateway.""" be = make_backend(tmp_path, catalog=_catalog('alpha')) - be.converge([vllm_ep('alpha', served='v1')], apply=False) + be.converge([vllm_ep('alpha'), vllm_ep('extra')], apply=False) before = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() + assert set(_registry(tmp_path)['entries']) == {'extra'} # alpha is derived + be.converge([vllm_ep('alpha')], apply=False) # 'extra' released + assert (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() == before + assert _aliases(tmp_path) == ['alpha', 'extra'] - with capture_warnings() as warnings: - be.converge([vllm_ep('alpha', served='v2')], apply=False) - after = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() - - assert before != after # a genuinely changed definition => one recreate - assert _registry(tmp_path)['entries']['alpha']['served'] == 'v2' - assert any('redefined' in m for m in warnings) - - -# -- 6. seeding from a pre-existing litellm_config.yaml (upgrade migration) - - - -def test_seed_from_existing_config_preserves_vllm_routes(tmp_path): - """A state dir upgraded in place: a rendered ``litellm_config.yaml`` exists - but no registry. The first converge under a *disjoint* catalog must still - route the old vLLM aliases (recovered by seeding). Ollama rows are skipped.""" - legacy = { - 'model_list': [ - { - 'model_name': 'old-vllm', - 'litellm_params': { - 'model': 'openai/old-served', - 'api_base': 'http://vllm-old-served:8000/v1', - 'api_key': 'EMPTY', - }, - }, - { - 'model_name': 'old-ollama', - 'litellm_params': { - 'model': 'ollama/llama3:8b', - 'api_base': 'http://ollama-gpuhost:11434', - }, - }, - ] - } - (tmp_path / LITELLM_CONFIG_FILENAME).write_text(yaml.safe_dump(legacy)) - assert not (tmp_path / LITELLM_REGISTRY_FILENAME).exists() - be = make_backend(tmp_path, catalog=_catalog('beta')) - be.converge([vllm_ep('beta')], apply=False) +def test_a_published_definition_wins_over_a_registry_row(tmp_path): + """A registry row (e.g. one an older binary wrote for a catalog endpoint) + is the lowest layer: the published definition renders instead.""" + _write_registry(tmp_path, {'alpha': {'engine': 'vllm', 'served': 'v1'}}) + be = make_backend(tmp_path, catalog=_catalog('alpha', served='v2')) + be.converge([], apply=False) + (entry,) = _model_list(tmp_path) + assert entry['litellm_params']['model'] == 'openai/v2' - aliases = _aliases(tmp_path) - assert 'old-vllm' in aliases # vLLM row recovered exactly - assert 'beta' in aliases # the new catalog merged in - assert 'old-ollama' not in aliases # Ollama row skipped (host not invertible) +# -- 4. registry rows from an older binary are still routed ----------------- -# -- 7. corrupt registry: fail-open, rebuilt from seed + catalog ----------- + +def test_catalog_less_converge_renders_existing_registry_rows(tmp_path): + """An upgraded state dir whose registry holds catalog rows: a catalog-less + converge still routes them (nothing is lost on upgrade), and the file is + not rewritten.""" + _write_registry(tmp_path, {'alpha': {'engine': 'vllm', 'served': 'alpha'}, + 'tiny': {'engine': 'ollama', 'model': 'tinyllama', + 'host': 'gpuhost'}}) + original = (tmp_path / LITELLM_REGISTRY_FILENAME).read_bytes() + bare = make_backend(tmp_path, catalog=None) + bare.converge([], apply=False) + assert _aliases(tmp_path) == ['alpha', 'tiny'] + assert (tmp_path / LITELLM_REGISTRY_FILENAME).read_bytes() == original -def test_corrupt_registry_is_rebuilt(tmp_path): +def test_corrupt_registry_is_ignored(tmp_path): (tmp_path / LITELLM_REGISTRY_FILENAME).write_text('{ this is not json') be = make_backend(tmp_path, catalog=_catalog('alpha')) be.converge([vllm_ep('alpha')], apply=False) # must not raise assert _aliases(tmp_path) == ['alpha'] - # A non-map ``entries`` is likewise structurally unusable -> rebuilt. (tmp_path / LITELLM_REGISTRY_FILENAME).write_text('{"entries": [1, 2, 3]}') be.converge([vllm_ep('alpha')], apply=False) assert _aliases(tmp_path) == ['alpha'] -# -- 8. dynamic-routing isolation ------------------------------------------ +def test_unknown_version_is_read_and_never_rewritten(tmp_path): + """A registry from a newer schema is rendered as-is with a warning and + never rewritten, even when a render has a row to remember.""" + path = _write_registry(tmp_path, {'old': {'engine': 'vllm', 'served': 'old'}}, + version=99) + original = path.read_bytes() + be = make_backend(tmp_path, catalog=_catalog('alpha')) + with capture_warnings() as warnings: + be.converge([vllm_ep('alpha'), vllm_ep('extra')], apply=False) + assert path.read_bytes() == original + assert _aliases(tmp_path) == ['alpha', 'extra', 'old'] + assert any('unknown schema version' in m for m in warnings) + + +# -- 5. dynamic routing ------------------------------------------------------ def test_dynamic_routing_creates_no_registry(tmp_path): be = make_backend(tmp_path, dynamic_routing=True) be.converge([vllm_ep('alpha')], apply=False) assert not (tmp_path / LITELLM_REGISTRY_FILENAME).exists() - # dynamic routing still renders its own desired route set file. assert (tmp_path / LITELLM_ROUTES_FILENAME).exists() -# -- 11. catalog-less converge renders from the accumulated registry -------- +# -- 6. derivations agree ---------------------------------------------------- -def test_catalog_less_converge_renders_full_registry(tmp_path): - """Seed a registry via a catalog-full converge, then converge with - ``catalog=None`` (as a bare release/gc does). The render must still contain - the full registry (no fall-through to the legacy empty-when-no-deployments - branch), bytes unchanged.""" - be = make_backend(tmp_path, catalog=_catalog('alpha')) - be.converge([vllm_ep('alpha')], apply=False) - before = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() - - bare = make_backend(tmp_path, catalog=None) - bare.converge([], apply=False) # no catalog, no live deployments - after = (tmp_path / LITELLM_CONFIG_FILENAME).read_bytes() - - assert _aliases(tmp_path) == ['alpha'] # NOT [] -> registry, not legacy - assert before == after # byte-stable: no strip, no blip - - -# -- 12. engine filter: RESERVED_ENGINE contributes no row ------------------ - - -def test_reserved_engine_contributes_no_row(): +def test_reserved_engine_routes_nowhere(): reserved = Deployment( 'grp-r', 'ck-r', RESERVED_ENGINE, 'dedicated', {}, {'engine': RESERVED_ENGINE, 'reserved_gpu_count': 1}, {'reserved-gpu': {}}, DeploymentState.LIVE, 0.0, 0.0, ) - incoming = _registry_incoming_from_deployments( - [reserved, vllm_ep('alpha')], {'grp-r': [0], 'grp-alpha': [1]} - ) - assert set(incoming) == {'alpha'} # reserved skipped, mirrors render loop - - -# -- 13. multi-alias deployment: two rows, one shared upstream -------------- + placed = {'grp-r': [0], 'grp-alpha': [1]} + assert [r.alias for r in deployment_routes([reserved, vllm_ep('alpha')], placed)] == ['alpha'] + assert set(remembered_rows([reserved, vllm_ep('alpha')], placed, defined=set())) == {'alpha'} -def test_multi_alias_deployment_matches_legacy_render(): - dep = Deployment( +def test_remembered_rows_render_back_to_the_live_route(): + """A remembered row renders exactly the route its deployment had, so a + release never moves the rendered bytes (vLLM, multi-alias, and Ollama).""" + multi = Deployment( 'grp-m', 'ck-m', 'vllm', 'shared-compatible', {}, - { - 'engine': 'vllm', - 'hf_model_id': 'org/model', - 'served_model_name': 'shared', - 'runtime': {'tensor_parallel_size': 1}, - 'reclaim': 'keep-warm', - }, - { - 'ep1': {'served_model_name': 'shared'}, - 'ep2': {'served_model_name': 'shared'}, - }, + {'engine': 'vllm', 'hf_model_id': 'org/model', 'served_model_name': 'shared', + 'runtime': {'tensor_parallel_size': 1}, 'reclaim': 'keep-warm'}, + {'ep1': {'served_model_name': 'shared'}, 'ep2': {'served_model_name': 'shared'}}, DeploymentState.LIVE, 0.0, 0.0, ) - assignments = {'grp-m': [0]} - incoming = _registry_incoming_from_deployments([dep], assignments) - assert incoming == { - 'ep1': {'engine': 'vllm', 'served': 'shared'}, - 'ep2': {'engine': 'vllm', 'served': 'shared'}, - } - registry = {'version': 1, 'entries': incoming} - rendered = _litellm_model_list_from_registry(registry) - legacy = _litellm_model_list([dep], assignments) - assert sorted(rendered, key=lambda e: e['model_name']) == sorted( - legacy, key=lambda e: e['model_name'] + tiny = Deployment( + 'grp-o', 'ck-o', 'ollama', 'shared-compatible', {}, + {'engine': 'ollama', 'host': 'gpuhost'}, {'tiny': {'model': 'tinyllama'}}, + DeploymentState.LIVE, 0.0, 0.0, ) + placed = {'grp-m': [0], 'grp-o': [1]} + live = deployment_routes([multi, tiny], placed) + rows = remembered_rows([multi, tiny], placed, defined=set()) + assert registry_routes({'entries': rows}) == sorted(live, key=lambda r: r.alias) + upstream = GatewayRoute('k', 'openai', 'model-k', 'http://kubeai/openai/v1', + origin='upstream') + rows = remembered_rows([], {}, defined=set(), extra=[upstream]) + assert registry_routes({'entries': rows}) == [upstream] -# -- 14. unknown-version tolerance ----------------------------------------- - - -def test_unknown_version_preserved_not_reseeded(tmp_path): - """A registry from a newer schema (``version: 99``) with a valid ``entries`` - map is rendered as-is with a warning and NOT rewritten/reseeded — a binary - rollback must not discard the accumulated union.""" - registry = { - 'version': 99, - 'entries': {'alpha': {'engine': 'vllm', 'served': 'alpha'}}, - } - path = tmp_path / LITELLM_REGISTRY_FILENAME - path.write_text(json.dumps(registry, sort_keys=True, indent=2) + '\n') - original = path.read_bytes() +def test_catalog_and_live_routes_coincide(): + assert catalog_routes(_catalog('alpha')) == deployment_routes( + [vllm_ep('alpha')], {'grp-alpha': [0]}) - # Converge with the same catalog+deployment already in the registry, so the - # merge is a no-op and the file must not be rewritten (version stays 99). - be = make_backend(tmp_path, catalog=_catalog('alpha')) - with capture_warnings() as warnings: - be.converge([vllm_ep('alpha')], apply=False) - assert path.read_bytes() == original # not rewritten/reseeded - assert _registry(tmp_path)['version'] == 99 - assert 'alpha' in _aliases(tmp_path) - assert any('unknown schema version' in m for m in warnings) +def test_dynamic_routes_are_one_per_managed_id(): + external = Catalog.from_dict({'endpoints': {'box': {'external': { + 'api_base': 'http://box:9000/v1', 'model': 'Org/Big', 'api_key_env': 'BOX_KEY'}}}}) + static, dynamic = front_door_routes([vllm_ep('alpha')], {'grp-alpha': [0]}, + catalog=external, dynamic=True) + assert static == [] + assert [r.alias for r in dynamic] == ['alpha', 'box'] + assert len({r.route_id for r in dynamic}) == 2 -# -- 10. concurrency smoke: two backends, one state dir, no lost update ----- +# -- 7. concurrency smoke ---------------------------------------------------- def test_concurrency_smoke_no_lost_update(tmp_path): - """Two backends sharing one state dir converge different catalogs (serialized - by the per-state-dir flock the converge already takes). The final registry - file must contain BOTH catalogs' routes — neither converge clobbers the - other's contribution.""" + """Two backends sharing one state dir remember different ad-hoc + deployments (serialized by the converge flock): both rows survive.""" be_a = make_backend(tmp_path, catalog=_catalog('alpha')) be_b = make_backend(tmp_path, catalog=_catalog('beta')) - be_a.converge([vllm_ep('alpha')], apply=False) - be_b.converge([vllm_ep('beta')], apply=False) - entries = _registry(tmp_path)['entries'] - assert set(entries) == {'alpha', 'beta'} - - -# -- extra: catalog vs live reduce to the identical row -------------------- - - -def test_catalog_and_live_rows_coincide(): - cat = _catalog('alpha') - from_cat = _registry_incoming_from_catalog(cat) - from_live = _registry_incoming_from_deployments( - [vllm_ep('alpha')], {'grp-alpha': [0]} - ) - assert from_cat == from_live == {'alpha': {'engine': 'vllm', 'served': 'alpha'}} + be_a.converge([vllm_ep('one')], apply=False) + be_b.converge([vllm_ep('two')], apply=False) + assert set(_registry(tmp_path)['entries']) == {'one', 'two'} From 2ac9a47dd44ca0b6c1a8871c95d664524bc3a1b3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:12:30 -0400 Subject: [PATCH 113/141] External keys by name on both gateways; a real-LiteLLM external e2e The host gateway lists each routed key variable in LiteLLM's environment (NAME: ${NAME}), so the fingerprint recreates it when the value changes; the in-cluster gateway's Secret carries them and its key-hash covers them (unchanged when there are none). `env NAME=...` says the gateway uses a published key after `apply`; `routes seed` refuses a key with no value and a render warns. profile_drift compared catalog sources by digest, so once catalogs merged every command warned of drift; it compares endpoint meaning now. dev/external_e2e.sh runs a real LiteLLM against the mock OpenAI server, static and dynamic, through a key rotation. Passes on the guest. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 7 ++ dev/external_e2e.sh | 161 +++++++++++++++++++++++++ dev/journals/claude.md | 15 +++ docs/queue.md | 21 +++- infer_stack/backends/kubeai_gateway.py | 38 +++++- infer_stack/cli/commands_leasing.py | 18 +++ infer_stack/leasing/compose.py | 2 + infer_stack/leasing/controller.py | 23 +++- infer_stack/leasing/endpoints.py | 20 +++ infer_stack/leasing/gateway.py | 35 +++++- infer_stack/leasing/profile.py | 45 ++++++- tests/test_external_endpoints.py | 149 +++++++++++++++++++++++ 12 files changed, 512 insertions(+), 22 deletions(-) create mode 100755 dev/external_e2e.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 3536f731..3672b342 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,13 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). `litellm_config.yaml`. - With dynamic routing, an external endpoint's route has a stable id from its alias, so pointing it at another server replaces one route. +- An external key reaches the gateway by name: the host gateway's + environment lists it (a changed value recreates the gateway on `apply`), + the in-cluster gateway's Secret holds it (a change rolls the pod). `env + NAME=...` says when the gateway picks it up; `routes seed` refuses a key + with no value. +- A runbook whose catalog is already published no longer warns that its + catalogs differ from the recovery snapshot. ### Hardening from a review of the backend-parity work diff --git a/dev/external_e2e.sh b/dev/external_e2e.sh new file mode 100755 index 00000000..c2ceb47b --- /dev/null +++ b/dev/external_e2e.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# End-to-end test for external endpoints against a REAL LiteLLM gateway. +# +# No GPU, no model weights: the compose backend runs only its front door, and +# the "external" server is infer-stack's own mock OpenAI server on this host, +# which accepts one bearer key and records every request. Runs on the guest +# VM (docker, the pinned litellm and postgres images). +# +# ./dev/external_e2e.sh +# +# For each routing mode (static, then dynamic): +# 1. an external endpoint whose key has no value is refused before publishing; +# 2. `routes seed` publishes it and the gateway routes the alias to the mock, +# sending the key from the managed .env by name (`os.environ/NAME`); +# 3. the key changes: `env` says an apply is needed, the old key is refused +# by the upstream until `apply`, the new key is sent after it; +# 4. `routes prune` unpublishes it and the alias stops routing. +# +# Knobs (env): +# E2E_MODES routing modes to run (default "static dynamic") +# E2E_PORT the mock's port (default 18911) +set -euo pipefail + +MODES="${E2E_MODES:-static dynamic}" +PORT="${E2E_PORT:-18911}" +HOST_IP="$(ip -4 -o addr show docker0 | awk '{print $4}' | cut -d/ -f1)" +[ -n "$HOST_IP" ] || { echo 'no docker0 address' >&2; exit 1; } + +fail() { echo "FAIL: $*" >&2; exit 1; } + +rm_work() { # Postgres owns its data dir as its own uid + [ -d "$1/data" ] && docker run --rm -v "$1/data:/d" --entrypoint sh postgres:16.8 \ + -c 'rm -rf /d/postgres-litellm /d/postgres-open-webui' >/dev/null 2>&1 + rm -rf "$1" +} + +MOCK_PID='' +start_mock() { # start_mock KEY + stop_mock + cat > "$WORK/mock.yaml" <"$WORK/mock.log" 2>&1 & + MOCK_PID=$! + for _ in $(seq 50); do + curl -sf "http://127.0.0.1:$PORT/health" >/dev/null && return 0 + sleep 0.2 + done + fail "mock server did not start: $(cat "$WORK/mock.log")" +} +stop_mock() { + if [ -n "$MOCK_PID" ]; then kill "$MOCK_PID" 2>/dev/null || true; wait "$MOCK_PID" 2>/dev/null || true; fi + MOCK_PID='' +} + +last_bearer() { # the Authorization the mock last received + curl -sf "http://127.0.0.1:$PORT/__mock__/requests" | python -c ' +import json, sys +reqs = [r for r in json.load(sys.stdin)["requests"] + if r.get("path", "").endswith("/chat/completions")] +headers = {k.lower(): v for k, v in reqs[-1]["headers"].items()} if reqs else {} +print(headers.get("authorization", ""))' +} + +chat() { # chat ALIAS -> http status; body in $WORK/chat.json + local base key + base="$(run_is env OPENAI_BASE_URL)" + key="$(run_is env LITELLM_MASTER_KEY)" + curl -s -o "$WORK/chat.json" -w '%{http_code}' "$base/chat/completions" \ + -H "Authorization: Bearer $key" -H 'Content-Type: application/json' \ + -d "{\"model\": \"$1\", \"max_tokens\": 8, + \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}]}" || true +} + +chat_until() { # chat_until ALIAS STATUS: routes settle after apply + local got='' + for _ in $(seq 60); do + got="$(chat "$1")" + [ "$got" = "$2" ] && return 0 + sleep 2 + done + fail "chat $1: wanted HTTP $2, got $got: $(cat "$WORK/chat.json")" +} + +run_mode() { + local mode=$1 + WORK="$(mktemp -d /tmp/infer-stack-external-e2e.XXXXXX)" + IS_ENV="INFER_STACK_CONFIG_DIR=$WORK/config INFER_STACK_DATA_DIR=$WORK/data" + run_is() { env $IS_ENV infer-stack "$@"; } + echo "== [$mode] work dir: $WORK" + mkdir -p "$WORK/config" + run_is config set backend compose >/dev/null + if [ "$mode" = dynamic ]; then run_is config set dynamic_routing true >/dev/null; fi + # The runbook's own catalog: nothing external. + cat > "$WORK/config/catalog.yaml" < "$WORK/external.yaml" <"$WORK/out" 2>&1; then + fail 'seed without the key succeeded' + fi + grep -q 'infer-stack env E2E_REMOTE_KEY=' "$WORK/out" || fail "$(cat "$WORK/out")" + + echo "== [$mode] 2. publish; the gateway sends the key by name" + run_is env E2E_REMOTE_KEY=key-one >/dev/null + run_is routes seed "$WORK/external.yaml" --yes + run_is routes list + run_is routes list --json | grep -q '"origin": "external"' || fail 'not listed as external' + if grep -rq 'key-one' "$WORK/data/leasing/compose/docker-compose.yml"; then + fail 'the key value is in the compose file' + fi + chat_until remote 200 + [ "$(last_bearer)" = 'Bearer key-one' ] || fail "sent $(last_bearer)" + + echo "== [$mode] 3. the key changes" + start_mock key-two # the upstream rotated + chat_until remote 401 + run_is env E2E_REMOTE_KEY=key-two | tee "$WORK/out" + grep -q 'after `infer-stack apply`' "$WORK/out" || fail 'env did not say apply' + run_is apply --yes + chat_until remote 200 + [ "$(last_bearer)" = 'Bearer key-two' ] || fail "sent $(last_bearer)" + + echo "== [$mode] 4. prune unpublishes it" + run_is routes prune --yes + if run_is routes list --json | grep -q '"remote"'; then fail 'still listed after prune'; fi + chat_until remote 400 + + run_is stack down >/dev/null 2>&1 || true + stop_mock + rm_work "$WORK" + echo "== [$mode] PASS" +} + +cleanup() { + status=$? + set +e + stop_mock + if [ -n "${WORK:-}" ] && [ -d "$WORK" ]; then + env $IS_ENV infer-stack stack down >/dev/null 2>&1 + rm_work "$WORK" + fi + exit $status +} +trap cleanup EXIT + +for mode in $MODES; do run_mode "$mode"; done +echo '== external endpoints e2e: PASS' diff --git a/dev/journals/claude.md b/dev/journals/claude.md index 470c77ac..17cf0210 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3410,3 +3410,18 @@ gateway byte-stable across an ad-hoc release. (2) Tests that called `render_compose` with deployments that serve no catalog endpoint were testing a branch production never took; making the deployments serve the catalog's aliases made them test the real property. + +## 2026-09-27 — items 33-34: external routes, keys by name, a real-LiteLLM e2e + +**Did.** Survival and replacement tests for external routes; keys by name +on both gateways (host env interpolation + fingerprint, in-cluster Secret + +key-hash); `env` says when a key takes effect; `routes seed` refuses a key +with no value. `dev/external_e2e.sh` drives a real LiteLLM against the mock +OpenAI server on the guest, static and dynamic, through a key rotation. + +**Found by the e2e, not the unit tests.** After `routes seed`, every command +warned that the catalogs drifted: `profile_drift` compared sources by digest, +and merging rewrites the union's sources. It now compares endpoint meaning. +Also: LiteLLM's upstream request lowercases `authorization`, and Postgres +leaves its data dir owned by its own uid (the script removes it through a +container). diff --git a/docs/queue.md b/docs/queue.md index 247080c1..a5303eba 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1183,10 +1183,12 @@ publication mechanisms (static registry vs dynamic reconciliation), not one storage for everything. Delete `render_front_door` branches that only tests reach (cleanup B), never a supported mode. -### 33. [ ] External routes in static and dynamic routing -*Partly done with 32:* static and dynamic routes, the `('external', alias)` -id, the key name in `model_info` for reconcile. Left: survival tests (gc, -release, dedicated going away) and retirement through a real reconcile. +### 33. [x] External routes in static and dynamic routing +*Done 2026-09-27* (with 32): static and dynamic routes on every backend, the +`('external', alias)` id, the key name in `model_info` for reconcile. Tests: +the route survives dedicated deployments coming and going, zero deployments, +release and gc; redefinition replaces one route in place; unpublishing +deletes it (`tests/test_external_endpoints.py`). Dynamic mode: an external route is standing desired state with a stable managed id from its logical owner, the endpoint alias (decision 28.5), not from target semantics; the semantic route decides whether it is replaced. It survives zero leases, @@ -1194,7 +1196,16 @@ releases, other acquires, gc, a dedicated deployment going away. Route retirement (item 17) stays intact; infer-stack never tears down an external upstream, only its route. -### 34. [ ] External credentials by env reference +### 34. [x] External credentials by env reference +*Done 2026-09-27.* Host gateway: `NAME: ${NAME}` in LiteLLM's environment, +so the fingerprint recreates it when the value changes. In-cluster: the +Secret carries each routed key, and the pod's key-hash covers them (the hash +is unchanged when there are none). `env NAME=...` says the gateway uses it +after `infer-stack apply`. `routes seed` refuses a key with no value, and a +render warns. Real LiteLLM: `dev/external_e2e.sh` (static and dynamic, key +rotation against a mock that accepts one key) passes on the guest. +Also fixed: `profile_drift` compared catalog sources by bytes, so after a +seed every command warned "catalogs" drifted; it now compares meaning. `api_key_env` only; never a literal key in the catalog, registry, rendered route or logs. Render as a LiteLLM env reference; the gateway container receives the variable from the managed `.env` (`infer-stack env KEY=...`). diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index 5e0ab3be..3dee1ec2 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -36,6 +36,7 @@ Gateway, remembered_rows, render_front_door, + route_key_envs, ) NAME = 'infer-stack-gateway' @@ -159,6 +160,13 @@ def access(self, endpoints: list[str]) -> dict[str, Any] | None: def catalog_routes(self, catalog) -> list[GatewayRoute]: return [] # the kubeai backend supplies its own routes + def _secret_values(self, routes: list[GatewayRoute]) -> dict[str, str]: + """What the Secret holds: the master key, and each routed external + key that has a value (never written into the manifests).""" + env = self.gateway.managed_env() + keys = {n: env[n] for n in route_key_envs(routes) if env.get(n)} + return {API_KEY_ENV: self.master_key(), **keys} + def static_routes(self) -> list[GatewayRoute]: """The static route table: the registry under the backend's routes.""" return route_table(self.gateway.registry_routes(), self.extra_routes) @@ -176,7 +184,9 @@ def _render_documents(self) -> dict[Path, str]: dynamic_routing=False, ) config = front.litellm_config or '' - key_hash = hashlib.sha256(self.master_key().encode()).hexdigest()[:12] + routes = self.static_routes() + self.gateway.warn_missing_keys(routes) + key_hash = secret_hash(self._secret_values(routes)) docs = gateway_manifests( namespace=self.namespace, image=self.images['litellm'], config=config, key_hash=key_hash, node_port=self.node_port) @@ -212,7 +222,8 @@ def apply(self) -> None: if not self.manifests_file.exists(): return - secret = yaml.safe_dump(secret_manifest(self.namespace, self.master_key())) + secret = yaml.safe_dump(secret_manifest( + self.namespace, self._secret_values(self.static_routes()))) path = self.state_dir / SECRET_FILENAME fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, 'w') as handle: @@ -275,11 +286,28 @@ def use_profile(self, profile: dict[str, Any]) -> None: self.images['litellm'] = profile.get('image') or self.images['litellm'] -def secret_manifest(namespace: str, key: str) -> dict[str, Any]: - """The master key as a Secret (applied, never rendered into a diff).""" +def secret_manifest(namespace: str, values: dict[str, str]) -> dict[str, Any]: + """The gateway's secrets as a Secret (applied, never rendered into a diff): + the master key, and the external keys its routes send by name.""" return {'apiVersion': 'v1', 'kind': 'Secret', 'metadata': {'name': NAME, 'namespace': namespace}, - 'type': 'Opaque', 'stringData': {API_KEY_ENV: key}} + 'type': 'Opaque', 'stringData': dict(values)} + + +def secret_hash(values: dict[str, str]) -> str: + """The pod-template annotation that rolls the gateway when a Secret value + changes. With only the master key it is that key's hash, as before. + + >>> secret_hash({API_KEY_ENV: 'k'}) == hashlib.sha256(b'k').hexdigest()[:12] + True + >>> secret_hash({API_KEY_ENV: 'k', 'X': '1'}) != secret_hash({API_KEY_ENV: 'k', 'X': '2'}) + True + """ + extra = {k: v for k, v in values.items() if k != API_KEY_ENV} + material = values.get(API_KEY_ENV, '') + if extra: + material += '\x00' + json.dumps(extra, sort_keys=True) + return hashlib.sha256(material.encode()).hexdigest()[:12] def gateway_manifests(*, namespace: str, image: str, config: str, key_hash: str, diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index 4af0aef6..d34825cf 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2443,6 +2443,19 @@ def _test_fail(config, base_url: str, reason: str) -> int: return 1 +def _published_key_users(ledger, key: str) -> list[str]: + """The published external endpoints that send ``key`` as their key.""" + from ..leasing.endpoints import key_references + from ..leasing.profile import CatalogUnion + + sources = (ledger.profile() or {}).get('catalogs') or [] + try: + union = CatalogUnion.from_sources(sources) if sources else None + except CatalogError: + return [] + return key_references(union).get(key, []) + + class EnvCLI(_PathOverridesMixin): """The managed env-file: print its path, read a value, or set one. @@ -2529,6 +2542,11 @@ def main(cls, argv=True, **kwargs): else: write_env_file(env_path, {key: value}) print(f'set {key} ({env_path})') + users = _published_key_users(ledger, key) + if users: + # A file write, not a publication: say when it takes effect. + print(f'{", ".join(users)} send(s) ${key} as the upstream key; ' + 'the gateway uses the new value after `infer-stack apply`') return 0 # Path first and foremost (it may not exist yet — that's fine). diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 7288bb1a..1705106c 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1516,6 +1516,8 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A self._ui_root_noted = True logger.info('Open WebUI runs as root: {}', why) routes, dynamic_routes = self._front_door_routes(desired, plan.assignments) + if self.litellm: + self.gateway.warn_missing_keys([*routes, *dynamic_routes]) rendered = render_compose( desired, plan.assignments, images=self.images, ports=self.ports, state=self.state, litellm=self.litellm, litellm_port=self.litellm_port, diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index e45e2bba..4f0986e3 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1893,13 +1893,32 @@ def plan_route_seed(self, catalogs: Iterable) -> RoutePlan: from .profile import CatalogUnion, catalog_sources from .routes import plan_seed - self._route_gateway() + gateway = self._route_gateway() sources = [s for c in catalogs for s in catalog_sources(c)] - incoming = self._meanings(CatalogUnion.from_sources(sources) if sources else None) + union = CatalogUnion.from_sources(sources) if sources else None + self._require_keys(gateway, union) + incoming = self._meanings(union) plan = plan_seed(self._published_meanings(), incoming) plan.sources = sources return plan + @staticmethod + def _require_keys(gateway, catalog) -> None: + """Refuse to publish an external endpoint whose key variable has no + value in the managed ``.env`` (docs/planning/external-endpoints.md, + decision 4): the gateway would send an empty key.""" + from .endpoints import key_references + from .profile import ProfileMismatch + + refs = key_references(catalog) + missing = gateway.missing_keys(sorted(refs)) + if missing: + name = missing[0] + raise ProfileMismatch( + f'endpoint {refs[name][0]!r} sends ${name} as its key, which ' + f'{gateway._env_path} does not set; set it first: ' + f'`infer-stack env {name}=...`') + def commit_route_seed(self, plan: RoutePlan, *, replace: bool = False ) -> tuple[RoutePlan, ReconcileResult]: """Merge the plan's catalogs into the published union and publish; diff --git a/infer_stack/leasing/endpoints.py b/infer_stack/leasing/endpoints.py index 4479758e..13a5f292 100644 --- a/infer_stack/leasing/endpoints.py +++ b/infer_stack/leasing/endpoints.py @@ -118,6 +118,26 @@ class ExternalTarget: kind = 'external' +def key_references(catalog: Any) -> dict[str, list[str]]: + """``{variable: [endpoint alias, ...]}`` for every external endpoint of + ``catalog`` that names a key. + + >>> from infer_stack.leasing.catalog import Catalog + >>> key_references(Catalog.from_dict({'endpoints': {'r': {'external': { + ... 'api_base': 'http://b/v1', 'model': 'm', 'api_key_env': 'R_KEY'}}}})) + {'R_KEY': ['r']} + """ + refs: dict[str, list[str]] = {} + for alias in sorted(getattr(catalog, 'endpoints', None) or {}): + try: + target = catalog.resolve_endpoint(alias).target + except Exception: # noqa: BLE001 - an invalid endpoint names no key + continue + if isinstance(target, ExternalTarget) and target.api_key_env: + refs.setdefault(target.api_key_env, []).append(alias) + return refs + + @dataclass(frozen=True) class ResolvedEndpoint: """One endpoint's meaning: ``alias`` (what clients request), ``protocol`` diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 5f9b2dff..534ca9b7 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -168,6 +168,11 @@ def deployment_routes( return routes +def route_key_envs(routes: Sequence[GatewayRoute]) -> list[str]: + """The variables ``routes`` send as upstream keys, sorted.""" + return sorted({r.key_env for r in routes if r.key_env}) + + def front_door_routes( deployments: list[Deployment], assignments: dict[str, list[int]], *, catalog: Any = None, dynamic: bool = False, @@ -342,7 +347,6 @@ def _postgres_service( def _litellm_service( - service_names: list[str], host_port: int, images: dict[str, str], aux_dir: str, @@ -351,6 +355,7 @@ def _litellm_service( *, dynamic_routing: bool = False, salt_key: bool = False, + key_envs: Sequence[str] = (), ) -> dict[str, Any]: # Reference the managed key via ${...} rather than baking the literal secret # into the compose YAML. Its value lives in the sidecar .env next to the @@ -377,6 +382,12 @@ def _litellm_service( f'@{POSTGRES_SERVICE}:{POSTGRES_CONTAINER_PORT}/{POSTGRES_DB_NAME}' ) environment['STORE_MODEL_IN_DB'] = 'True' + for name in sorted(set(key_envs)): + # An external route's key, by name: LiteLLM resolves `os.environ/NAME` + # from its own environment, and Compose interpolates the value from the + # managed .env, so the fingerprint (which hashes the values a stanza + # references) recreates the gateway when the key changes. + environment[name] = '${' + name + '}' labels = {ENGINE_LABEL: 'litellm'} if config_hash is not None: # LiteLLM reads its routing config once at startup; the file is bind- @@ -408,9 +419,6 @@ def _litellm_service( service['depends_on'] = { POSTGRES_SERVICE: {'condition': 'service_healthy'} } - elif service_names: - # Only wait on upstreams when there are any (zero models -> empty gateway). - service['depends_on'] = sorted(service_names) return service @@ -746,7 +754,6 @@ def render_front_door( if dynamic_routing: services[POSTGRES_SERVICE] = _postgres_service(images, state) services[LITELLM_SERVICE] = _litellm_service( - [], litellm_port, images, str(aux_dir or '.'), @@ -754,6 +761,7 @@ def render_front_door( config_hash=config_hash, dynamic_routing=dynamic_routing, salt_key=litellm_salt_key, + key_envs=route_key_envs([*routes, *(dynamic_routes or [])]), ) # Open WebUI is its own standing front door, rendered whenever ``ui`` is set @@ -1038,6 +1046,23 @@ def route_entries(self) -> dict[str, dict[str, Any]]: entries = self.route_registry().get('entries') return dict(entries) if isinstance(entries, dict) else {} + def missing_keys(self, names: Sequence[str]) -> list[str]: + """Which of ``names`` the managed ``.env`` does not set (or sets empty).""" + env = self.managed_env() + return [n for n in names if not env.get(n)] + + def warn_missing_keys(self, routes: Sequence[GatewayRoute]) -> None: + """Log each route whose key variable has no value (once per render).""" + from .._log import logger + + missing = set(self.missing_keys(route_key_envs(routes))) + for route in routes: + if route.key_env in missing: + logger.warning( + ' route {} sends ${}, which {} does not set; set it with ' + '`infer-stack env {}=...`, then `infer-stack apply`', + route.alias, route.key_env, self._env_path, route.key_env) + def registry_routes(self) -> list[GatewayRoute]: """The registry's routes, the lowest-precedence route layer.""" return registry_routes(self._load_route_registry()) diff --git a/infer_stack/leasing/profile.py b/infer_stack/leasing/profile.py index 865d2d6e..1e5e5ee4 100644 --- a/infer_stack/leasing/profile.py +++ b/infer_stack/leasing/profile.py @@ -200,17 +200,28 @@ def catalog_sources(catalog: Any) -> list[dict[str, Any]]: def profile_drift(published: dict[str, Any], invocation: dict[str, Any]) -> list[str]: """Keys where this invocation's settings differ from the recovery snapshot. - Catalogs drift only when the invocation names one that is not part of the - published union; naming a subset is the normal multi-runbook case. + Catalogs drift only when the invocation defines an endpoint the published + union does not define identically (by meaning, not by source bytes: the + union's sources are rewritten as catalogs merge); defining a subset is the + normal multi-runbook case. + + Example: + >>> ep = {'engine': 'vllm', 'model': 'm'} + >>> src = lambda **eps: {'models': {'m': {'source': 'hf://o/m'}}, 'endpoints': eps} + >>> published = {'catalogs': [src(a=ep, b=ep)]} + >>> profile_drift(published, {'catalogs': [src(a=ep)]}) + [] + >>> profile_drift(published, {'catalogs': [{'endpoints': {}}]}) + [] + >>> profile_drift(published, {'catalogs': [src(c=ep)]}) + ['catalogs'] """ drift = [] for key in sorted(set(published) | set(invocation)): if key in {'version'}: continue if key == 'catalogs': - have = {canonical_digest(s) for s in published.get(key) or []} - want = {canonical_digest(s) for s in invocation.get(key) or []} - if not want <= have: + if not _catalogs_within(invocation.get(key) or [], published.get(key) or []): drift.append(key) continue if published.get(key) != invocation.get(key): @@ -218,6 +229,30 @@ def profile_drift(published: dict[str, Any], invocation: dict[str, Any]) -> list return drift +def _catalogs_within(want: list[dict[str, Any]], have: list[dict[str, Any]]) -> bool: + """Whether every endpoint ``want`` defines, ``have`` defines identically.""" + have_digests = {canonical_digest(s) for s in have} + if all(canonical_digest(s) in have_digests for s in want): + return True + try: + mine = CatalogUnion.from_sources(want) if want else None + theirs = CatalogUnion.from_sources(have) if have else None + except CatalogError: + return False + if mine is None: + return True + for name in mine.endpoints: + if theirs is None or name not in theirs.endpoints: + return False + try: + if (mine.resolve_endpoint(name).semantic_key() + != theirs.resolve_endpoint(name).semantic_key()): + return False + except CatalogError: + return False + return True + + def merge_catalog_sources( published: list[dict[str, Any]], incoming: list[dict[str, Any]] ) -> list[dict[str, Any]]: diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 362727b5..7c515404 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -169,9 +169,15 @@ def _published(ledger): def test_seed_publishes_an_external_endpoint_and_prune_unpublishes_it(tmp_path): + from infer_stack.env_utils import write_env_file + from infer_stack.leasing.profile import ProfileMismatch + mine = Catalog.from_dict(catalog()) ledger, ctl = _ctl(tmp_path, mine) other = Catalog.from_dict({'endpoints': {'remote': REMOTE}}) + with pytest.raises(ProfileMismatch, match=r'infer-stack env REMOTE_QWEN_KEY='): + ctl.plan_route_seed([other]) # no key: nothing to send + write_env_file(ctl.backend.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) plan = ctl.plan_route_seed([other]) assert list(plan.added) == ['remote'] ctl.commit_route_seed(plan) @@ -228,3 +234,146 @@ def test_kubeai_catalog_routes_are_not_remembered_but_ad_hoc_models_are(tmp_path be.converge([local, adhoc], apply=False) registry = json.loads((tmp_path / 'gateway' / 'litellm_registry.json').read_text()) assert set(registry['entries']) == {'adhoc'} + + +# -- item 33: an external route is standing desired state --------------------------- + + +def _dynamic(tmp_path, cat, http): + from test_leasing_dynamic_routing import FakeDocker, IMAGES, PORTS, STATE + + from infer_stack.hardware import simulate_inventory + from infer_stack.leasing.compose import ComposeBackend + + return ComposeBackend(state_dir=tmp_path, inventory=simulate_inventory('4x80'), + run=FakeDocker(), http=http, images=IMAGES, ports=PORTS, + state=STATE, ui=False, dynamic_routing=True, catalog=cat) + + +def test_a_dynamic_external_route_survives_deployments_coming_and_going(tmp_path): + from test_leasing_dynamic_routing import RecordingGateway, _managed, dep + + from infer_stack.leasing.gateway import _route_id + + gw = RecordingGateway() + be = _dynamic(tmp_path, Catalog.from_dict({'endpoints': {'remote': REMOTE}}), gw) + ext = _route_id('external', 'remote') + a, b = dep('grp-aaaaaa', served='smol', t=0), dep('grp-bbbbbb', served='smol', t=1) + be.converge([a, b], apply=True) # two dedicated deployments + assert _managed(gw) == {ext, _route_id(a.id, 'smol'), _route_id(b.id, 'smol')} + gw.calls.clear() + be.converge([a], apply=True) # a dedicated one goes away + be.converge([], apply=True) # zero deployments + assert _managed(gw) == {ext} + assert all(rid != ext for _, rid in gw.calls) # never touched + + +def test_redefining_or_unpublishing_an_external_endpoint_changes_one_route(tmp_path): + from test_leasing_dynamic_routing import RecordingGateway, _managed + + from infer_stack.leasing.gateway import _route_id + + gw = RecordingGateway() + ext = _route_id('external', 'remote') + _dynamic(tmp_path, Catalog.from_dict({'endpoints': {'remote': REMOTE}}), gw).converge( + [], apply=True) + moved = {'remote': {**REMOTE, 'external': {**REMOTE['external'], + 'api_base': 'http://other:8000/v1'}}} + gw.calls.clear() + _dynamic(tmp_path, Catalog.from_dict({'endpoints': moved}), gw).converge([], apply=True) + assert gw.calls == [('delete', ext), ('new', ext)] # replaced in place + assert gw.models[ext]['litellm_params']['api_base'] == 'http://other:8000/v1' + gw.calls.clear() + _dynamic(tmp_path, None, gw).converge([], apply=True) # unpublished + assert gw.calls == [('delete', ext)] and _managed(gw) == set() + + +def test_release_and_gc_leave_an_external_route_published(tmp_path): + from test_leasing_compose import FakeDocker + + both = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _ctl(tmp_path, both, FakeDocker()) + out = ctl.acquire('w', both.resolve_names(['local']), wait=False) + ctl.release(out.lease.id) + ctl.gc() + assert {r.alias: r.origin for r in ctl.route_view()}['remote'] == 'external' + assert _published(ledger) == ['local', 'remote'] + + +# -- item 34: credentials by reference ----------------------------------------------- + + +def test_the_host_gateway_gets_the_key_by_name_and_recreates_when_it_changes(tmp_path): + from infer_stack.env_utils import write_env_file + from infer_stack.leasing.compose import FINGERPRINT_LABEL + + be = _compose(tmp_path, Catalog.from_dict(catalog(remote=REMOTE))) + write_env_file(be.gateway._env_path, {'REMOTE_QWEN_KEY': 'one'}) + be.converge([], apply=False) + text = be.compose_file.read_text() + litellm = yaml.safe_load(text)['services']['litellm'] + assert litellm['environment']['REMOTE_QWEN_KEY'] == '${REMOTE_QWEN_KEY}' + assert 'one' not in text # the name, never the value + before = litellm['labels'][FINGERPRINT_LABEL] + write_env_file(be.gateway._env_path, {'REMOTE_QWEN_KEY': 'two'}) + be.converge([], apply=False) + after = yaml.safe_load(be.compose_file.read_text())['services']['litellm'] + assert after['labels'][FINGERPRINT_LABEL] != before # recreated on apply + + +def test_managed_only_gateways_render_exactly_as_before(tmp_path): + be = _compose(tmp_path, Catalog.from_dict(catalog())) + be.converge([], apply=False) + env = yaml.safe_load(be.compose_file.read_text())['services']['litellm']['environment'] + assert set(env) == {'LITELLM_MASTER_KEY'} + + +def test_a_render_warns_about_a_key_with_no_value(tmp_path): + from test_leasing_route_registry import capture_warnings + + be = _compose(tmp_path, Catalog.from_dict(catalog(remote=REMOTE))) + with capture_warnings() as warnings: + be.converge([], apply=False) + assert any('REMOTE_QWEN_KEY' in m and 'infer-stack env' in m for m in warnings) + + +def test_the_cluster_gateway_puts_keys_in_its_secret_and_rolls_on_a_change(tmp_path): + from infer_stack.backends.kubeai_gateway import ClusterGateway + from infer_stack.env_utils import write_env_file + from infer_stack.leasing.gateway import catalog_routes + + calls = [] + gw = ClusterGateway(state_dir=tmp_path, namespace='kubeai', url='http://gw:30442', + run=lambda argv: calls.append(argv) or '') + gw.extra_routes = catalog_routes(Catalog.from_dict({'endpoints': {'remote': REMOTE}})) + + def rendered(value): + write_env_file(gw.gateway._env_path, {'REMOTE_QWEN_KEY': value}) + text = gw._render_documents()[gw.manifests_file] + assert value not in text # never in a manifest + deploy = [d for d in yaml.safe_load_all(text) if d['kind'] == 'Deployment'][0] + return deploy['spec']['template']['metadata']['annotations']['infer-stack/key-hash'] + + assert rendered('one') != rendered('two') + gw.converge() + secret = yaml.safe_load((tmp_path / 'gateway-secret.yaml').read_text()) + assert secret['stringData']['REMOTE_QWEN_KEY'] == 'two' + assert 'LITELLM_MASTER_KEY' in secret['stringData'] + + +def test_env_says_when_a_published_key_takes_effect(tmp_path, monkeypatch, capsys): + from infer_stack.cli import commands_leasing as cl + from infer_stack.leasing import Ledger, SqliteStore + from infer_stack.leasing.profile import catalog_sources + + db = tmp_path / 'ledger.db' + ledger = Ledger(SqliteStore(str(db))) + ledger.set_profile({'backend': 'compose', 'catalogs': catalog_sources( + Catalog.from_dict(catalog(remote=REMOTE)))}) + monkeypatch.setattr(cl, 'default_ledger_path', lambda: db) + monkeypatch.setattr(cl, '_secret_env_path', lambda config=None: tmp_path / '.env') + assert cl.EnvCLI.main(argv=['REMOTE_QWEN_KEY=sk-x']) == 0 + out = capsys.readouterr().out + assert "remote send(s) $REMOTE_QWEN_KEY" in out and 'infer-stack apply' in out + assert cl.EnvCLI.main(argv=['UNRELATED=1']) == 0 + assert 'apply' not in capsys.readouterr().out From 93823c6e40c51d0ca66781ead1edbc26ab738a80 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:22:48 -0400 Subject: [PATCH 114/141] Access above leasing: `infer-stack access`, typed connection info Controller.access partitions resolved endpoints: the managed members take one lease through acquire (whose transaction also publishes the invocation's catalogs), and an external-only access publishes through the same preview, approved marker and publication with no ledger mutation (publish_endpoints). External members need the LiteLLM front door and a value for their key; access waits until the front door accepts its key and releases the lease it took if it never does. The untyped backend.access(endpoints) dict becomes connection_info() (ConnectionInfo: base URL, key, UI and proxy URLs) and request_names() in the backend protocol. The descriptor takes the access's endpoints and omits INFER_STACK_LEASE_ID when there is no lease. `infer-stack access` writes it; `run` uses access; `release --env-file` of a lease-less file has nothing to release. FrontDoor the capability is FrontDoorControl, the render RenderedFrontDoor. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 7 + docs/queue.md | 22 ++- infer_stack/backends/kubeai.py | 31 ++-- infer_stack/backends/kubeai_gateway.py | 4 +- infer_stack/cli/__init__.py | 2 + infer_stack/cli/commands_leasing.py | 192 +++++++++++++++++++++---- infer_stack/leasing/backend.py | 51 ++++++- infer_stack/leasing/compose.py | 8 +- infer_stack/leasing/controller.py | 152 +++++++++++++++++++- infer_stack/leasing/envfile.py | 53 ++++--- infer_stack/leasing/gateway.py | 39 +++-- tests/test_cli_leasing.py | 18 ++- tests/test_external_endpoints.py | 179 +++++++++++++++++++++++ tests/test_leasing_compose.py | 27 +++- tests/test_leasing_kubeai.py | 12 +- tests/test_parity.py | 4 +- 16 files changed, 687 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3672b342..966b110b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,13 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). with no value. - A runbook whose catalog is already published no longer warns that its catalogs differ from the recovery snapshot. +- `infer-stack access NAME...` makes endpoints reachable whoever runs them: + it leases the managed ones (as `acquire`), publishes external ones with no + lease, waits for the front door, and writes the same env-file (with no + `INFER_STACK_LEASE_ID` when nothing is leased). `run` uses it, so a mixed + bundle takes one lease for its managed members. `acquire` of an external + endpoint points at `access`; `release --env-file` of a lease-less env-file + has nothing to release. ### Hardening from a review of the backend-parity work diff --git a/docs/queue.md b/docs/queue.md index a5303eba..22848a9e 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1220,7 +1220,12 @@ read keys back; the route identity/fingerprint includes the env NAME (a changed reference forces replacement), never the value; a changed value triggers what the gateway needs to see it, without leaking it. -### 35. [ ] Publication without a ledger mutation (one transaction, 28.3) +### 35. [x] Publication without a ledger mutation (one transaction, 28.3) +*Done 2026-09-27.* `Controller.publish_endpoints`: under the lock, the +profile candidate (invocation catalogs merged into the union), a preview +with approval, `ledger.publish_profile` (profile + approved marker at once), +`_publish`. Mixed access is `acquire`, whose transaction already carries the +candidate. No CLI writes gateway state. External-only access may change nothing in the ledger but still needs: check/incorporate the invocation catalog/profile, render gateway state, persist routes, apply under the publication lock with the pending / @@ -1228,7 +1233,16 @@ approved-render machinery. Factor it so managed acquire and external-only access reach the same publication coordinator; no second ad-hoc gateway apply path; no private gateway writes from the CLI. -### 36. [ ] Access above leasing +### 36. [x] Access above leasing +*Done 2026-09-27.* `ConnectionInfo` + `backend.connection_info()` / +`request_names()` (typed, in the protocol) replace the `access` hook; +`Controller.access` -> `AccessResult` (endpoints, external, request names, +connection, lease or `None`, the acquire's outcome, front-door readiness); +a front door that never accepts its key releases the lease the call took. +`infer-stack access NAMES [--env-file]`; `run` uses it; the descriptor +omits `INFER_STACK_LEASE_ID` without a lease; `release --env-file` of such +a file says there is nothing to release (exit 0); `acquire` of an external +endpoint points at `access`. *Review:* split the front door's part from the endpoints': a typed `FrontDoorControl.connection_info()` (base URL, front-door credential, optional UI URL), plus the resolved endpoints / routes (with LiteLLM the @@ -1259,7 +1273,9 @@ Status, TUI and catalog views show external targets as `external`. deployments. Show external endpoints in an endpoint/routing section (`status`, the TUI's catalog or routes view, `catalog show`). -### 38. [ ] `FrontDoor` naming collision (cleanup C) +### 38. [x] `FrontDoor` naming collision (cleanup C) +*Done 2026-09-27:* `FrontDoorControl` (the capability, now with +`connection_info`) and `RenderedFrontDoor` (the render). If both the controller capability and the rendered artifact are still called `FrontDoor`, rename (e.g. `FrontDoorControl`, `RenderedFrontDoor`). diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index cc72398f..b17b148f 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -1217,26 +1217,23 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: return Readiness(False, f'{reason} (pod: {waiting})' if waiting else reason, needs_room=bool(self._capacity_shortage(deployment))) - def access(self, endpoints: list[str]) -> dict[str, Any] | None: - """Where a client reaches these endpoints (env-file descriptor). + def connection_info(self): + """Where a client reaches these endpoints: the gateway's front door, or + without one KubeAI's own OpenAI server, which is unauthenticated (the + api key is the literal ``EMPTY`` placeholder, and no key variable).""" + from ..leasing.backend import ConnectionInfo - One gateway ``base_url`` for everything; the request name is the Model - CR name (from the render sidecar), not the endpoint alias — KubeAI has - no alias layer. The gateway is unauthenticated, so the api key is the - literal ``EMPTY`` placeholder and no key env var is advertised. - """ if self.gateway is not None: - return self.gateway.access(endpoints) + return self.gateway.connection_info() + return ConnectionInfo(self.base_url, api_key_env=None, api_key='EMPTY') + + def request_names(self, endpoints: list[str]) -> dict[str, str]: + """The alias behind the gateway; without it the Model CR name (from the + render sidecar): KubeAI has no alias layer.""" + if self.gateway is not None: + return {ep: ep for ep in endpoints} request_names = self._load_sidecar().get('request_names') or {} - return { - 'base_url': self.base_url, - 'api_key_env': None, - 'api_key': 'EMPTY', - 'request_names': { - ep: request_names.get(ep, model_name_for(ep)) - for ep in endpoints - }, - } + return {ep: request_names.get(ep, model_name_for(ep)) for ep in endpoints} #: No containers of this host to manage: the cluster owns network and pods. host_runtime = None diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index 3dee1ec2..ddb5d682 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -154,8 +154,8 @@ def restore_env(self, values) -> None: def gateway_accepts(self, key: str, *, wait: float = 0.0): return self.gateway.gateway_accepts(key, wait=wait) - def access(self, endpoints: list[str]) -> dict[str, Any] | None: - return self.gateway.access(endpoints) + def connection_info(self): + return self.gateway.connection_info() def catalog_routes(self, catalog) -> list[GatewayRoute]: return [] # the kubeai backend supplies its own routes diff --git a/infer_stack/cli/__init__.py b/infer_stack/cli/__init__.py index 5e3d44a3..f0da366a 100644 --- a/infer_stack/cli/__init__.py +++ b/infer_stack/cli/__init__.py @@ -40,6 +40,7 @@ from .commands_catalog import CatalogModalCLI from .commands_mock import MockModalCLI from .commands_leasing import ( + AccessCLI, AcquireCLI, ApplyCLI, CleanCLI, @@ -129,6 +130,7 @@ class ManageCLI(kw.ModalCLI): # Leasing model (acquire/release/run + status) acquire = AcquireCLI # stand up endpoints: lease + up + wait (--ttl for soft TTL) + access = AccessCLI # reach endpoints, managed or external: lease only what runs here release = ReleaseCLI evict = EvictCLI # force-tear-down released (idle) models to free GPUs gc = GcCLI # reclaim TTL-expired (leaked) leases + free their GPUs diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index d34825cf..e641fe9e 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -452,12 +452,21 @@ def _resolve_lease(config, verb: str) -> str | None: 'what is held)') sid = read_lease_id(path) if not sid: + if verb == 'release': + # An access to external endpoints only holds nothing. + print(f'release: {path} names no lease (an access to external ' + 'endpoints only); nothing to release') + raise SystemExit(0) raise SystemExit(f'{verb}: {path} names no lease ({LEASE_ENV})') return sid -def _descriptor_for(controller, lease, deployments, config, *, assignments=None): - """Build the descriptor, preferring backend-supplied access (real base_url). +def _descriptor_for(controller, lease, deployments, config, *, assignments=None, + endpoints=None): + """Build the descriptor, preferring the backend's connection (real base_url). + + ``endpoints`` (an access's aliases, external ones included) default to the + lease's; ``lease`` is ``None`` for an access to external endpoints only. For a GPU *reservation* the env-file's only useful payload is the reserved GPU index(es): fold the placement assignment into ``cuda_visible_devices`` so @@ -467,14 +476,12 @@ def _descriptor_for(controller, lease, deployments, config, *, assignments=None) base_url = config.base_url api_key_env = config.api_key_env api_key = None - request_names = None - access = getattr(controller.backend, 'access', None) - info = access(list(lease.endpoints)) if access else None - if info: - base_url = info.get('base_url', base_url) - api_key_env = info.get('api_key_env', api_key_env) - api_key = info.get('api_key') - request_names = info.get('request_names') + wanted = list(endpoints if endpoints is not None else lease.endpoints) + info = controller.backend.connection_info() + if info is not None and info.base_url is not None: + # Otherwise (Open WebUI alone, or nothing) keep the configured defaults. + base_url, api_key_env, api_key = info.base_url, info.api_key_env, info.api_key + request_names = controller.backend.request_names(wanted) or None cuda_visible_devices = None if assignments: reserved_gpus: list[int] = [] @@ -491,6 +498,7 @@ def _descriptor_for(controller, lease, deployments, config, *, assignments=None) api_key=api_key, request_names=request_names, cuda_visible_devices=cuda_visible_devices, + endpoints=wanted, ) @@ -699,17 +707,74 @@ def _emit_acquire(config, controller, outcome) -> int: print(f' pending: {endpoint} ({gid})') for hint in _oom_hints(controller, outcome): print(f' {hint}') - access = getattr(controller.backend, 'access', None) - info = access(list(outcome.lease.endpoints)) if access else None - if info and info.get('ui_url'): - print(f' open webui: {info["ui_url"]}') - if info and info.get('proxy_url'): - print(f' front door: {info["proxy_url"]} (UI: / API: /v1)') + _print_front_door(controller.backend.connection_info()) if config.env_file: print(f' env-file: {config.env_file}') return 2 if not_ready else 0 +def _print_front_door(info) -> None: + """The UI and reverse-proxy lines of an acquire's or access's summary.""" + if info is not None and info.ui_url: + print(f' open webui: {info.ui_url}') + if info is not None and info.proxy_url: + print(f' front door: {info.proxy_url} (UI: / API: /v1)') + + +def _resolve_endpoints(catalog, names): + """``catalog.resolve(names)``: endpoint meanings, bundles expanded.""" + try: + return catalog.resolve(names) + except CatalogError as ex: + raise SystemExit(str(ex)) + + +def _emit_access(config, controller, result) -> int: + """Output for ``access``: the descriptor, and what is held.""" + outcome = result.acquire + released = (outcome is not None and outcome.released_on_timeout) or ( + result.front_door_ready is False and result.lease is not None) + descriptor = _descriptor_for( + controller, result.lease, result.deployments, config, + endpoints=result.endpoints, + assignments=outcome.reconcile.assignments if outcome is not None else None) + if config.env_file and not released: + Path(config.env_file).expanduser().write_text(render_env_file(descriptor)) + if config.json: + print(json.dumps({ + 'lease_id': result.lease.id if result.lease is not None else None, + 'endpoints': result.endpoints, + 'external': result.external, + 'descriptor': _public_descriptor(descriptor), + 'ready': result.ready, + 'front_door_ready': result.front_door_ready, + 'pending': [] if outcome is None or outcome.wait is None else outcome.wait.pending, + 'failures': [] if outcome is None or outcome.wait is None else outcome.wait.failures, + 'released': released, + }, indent=2)) + return 0 if result.ready else 2 + if result.front_door_ready is False: + print('the front door did not accept its key in time' + + (f'; lease {result.lease.id} released' if result.lease else '')) + return 2 + if outcome is not None and outcome.released_on_timeout: + return _emit_acquire(config, controller, outcome) + held = (f'lease {result.lease.id} (owner={result.lease.owner})' + if result.lease is not None else 'no lease: every endpoint is external') + print(f'access {", ".join(result.endpoints)}: {held}') + for endpoint, model in descriptor['endpoints'].items(): + tag = ' (external)' if endpoint in result.external else '' + print(f' endpoint {endpoint} -> {model}{tag}') + print(f' ready: {result.ready}') + if outcome is not None and outcome.wait is not None: + for gid, endpoint in outcome.wait.pending: + print(f' pending: {endpoint} ({gid})') + _print_front_door(result.connection) + if config.env_file: + print(f' env-file: {config.env_file}') + return 0 if result.ready else 2 + + def _do_acquire(config, *, owner: str, ttl_seconds: float | None) -> int: from .._log import logger from ..leasing.backend import ConvergeAborted, PlacementError @@ -996,6 +1061,76 @@ def main(cls, argv=True, **kwargs): ) +class AccessCLI(_AcquireFlagsMixin): + """Make endpoints reachable and write how to reach them. + + ``access NAME…`` is ``acquire`` for a workflow that does not care who + runs its models. A managed endpoint (a model infer-stack runs) is leased + and brought up exactly as ``acquire`` does; an external one (``external:`` + in the catalog, a server that already runs elsewhere) is published on the + LiteLLM front door and needs no lease. Either way the client gets one base + URL and asks for the alias, so moving an endpoint between the two changes + neither. Mixed bundles take one lease, for the managed members only. + + Write the descriptor with ``--env-file``; ``release --env-file`` releases + the lease, if there is one. (``run`` does all of this around a command.) + """ + + __command__ = 'access' + __epilog__ = """ + Examples: + # an external endpoint: publish its route, no lease + infer-stack env REMOTE_QWEN_KEY=sk-... + infer-stack access remote-qwen --env-file qwen.env + + # a bundle of a managed and an external endpoint: one lease + infer-stack access pair --env-file pair.env + infer-stack release --env-file pair.env + """ + + names = kw.Value( + [], nargs='*', position=1, type=str, help='Endpoint or bundle names.' + ) + ttl = kw.Value( + None, type=str, help='Soft TTL of the lease, if one is taken (e.g. 2h).' + ) + owner = kw.Value(None, type=str, help='Lease owner (default: $USER).') + dedicated = kw.Value( + False, isflag=True, + help='Force dedicated deployments for the managed members.', + ) + + @classmethod + def main(cls, argv=True, **kwargs): + from ..leasing.backend import ConvergeAborted, PlacementError + from ..leasing.profile import ProfileMismatch + + config = cls.cli(argv=argv, data=kwargs) + if not config.apply: + raise SystemExit('access: --no-apply stages a lease; use `acquire --no-apply`') + controller = _open_controller(config) + catalog = _requests_catalog(controller, config) + names = _collect_names(config.names) + if not names: + raise SystemExit('give at least one endpoint or bundle name') + endpoints = _resolve_endpoints(catalog, names) + try: + result = controller.access( + config.owner or _default_owner(), endpoints, + sharing=Sharing.DEDICATED if config.dedicated else None, + ttl_seconds=_parse_duration(config.ttl), wait=bool(config.wait), + timeout=float(config.timeout), interval=float(config.interval), + wait_for_placement=bool(config.queue)) + except ConvergeAborted: + raise SystemExit('aborted: changes not applied (no lease kept)') + except ProfileMismatch as ex: + raise SystemExit(f'access: {ex}') + except PlacementError as ex: + raise SystemExit('could not place every requested endpoint (no lease ' + 'kept):\n' + '\n'.join(f' {r}' for r in ex.reasons)) + return _emit_access(config, controller, result) + + class RenderCLI(_LeasingCommonMixin): """Write the rendered state for the current desired set, without bringing it up. @@ -1866,10 +2001,12 @@ def main(cls, argv=True, **kwargs): class RunCLI(_LeasingCommonMixin): - """Acquire endpoints, run a command with the endpoint env, then release. + """Access endpoints, run a command with the endpoint env, then release. - Everything after ``--`` is the command. The lease is always released on - exit; the TTL is the backstop if the process is hard-killed. + Everything after ``--`` is the command. Managed endpoints are leased + (``access``), external ones only published; the lease, if any, is always + released on exit, and the TTL is the backstop if the process is + hard-killed. """ __command__ = 'run' @@ -1910,11 +2047,11 @@ def main(cls, argv=True, **kwargs): raise SystemExit('run: --endpoint is required') if not command: raise SystemExit('run: give a command after --') - requests = _resolve(catalog, names) + endpoints = _resolve_endpoints(catalog, names) try: - outcome = controller.acquire( + result = controller.access( config.owner or _default_owner(), - requests, + endpoints, ttl_seconds=_parse_duration(config.ttl), wait=True, timeout=float(config.timeout), @@ -1923,7 +2060,8 @@ def main(cls, argv=True, **kwargs): ) except ProfileMismatch as ex: raise SystemExit(f'run: {ex}') - if outcome.wait is not None and not outcome.wait.ready: + outcome = result.acquire + if outcome is not None and outcome.wait is not None and not outcome.wait.ready: # The controller already released the lease on timeout # (released_on_timeout); just surface why we're not running. if outcome.wait.failures: @@ -1935,8 +2073,11 @@ def main(cls, argv=True, **kwargs): raise SystemExit( f'run: endpoints not ready: {outcome.wait.pending}' ) + if result.front_door_ready is False: + raise SystemExit('run: the front door did not accept its key in time') descriptor = _descriptor_for( - controller, outcome.lease, outcome.deployments, config + controller, result.lease, result.deployments, config, + endpoints=result.endpoints, ) env = dict(os.environ) env.update(descriptor_env(descriptor)) @@ -1944,7 +2085,8 @@ def main(cls, argv=True, **kwargs): proc = subprocess.run(command, env=env) return int(proc.returncode) finally: - controller.release(outcome.lease.id) + if result.lease is not None: + controller.release(result.lease.id) def _lease_ttl(le, now: float | None = None) -> str: diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 9094669a..1f2a4cef 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -184,7 +184,7 @@ def remove_containers(self, container_ids: list[str]) -> None: ... -class FrontDoor(Protocol): +class FrontDoorControl(Protocol): """The LiteLLM gateway as the controller and the CLI address it. ``backend.front_door()`` returns one, or ``None`` for a backend without a @@ -211,6 +211,29 @@ def restore_env(self, values: dict[str, str | None]) -> None: def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: ... + def connection_info(self) -> ConnectionInfo | None: + """Where a client reaches this front door; ``None`` when nothing does.""" + ... + + +@dataclass(frozen=True) +class ConnectionInfo: + """Where a client reaches the endpoints: a base URL and its credential. + + Through the LiteLLM front door, one ``base_url`` for every alias and the + managed master key (``api_key``, read from ``api_key_env``); without one, + whatever the backend's own server offers (KubeAI's, keyless: + ``api_key='EMPTY'``). ``ui_url`` / ``proxy_url`` are Open WebUI and the + single-port reverse proxy, when on. The request name per endpoint is the + backend's (:meth:`ServingBackend.request_names`), not the front door's. + """ + + base_url: str | None + api_key_env: str | None = None + api_key: str | None = None + ui_url: str | None = None + proxy_url: str | None = None + class RecoveryProfile(Protocol): """A backend's recovery snapshot: what it would render, and adopting one. @@ -335,7 +358,7 @@ def placement_notes(self) -> dict[str, list[str]]: residents that yielded their GPUs). Empty where nothing is placed.""" ... - def front_door(self) -> FrontDoor | None: + def front_door(self) -> FrontDoorControl | None: """What holds the LiteLLM gateway (its keys and routes), or ``None`` for a backend without one.""" ... @@ -345,6 +368,18 @@ def routes(self, desired: list[Deployment], placement: Any = None """The gateway routes a render of ``desired`` produces.""" ... + def connection_info(self) -> ConnectionInfo | None: + """Where a client reaches this backend's endpoints, or ``None`` + (an in-process backend serves nothing).""" + ... + + def request_names(self, endpoints: list[str]) -> dict[str, str]: + """The ``model`` a client sends for each of ``endpoints``, for those + this backend decides (the alias itself behind a LiteLLM front door; a + KubeAI Model's name without one). Others fall back to the deployment's + served name.""" + ... + def catalog_routes(self, catalog: Any) -> list[GatewayRoute]: """The routes every endpoint of ``catalog`` would get here.""" ... @@ -568,13 +603,19 @@ def reclaim_candidates(self, blocked: Deployment, """One scheduling domain in-process: any idle deployment frees room.""" return [g.id for g in idle] - def front_door(self) -> FrontDoor | None: + def front_door(self) -> FrontDoorControl | None: return None # no gateway in-process def routes(self, desired: list[Deployment], placement: Any = None ) -> list[GatewayRoute]: return [] + def connection_info(self) -> ConnectionInfo | None: + return None + + def request_names(self, endpoints: list[str]) -> dict[str, str]: + return {} + def catalog_routes(self, catalog: Any) -> list[GatewayRoute]: return [] @@ -784,10 +825,10 @@ def null(backend: NullBackend) -> ServingBackend: def host(backend: ComposeBackend) -> HostRuntime: return backend - def compose_front(backend: ComposeBackend) -> FrontDoor: + def compose_front(backend: ComposeBackend) -> FrontDoorControl: return backend - def cluster_front(backend: ClusterGateway) -> FrontDoor: + def cluster_front(backend: ClusterGateway) -> FrontDoorControl: return backend def compose_profile(backend: ComposeBackend) -> RecoveryProfile: diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 1705106c..c6d1b5d0 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1301,9 +1301,13 @@ def restore_env(self, values: dict[str, str | None]) -> None: def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: return self.gateway.gateway_accepts(key, wait=wait) - def access(self, endpoints: list[str]) -> dict[str, Any] | None: + def connection_info(self): """Where a client reaches these endpoints: the front door.""" - return self.gateway.access(endpoints) + return self.gateway.connection_info() + + def request_names(self, endpoints: list[str]) -> dict[str, str]: + """Behind LiteLLM a client sends the alias; without it, the served name.""" + return {ep: ep for ep in endpoints} if self.litellm else {} @property def _state_file(self) -> Path: diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 4f0986e3..e7ee6355 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import Callable, Iterable, TypeVar -from .backend import ApplyResult, HostRuntime, ServingBackend +from .backend import ApplyResult, ConnectionInfo, HostRuntime, ServingBackend from .routes import RoutePlan from .ledger import Ledger from .models import Deployment, DeploymentState, EndpointRequest, Lease, LeaseState @@ -117,6 +117,46 @@ class AcquireOutcome: released_on_timeout: bool = False +@dataclass +class AccessResult: + """What a workload needs to reach its endpoints, and what it holds. + + ``endpoints``: the aliases asked for, bundles expanded, in order; + ``external``: those an external server fulfils (no lease, no deployment); + ``request_names``: the ``model`` a client sends for each; ``connection``: + the base URL and credential (``None`` on an in-process backend); ``lease``: + the one real lease over the managed members, or ``None`` when every member + is external (the lease never lists an external name); ``acquire``: that + lease's outcome (readiness, placement). ``front_door_ready`` is ``False`` + when an external member's front door never accepted its key in time; the + lease this access took is then already released. + """ + + endpoints: list[str] + external: list[str] + request_names: dict[str, str] + connection: ConnectionInfo | None + lease: Lease | None = None + acquire: AcquireOutcome | None = None + front_door_ready: bool | None = None + + @property + def deployments(self) -> list[Deployment]: + return list(self.acquire.deployments) if self.acquire is not None else [] + + @property + def ready(self) -> bool: + """Every member can be asked now (managed ones generated through the + front door; external ones reachable through it).""" + if self.front_door_ready is False: + return False + if self.acquire is None: + return True + if self.acquire.released_on_timeout: + return False + return self.acquire.wait is None or bool(self.acquire.wait.ready) + + @dataclass class ReleaseOutcome: idled_deployment_ids: list[str] @@ -1614,6 +1654,116 @@ def acquire( return self._finish_acquire(result, rec, apply=apply, wait=wait, timeout=timeout, interval=interval) + #: Seconds ``access`` waits for the front door to accept its key, when an + #: external member has nothing else to wait for. + FRONT_DOOR_WAIT_S = 120.0 + + def access( + self, + owner: str, + endpoints: list, + *, + sharing: str | None = None, + ttl_seconds: float | None = None, + wait: bool = True, + timeout: float = 300.0, + interval: float = 2.0, + wait_for_placement: bool = False, + ) -> AccessResult: + """Make ``endpoints`` reachable: lease the managed ones, publish all. + + ``endpoints`` are :class:`~infer_stack.leasing.endpoints. + ResolvedEndpoint` s (``catalog.resolve(names)``). The managed members + take one lease through :meth:`acquire`, whose transaction also + publishes the invocation's catalogs, external definitions included + (docs/planning/external-endpoints.md, decision 3). With no managed + member there is no ledger mutation: :meth:`publish_endpoints` runs the + same preview, approval and publication without one. External members + need the LiteLLM front door, a value for their key, and a front door + that accepts its key (decision 6); nothing is sent to the external + server. If the front door never answers, the lease this call took is + released and ``front_door_ready`` is ``False``. + """ + from .endpoints import ExternalTarget + from .profile import ProfileMismatch + + aliases = [e.alias for e in endpoints] + external = [e for e in endpoints if not e.managed] + front = self.backend.front_door() if external else None + if external: + if front is None or not front.litellm: + raise ProfileMismatch( + f'{", ".join(repr(e.alias) for e in external)} ' + f'{"is" if len(external) == 1 else "are"} served by an external ' + 'server, reached only through the LiteLLM front door; this ' + 'backend has none (turn `litellm` on)') + missing = front.gateway.missing_keys(sorted({ + e.target.api_key_env for e in external + if isinstance(e.target, ExternalTarget) and e.target.api_key_env})) + if missing: + raise ProfileMismatch( + f'${missing[0]} has no value in {front.gateway._env_path}; set it ' + f'first: `infer-stack env {missing[0]}=...`') + managed = [e.to_request(sharing_override=sharing) for e in endpoints if e.managed] + outcome = None + if managed: + outcome = self.acquire( + owner, managed, ttl_seconds=ttl_seconds, wait=wait, timeout=timeout, + interval=interval, wait_for_placement=wait_for_placement) + else: + self.publish_endpoints() + result = AccessResult( + endpoints=aliases, external=[e.alias for e in external], + request_names=self.backend.request_names(aliases), + connection=self.backend.connection_info(), + lease=outcome.lease if outcome is not None else None, acquire=outcome) + if front is not None and wait and not (outcome and outcome.released_on_timeout): + accepted = front.gateway_accepts( + front.master_key(), wait=min(timeout, self.FRONT_DOOR_WAIT_S)) + result.front_door_ready = accepted is True + if not result.front_door_ready and outcome is not None: + self.release(outcome.lease.id) + return result + + def publish_endpoints(self) -> ReconcileResult: + """Publish the invocation's endpoint definitions with no lease. + + The same transaction an acquire runs, minus the ledger mutation: + under the lock, derive the profile candidate (the invocation's + catalogs merged into the published union), preview its render with + approval, then store it with the approved digest and publish. When the + union already has them it still publishes: the front door may be down + (a fresh stack, or after ``stack down``), and an unchanged render + applies as a no-op. + """ + from .residency import ResidencyUnknown + + recovery = self.backend.recovery_profile + with self._global_lock(): + self._sync_profile(create=True) + try: + residency = self.backend.residency() + except ResidencyUnknown: + residency = None + candidate = self._acquire_profile_candidate(residency=residency) + if candidate is None or recovery is None: + self._mark_pending(apply=True) + return self._publish() + stored = self.ledger.profile() + self._use_profile_candidate(candidate) + try: + self._prepare_network(self.backend.host_runtime) + desired, inputs = self._admission_view(residency, virtual_expiry=True) + self.backend.preview(desired, inputs, approve=True) + except BaseException: + self._restore_stored_profile(stored) + raise + self.ledger.publish_profile( + candidate, approved_digest=self.backend.last_preview_digest) + self._profile_error = None + self._applied_profile = candidate + return self._publish() + def _acquire_by_admission( self, owner, requests, *, ttl_seconds, apply, wait_for_placement, placement_timeout, placement_interval, diff --git a/infer_stack/leasing/envfile.py b/infer_stack/leasing/envfile.py index 3c262ae6..b636e82a 100644 --- a/infer_stack/leasing/envfile.py +++ b/infer_stack/leasing/envfile.py @@ -38,36 +38,53 @@ def _endpoint_var(endpoint: str) -> str: def build_descriptor( - lease: Lease, + lease: Lease | None, deployments: list[Deployment], *, - base_url: str, - api_key_env: str = 'LITELLM_MASTER_KEY', + base_url: str | None, + api_key_env: str | None = 'LITELLM_MASTER_KEY', api_key: str | None = None, request_names: dict[str, str] | None = None, cuda_visible_devices: str | None = None, + endpoints: list[str] | None = None, ) -> dict[str, Any]: - """Build the endpoint descriptor for one lease. + """Build the endpoint descriptor for one lease, or for an access. + + ``endpoints`` are the aliases to describe, in order (default: the + lease's). An endpoint a deployment serves gets its request name from + ``request_names`` (behind a LiteLLM front door the client asks for the + endpoint *alias*, not the upstream served name), else from the + deployment; one no deployment serves (an external endpoint, reached + through the front door) only from ``request_names``. Only the endpoints + asked for are included (a coalesced deployment may serve more). With no + lease -- an access to external endpoints only -- ``lease_id`` is + ``None``, and the env-file carries no lease id. - Only the endpoints this lease actually requested are included (a coalesced - deployment may serve more). ``request_names`` overrides the model name a client - must request per endpoint — e.g. behind a LiteLLM front door the client asks - for the endpoint *alias*, not the upstream served name. + Example: + >>> d = build_descriptor(None, [], base_url='http://h:1/v1', + ... endpoints=['remote'], request_names={'remote': 'remote'}) + >>> d['lease_id'], d['endpoints'] + (None, {'remote': 'remote'}) """ - endpoints: dict[str, str] = {} + wanted = list(lease.endpoints) if endpoints is None and lease is not None \ + else list(endpoints or []) + names: dict[str, str] = {} for deployment in deployments: for endpoint, payload in deployment.served.items(): - if endpoint in lease.endpoints: + if endpoint in wanted: if request_names and endpoint in request_names: - endpoints[endpoint] = request_names[endpoint] + names[endpoint] = request_names[endpoint] else: - endpoints[endpoint] = _request_model_name(payload, endpoint) - # preserve the lease's requested order where possible - ordered = {ep: endpoints[ep] for ep in lease.endpoints if ep in endpoints} + names[endpoint] = _request_model_name(payload, endpoint) + for endpoint in wanted: + if endpoint not in names and request_names and endpoint in request_names: + names[endpoint] = request_names[endpoint] + # preserve the requested order + ordered = {ep: names[ep] for ep in wanted if ep in names} descriptor: dict[str, Any] = { 'schema_version': 1, 'kind': 'infer-stack-endpoint', - 'lease_id': lease.id, + 'lease_id': lease.id if lease is not None else None, 'base_url': base_url, 'api_key_env': api_key_env, 'protocol': 'openai', @@ -83,7 +100,11 @@ def build_descriptor( def descriptor_env(descriptor: dict[str, Any]) -> dict[str, str]: """Flatten a descriptor into the env vars a job should see.""" - env: dict[str, str] = {LEASE_ENV: descriptor['lease_id']} + env: dict[str, str] = {} + if descriptor.get('lease_id'): + # No sentinel without a lease: `release --env-file` then has nothing + # to find, which is the truth. + env[LEASE_ENV] = descriptor['lease_id'] if descriptor.get('base_url'): env['OPENAI_BASE_URL'] = descriptor['base_url'] if descriptor.get('api_key'): diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index 534ca9b7..e06645c9 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -23,7 +23,7 @@ from ..config import PINNED_IMAGES from ..config import DEFAULT_PORTS from ..env_utils import ensure_secret, parse_env_file, write_env_file -from .backend import ConvergeScaffold +from .backend import ConnectionInfo, ConvergeScaffold from .endpoints import ExternalTarget from .models import Deployment, served_name from .naming import ( @@ -664,7 +664,7 @@ def set_master_key(env_path: Path, key: str) -> None: @dataclass -class FrontDoor: +class RenderedFrontDoor: """The rendered front door: its Compose services and config files.""" services: dict[str, Any] @@ -693,7 +693,7 @@ def render_front_door( dynamic_routing: bool, dynamic_routes: list[GatewayRoute] | None = None, ui_run_as: str | None = None, -) -> FrontDoor: +) -> RenderedFrontDoor: """Render the gateway, its database, Open WebUI and the reverse proxy. ``routes`` is the static route table (one per alias, @@ -815,7 +815,7 @@ def render_front_door( ).hexdigest()[:12], ) - return FrontDoor(services=services, litellm_config=litellm_config, + return RenderedFrontDoor(services=services, litellm_config=litellm_config, nginx_config=nginx_config, litellm_routes=litellm_routes) @@ -1372,29 +1372,22 @@ def _post_route( return False return True - def access(self, endpoints: list[str]) -> dict[str, Any] | None: - """Where a client reaches these endpoints, for the env-file descriptor. + def connection_info(self) -> ConnectionInfo | None: + """Where a client reaches the front door. - With the LiteLLM front door, that is one ``base_url`` and the request - model name is the endpoint alias itself. With LiteLLM off there is no - single base URL, but a managed Open WebUI (if on) is still a useful - access point, so report just its URL rather than ``None``. + With LiteLLM, one ``base_url``, the master key and its variable's + name. With LiteLLM off there is no single base URL, but a managed Open + WebUI (if on) is still a useful access point, so report just its URL + rather than ``None``. """ base_url, ui_url = self.urls() + proxy_url = (f'http://127.0.0.1:{self.reverse_proxy_port}' + if self.reverse_proxy and base_url is not None else None) if base_url is None: - return {'ui_url': ui_url} if ui_url else None - info: dict[str, Any] = { - 'base_url': base_url, - 'api_key_env': API_KEY_ENV, - 'api_key': self.master_key(), - 'request_names': {ep: ep for ep in endpoints}, - } - if ui_url: - info['ui_url'] = ui_url - if self.reverse_proxy: - # The unified front door: one origin, UI at / and the API at /v1. - info['proxy_url'] = f'http://127.0.0.1:{self.reverse_proxy_port}' - return info + return ConnectionInfo(None, ui_url=ui_url) if ui_url else None + return ConnectionInfo(base_url, api_key_env=API_KEY_ENV, + api_key=self.master_key(), ui_url=ui_url, + proxy_url=proxy_url) def front_door_urls(backend) -> tuple[str | None, str | None]: diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index c5d48f5a..83d71776 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -820,13 +820,12 @@ def test_acquire_json_redacts_api_key(env, capsys, monkeypatch): from infer_stack.cli import commands_leasing as cl from infer_stack.leasing import MemoryBackend + from infer_stack.leasing.backend import ConnectionInfo + class KeyedBackend(MemoryBackend): - def access(self, endpoints): - return { - 'base_url': 'http://x:1/v1', - 'api_key_env': 'LITELLM_MASTER_KEY', - 'api_key': 'sk-secret123', - } + def connection_info(self): + return ConnectionInfo('http://x:1/v1', api_key_env='LITELLM_MASTER_KEY', + api_key='sk-secret123') monkeypatch.setattr( cl, '_make_backend', @@ -1176,9 +1175,14 @@ def test_release_and_renew_name_a_missing_env_file(env, tmp_path): ReleaseCLI.main(argv=['--ledger', env.db, '--env-file', missing]) with pytest.raises(SystemExit, match='renew: no env-file at'): RenewCLI.main(argv=['--ledger', env.db, '--env-file', missing, '--ttl', '1h']) + # An env-file with no lease (an access to external endpoints only): there + # is nothing to release, and that is not an error. (tmp_path / 'lease.env').write_text('export OPENAI_BASE_URL=x\n') - with pytest.raises(SystemExit, match='names no lease'): + with pytest.raises(SystemExit) as info: ReleaseCLI.main(argv=['--ledger', env.db, '--env-file', missing]) + assert info.value.code == 0 + with pytest.raises(SystemExit, match='names no lease'): + RenewCLI.main(argv=['--ledger', env.db, '--env-file', missing, '--ttl', '1h']) def test_routes_seed_adds_but_refuses_to_redefine_without_replace(tmp_path, monkeypatch, capsys): diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 7c515404..c3fdb958 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -7,6 +7,8 @@ from __future__ import annotations +import json + import pytest import yaml @@ -377,3 +379,180 @@ def test_env_says_when_a_published_key_takes_effect(tmp_path, monkeypatch, capsy assert "remote send(s) $REMOTE_QWEN_KEY" in out and 'infer-stack apply' in out assert cl.EnvCLI.main(argv=['UNRELATED=1']) == 0 assert 'apply' not in capsys.readouterr().out + + +# -- items 35-36: access above leasing ------------------------------------------------ + + +def _keyed(tmp_path, cat, docker=None): + """A controller on a compose stack whose managed .env holds the remote key.""" + from infer_stack.env_utils import write_env_file + + ledger, ctl = _ctl(tmp_path, cat, docker) + write_env_file(ctl.backend.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + return ledger, ctl + + +def _demand(ledger): + leases, deployments = ledger.status() + return leases, deployments + + +def test_external_only_access_publishes_a_route_and_takes_no_lease(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + result = ctl.access('me', cat.resolve(['remote'])) + assert result.lease is None and result.external == ['remote'] + assert result.ready and result.front_door_ready is True + assert result.request_names == {'remote': 'remote'} + assert _demand(ledger) == ([], []) # zero ledger demand + assert ledger.publication_pending() is None + config = yaml.safe_load((ctl.backend.state_dir / 'litellm_config.yaml').read_text()) + assert 'remote' in [e['model_name'] for e in config['model_list']] + + +def test_a_mixed_bundle_leases_only_its_managed_member(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) # bundle 'pair' = local + remote + ledger, ctl = _keyed(tmp_path, cat) + result = ctl.access('me', cat.resolve(['pair'])) + assert result.endpoints == ['local', 'remote'] and result.external == ['remote'] + assert result.lease is not None and result.lease.endpoints == ['local'] + leases, deployments = _demand(ledger) + assert len(leases) == 1 and [sorted(d.served) for d in deployments] == [['local']] + ctl.release(result.lease.id) # releases only the real lease + assert {r.alias: r.origin for r in ctl.route_view()}['remote'] == 'external' + + +def test_access_to_an_external_endpoint_needs_the_front_door_and_its_key(tmp_path): + from infer_stack.leasing.profile import ProfileMismatch + + cat = Catalog.from_dict(catalog(remote=REMOTE)) + _, ctl = _ctl(tmp_path, cat) + with pytest.raises(ProfileMismatch, match='infer-stack env REMOTE_QWEN_KEY='): + ctl.access('me', cat.resolve(['remote'])) + _, lean = _ctl(tmp_path / 'lean', cat) + lean.backend.litellm = False + with pytest.raises(ProfileMismatch, match='LiteLLM front door'): + lean.access('me', cat.resolve(['remote'])) + + +def test_a_front_door_that_never_answers_releases_the_lease_access_took(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + ctl.FRONT_DOOR_WAIT_S = 0.0 + ctl.backend.gateway_accepts = lambda key, wait=0.0: None + result = ctl.access('me', cat.resolve(['pair'])) + assert result.front_door_ready is False and not result.ready + leases, _ = ledger.status() + assert [le.state for le in leases] == ['released'] + + +def test_a_live_managed_endpoint_is_not_silently_redefined_external(tmp_path): + from infer_stack.leasing.profile import ProfileMismatch + from test_leasing_compose import FakeDocker + + docker = FakeDocker() + managed = Catalog.from_dict(catalog()) + ledger, ctl = _ctl(tmp_path, managed, docker) + held = ctl.acquire('me', managed.resolve_requests(['local']), wait=False) + moved = Catalog.from_dict({'endpoints': {'local': REMOTE}}) + _, other = _keyed(tmp_path, moved, docker) + with pytest.raises(ProfileMismatch, match="'local'"): + other.access('me', moved.resolve(['local'])) + ctl.release(held.lease.id) + ctl.evict() # the keep-warm one too + result = other.access('me', moved.resolve(['local'])) # unpinned: it moves + assert result.external == ['local'] and result.lease is None + back = Catalog.from_dict(catalog()) # and back to managed + _, again = _ctl(tmp_path, back, docker) + out = again.access('me', back.resolve(['local'])) + assert out.lease is not None and out.external == [] + + +def test_external_routes_come_back_from_durable_state(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + _, ctl = _keyed(tmp_path, cat) + ctl.access('me', cat.resolve(['remote'])) + _, fresh = _ctl(tmp_path, None) # a new process, no catalog + routes = {r.alias: r for r in fresh.route_view()} + assert routes['remote'].api_base == 'http://box:8000/v1' + + +def test_routes_list_shows_what_the_gateway_renders(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + _, ctl = _keyed(tmp_path, cat) + ctl.access('me', cat.resolve(['remote'])) + config = yaml.safe_load((ctl.backend.state_dir / 'litellm_config.yaml').read_text()) + assert [r.entry() for r in ctl.route_view()] == config['model_list'] + + +# -- the CLI: access, run, acquire ------------------------------------------------------ + + +@pytest.fixture +def cli(tmp_path, monkeypatch): + """The CLI on a compose stack (fake docker, shared between commands).""" + from types import SimpleNamespace + + from infer_stack.cli import commands_leasing as cl + from infer_stack.env_utils import write_env_file + from test_leasing_compose import FakeDocker + from test_leasing_profile import backend + + docker = FakeDocker() + cat_path = tmp_path / 'catalog.yaml' + cat_path.write_text(yaml.safe_dump(catalog(remote=REMOTE))) + state = tmp_path / 'state' + + def make(config, *, interactive=False): + be = backend(state, catalog=Catalog.load(cat_path), docker=docker) + write_env_file(be.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + return be + + monkeypatch.setattr(cl, '_make_backend', make) + base = ['--ledger', str(tmp_path / 'ledger.db'), '--catalog', str(cat_path)] + return SimpleNamespace(cl=cl, base=base, tmp=tmp_path) + + +def test_access_cli_writes_a_descriptor_without_a_fake_lease_id(cli, capsys): + envf = cli.tmp / 'remote.env' + assert cli.cl.AccessCLI.main(argv=['remote', *cli.base, '--env-file', str(envf)]) == 0 + out = capsys.readouterr().out + assert 'no lease: every endpoint is external' in out + text = envf.read_text() + assert 'INFER_STACK_LEASE_ID' not in text + assert 'INFER_STACK_ENDPOINT_REMOTE=remote' in text and 'OPENAI_BASE_URL=' in text + with pytest.raises(SystemExit) as info: # nothing to release + cli.cl.ReleaseCLI.main(argv=[*cli.base[:2], '--env-file', str(envf)]) + assert info.value.code == 0 + + +def test_access_cli_mixed_bundle_and_release_by_env_file(cli, capsys): + envf = cli.tmp / 'pair.env' + assert cli.cl.AccessCLI.main(argv=['pair', *cli.base, '--env-file', str(envf), + '--json']) == 0 + data = json.loads(capsys.readouterr().out) + assert data['lease_id'] and data['external'] == ['remote'] + assert set(data['descriptor']['endpoints']) == {'local', 'remote'} + assert f"INFER_STACK_LEASE_ID={data['lease_id']}" in envf.read_text() + cli.cl.ReleaseCLI.main(argv=[*cli.base[:2], '--env-file', str(envf)]) + + +def test_acquire_points_an_external_endpoint_at_access(cli): + with pytest.raises(SystemExit, match=r'does not require a lease; use `infer-stack access remote`'): + cli.cl.AcquireCLI.main(argv=['remote', *cli.base]) + + +@pytest.mark.parametrize('names', ['remote', 'pair']) +def test_run_works_external_only_and_mixed(cli, names): + import sys + + probe = ('import os,sys; sys.exit(0 if os.environ.get("INFER_STACK_ENDPOINT_REMOTE")' + ' == "remote" and os.environ.get("OPENAI_BASE_URL") else 3)') + rc = cli.cl.RunCLI.main(argv=[*cli.base, '--endpoint', names, '--', sys.executable, '-c', probe]) + assert rc == 0 + from infer_stack.leasing import Ledger, SqliteStore + + leases, _ = Ledger(SqliteStore(cli.base[1])).status() + assert all(le.state == 'released' for le in leases) # run released its lease + assert len(leases) == (1 if names == 'pair' else 0) diff --git a/tests/test_leasing_compose.py b/tests/test_leasing_compose.py index 9afc109f..2e6bf568 100644 --- a/tests/test_leasing_compose.py +++ b/tests/test_leasing_compose.py @@ -39,6 +39,19 @@ PORTS = {'ollama': 11434} + +def access_info(be, endpoints): + """What a client is handed for ``endpoints`` (the descriptor's inputs), as + a dict: the backend's connection info and its request names.""" + info = be.connection_info() + if info is None: + return None + out = {k: v for k, v in vars(info).items() + if v is not None or (k == 'api_key_env' and info.base_url is not None)} + if info.base_url is not None: + out['request_names'] = be.request_names(list(endpoints)) + return out + def vllm(gid, *, hf='org/model', served=None, tp=1, max_len=32768, reclaim='keep-warm', protocol='chat', t=0.0): # endpoint (public alias) is the deployment id; served_model_name is the upstream @@ -637,7 +650,7 @@ def test_converge_writes_nginx_conf_and_access_reports_proxy(tmp_path): be.converge([vllm('a', served='aa')]) assert (tmp_path / 'nginx.conf').exists() assert 'reverse-proxy' in yaml.safe_load(be.compose_file.read_text())['services'] - info = be.access(['aa']) + info = access_info(be, ['aa']) assert info['proxy_url'] == 'http://127.0.0.1:8080' @@ -1059,9 +1072,9 @@ def test_litellm_router_settings_present(tmp_path): def test_access_includes_ui_url_when_ui_on(tmp_path): be = make_backend(tmp_path, ui=True) - assert be.access(['a'])['ui_url'] == 'http://127.0.0.1:13000' + assert access_info(be, ['a'])['ui_url'] == 'http://127.0.0.1:13000' be_noui = make_backend(tmp_path, ui=False) - assert 'ui_url' not in be_noui.access(['a']) + assert 'ui_url' not in access_info(be_noui, ['a']) def test_converge_diff_decline_aborts(tmp_path, monkeypatch): @@ -1196,7 +1209,7 @@ def test_controller_acquire_render_only_stages(tmp_path): def test_access_reports_litellm_base_url(tmp_path): be = make_backend(tmp_path) - info = be.access(['qwen-coder', 'reranker']) + info = access_info(be, ['qwen-coder', 'reranker']) assert info['base_url'] == 'http://127.0.0.1:14042/v1' assert info['api_key_env'] == 'LITELLM_MASTER_KEY' assert info['api_key'].startswith('sk-') # infer-stack manages the key @@ -1238,7 +1251,7 @@ def test_envfile_carries_managed_api_key(tmp_path): from infer_stack.leasing.models import Lease be = make_backend(tmp_path) - info = be.access(['qwen-coder']) + info = access_info(be, ['qwen-coder']) lease = Lease('sess-x', 'me', 'active', 0.0, None, None, 0.0, endpoints=['qwen-coder']) g = Deployment('g', 'ck', 'vllm', 'shared-compatible', {}, {}, @@ -1255,13 +1268,13 @@ def test_envfile_carries_managed_api_key(tmp_path): def test_access_none_without_litellm(tmp_path): # No gateway and no UI -> no single access point. be = make_backend(tmp_path, litellm=False, ui=False) - assert be.access(['x']) is None + assert access_info(be, ['x']) is None def test_access_reports_ui_url_without_litellm(tmp_path): # No gateway but a managed UI -> the UI is still a useful access point. be = make_backend(tmp_path, litellm=False, ui=True) - info = be.access(['x']) + info = access_info(be, ['x']) assert info == {'ui_url': 'http://127.0.0.1:13000'} diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 28edbbbc..d7dc5dca 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -7,6 +7,8 @@ from __future__ import annotations +from test_leasing_compose import access_info + import json from pathlib import Path @@ -345,7 +347,7 @@ def test_probe_ready_completions_protocol(tmp_path): def test_access_maps_endpoints_to_model_names(tmp_path): be, _ = make_backend(tmp_path, base_url='http://10.0.0.5:8000/openai/v1/') be.converge([vllm('grp-a', served='qwen-32b')]) - info = be.access(['grp-a']) + info = access_info(be, ['grp-a']) assert info['base_url'] == 'http://10.0.0.5:8000/openai/v1' assert info['api_key'] == 'EMPTY' # unauthenticated gateway assert info['api_key_env'] is None @@ -486,7 +488,7 @@ def test_corrupt_sidecar_degrades_gracefully(tmp_path): be, _ = make_backend(tmp_path) be.converge([vllm('grp-a', served='qwen')]) be._state_file.write_text('{not json') - info = be.access(['grp-a']) + info = access_info(be, ['grp-a']) # falls back to the slug-of-endpoint guess rather than crashing assert info['request_names'] == {'grp-a': 'grp-a'} be.apply() # and apply still works (prunes against an empty wanted-set... @@ -641,7 +643,7 @@ def test_with_a_gateway_clients_use_the_alias_and_the_managed_key(tmp_path): dep = vllm('grp-a', served='Qwen/Qwen2.5-0.5B') dep.served = {'tiny': {'served_model_name': 'Qwen/Qwen2.5-0.5B', 'protocol': 'chat'}} be.converge([dep]) - info = be.access(['tiny']) + info = access_info(be, ['tiny']) assert info['request_names'] == {'tiny': 'tiny'} # the alias, as on compose assert info['base_url'].startswith('http://127.0.0.1:') assert info['api_key'] == be.gateway.master_key() @@ -653,7 +655,7 @@ def test_without_a_gateway_clients_still_see_kubeai_directly(tmp_path): be, _ = make_backend(tmp_path) be.converge([vllm('grp-a', served='qwen-32b')]) assert be.litellm is False - assert be.access(['grp-a'])['request_names'] == {'grp-a': 'qwen-32b'} + assert access_info(be, ['grp-a'])['request_names'] == {'grp-a': 'qwen-32b'} def test_rotating_the_key_recreates_the_gateway_in_front_of_the_cluster(tmp_path): @@ -1063,7 +1065,7 @@ def test_the_in_cluster_gateway_routes_by_cluster_dns(tmp_path): 'http://kubeai.kubeai.svc.cluster.local/openai/v1' assert kubectl.gateway_objects['Service']['spec']['type'] == 'NodePort' assert kubectl.rollouts == 1 - assert be.access(['tiny'])['base_url'] == 'http://10.0.0.7:30442/v1' + assert access_info(be, ['tiny'])['base_url'] == 'http://10.0.0.7:30442/v1' assert be.compose_project() is None and be.front_door() is be.gateway diff --git a/tests/test_parity.py b/tests/test_parity.py index 0511aeb5..aba859f4 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -10,6 +10,8 @@ from __future__ import annotations +from test_leasing_compose import access_info + import json from dataclasses import dataclass from typing import Any @@ -223,7 +225,7 @@ def make(**gateway_kw) -> Stack: def test_one_base_url_the_managed_key_and_the_alias(make_stack): stack = make_stack() stack.acquire('one') - info = stack.backend.access(['one']) + info = access_info(stack.backend, ['one']) assert info['base_url'].startswith('http://127.0.0.1:') assert info['api_key'] == stack.front.master_key() assert info['request_names'] == {'one': 'one'} From 3562bcf5849a67910f36291e749f3fd483168592 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:26:01 -0400 Subject: [PATCH 115/141] Show external endpoints as external: status, TUI catalog status lists published external endpoints in their own section (no pseudo-deployment rows); the TUI catalog shows them with engine `external` and their upstream model, and its endpoint editor refuses one rather than saving it back as a vLLM endpoint. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 8 +++++- infer_stack/cli/commands_runtime.py | 40 +++++++++++++++++++++++++++++ infer_stack/leasing/endpoints.py | 13 ++++++++++ infer_stack/tui.py | 12 +++++++++ tests/test_external_endpoints.py | 20 +++++++++++++++ 5 files changed, 92 insertions(+), 1 deletion(-) diff --git a/docs/queue.md b/docs/queue.md index 22848a9e..3319b5ca 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1267,7 +1267,13 @@ through access. `release` of external-only access has nothing to release. Access is a typed capability, not a `getattr(backend, 'access')` hook (cleanup D). -### 37. [ ] Views: `external` +### 37. [x] Views: `external` +*Done 2026-09-27.* `status` lists published external endpoints in their own +section (alias, upstream model, server; "no lease"); `leases` is unchanged; +`routes list` shows origin `external`; the TUI's catalog shows engine +`external` with the upstream model, and its endpoint editor refuses an +external endpoint (saving would have turned it into a vLLM one). `catalog +show` prints the `external:` block as written. Status, TUI and catalog views show external targets as `external`. *Review:* no pseudo-deployment rows: `leases` stays leases and deployments. Show external endpoints in an endpoint/routing section diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 5ca4e5aa..f5583f78 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -144,6 +144,31 @@ def _leasing_status() -> dict[str, Any]: ] out['summary'] = (active, len(leases), live, len(deployments)) out['live_deployments'] = [d for d in deployments if d.state == DeploymentState.LIVE] + out['external'] = _published_external(ledger) + return out + + +def _published_external(ledger) -> list[tuple[str, str, str]]: + """``(alias, upstream model, api_base)`` for each published external + endpoint: routed by the front door, held by no lease.""" + from ..leasing.endpoints import external_endpoints + from ..leasing.profile import CatalogUnion + + try: + sources = (ledger.profile() or {}).get('catalogs') or [] + union = CatalogUnion.from_sources(sources) if sources else None + except Exception: # noqa: BLE001 - status never fails on this + return [] + return [(alias, t.model, t.api_base) for alias, t in external_endpoints(union)] + + +def _external_lines(rows) -> list[str]: + """Plain-text 'external endpoints' block; empty when there are none.""" + if not rows: + return [] + width = max(8, *(len(r[0]) for r in rows)) + out = ['', 'external endpoints (through the front door; no lease)'] + out += [f' {alias.ljust(width)} {model} @ {base}' for alias, model, base in rows] return out @@ -285,6 +310,8 @@ def _print_status_plain(d: dict[str, Any]) -> None: f'{live} live / {total_d} deployment(s) (infer-stack leases)') for line in _served_lines(lz.get('served') or []): print(line) + for line in _external_lines(lz.get('external') or []): + print(line) if not d['configured']: print() for cmd, comment in _GETTING_STARTED: @@ -362,6 +389,19 @@ def _print_status_rich(d: dict[str, Any], console) -> None: ' STALE = recorded live but nothing is running for it; ' '`infer-stack apply` or `gc`', style='dim')) + external = lz.get('external') or [] + if external: + ext_table = Table(box=None, pad_edge=False, padding=(0, 2, 0, 0)) + ext_table.add_column('endpoint', style='bold cyan', no_wrap=True) + ext_table.add_column('upstream model', overflow='fold') + ext_table.add_column('server', style='dim', overflow='fold') + for alias, model, base in external: + ext_table.add_row(alias, model, base) + console.print() + console.print(Text('external endpoints (through the front door; no lease)', + style='bold')) + console.print(ext_table) + console.print() if d['configured']: console.print(Text('dig deeper', style='bold')) diff --git a/infer_stack/leasing/endpoints.py b/infer_stack/leasing/endpoints.py index 13a5f292..8c978657 100644 --- a/infer_stack/leasing/endpoints.py +++ b/infer_stack/leasing/endpoints.py @@ -118,6 +118,19 @@ class ExternalTarget: kind = 'external' +def external_endpoints(catalog: Any) -> list[tuple[str, ExternalTarget]]: + """``[(alias, target), ...]`` for every external endpoint of ``catalog``.""" + out = [] + for alias in sorted(getattr(catalog, 'endpoints', None) or {}): + try: + target = catalog.resolve_endpoint(alias).target + except Exception: # noqa: BLE001 - an invalid endpoint is not listed + continue + if isinstance(target, ExternalTarget): + out.append((alias, target)) + return out + + def key_references(catalog: Any) -> dict[str, list[str]]: """``{variable: [endpoint alias, ...]}`` for every external endpoint of ``catalog`` that names a key. diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 1762a244..b1ac1b74 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1549,6 +1549,12 @@ def _fill_catalog(self) -> None: self._endpoint_names = [] for name in sorted(self.catalog.endpoints): ep = self.catalog.endpoints[name] + external = getattr(ep, 'external', None) + if external is not None: + # A server that runs elsewhere: no model entry, GPU or reclaim. + eps.add_row(name, external.model, 'external', '-', '-') + self._endpoint_names.append(name) + continue indices = (getattr(ep, 'placement', {}) or {}).get('gpu_indices') gpu = ','.join(str(i) for i in indices) if indices else 'auto' eps.add_row( @@ -2931,6 +2937,12 @@ def action_edit_endpoint(self) -> None: return from .cli.commands_catalog import _load_raw entry = _load_raw(self.catalog_path)['endpoints'].get(name, {}) + if 'external' in (entry or {}): + # The editor describes a runtime infer-stack runs; saving would + # turn the external endpoint into one. + self._refuse(f'{name} is external — edit it with `infer-stack catalog ' + 'endpoint add --force --external-...` or `catalog edit`') + return self._status(f'inspecting GPUs for {name} editor…') self._prepare_endpoint_editor(name, entry) diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index c3fdb958..ad204271 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -556,3 +556,23 @@ def test_run_works_external_only_and_mixed(cli, names): leases, _ = Ledger(SqliteStore(cli.base[1])).status() assert all(le.state == 'released' for le in leases) # run released its lease assert len(leases) == (1 if names == 'pair' else 0) + + +# -- item 37: views -------------------------------------------------------------------- + + +def test_status_lists_published_external_endpoints_apart_from_deployments(tmp_path, monkeypatch): + from infer_stack.cli import commands_runtime as rt + from infer_stack.leasing import Ledger, SqliteStore + from infer_stack.leasing.profile import catalog_sources + + db = tmp_path / 'ledger.db' + ledger = Ledger(SqliteStore(str(db))) + ledger.set_profile({'backend': 'compose', 'catalogs': catalog_sources( + Catalog.from_dict(catalog(remote=REMOTE)))}) + import infer_stack.leasing as leasing + monkeypatch.setattr(leasing, 'default_ledger_path', lambda: db) + status = rt._leasing_status() + assert status['external'] == [('remote', 'Qwen/Qwen3-32B', 'http://box:8000/v1')] + assert status['deployments'] == [] # no pseudo-deployment + assert 'remote' in '\n'.join(rt._external_lines(status['external'])) From 19fc8fd8cccdceab2b529b054fff13ee1d2cd9b3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:36:19 -0400 Subject: [PATCH 116/141] External e2e covers access; bundles survive a member's redefinition dev/external_e2e.sh adds external-only, managed-only (llm-d-inference-sim on CPU) and mixed-bundle access, and a managed alias moved to an external server, in static and dynamic routing. Passes on the guest. It found that redefining a bundle member left the published copy of the bundle naming an endpoint its source had dropped, so the union could not be built; drop_catalog_names drops such bundles. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/external_e2e.sh | 95 ++++++++++++++++++++++++++++++- dev/journals/claude.md | 21 +++++++ docs/queue.md | 15 ++++- infer_stack/leasing/controller.py | 6 +- infer_stack/leasing/profile.py | 15 ++++- tests/test_external_endpoints.py | 27 +++++++++ 6 files changed, 169 insertions(+), 10 deletions(-) diff --git a/dev/external_e2e.sh b/dev/external_e2e.sh index c2ceb47b..cb9a9022 100755 --- a/dev/external_e2e.sh +++ b/dev/external_e2e.sh @@ -14,15 +14,25 @@ # sending the key from the managed .env by name (`os.environ/NAME`); # 3. the key changes: `env` says an apply is needed, the old key is refused # by the upstream until `apply`, the new key is sent after it; -# 4. `routes prune` unpublishes it and the alias stops routing. +# 4. `routes prune` unpublishes it and the alias stops routing; +# 5. `access` of the external endpoint alone: no lease, no lease id in the +# env-file, a client that knows only the env-file reaches it; +# 6. `access` of a managed endpoint alone (llm-d-inference-sim, on CPU): +# one lease, the same env-file shape; +# 7. `access` of a mixed bundle: one lease for the managed member only; +# `release --env-file` releases it and the external route stays; +# 8. the managed alias becomes external after release and eviction: +# same alias, same workflow, no lease. # # Knobs (env): # E2E_MODES routing modes to run (default "static dynamic") # E2E_PORT the mock's port (default 18911) +# E2E_SIM the simulator image (default ghcr.io/llm-d/llm-d-inference-sim:v0.9.0) set -euo pipefail MODES="${E2E_MODES:-static dynamic}" PORT="${E2E_PORT:-18911}" +SIM="${E2E_SIM:-ghcr.io/llm-d/llm-d-inference-sim:v0.9.0}" HOST_IP="$(ip -4 -o addr show docker0 | awk '{print $4}' | cut -d/ -f1)" [ -n "$HOST_IP" ] || { echo 'no docker0 address' >&2; exit 1; } @@ -74,6 +84,24 @@ chat() { # chat ALIAS -> http status; body in $WORK/chat. \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}]}" || true } +chat_env() { # chat_env ENV_FILE ALIAS -> http status, as a card would + ( + # shellcheck disable=SC1090 + source "$1" + var="INFER_STACK_ENDPOINT_$(echo "$2" | tr '[:lower:]-' '[:upper:]_')" + curl -s -o "$WORK/chat.json" -w '%{http_code}' "$OPENAI_BASE_URL/chat/completions" \ + -H "Authorization: Bearer $OPENAI_API_KEY" -H 'Content-Type: application/json' \ + -d "{\"model\": \"${!var}\", \"max_tokens\": 8, + \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}]}" || true + ) +} + +leases_active() { # how many ACTIVE leases the ledger holds + run_is leases --json | python -c ' +import json, sys +print(sum(1 for le in json.load(sys.stdin).get("leases", []) if le.get("state") == "active"))' +} + chat_until() { # chat_until ALIAS STATUS: routes settle after apply local got='' for _ in $(seq 60); do @@ -139,6 +167,71 @@ EOF if run_is routes list --json | grep -q '"remote"'; then fail 'still listed after prune'; fi chat_until remote 400 + # The runbook's catalog now names a managed endpoint, the external one, + # and a bundle of both. + cat > "$WORK/config/catalog.yaml" < "$WORK/pair.json" + python - "$WORK/pair.json" <<'PY' || fail "pair: $(cat "$WORK/pair.json")" +import json, sys +d = json.load(open(sys.argv[1])) +assert d['lease_id'] and d['external'] == ['remote'], d +assert set(d['descriptor']['endpoints']) == {'local', 'remote'}, d +PY + [ "$(leases_active)" = 1 ] || fail 'a mixed bundle took other than one lease' + [ "$(chat_env "$WORK/pair.env" local)" = 200 ] || fail "pair/local: $(cat "$WORK/chat.json")" + [ "$(chat_env "$WORK/pair.env" remote)" = 200 ] || fail "pair/remote: $(cat "$WORK/chat.json")" + run_is release --env-file "$WORK/pair.env" --yes + [ "$(leases_active)" = 0 ] || fail 'the bundle lease is still held' + [ "$(chat_env "$WORK/pair.env" remote)" = 200 ] || fail 'remote stopped routing after release' + + echo "== [$mode] 8. the managed alias moves to an external server" + run_is evict --all --yes + python - "$WORK/config/catalog.yaml" <<'PY' +import sys, yaml +path = sys.argv[1] +cat = yaml.safe_load(open(path)) +cat['endpoints']['local'] = {'external': dict(cat['endpoints']['remote']['external'])} +yaml.safe_dump(cat, open(path, 'w'), sort_keys=False) +PY + run_is access local --yes --env-file "$WORK/moved.env" + if grep -q INFER_STACK_LEASE_ID "$WORK/moved.env"; then fail 'moved alias took a lease'; fi + [ "$(chat_env "$WORK/moved.env" local)" = 200 ] || fail "moved: $(cat "$WORK/chat.json")" + [ "$(last_bearer)" = 'Bearer key-two' ] || fail "moved sent $(last_bearer)" + run_is stack down >/dev/null 2>&1 || true stop_mock rm_work "$WORK" diff --git a/dev/journals/claude.md b/dev/journals/claude.md index 17cf0210..73a3b2cb 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3425,3 +3425,24 @@ and merging rewrites the union's sources. It now compares endpoint meaning. Also: LiteLLM's upstream request lowercases `authorization`, and Postgres leaves its data dir owned by its own uid (the script removes it through a container). + +## 2026-09-27 — items 35-40: access above leasing, and the e2e that checked it + +**Did.** `Controller.access` over `acquire` (managed members, one lease) and +`publish_endpoints` (external-only: the same preview, approved marker and +publication, no ledger mutation); typed `ConnectionInfo` / +`request_names()` replace the `access` dict hook; `infer-stack access`, +`run` on top; lease-less env-files; `status` and TUI views; the +`FrontDoorControl` / `RenderedFrontDoor` names. `dev/external_e2e.sh` +covers eight phases in both routing modes on the guest. + +**Found by the e2e.** Redefining a bundle member (managed -> external) left +the published copy of the bundle naming an endpoint its own source had +dropped; every source must validate alone, so `drop_catalog_names` now +drops such bundles. The first external-only access on a fresh ledger +published nothing: freezing the snapshot consumed the candidate. Access now +always publishes. + +**Mistake.** A nested heredoc (`< external after release; external under dynamic routing if practical. Run from a frozen copy. diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index e7ee6355..489bfd69 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -2166,12 +2166,8 @@ def change(): kept = {k: v for k, v in legacy.items() if k not in drop} if kept != legacy: gateway.replace_route_entries(kept) - sources = self._published_sources() # A bundle naming an unpublished endpoint goes with it. - bundles = {b for src in sources - for b, members in (src.get('bundles') or {}).items() - if set(members or []) & set(drop)} - self._store_catalogs(drop_catalog_names(sources, set(drop) | bundles)) + self._store_catalogs(drop_catalog_names(self._published_sources(), set(drop))) return drop return self.publish_change(change) diff --git a/infer_stack/leasing/profile.py b/infer_stack/leasing/profile.py index 1e5e5ee4..f8f33816 100644 --- a/infer_stack/leasing/profile.py +++ b/infer_stack/leasing/profile.py @@ -286,13 +286,24 @@ def drop_catalog_names( sources: list[dict[str, Any]], names: set[str] ) -> list[dict[str, Any]]: """``sources`` without the endpoints and bundles named ``names``; a source - left with neither goes.""" + left with neither goes. + + A bundle goes too when a member it names is dropped: a catalog's bundles + name its own endpoints, and a source must stay a valid catalog. (The + catalog that redefined the member publishes its own bundles.) + + Example: + >>> src = {'endpoints': {'a': {}, 'b': {}}, 'bundles': {'ab': ['a', 'b']}} + >>> drop_catalog_names([src], {'a'}) + [{'endpoints': {'b': {}}, 'bundles': {}}] + """ out: list[dict[str, Any]] = [] for source in sources or []: endpoints = {n: s for n, s in (source.get('endpoints') or {}).items() if n not in names} + dropped = set(source.get('endpoints') or {}) - set(endpoints) bundles = {n: m for n, m in (source.get('bundles') or {}).items() - if n not in names} + if n not in names and not set(m or []) & dropped} if not endpoints and not bundles: continue kept = dict(source) diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index ad204271..ddc2b3a2 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -576,3 +576,30 @@ def test_status_lists_published_external_endpoints_apart_from_deployments(tmp_pa assert status['external'] == [('remote', 'Qwen/Qwen3-32B', 'http://box:8000/v1')] assert status['deployments'] == [] # no pseudo-deployment assert 'remote' in '\n'.join(rt._external_lines(status['external'])) + + +def test_two_catalogs_agree_on_an_external_alias_or_conflict(): + from infer_stack.leasing.profile import CatalogConflict, CatalogUnion + + a = {'endpoints': {'remote': REMOTE}} + b = {'models': {'x': {'source': 'hf://o/x'}}, 'endpoints': {'remote': REMOTE}} + union = CatalogUnion.from_sources([a, b]) # same alias, same target + assert list(union.endpoints) == ['remote'] + other = {'endpoints': {'remote': {**REMOTE, 'external': { + **REMOTE['external'], 'model': 'Other/Model'}}}} + with pytest.raises(CatalogConflict, match="'remote'"): + CatalogUnion.from_sources([a, other]) + + +def test_moving_a_bundled_endpoint_to_external_keeps_the_union_valid(): + """Found by dev/external_e2e.sh phase 8: redefining a bundle member left the + published copy of the bundle naming an endpoint its source no longer had.""" + from infer_stack.leasing.profile import CatalogUnion, adopt_catalog_sources + + before = catalog(remote=REMOTE) # pair = local + remote + after = catalog(remote=REMOTE) + after['endpoints']['local'] = {'external': dict(REMOTE['external'])} + sources = adopt_catalog_sources([before], [after], pinned=set()) + union = CatalogUnion.from_sources(sources) + assert not union.resolve_endpoint('local').managed + assert union.bundles['pair'] == ['local', 'remote'] From 501688e8c1b980c4c31db1c2064cabd664156335 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:40:32 -0400 Subject: [PATCH 117/141] Document external endpoints and access; audit covers access README (catalog model, main commands, render shapes), the catalog schema docstring and the design doc (status, the workflow, moving an alias between managed and external) describe external endpoints and `access`. The journal records the alternatives and why they lost. dev/ux_audit.sh now exercises `access` (a mistake and a day-2 run); 0 flags on compose. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- README.md | 17 +++++++++++++++ dev/journals/claude.md | 29 +++++++++++++++++++++++++ dev/ux_audit.sh | 3 +++ docs/planning/external-endpoints.md | 33 ++++++++++++++++++++++++++--- docs/queue.md | 8 ++++++- infer_stack/leasing/catalog.py | 10 +++++++++ 6 files changed, 96 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5c89183f..8a028c67 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ infer-stack config init # data dir + default backend -> settings.y infer-stack catalog suggest --apply # seed catalog.yaml from this host's GPUs infer-stack catalog show # what can be acquired infer-stack acquire # lease, render, bring up, wait for a real generation +infer-stack access --env-file e.env # reach it, managed or external; leases only what runs here infer-stack test # one generation through the gateway infer-stack leases # desired vs running, per deployment infer-stack status # paths, backend and a lease summary @@ -162,6 +163,21 @@ sections: * `runtime_hosts`: Ollama daemons, each with its GPUs and daemon settings; * `bundles`: named lists of endpoints to acquire together. +An endpoint can instead name a server that already runs elsewhere, with +`external: {api_base, model, api_key_env}` in place of `engine`/`model`. +It has no lease: `infer-stack access` (or `run`) publishes its route on the +LiteLLM front door, and the client asks for the alias through the same base +URL as any other endpoint. Moving an alias between a managed runtime and an +external server changes neither the alias nor the workflow: + +```bash +infer-stack env REMOTE_QWEN_KEY=sk-... # the upstream's key, by name +infer-stack access qwen-remote --env-file qwen.env +source qwen.env # OPENAI_BASE_URL, OPENAI_API_KEY, ... +``` + +See [docs/planning/external-endpoints.md](docs/planning/external-endpoints.md). + ```yaml models: smol135: @@ -195,6 +211,7 @@ Shapes the stack renders: ```text Open WebUI -> LiteLLM -> vLLM / Ollama # the default + \-> an external OpenAI-compatible server (external:) Open WebUI -> vLLM / Ollama # config set litellm false ``` diff --git a/dev/journals/claude.md b/dev/journals/claude.md index 73a3b2cb..e01c0c0f 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3446,3 +3446,32 @@ always publishes. **Mistake.** A nested heredoc (`</dev/null | sed -E 's/^[│ ├└─]+//' \ echo '== mistakes: each names its cause, none is a traceback' run acquire nope --yes +run access nope --yes run release lease-nope --yes run wait nope --timeout 5 run logs nope @@ -104,3 +105,5 @@ for cmd in leases status ps env "env OPENAI_BASE_URL" doctor clean gc \ run $cmd done run release --env-file "$WORK/lease.env" --yes +run access "$ENDPOINT" --yes --timeout 600 --env-file "$WORK/access.env" +run release --env-file "$WORK/access.env" --yes diff --git a/docs/planning/external-endpoints.md b/docs/planning/external-endpoints.md index 94b09019..2b73482a 100644 --- a/docs/planning/external-endpoints.md +++ b/docs/planning/external-endpoints.md @@ -1,8 +1,35 @@ # External endpoints, and access above leasing (design) -**Status:** design (queue item 28), 2026-09-27. Implementation: queue items -29-41 in [../queue.md](../queue.md). Parent: the backend-parity campaign -([backend-parity-roadmap.md](backend-parity-roadmap.md)). +**Status:** implemented, 2026-09-27 (queue items 28-41 in +[../queue.md](../queue.md); decision 2 revised during implementation, see +there). Parent: the backend-parity campaign +([backend-parity-roadmap.md](backend-parity-roadmap.md)). Verified by +`tests/test_external_endpoints.py` and `dev/external_e2e.sh` (real LiteLLM, +static and dynamic routing). + +## The workflow + +```yaml +endpoints: + qwen: + external: + api_base: http://gpu-box:8000/v1 + model: Qwen/Qwen3-32B + api_key_env: REMOTE_QWEN_KEY +``` + +```bash +infer-stack env REMOTE_QWEN_KEY=sk-... # the key, by name; never in the catalog +infer-stack access qwen --env-file qwen.env # publish the route; no lease +source qwen.env && my-eval --model "$INFER_STACK_ENDPOINT_QWEN" +``` + +Switching `qwen` back to a managed runtime (`engine: vllm`, `model: ...`) +changes neither the alias nor these commands: `access` then takes a lease and +waits for the model, and the env-file gains `INFER_STACK_LEASE_ID`. A bundle +may mix both; it takes one lease, for its managed members. `run --endpoint +qwen -- CMD` does the same around a command. `acquire` stays the lease +operation and points an external endpoint at `access`. An endpoint is the public name a workflow asks for (`qwen`). Today every endpoint is a model infer-stack runs, so asking for one means taking a diff --git a/docs/queue.md b/docs/queue.md index f9dc13ff..c099943d 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1325,7 +1325,13 @@ No internet provider. Phases: external only; managed only; mixed bundle; managed alias -> external after release; external under dynamic routing if practical. Run from a frozen copy. -### 41. [ ] Docs, journal, verification +### 41. [x] Docs, journal, verification +*Done 2026-09-27.* Design doc: status, the workflow, switching back to +managed; README (catalog model, main commands, shapes); the catalog schema +docstring; routing doc. Journal: the four alternatives and why not. Suite +(1094), ty, flake8, `dev/ux_audit.sh compose` (0 flags, now covering +`access`), `dev/external_e2e.sh` (both routing modes). Not run here: +`ux_audit.sh kubeai` and a KubeAI external e2e. Design doc: Endpoint (public contract) / Fulfillment (managed runtime or external upstream) / Lease (only when infer-stack owns runtime), with the workflow (`external:` in the catalog; `infer-stack access qwen --env-file diff --git a/infer_stack/leasing/catalog.py b/infer_stack/leasing/catalog.py index bbeb4c7b..db934ccb 100644 --- a/infer_stack/leasing/catalog.py +++ b/infer_stack/leasing/catalog.py @@ -56,6 +56,16 @@ engine: ollama host: local-ollama model: qwen3.5:4b + # An OpenAI-compatible server that already runs elsewhere: no model + # entry, no lease; the LiteLLM front door routes the alias to it, sending + # the key held in the named variable (`infer-stack env NAME=...`). + # Reached with `infer-stack access` or `run`, not `acquire`. + qwen-remote: + external: + api_base: http://gpu-box:8000/v1 + model: Qwen/Qwen3-32B # what that server expects as `model` + api_key_env: REMOTE_QWEN_KEY + protocol: chat runtime_hosts: # Ollama daemons (one daemon, many tags) local-ollama: From 8797fddf6cf862457c3891e03f90c2a3cc6efc66 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:47:18 -0400 Subject: [PATCH 118/141] routes seed --replace is compare-and-swap; CLI decisions use typed members The seed plan carries the meaning each redefinition showed; under the lock an alias another process changed since refuses (nothing written, no publication pending). The CLI no longer decides on getattr probes: compose_project(), down(), doctor(), deployment_logs(), runs_engines, catalog and measurements are ServingBackend members (in-process defaults on SimpleAdmission), and pull_images() / upstream_check() are HostRuntime members that network migrate, network check and config publish --pull reach through backend.host_runtime. The getattrs left serve display only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 3 ++ docs/planning/known-limitations.md | 12 +++---- docs/queue.md | 23 +++++++------ infer_stack/backends/kubeai.py | 3 ++ infer_stack/cli/commands_leasing.py | 29 ++++++++-------- infer_stack/cli/commands_runtime.py | 13 +++----- infer_stack/leasing/backend.py | 52 +++++++++++++++++++++++++++++ infer_stack/leasing/compose.py | 3 ++ infer_stack/leasing/controller.py | 11 +++++- infer_stack/leasing/routes.py | 21 +++++++++--- tests/test_cli_leasing.py | 35 +++++++++++++++++++ tests/test_day2.py | 5 ++- 12 files changed, 164 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 966b110b..bc7e4c92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). ### Hardening from a review of the backend-parity work +- `routes seed --replace` replaces exactly what it showed: an alias another + process redefined meanwhile refuses, and nothing is written. + - A KubeAI Model with several replicas (`min_replicas`) is resident like one pod: releasing its last keep-warm lease no longer prunes it, `leases` shows it running, and it fails fast when every replica crash-loops. diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index a951b5a2..7e910805 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -314,12 +314,10 @@ migrated keep Docker's dynamic addresses. - setting `AIOHTTP_TTL_DNS_CACHE` low on the gateway narrows the window but does not close it. -### `routes seed --replace` confirms against a snapshot (current, deferred) +### `routes seed --replace` is compare-and-swap (fixed 2026-09-27) `routes seed --replace` shows each redefinition (`A0 -> A1`) and asks, then -commits under the publication lock. If another process changes the alias -to `A2` in between, the commit writes `A1` over `A2` without showing `A2`: -the replace is "replace whatever is there", not compare-and-swap. A -conflict that appears without `--replace` still refuses, and leaves no -publication pending. A strict version would carry the expected old row in -the plan and ask again on a mismatch under the lock. +commits under the publication lock only if every alias still means what was +shown. If another process changed one to `A2` in between, nothing is +written, no publication is left pending, and the refusal says to run the +seed again to see `A2`. diff --git a/docs/queue.md b/docs/queue.md index c099943d..fb8539e2 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -951,16 +951,19 @@ campaign 2. None blocks campaign 2. - [ ] **In-cluster gateway: deferred features (P5).** Dynamic routing and Open WebUI with `kubeai_gateway cluster` (they need Postgres and a UI in the cluster), and an Ingress (`kubeai_gateway_url` accepts one; none tested). -- [ ] **`routes seed --replace` compare-and-swap.** Deferred; see - `known-limitations.md`. Carry the expected old row in the plan and ask - again on a mismatch under the lock. -- [ ] **CLI capability probes that decide behaviour.** The controller has no - optional-hook `getattr` left, but the CLI still branches on - `getattr(backend, ...)` for `down`, `doctor`, `compose_project`, - `rendered_file`/`compose_file`, `deployment_logs` and `access` (the last is - campaign 2's cleanup D). By the re-review's rule a `getattr` default may - serve display, never a decision: make the deciding ones protocol members - or typed capabilities. +- [x] **`routes seed --replace` compare-and-swap.** *Done 2026-09-27:* the + plan carries each alias's shown meaning; under the lock a changed one + refuses (`RouteConflict(changed=True)`), writes nothing, leaves no marker. +- [x] **CLI capability probes that decide behaviour.** *Done 2026-09-27:* + `compose_project()`, `down()`, `doctor()`, `deployment_logs()` and + `runs_engines` are `ServingBackend` members (in-process defaults on + `SimpleAdmission`); `pull_images()` and `upstream_check()` are + `HostRuntime` members, and `network migrate` / `network check` / `config + publish --pull` ask `backend.host_runtime`; `access` became + `connection_info()` / `request_names()` (campaign 2). The `getattr`s left + in the CLI serve display only (file paths in JSON, the GPU inventory in an + OOM hint, the cache path in `measure`'s narration). `catalog` and + `measurements` are protocol attributes (`None` in-process). - [ ] **TUI catalog-reload flake.** The fix counts only catalog refusals; the cause (a background worker refusing meanwhile) was inferred, not reproduced. Reopen if it recurs. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index b17b148f..deb65136 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -566,6 +566,9 @@ def models_file(self) -> Path: #: The file a render writes, whatever the backend (``status`` shows it). rendered_file = models_file + #: Its engines are pods in the cluster. + runs_engines = True + def compose_project(self): """The Compose project on this host: the gateway's, or ``None`` (no gateway, or the gateway runs in the cluster).""" diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index e641fe9e..d52eee03 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -412,7 +412,7 @@ def _requests_catalog(controller, config): """ from ..leasing.profile import CatalogUnion - published = getattr(controller.backend, 'catalog', None) + published = controller.backend.catalog explicit = ( getattr(config, 'catalog', None) or os.environ.get('INFER_STACK_CATALOG', '').strip() @@ -601,8 +601,8 @@ def _oom_hints(controller, outcome) -> list[str]: from ..leasing.vram import looks_like_cuda_oom backend = controller.backend - logs_fn = getattr(backend, 'deployment_logs', None) - if logs_fn is None or outcome.wait is None or outcome.wait.ready: + logs_fn = backend.deployment_logs + if outcome.wait is None or outcome.wait.ready: return [] by_id = {g.id: g for g in outcome.deployments} try: @@ -612,7 +612,7 @@ def _oom_hints(controller, outcome) -> list[str]: } except Exception: memory = {} - assignments = dict(getattr(backend, 'last_assignments', {}) or {}) + assignments = dict(backend.last_assignments or {}) hints: list[str] = [] for gid, endpoint in outcome.wait.pending: deployment = by_id.get(gid) @@ -1783,8 +1783,8 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) backend = controller.backend - logs_fn = getattr(backend, 'deployment_logs', None) - if logs_fn is None: + logs_fn = backend.deployment_logs + if not backend.runs_engines: raise SystemExit( 'measure reads the engine\'s log, and this backend runs no engine ' '(set `--backend compose` or `kubeai`).' @@ -1858,7 +1858,7 @@ def main(cls, argv=True, **kwargs): ) recorded_to = None if config.record: - store = getattr(backend, 'measurements', None) + store = backend.measurements if store is None: raise SystemExit( '--record: this backend keeps no measurements overlay.' @@ -3019,7 +3019,8 @@ def main(cls, argv=True, **kwargs): try: if profile.get('catalogs'): CatalogUnion.from_sources(profile['catalogs']) # conflicts - pull = getattr(controller.backend, 'pull_images', None) + host = controller.backend.host_runtime + pull = host.pull_images if host is not None else None if config.pull and pull is not None and profile.get('backend') == 'compose': # Outside the lock, before anything is published: a steady-state # apply must never wait on a registry, and a missing image must @@ -3072,7 +3073,7 @@ def main(cls, argv=True, **kwargs): if not config.subnet: raise SystemExit('network migrate: --subnet is required') controller = _open_controller(config, interactive=True) - if not hasattr(controller.backend, 'network'): + if controller.backend.host_runtime is None: raise SystemExit('network migrate gives compose containers stable ' 'addresses; this backend runs none (n/a on kubeai)') try: @@ -3103,10 +3104,10 @@ class NetworkCheckCLI(_LeasingCommonMixin): def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config) - check = getattr(controller.backend, 'upstream_check', None) - if check is None: + host = controller.backend.host_runtime + if host is None: raise SystemExit('network check needs the compose backend') - result = check() + result = host.upstream_check() if config.json: print(json.dumps(result, indent=2)) else: @@ -3179,8 +3180,8 @@ def main(cls, argv=True, **kwargs): 'rejects the OLD key') else: print(' gateway: new key accepted, old key rejected') - front = getattr(backend, 'compose_project', lambda: None)() - if getattr(front, 'ui', False): + front = backend.compose_project() + if front is not None and front.ui: print(' Open WebUI may keep the old key in its own settings: ' 'update it under Admin > Settings > Connections') return 0 diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index f5583f78..1536c550 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -72,7 +72,7 @@ def _compose_argv(config) -> list[str]: kubeai backend. Exits when there is none, or nothing is rendered yet. """ backend = _day2_backend(config) - project = getattr(backend, 'compose_project', lambda: None)() + project = backend.compose_project() if project is None: raise SystemExit( f'the {_backend_name(backend)} backend has no Compose project on this ' @@ -831,11 +831,10 @@ def main(cls, argv=True, **kwargs): cmd = _compose_argv(config) + ['down', '--remove-orphans', '--volumes'] return int(subprocess.run(cmd, env=_docker_env()).returncode) backend = _day2_backend(config) - down = getattr(backend, 'down', None) - if down is None: + if not backend.runs_engines: print(f'the {_backend_name(backend)} backend runs nothing to bring down') return 0 - down() + backend.down() return 0 @@ -910,15 +909,13 @@ def main(cls, argv=True, **kwargs): from .commands_leasing import _make_backend backend = _make_backend(config) - doctor = getattr(backend, 'doctor', None) name = type(backend).__name__ - if doctor is None: + checks = list(backend.doctor()) + if not checks: print(f'{name}: no preflight checks defined — nothing to verify.') if not (config['gpu'] or config['sudo']): return 0 - doctor = list # GPU checks still apply; the backend just has none failed = 0 - checks = list(doctor()) if config['gpu'] or config['sudo']: from ..gpu_doctor import gpu_checks checks += list(gpu_checks(use_sudo=bool(config['sudo']))) diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 1f2a4cef..591397c8 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -183,6 +183,14 @@ def remove_containers(self, container_ids: list[str]) -> None: """Remove these containers now (orphans the operator agreed to remove).""" ... + def pull_images(self, images: Any) -> list[str]: + """Pull these images now; return the ones that failed.""" + ... + + def upstream_check(self) -> dict[str, dict[str, Any]]: + """Probe each model upstream by name from inside the gateway's network.""" + ... + class FrontDoorControl(Protocol): """The LiteLLM gateway as the controller and the CLI address it. @@ -373,6 +381,33 @@ def connection_info(self) -> ConnectionInfo | None: (an in-process backend serves nothing).""" ... + #: Whether this backend runs engine processes (containers, pods) whose + #: logs can be read; ``False`` for the in-process backends. + runs_engines: bool + #: The catalog it renders from (the published union once a profile is in + #: use), or ``None``. + catalog: Any + #: Recorded VRAM measurements (``measure --record``), or ``None`` where + #: nothing is measured. + measurements: Any + + def compose_project(self) -> Any | None: + """The Compose project on this host (the stack itself, or KubeAI's + gateway), or ``None`` where there is none.""" + ... + + def down(self) -> None: + """Take down everything this backend runs (``stack down``).""" + ... + + def doctor(self) -> list[tuple[str, bool, str]]: + """Preflight checks: ``(check, ok, detail)``; empty when there are none.""" + ... + + def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: + """The tail of a deployment's engine log; ``''`` when there is none.""" + ... + def request_names(self, endpoints: list[str]) -> dict[str, str]: """The ``model`` a client sends for each of ``endpoints``, for those this backend decides (the alias itself behind a LiteLLM front door; a @@ -616,6 +651,23 @@ def connection_info(self) -> ConnectionInfo | None: def request_names(self, endpoints: list[str]) -> dict[str, str]: return {} + #: In-process: nothing runs, so nothing to log, check, measure or bring down. + runs_engines = False + catalog: Any = None + measurements: Any = None + + def compose_project(self) -> Any | None: + return None + + def down(self) -> None: + return None + + def doctor(self) -> list[tuple[str, bool, str]]: + return [] + + def deployment_logs(self, deployment: Deployment, *, tail: int = 400) -> str: + return '' + def catalog_routes(self, catalog: Any) -> list[GatewayRoute]: return [] diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index c6d1b5d0..aecd58e2 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1184,6 +1184,9 @@ def compose_file(self) -> Path: #: The file a render writes, whatever the backend (``status`` shows it). rendered_file = compose_file + #: Its engines are containers on this host. + runs_engines = True + def compose_project(self): """The Compose project on this host: this one.""" return self diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 489bfd69..7868564c 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -2076,7 +2076,9 @@ def commit_route_seed(self, plan: RoutePlan, *, replace: bool = False workload runs. Rechecked under the lock: a conflict that appeared since the plan refuses too (:class:`~infer_stack.leasing.routes.RouteConflict`) and nothing is - written.""" + written. ``replace`` is compare-and-swap: each redefinition replaces + exactly the meaning the plan showed; one another process changed + meanwhile refuses (``RouteConflict(changed=True)``).""" from .profile import CatalogConflict, ProfileMismatch, adopt_catalog_sources from .residency import ResidencyUnknown from .routes import RouteConflict, plan_seed @@ -2092,6 +2094,13 @@ def preflight(): now = plan_seed(self._published_meanings(), plan.incoming) if now.conflicted and not replace: raise RouteConflict(sorted(now.conflicted)) + if replace: + # Compare-and-swap: replace only what was shown and confirmed. + shown = {n: old for n, (old, _) in plan.conflicted.items()} + moved = sorted(n for n, (old, _) in now.conflicted.items() + if n not in shown or not (shown[n] == old)) + if moved: + raise RouteConflict(moved, changed=True) def change(): now = plan_seed(self._published_meanings(), plan.incoming) diff --git a/infer_stack/leasing/routes.py b/infer_stack/leasing/routes.py index ee1f78cd..14d59e0a 100644 --- a/infer_stack/leasing/routes.py +++ b/infer_stack/leasing/routes.py @@ -96,13 +96,24 @@ def route_table(*layers: Iterable[GatewayRoute]) -> list[GatewayRoute]: class RouteConflict(ValueError): - """A seed would redefine existing aliases; ``names`` are the conflicts.""" + """A seed would redefine existing aliases; ``names`` are the conflicts. - def __init__(self, names: list[str]): + ``changed``: a ``--replace`` whose confirmed redefinitions no longer hold, + because another process changed those aliases after the plan was shown. + """ + + def __init__(self, names: list[str], *, changed: bool = False): self.names = list(names) - super().__init__( - f'routes seed would redefine {len(self.names)} existing route(s): ' - f'{", ".join(self.names)}; pass --replace to redefine them') + self.changed = changed + if changed: + message = (f'{", ".join(self.names)} changed since the redefinition was ' + 'shown; nothing was written, run `routes seed` again to see ' + 'what is there now') + else: + message = (f'routes seed would redefine {len(self.names)} existing ' + f'route(s): {", ".join(self.names)}; pass --replace to ' + 'redefine them') + super().__init__(message) @dataclass(frozen=True, eq=False) diff --git a/tests/test_cli_leasing.py b/tests/test_cli_leasing.py index 83d71776..a46313d0 100644 --- a/tests/test_cli_leasing.py +++ b/tests/test_cli_leasing.py @@ -1252,3 +1252,38 @@ def racing_lock(): controller.commit_route_seed(plan) assert controller.ledger.publication_pending() is None assert gateway.route_entries() == {'alpha': {'engine': 'vllm', 'served': 'elsewhere'}} + + +def test_replace_is_compare_and_swap_against_what_was_shown(tmp_path, monkeypatch): + """`routes seed --replace` replaces exactly the meaning it showed: if + another process redefined the alias between the plan and the commit, + nothing is written and no publication is left pending.""" + from infer_stack.cli import commands_leasing as cl + from infer_stack.leasing import Catalog + from infer_stack.leasing.routes import RouteConflict + + state = tmp_path / 'state' + state.mkdir() + db = str(tmp_path / 'ledger.db') + _patch_backend(monkeypatch, state) + config = cl.RoutesSeedCLI.cli(argv=['--ledger', db, str(tmp_path / 'x.yaml')]) + controller = cl._open_controller(config, interactive=False) + gateway = controller.backend.front_door().gateway + gateway.replace_route_entries({'alpha': {'engine': 'vllm', 'served': 'shown'}}) + plan = controller.plan_route_seed([Catalog.from_dict(_one_endpoint_catalog('alpha'))]) + assert list(plan.conflicted) == ['alpha'] # shown: 'shown' -> alpha + real_lock = controller._global_lock + + def racing_lock(): + gateway.replace_route_entries({'alpha': {'engine': 'vllm', 'served': 'moved'}}) + return real_lock() + + controller._global_lock = racing_lock + with pytest.raises(RouteConflict, match='changed since the redefinition was shown'): + controller.commit_route_seed(plan, replace=True) + assert controller.ledger.publication_pending() is None + assert gateway.route_entries() == {'alpha': {'engine': 'vllm', 'served': 'moved'}} + controller._global_lock = real_lock # unchanged: it replaces + fresh = controller.plan_route_seed([Catalog.from_dict(_one_endpoint_catalog('alpha'))]) + controller.commit_route_seed(fresh, replace=True) + assert 'alpha' not in gateway.route_entries() # the published definition won diff --git a/tests/test_day2.py b/tests/test_day2.py index 0e787b21..236e6de2 100644 --- a/tests/test_day2.py +++ b/tests/test_day2.py @@ -63,6 +63,8 @@ def test_a_target_resolves_by_alias_deployment_name_or_id(): class _Backend: """A backend as the day-2 verbs see it.""" + runs_engines = True + def __init__(self, instances, *, residency=None): self._instances = instances self._residency = residency @@ -181,7 +183,8 @@ def test_stack_compose_targets_the_compose_project_on_this_host(monkeypatch, tmp compose_file.write_text('services: {}\n') assert rt._compose_argv(SimpleNamespace()) == ['docker', 'compose', '-p', 'gw'] - monkeypatch.setattr(rt, '_day2_backend', lambda config: SimpleNamespace()) + monkeypatch.setattr(rt, '_day2_backend', + lambda config: SimpleNamespace(compose_project=lambda: None)) with pytest.raises(SystemExit, match='no Compose project on this host'): rt._compose_argv(SimpleNamespace()) From 09b72fa01e03a0308c48d82a9e6482167538f6e7 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:56:04 -0400 Subject: [PATCH 119/141] External e2e runs on KubeAI's gateway too E2E_BACKEND=kubeai runs the external phases (key refusal, publish, key rotation, prune, lease-less access) through KubeAI's host gateway, in static and dynamic routing; both pass on the dev cluster, as does ux_audit.sh kubeai with `access` in it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/external_e2e.sh | 20 ++++++++++++++++++-- docs/queue.md | 8 +++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/dev/external_e2e.sh b/dev/external_e2e.sh index cb9a9022..35f0f07c 100755 --- a/dev/external_e2e.sh +++ b/dev/external_e2e.sh @@ -28,11 +28,15 @@ # E2E_MODES routing modes to run (default "static dynamic") # E2E_PORT the mock's port (default 18911) # E2E_SIM the simulator image (default ghcr.io/llm-d/llm-d-inference-sim:v0.9.0) +# E2E_BACKEND compose (default) or kubeai: the external route through +# KubeAI's host gateway (phases 1-5; 6-8 need the simulator, +# which runs on Compose only). Needs KUBECONFIG and the chart. set -euo pipefail MODES="${E2E_MODES:-static dynamic}" PORT="${E2E_PORT:-18911}" SIM="${E2E_SIM:-ghcr.io/llm-d/llm-d-inference-sim:v0.9.0}" +BACKEND="${E2E_BACKEND:-compose}" HOST_IP="$(ip -4 -o addr show docker0 | awk '{print $4}' | cut -d/ -f1)" [ -n "$HOST_IP" ] || { echo 'no docker0 address' >&2; exit 1; } @@ -119,7 +123,7 @@ run_mode() { run_is() { env $IS_ENV infer-stack "$@"; } echo "== [$mode] work dir: $WORK" mkdir -p "$WORK/config" - run_is config set backend compose >/dev/null + run_is config set backend "$BACKEND" >/dev/null if [ "$mode" = dynamic ]; then run_is config set dynamic_routing true >/dev/null; fi # The runbook's own catalog: nothing external. cat > "$WORK/config/catalog.yaml" </dev/null 2>&1 || true + stop_mock + rm_work "$WORK" + echo "== [$mode/$BACKEND] PASS (phases 1-5)" + return 0 + fi # The runbook's catalog now names a managed endpoint, the external one, # and a bundle of both. cat > "$WORK/config/catalog.yaml" < external after release; external under dynamic routing if practical. Run from a frozen copy. @@ -1333,8 +1335,8 @@ practical. Run from a frozen copy. managed; README (catalog model, main commands, shapes); the catalog schema docstring; routing doc. Journal: the four alternatives and why not. Suite (1094), ty, flake8, `dev/ux_audit.sh compose` (0 flags, now covering -`access`), `dev/external_e2e.sh` (both routing modes). Not run here: -`ux_audit.sh kubeai` and a KubeAI external e2e. +`access`), `dev/external_e2e.sh` (both routing modes, Compose and KubeAI), +`ux_audit.sh kubeai` (0 flags, `access` included). Design doc: Endpoint (public contract) / Fulfillment (managed runtime or external upstream) / Lease (only when infer-stack owns runtime), with the workflow (`external:` in the catalog; `infer-stack access qwen --env-file From 21ade9a814217d33934e3de537a7af95f3c13cbf Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 12:56:23 -0400 Subject: [PATCH 120/141] known-limitations: the in-cluster gateway routes only Dynamic routing, Open WebUI and a tested Ingress with kubeai_gateway cluster were deferred by P5 but not recorded where deferred items go. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/planning/known-limitations.md | 9 +++++++++ docs/queue.md | 2 ++ 2 files changed, 11 insertions(+) diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index 7e910805..369ac15b 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -171,6 +171,15 @@ the cluster's nodes (verified with a second node in a container; a run across two real machines is `dev/handover/p5_two_hosts.sh`). One deployment spanning machines (tensor parallel across nodes) is supported on neither. +### The in-cluster gateway routes only (current, deferred) + +With `kubeai_gateway cluster` the LiteLLM gateway is a Deployment and a +NodePort Service with static routes. Dynamic routing and Open WebUI need +the host placement (`kubeai_gateway host`): in the cluster they would need +Postgres and a UI there, which P5 deferred. `kubeai_gateway_url` accepts an +Ingress URL, but no Ingress has been tested. External endpoints work in both +placements (their keys go into the gateway's Secret). + ### Forged ownership labels are outside the threat model (design boundary) infer-stack identifies its containers by the Compose project and its own labels. diff --git a/docs/queue.md b/docs/queue.md index 4a381e82..3effd0c2 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -951,6 +951,8 @@ campaign 2. None blocks campaign 2. - [ ] **In-cluster gateway: deferred features (P5).** Dynamic routing and Open WebUI with `kubeai_gateway cluster` (they need Postgres and a UI in the cluster), and an Ingress (`kubeai_gateway_url` accepts one; none tested). + *2026-09-27:* recorded in `known-limitations.md` (current, deferred). A new + feature, not a fix: build it only against a reviewed plan. - [x] **`routes seed --replace` compare-and-swap.** *Done 2026-09-27:* the plan carries each alias's shown meaning; under the lock a changed one refuses (`RouteConflict(changed=True)`), writes nothing, leaves no marker. From 2af9c70fb3ef55281a481edc992d241ff913446f Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:17:42 -0400 Subject: [PATCH 121/141] queue: campaign 2 review items 42-50 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/queue.md b/docs/queue.md index 3effd0c2..f65bf7ac 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1349,6 +1349,79 @@ docs, not only CLI help. Journal: fake lease vs access above leasing; state; direct remote vs stable front door. Full suite, `ty`, flake8, UX audits, the relevant real e2e. +## Campaign 2 review (2026-09-27, reviewed through `21ade9a`) + +The architecture stands; these are places where the publication/profile +transaction model had not caught up with published catalogs being real +desired state. Items 42-45 must be fixed before campaign 2 is closed. + +### 42. [ ] The first profile, and a profile candidate, commit with publication intent +`_sync_profile(create=True)` persists an absent profile at once, before +preview, approval or marker. Since `profile.catalogs` is the published +endpoint set, that publishes the invocation's endpoints outside the +transaction: a crash or a declined approval leaves them durable with no +marker. Acquire has the same gap on an existing profile (candidate, marker +and lease are three transactions). Do: an absent profile is a candidate, +never a write; a non-endpoint mutation (gc, release, apply) on a fresh +ledger persists settings with no catalogs, together with its marker; an +acquire commits profile + marker + lease in one transaction; external-only +access already uses `publish_profile`. Poison tests: fresh external access +declined (no profile, marker, route); crash before the commit; fresh managed +acquire declined (no lease, no profile); first access commits profile and +marker together; `gc` on a fresh ledger publishes no endpoint. + +### 43. [ ] Access readiness includes route publication +External-only `access` returns ready when LiteLLM answers the master key, +even if the route reconcile failed (`routes=False`, publication pending). +Carry the publication result into `AccessResult`; ready requires it +complete and the front door authenticated; never a request to the upstream. +Readiness tri-state (`None` = not checked, e.g. `--no-wait`). Poison: route +reconcile fails, auth succeeds -> not ready, nonzero, pending; a retry that +succeeds makes it ready. + +### 44. [ ] Missing external keys refuse publication, whatever the command +Only `access` and `routes seed` check; renders warn. An unrelated +`acquire`/`apply`/`release`/`gc`/`config publish` can recreate the gateway +without a key a published route needs. Validate the full desired route set +at the render (both gateways), before any mutation; a typed front-door +operation, not callers reading the gateway's `.env` path. + +### 45. [ ] Pinning protects every resident alias, not only catalog ones +`routes seed --replace` (and an acquire's catalog adoption) can publish a +new meaning for an alias a resident ad-hoc deployment serves (no catalog +definition, registry-only or dynamic route). Pinning is an endpoint rule: +refuse any change to what a resident alias means, whatever source held it. +Poison: resident ad-hoc `qwen`, `routes seed --replace` qwen -> external, +dynamic routing: refuse, no catalog/registry change, no marker. + +### 46. [ ] Finish `FrontDoorControl` +Upper layers still reach `front.gateway` (`missing_keys`, `_env_path`, +registry rows) and backend-level gateway proxies; the secrets CLI uses +`getattr(backend, 'litellm')`, `backend.master_key()`. Give the capability +the operations callers need; drop `gateway: Any` from it and proxies whose +only job was exposing the gateway. + +### 47. [ ] Published catalog state: one mutation API (design candidate) +`profile.catalogs` is a list of source snapshots edited by +`adopt_catalog_sources`, `drop_catalog_names`, bundle repair. Candidate: one +`PublishedCatalogSet` API for every mutation (serialization unchanged). +Record; do only if more bugs appear here. + +### 48. [ ] `routes list` is desired state; say so +It derives routes from published definitions and deployments, not from the +gateway. Label it desired, show whether a publication is pending; a live +inspection, if ever, is a separate command. + +### 49. [ ] External `api_base`: stricter parsing, networking documented +`urlsplit`: http(s), a hostname, no whitespace. Document that `api_base` is +resolved in the gateway's network (container or pod): `localhost` is the +gateway itself. + +### 50. [ ] Pre-registry state directories: decide the migration explicitly +The seed-from-`litellm_config.yaml` path was dropped as incidental. Decide: +a one-time versioned migration of only what has no other source, or none, +recorded with the reason. + ## Non-goals Direct external access (a second access mode where the alias stops being the model name) is a possible later optimization, not campaign 2. From df62475b75b85740b56d985a5ef0de4fefb14f7e Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:25:08 -0400 Subject: [PATCH 122/141] Published endpoints commit with publication intent, never before profile.catalogs is the published endpoint set, but _sync_profile froze a fresh ledger's profile on any mutation before preview, approval or marker, and acquire wrote its candidate profile before its marker and lease. A crash or a declined approval could leave endpoints published with nothing recording the publication. _sync_profile no longer writes: a fresh ledger renders from settings with no catalogs, committed with the first marker. An acquire commits its candidate in the lease's transaction; access and a first apply / stack up publish through publish_profile after an approved preview; gc, release and evict publish no endpoints. Poison tests cover declined and crashed first access, acquire and apply, and gc on a fresh ledger. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 5 + docs/adr/0001-user-config-is-authoritative.md | 12 +- docs/queue.md | 10 +- infer_stack/leasing/controller.py | 83 +++++++--- infer_stack/leasing/ledger.py | 17 +- infer_stack/leasing/store.py | 30 ++-- tests/test_external_endpoints.py | 10 +- tests/test_leasing_profile.py | 6 +- tests/test_publication_transaction.py | 149 ++++++++++++++++++ 9 files changed, 278 insertions(+), 44 deletions(-) create mode 100644 tests/test_publication_transaction.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bc7e4c92..20393cd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,11 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). with no value. - A runbook whose catalog is already published no longer warns that its catalogs differ from the recovery snapshot. +- Publishing endpoints is transactional: an acquire commits its catalogs + with its lease and publication marker, and a fresh ledger's first snapshot + is written with its first marker, never before. `gc`, `release` and + `evict` publish no endpoints on a fresh ledger; `acquire`, `access` and a + first `apply` / `stack up` do. - `infer-stack access NAME...` makes endpoints reachable whoever runs them: it leases the managed ones (as `acquire`), publishes external ones with no lease, waits for the front door, and writes the same env-file (with no diff --git a/docs/adr/0001-user-config-is-authoritative.md b/docs/adr/0001-user-config-is-authoritative.md index 1690e95a..8209b7c6 100644 --- a/docs/adr/0001-user-config-is-authoritative.md +++ b/docs/adr/0001-user-config-is-authoritative.md @@ -48,8 +48,16 @@ On acquire, under the publication lock: quiescence used to replace the catalogs wholesale too, but an external endpoint has no lease to keep it published, so only an explicit `routes prune` unpublishes.* -5. Preview/admit/render using that exact candidate snapshot, then persist the - snapshot before committing the acquire's desired-state mutation. +5. Preview/admit/render using that exact candidate snapshot, then commit the + snapshot in the same transaction as the lease and the publication marker. + *Amended 2026-09-27 (queue item 42):* the snapshot's catalogs are the + published endpoint set, so writing them is publishing. It used to be + written before the lease, and a ledger's first snapshot on its first + mutation of any kind; now a crash or a declined approval before the commit + publishes nothing. A first non-endpoint mutation (gc, release, evict) + commits settings with no catalogs, with its marker; `access` and a first + `apply` / `stack up` publish the invocation's endpoints through the same + approved single commit. Compatible catalog additions therefore support the common pattern of adding a new model while another model is already serving. A conflicting edit is a real diff --git a/docs/queue.md b/docs/queue.md index f65bf7ac..66dfd8b8 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1355,7 +1355,15 @@ The architecture stands; these are places where the publication/profile transaction model had not caught up with published catalogs being real desired state. Items 42-45 must be fixed before campaign 2 is closed. -### 42. [ ] The first profile, and a profile candidate, commit with publication intent +### 42. [x] The first profile, and a profile candidate, commit with publication intent +*Done 2026-09-27.* `_sync_profile` never writes; a fresh ledger renders from +settings with no catalogs, committed by the first marker +(`mark_publication_pending(profile=...)`) or network migration; an acquire +commits its candidate inside `ledger.acquire` with the lease; access (and a +first `apply` / `stack up`) through `publish_profile` after an approved +preview. `gc`, `release`, `evict` never publish endpoints; `routes seed` +publishes only what it names. Poison tests: +`tests/test_publication_transaction.py` (8). ADR 0001 amended. `_sync_profile(create=True)` persists an absent profile at once, before preview, approval or marker. Since `profile.catalogs` is the published endpoint set, that publishes the invocation's endpoints outside the diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 7868564c..0eeaae01 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -233,6 +233,9 @@ def __init__( # Recovery snapshot (see leasing/profile.py): this process's own # resolved settings, captured before the backend is switched to the # resolved copy, and the snapshot the backend currently renders from. + #: The first snapshot, rendered from but not yet written (see + #: :meth:`_sync_profile`). + self._fresh_profile: dict | None = None self._invocation_profile: dict | None = None self._applied_profile: dict | None = None self._profile_drift_warned = False @@ -706,6 +709,7 @@ def network_migrate(self, subnet: str, *, force: bool = False) -> ReconcileResul self.ledger.migrate_network( subnet=subnet, reset_addresses=reset, approved_digest=self.backend.last_preview_digest, + profile=self._take_fresh_profile(), ) return self._publish() @@ -1139,7 +1143,11 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: stored = self.ledger.profile() invocation = self._invocation_profile - if stored is None or invocation is None or stored == invocation: + if invocation is None: + return None + if stored is None: + return dict(invocation) # the first publication: all of it + if stored == invocation: return None if stored.get('backend') != invocation.get('backend'): raise ProfileMismatch( @@ -1205,6 +1213,7 @@ def _commit_profile_candidate(self, profile: dict) -> None: self._applied_profile = profile def _restore_stored_profile(self, stored: dict | None) -> None: + stored = stored if stored is not None else self._fresh_profile if stored is None: return self._use_profile_candidate(stored) @@ -1214,9 +1223,13 @@ def _sync_profile(self, *, create: bool) -> None: """Make the backend render from the active recovery snapshot. With ``create`` (every mutation, under the lock), a ledger without a - snapshot gets this invocation's resolved settings frozen as the initial - one. Drift is reported once; acquire decides under the same lock whether - current user config can advance the snapshot. Backends without a render + snapshot renders from this invocation's settings with no published + endpoints (:attr:`_fresh_profile`). That is not written here: it + commits with the first publication marker (:meth:`_mark_pending`), and + an acquire or access commits the invocation's catalogs with its own + transaction, because writing ``catalogs`` is publishing them. Drift is + reported once; acquire decides under the same lock whether current + user config can advance the snapshot. Backends without a render profile (null, test fakes) are left alone. """ from .._log import logger @@ -1234,13 +1247,9 @@ def _sync_profile(self, *, create: bool) -> None: if self._invocation_profile is None: self._invocation_profile = render() if stored is None: - stored = self._invocation_profile - self.ledger.set_profile(stored) - logger.info( - 'Froze the initial recovery snapshot ({} backend, {} catalog(s)); ' - 'normal acquire advances it automatically from current user config', - stored.get('backend'), len(stored.get('catalogs') or []), - ) + # Settings only: endpoints are published by the operations that + # publish them, in their own transaction. + stored = self._fresh_profile = {**self._invocation_profile, 'catalogs': []} elif not self._profile_drift_warned: drift = profile_drift(stored, self._invocation_profile) if drift: @@ -1275,7 +1284,20 @@ def _mark_pending( current = self.ledger.publication_pending() if current and current.get('placement_context'): self.ledger.clear_placement_context() - return self.ledger.mark_publication_pending(apply_requested=apply) + return self.ledger.mark_publication_pending( + apply_requested=apply, profile=self._take_fresh_profile()) + + def _take_fresh_profile(self) -> dict | None: + """The first snapshot to commit with the next marker, or ``None`` + when the ledger already has one.""" + from .._log import logger + + fresh, self._fresh_profile = self._fresh_profile, None + if fresh is None or self.ledger.profile() is not None: + return None + logger.info('Froze the initial recovery snapshot ({} backend); acquire and ' + 'access publish endpoints into it', fresh.get('backend')) + return fresh def _apply_pending(self, rec: ReconcileResult) -> ReconcileResult: """Apply the last render if the marker requests it; clear on success. @@ -1422,9 +1444,18 @@ def apply_now(self) -> ReconcileResult: Heals drift (re-ups a container that died out-of-band) and publishes anything pending, including leases staged with ``--no-apply``. It is also the explicit approval that clears an approved-digest mismatch. + + On a ledger with no profile yet (``stack up`` on a fresh stack) it + publishes the invocation's endpoints first, through the same preview, + approval and single commit as ``access`` (:meth:`publish_endpoints`), so + catalog routes exist before any model runs and the first acquire does + not recreate the gateway. Other mutations (gc, release, evict) never + publish endpoints. """ self._explicit_apply = True try: + if self.backend.recovery_profile is not None and self.ledger.profile() is None: + return self.publish_endpoints() return self.reconcile(apply=True) finally: self._explicit_apply = False @@ -1758,8 +1789,14 @@ def publish_endpoints(self) -> ReconcileResult: except BaseException: self._restore_stored_profile(stored) raise - self.ledger.publish_profile( - candidate, approved_digest=self.backend.last_preview_digest) + try: + # Profile (the published endpoints) and approved marker at once. + self.ledger.publish_profile( + candidate, approved_digest=self.backend.last_preview_digest) + except BaseException: + self._restore_stored_profile(stored) + raise + self._fresh_profile = None self._profile_error = None self._applied_profile = candidate return self._publish() @@ -1816,8 +1853,6 @@ def _acquire_by_admission( self._restore_stored_profile(stored_profile) raise if not reasons: - if candidate_profile is not None: - self._commit_profile_candidate(candidate_profile) # Allocations are committed with the lease, so no placement # scope needs recording for recovery. self._mark_pending(apply=apply) @@ -1828,9 +1863,18 @@ def _acquire_by_admission( # Nothing is applied for --no-apply, so there is no # approval to guard; staged state stays discardable. approved_digest=self._admission_digest if apply else None, + # The published endpoints commit with the lease. + profile=candidate_profile, ) except AdmissionConflict: continue # the ledger moved; preview again + except BaseException: + if candidate_profile is not None: + self._restore_stored_profile(self.ledger.profile()) + raise + if candidate_profile is not None: + self._profile_error = None + self._applied_profile = candidate_profile try: rec = self._render() except ConvergeAborted: @@ -1993,10 +2037,9 @@ def route_view(self) -> list: return self.backend.routes(desired, inputs) def _published_sources(self) -> list[dict]: - """The published catalog union's sources (the stored profile's, else - the ones this invocation would freeze).""" - profile = self.ledger.profile() or self.invocation_profile() or {} - return list(profile.get('catalogs') or []) + """The published catalog union's sources: the stored profile's, and + none before the first publication.""" + return list((self.ledger.profile() or {}).get('catalogs') or []) def _meanings(self, union) -> dict: """``{alias: Meaning}`` for every endpoint of ``union``.""" diff --git a/infer_stack/leasing/ledger.py b/infer_stack/leasing/ledger.py index 56c33c5f..d1bbef38 100644 --- a/infer_stack/leasing/ledger.py +++ b/infer_stack/leasing/ledger.py @@ -238,10 +238,11 @@ def publish_profile(self, profile: dict, *, approved_digest: str | None) -> None self.store.publish_profile(profile, approved_digest=approved_digest) def migrate_network(self, *, subnet: str, reset_addresses: bool, - approved_digest: str | None) -> None: - """Switch the stable-address subnet and mark it pending, at once.""" + approved_digest: str | None, profile: dict | None = None) -> None: + """Switch the stable-address subnet and mark it pending, at once + (with a first ``profile``, if there is none yet).""" self.store.migrate_network(subnet=subnet, reset_addresses=reset_addresses, - approved_digest=approved_digest) + approved_digest=approved_digest, profile=profile) def acquire( self, @@ -252,6 +253,7 @@ def acquire( overlay: AcquireOverlay | None = None, allocations: dict[str, list[int]] | None = None, approved_digest: str | None = None, + profile: dict | None = None, ) -> AcquireResult: """Create a lease and coalesce its endpoints onto deployment deployments. @@ -262,6 +264,11 @@ def acquire( committed as they are, together with ``allocations`` (deployment id -> GPUs), in one transaction. If the admission state changed since the preview, :class:`AdmissionConflict` is raised and nothing is written. + + ``profile`` (the acquire's recovery-profile candidate, whose catalogs + are the published endpoint set) commits in the same transaction: the + endpoints it publishes are never durable without the lease and the + publication intent that go with them. """ lease_id = self.id_factory('lease') deployment_ids: list[str] = [] @@ -290,6 +297,8 @@ def acquire( self.store.update_deployment_served(gid, served, now) for gid, gpus in (allocations or {}).items(): self.store.set_deployment_allocation(gid, gpus) + if profile is not None: + self.store._write_profile(profile) if approved_digest is not None: # In the same transaction as the lease: the digest can never # describe a candidate that was not committed. @@ -509,11 +518,13 @@ def status( def mark_publication_pending( self, *, apply_requested: bool, interrupted: bool = False, placement_context: dict | None = None, approved_digest: str | None = None, + profile: dict | None = None, ) -> dict: """See :meth:`SqliteStore.mark_publication_pending`.""" return self.store.mark_publication_pending( apply_requested=apply_requested, interrupted=interrupted, placement_context=placement_context, approved_digest=approved_digest, + profile=profile, ) def clear_placement_context(self) -> None: diff --git a/infer_stack/leasing/store.py b/infer_stack/leasing/store.py index 586b0bfa..f1e6c1c6 100644 --- a/infer_stack/leasing/store.py +++ b/infer_stack/leasing/store.py @@ -385,18 +385,26 @@ def profile(self) -> dict | None: def set_profile(self, profile: dict) -> None: with self.transaction(): - self._conn.execute( - "INSERT INTO meta(key, value) VALUES ('profile', ?) " - 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', - (json.dumps(profile, sort_keys=True),), - ) + self._write_profile(profile) + + def _write_profile(self, profile: dict) -> None: + """The profile upsert; the caller holds a :meth:`transaction`.""" + self._conn.execute( + "INSERT INTO meta(key, value) VALUES ('profile', ?) " + 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', + (json.dumps(profile, sort_keys=True),), + ) def mark_publication_pending( self, *, apply_requested: bool, interrupted: bool = False, placement_context: dict | None = None, approved_digest: str | None = None, + profile: dict | None = None, ) -> dict: """Record that desired state is changing; return the marker written. + ``profile``, when given, is written in the same transaction (a first + recovery profile commits with the publication intent it belongs to). + Both flags only ever turn on until the marker is cleared. ``interrupted`` records that an apply was killed mid-flight, so the runtime may still be changing underneath the next one. @@ -404,6 +412,8 @@ def mark_publication_pending( ``allowed_gpus``) replaces any stored one; ``None`` keeps it. """ with self.transaction(): + if profile is not None: + self._write_profile(profile) return self._write_marker( apply_requested=apply_requested, interrupted=interrupted, placement_context=placement_context, approved_digest=approved_digest, @@ -455,15 +465,11 @@ def publish_profile(self, profile: dict, *, approved_digest: str | None) -> None in one transaction, so a crash cannot leave a published profile whose approval nothing records.""" with self.transaction(): - self._conn.execute( - "INSERT INTO meta(key, value) VALUES ('profile', ?) " - 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', - (json.dumps(profile, sort_keys=True),), - ) + self._write_profile(profile) self._write_marker(apply_requested=True, approved_digest=approved_digest) def migrate_network(self, *, subnet: str, reset_addresses: bool, - approved_digest: str | None) -> None: + approved_digest: str | None, profile: dict | None = None) -> None: """Switch subnet, reset addresses and mark the change pending, atomically. A crash leaves either the old subnet with its address table, or the new @@ -471,6 +477,8 @@ def migrate_network(self, *, subnet: str, reset_addresses: bool, which could hand a service's address to another service. """ with self.transaction(): + if profile is not None: + self._write_profile(profile) self._conn.execute( "INSERT INTO meta(key, value) VALUES ('network_config', ?) " 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index ddc2b3a2..6b1f44c6 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -183,16 +183,18 @@ def test_seed_publishes_an_external_endpoint_and_prune_unpublishes_it(tmp_path): plan = ctl.plan_route_seed([other]) assert list(plan.added) == ['remote'] ctl.commit_route_seed(plan) - assert _published(ledger) == ['local', 'remote'] + # The seed publishes what it names; the invocation's own catalog is + # published by the operations that publish it (acquire, access, apply). + assert _published(ledger) == ['remote'] view = {r.alias: r.origin for r in ctl.route_view()} - assert view == {'local': 'catalog', 'remote': 'external'} + assert view['remote'] == 'external' plan = ctl.plan_route_prune() # the invocation knows only 'local' assert plan.dropped == ['remote'] dropped, _ = ctl.commit_route_prune(plan) assert dropped == ['remote'] - assert _published(ledger) == ['local'] - assert [r.alias for r in ctl.route_view()] == ['local'] + assert _published(ledger) == [] + assert 'remote' not in [r.alias for r in ctl.route_view()] def test_prune_keeps_what_a_live_deployment_serves(tmp_path): diff --git a/tests/test_leasing_profile.py b/tests/test_leasing_profile.py index 413ba56d..b7071ddb 100644 --- a/tests/test_leasing_profile.py +++ b/tests/test_leasing_profile.py @@ -92,7 +92,7 @@ def test_a_quiescent_acquire_publishes_its_catalog_beside_the_others(tmp_path): a = Catalog.from_dict(cat('alpha')) b = Catalog.from_dict(cat('beta')) ledger, ctl = controller(tmp_path, catalog=a) - ctl.gc() # freezes {alpha} + ctl.apply_now() # publishes {alpha} _, ctl2 = controller(tmp_path, catalog=b) out = ctl2.acquire('x', b.resolve_names(['beta']), wait=False) assert out.lease.endpoints == ['beta'] @@ -137,7 +137,7 @@ def test_changed_live_definition_waits_for_quiescence_then_auto_adopts(tmp_path) def test_union_of_catalogs_serves_either_runbook(tmp_path): union = CatalogUnion.from_sources([cat('alpha'), cat('beta')]) ledger, ctl = controller(tmp_path, catalog=union) - ctl.gc() + ctl.apply_now() # A runbook that only knows `beta` passes its own catalog: no drift, accepted. b = Catalog.from_dict(cat('beta')) _, ctl2 = controller(tmp_path, catalog=b) @@ -647,7 +647,7 @@ def test_quiescence_adopts_new_settings_but_keeps_published_endpoints(tmp_path): a = Catalog.from_dict(cat('alpha')) b = Catalog.from_dict(cat('beta')) ledger, ctl = controller(tmp_path, catalog=a, ui=True) - ctl.gc() # freezes ui=True, {alpha} + ctl.apply_now() # freezes ui=True, {alpha} _, ctl2 = controller(tmp_path, catalog=b, ui=False) ctl2.acquire('x', b.resolve_names(['beta']), wait=False) assert ledger.profile()['ui'] is False # settings: the invocation's diff --git a/tests/test_publication_transaction.py b/tests/test_publication_transaction.py new file mode 100644 index 00000000..5b9bde5d --- /dev/null +++ b/tests/test_publication_transaction.py @@ -0,0 +1,149 @@ +"""The published endpoint set commits with publication intent, never before. + +``profile.catalogs`` is the published catalog union, from which gateway routes +are derived (docs/planning/external-endpoints.md, decision 1). Writing it is +publishing, so it commits in the same transaction as the publication marker +(and, for an acquire, the lease): a declined approval or a crash before that +commit leaves nothing published. Queue item 42. +""" + +from __future__ import annotations + +import pytest + +from infer_stack.env_utils import write_env_file +from infer_stack.leasing import Catalog +from infer_stack.leasing.backend import ConvergeAborted +from test_external_endpoints import REMOTE, catalog +from test_leasing_profile import controller + + +def _keyed(tmp_path, cat): + ledger, ctl = controller(tmp_path, catalog=cat) + write_env_file(ctl.backend.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + return ledger, ctl + + +def _decline(ctl): + def decline(planned): + raise ConvergeAborted('declined') + + ctl.backend._approve_changes = decline + + +def _published(ledger): + return sorted(n for s in (ledger.profile() or {}).get('catalogs') or [] + for n in (s.get('endpoints') or {})) + + +def test_a_declined_first_external_access_publishes_nothing(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + _decline(ctl) + with pytest.raises(ConvergeAborted): + ctl.access('me', cat.resolve(['remote'])) + assert ledger.profile() is None + assert ledger.publication_pending() is None + assert not (ctl.backend.state_dir / 'litellm_config.yaml').exists() + + +def test_a_crash_before_the_first_commit_publishes_nothing(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + + def crash(*args, **kwargs): + raise RuntimeError('killed') + + ledger.publish_profile = crash + with pytest.raises(RuntimeError, match='killed'): + ctl.access('me', cat.resolve(['remote'])) + assert ledger.profile() is None and ledger.publication_pending() is None + + +def test_a_declined_first_acquire_leaves_no_lease_and_no_profile(tmp_path): + cat = Catalog.from_dict(catalog()) + ledger, ctl = controller(tmp_path, catalog=cat) + _decline(ctl) + with pytest.raises(ConvergeAborted): + ctl.acquire('me', cat.resolve_requests(['local']), wait=False) + assert ledger.profile() is None + assert ledger.status() == ([], []) + + +def test_a_crash_at_the_first_acquire_commit_publishes_nothing(tmp_path): + cat = Catalog.from_dict(catalog()) + ledger, ctl = controller(tmp_path, catalog=cat) + + def crash(*args, **kwargs): + raise RuntimeError('killed') + + ledger.acquire = crash + with pytest.raises(RuntimeError, match='killed'): + ctl.acquire('me', cat.resolve_requests(['local']), wait=False) + assert _published(ledger) == [] + + +def test_the_first_external_access_commits_profile_and_marker_together(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + seen = [] + real = ledger.publish_profile + + def watch(profile, **kwargs): + seen.append((ledger.profile(), ledger.publication_pending())) + real(profile, **kwargs) + seen.append((ledger.profile(), ledger.publication_pending())) + + ledger.publish_profile = watch + ctl.access('me', cat.resolve(['remote'])) + (before_profile, before_marker), (after_profile, after_marker) = seen + assert before_profile is None and before_marker is None + assert 'remote' in _published(ledger) and after_marker is not None + + +def test_an_acquire_commits_its_catalog_with_its_lease(tmp_path): + """A candidate profile (a new catalog on an existing ledger) is written in + the lease's transaction, not before it.""" + first = Catalog.from_dict(catalog()) + ledger, ctl = controller(tmp_path, catalog=first) + ctl.acquire('me', first.resolve_requests(['local']), wait=False) + more = catalog(remote=REMOTE) + more['endpoints']['other'] = {'engine': 'vllm', 'model': 'm', + 'runtime': {'max_model_len': 1024}} + wider = Catalog.from_dict(more) + _, ctl2 = controller(tmp_path, catalog=wider) + + def crash(*args, **kwargs): + raise RuntimeError('killed') + + ledger2 = ctl2.ledger + ledger2.acquire = crash + with pytest.raises(RuntimeError, match='killed'): + ctl2.acquire('me', wider.resolve_requests(['other']), wait=False) + assert 'other' not in _published(ledger) + + +def test_gc_on_a_fresh_ledger_publishes_no_endpoint(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + ctl.gc() + assert _published(ledger) == [] + profile = ledger.profile() + assert profile is None or profile.get('backend') == 'compose' + + +def test_a_first_apply_publishes_with_approval_or_not_at_all(tmp_path): + """`stack up` / `apply` on a fresh ledger publishes the invocation's + endpoints (so catalog routes exist before any model runs), through the + same approved single commit as access.""" + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _keyed(tmp_path, cat) + real = ctl.backend._approve_changes + _decline(ctl) + with pytest.raises(ConvergeAborted): + ctl.apply_now() + assert ledger.profile() is None and ledger.publication_pending() is None + ctl.backend._approve_changes = real + ctl.apply_now() + assert _published(ledger) == ['local', 'remote'] + assert ledger.publication_pending() is None From 852753bd18fa2d5c62362cc663552f85018c40b0 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:28:17 -0400 Subject: [PATCH 123/141] Access is ready only once its routes are published External-only access reported ready when LiteLLM accepted the master key, even if the dynamic route reconcile failed and the publication stayed pending. AccessResult now carries the publication's outcome: an access with external members whose publication is pending is not ready (exit 2; `apply` retries it), and readiness is tri-state, None when it was not checked (--no-wait). The upstream is never asked. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- docs/queue.md | 7 +++- infer_stack/cli/commands_leasing.py | 20 ++++++++--- infer_stack/leasing/controller.py | 41 ++++++++++++++-------- tests/test_external_endpoints.py | 53 +++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 20 deletions(-) diff --git a/docs/queue.md b/docs/queue.md index 66dfd8b8..4c8fc811 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1378,7 +1378,12 @@ declined (no profile, marker, route); crash before the commit; fresh managed acquire declined (no lease, no profile); first access commits profile and marker together; `gc` on a fresh ledger publishes no endpoint. -### 43. [ ] Access readiness includes route publication +### 43. [x] Access readiness includes route publication +*Done 2026-09-27.* `AccessResult.published` (the publication's result; a +pending one is not ready, exit 2, and `apply` retries it) and a tri-state +`ready` (`None` with `--no-wait`); the front-door key check runs only after +a completed publication. Test: route adds refused, the gateway healthy -> +not ready and pending; the retry succeeds -> ready. External-only `access` returns ready when LiteLLM answers the master key, even if the route reconcile failed (`routes=False`, publication pending). Carry the publication result into `AccessResult`; ready requires it diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index d52eee03..dd411d6d 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -747,12 +747,17 @@ def _emit_access(config, controller, result) -> int: 'external': result.external, 'descriptor': _public_descriptor(descriptor), 'ready': result.ready, + 'published': result.published, 'front_door_ready': result.front_door_ready, 'pending': [] if outcome is None or outcome.wait is None else outcome.wait.pending, 'failures': [] if outcome is None or outcome.wait is None else outcome.wait.failures, 'released': released, }, indent=2)) - return 0 if result.ready else 2 + return 2 if result.ready is False else 0 + if not result.published: + print('the gateway\'s routes for this access were not published; the ' + 'publication stays pending and `infer-stack apply` retries it') + return 2 if result.front_door_ready is False: print('the front door did not accept its key in time' + (f'; lease {result.lease.id} released' if result.lease else '')) @@ -765,14 +770,15 @@ def _emit_access(config, controller, result) -> int: for endpoint, model in descriptor['endpoints'].items(): tag = ' (external)' if endpoint in result.external else '' print(f' endpoint {endpoint} -> {model}{tag}') - print(f' ready: {result.ready}') + print(' ready: ' + ('not checked (--no-wait)' if result.ready is None + else str(result.ready))) if outcome is not None and outcome.wait is not None: for gid, endpoint in outcome.wait.pending: print(f' pending: {endpoint} ({gid})') _print_front_door(result.connection) if config.env_file: print(f' env-file: {config.env_file}') - return 0 if result.ready else 2 + return 2 if result.ready is False else 0 def _do_acquire(config, *, owner: str, ttl_seconds: float | None) -> int: @@ -2073,8 +2079,12 @@ def main(cls, argv=True, **kwargs): raise SystemExit( f'run: endpoints not ready: {outcome.wait.pending}' ) - if result.front_door_ready is False: - raise SystemExit('run: the front door did not accept its key in time') + if not result.published or result.front_door_ready is False: + if result.lease is not None and result.front_door_ready is not False: + controller.release(result.lease.id) + raise SystemExit('run: the gateway\'s routes were not published' + if not result.published else + 'run: the front door did not accept its key in time') descriptor = _descriptor_for( controller, result.lease, result.deployments, config, endpoints=result.endpoints, diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index 0eeaae01..f4f2b5e0 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -127,9 +127,12 @@ class AccessResult: the base URL and credential (``None`` on an in-process backend); ``lease``: the one real lease over the managed members, or ``None`` when every member is external (the lease never lists an external name); ``acquire``: that - lease's outcome (readiness, placement). ``front_door_ready`` is ``False`` - when an external member's front door never accepted its key in time; the - lease this access took is then already released. + lease's outcome (readiness, placement). ``published`` is whether the + publication (the gateway's routes included) completed; ``False`` leaves it + pending for the next apply. ``front_door_ready`` is ``False`` when an + external member's front door never accepted its key in time; the lease + this access took is then already released. ``waited`` is whether + readiness was checked at all. """ endpoints: list[str] @@ -139,22 +142,27 @@ class AccessResult: lease: Lease | None = None acquire: AcquireOutcome | None = None front_door_ready: bool | None = None + published: bool = True + waited: bool = True @property def deployments(self) -> list[Deployment]: return list(self.acquire.deployments) if self.acquire is not None else [] @property - def ready(self) -> bool: - """Every member can be asked now (managed ones generated through the - front door; external ones reachable through it).""" - if self.front_door_ready is False: + def ready(self) -> bool | None: + """``True``: verified that every member can be asked now (infer-stack's + routes for them are published, managed members generate through the + front door, and the front door accepts its key; the external server + itself is not asked). ``False``: verified not. ``None``: not checked + (``wait=False``).""" + if not self.published or self.front_door_ready is False: return False - if self.acquire is None: - return True - if self.acquire.released_on_timeout: + if self.acquire is not None and ( + self.acquire.released_on_timeout + or (self.acquire.wait is not None and not self.acquire.wait.ready)): return False - return self.acquire.wait is None or bool(self.acquire.wait.ready) + return True if self.waited else None @dataclass @@ -1741,14 +1749,19 @@ def access( outcome = self.acquire( owner, managed, ttl_seconds=ttl_seconds, wait=wait, timeout=timeout, interval=interval, wait_for_placement=wait_for_placement) + rec = outcome.reconcile else: - self.publish_endpoints() + rec = self.publish_endpoints() result = AccessResult( endpoints=aliases, external=[e.alias for e in external], request_names=self.backend.request_names(aliases), connection=self.backend.connection_info(), - lease=outcome.lease if outcome is not None else None, acquire=outcome) - if front is not None and wait and not (outcome and outcome.released_on_timeout): + lease=outcome.lease if outcome is not None else None, acquire=outcome, + # An external member is reached only through the routes this + # publication installs: gateway liveness alone does not prove them. + published=not (external and rec.publication_pending), waited=wait) + if (front is not None and wait and result.published + and not (outcome and outcome.released_on_timeout)): accepted = front.gateway_accepts( front.master_key(), wait=min(timeout, self.FRONT_DOOR_WAIT_S)) result.front_door_ready = accepted is True diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 6b1f44c6..7a90601a 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -12,6 +12,7 @@ import pytest import yaml +from infer_stack.env_utils import write_env_file from infer_stack.leasing import Catalog from infer_stack.leasing.catalog import CatalogError from infer_stack.leasing.endpoints import ExternalTarget @@ -605,3 +606,55 @@ def test_moving_a_bundled_endpoint_to_external_keeps_the_union_valid(): union = CatalogUnion.from_sources(sources) assert not union.resolve_endpoint('local').managed assert union.bundles['pair'] == ['local', 'remote'] + + +# -- item 43: access readiness includes route publication ----------------------------- + + +class _LiteLLM: + """A healthy LiteLLM (it accepts its key) whose admin API can refuse adds.""" + + def __init__(self): + from test_leasing_dynamic_routing import RecordingGateway + + self.admin = RecordingGateway() + self.refuse = True + + def get(self, url, **kw): + from test_leasing_dynamic_routing import FakeResp + + if url.endswith('/v1/models'): + return FakeResp(200, {'data': []}) + return self.admin.get(url, **kw) + + def post(self, url, **kw): + from test_leasing_dynamic_routing import FakeResp + + if self.refuse and url.endswith('/model/new'): + return FakeResp(500, {'detail': 'db unavailable'}) + return self.admin.post(url, **kw) + + +def test_external_access_is_not_ready_until_its_route_is_published(tmp_path): + from test_leasing_dynamic_routing import FakeTime, _timed_backend + + from infer_stack.leasing import Controller, Ledger, SqliteStore + + time, http = FakeTime(), _LiteLLM() + be = _timed_backend(tmp_path / 'state', http, time) + be.catalog = Catalog.from_dict(catalog(remote=REMOTE)) + write_env_file(be.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + ledger = Ledger(SqliteStore(str(tmp_path / 'ledger.db'))) + ctl = Controller(ledger, be, clock=time.clock, sleep=time.sleep) + result = ctl.access('me', be.catalog.resolve(['remote'])) + assert result.ready is False and not result.published + assert ledger.publication_pending() is not None # stays pending + http.refuse = False # the DB is back + again = ctl.access('me', be.catalog.resolve(['remote'])) + assert again.ready is True and ledger.publication_pending() is None + + +def test_unchecked_access_is_not_called_ready(tmp_path): + cat = Catalog.from_dict(catalog(remote=REMOTE)) + _, ctl = _keyed(tmp_path, cat) + assert ctl.access('me', cat.resolve(['remote']), wait=False).ready is None From 7df1352eb1430cd3242424278d1ea18168de6e7d Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:33:35 -0400 Subject: [PATCH 124/141] Missing keys refuse every publication; FrontDoorControl is the whole interface A route whose key variable has no value used to warn at render, so an unrelated acquire, apply or config publish could recreate the gateway without a key a published endpoint needs. The render (host and in-cluster gateways) now raises MissingRouteKey, a ProfileMismatch the CLI reports at the top level, in every preview before any commit or apply; routes seed checks the published set plus its catalogs under the lock, before the marker. access's own check is gone. FrontDoorControl loses `gateway: Any` and gains the operations callers used to reach through it (env_path, require_route_keys, the route registry). The controller, `env` and `secrets rotate` use only the capability; KubeAI's backend-level key proxies are removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 4 ++ docs/planning/external-endpoints.md | 4 +- docs/queue.md | 17 ++++- infer_stack/backends/kubeai.py | 12 ---- infer_stack/backends/kubeai_gateway.py | 18 ++++- infer_stack/cli/__init__.py | 4 ++ infer_stack/cli/commands_leasing.py | 23 ++++--- infer_stack/leasing/backend.py | 27 +++++++- infer_stack/leasing/compose.py | 19 +++++- infer_stack/leasing/controller.py | 51 +++++++-------- infer_stack/leasing/gateway.py | 37 +++++++++-- tests/test_external_endpoints.py | 91 ++++++++++++++++++++++---- tests/test_leasing_kubeai.py | 10 +-- tests/test_publication_transaction.py | 2 +- 14 files changed, 235 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20393cd8..20b51d91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). with no value. - A runbook whose catalog is already published no longer warns that its catalogs differ from the recovery snapshot. +- A published external endpoint whose key has no value refuses every + publication (acquire, apply, config publish, ...) before anything changes, + with the `infer-stack env NAME=...` to fix it; no command recreates the + gateway without the key. - Publishing endpoints is transactional: an acquire commits its catalogs with its lease and publication marker, and a fresh ledger's first snapshot is written with its first marker, never before. `gc`, `release` and diff --git a/docs/planning/external-endpoints.md b/docs/planning/external-endpoints.md index 2b73482a..e0ecae4b 100644 --- a/docs/planning/external-endpoints.md +++ b/docs/planning/external-endpoints.md @@ -150,7 +150,9 @@ an external-only request with a pointer to `access`. `LITELLM_DB_PASSWORD`, `WEBUI_SECRET_KEY`, `HF_TOKEN`): a catalog must not be able to send the gateway's own credentials to a server it names. A referenced name with no value fails before apply; provider keys are never - generated. + generated. *Implemented at the render* (queue item 44): every publication, + whichever command caused it, refuses while a published route's key has no + value, so an unrelated operation cannot recreate the gateway without it. ### 5. Stable route identity diff --git a/docs/queue.md b/docs/queue.md index 4c8fc811..b17578cd 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1392,7 +1392,15 @@ Readiness tri-state (`None` = not checked, e.g. `--no-wait`). Poison: route reconcile fails, auth succeeds -> not ready, nonzero, pending; a retry that succeeds makes it ready. -### 44. [ ] Missing external keys refuse publication, whatever the command +### 44. [x] Missing external keys refuse publication, whatever the command +*Done 2026-09-27.* The render (both gateways) raises `MissingRouteKey` (a +`ProfileMismatch`, and caught at the CLI's top level) for any desired route +whose key has no value: it runs in every preview, so acquire, access, apply, +config publish refuse before any commit or apply. `routes seed` checks the +published set plus its catalogs under the lock, before the marker. The +command-local check in `access` is gone. Tests: an unrelated acquire / +apply / seed / config publish after the key is removed leaves config, +ledger and profile unchanged; the in-cluster gateway writes no manifest. Only `access` and `routes seed` check; renders warn. An unrelated `acquire`/`apply`/`release`/`gc`/`config publish` can recreate the gateway without a key a published route needs. Validate the full desired route set @@ -1407,7 +1415,12 @@ refuse any change to what a resident alias means, whatever source held it. Poison: resident ad-hoc `qwen`, `routes seed --replace` qwen -> external, dynamic routing: refuse, no catalog/registry change, no marker. -### 46. [ ] Finish `FrontDoorControl` +### 46. [x] Finish `FrontDoorControl` +*Done 2026-09-27.* No `gateway: Any`: the capability has `env_path`, +`require_route_keys`, `registry_routes`, `route_entries`, +`replace_route_entries` beside the key operations and `connection_info`; +the controller's route commands, `env` and `secrets rotate` use only it. +KubeAI's backend-level key proxies are gone (its front door is its gateway). Upper layers still reach `front.gateway` (`missing_keys`, `_env_path`, registry rows) and backend-level gateway proxies; the secrets CLI uses `getattr(backend, 'litellm')`, `backend.master_key()`. Give the capability diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index deb65136..76d766d6 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -623,18 +623,6 @@ def litellm(self) -> bool: def litellm_port(self) -> int | None: return self.gateway.litellm_port if self.gateway is not None else None - def master_key(self) -> str: - return self.gateway.master_key() - - def rotate_master_key(self) -> dict[str, str]: - return self.gateway.rotate_master_key() - - def restore_env(self, values) -> None: - self.gateway.restore_env(values) - - def gateway_accepts(self, key: str, *, wait: float = 0.0): - return self.gateway.gateway_accepts(key, wait=wait) - def _upstream_url(self) -> str: """Where the gateway sends requests for this cluster's Models.""" if not self.gateway_upstream and getattr(self.gateway, 'in_cluster', False): diff --git a/infer_stack/backends/kubeai_gateway.py b/infer_stack/backends/kubeai_gateway.py index ddb5d682..9069aaae 100644 --- a/infer_stack/backends/kubeai_gateway.py +++ b/infer_stack/backends/kubeai_gateway.py @@ -154,6 +154,22 @@ def restore_env(self, values) -> None: def gateway_accepts(self, key: str, *, wait: float = 0.0): return self.gateway.gateway_accepts(key, wait=wait) + @property + def env_path(self) -> Path: + return self.gateway.env_path + + def require_route_keys(self, routes) -> None: + self.gateway.require_route_keys(routes) + + def registry_routes(self): + return self.gateway.registry_routes() + + def route_entries(self) -> dict[str, dict[str, Any]]: + return self.gateway.route_entries() + + def replace_route_entries(self, entries: dict[str, dict[str, Any]]) -> None: + self.gateway.replace_route_entries(entries) + def connection_info(self): return self.gateway.connection_info() @@ -185,7 +201,7 @@ def _render_documents(self) -> dict[Path, str]: ) config = front.litellm_config or '' routes = self.static_routes() - self.gateway.warn_missing_keys(routes) + self.gateway.require_route_keys(routes) key_hash = secret_hash(self._secret_values(routes)) docs = gateway_manifests( namespace=self.namespace, image=self.images['litellm'], config=config, diff --git a/infer_stack/cli/__init__.py b/infer_stack/cli/__init__.py index f0da366a..f4f82803 100644 --- a/infer_stack/cli/__init__.py +++ b/infer_stack/cli/__init__.py @@ -216,6 +216,7 @@ def main(argv=None) -> int: from ..leasing import LeaseLockError from ..leasing.backend import BackendTimeout + from ..leasing.gateway import MissingRouteKey try: rv = ManageCLI.main(argv=argv) @@ -230,6 +231,9 @@ def main(argv=None) -> int: raise SystemExit(runtime_failure(exc)) except BackendTimeout as exc: raise SystemExit(str(exc)) + except MissingRouteKey as exc: + # Any publishing verb: the render refused before anything changed. + raise SystemExit(str(exc)) return int(rv) if rv is not None else 0 diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index dd411d6d..d55f2bbe 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2384,7 +2384,9 @@ def _gateway_state(config) -> tuple[Path, str]: backend = None base_url, _ = front_door_urls(backend) if backend is not None and base_url is not None: - return backend.front_door().gateway._env_path, base_url + front = backend.front_door() + if front is not None: + return front.env_path, base_url return (data_root() / 'leasing' / 'compose' / '.env', f'http://127.0.0.1:{DEFAULT_PORTS["litellm"]}/v1') @@ -3126,7 +3128,7 @@ def main(cls, argv=True, **kwargs): return 4 if any(r['status'] == 'routing-fault' for r in result.values()) else 0 -def _old_key_rejected(backend, key, *, within: float = 60.0) -> bool: +def _old_key_rejected(front, key, *, within: float = 60.0) -> bool: """Whether the gateway comes to reject ``key`` within ``within`` seconds. A replaced gateway can still answer for a moment: a Kubernetes rollout is @@ -3137,7 +3139,7 @@ def _old_key_rejected(backend, key, *, within: float = 60.0) -> bool: deadline = time.monotonic() + within while True: - if backend.gateway_accepts(key) is False: + if front.gateway_accepts(key) is False: return True if time.monotonic() >= deadline: return False @@ -3165,10 +3167,9 @@ def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config, interactive=True) - # Any: rotate_gateway_key below refuses a backend without a gateway, - # so past it these gateway methods exist. - backend: Any = controller.backend - old = backend.master_key() if getattr(backend, 'litellm', False) else None + backend = controller.backend + front = backend.front_door() + old = front.master_key() if front is not None and front.litellm else None try: rec = controller.rotate_gateway_key(force=bool(config.force)) except ProfileMismatch as ex: @@ -3179,13 +3180,15 @@ def main(cls, argv=True, **kwargs): if rec.publication_pending: print(' the gateway has not restarted yet; run `infer-stack apply`') return 3 - new = backend.master_key() - accepted = backend.gateway_accepts(new, wait=60.0) + if front is None: # rotate_gateway_key refused already + return 1 + new = front.master_key() + accepted = front.gateway_accepts(new, wait=60.0) if accepted is None: print(' gateway not running: it will use the new key when it starts') elif not accepted: raise SystemExit('secrets rotate: the gateway rejects the new key') - elif not _old_key_rejected(backend, old): + elif not _old_key_rejected(front, old): raise SystemExit('secrets rotate: could not confirm the gateway ' 'rejects the OLD key') else: diff --git a/infer_stack/leasing/backend.py b/infer_stack/leasing/backend.py index 591397c8..9b6a1550 100644 --- a/infer_stack/leasing/backend.py +++ b/infer_stack/leasing/backend.py @@ -198,13 +198,34 @@ class FrontDoorControl(Protocol): ``backend.front_door()`` returns one, or ``None`` for a backend without a gateway: the Compose project itself, KubeAI's gateway on this host or in the cluster. Keys, the route registry, and "does it accept this key" live - here, not as copies on every serving backend. + here, not as copies on every serving backend; callers never reach the + gateway object behind it. """ #: Whether LiteLLM is on (a front door can be Open WebUI alone). litellm: bool - #: The gateway's state: settings, managed keys, the route registry. - gateway: Any + + @property + def env_path(self) -> Path: + """The managed ``.env`` holding the gateway's keys (``infer-stack env``).""" + ... + + def require_route_keys(self, routes: list[GatewayRoute]) -> None: + """Refuse (:class:`~infer_stack.leasing.gateway.MissingRouteKey`) when a + route sends a key variable the managed ``.env`` does not set.""" + ... + + def registry_routes(self) -> list[GatewayRoute]: + """The route registry's routes (ad-hoc deployments, older rows).""" + ... + + def route_entries(self) -> dict[str, dict[str, Any]]: + """The route registry's rows: alias -> row.""" + ... + + def replace_route_entries(self, entries: dict[str, dict[str, Any]]) -> None: + """Write the route registry's rows.""" + ... def master_key(self) -> str: ... diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index aecd58e2..8ebc845a 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -1304,6 +1304,22 @@ def restore_env(self, values: dict[str, str | None]) -> None: def gateway_accepts(self, key: str, *, wait: float = 0.0) -> bool | None: return self.gateway.gateway_accepts(key, wait=wait) + @property + def env_path(self) -> Path: + return self.gateway.env_path + + def require_route_keys(self, routes) -> None: + self.gateway.require_route_keys(routes) + + def registry_routes(self): + return self.gateway.registry_routes() + + def route_entries(self) -> dict[str, dict[str, Any]]: + return self.gateway.route_entries() + + def replace_route_entries(self, entries: dict[str, dict[str, Any]]) -> None: + self.gateway.replace_route_entries(entries) + def connection_info(self): """Where a client reaches these endpoints: the front door.""" return self.gateway.connection_info() @@ -1524,7 +1540,8 @@ def _render_documents(self, desired: list[Deployment], placement) -> dict[str, A logger.info('Open WebUI runs as root: {}', why) routes, dynamic_routes = self._front_door_routes(desired, plan.assignments) if self.litellm: - self.gateway.warn_missing_keys([*routes, *dynamic_routes]) + # Before any approval or apply, whichever command is publishing. + self.gateway.require_route_keys([*routes, *dynamic_routes]) rendered = render_compose( desired, plan.assignments, images=self.images, ports=self.ports, state=self.state, litellm=self.litellm, litellm_port=self.litellm_port, diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index f4f2b5e0..b53b7c2c 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -30,7 +30,13 @@ from pathlib import Path from typing import Callable, Iterable, TypeVar -from .backend import ApplyResult, ConnectionInfo, HostRuntime, ServingBackend +from .backend import ( + ApplyResult, + ConnectionInfo, + FrontDoorControl, + HostRuntime, + ServingBackend, +) from .routes import RoutePlan from .ledger import Ledger from .models import Deployment, DeploymentState, EndpointRequest, Lease, LeaseState @@ -1723,7 +1729,6 @@ def access( server. If the front door never answers, the lease this call took is released and ``front_door_ready`` is ``False``. """ - from .endpoints import ExternalTarget from .profile import ProfileMismatch aliases = [e.alias for e in endpoints] @@ -1736,13 +1741,6 @@ def access( f'{"is" if len(external) == 1 else "are"} served by an external ' 'server, reached only through the LiteLLM front door; this ' 'backend has none (turn `litellm` on)') - missing = front.gateway.missing_keys(sorted({ - e.target.api_key_env for e in external - if isinstance(e.target, ExternalTarget) and e.target.api_key_env})) - if missing: - raise ProfileMismatch( - f'${missing[0]} has no value in {front.gateway._env_path}; set it ' - f'first: `infer-stack env {missing[0]}=...`') managed = [e.to_request(sharing_override=sharing) for e in endpoints if e.managed] outcome = None if managed: @@ -2024,8 +2022,8 @@ def release_leases( # -- the gateway's route registry (routes seed / prune) ---------------- - def _route_gateway(self): - """The front door's gateway, or a ProfileMismatch saying there is none.""" + def _route_gateway(self) -> FrontDoorControl: + """The front door, or a ProfileMismatch saying there is none.""" from .profile import ProfileMismatch front = self.backend.front_door() @@ -2033,7 +2031,7 @@ def _route_gateway(self): raise ProfileMismatch( 'the `routes` commands need a LiteLLM gateway (the compose or ' 'kubeai backend, with `litellm` on)') - return front.gateway + return front def route_view(self) -> list: """The routes the gateway serves now (``routes list``): derived from the @@ -2108,22 +2106,17 @@ def plan_route_seed(self, catalogs: Iterable) -> RoutePlan: plan.sources = sources return plan - @staticmethod - def _require_keys(gateway, catalog) -> None: - """Refuse to publish an external endpoint whose key variable has no - value in the managed ``.env`` (docs/planning/external-endpoints.md, - decision 4): the gateway would send an empty key.""" - from .endpoints import key_references - from .profile import ProfileMismatch + def _require_keys(self, front: FrontDoorControl, catalog) -> None: + """Refuse a seed when the routes it would publish -- everything + published plus ``catalog`` -- send a key with no value. Checked before + the marker: the render refuses too, but only after the seed stored its + catalogs.""" + from .profile import CatalogUnion - refs = key_references(catalog) - missing = gateway.missing_keys(sorted(refs)) - if missing: - name = missing[0] - raise ProfileMismatch( - f'endpoint {refs[name][0]!r} sends ${name} as its key, which ' - f'{gateway._env_path} does not set; set it first: ' - f'`infer-stack env {name}=...`') + sources = self._published_sources() + published = CatalogUnion.from_sources(sources) if sources else None + front.require_route_keys([*self.backend.catalog_routes(published), + *self.backend.catalog_routes(catalog)]) def commit_route_seed(self, plan: RoutePlan, *, replace: bool = False ) -> tuple[RoutePlan, ReconcileResult]: @@ -2157,6 +2150,10 @@ def preflight(): if n not in shown or not (shown[n] == old)) if moved: raise RouteConflict(moved, changed=True) + from .profile import CatalogUnion + + self._require_keys(gateway, CatalogUnion.from_sources(plan.sources) + if plan.sources else None) def change(): now = plan_seed(self._published_meanings(), plan.incoming) diff --git a/infer_stack/leasing/gateway.py b/infer_stack/leasing/gateway.py index e06645c9..85d5c023 100644 --- a/infer_stack/leasing/gateway.py +++ b/infer_stack/leasing/gateway.py @@ -35,6 +35,7 @@ vllm_service_name_for, ) from .residency import ENGINE_LABEL +from .profile import ProfileMismatch from .routes import GatewayRoute, route_table LITELLM_CONTAINER_PORT = 4000 @@ -168,6 +169,24 @@ def deployment_routes( return routes +class MissingRouteKey(ProfileMismatch): + """A route sends a key variable the managed ``.env`` does not set. + + Raised by the render, before anything is approved or applied, whichever + command published: a gateway recreated without the key would send an + empty one (docs/planning/external-endpoints.md, decision 4). + """ + + def __init__(self, missing: dict[str, list[str]], env_path: Path): + self.missing = dict(missing) + name, aliases = next(iter(sorted(self.missing.items()))) + super().__init__( + f'{", ".join(repr(a) for a in aliases)} sends ${name} as its upstream ' + f'key, which {env_path} does not set; set it first ' + f'(`infer-stack env {name}=...`) or unpublish the endpoint ' + '(`infer-stack routes prune`)') + + def route_key_envs(routes: Sequence[GatewayRoute]) -> list[str]: """The variables ``routes`` send as upstream keys, sorted.""" return sorted({r.key_env for r in routes if r.key_env}) @@ -1051,17 +1070,21 @@ def missing_keys(self, names: Sequence[str]) -> list[str]: env = self.managed_env() return [n for n in names if not env.get(n)] - def warn_missing_keys(self, routes: Sequence[GatewayRoute]) -> None: - """Log each route whose key variable has no value (once per render).""" - from .._log import logger + @property + def env_path(self) -> Path: + """The managed ``.env`` (public spelling of ``_env_path``).""" + return self._env_path + def require_route_keys(self, routes: Sequence[GatewayRoute]) -> None: + """Raise :class:`MissingRouteKey` if any route's key has no value.""" missing = set(self.missing_keys(route_key_envs(routes))) + if not missing: + return + users: dict[str, list[str]] = {} for route in routes: if route.key_env in missing: - logger.warning( - ' route {} sends ${}, which {} does not set; set it with ' - '`infer-stack env {}=...`, then `infer-stack apply`', - route.alias, route.key_env, self._env_path, route.key_env) + users.setdefault(route.key_env, []).append(route.alias) + raise MissingRouteKey(users, self._env_path) def registry_routes(self) -> list[GatewayRoute]: """The registry's routes, the lowest-precedence route layer.""" diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 7a90601a..1d172da6 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -102,10 +102,14 @@ def test_the_cli_adds_an_external_endpoint_and_refuses_runtime_options(tmp_path, # -- routes: one GatewayRoute per alias, whoever runs it ---------------------------- -def _compose(tmp_path, cat, **kw): +def _compose(tmp_path, cat, *, key=True, **kw): + """A compose backend; ``key`` writes the remote's key into its .env.""" from test_leasing_profile import backend - return backend(tmp_path / 'state', catalog=cat, **kw) + be = backend(tmp_path / 'state', catalog=cat, **kw) + if key: + write_env_file(be.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + return be def test_an_external_endpoint_routes_to_its_own_server_with_its_key_by_name(tmp_path): @@ -160,10 +164,14 @@ def test_kubeai_routes_an_external_endpoint_directly_not_through_the_cluster(tmp # -- publication: routes seed publishes, routes prune unpublishes ------------------- -def _ctl(tmp_path, cat, docker=None): +def _ctl(tmp_path, cat, docker=None, *, key=True): + """A controller on a compose stack; ``key`` writes the remote's key.""" from test_leasing_profile import controller - return controller(tmp_path, catalog=cat, docker=docker) + ledger, ctl = controller(tmp_path, catalog=cat, docker=docker) + if key: + write_env_file(ctl.backend.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + return ledger, ctl def _published(ledger): @@ -176,7 +184,7 @@ def test_seed_publishes_an_external_endpoint_and_prune_unpublishes_it(tmp_path): from infer_stack.leasing.profile import ProfileMismatch mine = Catalog.from_dict(catalog()) - ledger, ctl = _ctl(tmp_path, mine) + ledger, ctl = _ctl(tmp_path, mine, key=False) other = Catalog.from_dict({'endpoints': {'remote': REMOTE}}) with pytest.raises(ProfileMismatch, match=r'infer-stack env REMOTE_QWEN_KEY='): ctl.plan_route_seed([other]) # no key: nothing to send @@ -250,9 +258,11 @@ def _dynamic(tmp_path, cat, http): from infer_stack.hardware import simulate_inventory from infer_stack.leasing.compose import ComposeBackend - return ComposeBackend(state_dir=tmp_path, inventory=simulate_inventory('4x80'), - run=FakeDocker(), http=http, images=IMAGES, ports=PORTS, - state=STATE, ui=False, dynamic_routing=True, catalog=cat) + be = ComposeBackend(state_dir=tmp_path, inventory=simulate_inventory('4x80'), + run=FakeDocker(), http=http, images=IMAGES, ports=PORTS, + state=STATE, ui=False, dynamic_routing=True, catalog=cat) + write_env_file(be.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) + return be def test_a_dynamic_external_route_survives_deployments_coming_and_going(tmp_path): @@ -333,13 +343,13 @@ def test_managed_only_gateways_render_exactly_as_before(tmp_path): assert set(env) == {'LITELLM_MASTER_KEY'} -def test_a_render_warns_about_a_key_with_no_value(tmp_path): - from test_leasing_route_registry import capture_warnings +def test_a_render_refuses_a_route_whose_key_has_no_value(tmp_path): + from infer_stack.leasing.gateway import MissingRouteKey - be = _compose(tmp_path, Catalog.from_dict(catalog(remote=REMOTE))) - with capture_warnings() as warnings: + be = _compose(tmp_path, Catalog.from_dict(catalog(remote=REMOTE)), key=False) + with pytest.raises(MissingRouteKey, match='infer-stack env REMOTE_QWEN_KEY='): be.converge([], apply=False) - assert any('REMOTE_QWEN_KEY' in m and 'infer-stack env' in m for m in warnings) + assert not be.compose_file.exists() # nothing written def test_the_cluster_gateway_puts_keys_in_its_secret_and_rolls_on_a_change(tmp_path): @@ -430,9 +440,10 @@ def test_access_to_an_external_endpoint_needs_the_front_door_and_its_key(tmp_pat from infer_stack.leasing.profile import ProfileMismatch cat = Catalog.from_dict(catalog(remote=REMOTE)) - _, ctl = _ctl(tmp_path, cat) + ledger, ctl = _ctl(tmp_path, cat, key=False) with pytest.raises(ProfileMismatch, match='infer-stack env REMOTE_QWEN_KEY='): ctl.access('me', cat.resolve(['remote'])) + assert ledger.profile() is None # refused before any commit _, lean = _ctl(tmp_path / 'lean', cat) lean.backend.litellm = False with pytest.raises(ProfileMismatch, match='LiteLLM front door'): @@ -658,3 +669,55 @@ def test_unchecked_access_is_not_called_ready(tmp_path): cat = Catalog.from_dict(catalog(remote=REMOTE)) _, ctl = _keyed(tmp_path, cat) assert ctl.access('me', cat.resolve(['remote']), wait=False).ready is None + + + +# -- item 44: a missing key refuses every publication -------------------------------- + + +def _remove_key(ctl): + path = ctl.backend.gateway._env_path + path.write_text(''.join(line + '\n' for line in path.read_text().splitlines() + if not line.startswith('REMOTE_QWEN_KEY='))) + + +@pytest.mark.parametrize('operation', ['acquire', 'apply', 'seed', 'config publish']) +def test_an_unrelated_publication_refuses_when_a_published_key_is_gone(tmp_path, operation): + """A working external endpoint, then its key removed: no later publication + (whatever it is for) may recreate the gateway without it.""" + from infer_stack.leasing.gateway import MissingRouteKey + + cat = Catalog.from_dict(catalog(remote=REMOTE)) + ledger, ctl = _ctl(tmp_path, cat) + ctl.access('me', cat.resolve(['remote'])) + config = (ctl.backend.state_dir / 'litellm_config.yaml').read_text() + _remove_key(ctl) + before = ledger.status() + profile = ledger.profile() + with pytest.raises(MissingRouteKey): + if operation == 'acquire': + ctl.acquire('me', cat.resolve_requests(['local']), wait=False) + elif operation == 'apply': + ctl.apply_now() + elif operation == 'seed': + extra = Catalog.from_dict({'models': {'x': {'source': 'hf://o/x'}}, + 'endpoints': {'x': {'engine': 'vllm', 'model': 'x'}}}) + ctl.commit_route_seed(ctl.plan_route_seed([extra])) + else: + ctl.publish_profile({**ledger.profile(), 'ui': False}) + assert (ctl.backend.state_dir / 'litellm_config.yaml').read_text() == config + assert ledger.status() == before and ledger.profile() == profile + if operation != 'apply': # apply retries a pending one + assert ledger.publication_pending() is None + + +def test_the_cluster_gateway_refuses_a_route_whose_key_has_no_value(tmp_path): + from infer_stack.backends.kubeai_gateway import ClusterGateway + from infer_stack.leasing.gateway import MissingRouteKey, catalog_routes + + gw = ClusterGateway(state_dir=tmp_path, namespace='kubeai', url='http://gw:30442', + run=lambda argv: '') + gw.extra_routes = catalog_routes(Catalog.from_dict({'endpoints': {'remote': REMOTE}})) + with pytest.raises(MissingRouteKey): + gw.converge() + assert not gw.manifests_file.exists() diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index d7dc5dca..a48e6351 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -666,7 +666,7 @@ def test_rotating_the_key_recreates_the_gateway_in_front_of_the_cluster(tmp_path ctl = Controller(ledger, be) out = ctl.acquire('alice', [_req('qwen', profile='cpu')], wait=False) ctl.release(out.lease.id) - old = be.master_key() + old = be.front_door().master_key() def fingerprint(): doc = yaml.safe_load(be.gateway.compose_file.read_text()) @@ -674,7 +674,7 @@ def fingerprint(): before = fingerprint() ctl.rotate_gateway_key() - assert be.master_key() != old + assert be.front_door().master_key() != old assert fingerprint() != before @@ -1074,7 +1074,7 @@ def test_the_key_is_a_secret_never_a_diff_and_rotating_it_rolls_the_pods(tmp_pat be, kubectl = make_cluster_gateway_backend(tmp_path) be.converge([]) - key = be.master_key() + key = be.front_door().master_key() assert key not in be.gateway.manifests_file.read_text() secret = be.gateway.state_dir / 'gateway-secret.yaml' assert stat.S_IMODE(secret.stat().st_mode) == 0o600 @@ -1085,7 +1085,7 @@ def key_hash(): return tmpl['metadata']['annotations']['infer-stack/key-hash'] before = key_hash() - be.rotate_master_key() + be.front_door().rotate_master_key() be.converge([]) assert key_hash() != before @@ -1132,7 +1132,7 @@ def test_env_and_test_use_the_kubeai_gateways_key(tmp_path, monkeypatch): be, _ = make_front_door_backend(tmp_path) monkeypatch.setattr(commands_leasing, '_make_backend', lambda config, **kw: be) - key = be.master_key() + key = be.front_door().master_key() env_path, base = commands_leasing._gateway_state(None) assert env_path == be.gateway.gateway._env_path assert commands_leasing._front_door(None) == (base.rstrip('/'), key) diff --git a/tests/test_publication_transaction.py b/tests/test_publication_transaction.py index 5b9bde5d..acfaf260 100644 --- a/tests/test_publication_transaction.py +++ b/tests/test_publication_transaction.py @@ -111,7 +111,7 @@ def test_an_acquire_commits_its_catalog_with_its_lease(tmp_path): more['endpoints']['other'] = {'engine': 'vllm', 'model': 'm', 'runtime': {'max_model_len': 1024}} wider = Catalog.from_dict(more) - _, ctl2 = controller(tmp_path, catalog=wider) + _, ctl2 = _keyed(tmp_path, wider) def crash(*args, **kwargs): raise RuntimeError('killed') From 5fc6e6c0beb3db032c38c86dbcdacc808a43eec3 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:40:15 -0400 Subject: [PATCH 125/141] Pinning protects every resident alias, not only catalog-defined ones The resident-definition guard ran only through catalog conflicts, so an ad-hoc deployment's alias (no catalog definition; a registry row or a dynamic route) could be given a new meaning by `routes seed --replace` or an acquire's catalog adoption while it served; under dynamic routing the alias then had two targets. A resident alias may now take a new published meaning only if it matches the deployment running it; seed checks before its marker, acquire and access in their candidate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- CHANGELOG.md | 4 ++ docs/queue.md | 10 +++- infer_stack/leasing/controller.py | 82 ++++++++++++++++++++++++++++--- tests/test_external_endpoints.py | 59 +++++++++++++++++++++- 4 files changed, 146 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b51d91..ef9b3ea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). with no value. - A runbook whose catalog is already published no longer warns that its catalogs differ from the recovery snapshot. +- An alias a resident deployment serves keeps its meaning even when no + catalog defines it (an ad-hoc acquire): `routes seed --replace`, `access` + and `acquire` refuse to publish a different definition until it is + released or evicted. - A published external endpoint whose key has no value refuses every publication (acquire, apply, config publish, ...) before anything changes, with the `infer-stack env NAME=...` to fix it; no command recreates the diff --git a/docs/queue.md b/docs/queue.md index b17578cd..43cc0643 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1407,7 +1407,15 @@ without a key a published route needs. Validate the full desired route set at the render (both gateways), before any mutation; a typed front-door operation, not callers reading the gateway's `.env` path. -### 45. [ ] Pinning protects every resident alias, not only catalog ones +### 45. [x] Pinning protects every resident alias, not only catalog ones +*Done 2026-09-27.* `Controller._pinned_changes`: a resident alias (LIVE, or +IDLE and resident) may take a new published meaning only if it matches the +deployment running it (same compat key); checked for acquire/access +candidates and in `routes seed`'s preflight (before the marker, with or +without `--replace`). Tests: a resident ad-hoc `qwen` (no catalog), dynamic +routing, `seed --replace` qwen -> external refuses with profile, registry +and marker unchanged; an access publishing one refuses; a catalog matching +the running deployment is accepted. Both refusal tests fail on the old code. `routes seed --replace` (and an acquire's catalog adoption) can publish a new meaning for an alias a resident ad-hoc deployment serves (no catalog definition, registry-only or dynamic route). Pinning is an endpoint rule: diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index b53b7c2c..f0783ea8 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1103,16 +1103,69 @@ def _pinned_endpoints(self, residency=None) -> set[str]: while its container is actually resident, which is the same rule placement uses. Anything else is free to be redefined. """ + return set(self._pinned_deployments(residency)) + + def _pinned_deployments(self, residency=None) -> dict[str, list[Deployment]]: + """``{alias: [resident deployments serving it]}`` (see + :meth:`_pinned_endpoints`).""" _, deployments = self.ledger.status(virtual_expiry=True) - pinned: set[str] = set() + pinned: dict[str, list[Deployment]] = {} for deployment in deployments: - if deployment.state == DeploymentState.LIVE: - pinned.update(deployment.served) - elif deployment.state == DeploymentState.IDLE and residency is not None: - if residency.is_resident(deployment.id): - pinned.update(deployment.served) + resident = deployment.state == DeploymentState.LIVE or ( + deployment.state == DeploymentState.IDLE and residency is not None + and residency.is_resident(deployment.id)) + if resident: + for alias in deployment.served: + pinned.setdefault(alias, []).append(deployment) return pinned + def _pinned_changes(self, before: list[dict], after: list[dict], + residency=None) -> list[str]: + """Resident aliases whose meaning ``after`` (catalog sources) changes + to one their running deployment does not have. + + Pinning is an endpoint rule, not a catalog-conflict case (queue item + 45): an ad-hoc deployment serving ``qwen`` pins ``qwen`` although no + catalog defines it, so publishing ``qwen -> external`` must refuse + like redefining a catalog one. A definition that matches what runs + (same engine and structure) is not a change of meaning. + """ + from .catalog import CatalogError + from .profile import CatalogUnion + + pinned = self._pinned_deployments(residency) + if not pinned: + return [] + old = CatalogUnion.from_sources(before) if before else None + new = CatalogUnion.from_sources(after) if after else None + blocked = [] + for alias in sorted(pinned): + if new is None or alias not in new.endpoints: + continue + try: + meaning = new.resolve_endpoint(alias) + if (old is not None and alias in old.endpoints + and old.resolve_endpoint(alias).semantic_key() + == meaning.semantic_key()): + continue + if meaning.managed and meaning.to_request().compat_key in { + d.compat_key for d in pinned[alias]}: + continue + except CatalogError: + pass + blocked.append(alias) + return blocked + + @staticmethod + def _pinned_refusal(blocked: list[str], what: str) -> Exception: + from .profile import ProfileMismatch + + return ProfileMismatch( + f'{what} redefines {", ".join(repr(b) for b in blocked)}, which a ' + 'resident deployment is running. Release or evict it ' + f'(`infer-stack evict {blocked[0]}`), then retry; definitions nothing ' + 'is running are updated automatically') + def _profile_quiescent(self, residency=None) -> bool: """Whether it is safe to replace the recovery snapshot wholesale. @@ -1192,6 +1245,10 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: f'(`infer-stack evict {blocked[0]}`), then retry; definitions ' 'nothing is running are updated automatically' ) from pinned_ex + blocked = self._pinned_changes( + stored.get('catalogs') or [], candidate['catalogs'], residency) + if blocked: + raise self._pinned_refusal(blocked, 'the current catalog') if quiescent: return candidate if candidate != stored else None @@ -2154,6 +2211,19 @@ def preflight(): self._require_keys(gateway, CatalogUnion.from_sources(plan.sources) if plan.sources else None) + # A resident alias keeps its meaning, wherever that meaning lived + # (a catalog, a registry row, a dynamic route): refused before the + # marker, with or without --replace. + try: + residency = self.backend.residency() + except ResidencyUnknown: + residency = None + published = self._published_sources() + blocked = self._pinned_changes( + published, adopt_catalog_sources(published, plan.sources, set()), + residency) + if blocked: + raise self._pinned_refusal(blocked, 'routes seed') def change(): now = plan_seed(self._published_meanings(), plan.incoming) diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 1d172da6..5670d56b 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -164,11 +164,11 @@ def test_kubeai_routes_an_external_endpoint_directly_not_through_the_cluster(tmp # -- publication: routes seed publishes, routes prune unpublishes ------------------- -def _ctl(tmp_path, cat, docker=None, *, key=True): +def _ctl(tmp_path, cat, docker=None, *, key=True, **kw): """A controller on a compose stack; ``key`` writes the remote's key.""" from test_leasing_profile import controller - ledger, ctl = controller(tmp_path, catalog=cat, docker=docker) + ledger, ctl = controller(tmp_path, catalog=cat, docker=docker, **kw) if key: write_env_file(ctl.backend.gateway._env_path, {'REMOTE_QWEN_KEY': 'sk-remote'}) return ledger, ctl @@ -721,3 +721,58 @@ def test_the_cluster_gateway_refuses_a_route_whose_key_has_no_value(tmp_path): with pytest.raises(MissingRouteKey): gw.converge() assert not gw.manifests_file.exists() + + +# -- item 45: pinning protects every resident alias ------------------------------------ + + +def _ad_hoc(tmp_path, **kw): + """A LIVE deployment serving `qwen` that no catalog defines (an ad-hoc + acquire through a catalog-less controller).""" + from test_leasing_compose import FakeDocker + + docker = FakeDocker() + ledger, ctl = _ctl(tmp_path, None, docker, **kw) + managed = catalog() + managed['endpoints'] = {'qwen': managed['endpoints']['local']} + request = Catalog.from_dict(managed).resolve_requests(['qwen']) + ctl.acquire('me', request, wait=False, apply=False) + return ledger, ctl, docker + + +def test_seed_replace_cannot_redefine_a_resident_ad_hoc_alias(tmp_path): + from infer_stack.leasing.profile import ProfileMismatch + + ledger, ctl, _ = _ad_hoc(tmp_path, dynamic_routing=True) + front = ctl.backend.front_door() + profile, registry = ledger.profile(), front.route_entries() + marker = ledger.publication_pending() + moved = Catalog.from_dict({'endpoints': {'qwen': REMOTE}}) + plan = ctl.plan_route_seed([moved]) + with pytest.raises(ProfileMismatch, match="redefines 'qwen'.*resident"): + ctl.commit_route_seed(plan, replace=True) + assert ledger.profile() == profile and front.route_entries() == registry + assert ledger.publication_pending() == marker # nothing new marked + + +def test_an_acquire_cannot_publish_a_new_meaning_for_a_resident_ad_hoc_alias(tmp_path): + from infer_stack.leasing.profile import ProfileMismatch + + ledger, _, docker = _ad_hoc(tmp_path) + moved = catalog(remote=REMOTE) + moved['endpoints']['qwen'] = REMOTE + cat = Catalog.from_dict(moved) + _, other = _ctl(tmp_path, cat, docker) + with pytest.raises(ProfileMismatch, match="redefines 'qwen'"): + other.access('me', cat.resolve(['remote'])) + assert 'qwen' not in _published(ledger) + + +def test_a_catalog_matching_the_resident_deployment_is_not_a_redefinition(tmp_path): + ledger, _, docker = _ad_hoc(tmp_path) + same = catalog() + same['endpoints']['qwen'] = dict(same['endpoints']['local']) + cat = Catalog.from_dict(same) + _, other = _ctl(tmp_path, cat, docker) + other.acquire('me', cat.resolve_requests(['local']), wait=False, apply=False) + assert 'qwen' in _published(ledger) From 7d158fed758b10569bbcb401464089b3017664c6 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:43:43 -0400 Subject: [PATCH 126/141] routes list says it is desired state; api_base parsed strictly; no pre-registry migration routes list names itself desired state (text, help and JSON) and says when a publication is pending. External api_base is parsed with urlsplit (http(s), a hostname, a valid port, no whitespace), and the docs say it is resolved in the gateway's network: localhost is the gateway. The pre-registry migration is decided against, with the reason, in known-limitations; the published-catalog mutation API is recorded as a design candidate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- README.md | 3 +++ docs/litellm-gateway-routing.md | 9 ++++++--- docs/planning/external-endpoints.md | 11 +++++++++++ docs/planning/known-limitations.md | 14 ++++++++++++++ docs/queue.md | 21 ++++++++++++++++++--- infer_stack/cli/commands_leasing.py | 17 ++++++++++++++--- infer_stack/leasing/endpoints.py | 21 ++++++++++++++++++++- tests/test_external_endpoints.py | 13 +++++++++++++ 8 files changed, 99 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8a028c67..7304d0f2 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,9 @@ infer-stack access qwen-remote --env-file qwen.env source qwen.env # OPENAI_BASE_URL, OPENAI_API_KEY, ... ``` +`api_base` is resolved from inside the gateway's container or pod: +`localhost` there is the gateway itself, so a server on the same host is +reached through the Docker bridge address (e.g. `http://172.17.0.1:8000/v1`). See [docs/planning/external-endpoints.md](docs/planning/external-endpoints.md). ```yaml diff --git a/docs/litellm-gateway-routing.md b/docs/litellm-gateway-routing.md index dd6c8133..c90698df 100644 --- a/docs/litellm-gateway-routing.md +++ b/docs/litellm-gateway-routing.md @@ -52,9 +52,12 @@ ad-hoc deployment) and a genuinely changed endpoint definition. ### Managing routes: `infer-stack routes` -- `infer-stack routes list`: every route the next render has (alias, origin, - upstream model, upstream, and whether a live deployment backs it). Origins - are `catalog`, `external`, `deployment`, `upstream` (KubeAI) and `registry`. +- `infer-stack routes list`: the routes the gateway is to serve (alias, + origin, upstream model, upstream, and whether a live deployment backs it). + It is desired state, derived from the published endpoints and placed + deployments, and says when a publication is pending (the gateway may not + have them yet; `infer-stack apply` finishes it). Origins are `catalog`, + `external`, `deployment`, `upstream` (KubeAI) and `registry`. - `infer-stack routes seed [...]`: publish sibling runbooks' catalogs (merge them into the published union) and publish once. The operational key to blip-free concurrency: **seed all overlapping runbooks' diff --git a/docs/planning/external-endpoints.md b/docs/planning/external-endpoints.md index e0ecae4b..bbc4bdce 100644 --- a/docs/planning/external-endpoints.md +++ b/docs/planning/external-endpoints.md @@ -24,6 +24,17 @@ infer-stack access qwen --env-file qwen.env # publish the route; no lease source qwen.env && my-eval --model "$INFER_STACK_ENDPOINT_QWEN" ``` +**`api_base` is resolved by the gateway, not by your shell.** LiteLLM runs +in a container (Compose, KubeAI's host gateway) or a pod (the in-cluster +gateway), so `http://localhost:8000/v1` means the gateway itself, not the +host you typed it on. A server on the same host is reached through the +host's address as the gateway's network sees it: on Compose, the Docker +bridge (`http://172.17.0.1:8000/v1` on a default install; `ip -4 addr show +docker0`), with the server listening on that interface, not only on +127.0.0.1; from the in-cluster gateway, a node address or a Service name. +A server elsewhere is reached by any name or address the gateway can +resolve. `infer-stack test ` checks the whole path. + Switching `qwen` back to a managed runtime (`engine: vllm`, `model: ...`) changes neither the alias nor these commands: `access` then takes a lease and waits for the model, and the env-file gains `INFER_STACK_LEASE_ID`. A bundle diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index 369ac15b..fe4ace67 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -171,6 +171,20 @@ the cluster's nodes (verified with a second node in a container; a run across two real machines is `dev/handover/p5_two_hosts.sh`). One deployment spanning machines (tensor parallel across nodes) is supported on neither. +### State from before the route registry is not migrated (design boundary) + +A state directory whose gateway predates `litellm_registry.json` (it has a +rendered `litellm_config.yaml` and no registry) is no longer seeded from +that config. Decided 2026-09-27 (queue item 50): seeding it again would make +every rendered route permanent registry state, the duplicate authority the +published catalogs replaced, and what it could recover has another source: +catalog endpoints are republished by their runbooks' next acquire (or at +once, `infer-stack routes seed CATALOG...`), and an ad-hoc deployment's +route by its next acquire. What is lost is an ad-hoc route whose deployment +is gone and whose runbook never runs again, whose upstream was already +gone. Such state directories also predate the recovery profile (2026-09-16; +the registry is from 2026-07-13), so they could not be resumed anyway. + ### The in-cluster gateway routes only (current, deferred) With `kubeai_gateway cluster` the LiteLLM gateway is a Deployment and a diff --git a/docs/queue.md b/docs/queue.md index 43cc0643..3dd9286b 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1436,22 +1436,37 @@ the operations callers need; drop `gateway: Any` from it and proxies whose only job was exposing the gateway. ### 47. [ ] Published catalog state: one mutation API (design candidate) +*Recorded 2026-09-27, not done:* today the mutations are +`adopt_catalog_sources`, `drop_catalog_names` (with its bundle rule) and +`merge_catalog_sources`, all in `profile.py`, each validated by rebuilding +the union. Revisit if another bug lands here. `profile.catalogs` is a list of source snapshots edited by `adopt_catalog_sources`, `drop_catalog_names`, bundle repair. Candidate: one `PublishedCatalogSet` API for every mutation (serialization unchanged). Record; do only if more bugs appear here. -### 48. [ ] `routes list` is desired state; say so +### 48. [x] `routes list` is desired state; say so +*Done 2026-09-27:* help, text and JSON (`state: desired`, +`publication_pending`) say so; a pending publication is announced above the +table. No live inspection command. It derives routes from published definitions and deployments, not from the gateway. Label it desired, show whether a publication is pending; a live inspection, if ever, is a separate command. -### 49. [ ] External `api_base`: stricter parsing, networking documented +### 49. [x] External `api_base`: stricter parsing, networking documented +*Done 2026-09-27:* `urlsplit` (http(s), a hostname, a valid port, no +whitespace); README and the design doc say `api_base` is resolved in the +gateway's network, with the same-host Docker bridge example (the one +`dev/external_e2e.sh` uses). `urlsplit`: http(s), a hostname, no whitespace. Document that `api_base` is resolved in the gateway's network (container or pod): `localhost` is the gateway itself. -### 50. [ ] Pre-registry state directories: decide the migration explicitly +### 50. [x] Pre-registry state directories: decide the migration explicitly +*Decided 2026-09-27: no migration*, recorded in `known-limitations.md` with +the reason (seeding would restore the duplicate authority; catalog routes +come back from their runbooks or `routes seed`; such states also predate the +recovery profile). Reopen if a real installation needs it. The seed-from-`litellm_config.yaml` path was dropped as incidental. Decide: a one-time versioned migration of only what has no other source, or none, recorded with the reason. diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index d55f2bbe..e0b7a952 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -2780,7 +2780,13 @@ def _route_engine(route) -> str: class RoutesListCLI(_LeasingCommonMixin): - """Print the gateway's routes, and where each comes from. + """Print the routes the gateway is to serve, and where each comes from. + + This is desired state, derived from the published endpoints and the + deployments the next render places; it is what the gateway has once the + last publication completed. While one is pending (``publication + pending`` below), the gateway may not have them yet: ``infer-stack + apply`` finishes it. One row per alias: where it routes (a compose service, a KubeAI cluster's gateway under the Model's name, or an external server), its origin -- @@ -2803,6 +2809,7 @@ def main(cls, argv=True, **kwargs): controller = _open_controller(config) routes = _routes_or_exit(controller.route_view) live = _live_endpoints(controller) + pending = controller.ledger.publication_pending() is not None rows = [{ 'name': r.alias, @@ -2814,12 +2821,16 @@ def main(cls, argv=True, **kwargs): } for r in routes] if config.json: - print(json.dumps({'routes': rows}, indent=2)) + print(json.dumps({'state': 'desired', 'publication_pending': pending, + 'routes': rows}, indent=2)) return 0 + if pending: + print('publication pending: the gateway may not have these yet ' + '(`infer-stack apply` finishes it)') if not rows: print('no routes (nothing is published or placed yet)') return 0 - print(f'{len(rows)} route(s):') + print(f'{len(rows)} route(s) the gateway is to serve:') for r in rows: flag = 'live' if r['live'] else ' -' print( diff --git a/infer_stack/leasing/endpoints.py b/infer_stack/leasing/endpoints.py index 8c978657..1eaea81f 100644 --- a/infer_stack/leasing/endpoints.py +++ b/infer_stack/leasing/endpoints.py @@ -86,7 +86,7 @@ def external_errors(name: str, raw: dict[str, Any], external: Any) -> list[str]: errors.append(f"endpoint {name!r}: unknown external key(s) {unknown} " '(api_base, model, api_key_env)') base = external.get('api_base') - if not isinstance(base, str) or not re.match(r'https?://[^/\s]+', base): + if not _is_http_url(base): errors.append(f"endpoint {name!r}: external.api_base must be an http(s) URL, " f'not {base!r}') if not external.get('model') or not isinstance(external.get('model'), str): @@ -103,6 +103,25 @@ def external_errors(name: str, raw: dict[str, Any], external: Any) -> list[str]: return errors +def _is_http_url(value: Any) -> bool: + """An absolute http(s) URL with a host, no whitespace, a valid port. + + >>> [_is_http_url(v) for v in ('http://box:8000/v1', 'https://h/v1', 'box:8000', + ... 'http:///v1', 'http://a b/v1', 'ftp://h/', 'http://h:99999/', None)] + [True, True, False, False, False, False, False, False] + """ + from urllib.parse import urlsplit + + if not isinstance(value, str) or any(c.isspace() for c in value): + return False + try: + parts = urlsplit(value) + parts.port # raises on a malformed port + except ValueError: + return False + return parts.scheme in ('http', 'https') and bool(parts.hostname) + + @dataclass(frozen=True) class ExternalTarget: """An OpenAI-compatible server infer-stack does not run. diff --git a/tests/test_external_endpoints.py b/tests/test_external_endpoints.py index 5670d56b..585d5d0b 100644 --- a/tests/test_external_endpoints.py +++ b/tests/test_external_endpoints.py @@ -776,3 +776,16 @@ def test_a_catalog_matching_the_resident_deployment_is_not_a_redefinition(tmp_pa _, other = _ctl(tmp_path, cat, docker) other.acquire('me', cat.resolve_requests(['local']), wait=False, apply=False) assert 'qwen' in _published(ledger) + + +def test_routes_list_says_it_is_desired_state_and_when_publication_is_pending(cli, capsys): + from infer_stack.leasing import Ledger, SqliteStore + + assert cli.cl.AccessCLI.main(argv=['remote', *cli.base]) == 0 + capsys.readouterr() + cli.cl.RoutesListCLI.main(argv=[*cli.base[:2], '--json']) + data = json.loads(capsys.readouterr().out) + assert data['state'] == 'desired' and data['publication_pending'] is False + Ledger(SqliteStore(cli.base[1])).mark_publication_pending(apply_requested=True) + cli.cl.RoutesListCLI.main(argv=cli.base[:2]) + assert 'publication pending' in capsys.readouterr().out From 93285f57ba1cfb55270ce075d8c8d2f4037db32a Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Sun, 27 Sep 2026 13:52:17 -0400 Subject: [PATCH 127/141] Journal and queue: campaign 2 review round closed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9TxTZLpgCyS6bJe2URhQe --- dev/journals/claude.md | 32 ++++++++++++++++++++++++++++++++ docs/queue.md | 2 ++ 2 files changed, 34 insertions(+) diff --git a/dev/journals/claude.md b/dev/journals/claude.md index e01c0c0f..ea5d049b 100644 --- a/dev/journals/claude.md +++ b/dev/journals/claude.md @@ -3475,3 +3475,35 @@ the arguments that decided them. the alias stops being the model name and the key is the provider's, so a workflow changes when an endpoint moves. Through the front door, moving `qwen` between managed and external changed nothing in the e2e's phase 8. + +## 2026-09-27 — campaign 2 review: the publication boundary caught up + +**Intent.** A review of campaign 2 found that the profile/publication +transaction model had not caught up with `profile.catalogs` being the +published endpoint set (items 42-50). Model: Claude Opus 5.5 +(claude-opus-5-5), Claude Code. + +**Did.** (42) Nothing writes catalogs outside a publication's commit: the +first profile rides the first marker, an acquire's candidate rides its lease +transaction, access and a first `apply` go through `publish_profile`; `gc` +no longer publishes on a fresh ledger. (43) Access readiness includes the +publication's result, tri-state. (44) A route key with no value refuses at +the render, for every command. (45) Pinning compares a resident alias's new +meaning with the deployment running it, whatever source held the old one. +(46) `FrontDoorControl` is the whole interface. (48-50) desired-state +labelling, strict `api_base`, and an explicit no-migration decision. + +**Wider than reported.** Item 42 was reported for a fresh ledger; the same +gap existed on every acquire with a catalog change (profile, marker and +lease were three transactions). Fixed together. + +**Kept deliberately.** `apply` on a fresh ledger still publishes the +invocation's endpoints (through the approved single commit): the parity test +`test_catalog_routes_exist_before_models_run_and_do_not_churn` pins the +no-blip property of `stack up` then `acquire`, and `gc` / `release` / +`evict` are the non-endpoint operations the review meant. + +**Evidence.** Each poison test fails on the old code (item 45's two ran the +full 180 s dynamic reconcile budget there, because the unfixed seed went on +to publish). Suite 1114, ty, flake8, `external_e2e.sh` on Compose (8 phases, +both routing modes) and on KubeAI (1-5, both). diff --git a/docs/queue.md b/docs/queue.md index 3dd9286b..cb708999 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -1354,6 +1354,8 @@ audits, the relevant real e2e. The architecture stands; these are places where the publication/profile transaction model had not caught up with published catalogs being real desired state. Items 42-45 must be fixed before campaign 2 is closed. +*2026-09-27: 42-46 and 48-50 done; 47 recorded as a design candidate. +Verified: suite, ty, flake8, `dev/external_e2e.sh` on Compose and KubeAI.* ### 42. [x] The first profile, and a profile candidate, commit with publication intent *Done 2026-09-27.* `_sync_profile` never writes; a fresh ledger renders from From 7a0163708bb3f9bf9e8772cef338132096f9204e Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 28 Sep 2026 13:24:07 -0400 Subject: [PATCH 128/141] Add Kubernetes integration setup CLI Co-authored-by: GPT-5.6 Sol --- CHANGELOG.md | 22 + README.md | 133 +++--- dev/journals/gpt-5.6-sol.md | 14 + docs/backend-parity.md | 2 +- docs/kubeai-backend.md | 132 +++--- infer_stack/backends/kubeai.py | 50 ++- infer_stack/cli/__init__.py | 6 +- infer_stack/cli/commands_catalog.py | 8 +- infer_stack/cli/commands_kube.py | 234 ++++++++++ infer_stack/kube/__init__.py | 10 + infer_stack/kube/k3s.py | 168 +++++++ infer_stack/kube/manage.py | 662 ++++++++++++++++++++++++++++ scripts/bootstrap_k3s.sh | 40 +- scripts/join_agent.sh | 17 +- tests/test_cli_kube.py | 360 +++++++++++++++ 15 files changed, 1677 insertions(+), 181 deletions(-) create mode 100644 infer_stack/cli/commands_kube.py create mode 100644 infer_stack/kube/__init__.py create mode 100644 infer_stack/kube/k3s.py create mode 100644 infer_stack/kube/manage.py mode change 100644 => 100755 scripts/bootstrap_k3s.sh mode change 100644 => 100755 scripts/join_agent.sh create mode 100644 tests/test_cli_kube.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ef9b3ea3..bf39cfca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### Kubernetes integration setup moves into `infer-stack kube` + +- Added `infer-stack kube nodes` and capability-driven `infer-stack kube setup`; + setup is read-only by default and `--apply` reconciles only infer-stack's + Kubernetes integration requirements. Existing GPU Operator / managed-cluster + NVIDIA resources and externally managed KubeAI installations are accepted + rather than replaced. +- `kube setup --apply` can reconcile the pinned NVIDIA device plugin with GPU + Feature Discovery, install/reconcile KubeAI, and merge discovered GPU + resource profiles without overwriting operator-owned profiles. Existing + KubeAI Helm versions and optional operator values are preserved unless an + explicit override is requested. Existing/operator `secrets.*` values are + carried only through a temporary mode-0600 Helm values file, never copied + into infer-stack's persistent generated YAML. +- Added explicit K3s convenience commands, `infer-stack kube k3s bootstrap` + and `infer-stack kube k3s join`; the old bootstrap/join scripts are now + compatibility wrappers so cluster lifecycle knowledge has one implementation. + Host NVIDIA driver/container-runtime installation remains outside infer-stack. +- KubeAI diagnostics and catalog guidance now point at the shared Kubernetes + setup surface instead of requiring users to compose README recipes and shell + scripts by hand. + ### External endpoints; gateway routes derived from the published catalogs - A catalog endpoint can name an OpenAI-compatible server infer-stack does diff --git a/README.md b/README.md index 7304d0f2..57b29c69 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Two backends run this: **Compose** (single host; vLLM and Ollama) and infer-stack config init # data dir + default backend -> settings.yaml infer-stack catalog suggest --apply # seed catalog.yaml from this host's GPUs infer-stack catalog show # what can be acquired +infer-stack kube setup # inspect/plan Kubernetes integration (read-only) infer-stack acquire # lease, render, bring up, wait for a real generation infer-stack access --env-file e.env # reach it, managed or external; leases only what runs here infer-stack test # one generation through the gateway @@ -549,97 +550,103 @@ the managed key and the endpoint alias on either backend. The short version: ```bash -./scripts/bootstrap_k3s.sh # a one-host cluster (k3s + helm) -./scripts/install_kubeai.sh kubeai-values.yaml # the chart, with your resourceProfiles -kubectl -n kubeai port-forward svc/kubeai 8000:80 & +# New cluster convenience path (omit --version to use the K3s install channel): +infer-stack kube k3s bootstrap --version= + +# Read-only first: inspect the current cluster and the changes infer-stack owns. +infer-stack kube setup +infer-stack kube setup --apply +infer-stack kube nodes + infer-stack config set backend kubeai -infer-stack doctor # cluster -> CRD -> namespace -> KubeAI's API +infer-stack config set kubeai_gateway cluster +infer-stack doctor infer-stack acquire --ttl 2h --env-file lease.env --yes ``` -### KubeAI prerequisites - -You need a Kubernetes cluster, `kubectl` and Helm. `scripts/bootstrap_k3s.sh` -installs k3s and helm on one host; the steps it runs, by hand: +### KubeAI cluster setup -```bash -curl -sfL https://get.k3s.io | sh - -# or pin -curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION='v1.34.3+k3s1' sh - -``` +`infer-stack kube` owns the integration knowledge, not Kubernetes itself. An +existing kubeadm/RKE2/EKS/etc. cluster is first-class: point `kubectl` at it +and start with `infer-stack kube setup`. K3s is only the convenient local +bootstrap path. -Make `kubectl` usable without `sudo`: +`kube setup` is a plan/apply command. The default is read-only and checks the +cluster, Ready nodes, NVIDIA scheduling/discovery capabilities, Helm, KubeAI, +and the resource profiles infer-stack can derive from GPU labels. It accepts +working externally-managed GPU/KubeAI integrations instead of replacing them. ```bash -sudo mkdir -p /etc/rancher/k3s/config.yaml.d -printf 'write-kubeconfig-mode: "0644"\n' | \ - sudo tee /etc/rancher/k3s/config.yaml.d/10-kubeconfig-mode.yaml >/dev/null -sudo systemctl restart k3s -kubectl get nodes +infer-stack kube setup +# [ok] / [WARN] / [NEED] / [FAIL] checklist + proposed managed changes + +infer-stack kube setup --apply ``` -Install Helm: +The apply side is deliberately narrow: it can reconcile the NVIDIA device +plugin + GPU Feature Discovery when an `nvidia` RuntimeClass shows that the +node runtime is already configured, install/reconcile KubeAI, and add missing +discovered GPU `resourceProfiles`. It does **not** install NVIDIA drivers or +the NVIDIA container runtime. On K3s, install those before the first start (or +restart K3s afterward), then rerun `kube setup`. + +For a fresh K3s server, `kube k3s bootstrap` installs/starts K3s, keeps K3s's +own kubeconfig as the authority (readable by the local user), waits for Ready, +and installs Helm if needed. On a clean account it symlinks `~/.kube/config` +to that authoritative file; an existing kubeconfig is never overwritten, and the +CLI tells you to select/merge the K3s context before continuing: ```bash -curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/83a46119086589a593a62ca544982977a60318ca/scripts/get-helm-4 -chmod 700 get_helm.sh -./get_helm.sh -helm version +infer-stack kube k3s bootstrap --version= +infer-stack kube setup +infer-stack kube setup --apply ``` -### NVIDIA GPU support +An exact K3s version is recommended once the cluster is real infrastructure. +If K3s is already active and the requested version differs, infer-stack refuses +to perform an implicit cluster upgrade. -Install the NVIDIA device plugin and GPU Feature Discovery so Kubernetes can expose GPU resources and labels: +`kube setup` generates its reconciled KubeAI values under the infer-stack data +root (`generated/kube/kubeai-values.yaml`). Existing named resource profiles +win over generated profiles, so hand-tuned scheduling rules are not silently +rewritten. An optional operator values file participates in the same merge: ```bash -helm repo add nvdp https://nvidia.github.io/k8s-device-plugin -helm repo update -helm upgrade -i nvdp nvdp/nvidia-device-plugin \ - --version 0.17.1 \ - --namespace nvidia-device-plugin \ - --create-namespace \ - --set gfd.enabled=true \ - --set runtimeClassName=nvidia +infer-stack kube setup --values ./kubeai-values.local.yaml +infer-stack kube setup --apply --values ./kubeai-values.local.yaml ``` -Check that GPU support is working: +On an existing KubeAI Helm release, `--apply` preserves the installed chart +version unless `--kubeai-version=...` explicitly requests another one. On a +fresh install, use that option when you want a fixed chart version. `HF_TOKEN`, +when set, is passed through a short-lived mode-0600 values file rather than on +the Helm command line. + +`infer-stack kube nodes` is the compact inventory used by the setup logic: ```bash -kubectl -n nvidia-device-plugin get pods -kubectl get node "$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')" \ - -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}' -kubectl get nodes --show-labels | tr ',' '\n' | grep 'nvidia.com/' || true +infer-stack kube nodes +infer-stack kube nodes --json ``` -You want a non-empty `nvidia.com/gpu` count and `nvidia.com/*` labels such as product and memory. - -### Resource profiles +The important GPU facts are allocatable `nvidia.com/gpu` plus GPU Feature +Discovery's `nvidia.com/gpu.product` and `nvidia.com/gpu.memory` labels. These +are also the authority used to derive one KubeAI resource profile per GPU +product. If a managed cluster already supplies those capabilities but has no +`nvidia` RuntimeClass, infer-stack accepts it and omits `runtimeClassName` from +its generated profiles. -A `resourceProfile` is what one "GPU unit" means on your cluster; the -catalog's `runtime.resource_profile` (or the `kubeai_resource_profile` -setting) names one, and infer-stack appends the GPU count. Include GPU -`requests`, GPU `limits` and `runtimeClassName: nvidia`: without them the -pod can land on the GPU node and still start without `libcuda.so.1`. +For CPU-only development, skip NVIDIA checks explicitly: ```bash -PRODUCT="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.product}')" - -cat > kubeai-values.yaml < sqlite commit -> post-commit render. With a minimal temporary ubelt stub (the extracted checkout lacks project dependencies), the full Compose + admission suites pass: 115 passed, 4 skipped. `compileall` and `git diff --check` also pass. No HyperQwen-specific runtime logic was added. + +## 2026-09-28 13:07:17 -0400 + +Summary of user intent: move from VM-only KubeAI testing toward practical real-cluster use, and consolidate the currently scattered K3s/NVIDIA/KubeAI setup knowledge into an ergonomic infer-stack surface without turning infer-stack into a general Kubernetes installer. The requested deliverable is an overlay implementing the design. + +Model: GPT-5.6 Sol. Configuration: tool-enabled reasoning session. + +I added a deliberately narrow `infer-stack kube` modal. `kube nodes` inventories Ready state and GPU scheduling/discovery facts. `kube setup` is a read-only capability plan by default; `--apply` reconciles only the cluster integration infer-stack needs. The planner treats working externally managed GPU resources/labels or KubeAI components as satisfying the capability instead of taking them over. When infer-stack does need to manage NVIDIA support, it requires an already-functional `nvidia` RuntimeClass and then installs/reconciles the pinned NVIDIA device-plugin chart with GPU Feature Discovery; host drivers and the NVIDIA container runtime remain explicitly outside infer-stack's ownership. KubeAI setup merges discovered resource profiles into existing/operator Helm values without overwriting same-named hand-tuned profiles, preserves an installed chart version unless explicitly changed, supports an operator `--values` file, and keeps chart `secrets.*` out of both Helm argv and the persistent generated values file by carrying them only through a short-lived mode-0600 values file. An exported `HF_TOKEN` overrides the preserved Hugging Face token in that ephemeral layer. + +K3s remains an explicit convenience implementation underneath Kubernetes rather than being implied by the generic kube surface. `kube k3s bootstrap` owns the former bootstrap flow, configures K3s's persistent kubeconfig mode instead of copying a second kubeconfig authority, waits against `/etc/rancher/k3s/k3s.yaml` explicitly, symlinks a clean `~/.kube/config` to that authority but never overwrites an existing kubeconfig, and installs Helm when absent; an explicit version mismatch on an already-running server refuses an implicit upgrade. `kube setup` also prints the active Kubernetes context before its capability plan so an operator can see which cluster an apply would target. `kube k3s join` reads the cluster token from a file and passes it through the installer environment rather than argv. The old `scripts/bootstrap_k3s.sh` and `scripts/join_agent.sh` are compatibility wrappers around these commands, so they no longer duplicate lifecycle logic. `scripts/install_kubeai.sh` remains available as a low-level/manual escape hatch rather than becoming a second recommended workflow. + +I also factored KubeAI resource-profile generation into a shared backend helper so `catalog suggest` and cluster setup use the same mapping, updated KubeAI doctor/catalog remediation text, and rewrote the README/KubeAI backend/parity docs around the new plan/apply authority. A late review caught two important edge cases: Helm may be absent only when every needed GPU/KubeAI capability is already externally managed; if the plan has any reconciliation action, missing Helm is a required blocker. Also, preserving existing Helm values must not copy `secrets.*` (including a prior Hugging Face token) into generated YAML; those values are now split into the ephemeral secret file and preserved across upgrades without being persisted by infer-stack. The explicit K3s subcommands inherit `kw.Config`, matching the rest of the kwconf command surface. + +Validation in this extracted/offline checkout: `python -m compileall -q infer_stack tests`, `bash -n` on the compatibility wrappers, and `git diff --check` pass. `tests/test_cli_kube.py` plus four focused existing KubeAI sizing/measurement tests pass (14 passed, 60 deselected). Running the complete two relevant test files reaches 67 passing tests; the remaining seven tests fail at collection/import only because `kwconf` is not installed in this container. A minimal temporary kwconf/ubelt shape stub outside the repository successfully imports the complete CLI tree and verifies the nested `ManageCLI.kube -> KubeModalCLI.k3s -> bootstrap/join` registration. Package-index DNS is unavailable, so I could not execute the real kwconf parser/help tree or the full project suite; those should be run in the normal development environment before merge. diff --git a/docs/backend-parity.md b/docs/backend-parity.md index 79006962..157c8b9b 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -23,7 +23,7 @@ cluster setup is [kubeai-backend.md](kubeai-backend.md). | | Compose | KubeAI adds | |---|---|---| -| tools | Docker with the NVIDIA runtime | a cluster (`scripts/bootstrap_k3s.sh` for one host, `scripts/join_agent.sh` for another), `kubectl`, `helm`, the KubeAI chart (`scripts/install_kubeai.sh`), the NVIDIA device plugin on GPU nodes | +| tools | Docker with the NVIDIA runtime | a Kubernetes cluster (`infer-stack kube k3s bootstrap/join` is the K3s convenience path), `kubectl`, `helm`, KubeAI, and GPU scheduling/discovery capabilities (`infer-stack kube setup [--apply]`) | | information | none: GPUs are discovered with `nvidia-smi` | `resourceProfiles` in the chart values (what one GPU unit requests, and on which nodes; `catalog suggest --backend kubeai` proposes them from the nodes' GPU labels), and per endpoint `runtime.resource_profile`, a `min_vram_gib` that picks one by size, or a default `kubeai_resource_profile` | | settings | `backend compose` | `backend kubeai`; `kubeai_gateway cluster` for more than one workstation; `kubeai_namespace`, `kubeai_base_url`, `kubeai_gateway_upstream` when the defaults (namespace `kubeai`, a port-forward on 8000, the Service's cluster IP) do not hold | | preflight | none needed | `infer-stack doctor`: cluster → CRD → namespace → KubeAI's API | diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 93cad3b9..974e5560 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -37,35 +37,44 @@ Where the two backends match and where they still differ, row by row: ## One-time cluster setup +The normal setup path is now capability-driven and lives in the CLI: + ```bash -# 1. A cluster. For a single GPU host, k3s works out of the box: -./scripts/bootstrap_k3s.sh - -# 2. Resource profiles: name -> the requests/limits/nodeSelector that one -# "GPU unit" means on your cluster. These names are what the catalog's -# `runtime.resource_profile` refers to. Without the GPU request and -# runtimeClassName the pod can land on a GPU node and still start without -# libcuda.so.1. `infer-stack catalog suggest --backend kubeai` proposes -# one per GPU product once the device plugin runs. -cat > kubeai-values.yaml <<'EOF' -resourceProfiles: - nvidia-gpu-rtx-4090: - runtimeClassName: nvidia - requests: - nvidia.com/gpu: "1" - limits: - nvidia.com/gpu: "1" - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GeForce-RTX-4090 -EOF - -# 3. Install the chart (HF_TOKEN, if exported, is passed to the chart secret): -./scripts/install_kubeai.sh kubeai-values.yaml kubeai - -# 4. A route to the gateway. The default base_url assumes a port-forward: -kubectl -n kubeai port-forward svc/kubeai 8000:80 & +# Existing cluster: just point kubectl at it. For a new local K3s server: +infer-stack kube k3s bootstrap --version= + +# Planning is read-only; apply only after reviewing it. +infer-stack kube setup +infer-stack kube setup --apply +infer-stack kube nodes ``` +`kube setup` checks the Kubernetes API, Ready nodes, GPU scheduling/discovery +facts, Helm, the KubeAI CRD/service, and discovered GPU resource profiles. It +is capability-based: an existing GPU Operator or externally-managed KubeAI is +accepted when it already supplies what infer-stack needs. It does not replace +working external components merely because they were installed differently. + +The managed NVIDIA path begins only after Kubernetes exposes an `nvidia` +RuntimeClass. Installing NVIDIA drivers/container-runtime packages is host +administration and remains outside infer-stack. With that runtime present, +`kube setup --apply` may install/reconcile the pinned NVIDIA device plugin + +GPU Feature Discovery and then derive profiles from `nvidia.com/gpu.product` +and `nvidia.com/gpu.memory`. + +KubeAI values are written to `/generated/kube/kubeai-values.yaml`. Named +profiles already supplied by the operator win over generated profiles. Use an +operator values file for other chart customization: + +```bash +infer-stack kube setup --values ./kubeai-values.local.yaml +infer-stack kube setup --apply --values ./kubeai-values.local.yaml +``` + +For an existing Helm-managed KubeAI release, apply preserves its installed +chart version by default; use `--kubeai-version=...` for an explicit change. +`scripts/install_kubeai.sh` remains only as a manual Helm escape hatch. + ## Point infer-stack at it ```bash @@ -105,12 +114,9 @@ Discovery) of the nodes its `nodeSelector` selects; a profile without a `nodeSelector`, like the chart's generic ones, has no size and is never picked this way. With none large enough, the `kubeai_resource_profile` default is used, or the acquire is refused with the sizes it found. -`infer-stack catalog suggest` proposes one sized profile per GPU product in -the cluster, ready for the helm values: - -```bash -infer-stack catalog suggest --backend kubeai # catalog on stdout, profiles on stderr -``` +`infer-stack kube setup` is the setup authority for these profiles; +`catalog suggest --backend kubeai` consumes the same profile-generation shape +while suggesting model/catalog entries. Verify the setup before the first acquire — `doctor` checks the chain in dependency order (cluster reachable → CRD installed → namespace → KubeAI's API): @@ -151,37 +157,37 @@ routes only: dynamic routing and Open WebUI need the host placement. ## Add a workstation -The cluster's first node is the one `scripts/bootstrap_k3s.sh` set up. To add -a GPU workstation as a second node: - -1. On the first node, collect the join facts: - ```bash - sudo cat /var/lib/rancher/k3s/server/node-token # the token - k3s --version # join with the same version - ``` -2. On the new workstation (NVIDIA driver and container toolkit installed), - join with the server's version: - ```bash - INSTALL_K3S_VERSION='v1.36.4+k3s1' \ - scripts/join_agent.sh https://:6443 - ``` - Between the nodes, open 6443/tcp (to the first node), 8472/udp (flannel) - and 10250/tcp; for clients, the NodePort (30442/tcp). -3. Back on the first node: the NVIDIA device plugin is a DaemonSet, so it - starts on the new node by itself. Check the new node reports its GPUs and - labels, then add a sized profile for its GPU product: - ```bash - kubectl get node -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}' - infer-stack catalog suggest --backend kubeai # profiles per GPU product (stderr) - ``` - Merge the proposed `resourceProfiles` into your helm values and rerun - `scripts/install_kubeai.sh `. An endpoint that names that profile, - or declares a `min_vram_gib` only that GPU meets, lands on the new node. -4. With the gateway in the cluster (above), a card on either workstation uses - the env file as it is: the NodePort answers on every node. - -`dev/k3s_agent_container.sh` makes a second node out of a container on one -host, for development; `dev/handover/p5_two_hosts.sh` checks a real one. +On the server node, copy the K3s token into a protected file on the new host +through whatever secure channel you normally use, and record the server's K3s +version. Then on the new workstation (after NVIDIA driver/container-runtime +setup when it is a GPU node): + +```bash +infer-stack kube k3s join \ + --server=https://:6443 \ + --token-file=~/.private/k3s-token \ + --node-name= \ + --version= +``` + +The token is read from a file so it does not land in shell history or the +installer argv. Between K3s nodes, allow the K3s-required cluster traffic +(6443/tcp to the server, the configured Flannel/backend traffic, and kubelet +traffic as appropriate for your network); expose the infer-stack NodePort only +on the trusted LAN/VPN. + +Back on the control node, inspect and reconcile the newly visible hardware: + +```bash +infer-stack kube nodes +infer-stack kube setup +infer-stack kube setup --apply +``` + +A new GPU product becomes a new discovered KubeAI resource profile without +rewriting any same-named operator profile. `dev/k3s_agent_container.sh` still +provides a synthetic second K3s node for development; +`dev/handover/p5_two_hosts.sh` is the real two-host evidence path. ## Semantics + limitations diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 76d766d6..7a0b365b 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -23,7 +23,7 @@ ``last_errors``), never silently overwritten. Cluster prerequisites (once per cluster, not per acquire): a reachable -kubeconfig, the KubeAI helm chart installed (``scripts/install_kubeai.sh``) +kubeconfig, KubeAI installed (normally via ``infer-stack kube setup --apply``) with ``resourceProfiles`` matching the ``resource_profile`` names your catalog uses, and a route to the gateway (the default ``base_url`` assumes ``kubectl port-forward svc/kubeai 8000:80``). See ``docs/kubeai-backend.md``. @@ -193,6 +193,33 @@ def node_gpus(nodes: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: return out +def resource_profiles_for_nodes( + nodes: list[dict[str, Any]], *, runtime_class_name: str | None = 'nvidia', +) -> dict[str, Any]: + """One KubeAI resource profile per discovered NVIDIA GPU product. + + This is shared by ``catalog suggest`` and ``kube setup`` so there is one + authority for the generated profile names and scheduling shape. Existing + operator profiles are merged/preserved by the setup layer. + """ + profiles: dict[str, Any] = {} + gpu_nodes = [f for f in node_gpus(nodes).values() + if f.get('count') and f.get('product')] + for facts in sorted(gpu_nodes, + key=lambda f: (f.get('memory_gib') or 0, f.get('product') or '')): + product = str(facts['product']) + profile = { + 'imageName': 'nvidia-gpu', + 'requests': {GPU_RESOURCE: '1'}, + 'limits': {GPU_RESOURCE: '1'}, + 'nodeSelector': {GPU_PRODUCT_LABEL: product}, + } + if runtime_class_name: + profile['runtimeClassName'] = runtime_class_name + profiles[f'nvidia-{dns_slug(product)}'] = profile + return profiles + + def sized_profiles(profiles: dict[str, Any], nodes: dict[str, dict[str, Any]]) -> dict[str, float]: """Per-GPU memory (GiB) of each resource profile that says where it runs. @@ -768,16 +795,11 @@ def suggestion_inventory(self) -> tuple[dict[str, Any], dict[str, Any]]: 'memory_gib': best['memory_gib'], 'memory_mib': int(best['memory_gib'] * 1024), 'display_active': False} for i in range(best['count'])] - profiles = {} - for f in sorted(gpu_nodes, key=lambda f: f['memory_gib']): - product = f['product'] or 'gpu' - profiles[f'nvidia-{dns_slug(product)}'] = { - 'imageName': 'nvidia-gpu', - 'runtimeClassName': 'nvidia', - 'requests': {GPU_RESOURCE: '1'}, - 'limits': {GPU_RESOURCE: '1'}, - 'nodeSelector': {GPU_PRODUCT_LABEL: product}, - } + profiles = resource_profiles_for_nodes([ + {'metadata': {'name': name, 'labels': facts['labels']}, + 'status': {'allocatable': {GPU_RESOURCE: str(facts['count'])}}} + for name, facts in facts.items() + ]) return {'gpu_count': len(gpus), 'gpus': gpus}, profiles def _enrich_min_vram(self, desired) -> None: @@ -997,7 +1019,7 @@ def apply(self) -> ApplyResult: raise RuntimeError( f'kubectl apply failed: {ex}\n' 'Is the KubeAI chart installed and the kubeconfig ' - 'reachable? See scripts/install_kubeai.sh and ' + 'reachable? See `infer-stack kube setup` and ' 'docs/kubeai-backend.md.' ) from ex # Prune: managed Models on the cluster that the render dropped. Their @@ -1276,13 +1298,13 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: if not _run_check( 'cluster reachable', ['version', '--client=false', '-o', 'json'], - 'is the kubeconfig set up? (scripts/bootstrap_k3s.sh)', + 'is the kubeconfig set up? (`infer-stack kube setup`; for new K3s: `infer-stack kube k3s bootstrap`)', ): return checks if not _run_check( 'KubeAI Model CRD installed', ['get', 'crd', 'models.kubeai.org', '-o', 'name'], - 'install the chart: scripts/install_kubeai.sh', + 'run `infer-stack kube setup` to inspect, then `infer-stack kube setup --apply`', ): return checks _run_check( diff --git a/infer_stack/cli/__init__.py b/infer_stack/cli/__init__.py index f4f82803..ed3f7071 100644 --- a/infer_stack/cli/__init__.py +++ b/infer_stack/cli/__init__.py @@ -15,6 +15,7 @@ * ``compose`` — compose + LiteLLM helpers. * ``options`` — shared ``DataConfig`` mixins for override flags. * ``commands_catalog`` — catalog editor (models/endpoints/hosts/bundles). +* ``commands_kube`` — Kubernetes integration inspection/setup. * ``commands_leasing`` — acquire/release/run/leases/test + reconcile. * ``commands_runtime`` — ``status`` + ``stack`` day-2 compose wrappers. * ``commands_meta`` — version/help/config introspection. @@ -30,6 +31,7 @@ # so tests can patch seams where they are actually looked up. from . import ( # noqa: F401 commands_catalog, + commands_kube, commands_leasing, commands_meta, commands_mock, @@ -38,7 +40,7 @@ options, ) from .commands_catalog import CatalogModalCLI -from .commands_mock import MockModalCLI +from .commands_kube import KubeModalCLI from .commands_leasing import ( AccessCLI, AcquireCLI, @@ -60,6 +62,7 @@ ) from .commands_leasing import EnvCLI as LeasingEnvCLI from .commands_leasing import RenderCLI as LeasingRenderCLI +from .commands_mock import MockModalCLI from .commands_meta import ( ConfigModalCLI, ConfigPathsCLI, @@ -121,6 +124,7 @@ class ManageCLI(kw.ModalCLI): paths = ConfigPathsCLI # top-level alias for `config paths` status = StatusCLI doctor = DoctorCLI # preflight the configured backend's prerequisites + kube = KubeModalCLI # inspect/setup the Kubernetes integration infer-stack needs # Catalog editor (models / endpoints / hosts / bundles — no raw YAML) catalog = CatalogModalCLI diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index 79c460f4..dcf7e31e 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -386,8 +386,8 @@ def main(cls, argv=True, **kwargs): ) _print_yaml(text) if values: - print('# helm values for the KubeAI chart (merge, then ' - '`scripts/install_kubeai.sh `):\n' + values, + print('# discovered KubeAI resourceProfiles (the same shape reconciled by ' + '`infer-stack kube setup`; shown here for inspection/manual Helm use):\n' + values, file=sys.stderr) return 0 @@ -416,8 +416,8 @@ def main(cls, argv=True, **kwargs): f'{", ".join(skipped)}' ) if values: - print('resource profiles for the KubeAI chart (merge into your helm ' - 'values, then `scripts/install_kubeai.sh `):') + print('discovered KubeAI resource profiles (normally reconciled by ' + '`infer-stack kube setup --apply`; shown here for inspection/manual Helm use):') print(values, end='') return 0 diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py new file mode 100644 index 00000000..d93aa201 --- /dev/null +++ b/infer_stack/cli/commands_kube.py @@ -0,0 +1,234 @@ +"""Kubernetes integration commands. + +``kube`` intentionally exposes only infer-stack's integration surface. Use +``kubectl``/``helm`` for arbitrary cluster administration. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import kwconf as kw + +from ..kube import KubeManager +from ..kube.manage import NVIDIA_DEVICE_PLUGIN_VERSION +from .context import _apply_path_overrides +from .options import _PathOverridesMixin + + +def _namespace(config) -> str: + from ..paths import get_setting + + return str(config.namespace or get_setting('kubeai_namespace') or 'kubeai') + + +def _print_plan(plan) -> None: + if plan.context: + print(f'Kubernetes context: {plan.context}') + for check in plan.checks: + if check.ok: + mark = 'ok ' + elif check.level == 'action': + mark = 'NEED' + elif check.level == 'advisory': + mark = 'WARN' + else: + mark = 'FAIL' + line = f'[{mark}] {check.name}' + if check.detail: + line += f' — {check.detail}' + print(line) + if plan.resource_profiles: + print('\nresource profiles discovered:') + for name, spec in plan.resource_profiles.items(): + product = (spec.get('nodeSelector') or {}).get('nvidia.com/gpu.product', '?') + print(f' {name}: {product}') + if plan.actions: + print('\nplan:') + for action in plan.actions: + print(f' [plan] {action}') + else: + print('\nplan: no managed changes needed') + + +class KubeNodesCLI(_PathOverridesMixin): + """Show the cluster nodes and the GPU facts infer-stack uses.""" + + __command__ = 'nodes' + json = kw.Value(False, isflag=True, help='Emit machine-readable JSON.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = KubeManager() + try: + rows = manager.node_rows(manager.nodes()) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + if config.json: + print(json.dumps(rows, indent=2, sort_keys=True)) + return 0 + if not rows: + print('(cluster has no nodes)') + return 1 + headers = ('NAME', 'READY', 'GPUS', 'GPU PRODUCT', 'GPU MEMORY') + print(f'{headers[0]:<28} {headers[1]:<7} {headers[2]:>4} {headers[3]:<36} {headers[4]}') + for row in rows: + memory = row['gpu_memory_gib'] + memory_text = f'{memory:g} GiB' if memory is not None else '-' + print( + f"{row['name']:<28} " + f"{('yes' if row['ready'] else 'NO'):<7} " + f"{row['gpu_count']:>4} " + f"{str(row['gpu_product'] or '-'):<36} {memory_text}" + ) + return 0 + + +class KubeSetupCLI(_PathOverridesMixin): + """Inspect or reconcile the Kubernetes capabilities infer-stack needs. + + The default is read-only. ``--apply`` installs/reconciles only components + infer-stack owns: the NVIDIA device-plugin/GFD convenience path (when an + NVIDIA RuntimeClass says the host runtime is ready), and KubeAI with + discovered resource profiles. Existing working GPU integrations and + operator-customized named profiles are preserved. + """ + + __command__ = 'setup' + apply = kw.Value(False, isflag=True, help='Apply the displayed managed changes.') + namespace = kw.Value(None, type=str, help='KubeAI namespace (default: configured namespace or kubeai).') + release = kw.Value('kubeai', type=str, help='KubeAI Helm release name.') + gpu = kw.Value( + 'auto', type=str, choices=['auto', 'nvidia', 'none'], + help='GPU integration: auto-detect, require/manage NVIDIA, or skip GPU checks.', + ) + nvidia_plugin_version = kw.Value( + NVIDIA_DEVICE_PLUGIN_VERSION, type=str, + help='NVIDIA device-plugin chart version used when infer-stack installs it.', + ) + kubeai_version = kw.Value( + None, type=str, + help='Optional KubeAI chart version pin (existing/default chart version otherwise).', + ) + values = kw.Value( + None, type=str, + help='Optional operator KubeAI Helm values file. Its settings win; discovered profiles fill missing names.', + ) + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + namespace = _namespace(config) + manager = KubeManager() + try: + operator_values = manager.load_values_file(config.values) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + if not config.apply: + try: + plan = manager.plan_setup( + namespace=namespace, release=config.release, gpu=config.gpu, + operator_values=operator_values, + ) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + _print_plan(plan) + print('\nNo changes made. Re-run with --apply to reconcile managed components.') + return 1 if plan.failed else 0 + + try: + plan, values_path = manager.apply_setup( + namespace=namespace, + release=config.release, + gpu=config.gpu, + nvidia_plugin_version=config.nvidia_plugin_version, + kubeai_version=config.kubeai_version, + operator_values=operator_values, + ) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + _print_plan(plan) + if values_path is not None: + print(f'\nKubeAI values authority: {values_path}') + return 1 if plan.failed else 0 + + +class K3sBootstrapCLI(kw.Config): + """Install/start a local K3s server (explicit convenience path).""" + + __command__ = 'bootstrap' + version = kw.Value( + None, type=str, + help='Optional exact K3s version, e.g. v1.34.3+k3s1. Omit to use the K3s install channel.', + ) + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + from ..kube.k3s import bootstrap + + try: + default_kubeconfig_ready = bootstrap(version=config.version) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + print('K3s server is active and all nodes are Ready.') + print('K3s kubeconfig: /etc/rancher/k3s/k3s.yaml') + if not default_kubeconfig_ready: + print( + 'Existing ~/.kube/config was left unchanged. Make sure its ' + 'current context selects this K3s cluster (or set KUBECONFIG) ' + 'before running setup.' + ) + print('Next: infer-stack kube setup') + return 0 + + +class K3sJoinCLI(kw.Config): + """Join this machine to a K3s server as an agent.""" + + __command__ = 'join' + server = kw.Value(None, type=str, required=True, help='K3s server URL, e.g. https://host:6443.') + token_file = kw.Value( + None, type=str, required=True, + help='File containing the K3s node token (keeps the secret out of argv/history).', + ) + node_name = kw.Value(None, type=str, help='Optional Kubernetes node name.') + version = kw.Value(None, type=str, help='Exact K3s version; normally match the server.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + from ..kube.k3s import join + + try: + join( + server=config.server, + token_file=Path(config.token_file), + node_name=config.node_name, + version=config.version, + ) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + print('K3s agent is active.') + return 0 + + +class K3sModalCLI(kw.ModalCLI): + """Create/join K3s clusters; not required for existing Kubernetes clusters.""" + + __command__ = 'k3s' + bootstrap = K3sBootstrapCLI + join = K3sJoinCLI + + +class KubeModalCLI(kw.ModalCLI): + """Inspect/setup the Kubernetes capabilities used by infer-stack.""" + + __command__ = 'kube' + nodes = KubeNodesCLI + setup = KubeSetupCLI + k3s = K3sModalCLI diff --git a/infer_stack/kube/__init__.py b/infer_stack/kube/__init__.py new file mode 100644 index 00000000..53010e2c --- /dev/null +++ b/infer_stack/kube/__init__.py @@ -0,0 +1,10 @@ +"""Kubernetes integration helpers used by ``infer-stack kube``. + +This package deliberately manages only the Kubernetes capabilities infer-stack +needs. It is not a kubectl replacement: arbitrary cluster administration stays +with Kubernetes-native tools. +""" + +from .manage import KubeManager, SetupPlan + +__all__ = ['KubeManager', 'SetupPlan'] diff --git a/infer_stack/kube/k3s.py b/infer_stack/kube/k3s.py new file mode 100644 index 00000000..f1ed8a55 --- /dev/null +++ b/infer_stack/kube/k3s.py @@ -0,0 +1,168 @@ +"""Explicit K3s convenience path for creating/joining infer-stack clusters.""" + +from __future__ import annotations + +import os +import shutil +import tempfile +from pathlib import Path +from typing import Callable + +from .manage import default_run + +RunFunc = Callable[..., str] +K3S_INSTALL_URL = 'https://get.k3s.io' +HELM_INSTALL_URL = ( + 'https://raw.githubusercontent.com/helm/helm/' + '83a46119086589a593a62ca544982977a60318ca/scripts/get-helm-4' +) + + +def _require_local_tool(name: str) -> None: + if shutil.which(name) is None: + raise RuntimeError(f'{name} is required') + + +def _fetch_installer(run: RunFunc) -> str: + _require_local_tool('curl') + return run(['curl', '-sfL', K3S_INSTALL_URL]) + + +def _active(run: RunFunc, service: str) -> bool: + try: + run(['systemctl', 'is-active', '--quiet', service]) + except Exception: + return False + return True + + +K3S_KUBECONFIG = Path('/etc/rancher/k3s/k3s.yaml') + + +def _ensure_default_kubeconfig_link() -> bool: + """Point a clean user's default kubeconfig at K3s without copying it. + + Existing kubeconfig state is operator authority and is never replaced. + """ + target = Path.home() / '.kube' / 'config' + if target.exists() or target.is_symlink(): + if target.is_symlink(): + try: + return target.resolve() == K3S_KUBECONFIG + except OSError: + return False + return False + target.parent.mkdir(parents=True, exist_ok=True) + target.symlink_to(K3S_KUBECONFIG) + return True + + +def _ensure_helm(run: RunFunc) -> None: + if shutil.which('helm') is not None: + return + installer = run(['curl', '-fsSL', HELM_INSTALL_URL]) + run(['bash', '-s', '--'], input_text=installer) + + +def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool: + """Install/start a local K3s server and make its kubeconfig readable. + + NVIDIA drivers/container runtime are intentionally not installed here. + K3s detects an already-installed NVIDIA container runtime when it starts; + ``infer-stack kube setup`` reports the resulting RuntimeClass/capabilities. + """ + run = run or default_run + _require_local_tool('sudo') + already_active = _active(run, 'k3s') + if already_active and version: + installed = run(['k3s', '--version']).splitlines()[0] + if version not in installed: + raise RuntimeError( + f'K3s is already active as {installed!r}; requested {version!r}. ' + 'Refusing an implicit cluster upgrade; upgrade K3s explicitly.' + ) + + # Keep this as persistent K3s configuration rather than copying the + # root-owned kubeconfig into ~/.kube (which creates a second authority that + # can drift when K3s rotates credentials). + with tempfile.NamedTemporaryFile('w', prefix='infer-stack-k3s-', delete=False) as file: + file.write('write-kubeconfig-mode: "0644"\n') + source = file.name + try: + run(['sudo', 'mkdir', '-p', '/etc/rancher/k3s/config.yaml.d']) + run([ + 'sudo', 'install', '-m', '0644', source, + '/etc/rancher/k3s/config.yaml.d/10-infer-stack-kubeconfig-mode.yaml', + ]) + finally: + Path(source).unlink(missing_ok=True) + + if already_active: + run(['sudo', 'systemctl', 'restart', 'k3s']) + else: + installer = _fetch_installer(run) + env = os.environ.copy() + preserve = [] + if version: + env['INSTALL_K3S_VERSION'] = version + preserve.append('INSTALL_K3S_VERSION') + cmd = ['sudo'] + if preserve: + cmd.append('--preserve-env=' + ','.join(preserve)) + cmd.extend(['sh', '-']) + run(cmd, input_text=installer, env=env) + + run([ + 'kubectl', '--kubeconfig', str(K3S_KUBECONFIG), + 'wait', '--for=condition=Ready', 'node', '--all', '--timeout=180s', + ]) + default_kubeconfig_ready = _ensure_default_kubeconfig_link() + _ensure_helm(run) + return default_kubeconfig_ready + + +def join( + *, + server: str, + token_file: str | Path, + node_name: str | None = None, + version: str | None = None, + run: RunFunc | None = None, +) -> None: + """Join this host to an existing K3s server without putting its token on argv.""" + run = run or default_run + _require_local_tool('sudo') + if _active(run, 'k3s-agent'): + if version: + installed = run(['k3s', '--version']).splitlines()[0] + if version not in installed: + raise RuntimeError( + f'K3s agent is already active as {installed!r}; requested ' + f'{version!r}. Refusing an implicit agent upgrade.' + ) + return + + token_path = Path(token_file).expanduser() + if not token_path.is_file(): + raise RuntimeError(f'K3s token file does not exist: {token_path}') + token = token_path.read_text(encoding='utf-8').strip() + if not token: + raise RuntimeError(f'K3s token file is empty: {token_path}') + + installer = _fetch_installer(run) + env = os.environ.copy() + env['K3S_URL'] = server + env['K3S_TOKEN'] = token + preserve = ['K3S_URL', 'K3S_TOKEN'] + if node_name: + env['K3S_NODE_NAME'] = node_name + preserve.append('K3S_NODE_NAME') + if version: + env['INSTALL_K3S_VERSION'] = version + preserve.append('INSTALL_K3S_VERSION') + run( + ['sudo', '--preserve-env=' + ','.join(preserve), 'sh', '-'], + input_text=installer, + env=env, + ) + run(['systemctl', 'is-active', '--quiet', 'k3s-agent']) diff --git a/infer_stack/kube/manage.py b/infer_stack/kube/manage.py new file mode 100644 index 00000000..b7020678 --- /dev/null +++ b/infer_stack/kube/manage.py @@ -0,0 +1,662 @@ +"""Capability-driven Kubernetes setup and inspection. + +The important boundary here is between *what infer-stack needs* and *how a +cluster happened to provide it*. Existing GPU Operator / managed-cluster +installations are accepted when they already expose the resources and labels +we need; the NVIDIA device-plugin chart is only a convenience remediation. +""" + +from __future__ import annotations + +import copy +import json +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable + +import yaml + +from ..backends.kubeai import ( + GPU_MEMORY_LABEL, + GPU_PRODUCT_LABEL, + node_gpus, + resource_profiles_for_nodes, +) +from ..paths import data_root + +NVIDIA_DEVICE_PLUGIN_VERSION = '0.17.1' +NVIDIA_DEVICE_PLUGIN_REPO = 'https://nvidia.github.io/k8s-device-plugin' +KUBEAI_REPO = 'https://www.kubeai.org' + +RunFunc = Callable[..., str] + + +def default_run( + args: list[str], + *, + input_text: str | None = None, + env: dict[str, str] | None = None, +) -> str: + """Run a command and return stdout while preserving useful stderr errors.""" + proc = subprocess.run( + args, + input=input_text, + env=env, + text=True, + capture_output=True, + check=False, + ) + if proc.returncode: + detail = (proc.stderr or proc.stdout or '').strip() + message = f'`{" ".join(args)}` failed with exit code {proc.returncode}' + if detail: + message += f': {detail}' + raise RuntimeError(message) + return proc.stdout + + +@dataclass +class Check: + """One read-only capability observation.""" + + name: str + ok: bool + detail: str = '' + level: str = 'required' # required | action | advisory + + +@dataclass +class SetupPlan: + """What ``kube setup`` found and what an apply would reconcile.""" + + checks: list[Check] = field(default_factory=list) + actions: list[str] = field(default_factory=list) + resource_profiles: dict[str, Any] = field(default_factory=dict) + namespace: str = 'kubeai' + context: str = '' + + @property + def failed(self) -> bool: + return any(not c.ok and c.level == 'required' for c in self.checks) + + +class KubeManager: + """Inspect/reconcile the small slice of Kubernetes infer-stack depends on.""" + + def __init__(self, *, run: RunFunc | None = None) -> None: + self.run = run or default_run + + def _json(self, args: list[str]) -> dict[str, Any]: + return json.loads(self.run(args) or '{}') + + def kubectl_json(self, args: list[str]) -> dict[str, Any]: + return self._json(['kubectl', *args]) + + def nodes(self) -> list[dict[str, Any]]: + result = self.kubectl_json(['get', 'nodes', '-o', 'json']) + return list(result.get('items') or []) + + @staticmethod + def node_rows(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Compact node inventory for people and JSON output.""" + gpu = node_gpus(nodes) + rows = [] + for node in nodes: + meta = node.get('metadata') or {} + status = node.get('status') or {} + conditions = status.get('conditions') or [] + ready = any( + c.get('type') == 'Ready' and c.get('status') == 'True' + for c in conditions + ) + name = str(meta.get('name') or '') + facts = gpu.get(name, {}) + rows.append({ + 'name': name, + 'ready': ready, + 'gpu_count': facts.get('count', 0), + 'gpu_product': facts.get('product'), + 'gpu_memory_gib': facts.get('memory_gib'), + }) + return rows + + def command_exists(self, name: str) -> bool: + return shutil.which(name) is not None + + def _exists(self, args: list[str]) -> bool: + try: + self.run(args) + except Exception: + return False + return True + + def runtime_class_exists(self, name: str = 'nvidia') -> bool: + return self._exists( + ['kubectl', 'get', 'runtimeclass', name, '-o', 'name'] + ) + + def crd_exists(self, name: str = 'models.kubeai.org') -> bool: + return self._exists(['kubectl', 'get', 'crd', name, '-o', 'name']) + + def namespace_exists(self, namespace: str) -> bool: + return self._exists( + ['kubectl', 'get', 'namespace', namespace, '-o', 'name'] + ) + + def kubeai_service_exists(self, namespace: str) -> bool: + return self._exists([ + 'kubectl', '-n', namespace, 'get', 'service', 'kubeai', '-o', 'name' + ]) + + def current_context(self) -> str: + try: + return self.run(['kubectl', 'config', 'current-context']).strip() + except Exception: + return '' + + def cluster_reachable(self) -> tuple[bool, str]: + try: + self.run(['kubectl', 'version', '--client=false', '-o', 'json']) + except Exception as ex: # noqa: BLE001 - diagnostic boundary + return False, str(ex) + return True, '' + + def helm_releases(self) -> list[dict[str, Any]]: + value = json.loads(self.run(['helm', 'list', '-A', '-o', 'json']) or '[]') + return value if isinstance(value, list) else [] + + def kubeai_release(self, release: str = 'kubeai') -> dict[str, Any] | None: + if not self.command_exists('helm'): + return None + try: + for item in self.helm_releases(): + if item.get('name') == release: + return item + except Exception: + return None + return None + + @staticmethod + def resource_profiles( + nodes: list[dict[str, Any]], *, runtime_class_name: str | None = 'nvidia', + ) -> dict[str, Any]: + """One KubeAI profile per discovered GPU product.""" + return resource_profiles_for_nodes( + nodes, runtime_class_name=runtime_class_name, + ) + + @staticmethod + def gpu_summary(nodes: list[dict[str, Any]]) -> tuple[bool, bool, str]: + """``(resource exposed, discovery labels complete, detail)``.""" + facts = node_gpus(nodes) + gpu_nodes = {name: f for name, f in facts.items() if f.get('count')} + resource_ok = bool(gpu_nodes) + labels_ok = bool(gpu_nodes) and all( + f.get('product') and f.get('memory_gib') + for f in gpu_nodes.values() + ) + if not gpu_nodes: + return False, False, 'no node exposes allocatable nvidia.com/gpu' + total = sum(int(f.get('count') or 0) for f in gpu_nodes.values()) + missing = [ + name for name, f in gpu_nodes.items() + if not f.get('product') or not f.get('memory_gib') + ] + if missing: + return True, False, ( + f'{total} GPU(s) allocatable, but product/memory labels are ' + 'missing on: ' + ', '.join(sorted(missing)) + ) + products = sorted({str(f['product']) for f in gpu_nodes.values()}) + return True, True, f'{total} GPU(s); ' + ', '.join(products) + + def _kubeai_capability( + self, namespace: str, + ) -> tuple[bool, bool, bool, bool]: + crd = self.crd_exists() + namespace_ok = self.namespace_exists(namespace) + service_ok = namespace_ok and self.kubeai_service_exists(namespace) + return crd and namespace_ok and service_ok, crd, namespace_ok, service_ok + + def _existing_kubeai_values( + self, release: str, namespace: str, + ) -> dict[str, Any]: + found = self.kubeai_release(release) + if found is None or str(found.get('namespace') or '') != namespace: + return {} + try: + text = self.run([ + 'helm', 'get', 'values', release, + '-n', namespace, '-o', 'yaml', + ]) + except Exception: + return {} + return yaml.safe_load(text or '') or {} + + @staticmethod + def load_values_file(path: str | Path | None) -> dict[str, Any]: + if path is None: + return {} + source = Path(path).expanduser() + if not source.is_file(): + raise RuntimeError(f'KubeAI values file does not exist: {source}') + value = yaml.safe_load(source.read_text(encoding='utf-8')) or {} + if not isinstance(value, dict): + raise RuntimeError(f'KubeAI values file must contain a mapping: {source}') + return value + + @classmethod + def _deep_merge(cls, base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + merged = dict(base or {}) + for key, value in (override or {}).items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = cls._deep_merge(merged[key], value) + else: + merged[key] = value + return merged + + def plan_setup( + self, + *, + namespace: str = 'kubeai', + release: str = 'kubeai', + gpu: str = 'auto', + operator_values: dict[str, Any] | None = None, + ) -> SetupPlan: + """Inspect a cluster without mutating it.""" + if gpu not in {'auto', 'nvidia', 'none'}: + raise ValueError("gpu must be 'auto', 'nvidia', or 'none'") + plan = SetupPlan(namespace=namespace) + + if not self.command_exists('kubectl'): + plan.checks.append(Check( + 'kubectl available', False, + 'install kubectl or use `infer-stack kube k3s bootstrap` ' + 'on a new K3s server', + )) + return plan + plan.checks.append(Check('kubectl available', True)) + plan.context = self.current_context() + + reachable, detail = self.cluster_reachable() + plan.checks.append(Check( + 'cluster reachable', reachable, + detail or 'current kubeconfig context answers', + )) + if not reachable: + return plan + + nodes = self.nodes() + rows = self.node_rows(nodes) + not_ready = [row['name'] for row in rows if not row['ready']] + plan.checks.append(Check( + 'nodes Ready', bool(rows) and not not_ready, + (f'{len(rows)} node(s)' if rows and not not_ready + else ('not Ready: ' + ', '.join(not_ready) + if not_ready else 'cluster has no nodes')), + )) + + resource_ok, labels_ok, gpu_detail = self.gpu_summary(nodes) + runtime_ok = self.runtime_class_exists('nvidia') + if gpu == 'none': + plan.checks.append(Check( + 'NVIDIA GPU support', True, 'skipped (--gpu=none)', 'advisory' + )) + else: + wants_nvidia = gpu == 'nvidia' or runtime_ok or resource_ok + if not wants_nvidia: + plan.checks.append(Check( + 'NVIDIA GPU support', False, + 'no NVIDIA RuntimeClass or GPU resource detected; configure ' + 'the NVIDIA driver/container runtime on GPU nodes, then ' + 'rerun (or use --gpu=none)', + 'advisory', + )) + elif resource_ok and labels_ok: + # Existing GPU Operator / managed-cluster integration wins. + plan.checks.append(Check( + 'nvidia.com/gpu allocatable', True, gpu_detail + )) + plan.checks.append(Check( + 'GPU product + memory labels', True, gpu_detail + )) + plan.checks.append(Check( + 'RuntimeClass nvidia', True, + ('available' if runtime_ok else + 'not present; external GPU runtime accepted and generated ' + 'profiles omit runtimeClassName'), + 'advisory', + )) + else: + # Our NVIDIA chart path needs a runtime the node can actually + # select. Missing RuntimeClass is host setup, not something a + # Kubernetes Helm chart can repair. + plan.checks.append(Check( + 'RuntimeClass nvidia', runtime_ok, + '' if runtime_ok else ( + 'Kubernetes has no nvidia RuntimeClass; on K3s install ' + 'nvidia-container-runtime before starting/restarting K3s' + ), + )) + level = 'action' if runtime_ok else 'required' + plan.checks.append(Check( + 'nvidia.com/gpu allocatable', resource_ok, gpu_detail, level + )) + plan.checks.append(Check( + 'GPU product + memory labels', labels_ok, + gpu_detail if labels_ok else ( + f'{gpu_detail}; GPU Feature Discovery must provide ' + f'{GPU_PRODUCT_LABEL} and {GPU_MEMORY_LABEL}' + ), + level, + )) + if runtime_ok: + plan.actions.append( + 'install/reconcile NVIDIA device plugin ' + f'{NVIDIA_DEVICE_PLUGIN_VERSION} with GPU Feature Discovery' + ) + + capability_ok, crd, namespace_ok, service_ok = \ + self._kubeai_capability(namespace) + runtime_class = 'nvidia' if runtime_ok else None + plan.resource_profiles = ( + {} if gpu == 'none' else + self.resource_profiles(nodes, runtime_class_name=runtime_class) + ) + + helm_ok = self.command_exists('helm') + if not helm_ok: + needs_helm = bool(plan.actions) or not capability_ok + if needs_helm: + helm_detail = ( + 'install Helm before `infer-stack kube setup --apply`' + ) + helm_level = 'required' + else: + helm_detail = ( + 'not installed; existing KubeAI/GPU capabilities are ' + 'externally managed' + ) + helm_level = 'advisory' + plan.checks.append(Check( + 'helm available', not needs_helm, helm_detail, helm_level, + )) + plan.checks.append(Check( + 'KubeAI Model CRD', crd, + '' if crd else 'models.kubeai.org is not installed', + )) + plan.checks.append(Check( + f'namespace {namespace!r}', namespace_ok, + '' if namespace_ok else 'configured KubeAI namespace is missing', + )) + plan.checks.append(Check( + f'KubeAI service in namespace {namespace!r}', service_ok, + '' if service_ok else 'service/kubeai is not available', + )) + return plan + plan.checks.append(Check('helm available', True)) + + found_release = self.kubeai_release(release) + managed_here = False + if found_release: + found_ns = str(found_release.get('namespace') or '') + same_ns = found_ns == namespace + managed_here = same_ns + plan.checks.append(Check( + f'KubeAI Helm release {release!r}', same_ns, + (f'installed in namespace {found_ns}' if same_ns else + f'installed in namespace {found_ns}; use --namespace={found_ns} ' + 'or choose another release'), + )) + if same_ns and not capability_ok: + plan.actions.append( + f'reconcile KubeAI release {release!r} in namespace {namespace}' + ) + elif capability_ok: + plan.checks.append(Check( + f'KubeAI Helm release {release!r}', True, + 'not owned by this Helm release; existing KubeAI capability is ' + 'accepted as externally managed', + 'advisory', + )) + else: + managed_here = True + plan.checks.append(Check( + f'KubeAI Helm release {release!r}', False, + f'not installed in namespace {namespace}', 'action', + )) + plan.actions.append( + f'install KubeAI release {release!r} in namespace {namespace}' + ) + + component_level = 'action' if managed_here else 'required' + plan.checks.append(Check( + 'KubeAI Model CRD', crd, + '' if crd else 'models.kubeai.org will be installed with KubeAI', + component_level if not crd else 'required', + )) + plan.checks.append(Check( + f'namespace {namespace!r}', namespace_ok, + '' if namespace_ok else 'will be created with KubeAI', + component_level if not namespace_ok else 'required', + )) + plan.checks.append(Check( + f'KubeAI service in namespace {namespace!r}', service_ok, + '' if service_ok else 'service/kubeai will be installed with KubeAI', + component_level if not service_ok else 'required', + )) + + if plan.resource_profiles and managed_here: + effective_values = self._deep_merge( + self._existing_kubeai_values(release, namespace), + operator_values or {}, + ) + existing_profiles = dict(effective_values.get('resourceProfiles') or {}) + missing = sorted(set(plan.resource_profiles) - set(existing_profiles)) + if missing: + plan.actions.append( + f'reconcile {len(missing)} discovered GPU resource profile(s) ' + f'into KubeAI values: {", ".join(missing)}' + ) + return plan + + def install_nvidia_device_plugin( + self, + *, + version: str = NVIDIA_DEVICE_PLUGIN_VERSION, + ) -> None: + self.run([ + 'helm', 'repo', 'add', 'nvdp', NVIDIA_DEVICE_PLUGIN_REPO, + '--force-update', + ]) + self.run(['helm', 'repo', 'update']) + self.run([ + 'helm', 'upgrade', '--install', 'nvdp', + 'nvdp/nvidia-device-plugin', + '--version', version, + '--namespace', 'nvidia-device-plugin', '--create-namespace', + '--set', 'gfd.enabled=true', + '--set', 'runtimeClassName=nvidia', + '--wait', + ]) + + @staticmethod + def _split_secret_values( + values: dict[str, Any], + ) -> tuple[dict[str, Any], dict[str, Any]]: + """Keep Helm ``secrets`` out of the persistent generated values file.""" + public_values = copy.deepcopy(values or {}) + secrets = public_values.pop('secrets', None) + secret_values = {'secrets': secrets} if secrets else {} + return public_values, secret_values + + @staticmethod + def _merge_profiles( + values: dict[str, Any], generated: dict[str, Any], + ) -> dict[str, Any]: + values = dict(values or {}) + existing = dict(values.get('resourceProfiles') or {}) + # Existing named profiles are operator authority. Add only missing + # discovered products rather than rewriting customized scheduling rules. + merged = dict(generated) + merged.update(existing) + values['resourceProfiles'] = merged + return values + + @staticmethod + def _release_chart_version( + release: dict[str, Any] | None, + ) -> str | None: + chart = str((release or {}).get('chart') or '') + prefix = 'kubeai-' + return chart[len(prefix):] if chart.startswith(prefix) else None + + def install_kubeai( + self, + *, + namespace: str, + release: str, + resource_profiles: dict[str, Any], + version: str | None = None, + operator_values: dict[str, Any] | None = None, + ) -> Path: + self.run([ + 'helm', 'repo', 'add', 'kubeai', KUBEAI_REPO, '--force-update' + ]) + self.run(['helm', 'repo', 'update']) + base_values = self._deep_merge( + self._existing_kubeai_values(release, namespace), + operator_values or {}, + ) + public_values, secret_values = self._split_secret_values(base_values) + values = self._merge_profiles(public_values, resource_profiles) + values_path = data_root() / 'generated' / 'kube' / 'kubeai-values.yaml' + values_path.parent.mkdir(parents=True, exist_ok=True) + values_path.write_text( + yaml.safe_dump(values, sort_keys=False), encoding='utf-8' + ) + + cmd = [ + 'helm', 'upgrade', '--install', release, 'kubeai/kubeai', + '-n', namespace, '--create-namespace', + '-f', str(values_path), '--wait', + ] + if version: + cmd.extend(['--version', version]) + + # Keep chart secrets out of persistent generated YAML and process argv. + # Existing/operator secret values are preserved in-memory; HF_TOKEN, if + # exported, deliberately overrides only the Hugging Face token. + token = os.environ.get('HF_TOKEN', '').strip() + if token: + huggingface = secret_values.setdefault('secrets', {}).setdefault( + 'huggingface', {} + ) + huggingface['token'] = token + secret_path: str | None = None + try: + if secret_values: + with tempfile.NamedTemporaryFile( + 'w', prefix='infer-stack-kubeai-secret-', + suffix='.yaml', delete=False, + ) as file: + os.chmod(file.name, 0o600) + yaml.safe_dump(secret_values, file, sort_keys=False) + secret_path = file.name + cmd.extend(['-f', secret_path]) + self.run(cmd) + finally: + if secret_path: + Path(secret_path).unlink(missing_ok=True) + return values_path + + def apply_setup( + self, + *, + namespace: str = 'kubeai', + release: str = 'kubeai', + gpu: str = 'auto', + nvidia_plugin_version: str = NVIDIA_DEVICE_PLUGIN_VERSION, + kubeai_version: str | None = None, + operator_values: dict[str, Any] | None = None, + ) -> tuple[SetupPlan, Path | None]: + """Reconcile managed prerequisites, then return the fresh plan.""" + plan = self.plan_setup( + namespace=namespace, release=release, gpu=gpu, + operator_values=operator_values, + ) + if any( + c.name == 'cluster reachable' and not c.ok for c in plan.checks + ): + raise RuntimeError('Kubernetes cluster is not reachable') + if plan.failed: + blockers = '; '.join( + c.name for c in plan.checks + if not c.ok and c.level == 'required' + ) + raise RuntimeError(f'cannot apply while required checks fail: {blockers}') + if not self.command_exists('helm'): + # Reaching here means every needed capability was accepted as + # externally managed, so there is nothing for apply to mutate. + return plan, None + + if any( + action.startswith('install/reconcile NVIDIA') + for action in plan.actions + ): + self.install_nvidia_device_plugin( + version=nvidia_plugin_version, + ) + + # Re-read after the DaemonSet/GFD rollout: those labels are the source + # of truth for generated profiles. + nodes = self.nodes() + if any( + action.startswith('install/reconcile NVIDIA') + for action in plan.actions + ): + resource_ok, labels_ok, detail = self.gpu_summary(nodes) + if not (resource_ok and labels_ok): + raise RuntimeError( + 'NVIDIA device-plugin/GFD reconciliation completed but GPU ' + f'capability is still incomplete: {detail}' + ) + runtime_ok = self.runtime_class_exists('nvidia') + profiles = ( + {} if gpu == 'none' else self.resource_profiles( + nodes, + runtime_class_name='nvidia' if runtime_ok else None, + ) + ) + found = self.kubeai_release(release) + if found and str(found.get('namespace') or '') != namespace: + raise RuntimeError( + f'KubeAI release {release!r} already exists in namespace ' + f'{found.get("namespace")!r}; use that namespace or another ' + 'release name' + ) + + values_path: Path | None = None + kubeai_actions = [ + action for action in plan.actions + if 'KubeAI release' in action or 'resource profile' in action + ] + if kubeai_actions: + version = kubeai_version or self._release_chart_version(found) + values_path = self.install_kubeai( + namespace=namespace, + release=release, + resource_profiles=profiles, + version=version, + operator_values=operator_values, + ) + + fresh = self.plan_setup( + namespace=namespace, release=release, gpu=gpu, + operator_values=operator_values, + ) + return fresh, values_path diff --git a/scripts/bootstrap_k3s.sh b/scripts/bootstrap_k3s.sh old mode 100644 new mode 100755 index ceab1db6..fcf27fc7 --- a/scripts/bootstrap_k3s.sh +++ b/scripts/bootstrap_k3s.sh @@ -1,35 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -if ! command -v curl >/dev/null 2>&1; then - echo "curl is required" >&2 - exit 1 -fi - -if ! command -v sudo >/dev/null 2>&1; then - echo "sudo is required" >&2 - exit 1 -fi - -if systemctl is-active --quiet k3s; then - echo "k3s is already active" -else - curl -sfL https://get.k3s.io | sh - -fi - -if ! command -v helm >/dev/null 2>&1; then - echo "Installing helm" - curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -fi - -mkdir -p "$HOME/.kube" -sudo cp /etc/rancher/k3s/k3s.yaml "$HOME/.kube/config" -sudo chown "$(id -u):$(id -g)" "$HOME/.kube/config" -chmod 600 "$HOME/.kube/config" - -export KUBECONFIG="$HOME/.kube/config" -kubectl wait --for=condition=Ready node --all --timeout=180s - -echo "K3s installed and ready" -echo "Kubeconfig: $HOME/.kube/config" -echo "Node token: /var/lib/rancher/k3s/server/node-token" +# Compatibility wrapper. The implementation lives in infer-stack now so the +# CLI, tests, and docs share one setup authority. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ARGS=(kube k3s bootstrap) +if [[ -n "${INSTALL_K3S_VERSION:-}" ]]; then + ARGS+=(--version="${INSTALL_K3S_VERSION}") +fi +exec python "$ROOT/manage.py" "${ARGS[@]}" "$@" diff --git a/scripts/join_agent.sh b/scripts/join_agent.sh old mode 100644 new mode 100755 index d87c7c21..31406e64 --- a/scripts/join_agent.sh +++ b/scripts/join_agent.sh @@ -1,17 +1,28 @@ #!/usr/bin/env bash set -euo pipefail +# Compatibility wrapper for the historical positional interface. New callers +# should prefer: +# infer-stack kube k3s join --server=... --token-file=... [--node-name=...] if [[ $# -lt 2 ]]; then echo "Usage: $0 [node-name]" >&2 exit 1 fi +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SERVER_URL="$1" NODE_TOKEN="$2" NODE_NAME="${3:-}" +TOKEN_FILE="$(mktemp)" +trap 'rm -f "$TOKEN_FILE"' EXIT +chmod 600 "$TOKEN_FILE" +printf '%s\n' "$NODE_TOKEN" > "$TOKEN_FILE" +ARGS=(kube k3s join --server="$SERVER_URL" --token-file="$TOKEN_FILE") if [[ -n "$NODE_NAME" ]]; then - curl -sfL https://get.k3s.io | K3S_URL="$SERVER_URL" K3S_TOKEN="$NODE_TOKEN" K3S_NODE_NAME="$NODE_NAME" sh - -else - curl -sfL https://get.k3s.io | K3S_URL="$SERVER_URL" K3S_TOKEN="$NODE_TOKEN" sh - + ARGS+=(--node-name="$NODE_NAME") fi +if [[ -n "${INSTALL_K3S_VERSION:-}" ]]; then + ARGS+=(--version="${INSTALL_K3S_VERSION}") +fi +python "$ROOT/manage.py" "${ARGS[@]}" diff --git a/tests/test_cli_kube.py b/tests/test_cli_kube.py new file mode 100644 index 00000000..8f7e54e2 --- /dev/null +++ b/tests/test_cli_kube.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from infer_stack.kube.manage import KubeManager + + +GPU_PRODUCT = 'nvidia.com/gpu.product' +GPU_MEMORY = 'nvidia.com/gpu.memory' +GPU_RESOURCE = 'nvidia.com/gpu' + + +def node(name='gpu-a', *, product='RTX-TEST', memory='24576', count='1'): + labels = {} + if product is not None: + labels[GPU_PRODUCT] = product + if memory is not None: + labels[GPU_MEMORY] = memory + allocatable = {} + if count is not None: + allocatable[GPU_RESOURCE] = count + return { + 'metadata': {'name': name, 'labels': labels}, + 'status': { + 'conditions': [{'type': 'Ready', 'status': 'True'}], + 'allocatable': allocatable, + }, + } + + +class FakeCluster: + def __init__( + self, + *, + nodes=None, + runtime_class=True, + crd=True, + namespace=True, + service=True, + releases=None, + values=None, + ): + self.nodes = nodes if nodes is not None else [node()] + self.runtime_class = runtime_class + self.crd = crd + self.namespace = namespace + self.service = service + self.releases = releases if releases is not None else [] + self.values = values or {} + self.applied_values_documents = [] + self.calls = [] + + def __call__(self, args, **kwargs): + self.calls.append((list(args), dict(kwargs))) + if args == ['kubectl', 'config', 'current-context']: + return 'test-cluster\n' + if args[:4] == ['kubectl', 'version', '--client=false', '-o']: + return '{}' + if args == ['kubectl', 'get', 'nodes', '-o', 'json']: + return json.dumps({'items': self.nodes}) + if args[:4] == ['kubectl', 'get', 'runtimeclass', 'nvidia']: + if not self.runtime_class: + raise RuntimeError('not found') + return 'runtimeclass.node.k8s.io/nvidia\n' + if args[:4] == ['kubectl', 'get', 'crd', 'models.kubeai.org']: + if not self.crd: + raise RuntimeError('not found') + return 'customresourcedefinition.apiextensions.k8s.io/models.kubeai.org\n' + if args[:3] == ['kubectl', 'get', 'namespace']: + if not self.namespace: + raise RuntimeError('not found') + return f'namespace/{args[3]}\n' + if len(args) >= 7 and args[:3] == ['kubectl', '-n', 'kubeai'] and args[3:6] == ['get', 'service', 'kubeai']: + if not self.service: + raise RuntimeError('not found') + return 'service/kubeai\n' + if args == ['helm', 'list', '-A', '-o', 'json']: + return json.dumps(self.releases) + if args[:4] == ['helm', 'get', 'values', 'kubeai']: + return yaml.safe_dump(self.values, sort_keys=False) + if args[:3] == ['helm', 'repo', 'add']: + return '' + if args == ['helm', 'repo', 'update']: + return '' + if args[:3] == ['helm', 'upgrade', '--install']: + # Reflect successful KubeAI/NVIDIA installs into the fake cluster. + if len(args) > 3 and args[3] == 'nvdp': + self.runtime_class = True + for item in self.nodes: + alloc = item.setdefault('status', {}).setdefault('allocatable', {}) + alloc.setdefault(GPU_RESOURCE, '1') + labels = item.setdefault('metadata', {}).setdefault('labels', {}) + labels.setdefault(GPU_PRODUCT, 'RTX-TEST') + labels.setdefault(GPU_MEMORY, '24576') + if len(args) > 3 and args[3] == 'kubeai': + self.crd = self.namespace = self.service = True + chart_version = '0.22.0' + if '--version' in args: + chart_version = args[args.index('--version') + 1] + self.releases = [{ + 'name': 'kubeai', 'namespace': 'kubeai', + 'chart': f'kubeai-{chart_version}', + }] + value_docs = [] + for idx, arg in enumerate(args): + if arg == '-f': + value_docs.append( + yaml.safe_load(Path(args[idx + 1]).read_text()) or {} + ) + self.applied_values_documents = value_docs + if value_docs: + merged = {} + for doc in value_docs: + merged = KubeManager._deep_merge(merged, doc) + self.values = merged + return '' + raise AssertionError(f'unhandled fake command: {args!r}') + + +def manager_for(fake): + manager = KubeManager(run=fake) + manager.command_exists = lambda name: name in {'kubectl', 'helm'} + return manager + + +def test_setup_accepts_external_gpu_and_kubeai_without_runtimeclass(): + fake = FakeCluster(runtime_class=False, releases=[]) + manager = manager_for(fake) + plan = manager.plan_setup() + assert not plan.failed + assert plan.context == 'test-cluster' + assert not any('install KubeAI' in action for action in plan.actions) + assert not any('NVIDIA device plugin' in action for action in plan.actions) + (profile,) = plan.resource_profiles.values() + assert 'runtimeClassName' not in profile + assert any('externally managed' in check.detail for check in plan.checks) + + +def test_setup_turns_missing_gpu_plugin_and_kubeai_into_actions(): + fake = FakeCluster( + nodes=[node(product=None, memory=None, count=None)], + runtime_class=True, + crd=False, + namespace=False, + service=False, + releases=[], + ) + manager = manager_for(fake) + plan = manager.plan_setup() + assert not plan.failed + assert any('NVIDIA device plugin' in action for action in plan.actions) + assert any('install KubeAI release' in action for action in plan.actions) + needs = [check for check in plan.checks if not check.ok] + assert needs and all(check.level == 'action' for check in needs) + + +def test_setup_is_noop_when_managed_profiles_are_already_present(): + profile = { + 'requests': {GPU_RESOURCE: '1'}, + 'limits': {GPU_RESOURCE: '1'}, + 'nodeSelector': {GPU_PRODUCT: 'RTX-TEST'}, + } + fake = FakeCluster( + releases=[{'name': 'kubeai', 'namespace': 'kubeai', 'chart': 'kubeai-0.22.0'}], + values={'resourceProfiles': {'nvidia-rtx-test': profile}}, + ) + manager = manager_for(fake) + plan = manager.plan_setup() + assert not plan.failed + assert plan.actions == [] + + +def test_apply_preserves_custom_profile_and_existing_chart_version(tmp_path, monkeypatch): + custom = { + 'runtimeClassName': 'custom-runtime', + 'requests': {GPU_RESOURCE: '1'}, + 'limits': {GPU_RESOURCE: '1'}, + 'nodeSelector': {GPU_PRODUCT: 'RTX-A'}, + } + fake = FakeCluster( + nodes=[ + node('a', product='RTX-A', memory='24576'), + node('b', product='RTX-B', memory='49152'), + ], + releases=[{'name': 'kubeai', 'namespace': 'kubeai', 'chart': 'kubeai-0.22.0'}], + values={'resourceProfiles': {'nvidia-rtx-a': custom}}, + ) + manager = manager_for(fake) + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + fresh, values_path = manager.apply_setup() + assert not fresh.failed + assert values_path == tmp_path / 'generated' / 'kube' / 'kubeai-values.yaml' + values = yaml.safe_load(values_path.read_text()) + assert values['resourceProfiles']['nvidia-rtx-a'] == custom + assert 'nvidia-rtx-b' in values['resourceProfiles'] + helm_upgrade = [ + args for args, _ in fake.calls + if args[:4] == ['helm', 'upgrade', '--install', 'kubeai'] + ][-1] + assert helm_upgrade[helm_upgrade.index('--version') + 1] == '0.22.0' + + +def test_k3s_join_keeps_token_out_of_argv(tmp_path, monkeypatch): + from infer_stack.kube import k3s + + token = 'K10-secret-token::server:abc123' + token_file = tmp_path / 'token' + token_file.write_text(token) + calls = [] + active_checks = 0 + + def fake_run(args, **kwargs): + nonlocal active_checks + calls.append((list(args), dict(kwargs))) + if args[:2] == ['curl', '-sfL']: + return '# installer\n' + if args[:3] == ['systemctl', 'is-active', '--quiet']: + active_checks += 1 + if active_checks == 1: + raise RuntimeError('inactive') + return 'active\n' + return '' + + monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) + k3s.join( + server='https://server:6443', + token_file=token_file, + node_name='worker-b', + version='v1.34.3+k3s1', + run=fake_run, + ) + assert all(token not in ' '.join(args) for args, _ in calls) + install = [item for item in calls if item[0] and item[0][0] == 'sudo'][0] + assert install[1]['env']['K3S_TOKEN'] == token + assert 'K3S_TOKEN' in install[0][1] + + +def test_missing_helm_is_required_when_setup_has_managed_actions(): + fake = FakeCluster( + nodes=[node(product=None, memory=None, count=None)], + runtime_class=True, + # KubeAI itself is external/healthy, but NVIDIA reconciliation needs Helm. + crd=True, + namespace=True, + service=True, + releases=[], + ) + manager = KubeManager(run=fake) + manager.command_exists = lambda name: name == 'kubectl' + plan = manager.plan_setup() + assert plan.failed + helm = [check for check in plan.checks if check.name == 'helm available'][0] + assert not helm.ok + assert helm.level == 'required' + assert any('NVIDIA device plugin' in action for action in plan.actions) + + +def test_gpu_none_does_not_generate_or_reconcile_gpu_profiles(): + fake = FakeCluster( + releases=[{'name': 'kubeai', 'namespace': 'kubeai', 'chart': 'kubeai-0.22.0'}], + values={}, + ) + manager = manager_for(fake) + plan = manager.plan_setup(gpu='none') + assert not plan.failed + assert plan.resource_profiles == {} + assert not any('resource profile' in action for action in plan.actions) + + +def test_apply_fresh_cluster_installs_gpu_then_kubeai(tmp_path, monkeypatch): + fake = FakeCluster( + nodes=[node(product=None, memory=None, count=None)], + runtime_class=True, + crd=False, + namespace=False, + service=False, + releases=[], + ) + manager = manager_for(fake) + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + fresh, values_path = manager.apply_setup(kubeai_version='0.22.0') + assert not fresh.failed + assert fresh.actions == [] + assert values_path is not None + assert 'nvidia-rtx-test' in yaml.safe_load(values_path.read_text())['resourceProfiles'] + upgrades = [ + args for args, _ in fake.calls + if args[:3] == ['helm', 'upgrade', '--install'] + ] + assert upgrades[0][3] == 'nvdp' + assert upgrades[1][3] == 'kubeai' + + +def test_existing_helm_secrets_are_preserved_but_not_persisted(tmp_path, monkeypatch): + fake = FakeCluster( + nodes=[ + node('a', product='RTX-A', memory='24576'), + node('b', product='RTX-B', memory='49152'), + ], + releases=[{'name': 'kubeai', 'namespace': 'kubeai', 'chart': 'kubeai-0.22.0'}], + values={ + 'secrets': {'huggingface': {'token': 'existing-secret'}}, + 'resourceProfiles': { + 'nvidia-rtx-a': { + 'nodeSelector': {GPU_PRODUCT: 'RTX-A'}, + 'requests': {GPU_RESOURCE: '1'}, + 'limits': {GPU_RESOURCE: '1'}, + }, + }, + }, + ) + manager = manager_for(fake) + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + fresh, values_path = manager.apply_setup() + assert not fresh.failed + persisted = yaml.safe_load(values_path.read_text()) + assert 'secrets' not in persisted + assert len(fake.applied_values_documents) == 2 + assert fake.applied_values_documents[1] == { + 'secrets': {'huggingface': {'token': 'existing-secret'}} + } + helm_upgrade = [ + args for args, _ in fake.calls + if args[:4] == ['helm', 'upgrade', '--install', 'kubeai'] + ][-1] + value_paths = [ + Path(helm_upgrade[idx + 1]) + for idx, arg in enumerate(helm_upgrade) if arg == '-f' + ] + assert value_paths[0] == values_path + assert not value_paths[1].exists() + + +def test_k3s_bootstrap_existing_server_does_not_fetch_installer(monkeypatch): + from infer_stack.kube import k3s + + calls = [] + + def fake_run(args, **kwargs): + calls.append((list(args), dict(kwargs))) + if args == ['systemctl', 'is-active', '--quiet', 'k3s']: + return 'active\n' + if args == ['k3s', '--version']: + return 'k3s version v1.34.3+k3s1 (test)\n' + return '' + + monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) + monkeypatch.setattr(k3s, '_ensure_helm', lambda run: None) + monkeypatch.setattr(k3s, '_ensure_default_kubeconfig_link', lambda: True) + ready = k3s.bootstrap(version='v1.34.3+k3s1', run=fake_run) + assert ready is True + assert not any(args and args[0] == 'curl' for args, _ in calls) + assert any(args[:3] == ['sudo', 'systemctl', 'restart'] for args, _ in calls) + assert any( + args[:3] == ['kubectl', '--kubeconfig', str(k3s.K3S_KUBECONFIG)] + for args, _ in calls + ) From 2608d62fd26a1e0cead0eeeee330479e0e5d1083 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 28 Sep 2026 14:06:33 -0400 Subject: [PATCH 129/141] Keep Kubernetes integration distribution neutral Co-authored-by: GPT-5.6 Sol --- CHANGELOG.md | 6 + README.md | 18 ++- dev/journals/gpt-5.6-sol.md | 36 +++++ docs/cluster-setup.md | 231 +++++++++++++++++++++++++++++++ docs/kubeai-backend.md | 8 +- infer_stack/backends/kubeai.py | 2 +- infer_stack/cli/commands_kube.py | 20 ++- infer_stack/kube/__init__.py | 11 +- infer_stack/kube/k3s.py | 7 +- infer_stack/kube/manage.py | 9 +- tests/test_cli_kube.py | 27 ++++ tests/test_leasing_kubeai.py | 1 + 12 files changed, 352 insertions(+), 24 deletions(-) create mode 100644 docs/cluster-setup.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bf39cfca..71bb8c5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). ### Kubernetes integration setup moves into `infer-stack kube` +- Kept the generic Kubernetes manager/backend distribution-neutral: generic + diagnostics now describe Kubernetes capabilities rather than K3s remediation, + while K3s provisioning remains isolated under `infer-stack kube k3s`. Added a + dedicated cluster setup runbook that documents the control-plane/worker model, + K3s as the first provisioning target, and the contract future distro-specific + integrations must preserve. - Added `infer-stack kube nodes` and capability-driven `infer-stack kube setup`; setup is read-only by default and `--apply` reconciles only infer-stack's Kubernetes integration requirements. Existing GPU Operator / managed-cluster diff --git a/README.md b/README.md index 57b29c69..2922bfd6 100644 --- a/README.md +++ b/README.md @@ -542,7 +542,10 @@ services and the cluster scheduler in place of the local GPU planner. The LiteLLM gateway still fronts everything, so a card sees one `OPENAI_BASE_URL`, the managed key and the endpoint alias on either backend. -* Setup, settings and semantics: [docs/kubeai-backend.md](docs/kubeai-backend.md). +* Creating/joining a cluster and the distribution boundary: + [docs/cluster-setup.md](docs/cluster-setup.md). +* KubeAI setup, settings and semantics: + [docs/kubeai-backend.md](docs/kubeai-backend.md). * What matches Compose, what does not yet, and what is deliberate: [docs/backend-parity.md](docs/backend-parity.md); the plan to close the rest: [docs/planning/backend-parity-roadmap.md](docs/planning/backend-parity-roadmap.md). @@ -564,12 +567,15 @@ infer-stack doctor infer-stack acquire --ttl 2h --env-file lease.env --yes ``` -### KubeAI cluster setup +### Kubernetes cluster setup and KubeAI integration -`infer-stack kube` owns the integration knowledge, not Kubernetes itself. An -existing kubeadm/RKE2/EKS/etc. cluster is first-class: point `kubectl` at it -and start with `infer-stack kube setup`. K3s is only the convenient local -bootstrap path. +See [docs/cluster-setup.md](docs/cluster-setup.md) for the server/worker mental +model and the complete first-cluster runbook. `infer-stack kube` owns the +integration knowledge, not Kubernetes itself. Its generic commands are +distribution-neutral: an existing kubeadm/RKE2/k0s/EKS/etc. cluster is +first-class once `kubectl` selects it. K3s is the first explicitly supported +**provisioning** integration and therefore lives under `infer-stack kube k3s`, +not in the generic manager/backend. `kube setup` is a plan/apply command. The default is read-only and checks the cluster, Ready nodes, NVIDIA scheduling/discovery capabilities, Helm, KubeAI, diff --git a/dev/journals/gpt-5.6-sol.md b/dev/journals/gpt-5.6-sol.md index 7ab7e6d9..521ba521 100644 --- a/dev/journals/gpt-5.6-sol.md +++ b/dev/journals/gpt-5.6-sol.md @@ -155,3 +155,39 @@ K3s remains an explicit convenience implementation underneath Kubernetes rather I also factored KubeAI resource-profile generation into a shared backend helper so `catalog suggest` and cluster setup use the same mapping, updated KubeAI doctor/catalog remediation text, and rewrote the README/KubeAI backend/parity docs around the new plan/apply authority. A late review caught two important edge cases: Helm may be absent only when every needed GPU/KubeAI capability is already externally managed; if the plan has any reconciliation action, missing Helm is a required blocker. Also, preserving existing Helm values must not copy `secrets.*` (including a prior Hugging Face token) into generated YAML; those values are now split into the ephemeral secret file and preserved across upgrades without being persisted by infer-stack. The explicit K3s subcommands inherit `kw.Config`, matching the rest of the kwconf command surface. Validation in this extracted/offline checkout: `python -m compileall -q infer_stack tests`, `bash -n` on the compatibility wrappers, and `git diff --check` pass. `tests/test_cli_kube.py` plus four focused existing KubeAI sizing/measurement tests pass (14 passed, 60 deselected). Running the complete two relevant test files reaches 67 passing tests; the remaining seven tests fail at collection/import only because `kwconf` is not installed in this container. A minimal temporary kwconf/ubelt shape stub outside the repository successfully imports the complete CLI tree and verifies the nested `ManageCLI.kube -> KubeModalCLI.k3s -> bootstrap/join` registration. Package-index DNS is unavailable, so I could not execute the real kwconf parser/help tree or the full project suite; those should be run in the normal development environment before merge. + +## 2026-09-28 13:52:00 -0400 + +Summary of user intent: keep the new Kubernetes integration from acquiring K3s +as an architectural dependency while retaining K3s as infer-stack's first +well-supported workstation-cluster provisioning path. + +Model: GPT-5.6 Sol. Configuration: tool-enabled reasoning session. + +I tightened the distribution boundary rather than inventing a provider +abstraction before a second provisioning integration exists. The generic +`KubeManager` and KubeAI backend diagnostics no longer mention K3s-specific +remediation; they report missing Kubernetes/container-runtime capabilities in +distribution-neutral terms. K3s knowledge remains in the explicit +`infer_stack.kube.k3s` leaf module and `infer-stack kube k3s` CLI namespace. +Package/CLI docstrings now state that separation explicitly. + +I added `docs/cluster-setup.md` as the operator runbook. It distinguishes the +Kubernetes control plane from infer-stack authority, shows the initial K3s +server + agent topology (with the server also usable as a GPU worker), gives the +create/join/reconcile sequence, and gives an existing-cluster path that starts +directly at `kube nodes` / `kube setup`. README and the KubeAI backend guide +link to this runbook and identify K3s as the first provisioning target rather +than the generic backend. Focused tests assert that generic missing-kubectl and +missing-NVIDIA-runtime diagnostics remain distribution-neutral. + +Validation: with the same minimal offline ubelt/kwconf stubs used for the prior +overlay, `tests/test_cli_kube.py` passes 12/12. The Kubernetes CLI suite plus +the non-parser KubeAI sizing/doctor checks pass 16/16. Running both complete +relevant files reaches 71 passed / 5 failed; every failure is a CLI-entry test +that reaches the intentional `kwconf.Config.cli` stub, so the real kwconf parser +remains the only unavailable test dependency in this environment. `compileall`, +`bash -n` on both compatibility wrappers, and `git diff --check` pass. A +cumulative archive was applied to a pristine extraction of the supplied source; +all overlay files compared byte-identical, executable modes were preserved, and +the 16-test focused verification passed there as well. diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md new file mode 100644 index 00000000..8ccf6256 --- /dev/null +++ b/docs/cluster-setup.md @@ -0,0 +1,231 @@ +# Cluster setup for infer-stack + +`infer-stack` targets Kubernetes capabilities, not a particular Kubernetes +distribution. K3s is the first provisioning integration because it gives a +small workstation cluster a short path to a conformant Kubernetes API without +making cluster administration the main project. + +The architecture boundary is: + +```text +any Kubernetes distribution + | + | kubectl / Kubernetes API + v +infer-stack kube nodes / setup + | + v +GPU scheduling + discovery, KubeAI, resource profiles +``` + +Distribution-specific lifecycle commands live below that boundary: + +```text +infer-stack kube k3s bootstrap +infer-stack kube k3s join + +# A future integration could add, for example: +# infer-stack kube k0s ... +``` + +Nothing in `infer-stack kube setup`, model placement, the lease controller, or +the KubeAI backend requires K3s. An existing kubeadm, RKE2, k0s, EKS, or other +Kubernetes cluster should start at [Use an existing cluster](#use-an-existing-cluster). + +## Mental model + +A small cluster has one or more Kubernetes **control-plane** nodes and one or +more **worker** nodes. The control plane owns Kubernetes API/state/scheduling; +workers run pods. + +For the first K3s topology, use one workstation as a K3s **server** and join +other workstations as K3s **agents**: + +```text +workstation A + K3s server + Kubernetes control plane + worker too (default K3s behavior) + may run GPU model pods + + | Kubernetes cluster + +---------------- workstation B + | K3s agent / worker / GPU + | + +---------------- workstation C + K3s agent / worker / GPU +``` + +The server is not required to be a dedicated appliance. In the initial +workstation cluster it can keep contributing its GPU. + +This is separate from infer-stack's own authority. Use one infer-stack control +plane for a Kubernetes namespace; do not have independent infer-stack state +directories concurrently acquire/release against the same namespace. The +machine running that authority can initially be the K3s server for simplicity, +but Kubernetes control-plane ownership and infer-stack lease/config ownership +are different concepts. + +## What generic setup expects + +Before `infer-stack kube setup --apply`, the generic integration cares about +observable capabilities rather than how the cluster was installed: + +- `kubectl` selects the intended cluster context and can reach its API. +- Nodes are Ready. +- GPU nodes expose allocatable `nvidia.com/gpu` when NVIDIA scheduling is + desired. +- GPU product/memory discovery labels are available for profile generation. +- The node/container runtime can actually launch NVIDIA workloads. A working + externally managed GPU integration is accepted; infer-stack does not require + a particular installer. +- Helm is available when infer-stack needs to reconcile a Helm-managed + component. +- KubeAI's CRD/service are present, or infer-stack can install/reconcile them. + +The read-only check is always the first command: + +```bash +infer-stack kube nodes +infer-stack kube setup +``` + +Only after reviewing the target context and plan: + +```bash +infer-stack kube setup --apply +``` + +Host NVIDIA driver/container-runtime installation stays outside infer-stack. +The exact host procedure depends on the distribution and operating system. + +## First provisioning target: K3s + +### 1. Prepare the first workstation + +Choose the first workstation's stable LAN/VPN address. Install and verify its +NVIDIA driver/container runtime before starting K3s when this node will serve +GPU workloads. + +Record an exact K3s version for a real cluster so every node joins with the +same version. + +### 2. Create the K3s server + +On workstation A: + +```bash +infer-stack kube k3s bootstrap --version= +``` + +This command is intentionally scoped under `kube k3s`: it installs/starts a +K3s server, waits for its node to become Ready, and makes the K3s kubeconfig +usable without overwriting an existing `~/.kube/config`. + +Then inspect the generic Kubernetes integration: + +```bash +infer-stack kube nodes +infer-stack kube setup +infer-stack kube setup --apply +``` + +The setup commands above are not K3s-specific. + +### 3. Obtain the join information + +On the K3s server, the agent token is stored at: + +```text +/var/lib/rancher/k3s/server/node-token +``` + +Copy that token to a mode-0600 file on the workstation being joined using your +normal secure transport. Also record workstation A's reachable server URL: + +```text +https://:6443 +``` + +Do not put the token directly on a command line or in shell history. + +### 4. Join another workstation + +After preparing the NVIDIA host runtime on workstation B when applicable: + +```bash +infer-stack kube k3s join \ + --server=https://:6443 \ + --token-file=~/.private/k3s-token \ + --node-name= \ + --version= +``` + +Repeat that command on each additional workstation with its own node name. + +K3s networking/firewall requirements are distribution-specific operational +requirements, not infer-stack requirements. In particular, agents must be able +to reach the K3s API on the server, and inter-node traffic required by the +configured K3s networking backend must be allowed on the trusted cluster +network. + +### 5. Reconcile newly visible hardware + +Back on the machine operating infer-stack: + +```bash +infer-stack kube nodes +infer-stack kube setup +infer-stack kube setup --apply +``` + +A newly joined GPU product may produce a new KubeAI resource profile. Existing +operator-owned profiles with the same names remain authoritative. + +### 6. Verify the serving path + +After the cluster inventory is correct: + +```bash +infer-stack config set backend kubeai +infer-stack config set kubeai_gateway cluster +infer-stack doctor +``` + +Then exercise a real acquire/generation/release and the multi-host handoff +checks described in [the KubeAI backend guide](kubeai-backend.md). + +## Use an existing cluster + +If Kubernetes already exists, do not run a provisioning command. Select its +kubeconfig/context and begin at the generic layer: + +```bash +kubectl config current-context +infer-stack kube nodes +infer-stack kube setup +infer-stack kube setup --apply +infer-stack doctor +``` + +The generic setup path must not depend on K3s files, services, tokens, Flannel, +or installer behavior. If a future Kubernetes distribution needs first-class +provisioning convenience, add it under its own `infer-stack kube ` +namespace while preserving this generic contract. + +## Ownership boundary + +`infer-stack kube` deliberately stops short of being a general Kubernetes +administrator: + +- `kube setup` owns only the integration components/capabilities infer-stack + needs. +- Distribution-specific provisioning stays in scoped subcommands such as + `kube k3s`. +- Arbitrary cluster administration remains `kubectl`, Helm, and the chosen + distribution's tooling. +- Existing externally managed GPU/KubeAI capabilities are accepted instead of + replaced merely because they were installed differently. + +This keeps K3s easy to use today without making it part of infer-stack's +backend or scheduling architecture. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 974e5560..4a61c2be 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -37,7 +37,11 @@ Where the two backends match and where they still differ, row by row: ## One-time cluster setup -The normal setup path is now capability-driven and lives in the CLI: +For the control-plane/worker mental model and the full create/join runbook, see +[cluster-setup.md](cluster-setup.md). The generic integration is +distribution-neutral; K3s is only the first scoped provisioning target. + +The normal setup path is capability-driven and lives in the CLI: ```bash # Existing cluster: just point kubectl at it. For a new local K3s server: @@ -155,7 +159,7 @@ needed. `secrets rotate` updates its Secret and rolls it; `doctor` checks it; `kubeai_gateway_url` (an ingress URL clients should use instead). Static routes only: dynamic routing and Open WebUI need the host placement. -## Add a workstation +## Add a workstation with the K3s provisioning integration On the server node, copy the K3s token into a protected file on the new host through whatever secure channel you normally use, and record the server's K3s diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 7a0b365b..e2ed4d2e 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -1298,7 +1298,7 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: if not _run_check( 'cluster reachable', ['version', '--client=false', '-o', 'json'], - 'is the kubeconfig set up? (`infer-stack kube setup`; for new K3s: `infer-stack kube k3s bootstrap`)', + 'select a reachable Kubernetes kubeconfig/context, then run `infer-stack kube setup`', ): return checks if not _run_check( diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index d93aa201..ce8cc368 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -1,7 +1,9 @@ """Kubernetes integration commands. -``kube`` intentionally exposes only infer-stack's integration surface. Use -``kubectl``/``helm`` for arbitrary cluster administration. +The top-level ``kube`` surface is distribution-neutral and exposes only the +capabilities infer-stack needs. Provisioning integrations live in explicitly +scoped subcommands such as ``kube k3s``. Use ``kubectl``/``helm`` for arbitrary +cluster administration. """ from __future__ import annotations @@ -169,7 +171,7 @@ class K3sBootstrapCLI(kw.Config): @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - from ..kube.k3s import bootstrap + from ..kube.k3s import K3S_NODE_TOKEN, bootstrap try: default_kubeconfig_ready = bootstrap(version=config.version) @@ -183,7 +185,9 @@ def main(cls, argv=True, **kwargs): 'current context selects this K3s cluster (or set KUBECONFIG) ' 'before running setup.' ) - print('Next: infer-stack kube setup') + print(f'K3s join token: sudo cat {K3S_NODE_TOKEN}') + print('Join workers with: infer-stack kube k3s join --server=... --token-file=...') + print('Then: infer-stack kube setup') return 0 @@ -218,7 +222,11 @@ def main(cls, argv=True, **kwargs): class K3sModalCLI(kw.ModalCLI): - """Create/join K3s clusters; not required for existing Kubernetes clusters.""" + """First supported provisioning target: create/join K3s clusters. + + This namespace is intentionally distribution-specific. Existing clusters + and future provisioning integrations use the same generic ``kube setup``. + """ __command__ = 'k3s' bootstrap = K3sBootstrapCLI @@ -226,7 +234,7 @@ class K3sModalCLI(kw.ModalCLI): class KubeModalCLI(kw.ModalCLI): - """Inspect/setup the Kubernetes capabilities used by infer-stack.""" + """Inspect/setup infer-stack capabilities on any Kubernetes distribution.""" __command__ = 'kube' nodes = KubeNodesCLI diff --git a/infer_stack/kube/__init__.py b/infer_stack/kube/__init__.py index 53010e2c..f2723572 100644 --- a/infer_stack/kube/__init__.py +++ b/infer_stack/kube/__init__.py @@ -1,8 +1,11 @@ -"""Kubernetes integration helpers used by ``infer-stack kube``. +"""Distribution-neutral Kubernetes integration helpers. -This package deliberately manages only the Kubernetes capabilities infer-stack -needs. It is not a kubectl replacement: arbitrary cluster administration stays -with Kubernetes-native tools. +The generic layer models only capabilities visible through Kubernetes APIs and +Helm. Distribution provisioning is deliberately kept in leaf modules such as +:mod:`infer_stack.kube.k3s`; generic setup must not depend on one distribution. + +This package is not a kubectl replacement: arbitrary cluster administration +stays with Kubernetes-native tools. """ from .manage import KubeManager, SetupPlan diff --git a/infer_stack/kube/k3s.py b/infer_stack/kube/k3s.py index f1ed8a55..8143896d 100644 --- a/infer_stack/kube/k3s.py +++ b/infer_stack/kube/k3s.py @@ -1,4 +1,8 @@ -"""Explicit K3s convenience path for creating/joining infer-stack clusters.""" +"""K3s-specific cluster provisioning convenience. + +This is a leaf integration beneath the distribution-neutral ``infer_stack.kube`` +capability layer. Generic setup/backends must not depend on this module. +""" from __future__ import annotations @@ -37,6 +41,7 @@ def _active(run: RunFunc, service: str) -> bool: K3S_KUBECONFIG = Path('/etc/rancher/k3s/k3s.yaml') +K3S_NODE_TOKEN = Path('/var/lib/rancher/k3s/server/node-token') def _ensure_default_kubeconfig_link() -> bool: diff --git a/infer_stack/kube/manage.py b/infer_stack/kube/manage.py index b7020678..43519892 100644 --- a/infer_stack/kube/manage.py +++ b/infer_stack/kube/manage.py @@ -275,8 +275,8 @@ def plan_setup( if not self.command_exists('kubectl'): plan.checks.append(Check( 'kubectl available', False, - 'install kubectl or use `infer-stack kube k3s bootstrap` ' - 'on a new K3s server', + 'install kubectl and select a Kubernetes context; cluster ' + 'provisioning is distribution-specific', )) return plan plan.checks.append(Check('kubectl available', True)) @@ -338,8 +338,9 @@ def plan_setup( plan.checks.append(Check( 'RuntimeClass nvidia', runtime_ok, '' if runtime_ok else ( - 'Kubernetes has no nvidia RuntimeClass; on K3s install ' - 'nvidia-container-runtime before starting/restarting K3s' + 'Kubernetes has no nvidia RuntimeClass; configure the ' + 'NVIDIA container runtime for this cluster/distribution ' + 'before rerunning setup' ), )) level = 'action' if runtime_ok else 'required' diff --git a/tests/test_cli_kube.py b/tests/test_cli_kube.py index 8f7e54e2..1ad65fc8 100644 --- a/tests/test_cli_kube.py +++ b/tests/test_cli_kube.py @@ -358,3 +358,30 @@ def fake_run(args, **kwargs): args[:3] == ['kubectl', '--kubeconfig', str(k3s.K3S_KUBECONFIG)] for args, _ in calls ) + + +def test_generic_setup_missing_kubectl_hint_is_distribution_neutral(): + manager = KubeManager(run=lambda args, **kwargs: '') + manager.command_exists = lambda name: False + plan = manager.plan_setup() + assert plan.failed + details = '\n'.join(check.detail for check in plan.checks) + assert 'k3s' not in details.lower() + assert 'distribution-specific' in details + + +def test_generic_setup_runtime_hint_is_distribution_neutral(): + fake = FakeCluster( + nodes=[node(product=None, memory=None, count=None)], + runtime_class=False, + crd=True, + namespace=True, + service=True, + releases=[], + ) + manager = manager_for(fake) + plan = manager.plan_setup(gpu='nvidia') + assert plan.failed + details = '\n'.join(check.detail for check in plan.checks) + assert 'k3s' not in details.lower() + assert 'cluster/distribution' in details diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index a48e6351..0cc31acb 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -530,6 +530,7 @@ def no_cluster(args): checks = be.doctor() assert checks[0][1] is False assert 'kubeconfig' in checks[0][2] + assert 'k3s' not in checks[0][2].lower() assert len(checks) == 1 # later checks would only cascade From 81415c6dd98c99af5d931b624900b58732fa8535 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 28 Sep 2026 15:20:03 -0400 Subject: [PATCH 130/141] Add reversible Kubernetes Compose node handoff Co-authored-by: GPT-5.6 Sol --- CHANGELOG.md | 19 +++ README.md | 9 ++ dev/journals/gpt-5.6-sol.md | 16 +++ docs/cluster-setup.md | 150 ++++++++++++++++++++++- docs/kubeai-backend.md | 20 +++ infer_stack/cli/commands_kube.py | 194 ++++++++++++++++++++++++++++- infer_stack/kube/__init__.py | 4 +- infer_stack/kube/manage.py | 203 ++++++++++++++++++++++++++++++- tests/test_cli_kube.py | 185 +++++++++++++++++++++++++++- 9 files changed, 787 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71bb8c5d..20c82ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ We [keep a changelog](https://keepachangelog.com/en/1.0.0/). We aim to adhere to [semantic versioning](https://semver.org/spec/v2.0.0.html). +### Reversible Kubernetes / Compose workstation handoff + +- Added `infer-stack kube node status|detach|attach`. `detach` is a preview by + default; `--yes` marks an infer-stack-owned maintenance cordon, drains + controller-managed workloads, and leaves the Kubernetes agent/control plane + running so the host can temporarily use its GPU through the local Compose + backend. `attach --yes` waits for the same node identity to be Ready and + uncordons it after the operator has stopped local Compose GPU workloads. +- The lifecycle refuses bare/unmanaged pods rather than force-deleting them and + refuses to attach/detach through a cordon owned by unrelated cluster + maintenance. Interrupted detaches are marked so they can be retried or + reversed safely. +- `kube nodes` now exposes schedulability and reports an infer-stack-detached + node as `SCHED=compose`. +- Documented the practical two-workstation test workflow: preserve existing + Compose config/ledger, run KubeAI from a separate data root with + `INFER_STACK_BACKEND=kubeai`, and switch physical nodes between Kubernetes + scheduling and direct Compose use without uninstalling/rejoining Kubernetes. + ### Kubernetes integration setup moves into `infer-stack kube` - Kept the generic Kubernetes manager/backend distribution-neutral: generic diff --git a/README.md b/README.md index 2922bfd6..bb441940 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ infer-stack config init # data dir + default backend -> settings.y infer-stack catalog suggest --apply # seed catalog.yaml from this host's GPUs infer-stack catalog show # what can be acquired infer-stack kube setup # inspect/plan Kubernetes integration (read-only) +infer-stack kube node detach NODE # preview handing a cluster GPU host to Compose infer-stack acquire # lease, render, bring up, wait for a real generation infer-stack access --env-file e.env # reach it, managed or external; leases only what runs here infer-stack test # one generation through the gateway @@ -567,6 +568,14 @@ infer-stack doctor infer-stack acquire --ttl 2h --env-file lease.env --yes ``` +If the workstation already has a Compose ledger/configuration that you plan to +return to, keep that authority intact rather than switching backend kinds in +the same ledger. The cluster setup guide documents the temporary test pattern: +use `INFER_STACK_BACKEND=kubeai` with a separate KubeAI data root, and use +`infer-stack kube node detach/attach` to hand each physical GPU host between +Kubernetes scheduling and direct Compose use without uninstalling or rejoining +the node. See [docs/cluster-setup.md](docs/cluster-setup.md). + ### Kubernetes cluster setup and KubeAI integration See [docs/cluster-setup.md](docs/cluster-setup.md) for the server/worker mental diff --git a/dev/journals/gpt-5.6-sol.md b/dev/journals/gpt-5.6-sol.md index 521ba521..45fcc21e 100644 --- a/dev/journals/gpt-5.6-sol.md +++ b/dev/journals/gpt-5.6-sol.md @@ -191,3 +191,19 @@ remains the only unavailable test dependency in this environment. `compileall`, cumulative archive was applied to a pristine extraction of the supplied source; all overlay files compared byte-identical, executable modes were preserved, and the 16-test focused verification passed there as well. + +## 2026-09-28 14:42:27 -0400 + +Summary of user intent: make two workstations that already have working Compose-backed infer-stack installations practical for real KubeAI testing without destroying their existing local state. The machines need to be pooled into Kubernetes, then temporarily handed back to direct Compose use, and later returned to the same cluster only when the operator explicitly requests it. + +Model: GPT-5.6 Sol. Configuration: tool-enabled reasoning session. + +The main design decision was to avoid treating temporary Compose use as Kubernetes leave/rejoin. A worker does not need its K3s/kubelet agent stopped merely to make its GPU safe for local Compose; doing so would create more lifecycle state, and stopping the sole K3s server would also take down the cluster control plane. Instead, the generic Kubernetes layer now models a reversible scheduling handoff. `infer-stack kube node detach NODE` is read-only by default; `--yes` marks an infer-stack-owned detach, cordons the node, and drains controller-managed workloads while leaving DaemonSet/static pods and the distribution agent/control plane running. `attach --yes` waits for that same node identity to be Ready, uncordons it, and removes the marker. This is distribution-neutral because it uses only Kubernetes APIs, so it does not weaken the earlier K3s boundary. + +The cordon has explicit ownership and crash recovery. Infer-stack refuses to adopt or undo an unrelated operator cordon. Detach writes a `requested` annotation before mutation and changes it to `true` only after a successful drain; an interrupted operation can therefore be retried or reversed instead of becoming an ambiguous foreign cordon. Bare/unmanaged pods are a hard stop rather than a reason to add `kubectl drain --force`. `kube nodes` now exposes schedulability, `compose` for a completed temporary detach, `pending` for an interrupted detach, and `CONFLICT` if someone externally makes a Compose-marked node schedulable. + +A second issue was backend recovery state. The existing leasing design intentionally refuses to change backend kind inside one recovery ledger, so switching a Compose-configured control workstation to `config set backend kubeai` and reusing its data root would make the physical-node workflow look easy while leaving a recovery-snapshot mismatch. I preserved that invariant rather than weakening it. The documented temporary-testing path keeps the persisted Compose default/ledger untouched and runs the one KubeAI authority with `INFER_STACK_BACKEND=kubeai` plus a separate `INFER_STACK_DATA_DIR=$HOME/.local/share/infer_stack-kubeai`. The normal config/catalog root remains shared. Unsetting those variables returns the control workstation to its original Compose authority; ordinary worker nodes never need a local KubeAI authority at all. + +The resulting two-machine loop is: quiesce local Compose on A/B, create/join Kubernetes once, run KubeAI from its separate authority/data root, release KubeAI workloads, detach A/B for Compose, use their old Compose installations, quiesce Compose, attach A/B, and resume the same KubeAI authority. There is no second `kube k3s join` because detach preserves membership and agent state. A K3s server that is also the sole control-plane node can be detached from workload scheduling without stopping the server; cluster workloads may become Pending when all workers are cordoned, but the control plane remains available to attach them again. + +Validation in this extracted/offline checkout: `tests/test_cli_kube.py` passes 18/18 with the same minimal ubelt stub used for the Kubernetes work; the tests cover reversible detach/attach, retained DaemonSet/static pods, refusal to force-delete unmanaged pods, refusal to undo foreign cordons, interrupted-detach retry, control-plane handling, and node inventory state. Four existing KubeAI doctor/sizing tests also pass. A minimal external kwconf/ubelt shape stub imports the full CLI tree and verifies `ManageCLI.kube -> KubeModalCLI.node -> status/detach/attach`. `python -m compileall -q infer_stack tests` passes. The container still lacks the real ubelt/kwconf and ruff packages, so the real parser/help path and repository lint suite should run in the normal development environment before merge. diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 8ccf6256..7398e93e 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -184,7 +184,8 @@ operator-owned profiles with the same names remain authoritative. ### 6. Verify the serving path -After the cluster inventory is correct: +After the cluster inventory is correct, a fresh Kubernetes-only control host +can make KubeAI its durable default: ```bash infer-stack config set backend kubeai @@ -192,9 +193,153 @@ infer-stack config set kubeai_gateway cluster infer-stack doctor ``` +If this machine already has a Compose deployment history that you want to keep +and return to later, do **not** reuse that Compose ledger as the KubeAI ledger. +Use the temporary-testing workflow below instead. + Then exercise a real acquire/generation/release and the multi-host handoff checks described in [the KubeAI backend guide](kubeai-backend.md). +## Temporarily swap Compose-configured workstations into a cluster + +It is common to start with workstations that already use infer-stack's Compose +backend and temporarily pool them for KubeAI testing. There are two independent +pieces of state to preserve: + +1. the workstation's existing Compose configuration/ledger; and +2. the Kubernetes node's cluster identity. + +Neither needs to be destroyed. Use a separate KubeAI data root for the cluster +authority, and temporarily **cordon + drain** a node when its physical GPU is +being handed back to local Compose. The Kubernetes distribution agent keeps +running throughout; `detach` does not uninstall, stop, or remove the node. + +### Keep the existing Compose authority intact + +Before using a Compose-configured machine as a Kubernetes GPU worker, quiesce +its local Compose backend: + +```bash +infer-stack release --all --evict +infer-stack ps +``` + +On the one machine that will operate the KubeAI infer-stack authority, use an +independent data root instead of changing the existing Compose default: + +```bash +export INFER_STACK_BACKEND=kubeai +export INFER_STACK_DATA_DIR="$HOME/.local/share/infer_stack-kubeai" + +# KubeAI-only settings can coexist with a persisted Compose default. +infer-stack config set kubeai_gateway cluster +infer-stack doctor +``` + +`INFER_STACK_BACKEND` overrides the persisted `backend: compose` only for the +current shell. `INFER_STACK_DATA_DIR` gives KubeAI its own ledger and rendered +state, which is important because recovery snapshots intentionally do not +change backend kind inside one ledger. The normal catalog/config root remains +shared, so the same endpoint definitions can be exercised on both backends. + +Workstation B does not need a local KubeAI infer-stack authority merely because +it is a Kubernetes worker. Its existing Compose configuration can remain +untouched while the authority on workstation A schedules KubeAI pods onto it. + +### Hand one node back to Compose + +If you are ending the KubeAI test entirely rather than removing only one +worker, first quiesce the KubeAI authority while its temporary environment is +still active: + +```bash +infer-stack release --all --evict +infer-stack ps +``` + +For a single-node handoff, detach can instead evict that node's workload pods; +their controllers may reschedule them onto other attached workers. + +Run the node lifecycle command from any shell whose `kubectl` context has +permission to drain the target node. The first invocation is a read-only +preview: + +```bash +infer-stack kube node detach +infer-stack kube node detach --yes +``` + +Detach: + +- records that infer-stack owns this temporary cordon; +- cordons the node so Kubernetes cannot schedule new ordinary workloads there; +- drains controller-managed workload pods; +- keeps DaemonSet/static pods and the Kubernetes agent/control plane running; +- refuses unmanaged/bare pods instead of using `kubectl drain --force`. + +`infer-stack kube nodes` reports such a node with `SCHED=compose`. A K3s server +that is also the sole control-plane node may be detached this way: the control +plane remains running because detach does not stop the K3s service. Cluster +workloads can of course become Pending if every worker is cordoned. + +After detach succeeds, use the target workstation's original Compose setup. +If the current shell was the KubeAI authority shell, return to the persisted +Compose default by dropping the temporary overrides: + +```bash +unset INFER_STACK_BACKEND INFER_STACK_DATA_DIR +infer-stack doctor +``` + +### Hand the node back to Kubernetes + +First stop/release every local Compose GPU workload on the target workstation: + +```bash +infer-stack release --all --evict +infer-stack ps +``` + +Then, from a cluster-admin shell, preview and explicitly re-enable scheduling: + +```bash +infer-stack kube node attach +infer-stack kube node attach --yes +``` + +`attach --yes` is the operator's assertion that local Compose no longer owns +the GPU. It waits for the existing node to be Ready, uncordons it, and removes +the infer-stack detach marker. It refuses to uncordon a node that was cordoned +by someone else, so temporary backend switching cannot silently undo unrelated +cluster maintenance. + +To operate KubeAI again on the authority machine: + +```bash +export INFER_STACK_BACKEND=kubeai +export INFER_STACK_DATA_DIR="$HOME/.local/share/infer_stack-kubeai" +infer-stack doctor +``` + +There is no second `kube k3s join`: attach reuses the node identity and agent +configuration that were preserved during detach. + +For two Compose-configured machines A and B, a complete test cycle is therefore: + +```text +Compose A + Compose B + -> quiesce both Compose stacks + -> create/join Kubernetes once + -> use a separate KubeAI data root on A + -> test KubeAI on A+B + -> release KubeAI workloads + -> kube node detach A/B + -> use Compose A/B again + -> quiesce Compose A/B + -> kube node attach A/B + -> resume the same KubeAI authority/data root +``` + ## Use an existing cluster If Kubernetes already exists, do not run a provisioning command. Select its @@ -220,6 +365,9 @@ administrator: - `kube setup` owns only the integration components/capabilities infer-stack needs. +- `kube node detach/attach` owns only a temporary scheduling cordon used to + hand a host's GPU between Kubernetes and local Compose; it does not remove + cluster membership. - Distribution-specific provisioning stays in scoped subcommands such as `kube k3s`. - Arbitrary cluster administration remains `kubectl`, Helm, and the chosen diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 4a61c2be..a7f71b9c 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -53,6 +53,13 @@ infer-stack kube setup --apply infer-stack kube nodes ``` +For temporary testing on workstations that already have Compose state you want +to preserve, do not point KubeAI at the existing Compose ledger. The +[cluster setup guide](cluster-setup.md#temporarily-swap-compose-configured-workstations-into-a-cluster) +uses a separate KubeAI data root and `kube node detach/attach` so the same +physical machines can move back to local Compose use without uninstalling or +rejoining Kubernetes. + `kube setup` checks the Kubernetes API, Ready nodes, GPU scheduling/discovery facts, Helm, the KubeAI CRD/service, and discovered GPU resource profiles. It is capability-based: an existing GPU Operator or externally-managed KubeAI is @@ -81,6 +88,8 @@ chart version by default; use `--kubeai-version=...` for an explicit change. ## Point infer-stack at it +For a Kubernetes-only control host: + ```bash infer-stack config set backend kubeai # optional overrides (defaults shown): @@ -93,6 +102,17 @@ infer-stack config set kubeai_resource_profile nvidia-gpu-rtx-4090 infer-stack config set kubeai_gateway_upstream http://kubeai.example/openai/v1 ``` +On a host whose persisted default remains Compose, use a separate KubeAI +authority/data root instead: + +```bash +export INFER_STACK_BACKEND=kubeai +export INFER_STACK_DATA_DIR="$HOME/.local/share/infer_stack-kubeai" +``` + +Unset those variables to return to that host's existing Compose authority after +the Kubernetes node has been detached as described in the cluster setup guide. + Catalog endpoints opt into a specific profile per endpoint; the GPU count is appended automatically from `tensor_parallel_size × pipeline_parallel_size × data_parallel_size`: diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index ce8cc368..2eb4d0b7 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -54,6 +54,30 @@ def _print_plan(plan) -> None: print('\nplan: no managed changes needed') +def _print_node_lifecycle(plan) -> None: + state = 'schedulable' if plan.schedulable else 'cordoned' + if plan.detached_for_compose: + if plan.schedulable: + state += ' (CONFLICT: Compose detach marker is active)' + else: + state += ' (detached for Compose)' + elif plan.detach_state == 'requested': + state += ' (detach incomplete; retry detach or attach)' + role = 'control-plane + worker' if plan.control_plane else 'worker' + print(f'Node: {plan.name}') + print(f'Role: {role}') + print(f'Ready: {"yes" if plan.ready else "NO"}') + print(f'Scheduling: {state}') + print(f'Workload pods drain would evict: {len(plan.workload_pods)}') + for name in plan.workload_pods: + print(f' - {name}') + print(f'DaemonSet/static pods retained: {len(plan.retained_pods)}') + if plan.unmanaged_pods: + print('Unmanaged pods block safe detach:') + for name in plan.unmanaged_pods: + print(f' - {name}') + + class KubeNodesCLI(_PathOverridesMixin): """Show the cluster nodes and the GPU facts infer-stack uses.""" @@ -75,20 +99,185 @@ def main(cls, argv=True, **kwargs): if not rows: print('(cluster has no nodes)') return 1 - headers = ('NAME', 'READY', 'GPUS', 'GPU PRODUCT', 'GPU MEMORY') - print(f'{headers[0]:<28} {headers[1]:<7} {headers[2]:>4} {headers[3]:<36} {headers[4]}') + headers = ('NAME', 'READY', 'SCHED', 'GPUS', 'GPU PRODUCT', 'GPU MEMORY') + print( + f'{headers[0]:<28} {headers[1]:<7} {headers[2]:<8} ' + f'{headers[3]:>4} {headers[4]:<36} {headers[5]}' + ) for row in rows: memory = row['gpu_memory_gib'] memory_text = f'{memory:g} GiB' if memory is not None else '-' + if row['detached_for_compose'] and row['schedulable']: + sched = 'CONFLICT' + elif row['detached_for_compose']: + sched = 'compose' + elif row.get('detach_state') == 'requested': + sched = 'pending' + else: + sched = 'yes' if row['schedulable'] else 'NO' print( f"{row['name']:<28} " f"{('yes' if row['ready'] else 'NO'):<7} " + f'{sched:<8} ' f"{row['gpu_count']:>4} " f"{str(row['gpu_product'] or '-'):<36} {memory_text}" ) return 0 +class KubeNodeStatusCLI(_PathOverridesMixin): + """Inspect one node's temporary Compose/Kubernetes scheduling state.""" + + __command__ = 'status' + name = kw.Value(None, position=1, type=str, required=True, help='Kubernetes node name.') + json = kw.Value(False, isflag=True, help='Emit machine-readable JSON.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = KubeManager() + try: + plan = manager.node_lifecycle_plan(config.name) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + if config.json: + payload = { + **plan.__dict__, + 'can_detach': plan.can_detach, + 'detach_owned': plan.detach_owned, + 'detached_for_compose': plan.detached_for_compose, + } + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + _print_node_lifecycle(plan) + return 0 + + +class KubeNodeDetachCLI(_PathOverridesMixin): + """Temporarily drain/cordon a node so its host can use Compose directly. + + This does not stop kubelet/K3s or remove the node from the cluster. The + default is a read-only preview; ``--yes`` performs the cordon + drain and + marks the cordon as infer-stack-owned so ``node attach`` can safely undo it. + """ + + __command__ = 'detach' + name = kw.Value(None, position=1, type=str, required=True, help='Kubernetes node name.') + yes = kw.Value(False, isflag=True, help='Apply the detach after reviewing the preview.') + timeout = kw.Value(300, type=int, help='kubectl drain timeout in seconds.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = KubeManager() + try: + plan = manager.node_lifecycle_plan(config.name) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + _print_node_lifecycle(plan) + if ( + plan.detached_for_compose + and not plan.schedulable + and not plan.workload_pods + ): + print('\nNode is already detached for temporary Compose use.') + return 0 + if not plan.schedulable and not plan.detach_owned: + raise SystemExit( + 'detach blocked: node is already cordoned by another operator; ' + 'infer-stack will not adopt that maintenance cordon' + ) + if not plan.can_detach: + raise SystemExit( + 'detach blocked: unmanaged pods are present; move/remove them ' + 'explicitly rather than force-deleting them' + ) + if plan.control_plane: + print( + '\nNote: this is a control-plane node. infer-stack will keep the ' + 'Kubernetes control plane running; only scheduled workloads are drained.' + ) + print( + '\nDetach keeps cluster membership and the Kubernetes agent running. ' + 'After the drain, this host may run the Compose backend directly.' + ) + if not config.yes: + print(f'Re-run with `infer-stack kube node detach {config.name} --yes` to apply.') + return 0 + try: + fresh = manager.detach_node_for_compose( + config.name, timeout_seconds=config.timeout, + ) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + print('\nDetached successfully:') + _print_node_lifecycle(fresh) + return 0 + + +class KubeNodeAttachCLI(_PathOverridesMixin): + """Return a Compose-detached node to Kubernetes scheduling. + + The operator must first stop/release local Compose GPU workloads. The + default is read-only; ``--yes`` is the explicit assertion that local GPU + ownership has been handed back to Kubernetes. + """ + + __command__ = 'attach' + name = kw.Value(None, position=1, type=str, required=True, help='Kubernetes node name.') + yes = kw.Value( + False, isflag=True, + help='Confirm local Compose GPU workloads are stopped and uncordon the node.', + ) + timeout = kw.Value(180, type=int, help='Ready wait timeout in seconds.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = KubeManager() + try: + plan = manager.node_lifecycle_plan(config.name) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + _print_node_lifecycle(plan) + if plan.schedulable and not plan.detach_owned: + print('\nNode is already attached to Kubernetes scheduling.') + return 0 + if not plan.detach_owned: + raise SystemExit( + 'attach blocked: this cordon is not marked as an infer-stack ' + 'temporary Compose detach; refusing to undo operator maintenance' + ) + print( + '\nBefore attaching, stop/release every local Compose GPU workload ' + f'on {config.name}. Kubernetes may schedule onto it immediately after uncordon.' + ) + if not config.yes: + print(f'Re-run with `infer-stack kube node attach {config.name} --yes` to apply.') + return 0 + try: + fresh = manager.attach_node_from_compose( + config.name, timeout_seconds=config.timeout, + ) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + print('\nAttached successfully:') + _print_node_lifecycle(fresh) + return 0 + + +class KubeNodeModalCLI(kw.ModalCLI): + """Temporarily hand a cluster node between Kubernetes and local Compose.""" + + __command__ = 'node' + status = KubeNodeStatusCLI + detach = KubeNodeDetachCLI + attach = KubeNodeAttachCLI + + class KubeSetupCLI(_PathOverridesMixin): """Inspect or reconcile the Kubernetes capabilities infer-stack needs. @@ -238,5 +427,6 @@ class KubeModalCLI(kw.ModalCLI): __command__ = 'kube' nodes = KubeNodesCLI + node = KubeNodeModalCLI setup = KubeSetupCLI k3s = K3sModalCLI diff --git a/infer_stack/kube/__init__.py b/infer_stack/kube/__init__.py index f2723572..d41228a9 100644 --- a/infer_stack/kube/__init__.py +++ b/infer_stack/kube/__init__.py @@ -8,6 +8,6 @@ stays with Kubernetes-native tools. """ -from .manage import KubeManager, SetupPlan +from .manage import KubeManager, NodeLifecyclePlan, SetupPlan -__all__ = ['KubeManager', 'SetupPlan'] +__all__ = ['KubeManager', 'NodeLifecyclePlan', 'SetupPlan'] diff --git a/infer_stack/kube/manage.py b/infer_stack/kube/manage.py index 43519892..be42b73d 100644 --- a/infer_stack/kube/manage.py +++ b/infer_stack/kube/manage.py @@ -31,6 +31,7 @@ NVIDIA_DEVICE_PLUGIN_VERSION = '0.17.1' NVIDIA_DEVICE_PLUGIN_REPO = 'https://nvidia.github.io/k8s-device-plugin' KUBEAI_REPO = 'https://www.kubeai.org' +COMPOSE_DETACH_ANNOTATION = 'infer-stack.io/compose-detached' RunFunc = Callable[..., str] @@ -84,6 +85,35 @@ def failed(self) -> bool: return any(not c.ok and c.level == 'required' for c in self.checks) +@dataclass +class NodeLifecyclePlan: + """Read-only view of a node before a temporary Compose/Kube swap.""" + + name: str + ready: bool + schedulable: bool + detach_state: str | None + control_plane: bool + workload_pods: list[str] = field(default_factory=list) + retained_pods: list[str] = field(default_factory=list) + unmanaged_pods: list[str] = field(default_factory=list) + + @property + def can_detach(self) -> bool: + """Whether ``kubectl drain`` can proceed without ``--force``.""" + return not self.unmanaged_pods + + @property + def detach_owned(self) -> bool: + """Whether infer-stack owns this cordon/detach transition.""" + return self.detach_state in {'requested', 'true'} + + @property + def detached_for_compose(self) -> bool: + """Whether the drain completed and the host is ready for Compose.""" + return self.detach_state == 'true' + + class KubeManager: """Inspect/reconcile the small slice of Kubernetes infer-stack depends on.""" @@ -100,6 +130,164 @@ def nodes(self) -> list[dict[str, Any]]: result = self.kubectl_json(['get', 'nodes', '-o', 'json']) return list(result.get('items') or []) + def node(self, name: str) -> dict[str, Any]: + try: + return self.kubectl_json(['get', 'node', name, '-o', 'json']) + except Exception as ex: + raise RuntimeError( + f'could not inspect Kubernetes node {name!r}: {ex}' + ) from ex + + def pods_on_node(self, name: str) -> list[dict[str, Any]]: + result = self.kubectl_json([ + 'get', 'pods', '-A', '--field-selector', f'spec.nodeName={name}', + '-o', 'json', + ]) + return list(result.get('items') or []) + + @staticmethod + def _node_ready(node: dict[str, Any]) -> bool: + conditions = (node.get('status') or {}).get('conditions') or [] + return any( + c.get('type') == 'Ready' and c.get('status') == 'True' + for c in conditions + ) + + @staticmethod + def _node_control_plane(node: dict[str, Any]) -> bool: + labels = (node.get('metadata') or {}).get('labels') or {} + return any( + key in labels + for key in ( + 'node-role.kubernetes.io/control-plane', + 'node-role.kubernetes.io/master', + ) + ) + + @staticmethod + def _pod_name(pod: dict[str, Any]) -> str: + meta = pod.get('metadata') or {} + namespace = str(meta.get('namespace') or 'default') + name = str(meta.get('name') or '?') + return f'{namespace}/{name}' + + @staticmethod + def _pod_retained_by_drain(pod: dict[str, Any]) -> bool: + """Mirror/static and DaemonSet pods are intentionally left by drain.""" + meta = pod.get('metadata') or {} + annotations = meta.get('annotations') or {} + if 'kubernetes.io/config.mirror' in annotations: + return True + owners = meta.get('ownerReferences') or [] + return any(owner.get('kind') == 'DaemonSet' for owner in owners) + + @staticmethod + def _pod_unmanaged(pod: dict[str, Any]) -> bool: + """Bare pods make ``kubectl drain`` refuse unless ``--force`` is used.""" + meta = pod.get('metadata') or {} + annotations = meta.get('annotations') or {} + if 'kubernetes.io/config.mirror' in annotations: + return False + return not bool(meta.get('ownerReferences')) + + def node_lifecycle_plan(self, name: str) -> NodeLifecyclePlan: + """Inspect a node for reversible Kubernetes <-> Compose use. + + ``detach`` intentionally means *scheduling detach*, not cluster leave: + the kubelet / distribution agent keeps running and the node identity is + preserved. This is what makes a later attach a cheap uncordon rather + than a reinstall/rejoin operation. + """ + node = self.node(name) + meta = node.get('metadata') or {} + spec = node.get('spec') or {} + annotations = meta.get('annotations') or {} + pods = self.pods_on_node(name) + retained = [p for p in pods if self._pod_retained_by_drain(p)] + workload = [p for p in pods if not self._pod_retained_by_drain(p)] + unmanaged = [p for p in workload if self._pod_unmanaged(p)] + return NodeLifecyclePlan( + name=name, + ready=self._node_ready(node), + schedulable=not bool(spec.get('unschedulable')), + detach_state=annotations.get(COMPOSE_DETACH_ANNOTATION), + control_plane=self._node_control_plane(node), + workload_pods=sorted(self._pod_name(p) for p in workload), + retained_pods=sorted(self._pod_name(p) for p in retained), + unmanaged_pods=sorted(self._pod_name(p) for p in unmanaged), + ) + + def detach_node_for_compose( + self, name: str, *, timeout_seconds: int = 300, + ) -> NodeLifecyclePlan: + """Cordon/drain a node while preserving its cluster membership. + + We deliberately do not stop kubelet/K3s/etc. Keeping the distribution + agent alive retains identity, networking and control-plane services; + the cordon is the ownership boundary that prevents new scheduled GPU + work while Compose uses the host directly. + """ + plan = self.node_lifecycle_plan(name) + if ( + plan.detached_for_compose + and not plan.schedulable + and not plan.workload_pods + ): + return plan + if not plan.schedulable and not plan.detach_owned: + raise RuntimeError( + f'node {name!r} is already cordoned by another operator. ' + 'Refusing to adopt an unrelated maintenance cordon as a ' + 'temporary Compose detach.' + ) + if not plan.can_detach: + names = ', '.join(plan.unmanaged_pods) + raise RuntimeError( + f'node {name!r} has unmanaged pod(s) that kubectl drain would ' + f'refuse without --force: {names}. Move/remove them explicitly; ' + 'infer-stack will not force-delete unrelated bare pods.' + ) + self.run([ + 'kubectl', 'annotate', 'node', name, + f'{COMPOSE_DETACH_ANNOTATION}=requested', '--overwrite', + ]) + self.run(['kubectl', 'cordon', name]) + self.run([ + 'kubectl', 'drain', name, + '--ignore-daemonsets', + '--delete-emptydir-data', + f'--timeout={int(timeout_seconds)}s', + ]) + self.run([ + 'kubectl', 'annotate', 'node', name, + f'{COMPOSE_DETACH_ANNOTATION}=true', '--overwrite', + ]) + return self.node_lifecycle_plan(name) + + def attach_node_from_compose( + self, name: str, *, timeout_seconds: int = 180, + ) -> NodeLifecyclePlan: + """Make a previously detached node schedulable again.""" + plan = self.node_lifecycle_plan(name) + if plan.schedulable and not plan.detach_owned: + return plan + if not plan.detach_owned: + raise RuntimeError( + f'node {name!r} is cordoned, but infer-stack did not mark it as ' + 'temporarily detached for Compose. Refusing to undo an ' + 'operator-owned maintenance cordon.' + ) + self.run([ + 'kubectl', 'wait', '--for=condition=Ready', f'node/{name}', + f'--timeout={int(timeout_seconds)}s', + ]) + self.run(['kubectl', 'uncordon', name]) + self.run([ + 'kubectl', 'annotate', 'node', name, + f'{COMPOSE_DETACH_ANNOTATION}-', + ]) + return self.node_lifecycle_plan(name) + @staticmethod def node_rows(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: """Compact node inventory for people and JSON output.""" @@ -107,17 +295,20 @@ def node_rows(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: rows = [] for node in nodes: meta = node.get('metadata') or {} - status = node.get('status') or {} - conditions = status.get('conditions') or [] - ready = any( - c.get('type') == 'Ready' and c.get('status') == 'True' - for c in conditions - ) + ready = KubeManager._node_ready(node) name = str(meta.get('name') or '') facts = gpu.get(name, {}) + spec = node.get('spec') or {} + annotations = meta.get('annotations') or {} rows.append({ 'name': name, 'ready': ready, + 'schedulable': not bool(spec.get('unschedulable')), + 'detached_for_compose': ( + annotations.get(COMPOSE_DETACH_ANNOTATION) == 'true' + ), + 'detach_state': annotations.get(COMPOSE_DETACH_ANNOTATION), + 'control_plane': KubeManager._node_control_plane(node), 'gpu_count': facts.get('count', 0), 'gpu_product': facts.get('product'), 'gpu_memory_gib': facts.get('memory_gib'), diff --git a/tests/test_cli_kube.py b/tests/test_cli_kube.py index 1ad65fc8..b96e79bb 100644 --- a/tests/test_cli_kube.py +++ b/tests/test_cli_kube.py @@ -13,17 +13,26 @@ GPU_RESOURCE = 'nvidia.com/gpu' -def node(name='gpu-a', *, product='RTX-TEST', memory='24576', count='1'): +def node( + name='gpu-a', *, product='RTX-TEST', memory='24576', count='1', + schedulable=True, detached=False, control_plane=False, +): labels = {} if product is not None: labels[GPU_PRODUCT] = product if memory is not None: labels[GPU_MEMORY] = memory + if control_plane: + labels['node-role.kubernetes.io/control-plane'] = 'true' allocatable = {} if count is not None: allocatable[GPU_RESOURCE] = count + annotations = {} + if detached: + annotations['infer-stack.io/compose-detached'] = 'true' return { - 'metadata': {'name': name, 'labels': labels}, + 'metadata': {'name': name, 'labels': labels, 'annotations': annotations}, + 'spec': {'unschedulable': not schedulable}, 'status': { 'conditions': [{'type': 'Ready', 'status': 'True'}], 'allocatable': allocatable, @@ -31,6 +40,15 @@ def node(name='gpu-a', *, product='RTX-TEST', memory='24576', count='1'): } +def pod(name, *, namespace='default', owner='ReplicaSet', mirror=False): + meta = {'name': name, 'namespace': namespace, 'annotations': {}} + if owner is not None: + meta['ownerReferences'] = [{'kind': owner, 'name': f'{name}-owner'}] + if mirror: + meta['annotations']['kubernetes.io/config.mirror'] = 'mirror-hash' + return {'metadata': meta} + + class FakeCluster: def __init__( self, @@ -42,8 +60,10 @@ def __init__( service=True, releases=None, values=None, + pods=None, ): self.nodes = nodes if nodes is not None else [node()] + self.pods = pods if pods is not None else {} self.runtime_class = runtime_class self.crd = crd self.namespace = namespace @@ -61,6 +81,57 @@ def __call__(self, args, **kwargs): return '{}' if args == ['kubectl', 'get', 'nodes', '-o', 'json']: return json.dumps({'items': self.nodes}) + if len(args) == 6 and args[:3] == ['kubectl', 'get', 'node'] and args[4:] == ['-o', 'json']: + wanted = args[3] + for item in self.nodes: + if item.get('metadata', {}).get('name') == wanted: + return json.dumps(item) + raise RuntimeError('not found') + if ( + len(args) == 8 + and args[:4] == ['kubectl', 'get', 'pods', '-A'] + and args[4] == '--field-selector' + and args[6:] == ['-o', 'json'] + ): + wanted = args[5].split('=', 1)[1] + return json.dumps({'items': self.pods.get(wanted, [])}) + if args[:3] == ['kubectl', 'annotate', 'node']: + wanted = args[3] + annotation = args[4] + for item in self.nodes: + if item.get('metadata', {}).get('name') != wanted: + continue + annotations = item.setdefault('metadata', {}).setdefault('annotations', {}) + if annotation.endswith('-'): + annotations.pop(annotation[:-1], None) + else: + key, value = annotation.split('=', 1) + annotations[key] = value + return f'node/{wanted} annotated\n' + raise RuntimeError('not found') + if args[:2] == ['kubectl', 'cordon']: + wanted = args[2] + for item in self.nodes: + if item.get('metadata', {}).get('name') == wanted: + item.setdefault('spec', {})['unschedulable'] = True + return f'node/{wanted} cordoned\n' + raise RuntimeError('not found') + if args[:2] == ['kubectl', 'drain']: + wanted = args[2] + self.pods[wanted] = [ + item for item in self.pods.get(wanted, []) + if KubeManager._pod_retained_by_drain(item) + ] + return f'node/{wanted} drained\n' + if args[:3] == ['kubectl', 'wait', '--for=condition=Ready']: + return 'condition met\n' + if args[:2] == ['kubectl', 'uncordon']: + wanted = args[2] + for item in self.nodes: + if item.get('metadata', {}).get('name') == wanted: + item.setdefault('spec', {})['unschedulable'] = False + return f'node/{wanted} uncordoned\n' + raise RuntimeError('not found') if args[:4] == ['kubectl', 'get', 'runtimeclass', 'nvidia']: if not self.runtime_class: raise RuntimeError('not found') @@ -385,3 +456,113 @@ def test_generic_setup_runtime_hint_is_distribution_neutral(): details = '\n'.join(check.detail for check in plan.checks) assert 'k3s' not in details.lower() assert 'cluster/distribution' in details + + +def test_node_detach_is_reversible_and_keeps_daemonsets(): + fake = FakeCluster( + nodes=[node('gpu-a', control_plane=True)], + pods={ + 'gpu-a': [ + pod('model', namespace='kubeai'), + pod('nvidia-plugin', namespace='nvidia-device-plugin', owner='DaemonSet'), + pod('kube-apiserver', namespace='kube-system', mirror=True), + ], + }, + ) + manager = manager_for(fake) + before = manager.node_lifecycle_plan('gpu-a') + assert before.schedulable + assert before.control_plane + assert before.workload_pods == ['kubeai/model'] + assert sorted(before.retained_pods) == [ + 'kube-system/kube-apiserver', + 'nvidia-device-plugin/nvidia-plugin', + ] + + detached = manager.detach_node_for_compose('gpu-a', timeout_seconds=42) + assert detached.detached_for_compose + assert not detached.schedulable + assert detached.workload_pods == [] + assert any( + args == [ + 'kubectl', 'drain', 'gpu-a', '--ignore-daemonsets', + '--delete-emptydir-data', '--timeout=42s', + ] + for args, _ in fake.calls + ) + assert not any( + args[:2] == ['systemctl', 'stop'] or args[:2] == ['sudo', 'systemctl'] + for args, _ in fake.calls + ) + + attached = manager.attach_node_from_compose('gpu-a', timeout_seconds=19) + assert attached.schedulable + assert not attached.detached_for_compose + assert any(args == ['kubectl', 'uncordon', 'gpu-a'] for args, _ in fake.calls) + + +def test_node_detach_refuses_unmanaged_pods_instead_of_force_deleting(): + fake = FakeCluster( + nodes=[node('gpu-a')], + pods={'gpu-a': [pod('manual-debug', owner=None)]}, + ) + manager = manager_for(fake) + plan = manager.node_lifecycle_plan('gpu-a') + assert plan.unmanaged_pods == ['default/manual-debug'] + assert not plan.can_detach + try: + manager.detach_node_for_compose('gpu-a') + except RuntimeError as ex: + assert 'without --force' in str(ex) + else: + raise AssertionError('expected unmanaged pod safety refusal') + assert not any(args[:2] == ['kubectl', 'drain'] for args, _ in fake.calls) + + +def test_node_attach_refuses_to_undo_foreign_cordon(): + fake = FakeCluster(nodes=[node('gpu-a', schedulable=False, detached=False)]) + manager = manager_for(fake) + try: + manager.attach_node_from_compose('gpu-a') + except RuntimeError as ex: + assert 'operator-owned maintenance cordon' in str(ex) + else: + raise AssertionError('expected foreign cordon safety refusal') + assert not any(args[:2] == ['kubectl', 'uncordon'] for args, _ in fake.calls) + + +def test_interrupted_detach_marker_can_be_retried(): + interrupted = node('gpu-a', schedulable=False) + interrupted['metadata']['annotations']['infer-stack.io/compose-detached'] = 'requested' + fake = FakeCluster( + nodes=[interrupted], + pods={'gpu-a': [pod('model', namespace='kubeai')]}, + ) + manager = manager_for(fake) + before = manager.node_lifecycle_plan('gpu-a') + assert before.detach_owned + assert not before.detached_for_compose + assert before.workload_pods == ['kubeai/model'] + + after = manager.detach_node_for_compose('gpu-a') + assert after.detached_for_compose + assert after.workload_pods == [] + + +def test_node_rows_show_compose_detach_state(): + rows = KubeManager.node_rows([ + node('a'), + node('b', schedulable=False, detached=True), + ]) + assert rows[0]['schedulable'] is True + assert rows[0]['detached_for_compose'] is False + assert rows[1]['schedulable'] is False + assert rows[1]['detached_for_compose'] is True + + +def test_detach_marker_on_schedulable_node_is_detectable_conflict(): + row = KubeManager.node_rows([ + node('gpu-a', schedulable=True, detached=True), + ])[0] + assert row['schedulable'] is True + assert row['detached_for_compose'] is True From 882758c0e6d488762f100f5e1767ac1560e008ce Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 19:48:33 -0400 Subject: [PATCH 131/141] Add Kubernetes readiness setup and safe backend recovery transitions Co-authored-by: GPT-6.1-Sol --- Makefile | 8 +- README.md | 190 +++++------- dev/journals/codex.md | 60 ++++ dev/kubeai_e2e.sh | 13 +- docs/backend-parity.md | 2 +- docs/cluster-setup.md | 50 ++-- docs/kubeai-backend.md | 131 ++++++--- docs/planning/known-limitations.md | 7 +- infer_stack/backends/kubeai.py | 68 +++-- infer_stack/cli/__init__.py | 2 + infer_stack/cli/commands_catalog.py | 4 +- infer_stack/cli/commands_kube.py | 213 +++++++++++++- infer_stack/cli/commands_leasing.py | 5 + infer_stack/cli/commands_ledger.py | 80 ++++++ infer_stack/cli/commands_runtime.py | 32 ++- infer_stack/gpu_doctor.py | 8 +- infer_stack/kube/inspect.py | 191 ++++++++++++ infer_stack/kube/k3s.py | 20 +- infer_stack/kube/manage.py | 81 ++---- infer_stack/kube/operations.py | 108 +++++++ infer_stack/kubeai_ops.py | 68 +++-- infer_stack/leasing/compose.py | 4 +- infer_stack/leasing/store.py | 55 ++++ infer_stack/leasing/transition.py | 76 +++++ infer_stack/profile_runtime.py | 8 +- scripts/bootstrap_k3s.sh | 4 +- scripts/install_kubeai.sh | 37 +-- tests/test_backend_transition.py | 201 +++++++++++++ tests/test_cli_kube.py | 4 +- tests/test_gpu_doctor.py | 11 +- tests/test_kube_readiness.py | 431 ++++++++++++++++++++++++++++ tests/test_leasing_kubeai.py | 37 ++- 32 files changed, 1849 insertions(+), 360 deletions(-) create mode 100644 infer_stack/cli/commands_ledger.py create mode 100644 infer_stack/kube/inspect.py create mode 100644 infer_stack/kube/operations.py create mode 100644 infer_stack/leasing/transition.py create mode 100644 tests/test_backend_transition.py create mode 100644 tests/test_kube_readiness.py diff --git a/Makefile b/Makefile index 4a94f8ec..55eefcb8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python -# The KubeAI chart values: your resourceProfiles (see docs/kubeai-backend.md). -VALUES ?= kubeai-values.yaml +# Optional extra Helm values; the common case infers GPU profiles. +VALUES ?= status: $(PYTHON) manage.py status @@ -9,10 +9,10 @@ render: $(PYTHON) manage.py render bootstrap-k3s: - bash scripts/bootstrap_k3s.sh + $(PYTHON) manage.py kube bootstrap --provider=k3s --apply install-kubeai: - bash scripts/install_kubeai.sh $(VALUES) kubeai + $(PYTHON) manage.py kube install --apply $(if $(VALUES),--values "$(VALUES)",) port-forward-kubeai: kubectl -n kubeai port-forward svc/kubeai 8000:80 diff --git a/README.md b/README.md index bb441940..f9505297 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,8 @@ Two backends run this: **Compose** (single host; vLLM and Ollama) and infer-stack config init # data dir + default backend -> settings.yaml infer-stack catalog suggest --apply # seed catalog.yaml from this host's GPUs infer-stack catalog show # what can be acquired -infer-stack kube setup # inspect/plan Kubernetes integration (read-only) +infer-stack kube inventory # Kubernetes/GPU/KubeAI facts, before or after install +infer-stack kube doctor # detailed readiness + actionable fixes infer-stack kube node detach NODE # preview handing a cluster GPU host to Compose infer-stack acquire # lease, render, bring up, wait for a real generation infer-stack access --env-file e.env # reach it, managed or external; leases only what runs here @@ -554,17 +555,20 @@ the managed key and the endpoint alias on either backend. The short version: ```bash -# New cluster convenience path (omit --version to use the K3s install channel): -infer-stack kube k3s bootstrap --version= - -# Read-only first: inspect the current cluster and the changes infer-stack owns. -infer-stack kube setup -infer-stack kube setup --apply -infer-stack kube nodes - +infer-stack kube inventory +infer-stack kube doctor +# New local cluster / NVIDIA plugin + GFD prerequisites (review before applying): +infer-stack kube bootstrap --provider=k3s +sudo -v +infer-stack kube bootstrap --provider=k3s --apply +# Automatic profiles from Kubernetes node labels; no temporary values file needed: +infer-stack kube install +infer-stack kube install --apply +infer-stack kube doctor +infer-stack catalog suggest --backend kubeai infer-stack config set backend kubeai infer-stack config set kubeai_gateway cluster -infer-stack doctor +infer-stack doctor --backend kubeai infer-stack acquire --ttl 2h --env-file lease.env --yes ``` @@ -576,119 +580,73 @@ use `INFER_STACK_BACKEND=kubeai` with a separate KubeAI data root, and use Kubernetes scheduling and direct Compose use without uninstalling or rejoining the node. See [docs/cluster-setup.md](docs/cluster-setup.md). -### Kubernetes cluster setup and KubeAI integration - -See [docs/cluster-setup.md](docs/cluster-setup.md) for the server/worker mental -model and the complete first-cluster runbook. `infer-stack kube` owns the -integration knowledge, not Kubernetes itself. Its generic commands are -distribution-neutral: an existing kubeadm/RKE2/k0s/EKS/etc. cluster is -first-class once `kubectl` selects it. K3s is the first explicitly supported -**provisioning** integration and therefore lives under `infer-stack kube k3s`, -not in the generic manager/backend. - -`kube setup` is a plan/apply command. The default is read-only and checks the -cluster, Ready nodes, NVIDIA scheduling/discovery capabilities, Helm, KubeAI, -and the resource profiles infer-stack can derive from GPU labels. It accepts -working externally-managed GPU/KubeAI integrations instead of replacing them. - -```bash -infer-stack kube setup -# [ok] / [WARN] / [NEED] / [FAIL] checklist + proposed managed changes - -infer-stack kube setup --apply -``` - -The apply side is deliberately narrow: it can reconcile the NVIDIA device -plugin + GPU Feature Discovery when an `nvidia` RuntimeClass shows that the -node runtime is already configured, install/reconcile KubeAI, and add missing -discovered GPU `resourceProfiles`. It does **not** install NVIDIA drivers or -the NVIDIA container runtime. On K3s, install those before the first start (or -restart K3s afterward), then rerun `kube setup`. - -For a fresh K3s server, `kube k3s bootstrap` installs/starts K3s, keeps K3s's -own kubeconfig as the authority (readable by the local user), waits for Ready, -and installs Helm if needed. On a clean account it symlinks `~/.kube/config` -to that authoritative file; an existing kubeconfig is never overwritten, and the -CLI tells you to select/merge the K3s context before continuing: - -```bash -infer-stack kube k3s bootstrap --version= -infer-stack kube setup -infer-stack kube setup --apply -``` - -An exact K3s version is recommended once the cluster is real infrastructure. -If K3s is already active and the requested version differs, infer-stack refuses -to perform an implicit cluster upgrade. - -`kube setup` generates its reconciled KubeAI values under the infer-stack data -root (`generated/kube/kubeai-values.yaml`). Existing named resource profiles -win over generated profiles, so hand-tuned scheduling rules are not silently -rewritten. An optional operator values file participates in the same merge: +### Switching an existing recovery ledger to KubeAI -```bash -infer-stack kube setup --values ./kubeai-values.local.yaml -infer-stack kube setup --apply --values ./kubeai-values.local.yaml -``` - -On an existing KubeAI Helm release, `--apply` preserves the installed chart -version unless `--kubeai-version=...` explicitly requests another one. On a -fresh install, use that option when you want a fixed chart version. `HF_TOKEN`, -when set, is passed through a short-lived mode-0600 values file rather than on -the Helm command line. - -`infer-stack kube nodes` is the compact inventory used by the setup logic: - -```bash -infer-stack kube nodes -infer-stack kube nodes --json -``` - -The important GPU facts are allocatable `nvidia.com/gpu` plus GPU Feature -Discovery's `nvidia.com/gpu.product` and `nvidia.com/gpu.memory` labels. These -are also the authority used to derive one KubeAI resource profile per GPU -product. If a managed cluster already supplies those capabilities but has no -`nvidia` RuntimeClass, infer-stack accepts it and omits `runtimeClassName` from -its generated profiles. - -For CPU-only development, skip NVIDIA checks explicitly: +Stopping the Compose realization retains its recovery backend and historical +ledger. Backend kinds require separate recovery epochs. After releasing every +old lease and stopping the old runtime: ```bash -infer-stack kube setup --gpu=none +infer-stack release --all --backend compose +infer-stack stack down --backend compose +infer-stack config set backend kubeai +infer-stack status # configured kubeai / active recovery compose +infer-stack ledger rotate # verify quiescence and preview the archive +infer-stack ledger rotate --yes # archive old history; initialize KubeAI epoch +infer-stack ledger archives # archive paths + history inspection commands +infer-stack acquire --yes ``` -The older `scripts/bootstrap_k3s.sh` and `scripts/join_agent.sh` entry points -remain compatibility wrappers around the CLI. `scripts/install_kubeai.sh` is -retained as a low-level/manual Helm escape hatch; normal setup should use -`infer-stack kube setup [--apply]` so checks, profile discovery, and install -behavior share one authority. - -### Debugging checks - -`infer-stack acquire` reports pod-level failures itself (`ImagePullBackOff`, -`Unschedulable`, a crash with the engine's error quoted). `infer-stack ps` lists -the pods and `infer-stack logs -f ` follows one, as on compose. For -anything else, with `NS` the namespace and `MODEL` the Model's name -(`kubectl -n $NS get models`): +Rotation refuses active leases, remaining old runtime objects, and unknown +runtime state. It does not change configuration/catalogs or silently tear down +workloads. Old leases, deployments and the recovery profile remain in SQLite +archives under `/archives/`. Rotation is transactional and +safe to retry; a matching recovery backend makes the command a no-op. `gc` +reports backend mismatches with transition instructions; `gc --forget` remains +a history-only operation. -```bash -kubectl -n "$NS" describe model "$MODEL" -kubectl -n "$NS" get pods -l model="$MODEL" -kubectl -n "$NS" logs -f -l model="$MODEL" -c server # the engine -kubectl -n "$NS" logs -l model="$MODEL" -c server --previous # after a restart -kubectl -n "$NS" logs deploy/kubeai --tail=200 -f # the KubeAI controller -kubectl -n "$NS" get events --sort-by=.lastTimestamp | tail -n 40 -``` - -Common bad states: +### Kubernetes cluster setup and KubeAI integration -* `libcuda.so.1: cannot open shared object file`: the pod did not request a - GPU; fix the resource profile (requests, limits, `runtimeClassName`). -* startup probe fails with `connection refused`: still pulling the image, - loading the model or warming up; the acquire's wait reports which. -* `/models` works but completions 404: the request bypassed the gateway and - used the alias; through the gateway the alias is the model name, directly - against KubeAI the Model name is (`INFER_STACK_ENDPOINT_*` in the env file). +[docs/cluster-setup.md](docs/cluster-setup.md) describes server/worker setup. +`kube inventory` and `kube status` report structured facts; `--json` includes +probe errors without discarding other discoveries. Inventory and catalog +suggestions work before the KubeAI chart or namespace exists. + +`kube doctor` checks Kubernetes, NVIDIA scheduling/discovery, and KubeAI in +dependency order. Top-level `doctor --backend kubeai` remains the operational +preflight for acquiring work. If Helm installation succeeds but the configured +KubeAI API URL is unavailable, doctor reports that remaining routing issue. +Set `kubeai_base_url` to a reachable OpenAI URL for the installed service. + +`kube bootstrap` defaults to a read-only plan; `--apply` (or `--yes`) authorizes +host/cluster changes. K3s is the implemented provisioning provider; inventory, +installation and the backend work with any Kubernetes distribution. Bootstrap +preserves working clusters and existing kubeconfigs, installs Helm if missing, +and reconciles NVIDIA device plugin **0.17.1** with GPU Feature Discovery. +Install the host NVIDIA driver and container toolkit first. K3s discovers the +installed runtime on startup; bootstrap restarts local K3s only when runtime +discovery needs repair. It waits for Ready nodes, GPU allocation and GFD labels. + +`kube install` shows inferred Helm values; `--apply` uses Helm upgrade/install +and checks readiness afterward. `--dry-run`/`--plan` forces read-only behavior. +`--namespace`, `--release`, `--chart`, `--version` and `--values` allow overrides. +Existing named profiles and custom values are preserved; `HF_TOKEN` overrides +the chart token using a temporary protected file. Generated public values live +at `/generated/kube/kubeai-values.yaml`. + +The older `kube setup` and `kube k3s` commands remain available for compatibility +and node join operations. Setup scripts are compatibility wrappers around the +package commands. + +### Debugging serving failures + +`infer-stack kube status` summarizes live Kubernetes state. `infer-stack acquire` +reports pod failures such as `ImagePullBackOff`, `Unschedulable`, and engine +crashes. Use `infer-stack ps` and `infer-stack logs -f ` to inspect +model serving. If `/models` works but completions return 404, check that clients +use the gateway alias; direct KubeAI requests use the Model name from the env +file (`INFER_STACK_ENDPOINT_*`). Missing `libcuda.so.1` usually indicates that +the model profile did not request GPUs or select the correct runtime. --- diff --git a/dev/journals/codex.md b/dev/journals/codex.md index 8af6437e..e6d179fb 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -131,3 +131,63 @@ Design takeaways: 1. Once a file becomes canonical input, stale detection must treat it as input everywhere apply-style commands rely on freshness checks. 2. A clean source-of-truth split is only complete when freshness logic follows the same split. 3. Focused tests around stale detection are worth adding because timestamp-based bugs are easy to reintroduce during unrelated CLI cleanup. + +## 2026-10-01 19:28:25 -0400 + +User intent: make Kubernetes inventory, detailed readiness, K3s bootstrap and +KubeAI installation first-class kwconf commands, then fix the real-machine +Compose-to-KubeAI migration failures (stack down left the old recovery identity, +gc threw ProfileMismatch, status hid the active backend, sudo diagnostics implied +an all-clear, and the vLLM pod received served-model-name twice). Validation is +on this VM with fake runtimes; the maintainer will exercise the GPU machine. + +Model/configuration: GPT-6 (Codex), Default collaboration mode, standard session +configuration; the exact reasoning-effort setting is not exposed to this agent. +No sub-agents used. No real-host bootstrap/install mutations performed. + +I extended the existing kube ModalCLI and KubeManager rather than introducing +another shell implementation. Independent probes retain their own errors; node +GPU facts cannot depend on installed chart configuration because they determine +that configuration. Inventory describes facts; readiness applies dependency +policy. Bootstrap is a provider adapter with a read-only default and explicit +apply; working clusters/kubeconfigs win over installation convenience. The local +K3s restart gate verifies cluster identity, and unknown runtime state never means +quiescent. NVIDIA driver/toolkit installation remains host-distribution work. +Helm 0.17.1 NVIDIA plugin/GFD behavior is retained. KubeAI upgrades share +kubeai_ops and protect tokens in temporary 0600 values files. + +The migration failure clarified that runtime teardown and recovery epoch +ownership are separate operations. The chosen ledger rotate command previews +and rechecks quiescence under the publication lock, archives SQLite using its +backup API (including WAL), then atomically clears archived rows and publishes +the new configured backend snapshot in the same database. Keeping the database +inode avoids stranding already-open processes on a renamed old ledger. Archive +publication precedes reset; interruption before reset retains old state and +interruption after commit is a no-op on retry. Active leases and strict runtime +objects block rotation. Catalog/settings files remain authority and untouched. +GC reports a mismatch before mutation; status names both backend identities and +reads old rows with their frozen backend. KubeAI's engine_vllm.go injects the CR +served name before spec.args; an optional flag in the shared arg builder removes +only infer-stack's own served-name emission for KubeAI, preserving Compose and +arbitrary extra arguments. + +I am confident in the node/config separation and archive transaction design; +remaining risk is integration behavior on a real K3s host, particularly runtime +PATH discovery, chart rollout timing and configured API routing. Final combined +focused validation passed 213 tests. The full suite passed 1098 tests with 7 skips +and 8 existing cleanup/ResourceWarnings. `ty check ./infer_stack`, the CI flake8 +E9/F63/F7/F82 gate, scoped Ruff checks of changed implementation/new tests, and +`git diff --check` passed. Actual entrypoint help tree, all five kube leaves, +ledger rotate, status, gc and both GPU/sudo doctor forms were inspected. A +standalone simulated integration using real Controllers/SQLite and fake runtime +commands exercised acquire/refusal, clean GC refusal, preview/rotation/retry, +archive inspection and subsequent KubeAI acquire successfully. Read-only probes +on the VM's existing CPU K3s cluster retained both node facts and diagnosed the +missing plugin, unready node and unavailable configured API. No setup mutation +was performed. Remaining acceptance is the maintainer's GPU host: inventory and +doctor, bootstrap/install preview then explicit apply where needed, followed by +both detailed and operational doctor checks with the intended namespace/API URL. + +Reusable takeaways: prerequisite inventory must precede configuration that +consumes it; teardown does not erase persistence ownership; SQLite history +rotation should preserve live connections and change epochs transactionally. diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 14a13709..10014fc6 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -2,11 +2,14 @@ # End-to-end smoke test for the kubeai backend against a REAL cluster. # # Prereqs (once): a cluster + the KubeAI chart. On a single GPU host: -# ./scripts/bootstrap_k3s.sh -# printf 'resourceProfiles:\n %s:\n runtimeClassName: nvidia\n requests: {nvidia.com/gpu: "1"}\n limits: {nvidia.com/gpu: "1"}\n' \ -# "${E2E_RESOURCE_PROFILE:-nvidia-gpu}" > /tmp/kubeai-values.yaml -# ./scripts/install_kubeai.sh /tmp/kubeai-values.yaml kubeai -# kubectl -n kubeai port-forward svc/kubeai 8000:80 & +# infer-stack kube inventory +# infer-stack kube bootstrap --provider=k3s +# sudo -v +# infer-stack kube bootstrap --provider=k3s --apply +# infer-stack kube install +# infer-stack kube install --apply +# infer-stack kube doctor +# Set kubeai_base_url / E2E_BASE_URL to a reachable KubeAI OpenAI URL. # # Without a GPU (verified on k3s): install the chart with # dev/e2e_tests/kubeai-cpu-values.yaml, whose `cpu` profile runs real vLLM on diff --git a/docs/backend-parity.md b/docs/backend-parity.md index 157c8b9b..b27119ae 100644 --- a/docs/backend-parity.md +++ b/docs/backend-parity.md @@ -23,7 +23,7 @@ cluster setup is [kubeai-backend.md](kubeai-backend.md). | | Compose | KubeAI adds | |---|---|---| -| tools | Docker with the NVIDIA runtime | a Kubernetes cluster (`infer-stack kube k3s bootstrap/join` is the K3s convenience path), `kubectl`, `helm`, KubeAI, and GPU scheduling/discovery capabilities (`infer-stack kube setup [--apply]`) | +| tools | Docker with the NVIDIA runtime | a Kubernetes cluster (`infer-stack kube bootstrap --provider=k3s` / `kube k3s join` is the K3s convenience path), `kubectl`, `helm`, KubeAI, and GPU scheduling/discovery capabilities (`infer-stack kube inventory/doctor/bootstrap/install`) | | information | none: GPUs are discovered with `nvidia-smi` | `resourceProfiles` in the chart values (what one GPU unit requests, and on which nodes; `catalog suggest --backend kubeai` proposes them from the nodes' GPU labels), and per endpoint `runtime.resource_profile`, a `min_vram_gib` that picks one by size, or a default `kubeai_resource_profile` | | settings | `backend compose` | `backend kubeai`; `kubeai_gateway cluster` for more than one workstation; `kubeai_namespace`, `kubeai_base_url`, `kubeai_gateway_upstream` when the defaults (namespace `kubeai`, a port-forward on 8000, the Service's cluster IP) do not hold | | preflight | none needed | `infer-stack doctor`: cluster → CRD → namespace → KubeAI's API | diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 7398e93e..7f9f31eb 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -12,7 +12,7 @@ any Kubernetes distribution | | kubectl / Kubernetes API v -infer-stack kube nodes / setup +infer-stack kube inventory / doctor / install | v GPU scheduling + discovery, KubeAI, resource profiles @@ -21,14 +21,14 @@ GPU scheduling + discovery, KubeAI, resource profiles Distribution-specific lifecycle commands live below that boundary: ```text -infer-stack kube k3s bootstrap +infer-stack kube bootstrap --provider=k3s infer-stack kube k3s join # A future integration could add, for example: # infer-stack kube k0s ... ``` -Nothing in `infer-stack kube setup`, model placement, the lease controller, or +Nothing in `infer-stack kube inventory`, model placement, the lease controller, or the KubeAI backend requires K3s. An existing kubeadm, RKE2, k0s, EKS, or other Kubernetes cluster should start at [Use an existing cluster](#use-an-existing-cluster). @@ -68,7 +68,7 @@ are different concepts. ## What generic setup expects -Before `infer-stack kube setup --apply`, the generic integration cares about +Before `infer-stack kube install --apply`, the generic integration cares about observable capabilities rather than how the cluster was installed: - `kubectl` selects the intended cluster context and can reach its API. @@ -87,13 +87,13 @@ The read-only check is always the first command: ```bash infer-stack kube nodes -infer-stack kube setup +infer-stack kube inventory ``` Only after reviewing the target context and plan: ```bash -infer-stack kube setup --apply +infer-stack kube install --apply ``` Host NVIDIA driver/container-runtime installation stays outside infer-stack. @@ -115,19 +115,22 @@ same version. On workstation A: ```bash -infer-stack kube k3s bootstrap --version= +infer-stack kube bootstrap --provider=k3s --version= +sudo -v +infer-stack kube bootstrap --provider=k3s --version= --apply ``` -This command is intentionally scoped under `kube k3s`: it installs/starts a +This provider-specific bootstrap plans by default; `--apply` installs/starts a K3s server, waits for its node to become Ready, and makes the K3s kubeconfig usable without overwriting an existing `~/.kube/config`. Then inspect the generic Kubernetes integration: ```bash -infer-stack kube nodes -infer-stack kube setup -infer-stack kube setup --apply +infer-stack kube inventory +infer-stack kube doctor +infer-stack kube install +infer-stack kube install --apply ``` The setup commands above are not K3s-specific. @@ -174,9 +177,10 @@ network. Back on the machine operating infer-stack: ```bash -infer-stack kube nodes -infer-stack kube setup -infer-stack kube setup --apply +infer-stack kube inventory +infer-stack kube doctor +infer-stack kube install +infer-stack kube install --apply ``` A newly joined GPU product may produce a new KubeAI resource profile. Existing @@ -347,29 +351,29 @@ kubeconfig/context and begin at the generic layer: ```bash kubectl config current-context -infer-stack kube nodes -infer-stack kube setup -infer-stack kube setup --apply +infer-stack kube inventory +infer-stack kube doctor +infer-stack kube install +infer-stack kube install --apply infer-stack doctor ``` The generic setup path must not depend on K3s files, services, tokens, Flannel, or installer behavior. If a future Kubernetes distribution needs first-class -provisioning convenience, add it under its own `infer-stack kube ` -namespace while preserving this generic contract. +provisioning convenience, add a provider adapter for `infer-stack kube bootstrap --provider=` while preserving this generic contract. ## Ownership boundary `infer-stack kube` deliberately stops short of being a general Kubernetes administrator: -- `kube setup` owns only the integration components/capabilities infer-stack - needs. +- `kube inventory/doctor` inspect prerequisites and installation; `kube install` + reconciles the KubeAI chart and resource profiles. - `kube node detach/attach` owns only a temporary scheduling cordon used to hand a host's GPU between Kubernetes and local Compose; it does not remove cluster membership. -- Distribution-specific provisioning stays in scoped subcommands such as - `kube k3s`. +- Distribution provisioning uses `kube bootstrap --provider=k3s`; joining nodes + remains under `kube k3s join`. - Arbitrary cluster administration remains `kubectl`, Helm, and the chosen distribution's tooling. - Existing externally managed GPU/KubeAI capabilities are accepted instead of diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index a7f71b9c..7d9b44c2 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -41,50 +41,98 @@ For the control-plane/worker mental model and the full create/join runbook, see [cluster-setup.md](cluster-setup.md). The generic integration is distribution-neutral; K3s is only the first scoped provisioning target. -The normal setup path is capability-driven and lives in the CLI: +The normal setup path uses package commands: ```bash -# Existing cluster: just point kubectl at it. For a new local K3s server: -infer-stack kube k3s bootstrap --version= +infer-stack kube inventory # works before KubeAI exists +infer-stack kube doctor # detailed dependency-ordered readiness +infer-stack kube bootstrap --provider=k3s # plan a new host or prerequisite repair +sudo -v +infer-stack kube bootstrap --apply # K3s/Helm/NVIDIA device plugin 0.17.1 + GFD +infer-stack kube install # inspect inferred resourceProfiles and values +infer-stack kube install --apply # Helm upgrade/install, wait, post-install doctor +infer-stack kube status # nodes, GPUs, plugin, pods/services/managed Models +``` + +Install host NVIDIA drivers/container toolkit before bootstrap. Inventory and +install are distribution-neutral. Bootstrap preserves working clusters and +existing kubeconfigs; it refuses to replace a selected unreachable context. +Use `--version` to pin K3s; active clusters are never implicitly upgraded. + +`inventory`, `doctor`, and `status` accept `--json`. Errors belong to individual +probes, so a missing KubeAI ConfigMap cannot erase GPU node facts. A cluster with +a leftover CRD but no namespace/chart can be diagnosed and repaired with these +commands. Kubernetes GPU allocation and GFD readiness are separate from host +GPU visibility. + +Mutation requires `--apply` (alias `--yes`). Both mutating commands support +`--dry-run` (alias `--plan`). Install derives one profile per GPU product with +GPU requests/limits, a product selector and `runtimeClassName=nvidia` when that +RuntimeClass exists. Kubernetes chooses placement; infer-stack assigns no host +GPU indices. Models request multiple GPUs via the existing profile/count syntax. + +```bash +infer-stack kube install --values ./kubeai-values.local.yaml --namespace kubeai +infer-stack kube install --values ./kubeai-values.local.yaml --namespace kubeai --apply +``` -# Planning is read-only; apply only after reviewing it. -infer-stack kube setup -infer-stack kube setup --apply -infer-stack kube nodes +Custom values extend the automatic common case. Existing/custom named profiles +win over generated profiles. The installed chart version is preserved unless +`--version` overrides it; `--chart` and `--release` select alternative settings. +`HF_TOKEN` is preserved using a temporary mode-0600 Helm values file, never +printed or written to the persistent generated public values. Inspect those +values at `/generated/kube/kubeai-values.yaml`. + +Installation waits for Helm readiness, then checks the configured +`kubeai_base_url`. If that URL is unavailable, installation reports a routing +failure even though the release is installed; set a reachable OpenAI URL with +`infer-stack config set kubeai_base_url ` and repeat doctor. No automatic +background port-forward is created. + +For temporary Compose/KubeAI testing, preserve the Compose authority as described +in [cluster setup](cluster-setup.md#temporarily-swap-compose-configured-workstations-into-a-cluster). +`kube setup` remains a compatibility plan/apply workflow, and the setup scripts +forward to the new commands. + +For explicit CPU development, preserve the existing CPU chart workflow: + +```bash +infer-stack kube install --gpu=none --values dev/e2e_tests/kubeai-cpu-values.yaml +infer-stack kube install --gpu=none --values dev/e2e_tests/kubeai-cpu-values.yaml --apply +infer-stack kube doctor --gpu=none ``` -For temporary testing on workstations that already have Compose state you want -to preserve, do not point KubeAI at the existing Compose ledger. The -[cluster setup guide](cluster-setup.md#temporarily-swap-compose-configured-workstations-into-a-cluster) -uses a separate KubeAI data root and `kube node detach/attach` so the same -physical machines can move back to local Compose use without uninstalling or -rejoining Kubernetes. - -`kube setup` checks the Kubernetes API, Ready nodes, GPU scheduling/discovery -facts, Helm, the KubeAI CRD/service, and discovered GPU resource profiles. It -is capability-based: an existing GPU Operator or externally-managed KubeAI is -accepted when it already supplies what infer-stack needs. It does not replace -working external components merely because they were installed differently. - -The managed NVIDIA path begins only after Kubernetes exposes an `nvidia` -RuntimeClass. Installing NVIDIA drivers/container-runtime packages is host -administration and remains outside infer-stack. With that runtime present, -`kube setup --apply` may install/reconcile the pinned NVIDIA device plugin + -GPU Feature Discovery and then derive profiles from `nvidia.com/gpu.product` -and `nvidia.com/gpu.memory`. - -KubeAI values are written to `/generated/kube/kubeai-values.yaml`. Named -profiles already supplied by the operator win over generated profiles. Use an -operator values file for other chart customization: +## Migrate an existing Compose recovery ledger + +`stack down --backend compose` removes runtime objects but retains the Compose +recovery snapshot. Changing `settings.yaml` alone cannot reinterpret historical +Compose rows as Kubernetes Models. Use an explicit transition: ```bash -infer-stack kube setup --values ./kubeai-values.local.yaml -infer-stack kube setup --apply --values ./kubeai-values.local.yaml +infer-stack release --all --backend compose +infer-stack stack down --backend compose +infer-stack config set backend kubeai +infer-stack ledger rotate # preview; strict old-runtime verification +infer-stack ledger rotate --yes # commit archived history and new epoch +infer-stack ledger archives +infer-stack acquire --yes ``` -For an existing Helm-managed KubeAI release, apply preserves its installed -chart version by default; use `--kubeai-version=...` for an explicit change. -`scripts/install_kubeai.sh` remains only as a manual Helm escape hatch. +All active leases must be released and all old runtime objects removed first. +An unreachable runtime blocks rotation, rather than being interpreted as empty. +KubeAI transitions also check managed Model CRs whose pods may not yet exist. +Stopped Compose containers must be removed with `stack down` as well, so restart +policies cannot resurrect them after transition. Configuration/catalogs are +untouched. SQLite backups under `/archives/` retain the old +leases/deployments/profile, including committed WAL contents. The archive is +published before a single transaction resets live rows and installs the new +backend snapshot. Retrying after interruption is safe. The live DB stays at the +same path/inode; already-open old-backend controllers refuse their next +mutation against the new snapshot. + +`status` names the configured and active recovery backends separately. `gc` +refuses backend mismatch cleanly with `ledger rotate` guidance. `gc --forget` +only prunes terminal historical rows; it never adopts the configured backend. ## Point infer-stack at it @@ -102,8 +150,8 @@ infer-stack config set kubeai_resource_profile nvidia-gpu-rtx-4090 infer-stack config set kubeai_gateway_upstream http://kubeai.example/openai/v1 ``` -On a host whose persisted default remains Compose, use a separate KubeAI -authority/data root instead: +For temporary testing while keeping an existing Compose recovery epoch, use a +separate KubeAI authority/data root: ```bash export INFER_STACK_BACKEND=kubeai @@ -138,7 +186,7 @@ Discovery) of the nodes its `nodeSelector` selects; a profile without a `nodeSelector`, like the chart's generic ones, has no size and is never picked this way. With none large enough, the `kubeai_resource_profile` default is used, or the acquire is refused with the sizes it found. -`infer-stack kube setup` is the setup authority for these profiles; +`infer-stack kube install` is the installation authority for these profiles; `catalog suggest --backend kubeai` consumes the same profile-generation shape while suggesting model/catalog entries. @@ -203,9 +251,10 @@ on the trusted LAN/VPN. Back on the control node, inspect and reconcile the newly visible hardware: ```bash -infer-stack kube nodes -infer-stack kube setup -infer-stack kube setup --apply +infer-stack kube inventory +infer-stack kube bootstrap --apply +infer-stack kube install --apply +infer-stack kube doctor ``` A new GPU product becomes a new discovered KubeAI resource profile without diff --git a/docs/planning/known-limitations.md b/docs/planning/known-limitations.md index fe4ace67..444c4c01 100644 --- a/docs/planning/known-limitations.md +++ b/docs/planning/known-limitations.md @@ -126,9 +126,10 @@ controller advances it automatically on acquire under the publication lock: away. `allowed_gpus` remains per caller rather than part of the recovery snapshot. -Changing backend kind (Compose to KubeAI or back) still requires tearing down -the old backend first; automatic snapshot advancement never crosses backend -kinds. +Changing backend kind (Compose to KubeAI or back) requires quiescing the old +backend, then `infer-stack ledger rotate --yes` to archive its history and start +a new recovery epoch. `stack down` alone retains the old snapshot. Automatic +snapshot advancement never crosses backend kinds. ### A queued acquire holds nothing while it waits (current, by design) diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index e2ed4d2e..4000c39a 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -23,7 +23,7 @@ ``last_errors``), never silently overwritten. Cluster prerequisites (once per cluster, not per acquire): a reachable -kubeconfig, KubeAI installed (normally via ``infer-stack kube setup --apply``) +kubeconfig, KubeAI installed (normally via ``infer-stack kube install --apply``) with ``resourceProfiles`` matching the ``resource_profile`` names your catalog uses, and a route to the gateway (the default ``base_url`` assumes ``kubectl port-forward svc/kubeai 8000:80``). See ``docs/kubeai-backend.md``. @@ -106,8 +106,9 @@ def _model_doc( ``spec.args`` reuses the exact arg pipeline the compose backend renders with (``_vllm_service_dict`` + ``vllm_args``), so every serving knob the compat key distinguishes (revision/quantization/dtype/pp/...) reaches the - engine here too. ``served_model_name`` is overridden to the CR name so the - gateway's request name and vLLM's served name agree. + engine here too. KubeAI injects the served name from the CR name before + spec.args (engine_vllm.go); emit only the remaining infer-stack flags. + Environment templates use that same CR identity. """ name = name or model_name(deployment) svc = vllm_service_dict(deployment) @@ -128,7 +129,7 @@ def _model_doc( 'resourceProfile': profile, 'minReplicas': min_replicas, 'maxReplicas': max_replicas, - 'args': vllm_args(svc), + 'args': vllm_args(svc, include_served_name=False), } doc: dict[str, Any] = { 'apiVersion': 'kubeai.org/v1', @@ -756,7 +757,7 @@ def _set_front_door(self, desired, rendered: RenderedModels) -> None: def gpu_facts(self) -> tuple[dict[str, dict[str, Any]], dict[str, float]]: """``(each node's GPUs, each sized profile's GPU GiB)`` from the cluster. - Empty when the cluster cannot say (no GPU labels, no chart config): + Node facts survive missing chart config; installed sizing is empty then: sizing then falls back to the default profile, as before. """ import time @@ -765,18 +766,29 @@ def gpu_facts(self) -> tuple[dict[str, dict[str, Any]], dict[str, float]]: if cached is not None and time.monotonic() - cached[0] < self.GPU_FACTS_TTL: return cached[1] try: - nodes = (json.loads(self._kubectl(['get', 'nodes', '-o', 'json']) or '{}') - .get('items') or []) - config = json.loads(self._kubectl( - ['get', 'configmap', self.CONFIG_MAP, '-o', 'json']) or '{}') - system = yaml.safe_load(((config.get('data') or {}).get('system.yaml')) or '') or {} - facts = node_gpus(nodes) - sized = sized_profiles(system.get('resourceProfiles') or {}, facts) - except Exception: # noqa: BLE001 - sizing is best-effort, never fatal - facts, sized = {}, {} + facts = self.node_gpu_facts() + except Exception: # node discovery is best-effort + facts = {} + try: + profiles = self.installed_resource_profiles() + sized = sized_profiles(profiles, facts) + except Exception: # missing/malformed chart config must not erase nodes + sized = {} self._gpu_facts_cache = (time.monotonic(), (facts, sized)) return facts, sized + def node_gpu_facts(self) -> dict[str, dict[str, Any]]: + """Discover Kubernetes GPUs independently of KubeAI installation.""" + nodes = json.loads(self._kubectl(['get', 'nodes', '-o', 'json']) or '{}') + return node_gpus(nodes.get('items') or []) + + def installed_resource_profiles(self) -> dict[str, Any]: + """Read installed chart profiles; callers decide how to handle absence.""" + config = json.loads(self._kubectl( + ['get', 'configmap', self.CONFIG_MAP, '-o', 'json']) or '{}') + system = yaml.safe_load((config.get('data') or {}).get('system.yaml', '')) or {} + return system.get('resourceProfiles') or {} + def suggestion_inventory(self) -> tuple[dict[str, Any], dict[str, Any]]: """``(inventory, resourceProfiles)`` for ``catalog suggest``. @@ -964,7 +976,9 @@ def use_profile(self, profile: dict) -> None: raise ProfileMismatch( f"the active recovery snapshot is for the {profile.get('backend')!r} backend; " - 'tear down the old backend before switching backend kinds' + 'quiesce the old backend (release leases and tear down workloads), then ' + 'start a new ledger/recovery epoch with `infer-stack ledger rotate --yes` ' + 'for the configured backend; stack down alone retains the old snapshot' ) from ..leasing.profile import CatalogUnion @@ -1019,7 +1033,7 @@ def apply(self) -> ApplyResult: raise RuntimeError( f'kubectl apply failed: {ex}\n' 'Is the KubeAI chart installed and the kubeconfig ' - 'reachable? See `infer-stack kube setup` and ' + 'reachable? See `infer-stack kube doctor` and ' 'docs/kubeai-backend.md.' ) from ex # Prune: managed Models on the cluster that the render dropped. Their @@ -1265,6 +1279,11 @@ def settle_snapshot(self): """ return self.gateway.settle_snapshot() if self.gateway is not None else None + def recovery_blockers(self) -> list[str]: + """Strict transition gate, including Models whose pods have not started.""" + return ([f'Model/{name}' for name in self._cluster_models()] + + [i.name for i in self.instances()]) + def down(self) -> None: """Delete every infer-stack-managed Model (explicit stop), and the gateway.""" for name in sorted(self._cluster_models()): @@ -1298,21 +1317,21 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: if not _run_check( 'cluster reachable', ['version', '--client=false', '-o', 'json'], - 'select a reachable Kubernetes kubeconfig/context, then run `infer-stack kube setup`', + 'select a reachable Kubernetes kubeconfig/context, then run `infer-stack kube doctor`', ): return checks if not _run_check( 'KubeAI Model CRD installed', ['get', 'crd', 'models.kubeai.org', '-o', 'name'], - 'run `infer-stack kube setup` to inspect, then `infer-stack kube setup --apply`', + 'run `infer-stack kube doctor` to inspect, then `infer-stack kube install --apply`', ): return checks - _run_check( + if not _run_check( f'namespace {self.namespace!r} exists', ['get', 'namespace', self.namespace, '-o', 'name'], - f'kubectl create namespace {self.namespace} (or install the ' - 'chart there)', - ) + f'infer-stack kube install --namespace={self.namespace} --apply', + ): + return checks if getattr(self.gateway, 'in_cluster', False): # Clients use the gateway in the cluster; no port-forward needed. checks.append(self.gateway.doctor_check()) @@ -1327,9 +1346,8 @@ def _run_check(name: str, args: list[str], hint: str) -> bool: except Exception as ex: # noqa: BLE001 - report, don't raise ok = False detail = ( - f'{ex} — is the gateway routed? e.g. ' - f'`kubectl -n {self.namespace} port-forward svc/kubeai ' - '8000:80` (or set kubeai_base_url)' + f'{ex} — inspect `infer-stack kube status`, then ' + 'set a reachable API with `infer-stack config set kubeai_base_url `' ) # KubeAI's own API (what the port-forward reaches), not infer-stack's # LiteLLM gateway, which starts with the first acquire. diff --git a/infer_stack/cli/__init__.py b/infer_stack/cli/__init__.py index ed3f7071..f75fbe5e 100644 --- a/infer_stack/cli/__init__.py +++ b/infer_stack/cli/__init__.py @@ -41,6 +41,7 @@ ) from .commands_catalog import CatalogModalCLI from .commands_kube import KubeModalCLI +from .commands_ledger import LedgerModalCLI from .commands_leasing import ( AccessCLI, AcquireCLI, @@ -146,6 +147,7 @@ class ManageCLI(kw.ModalCLI): apply = ApplyCLI # bring the desired set up (the trigger for acquire --no-apply) wait = WaitCLI # block until endpoints are ready (acquire --no-wait fan-out) leases = LeasesCLI + ledger = LedgerModalCLI routes = RoutesModalCLI # inspect/seed/prune the LiteLLM route registry network = NetworkModalCLI # stable per-service addresses + upstream routing check tui = TuiCLI # live Textual monitor + controls (opt-in: infer-stack[tui]) diff --git a/infer_stack/cli/commands_catalog.py b/infer_stack/cli/commands_catalog.py index dcf7e31e..14d0dab7 100644 --- a/infer_stack/cli/commands_catalog.py +++ b/infer_stack/cli/commands_catalog.py @@ -387,7 +387,7 @@ def main(cls, argv=True, **kwargs): _print_yaml(text) if values: print('# discovered KubeAI resourceProfiles (the same shape reconciled by ' - '`infer-stack kube setup`; shown here for inspection/manual Helm use):\n' + values, + '`infer-stack kube install`; review with `infer-stack kube install`):\n' + values, file=sys.stderr) return 0 @@ -417,7 +417,7 @@ def main(cls, argv=True, **kwargs): ) if values: print('discovered KubeAI resource profiles (normally reconciled by ' - '`infer-stack kube setup --apply`; shown here for inspection/manual Helm use):') + '`infer-stack kube install --apply`; review with `infer-stack kube install`):') print(values, end='') return 0 diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index 2eb4d0b7..e84d23f1 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -376,7 +376,7 @@ def main(cls, argv=True, **kwargs): ) print(f'K3s join token: sudo cat {K3S_NODE_TOKEN}') print('Join workers with: infer-stack kube k3s join --server=... --token-file=...') - print('Then: infer-stack kube setup') + print('Then: infer-stack kube inventory; infer-stack kube bootstrap; infer-stack kube install') return 0 @@ -422,10 +422,221 @@ class K3sModalCLI(kw.ModalCLI): join = K3sJoinCLI +def _inventory_for_config(config, manager): + from ..kube.inspect import inventory + from ..paths import get_setting + + return inventory(manager, namespace=_namespace(config), release=config.release, + base_url=config.base_url or get_setting('kubeai_base_url') or + 'http://127.0.0.1:8000/openai/v1') + + +def _print_inventory(report): + import yaml + + print('cluster') + for key, value in report['cluster'].items(): + print(f' {key}: {value}') + print(f"host GPUs: {report['host_gpus']}") + print('tools') + for key, value in report['tools'].items(): + print(f' {key}: {"available" if value else "missing"}') + for node in report['nodes']: + print(f"node {node['name']}") + for key in ('ready', 'nvidia_runtime_class', 'gpu_count', 'gpu_product', + 'gpu_memory_gib', 'gfd_labels'): + print(f' {key}: {node[key]}') + plugins = report['device_plugin'] + print('NVIDIA device plugin: ' + ('unknown' if plugins is None else 'missing' if not plugins else + ', '.join(f"{p['name']} ({p['ready']}/{p['desired']} ready)" for p in plugins))) + kubeai = report['kubeai'] + print('KubeAI') + print(f" Model CRD: {'installed' if kubeai['crd'] else 'missing' if kubeai['crd'] is False else 'unknown'}") + print(f" namespace {kubeai['namespace']}: {'present' if kubeai['namespace_exists'] else 'missing' if kubeai['namespace_exists'] is False else 'unknown'}") + release = kubeai['release'] + print(f" Helm release: {release.get('chart')} ({release.get('status')})" if release else + ' Helm release: missing/unknown') + api = kubeai['api'] + print(f" API: {api['base_url']} — {'available' if api['reachable'] else 'unavailable'}") + if not api['reachable']: + print(f" {api.get('error') or api.get('blocked') or api.get('status_code') or ''}") + for key in ('pods', 'services', 'models'): + items = kubeai[key] + print(f" {key}: {len(items) if items is not None else 'unknown/blocked'}") + for item in items or []: + name = item.get('metadata', {}).get('name', '?') + status = item.get('status', {}) + if key == 'pods': + containers = status.get('containerStatuses', []) + summary = f"{status.get('phase', '?')}; {sum(bool(c.get('ready')) for c in containers)}/{len(containers)} containers ready" + elif key == 'services': + spec = item.get('spec', {}) + summary = f"{spec.get('type', '?')} {spec.get('clusterIP', '')}" + else: + summary = f"{status.get('readyReplicas', 0)} ready replicas" + print(f' {name}: {summary}') + print('resource profiles proposed') + print(yaml.safe_dump(report['resource_profiles']['proposed'], sort_keys=False), end='') + print('resource profiles installed: ' + (', '.join(report['resource_profiles']['installed']) or 'none')) + for key, error in report['errors'].items(): + print(f' probe {key}: {error}') + + +def _print_checks(checks): + for check in checks: + mark = {'ok': 'ok ', 'fail': 'FAIL', 'blocked': 'SKIP'}[check['status']] + print(f"[{mark}] {check['name']} — {check['detail']}") + if check['fix']: + print(f" fix: {check['fix']}") + + +class KubeInventoryCLI(_PathOverridesMixin): + """Describe Kubernetes, GPU and KubeAI facts, including partial installations.""" + + __command__ = 'inventory' + namespace = kw.Value(None, type=str, help='Configured KubeAI namespace override.') + release = kw.Value('kubeai', type=str, help='KubeAI Helm release name.') + base_url = kw.Value(None, type=str, help='Configured KubeAI OpenAI API URL override.') + json = kw.Value(False, isflag=True, help='Emit structured facts and probe errors.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + report = _inventory_for_config(config, KubeManager()) + if config.json: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + _print_inventory(report) + return 0 + + +class KubeStatusCLI(KubeInventoryCLI): + """Show cluster GPUs, KubeAI release, pods, services and managed Models.""" + + __command__ = 'status' + + +class KubeDoctorCLI(KubeInventoryCLI): + """Check Kubernetes/GPU prerequisites and KubeAI readiness in dependency order.""" + + __command__ = 'doctor' + gpu = kw.Value('nvidia', type=str, choices=['nvidia', 'none'], help='Use none for explicitly configured CPU KubeAI.') + + @classmethod + def main(cls, argv=True, **kwargs): + from ..kube.inspect import failed, readiness + + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + report = _inventory_for_config(config, KubeManager()) + checks = readiness(report, gpu=config.gpu != 'none') + if config.json: + print(json.dumps({'inventory': report, 'checks': checks}, indent=2)) + else: + _print_checks(checks) + return int(failed(checks)) + + +class KubeInstallCLI(KubeInventoryCLI): + """Plan KubeAI values; --apply installs/upgrades the chart and checks readiness.""" + + __command__ = 'install' + gpu = kw.Value('nvidia', type=str, choices=['nvidia', 'none'], help='Use none with explicit CPU resource profiles.') + apply = kw.Value(False, isflag=True, alias=['yes'], help='Apply the displayed Helm installation plan.') + dry_run = kw.Value(False, isflag=True, alias=['plan'], help='Only inspect values, even with --apply.') + values = kw.Value(None, type=str, help='Additional Helm values; custom named profiles take precedence.') + chart = kw.Value('kubeai/kubeai', type=str, help='KubeAI chart reference.') + version = kw.Value(None, type=str, help='Chart version; preserves installed version by default.') + + @classmethod + def main(cls, argv=True, **kwargs): + import yaml + + from ..kube.inspect import failed, readiness + from ..kube.operations import install, install_plan + + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = KubeManager() + try: + operator = manager.load_values_file(config['values']) + report = _inventory_for_config(config, manager) + if config.gpu == 'none': + report['resource_profiles']['proposed'] = {} + values = install_plan(manager, report, operator_values=operator) + checks = readiness(report, installation=False, gpu=config.gpu != 'none') + if config.json: + if not config.apply or config.dry_run: + print(json.dumps({'values': values, 'checks': checks}, indent=2)) + else: + _print_checks(checks) + print(yaml.safe_dump(values, sort_keys=False), end='') + if not config.apply or config.dry_run: + if not config.json: + print('No changes made. Run infer-stack kube install --apply to install/upgrade.') + return int(failed(checks)) + install(manager, report, operator_values=operator, chart=config.chart, version=config.version, gpu=config.gpu != 'none') + checks = readiness(_inventory_for_config(config, manager), gpu=config.gpu != 'none') + if config.json: + print(json.dumps({'values': values, 'checks': checks}, indent=2)) + else: + _print_checks(checks) + return int(failed(checks)) + except RuntimeError as ex: + raise SystemExit(str(ex)) from ex + + +class KubeBootstrapCLI(KubeInventoryCLI): + """Plan host/cluster GPU prerequisites; --apply converges them with K3s.""" + + __command__ = 'bootstrap' + provider = kw.Value('k3s', choices=['k3s'], type=str, help='Cluster provisioning provider.') + version = kw.Value(None, type=str, help='Optional exact K3s version; no implicit upgrades.') + apply = kw.Value(False, isflag=True, alias=['yes'], help='Authorize host/cluster bootstrap changes.') + dry_run = kw.Value(False, isflag=True, alias=['plan'], help='Only display the plan.') + timeout = kw.Value(180, type=int, help='Ready/GPU discovery wait timeout in seconds.') + + @classmethod + def main(cls, argv=True, **kwargs): + from ..kube.inspect import failed, readiness + from ..kube.operations import K3sProvider + + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = KubeManager() + provider = K3sProvider() + report = _inventory_for_config(config, manager) + actions = provider.plan(manager, report) + if not config.apply or config.dry_run: + if config.json: + print(json.dumps({'inventory': report, 'actions': actions}, indent=2)) + else: + for action in actions: + print(f'[plan] {action}') + print('No changes made. Authenticate with sudo -v if needed; then run infer-stack kube bootstrap --apply.') + return 0 + try: + report = provider.apply(manager, report, version=config.version, timeout=config.timeout) + except RuntimeError as ex: + raise SystemExit(f'{ex}. If elevation failed, authenticate with sudo -v and retry.') from ex + checks = readiness(report, installation=False) + if config.json: + print(json.dumps({'inventory': report, 'checks': checks}, indent=2)) + else: + _print_checks(checks) + return int(failed(checks)) + + class KubeModalCLI(kw.ModalCLI): """Inspect/setup infer-stack capabilities on any Kubernetes distribution.""" __command__ = 'kube' + inventory = KubeInventoryCLI + doctor = KubeDoctorCLI + bootstrap = KubeBootstrapCLI + install = KubeInstallCLI + status = KubeStatusCLI nodes = KubeNodesCLI node = KubeNodeModalCLI setup = KubeSetupCLI diff --git a/infer_stack/cli/commands_leasing.py b/infer_stack/cli/commands_leasing.py index e0b7a952..81e67897 100644 --- a/infer_stack/cli/commands_leasing.py +++ b/infer_stack/cli/commands_leasing.py @@ -1479,6 +1479,7 @@ class GcCLI(_ApprovalMixin): @classmethod def main(cls, argv=True, **kwargs): from ..leasing.backend import ConvergeAborted + from ..leasing.profile import ProfileMismatch config = cls.cli(argv=argv, data=kwargs) controller = _open_controller(config, interactive=True) @@ -1491,6 +1492,8 @@ def main(cls, argv=True, **kwargs): print(f'gc --forget: forgot {n_leases} released/expired lease(s), ' f'{n_deployments} stopped deployment(s)') return 0 + if controller._profile_error is not None: + raise SystemExit(f'gc: {controller._profile_error}') if config.orphans: # Unlabelled units in residency. The kubeai backend reads only the # pods carrying its label, so it never has any. @@ -1511,6 +1514,8 @@ def confirm(found): return 0 try: outcome = controller.gc(evict_idle=bool(config.evict)) + except ProfileMismatch as ex: + raise SystemExit(f'gc: {ex}') from ex except ConvergeAborted: raise _declined_exit() if config.json: diff --git a/infer_stack/cli/commands_ledger.py b/infer_stack/cli/commands_ledger.py new file mode 100644 index 00000000..cd7b18a7 --- /dev/null +++ b/infer_stack/cli/commands_ledger.py @@ -0,0 +1,80 @@ +"""Recovery ledger lifecycle; configuration/catalogs remain user authority.""" +from __future__ import annotations + +import json + +import kwconf as kw + +from .commands_leasing import _ApprovalMixin, _make_backend, _open_controller + + +class LedgerRotateCLI(_ApprovalMixin): + """Archive a quiescent old backend ledger and start the configured backend. + + Default: verify and preview. --yes authorizes rotation; it never releases + leases or tears down workloads. Configuration and catalog files are retained. + """ + + __command__ = 'rotate' + json = kw.Value(False, isflag=True, help='Emit the transition summary as JSON.') + dry_run = kw.Value(False, isflag=True, alias=['plan'], help='Verify and preview even with --yes.') + + @classmethod + def main(cls, argv=True, **kwargs): + from ..leasing.profile import ProfileMismatch + from ..leasing.transition import rotate_backend + + config = cls.cli(argv=argv, data=kwargs) + controller = _open_controller(config) + current = controller.ledger.profile() + target = controller.invocation_profile() or {} + old_backend = None + if current and current.get('backend') != target.get('backend'): + old_config = cls.cli(argv=False, data={**config.asdict(), 'backend': current['backend']}) + old_backend = _make_backend(old_config) + try: + summary = rotate_backend(controller, old_backend, apply=bool(config.yes and not config.dry_run)) + except (ProfileMismatch, OSError, RuntimeError) as ex: + raise SystemExit(f'ledger rotate: {ex}') from ex + if config.json: + print(json.dumps(summary, indent=2)) + else: + print(f"Recovery backend: {summary['from_backend']} -> {summary['to_backend']}") + print(summary['detail']) + if summary['archive']: + print(f"History archive: {summary['archive']} ({summary['leases']} leases, {summary['deployments']} deployments)") + if summary['changed']: + print('New recovery epoch initialized; acquire now uses the configured backend.') + elif summary['archive']: + print('No changes made. Re-run infer-stack ledger rotate --yes to commit.') + return 0 + + +class LedgerArchivesCLI(_ApprovalMixin): + """List archived recovery epochs and their inspectable SQLite ledger paths.""" + + __command__ = 'archives' + json = kw.Value(False, isflag=True, help='Emit archive metadata as JSON.') + + @classmethod + def main(cls, argv=True, **kwargs): + config = cls.cli(argv=argv, data=kwargs) + controller = _open_controller(config) + archives = controller.ledger.store.meta_json('ledger_archives', []) + if config.json: + print(json.dumps(archives, indent=2)) + elif not archives: + print('No archived recovery epochs.') + else: + for archive in archives: + print(f"{archive['backend']}: {archive['path']} ({archive['leases']} leases, {archive['deployments']} deployments)") + print(f" inspect: infer-stack leases --backend={archive['backend']} --ledger={archive['path']}") + return 0 + + +class LedgerModalCLI(kw.ModalCLI): + """Preview/rotate backend recovery epochs and inspect preserved history.""" + + __command__ = 'ledger' + rotate = LedgerRotateCLI + archives = LedgerArchivesCLI diff --git a/infer_stack/cli/commands_runtime.py b/infer_stack/cli/commands_runtime.py index 1536c550..b4a04349 100644 --- a/infer_stack/cli/commands_runtime.py +++ b/infer_stack/cli/commands_runtime.py @@ -9,6 +9,7 @@ from __future__ import annotations +import os import subprocess import sys from pathlib import Path @@ -123,13 +124,14 @@ def _leasing_status() -> dict[str, Any]: path = default_ledger_path() out: dict[str, Any] = {'path': str(path), 'exists': path.exists(), - 'leases': [], 'deployments': [], 'summary': None} + 'leases': [], 'deployments': [], 'summary': None, 'recovery_profile': None} if not path.exists(): return out try: ledger = Ledger(SqliteStore(str(path))) leases, deployments = ledger.status(virtual_expiry=True) out['pending'] = ledger.publication_pending() is not None + out['recovery_profile'] = ledger.profile() except Exception: # noqa: BLE001 return out active = sum(1 for le in leases if le.state == LeaseState.ACTIVE) @@ -224,13 +226,29 @@ def _gather_status(config) -> dict[str, Any]: backend = None rendered = getattr(backend, 'rendered_file', None) leasing = _leasing_status() + active_profile = leasing.get('recovery_profile') + configured_backend = os.environ.get('INFER_STACK_BACKEND', '').strip() or get_setting('backend') or 'null' + recovery_backend = (active_profile or {}).get('backend') + # Interpret old deployment rows using their frozen backend, never the new + # configured kind. Failure to inspect it remains unknown in the status view. + if active_profile: + from types import SimpleNamespace + try: + active_config = SimpleNamespace(**{**config.asdict(), 'backend': recovery_backend}) + backend = _day2_backend(active_config) + backend.recovery_profile.use_profile(active_profile) + rendered = getattr(backend, 'rendered_file', None) + except (SystemExit, Exception): + backend = None + rendered = None if leasing.get('live_deployments'): leasing['served'] = _served_models(leasing.pop('live_deployments'), backend, pending=bool(leasing.get('pending'))) else: leasing.pop('live_deployments', None) return { - 'backend': str(get_setting('backend') or 'null'), + 'backend': str(configured_backend), + 'recovery_backend': recovery_backend, 'data_dir': str(data_root()), 'config_dir': str(config_root()), 'configured': settings_path().exists(), @@ -285,7 +303,10 @@ def _served_lines(served: list[tuple[str, str, str, str]]) -> list[str]: def _print_status_plain(d: dict[str, Any]) -> None: print('infer-stack status') - print(f' backend: {d["backend"]}') + print(f' configured backend: {d["backend"]}') + print(f' active recovery backend: {d["recovery_backend"] or "(none — no active snapshot)"}') + if d['recovery_backend'] and d['recovery_backend'] != d['backend']: + print(' transition required: quiesce the old backend, then infer-stack ledger rotate --yes') print(f' data dir: {d["data_dir"]}') print(f' config dir: {d["config_dir"]}') cat = d['catalog'] @@ -328,7 +349,10 @@ def _print_status_rich(d: dict[str, Any], console) -> None: table.add_column(style='bold', justify='left', no_wrap=True) table.add_column(overflow='fold') - table.add_row('backend', Text(d['backend'], style='bold cyan')) + table.add_row('configured backend', Text(d['backend'], style='bold cyan')) + table.add_row('active recovery backend', Text(d['recovery_backend'] or '(none — no active snapshot)', style='cyan')) + if d['recovery_backend'] and d['recovery_backend'] != d['backend']: + table.add_row('transition required', 'Quiesce the old backend, then infer-stack ledger rotate --yes') table.add_row('data dir', Text(d['data_dir'], style='cyan')) table.add_row('config dir', Text(d['config_dir'], style='cyan')) cat = d['catalog'] diff --git a/infer_stack/gpu_doctor.py b/infer_stack/gpu_doctor.py index 3e0e4960..69754684 100644 --- a/infer_stack/gpu_doctor.py +++ b/infer_stack/gpu_doctor.py @@ -32,7 +32,7 @@ import re import subprocess -from dataclasses import dataclass, field +from dataclasses import dataclass __all__ = ['GpuSample', 'Holder', 'gpu_checks'] @@ -210,10 +210,12 @@ def gpu_checks(use_sudo: bool = False, *, _sample=None, _apps=None, holders = _holders() if _holders else device_holders(use_sudo) if holders is None: - yield ('device holders', True, + yield ('device holders', False, 'not checked — needs root, and unprivileged this can only see ' 'your own processes, which would read as a false all-clear. ' - 'Re-run with --sudo.') + + ('Sudo was requested but non-interactive elevation or the scan failed. ' + 'Run sudo -v, then retry with --sudo.' if use_sudo else + 'Re-run with --sudo.')) return unexpected = [h for h in holders if not h.expected] diff --git a/infer_stack/kube/inspect.py b/infer_stack/kube/inspect.py new file mode 100644 index 00000000..afa6ce74 --- /dev/null +++ b/infer_stack/kube/inspect.py @@ -0,0 +1,191 @@ +"""Best-effort Kubernetes facts and dependency-ordered readiness policy. + +Each probe owns its error. In particular, chart configuration is never a +prerequisite for node discovery or proposed resource profiles. +""" +from __future__ import annotations + +import csv +import io +from typing import Any + +import requests +import yaml + +from ..backends.kubeai import node_gpus +from .manage import KubeManager + + +def api_probe(base_url: str, http: Any = requests) -> dict: + try: + code = http.get(base_url.rstrip('/') + '/models', timeout=5).status_code + return {'reachable': code == 200, 'status_code': code, 'base_url': base_url} + except Exception as ex: # diagnostic boundary + return {'reachable': False, 'error': str(ex), 'base_url': base_url} + + +def inventory(manager: KubeManager, *, namespace='kubeai', release='kubeai', + base_url='http://127.0.0.1:8000/openai/v1', http=requests) -> dict: + report: dict[str, Any] = { + 'tools': {name: manager.command_exists(name) for name in ('kubectl', 'helm')}, + 'cluster': {'context': '', 'reachable': False, 'distribution': None}, + 'host_gpus': None, 'gpu_count': 0, 'nodes': [], 'runtime_classes': None, 'device_plugin': None, + 'kubeai': {'namespace': namespace, 'release_name': release, 'crd': None, + 'namespace_exists': None, 'release': None, 'pods': None, + 'services': None, 'models': None, + 'api': {'base_url': base_url, 'reachable': False, 'blocked': 'Cluster unavailable'}}, + 'resource_profiles': {'proposed': {}, 'installed': {}}, 'errors': {}, + } + + def probe(key, func, default=None): + try: + return func() + except Exception as ex: # independent, best-effort diagnostic boundary + report['errors'][key] = str(ex) + return default + + if manager.command_exists('nvidia-smi'): + def host_gpus(): + text = manager.run(['nvidia-smi', '--query-gpu=index,name,memory.total', + '--format=csv,noheader,nounits']) + return [{'index': int(row[0]), 'product': row[1].strip(), + 'memory_mib': int(row[2])} + for row in csv.reader(io.StringIO(text)) if row] + report['host_gpus'] = probe('host_gpus', host_gpus) + if not report['tools']['kubectl']: + report['errors']['cluster'] = 'kubectl unavailable' + return report + report['cluster']['context'] = manager.current_context() + reachable, detail = manager.cluster_reachable() + report['cluster']['reachable'] = reachable + if not reachable: + report['errors']['cluster'] = detail + return report + nodes = probe('nodes', manager.nodes, []) + report['nodes'] = manager.node_rows(nodes) + facts = node_gpus(nodes) + for row in report['nodes']: + row['gfd_labels'] = bool(row['gpu_product'] and row['gpu_memory_gib']) + row['kubelet_version'] = next( + ((n.get('status', {}).get('nodeInfo') or {}).get('kubeletVersion', '') + for n in nodes if n.get('metadata', {}).get('name') == row['name']), '') + versions = [r['kubelet_version'] for r in report['nodes']] + report['cluster']['distribution'] = ('k3s' if any('k3s' in v for v in versions) + else 'unknown') + runtime = probe('runtime_classes', lambda: manager.kubectl_json( + ['get', 'runtimeclasses', '-o', 'json'])) + if runtime is not None: + report['runtime_classes'] = [i.get('metadata', {}).get('name') + for i in runtime.get('items', [])] + runtime_ok = 'nvidia' in (report['runtime_classes'] or []) + for row in report['nodes']: + row['nvidia_runtime_class'] = runtime_ok + ds = probe('device_plugin', lambda: manager.kubectl_json( + ['get', 'daemonsets', '-A', '-o', 'json'])) + if ds is not None: + plugins = [] + for item in ds.get('items', []): + containers = item.get('spec', {}).get('template', {}).get('spec', {}).get('containers', []) + if any('k8s-device-plugin' in c.get('image', '') for c in containers): + status = item.get('status', {}) + desired = status.get('desiredNumberScheduled', 0) + plugins.append({'name': manager._pod_name(item), 'desired': desired, + 'ready': status.get('numberReady', 0), + 'healthy': desired > 0 and status.get('numberReady', 0) == desired}) + report['device_plugin'] = plugins + report['resource_profiles']['proposed'] = manager.resource_profiles( + nodes, runtime_class_name='nvidia' if runtime_ok else None) + kubeai = report['kubeai'] + # --ignore-not-found distinguishes absence from permission/transport errors. + kubeai['crd'] = probe('crd', lambda: bool(manager.run([ + 'kubectl', 'get', 'crd', 'models.kubeai.org', '--ignore-not-found', '-o', 'name']).strip())) + kubeai['namespace_exists'] = probe('namespace', lambda: bool(manager.run([ + 'kubectl', 'get', 'namespace', namespace, '--ignore-not-found', '-o', 'name']).strip())) + kubeai['release'] = None + if report['tools']['helm']: + releases = probe('helm_releases', manager.helm_releases, []) + kubeai['release'] = next((r for r in releases if r.get('name') == release + and r.get('namespace') == namespace), None) + if kubeai['namespace_exists']: + for resource in ('pods', 'services'): + result = probe(resource, lambda resource=resource: manager.kubectl_json( + ['-n', namespace, 'get', resource, '-o', 'json'])) + kubeai[resource] = result.get('items', []) if result is not None else None + if kubeai['crd']: + result = probe('models', lambda: manager.kubectl_json([ + '-n', namespace, 'get', 'models.kubeai.org', + '-l', 'infer-stack/managed=true', '-o', 'json'])) + kubeai['models'] = result.get('items', []) if result is not None else None + config = probe('installed_profiles', lambda: manager.kubectl_json([ + '-n', namespace, 'get', 'configmap', 'kubeai-config', + '--ignore-not-found', '-o', 'json']), {}) + def installed_profiles(): + system = yaml.safe_load((config or {}).get('data', {}).get('system.yaml', '')) or {} + if not isinstance(system, dict): + raise ValueError('kubeai-config system.yaml must contain a mapping') + profiles = system.get('resourceProfiles') or {} + if not isinstance(profiles, dict): + raise ValueError('Installed resourceProfiles must contain a mapping') + return profiles + report['resource_profiles']['installed'] = probe('profile_yaml', installed_profiles, {}) + kubeai['api'] = api_probe(base_url, http) if kubeai['namespace_exists'] else { + 'base_url': base_url, 'reachable': False, 'blocked': 'KubeAI namespace missing'} + report['gpu_count'] = sum(f['count'] for f in facts.values()) + return report + + +def readiness(report: dict, *, installation=True, gpu=True) -> list[dict]: + """Policy over inventory; stop dependent chains at their first blocker.""" + checks = [] + + def check(name, ok, fix='', detail='', blocked=False): + checks.append({'name': name, 'ok': bool(ok), 'status': 'blocked' if blocked else + ('ok' if ok else 'fail'), 'detail': detail, 'fix': fix if not ok else ''}) + return ok + + if not check('kubectl available', report['tools']['kubectl'], + 'infer-stack kube bootstrap --provider=k3s'): + return checks + check('Helm available', report['tools']['helm'], 'infer-stack kube bootstrap') + if not check('cluster reachable', report['cluster']['reachable'], + 'Select a reachable kubeconfig/context or run infer-stack kube bootstrap', + report['errors'].get('cluster', '')): + return checks + rows = report['nodes'] + check('nodes Ready', bool(rows) and all(r['ready'] for r in rows), + 'infer-stack kube bootstrap', report['errors'].get('nodes', '')) + if gpu: + gpu_rows = [r for r in rows if r['gpu_count']] + runtime_ok = 'nvidia' in (report['runtime_classes'] or []) + # Working external integrations need not use a named NVIDIA RuntimeClass. + check('NVIDIA runtime', runtime_ok or bool(gpu_rows), + 'Install NVIDIA driver/container toolkit on GPU nodes; then infer-stack kube bootstrap', + 'RuntimeClass nvidia' if runtime_ok else 'No nvidia RuntimeClass; GPU allocation proves an external integration' if gpu_rows else '') + plugin = report['device_plugin'] + check('NVIDIA device plugin', bool(plugin) and all(p['healthy'] for p in plugin), + 'infer-stack kube bootstrap', report['errors'].get('device_plugin', '')) + if check('nvidia.com/gpu allocatable', bool(gpu_rows), 'infer-stack kube bootstrap'): + check('GFD product + memory labels', all(r['gfd_labels'] for r in gpu_rows), + 'infer-stack kube bootstrap') + else: + check('GFD product + memory labels', False, blocked=True, + detail='Waiting for allocatable GPUs') + if not installation: + return checks + kubeai = report['kubeai'] + fix = f"infer-stack kube install --namespace={kubeai['namespace']} --release={kubeai['release_name']}" + check('KubeAI Model CRD', kubeai.get('crd'), fix, report['errors'].get('crd', '')) + ns = check('KubeAI namespace', kubeai.get('namespace_exists'), fix, + report['errors'].get('namespace', '')) + release = kubeai.get('release') + check('KubeAI Helm release', bool(release) and release.get('status') == 'deployed', + fix, report['errors'].get('helm_releases', '')) + api = kubeai.get('api', {}) + check('KubeAI API', api.get('reachable'), + 'infer-stack config set kubeai_base_url ; inspect infer-stack kube status', + api.get('error', api.get('blocked', api.get('base_url', ''))), blocked=not ns) + return checks + + +def failed(checks: list[dict]) -> bool: + return any(c['status'] == 'fail' for c in checks) diff --git a/infer_stack/kube/k3s.py b/infer_stack/kube/k3s.py index 8143896d..f59708d3 100644 --- a/infer_stack/kube/k3s.py +++ b/infer_stack/kube/k3s.py @@ -49,6 +49,8 @@ def _ensure_default_kubeconfig_link() -> bool: Existing kubeconfig state is operator authority and is never replaced. """ + if os.environ.get('KUBECONFIG'): + return os.environ['KUBECONFIG'] == str(K3S_KUBECONFIG) target = Path.home() / '.kube' / 'config' if target.exists() or target.is_symlink(): if target.is_symlink(): @@ -66,7 +68,7 @@ def _ensure_helm(run: RunFunc) -> None: if shutil.which('helm') is not None: return installer = run(['curl', '-fsSL', HELM_INSTALL_URL]) - run(['bash', '-s', '--'], input_text=installer) + run(['sudo', '-n', 'bash', '-s', '--'], input_text=installer) def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool: @@ -74,10 +76,12 @@ def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool NVIDIA drivers/container runtime are intentionally not installed here. K3s detects an already-installed NVIDIA container runtime when it starts; - ``infer-stack kube setup`` reports the resulting RuntimeClass/capabilities. + ``infer-stack kube doctor`` reports the resulting RuntimeClass/capabilities. """ run = run or default_run _require_local_tool('sudo') + if _active(run, 'k3s-agent'): + raise RuntimeError('This host is a K3s agent; refusing to replace it with a server') already_active = _active(run, 'k3s') if already_active and version: installed = run(['k3s', '--version']).splitlines()[0] @@ -94,16 +98,20 @@ def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool file.write('write-kubeconfig-mode: "0644"\n') source = file.name try: - run(['sudo', 'mkdir', '-p', '/etc/rancher/k3s/config.yaml.d']) + run(['sudo', '-n', 'mkdir', '-p', '/etc/rancher/k3s/config.yaml.d']) run([ - 'sudo', 'install', '-m', '0644', source, + 'sudo', '-n', 'install', '-m', '0644', source, '/etc/rancher/k3s/config.yaml.d/10-infer-stack-kubeconfig-mode.yaml', ]) finally: Path(source).unlink(missing_ok=True) if already_active: - run(['sudo', 'systemctl', 'restart', 'k3s']) + # Updating the mode fragment does not require restarting a working + # control plane. Repair current access directly; next start uses it too. + run(['sudo', '-n', 'chmod', '0644', str(K3S_KUBECONFIG)]) + elif shutil.which('k3s') is not None: + run(['sudo', '-n', 'systemctl', 'start', 'k3s']) else: installer = _fetch_installer(run) env = os.environ.copy() @@ -111,7 +119,7 @@ def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool if version: env['INSTALL_K3S_VERSION'] = version preserve.append('INSTALL_K3S_VERSION') - cmd = ['sudo'] + cmd = ['sudo', '-n'] if preserve: cmd.append('--preserve-env=' + ','.join(preserve)) cmd.extend(['sh', '-']) diff --git a/infer_stack/kube/manage.py b/infer_stack/kube/manage.py index be42b73d..5471d1e8 100644 --- a/infer_stack/kube/manage.py +++ b/infer_stack/kube/manage.py @@ -10,10 +10,8 @@ import copy import json -import os import shutil import subprocess -import tempfile from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable @@ -49,6 +47,7 @@ def default_run( env=env, text=True, capture_output=True, + timeout=360 if ('upgrade' in args or 'wait' in args) else 60, check=False, ) if proc.returncode: @@ -385,11 +384,6 @@ def gpu_summary(nodes: list[dict[str, Any]]) -> tuple[bool, bool, str]: """``(resource exposed, discovery labels complete, detail)``.""" facts = node_gpus(nodes) gpu_nodes = {name: f for name, f in facts.items() if f.get('count')} - resource_ok = bool(gpu_nodes) - labels_ok = bool(gpu_nodes) and all( - f.get('product') and f.get('memory_gib') - for f in gpu_nodes.values() - ) if not gpu_nodes: return False, False, 'no node exposes allocatable nvidia.com/gpu' total = sum(int(f.get('count') or 0) for f in gpu_nodes.values()) @@ -419,14 +413,16 @@ def _existing_kubeai_values( found = self.kubeai_release(release) if found is None or str(found.get('namespace') or '') != namespace: return {} - try: - text = self.run([ - 'helm', 'get', 'values', release, - '-n', namespace, '-o', 'yaml', - ]) - except Exception: - return {} - return yaml.safe_load(text or '') or {} + # An unreadable existing release is not an empty values mapping: + # silently treating it as one would discard operator configuration. + text = self.run([ + 'helm', 'get', 'values', release, + '-n', namespace, '-o', 'yaml', + ]) + values = yaml.safe_load(text or '') or {} + if not isinstance(values, dict): + raise RuntimeError('Installed KubeAI Helm values must be a mapping') + return values @staticmethod def load_values_file(path: str | Path | None) -> dict[str, Any]: @@ -715,55 +711,16 @@ def install_kubeai( resource_profiles: dict[str, Any], version: str | None = None, operator_values: dict[str, Any] | None = None, + chart: str = 'kubeai/kubeai', ) -> Path: - self.run([ - 'helm', 'repo', 'add', 'kubeai', KUBEAI_REPO, '--force-update' - ]) - self.run(['helm', 'repo', 'update']) - base_values = self._deep_merge( - self._existing_kubeai_values(release, namespace), - operator_values or {}, - ) - public_values, secret_values = self._split_secret_values(base_values) - values = self._merge_profiles(public_values, resource_profiles) - values_path = data_root() / 'generated' / 'kube' / 'kubeai-values.yaml' - values_path.parent.mkdir(parents=True, exist_ok=True) - values_path.write_text( - yaml.safe_dump(values, sort_keys=False), encoding='utf-8' - ) + from ..kubeai_ops import install_chart - cmd = [ - 'helm', 'upgrade', '--install', release, 'kubeai/kubeai', - '-n', namespace, '--create-namespace', - '-f', str(values_path), '--wait', - ] - if version: - cmd.extend(['--version', version]) - - # Keep chart secrets out of persistent generated YAML and process argv. - # Existing/operator secret values are preserved in-memory; HF_TOKEN, if - # exported, deliberately overrides only the Hugging Face token. - token = os.environ.get('HF_TOKEN', '').strip() - if token: - huggingface = secret_values.setdefault('secrets', {}).setdefault( - 'huggingface', {} - ) - huggingface['token'] = token - secret_path: str | None = None - try: - if secret_values: - with tempfile.NamedTemporaryFile( - 'w', prefix='infer-stack-kubeai-secret-', - suffix='.yaml', delete=False, - ) as file: - os.chmod(file.name, 0o600) - yaml.safe_dump(secret_values, file, sort_keys=False) - secret_path = file.name - cmd.extend(['-f', secret_path]) - self.run(cmd) - finally: - if secret_path: - Path(secret_path).unlink(missing_ok=True) + values = self._merge_profiles(self._deep_merge( + self._existing_kubeai_values(release, namespace), operator_values or {}, + ), resource_profiles) + values_path = data_root() / 'generated' / 'kube' / 'kubeai-values.yaml' + install_chart(values=values, values_path=values_path, namespace=namespace, + release=release, chart=chart, version=version, run=self.run) return values_path def apply_setup( diff --git a/infer_stack/kube/operations.py b/infer_stack/kube/operations.py new file mode 100644 index 00000000..fbc51a64 --- /dev/null +++ b/infer_stack/kube/operations.py @@ -0,0 +1,108 @@ +"""Converge prerequisites and KubeAI using the shared inventory policy.""" +from __future__ import annotations + +import time + +from . import k3s +from .inspect import failed, inventory, readiness +from .manage import KubeManager + + +class K3sProvider: + """Host provisioning adapter; generic inventory never assumes K3s.""" + + def plan(self, manager: KubeManager, report: dict) -> list[str]: + actions = [] + if not report['cluster']['reachable']: + actions.append('Install/start K3s only if no existing cluster/service; establish kubeconfig access') + else: + actions.append('Preserve the selected working cluster and kubeconfig') + if not report['tools']['helm']: + actions.append('Install Helm') + if not report.get('gpu_count'): + actions.append('Verify installed NVIDIA container runtime; restart local K3s only if runtime discovery needs it') + if failed(readiness(report, installation=False)): + actions.append('Reconcile NVIDIA device plugin 0.17.1 + GFD when runtime is available') + actions.append('Wait for Ready nodes, allocatable GPUs and GFD labels; run prerequisite doctor') + return actions + + def apply(self, manager: KubeManager, report: dict, *, version=None, timeout=180) -> dict: + if not report['cluster']['reachable']: + # A configured but broken context is operator authority. Do not silently + # provision a second cluster beneath an unrelated kubeconfig. + if report['cluster']['context']: + raise RuntimeError('Selected context is unreachable; repair/select kubeconfig before bootstrap') + if not k3s.bootstrap(version=version, run=manager.run): + raise RuntimeError('Existing kubeconfig preserved. Select the K3s context, then retry bootstrap') + elif not report['tools']['helm']: + k3s._ensure_helm(manager.run) + fresh = inventory(manager) + if not fresh.get('gpu_count') and (manager.command_exists('k3s') + or 'nvidia' not in (fresh['runtime_classes'] or [])): + if not k3s._active(manager.run, 'k3s'): + raise RuntimeError('Configure NVIDIA runtime on this cluster provider; local K3s is not active') + if not manager.command_exists('nvidia-container-runtime'): + raise RuntimeError('Install NVIDIA driver and nvidia-container-toolkit, then retry infer-stack kube bootstrap') + selected_uid = manager.run(['kubectl', 'get', 'namespace', 'kube-system', + '-o', 'jsonpath={.metadata.uid}']).strip() + local_uid = manager.run(['sudo', '-n', 'k3s', 'kubectl', 'get', + 'namespace', 'kube-system', '-o', + 'jsonpath={.metadata.uid}']).strip() + if not selected_uid or selected_uid != local_uid: + raise RuntimeError('Selected cluster differs from local K3s; configure runtime on its GPU nodes') + config_path = '/var/lib/rancher/k3s/agent/etc/containerd/config.toml' + config = manager.run(['sudo', '-n', 'cat', config_path]) + if 'nvidia-container-runtime' not in config: + manager.run(['sudo', '-n', 'systemctl', 'restart', 'k3s']) + deadline = time.monotonic() + timeout + while True: + fresh = inventory(manager) + if fresh['cluster']['reachable'] and 'nvidia' in (fresh['runtime_classes'] or []): + break + if time.monotonic() >= deadline: + raise RuntimeError('K3s runtime discovery timed out; ensure nvidia-container-runtime is in the service PATH') + time.sleep(min(2, max(0, deadline - time.monotonic()))) + config = manager.run(['sudo', '-n', 'cat', + '/var/lib/rancher/k3s/agent/etc/containerd/config.toml']) + if 'nvidia-container-runtime' not in config or 'nvidia' not in (fresh['runtime_classes'] or []): + raise RuntimeError('K3s did not detect NVIDIA runtime; ensure nvidia-container-runtime is in the K3s service PATH') + manager.run(['kubectl', 'wait', '--for=condition=Ready', 'node', '--all', f'--timeout={timeout}s']) + plugin = fresh.get('device_plugin') or [] + if (not plugin or not all(p['healthy'] for p in plugin) + or not fresh.get('gpu_count') + or not all(r['gfd_labels'] for r in fresh['nodes'] if r['gpu_count'])): + if 'nvidia' not in (fresh['runtime_classes'] or []): + raise RuntimeError('NVIDIA chart remediation requires RuntimeClass nvidia; configure node runtimes first') + manager.install_nvidia_device_plugin() + deadline = time.monotonic() + timeout + while True: + fresh = inventory(manager) + if not failed(readiness(fresh, installation=False)): + return fresh + if time.monotonic() >= deadline: + raise RuntimeError('GPU readiness timed out; run infer-stack kube doctor') + time.sleep(min(2, max(0, deadline - time.monotonic()))) + + +def install_plan(manager: KubeManager, report: dict, *, operator_values=None) -> dict: + kubeai = report['kubeai'] + values = manager._merge_profiles(manager._deep_merge( + manager._existing_kubeai_values(kubeai['release_name'], kubeai['namespace']), + operator_values or {}), report['resource_profiles']['proposed']) + public, _ = manager._split_secret_values(values) + return public + + +def install(manager: KubeManager, report: dict, *, operator_values=None, + chart='kubeai/kubeai', version=None, gpu=True) -> None: + checks = readiness(report, installation=False, gpu=gpu) + if report['errors'].get('helm_releases'): + raise RuntimeError('Cannot inspect existing Helm releases; repair Helm access before installation') + if failed(checks): + names = ', '.join(c['name'] for c in checks if c['status'] == 'fail') + raise RuntimeError(f'KubeAI prerequisites failed: {names}. Run infer-stack kube doctor / kube bootstrap') + kubeai = report['kubeai'] + manager.install_kubeai(namespace=kubeai['namespace'], release=kubeai['release_name'], + resource_profiles=report['resource_profiles']['proposed'] if gpu else {}, + operator_values=operator_values, chart=chart, + version=version or manager._release_chart_version(kubeai.get('release'))) diff --git a/infer_stack/kubeai_ops.py b/infer_stack/kubeai_ops.py index be51cb3a..5d9a298f 100644 --- a/infer_stack/kubeai_ops.py +++ b/infer_stack/kubeai_ops.py @@ -1,9 +1,13 @@ from __future__ import annotations +import copy import os import subprocess +import tempfile from pathlib import Path +import yaml + from .config import KUBEAI_GENERATED_SUBDIR, normalized_output @@ -34,35 +38,9 @@ def deploy_rendered_artifacts(deployment: dict) -> None: ingress_file = generated / 'ingress.yaml' run(['kubectl', 'apply', '-f', str(namespace_file)]) - run( - [ - 'helm', - 'repo', - 'add', - 'kubeai', - 'https://www.kubeai.org', - '--force-update', - ] - ) - run(['helm', 'repo', 'update']) - - helm_cmd = [ - 'helm', - 'upgrade', - '--install', - release_name, - chart, - '-n', - namespace, - '--create-namespace', - '-f', - str(values_file), - '--wait', - ] - hf_token = os.environ.get('HF_TOKEN', '').strip() - if hf_token: - helm_cmd.extend(['--set', f'secrets.huggingface.token={hf_token}']) - run(helm_cmd) + install_chart(values=yaml.safe_load(values_file.read_text()) or {}, + values_path=values_file, namespace=namespace, + release=release_name, chart=chart, run=run) run(['kubectl', 'apply', '-f', str(models_file)]) if ingress_file.exists(): @@ -74,3 +52,35 @@ def print_status(namespace: str) -> None: run(['kubectl', '-n', namespace, 'get', 'svc']) run(['kubectl', '-n', namespace, 'get', 'ingress']) run(['kubectl', '-n', namespace, 'get', 'models']) + + +def install_chart(*, values: dict, values_path: Path, namespace: str, + release: str, chart: str = 'kubeai/kubeai', + version: str | None = None, run=run) -> None: + """Shared Helm upgrade authority; secrets live only in a temporary 0600 file.""" + public = copy.deepcopy(values) + secrets = public.pop('secrets', {}) or {} + token = os.environ.get('HF_TOKEN', '').strip() + if token: + secrets.setdefault('huggingface', {})['token'] = token + values_path.parent.mkdir(parents=True, exist_ok=True) + values_path.write_text(yaml.safe_dump(public, sort_keys=False), encoding='utf-8') + run(['helm', 'repo', 'add', 'kubeai', 'https://www.kubeai.org', '--force-update']) + run(['helm', 'repo', 'update']) + cmd = ['helm', 'upgrade', '--install', release, chart, '-n', namespace, + '--create-namespace', '-f', str(values_path), '--wait', '--timeout=300s'] + if version: + cmd.extend(['--version', version]) + secret_path = None + try: + if secrets: + with tempfile.NamedTemporaryFile('w', prefix='infer-stack-kubeai-secret-', + suffix='.yaml', delete=False) as file: + os.chmod(file.name, 0o600) + yaml.safe_dump({'secrets': secrets}, file) + secret_path = file.name + cmd.extend(['-f', secret_path]) + run(cmd) + finally: + if secret_path: + Path(secret_path).unlink(missing_ok=True) diff --git a/infer_stack/leasing/compose.py b/infer_stack/leasing/compose.py index 8ebc845a..d8d68660 100644 --- a/infer_stack/leasing/compose.py +++ b/infer_stack/leasing/compose.py @@ -2377,7 +2377,9 @@ def use_profile(self, profile: dict[str, Any]) -> None: raise ProfileMismatch( f"the active recovery snapshot is for the {profile.get('backend')!r} backend; " - 'tear down the old backend before switching backend kinds' + 'quiesce the old backend (release leases and tear down workloads), then ' + 'start a new ledger/recovery epoch with `infer-stack ledger rotate --yes` ' + 'for the configured backend; stack down alone retains the old snapshot' ) self.project = profile['project'] self.litellm = profile['litellm'] diff --git a/infer_stack/leasing/store.py b/infer_stack/leasing/store.py index f1e6c1c6..37b18193 100644 --- a/infer_stack/leasing/store.py +++ b/infer_stack/leasing/store.py @@ -234,6 +234,61 @@ def _add_missing_columns(self) -> None: if 'duplicate column' not in str(ex).lower(): raise + def backup_to(self, path: Path) -> None: + """Publish a consistent SQLite archive, including committed WAL contents. + + The caller holds the controller publication lock. A temporary file plus + atomic replace ensures an interrupted backup is never a valid archive. + """ + import os + import tempfile + + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix='.ledger-backup-', dir=path.parent) + os.close(fd) + try: + with self._lock, contextlib.closing(sqlite3.connect(temporary)) as destination: + self._conn._conn.backup(destination) + with open(temporary, 'rb') as file: + os.fsync(file.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + Path(temporary).unlink(missing_ok=True) + + def start_backend_epoch(self, profile: dict, archive: dict, *, + expected_profile: dict, expected_version: int) -> None: + """Atomically replace archived history with a new backend snapshot. + + Keep the same database inode: other processes may already hold SQLite + connections. Replacing/renaming the live DB would strand those writers + in the old epoch. The new profile rejects them at their next mutation. + """ + from .profile import ProfileMismatch + + with self.transaction() as conn: + if (self.profile() != expected_profile + or self.admission_state_version() != expected_version): + raise ProfileMismatch('Ledger changed during rotation; review and retry') + archives = self.meta_json('ledger_archives', []) + conn.execute('DELETE FROM claims') + conn.execute('DELETE FROM leases') + conn.execute('DELETE FROM deployments') + conn.execute('DELETE FROM service_addresses') + conn.execute("DELETE FROM meta WHERE key NOT IN " + "('schema_version', 'desired_gen', 'applied_gen', " + "'admission_state_version', 'ledger_archives')") + conn.execute( + "INSERT INTO meta(key, value) VALUES ('ledger_archives', ?) " + 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', + (_dumps([*archives, archive]),)) + self._write_profile(profile) + self.bump_desired_generation() + def close(self) -> None: """Close the owned sqlite connection. Safe to call more than once.""" self._conn.close() diff --git a/infer_stack/leasing/transition.py b/infer_stack/leasing/transition.py new file mode 100644 index 00000000..db34465b --- /dev/null +++ b/infer_stack/leasing/transition.py @@ -0,0 +1,76 @@ +"""Explicit, archived backend transitions under the publication lock.""" +from __future__ import annotations + +from pathlib import Path + +from .models import LeaseState +from .profile import CatalogUnion, ProfileMismatch, canonical_digest + + +def rotate_backend(controller, old_backend, *, apply=False) -> dict: + """Verify the frozen backend is quiescent, archive history, start a new epoch. + + Preview and commit both recheck leases and strict runtime observations. No + observe() fallback is allowed to interpret a probe failure as quiescence. + Archive-before-commit and one SQLite reset transaction make interruptions + safe: before commit the old epoch remains intact; afterward retries are no-op. + """ + store = controller.ledger.store + target = controller.invocation_profile() + if target is None or target.get('backend') not in {'compose', 'kubeai'}: + raise ProfileMismatch('ledger rotate requires a configured Compose or KubeAI backend') + # Validate proposed catalogs without publishing them or changing config files. + if target.get('catalogs'): + CatalogUnion.from_sources(target['catalogs']) + with controller._global_lock(): + current = store.profile() + archives = store.meta_json('ledger_archives', []) + result = {'from_backend': (current or {}).get('backend'), + 'to_backend': target['backend'], 'changed': False, + 'archive': None, 'archives': archives} + if current is None: + raise ProfileMismatch('No active recovery snapshot to rotate; acquire normally starts one') + if current.get('backend') == target['backend']: + result['detail'] = 'Recovery backend already matches the configured backend; no rotation needed' + return result + leases, deployments = controller.ledger.status(virtual_expiry=True) + active = [le.id for le in leases if le.state == LeaseState.ACTIVE] + if active: + raise ProfileMismatch('ledger rotate refused: active leases remain: ' + ', '.join(active) + + f". Release them using --backend={current['backend']} first") + if old_backend is None: + raise ProfileMismatch('Cannot verify the old backend; rotation refused') + old_backend.recovery_profile.use_profile(current) + try: + blockers = getattr(old_backend, 'recovery_blockers', None) + running = blockers() if blockers else [i.name for i in old_backend.instances()] + except Exception as ex: + raise ProfileMismatch(f'Cannot verify old backend quiescence: {ex}; rotation refused') from ex + if running: + raise ProfileMismatch('ledger rotate refused: old backend still owns runtime objects: ' + + ', '.join(running) + + f". Run infer-stack stack down --backend={current['backend']}") + version = store.admission_state_version() + signature = canonical_digest({'profile': current, 'state_version': version})[:16] + if store.path == ':memory:': + raise ProfileMismatch('ledger rotate requires a file-backed ledger for history archives') + path = Path(store.path).resolve() + archive_path = path.parent / 'archives' / f'{path.stem}-{current["backend"]}-{signature}.db' + result.update(archive=str(archive_path), leases=len(leases), deployments=len(deployments), + detail='Old runtime is quiescent; archive history and start a new recovery epoch') + if not apply: + return result + # Prepare the target backend before touching the old epoch. Catalogs are + # imported by the next acquire; no workloads/routes are started here. + fresh = {**target, 'catalogs': []} + controller._use_profile_candidate(fresh) + store.backup_to(archive_path) + archive = {'path': str(archive_path), 'backend': current['backend'], + 'next_backend': target['backend'], 'leases': len(leases), + 'deployments': len(deployments), 'created_at': controller.clock()} + store.start_backend_epoch(fresh, archive, expected_profile=current, expected_version=version) + controller._profile_error = None + controller._applied_profile = fresh + controller._fresh_profile = None + result.update(changed=True, archives=[*archives, archive]) + return result diff --git a/infer_stack/profile_runtime.py b/infer_stack/profile_runtime.py index e5f7a0cc..96b203a0 100644 --- a/infer_stack/profile_runtime.py +++ b/infer_stack/profile_runtime.py @@ -111,9 +111,11 @@ def simulator_args(service: dict[str, Any]) -> list[str]: return args -def vllm_args(service: dict[str, Any]) -> list[str]: - args = [ - f'--served-model-name={service["served_model_name"]}', +def vllm_args(service: dict[str, Any], *, include_served_name: bool = True) -> list[str]: + """Engine flags; KubeAI owns the served name through its Model identity.""" + args = ([f'--served-model-name={service["served_model_name"]}'] + if include_served_name else []) + args += [ f'--tensor-parallel-size={service["tensor_parallel_size"]}', # .get: older KubeAI lock data predates these keys. f'--pipeline-parallel-size={service.get("pipeline_parallel_size", 1)}', diff --git a/scripts/bootstrap_k3s.sh b/scripts/bootstrap_k3s.sh index fcf27fc7..c4c00ca0 100755 --- a/scripts/bootstrap_k3s.sh +++ b/scripts/bootstrap_k3s.sh @@ -4,8 +4,8 @@ set -euo pipefail # Compatibility wrapper. The implementation lives in infer-stack now so the # CLI, tests, and docs share one setup authority. ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ARGS=(kube k3s bootstrap) +ARGS=(kube bootstrap --provider=k3s --apply) if [[ -n "${INSTALL_K3S_VERSION:-}" ]]; then ARGS+=(--version="${INSTALL_K3S_VERSION}") fi -exec python "$ROOT/manage.py" "${ARGS[@]}" "$@" +exec python3 "$ROOT/manage.py" "${ARGS[@]}" "$@" diff --git a/scripts/install_kubeai.sh b/scripts/install_kubeai.sh index 238223b0..635862a0 100644 --- a/scripts/install_kubeai.sh +++ b/scripts/install_kubeai.sh @@ -1,27 +1,14 @@ #!/usr/bin/env bash set -euo pipefail - -VALUES_FILE="${1:-generated/kubeai/kubeai-values.yaml}" -NAMESPACE="${2:-kubeai}" - -if ! command -v helm >/dev/null 2>&1; then - echo "helm is required" >&2 - exit 1 -fi -if ! command -v kubectl >/dev/null 2>&1; then - echo "kubectl is required" >&2 - exit 1 -fi -if [[ ! -f "$VALUES_FILE" ]]; then - echo "missing values file: $VALUES_FILE" >&2 - exit 1 -fi - -helm repo add kubeai https://www.kubeai.org --force-update -helm repo update - -CMD=(helm upgrade --install kubeai kubeai/kubeai -n "$NAMESPACE" --create-namespace -f "$VALUES_FILE" --wait) -if [[ -n "${HF_TOKEN:-}" ]]; then - CMD+=(--set "secrets.huggingface.token=${HF_TOKEN}") -fi -"${CMD[@]}" +# Compatibility with the old optional VALUES_FILE NAMESPACE positional arguments. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ARGS=(kube install --apply) +if [[ $# -gt 0 && "$1" != --* ]]; then + ARGS+=(--values "$1") + shift + if [[ $# -gt 0 && "$1" != --* ]]; then + ARGS+=(--namespace "$1") + shift + fi +fi +exec python3 "$ROOT/manage.py" "${ARGS[@]}" "$@" diff --git a/tests/test_backend_transition.py b/tests/test_backend_transition.py new file mode 100644 index 00000000..2d1f3c8c --- /dev/null +++ b/tests/test_backend_transition.py @@ -0,0 +1,201 @@ +"""Real Compose/KubeAI controllers with stateful fake runtimes and SQLite WAL.""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml +from test_leasing_compose import FakeDocker +from test_leasing_kubeai import FakeHttp, FakeKubectl +from test_leasing_profile import backend as compose_backend + +from infer_stack.backends.kubeai import KubeaiBackend +from infer_stack.cli import ManageCLI +from infer_stack.cli.commands_leasing import AcquireCLI, GcCLI +from infer_stack.cli.commands_ledger import LedgerRotateCLI +from infer_stack.cli.commands_runtime import StatusCLI +from infer_stack.leasing import Catalog, Controller, Ledger, SqliteStore +from infer_stack.leasing.profile import ProfileMismatch +from infer_stack.leasing.transition import rotate_backend +from infer_stack.paths import config_root, data_root + + +@pytest.fixture +def environment(monkeypatch): + from infer_stack._log import logger + logger.disable('infer_stack') + monkeypatch.setattr('infer_stack._log.configure_logging', lambda *args, **kwargs: None) + from infer_stack.leasing import default_ledger_path + catalog_data = {'models': {'m': {'source': 'hf://org/model'}}, 'endpoints': { + 'smol135-kube': {'engine': 'vllm', 'model': 'm', + 'runtime': {'resource_profile': 'gpu-single-default'}}}} + catalog = Catalog.from_dict(catalog_data) + config_root().mkdir(parents=True) + (config_root() / 'catalog.yaml').write_text(yaml.safe_dump(catalog_data)) + settings = config_root() / 'settings.yaml' + settings.write_text('backend: kubeai\n') + docker = FakeDocker() + kubectl = FakeKubectl() + path = default_ledger_path() + def make_backend(config=None, **kwargs): + kind = getattr(config, 'backend', None) or 'kubeai' + if kind == 'compose': + return compose_backend(data_root() / 'leasing/compose', catalog=catalog, docker=docker, ui=False) + result = KubeaiBackend(state_dir=data_root() / 'leasing/kubeai', namespace='default', + run=kubectl, http=FakeHttp(kubectl), default_resource_profile='gpu-single-default') + result.catalog = catalog + return result + def controller(kind): + return Controller(Ledger(SqliteStore(path)), make_backend(SimpleNamespace(backend=kind))) + monkeypatch.setattr('infer_stack.cli.commands_leasing._make_backend', make_backend) + monkeypatch.setattr('infer_stack.cli.commands_ledger._make_backend', make_backend) + return SimpleNamespace(controller=controller, make_backend=make_backend, catalog=catalog, + path=path, docker=docker, kubectl=kubectl, settings=settings) + + +def history(env, *, active=False, live=False): + old = env.controller('compose') + request = env.catalog.resolve_requests(['smol135-kube']) + outcome = old.acquire('old-owner', request, wait=False) + if not active: + old.release(outcome.lease.id) + if not live: + old.backend.down() + return old, outcome + + +def test_full_real_machine_migration_sequence(environment, capsys): + env = environment + old, outcome = history(env) + profile = old.ledger.profile() + assert profile['backend'] == 'compose' + leases, deployments = old.ledger.status(virtual_expiry=True) + assert len(leases) == len(deployments) == 1 + assert str(leases[0].state) == 'released' + assert not env.docker.containers + config_before = env.settings.read_bytes() + catalog_path = config_root() / 'catalog.yaml' + catalog_before = catalog_path.read_bytes() + with pytest.raises(SystemExit, match='ledger rotate'): + AcquireCLI.main(argv=False, names=['smol135-kube'], wait=False, yes=True) + capsys.readouterr() + with pytest.raises(SystemExit, match='gc:.*ledger rotate'): + GcCLI.main(argv=False, yes=True) + assert not env.kubectl.applied # no reinterpretation of Compose rows + assert old.ledger.profile() == profile + assert LedgerRotateCLI.main(argv=False, json=True) == 0 + plan = json.loads(capsys.readouterr().out) + assert not plan['changed'] and not Path(plan['archive']).exists() + assert LedgerRotateCLI.main(argv=False, yes=True, json=True) == 0 + result = json.loads(capsys.readouterr().out) + assert result['changed'] + archived = Ledger(SqliteStore(result['archive'])) + assert archived.profile() == profile + assert archived.get_lease(outcome.lease.id).state == leases[0].state + assert archived.get_deployment(outcome.deployments[0].id) is not None + current = env.controller('kubeai') + assert current.ledger.status() == ([], []) + assert current.ledger.profile()['backend'] == 'kubeai' + assert current.ledger.publication_pending() is None + assert env.settings.read_bytes() == config_before + assert catalog_path.read_bytes() == catalog_before + assert AcquireCLI.main(argv=False, names=['smol135-kube'], wait=False, yes=True) == 0 + capsys.readouterr() + assert current.ledger.profile()['backend'] == 'kubeai' + assert current.ledger.profile()['catalogs'] + (doc,) = env.kubectl.applied.values() + assert doc['metadata']['namespace'] == 'default' + assert doc['metadata']['name'] == 'smol135-kube' + assert doc['spec']['resourceProfile'] == 'gpu-single-default:1' + for flag in ('tensor', 'pipeline', 'data'): + assert f'--{flag}-parallel-size=1' in doc['spec']['args'] + assert not any(a.startswith('--served-model-name') for a in doc['spec']['args']) + assert archived.get_deployment(outcome.deployments[0].id) is not None + assert ManageCLI.main(argv=['ledger', 'archives', '--json']) == 0 + assert json.loads(capsys.readouterr().out)[0]['path'] == result['archive'] + # A process opened against the old backend cannot mutate the new epoch. + with pytest.raises(ProfileMismatch): + old.gc() + + +@pytest.mark.parametrize('active,live,match', [(False, True, 'runtime objects'), (True, False, 'active leases')]) +def test_transition_refuses_live_objects_or_leases(environment, active, live, match): + env = environment + old, _ = history(env, active=active, live=live) + before = old.ledger.profile() + with pytest.raises(ProfileMismatch, match=match): + rotate_backend(env.controller('kubeai'), env.make_backend(SimpleNamespace(backend='compose')), apply=True) + assert old.ledger.profile() == before + assert not list(env.path.parent.glob('archives/*.db')) + + +def test_rotation_retry_after_archive_before_reset(environment, monkeypatch): + env = environment + history(env) + current = env.controller('kubeai') + store = current.ledger.store + original = store._write_profile + def interrupt(profile): + raise RuntimeError('simulated power loss inside reset transaction') + monkeypatch.setattr(store, '_write_profile', interrupt) + old = env.make_backend(SimpleNamespace(backend='compose')) + with pytest.raises(RuntimeError, match='power loss'): + rotate_backend(current, old, apply=True) + assert store.profile()['backend'] == 'compose' + assert len(store.list_leases()) == len(store.list_deployments(now=current.clock())) == 1 + archives = list(env.path.parent.glob('archives/*.db')) + assert len(archives) == 1 + monkeypatch.setattr(store, '_write_profile', original) + result = rotate_backend(env.controller('kubeai'), old, apply=True) + assert result['changed'] + assert Path(result['archive']) == archives[0] + assert len(list(env.path.parent.glob('archives/*.db'))) == 1 + retry = rotate_backend(env.controller('kubeai'), old, apply=True) + assert not retry['changed'] + assert len(retry['archives']) == 1 + + +def test_unknown_old_runtime_refuses_rotation(environment, monkeypatch): + history(environment) + old = environment.make_backend(SimpleNamespace(backend='compose')) + def unavailable(): + raise RuntimeError('Docker unreachable') + monkeypatch.setattr(old, 'instances', unavailable) + with pytest.raises(ProfileMismatch, match='Cannot verify old backend quiescence'): + rotate_backend(environment.controller('kubeai'), old, apply=True) + + +def test_gc_matching_and_history_only_mismatch(environment, capsys): + history(environment) + assert GcCLI.main(argv=False, backend='compose', yes=True) == 0 + capsys.readouterr() + with pytest.raises(SystemExit, match='gc:.*ledger rotate'): + GcCLI.main(argv=False, yes=True) + assert 'Traceback' not in capsys.readouterr().err + # Explicit history pruning does not adopt/configure the wrong backend. + assert GcCLI.main(argv=False, forget=True, json=True) == 0 + assert json.loads(capsys.readouterr().out)['leases'] == 1 + + +@pytest.mark.parametrize('snapshot,configured', [(None, 'kubeai'), ('compose', 'compose'), ('compose', 'kubeai')]) +def test_status_names_configured_and_recovery_backends(environment, capsys, snapshot, configured): + env = environment + if snapshot: + history(env) + env.settings.write_text(f'backend: {configured}\n') + assert StatusCLI.main(argv=False) == 0 + text = capsys.readouterr().out + assert f'configured backend: {configured}' in text + assert f'active recovery backend: {snapshot or "(none — no active snapshot)"}' in text + assert ('transition required' in text) == bool(snapshot and snapshot != configured) + + +def test_ledger_modal_real_help(): + for args in (['ledger', '--help'], ['ledger', 'rotate', '--help'], ['ledger', 'archives', '--help']): + proc = subprocess.run([sys.executable, '-m', 'infer_stack', *args], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + assert 'ledger' in proc.stdout diff --git a/tests/test_cli_kube.py b/tests/test_cli_kube.py index b96e79bb..141d83e9 100644 --- a/tests/test_cli_kube.py +++ b/tests/test_cli_kube.py @@ -412,6 +412,8 @@ def test_k3s_bootstrap_existing_server_does_not_fetch_installer(monkeypatch): def fake_run(args, **kwargs): calls.append((list(args), dict(kwargs))) + if args == ['systemctl', 'is-active', '--quiet', 'k3s-agent']: + raise RuntimeError('inactive') if args == ['systemctl', 'is-active', '--quiet', 'k3s']: return 'active\n' if args == ['k3s', '--version']: @@ -424,7 +426,7 @@ def fake_run(args, **kwargs): ready = k3s.bootstrap(version='v1.34.3+k3s1', run=fake_run) assert ready is True assert not any(args and args[0] == 'curl' for args, _ in calls) - assert any(args[:3] == ['sudo', 'systemctl', 'restart'] for args, _ in calls) + assert not any('restart' in args for args, _ in calls) assert any( args[:3] == ['kubectl', '--kubeconfig', str(k3s.K3S_KUBECONFIG)] for args, _ in calls diff --git a/tests/test_gpu_doctor.py b/tests/test_gpu_doctor.py index 24a442b4..7871b330 100644 --- a/tests/test_gpu_doctor.py +++ b/tests/test_gpu_doctor.py @@ -50,7 +50,7 @@ def test_unprivileged_holder_check_is_unknown_not_clear(): res = _checks(_sample=lambda: [GpuSample(0, 0, 2)], _apps=lambda: [], _holders=lambda: None) ok, detail = res['device holders'] - assert ok is True # not a failure, but... + assert ok is False # incomplete diagnostic must not pass assert 'not checked' in detail # ...explicitly not an all-clear assert '--sudo' in detail @@ -111,3 +111,12 @@ def test_sampling_takes_the_minimum_so_one_spike_is_not_load(): def test_no_gpus_is_a_failure_not_a_pass(): res = _checks(_sample=lambda: [], _apps=lambda: [], _holders=lambda: []) assert res['GPUs visible'][0] is False + + +def test_requested_sudo_unavailable_has_actionable_hint(): + res = _checks(use_sudo=True, _sample=lambda: [GpuSample(0, 0, 2)], + _apps=lambda: [], _holders=lambda: None) + ok, detail = res['device holders'] + assert not ok + assert 'sudo -v' in detail + assert 'Re-run with --sudo' not in detail diff --git a/tests/test_kube_readiness.py b/tests/test_kube_readiness.py new file mode 100644 index 00000000..8713e517 --- /dev/null +++ b/tests/test_kube_readiness.py @@ -0,0 +1,431 @@ +"""Kubernetes setup contracts without a cluster or privileged host changes.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from infer_stack.backends.kubeai import KubeaiBackend +from infer_stack.cli import ManageCLI +from infer_stack.cli.commands_catalog import CatalogSuggestCLI +from infer_stack.cli.commands_kube import ( + KubeBootstrapCLI, + KubeDoctorCLI, + KubeInstallCLI, + KubeInventoryCLI, +) +from infer_stack.kube.inspect import failed, inventory, readiness +from infer_stack.kube.manage import KubeManager +from infer_stack.kube.operations import K3sProvider, install + + +def node(name='gpu-a', product='RTX-4090', count=4, memory='24576'): + return {'metadata': {'name': name, 'labels': { + 'nvidia.com/gpu.product': product, 'nvidia.com/gpu.memory': memory} if product else {}}, + 'status': {'nodeInfo': {'kubeletVersion': 'v1.34.3+k3s1'}, + 'conditions': [{'type': 'Ready', 'status': 'True'}], + 'allocatable': {'nvidia.com/gpu': str(count)}}} + + +class Cluster: + def __init__(self, *, nodes=None, installed=False, crd=False, plugin=True, reachable=True): + self.nodes = [node()] if nodes is None else nodes + self.installed = installed + self.crd = installed or crd + self.plugin = plugin + self.reachable = reachable + self.calls = [] + self.documents = [] + + def __call__(self, args, **kwargs): + self.calls.append(args) + if args[:3] == ['kubectl', 'config', 'current-context']: + return 'test-context' + if args[:2] == ['kubectl', 'version']: + if not self.reachable: + raise RuntimeError('connection refused') + return '{}' + if args[0] == 'kubectl' and 'nodes' in args and 'get' in args: + return json.dumps({'items': self.nodes}) + if args[:3] == ['kubectl', 'get', 'runtimeclasses']: + return json.dumps({'items': [{'metadata': {'name': 'nvidia'}}]}) + if args[:3] == ['kubectl', 'get', 'daemonsets']: + return json.dumps({'items': [{ + 'metadata': {'name': 'nvdp', 'namespace': 'nvidia-device-plugin'}, + 'spec': {'template': {'spec': {'containers': [{'image': 'nvcr.io/nvidia/k8s-device-plugin:v0.17.1'}]}}}, + 'status': {'desiredNumberScheduled': 1, 'numberReady': 1}}] if self.plugin else []}) + if args[:3] == ['kubectl', 'get', 'crd']: + return 'models.kubeai.org' if self.crd else '' + if args[:3] == ['kubectl', 'get', 'namespace']: + return 'namespace/kubeai' if self.installed else '' + if args[0] == 'kubectl' and 'configmap' in args: + if not self.installed: + raise RuntimeError('NotFound: kubeai-config') + return json.dumps({'data': {'system.yaml': 'resourceProfiles: {custom: {}}'}}) + if args[0] == 'kubectl' and any(r in args for r in ('pods', 'services', 'models.kubeai.org')): + return '{"items": []}' + if args[:2] == ['helm', 'list']: + return json.dumps([{'name': 'kubeai', 'namespace': 'kubeai', + 'status': 'deployed', 'chart': 'kubeai-0.22.0'}] if self.installed else []) + if args[:3] == ['helm', 'get', 'values']: + return 'resourceProfiles: {}' + if args[:3] == ['helm', 'upgrade', '--install']: + self.documents.append([yaml.safe_load(Path(args[i + 1]).read_text()) + for i, a in enumerate(args) if a == '-f']) + if args[3] == 'nvdp': + self.plugin = True + self.nodes = [node()] + else: + self.installed = self.crd = True + return '' + if args[:2] == ['helm', 'repo'] or (args[0] == 'kubectl' and 'wait' in args): + return '' + raise AssertionError(f'Unexpected command: {args}') + + +def manager(cluster, tools=('kubectl', 'helm')): + result = KubeManager(run=cluster) + result.command_exists = lambda name: name in tools + return result + + +HTTP = SimpleNamespace(get=lambda *a, **kw: SimpleNamespace(status_code=200)) + + +@pytest.mark.parametrize('nodes,plugin,installed,crd', [ + ([node(count=0, product=None)], False, False, False), + ([node(count=1)], True, False, False), + ([node(count=4)], True, False, False), + ([node(), node('gpu-b', 'A100', 8, '81920')], True, False, False), + ([node(product=None)], True, False, False), + ([node()], False, False, False), + ([node()], True, False, True), + ([node()], True, True, True), +]) +def test_inventory_partial_states(nodes, plugin, installed, crd): + cluster = Cluster(nodes=nodes, plugin=plugin, installed=installed, crd=crd) + report = inventory(manager(cluster), http=HTTP) + assert report['cluster']['reachable'] + assert report['gpu_count'] == sum(int(n['status']['allocatable']['nvidia.com/gpu']) for n in nodes) + assert bool(report['device_plugin']) == plugin + assert report['kubeai']['crd'] == crd + assert report['kubeai']['namespace_exists'] == installed + assert bool(report['kubeai']['release']) == installed + assert len(report['resource_profiles']['proposed']) == len({ + n['metadata']['labels']['nvidia.com/gpu.product'] for n in nodes + if n['status']['allocatable']['nvidia.com/gpu'] != '0' and n['metadata']['labels']}) + + +def test_missing_kubectl_and_unreachable_do_not_cascade(): + cluster = Cluster(reachable=False) + report = inventory(manager(cluster, tools=())) + assert len(readiness(report)) == 1 + assert not cluster.calls + report = inventory(manager(cluster)) + assert failed(readiness(report)) + assert [c['name'] for c in readiness(report)] == ['kubectl available', 'Helm available', 'cluster reachable'] + assert not any('daemonsets' in c for c in cluster.calls) + + +def test_probe_error_keeps_gpu_facts(): + cluster = Cluster(installed=True) + def run(args, **kwargs): + if 'daemonsets' in args: + raise RuntimeError('Forbidden') + return cluster(args, **kwargs) + m = manager(cluster) + m.run = run + report = inventory(m, http=HTTP) + assert report['gpu_count'] == 4 + assert report['device_plugin'] is None + assert 'Forbidden' in report['errors']['device_plugin'] + assert report['resource_profiles']['proposed'] + + +def test_preinstall_configmap_notfound_inventory_and_catalog(tmp_path, monkeypatch, capsys): + cluster = Cluster(crd=True) + backend = KubeaiBackend(state_dir=tmp_path, run=cluster) + facts, sizes = backend.gpu_facts() + assert facts['gpu-a']['count'] == 4 + assert sizes == {} + inv, profiles = backend.suggestion_inventory() + assert inv['gpu_count'] == 4 + assert profiles + monkeypatch.setattr('infer_stack.cli.commands_leasing._make_backend', lambda *a: backend) + assert CatalogSuggestCLI.main(argv=False, backend='kubeai') == 0 + out = capsys.readouterr() + assert 'resourceProfiles' in out.err + assert 'infer-stack kube install' in out.err + assert 'models:' in out.out + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: manager(cluster)) + assert KubeInventoryCLI.main(argv=False, json=True) == 0 + report = json.loads(capsys.readouterr().out) + assert report['gpu_count'] == 4 + assert not report['kubeai']['namespace_exists'] + assert report['resource_profiles']['proposed'] == profiles + + +def test_doctor_green_and_fail_exit(monkeypatch, capsys): + cluster = Cluster(crd=True) + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: manager(cluster)) + monkeypatch.setattr('infer_stack.kube.inspect.requests.get', HTTP.get) + assert KubeDoctorCLI.main(argv=False) == 1 + out = capsys.readouterr().out + assert 'fix: infer-stack kube install' in out + assert '[SKIP] KubeAI API' in out + cluster.installed = True + assert KubeDoctorCLI.main(argv=['--json']) == 0 + assert not failed(json.loads(capsys.readouterr().out)['checks']) + + +def test_install_values_token_upgrade_dryrun(tmp_path, monkeypatch, capsys): + cluster = Cluster(nodes=[node(), node('gpu-b', 'A100', 8, '81920')]) + m = manager(cluster) + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: m) + monkeypatch.setattr('infer_stack.kube.inspect.requests.get', HTTP.get) + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + monkeypatch.setenv('HF_TOKEN', 'sensitive-token') + assert KubeInstallCLI.main(argv=['--dry-run', '--json']) == 0 + preview = json.loads(capsys.readouterr().out) + assert len(preview['values']['resourceProfiles']) == 2 + assert not any('upgrade' in c or 'add' in c for c in cluster.calls) + assert not (tmp_path / 'generated').exists() + assert KubeInstallCLI.main(argv=False, apply=True) == 0 + assert len(cluster.documents[-1][0]['resourceProfiles']) == 2 + assert cluster.documents[-1][1]['secrets']['huggingface']['token'] == 'sensitive-token' + cmd = next(c for c in cluster.calls if c[:3] == ['helm', 'upgrade', '--install']) + assert 'sensitive-token' not in ' '.join(cmd) + assert not Path(cmd[-1]).exists() # temporary secret values cleaned up + assert 'sensitive-token' not in (tmp_path / 'generated/kube/kubeai-values.yaml').read_text() + assert KubeInstallCLI.main(argv=False, apply=True) == 0 + upgrades = [c for c in cluster.calls if c[:3] == ['helm', 'upgrade', '--install']] + assert len(upgrades) == 2 + assert '--version' in upgrades[-1] + + +def test_install_custom_settings_and_profiles(tmp_path, monkeypatch): + cluster = Cluster() + m = manager(cluster) + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + report = inventory(m, namespace='custom-ns', release='custom-release', http=HTTP) + profiles = report['resource_profiles']['proposed'] + name = next(iter(profiles)) + install(m, report, operator_values={'resourceProfiles': {name: {'custom': True}}}, chart='custom/chart', version='1.2') + assert cluster.documents[-1][0]['resourceProfiles'][name] == {'custom': True} + cmd = next(c for c in cluster.calls if 'upgrade' in c) + assert cmd[3:5] == ['custom-release', 'custom/chart'] + assert cmd[cmd.index('-n') + 1] == 'custom-ns' + + +def test_install_requires_prerequisites(): + cluster = Cluster(nodes=[node(product=None)]) + m = manager(cluster) + with pytest.raises(RuntimeError, match='GFD'): + install(m, inventory(m)) + assert not any('upgrade' in c for c in cluster.calls) + + +def test_bootstrap_working_cluster_idempotent(): + cluster = Cluster() + m = manager(cluster) + provider = K3sProvider() + for _ in range(2): + provider.apply(m, inventory(m), timeout=0) + assert not any(c[0] in ('sudo', 'curl', 'systemctl') or 'upgrade' in c for c in cluster.calls) + + +def test_bootstrap_partial_state_resumes(): + cluster = Cluster(nodes=[node(count=0, product=None)], plugin=False) + m = manager(cluster) + provider = K3sProvider() + provider.apply(m, inventory(m), timeout=0) + provider.apply(m, inventory(m), timeout=0) + upgrades = [c for c in cluster.calls if 'upgrade' in c] + assert len(upgrades) == 1 + assert '--version' in upgrades[0] and '0.17.1' in upgrades[0] + assert 'gfd.enabled=true' in upgrades[0] and 'runtimeClassName=nvidia' in upgrades[0] + + +def test_fresh_bootstrap_plan_no_mutation(monkeypatch, capsys): + cluster = Cluster() + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: manager(cluster, tools=())) + assert KubeBootstrapCLI.main(argv=['--provider=k3s', '--plan', '--json']) == 0 + plan = json.loads(capsys.readouterr().out) + assert any('K3s' in a for a in plan['actions']) + assert any('Helm' in a for a in plan['actions']) + assert not cluster.calls + + +@pytest.mark.parametrize('leaf', ['inventory', 'doctor', 'bootstrap', 'install', 'status']) +def test_real_modal_help_registration(leaf, tmp_path): + env = os.environ | {'INFER_STACK_CONFIG_DIR': str(tmp_path), 'INFER_STACK_DATA_DIR': str(tmp_path)} + proc = subprocess.run([sys.executable, '-m', 'infer_stack', 'kube', leaf, '--help'], + capture_output=True, text=True, env=env) + assert proc.returncode == 0, proc.stderr + assert leaf in proc.stdout + assert '--config_dir' in proc.stdout + + +def test_modal_doctor_and_tree(monkeypatch, tmp_path, capsys): + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: manager(Cluster(), tools=())) + assert ManageCLI.main(argv=['kube', 'doctor']) == 1 + capsys.readouterr() + proc = subprocess.run([sys.executable, '-m', 'infer_stack', 'help', 'tree'], + capture_output=True, text=True) + assert proc.returncode == 0 + for leaf in ('inventory', 'doctor', 'bootstrap', 'install', 'status'): + assert leaf in proc.stdout + + +def test_host_gpu_facts_survive_missing_kubectl(): + def run(args, **kwargs): + assert args[0] == 'nvidia-smi' + return '0, RTX 4090, 24576\n1, RTX 4090, 24576\n' + m = KubeManager(run=run) + m.command_exists = lambda name: name == 'nvidia-smi' + report = inventory(m) + assert len(report['host_gpus']) == 2 + assert report['gpu_count'] == 0 + assert not report['cluster']['reachable'] + + +def test_existing_values_probe_failure_prevents_replacement(tmp_path, monkeypatch): + cluster = Cluster(installed=True) + m = manager(cluster) + def run(args, **kwargs): + if args[:3] == ['helm', 'get', 'values']: + raise RuntimeError('Forbidden') + return cluster(args, **kwargs) + m.run = run + monkeypatch.setenv('INFER_STACK_DATA_DIR', str(tmp_path)) + with pytest.raises(RuntimeError, match='Forbidden'): + install(m, inventory(m, http=HTTP)) + assert not any('upgrade' in c for c in cluster.calls) + + +def test_bootstrap_installs_helm_only_if_missing(monkeypatch): + cluster = Cluster() + m = manager(cluster, tools=('kubectl',)) + def ensure(run): + assert run == cluster + m.command_exists = lambda name: name in ('kubectl', 'helm') + monkeypatch.setattr('infer_stack.kube.k3s._ensure_helm', ensure) + result = K3sProvider().apply(m, inventory(m), timeout=0) + assert result['tools']['helm'] + + +def test_k3s_fresh_then_interrupted_start_never_reinstalls(monkeypatch): + from infer_stack.kube import k3s + + calls = [] + installed = False + def run(args, **kwargs): + nonlocal installed + calls.append(args) + if args[:2] == ['systemctl', 'is-active']: + raise RuntimeError('inactive') + if args == ['curl', '-sfL', k3s.K3S_INSTALL_URL]: + return '# installer' + if args[-2:] == ['sh', '-']: + installed = True + return '' + monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) + monkeypatch.setattr(k3s.shutil, 'which', lambda name: '/usr/local/bin/k3s' if name == 'k3s' and installed else None) + monkeypatch.setattr(k3s, '_ensure_default_kubeconfig_link', lambda: True) + monkeypatch.setattr(k3s, '_ensure_helm', lambda run: None) + k3s.bootstrap(run=run) + k3s.bootstrap(run=run) + assert sum(c[0] == 'curl' for c in calls) == 1 + assert ['sudo', '-n', 'systemctl', 'start', 'k3s'] in calls + assert not any('restart' in c for c in calls) + + +def test_kubeconfig_preserved(tmp_path, monkeypatch): + from infer_stack.kube import k3s + + monkeypatch.delenv('KUBECONFIG', raising=False) + monkeypatch.setattr(k3s.Path, 'home', lambda: tmp_path) + config = tmp_path / '.kube/config' + config.parent.mkdir() + config.write_text('operator config') + assert not k3s._ensure_default_kubeconfig_link() + assert config.read_text() == 'operator config' + + +def test_runtime_repair_refuses_unrelated_local_cluster(monkeypatch): + cluster = Cluster(nodes=[node(count=0)], plugin=False) + m = manager(cluster, tools=('kubectl', 'helm', 'nvidia-container-runtime')) + original = m.run + def run(args, **kwargs): + if args[:3] == ['kubectl', 'get', 'runtimeclasses']: + return '{"items": []}' + if 'jsonpath={.metadata.uid}' in args: + return 'local' if args[0] == 'sudo' else 'remote' + return original(args, **kwargs) + m.run = run + monkeypatch.setattr('infer_stack.kube.k3s._active', lambda *args: True) + with pytest.raises(RuntimeError, match='differs from local K3s'): + K3sProvider().apply(m, inventory(m), timeout=0) + assert not any('restart' in c for c in cluster.calls) + + +def test_k3s_runtimeclass_alone_does_not_prove_runtime_configured(monkeypatch): + cluster = Cluster(nodes=[node(count=0)], plugin=False) + m = manager(cluster, tools=('kubectl', 'helm', 'k3s')) + monkeypatch.setattr('infer_stack.kube.k3s._active', lambda *args: True) + with pytest.raises(RuntimeError, match='nvidia-container-toolkit'): + K3sProvider().apply(m, inventory(m), timeout=0) + assert not any('upgrade' in c for c in cluster.calls) + + +def test_k3s_configured_runtime_does_not_restart(monkeypatch): + cluster = Cluster(nodes=[node(count=0, product=None)], plugin=False) + m = manager(cluster, tools=('kubectl', 'helm', 'k3s', 'nvidia-container-runtime')) + original = m.run + host_calls = [] + def run(args, **kwargs): + host_calls.append(args) + if 'jsonpath={.metadata.uid}' in args: + return 'same-cluster' + if args[:3] == ['sudo', '-n', 'cat']: + return 'BinaryName = "/usr/bin/nvidia-container-runtime"' + return original(args, **kwargs) + m.run = run + monkeypatch.setattr('infer_stack.kube.k3s._active', lambda *args: True) + result = K3sProvider().apply(m, inventory(m), timeout=0) + assert result['gpu_count'] == 4 + assert not any('restart' in c for c in host_calls) + + +def test_explicit_cpu_install_escape_hatch(tmp_path, monkeypatch, capsys): + cluster = Cluster(nodes=[node(count=0, product=None)], plugin=False) + m = manager(cluster) + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: m) + monkeypatch.setattr('infer_stack.kube.inspect.requests.get', HTTP.get) + values = tmp_path / 'cpu.yaml' + values.write_text('resourceProfiles: {cpu: {imageName: cpu, requests: {cpu: 8}}}') + assert KubeInstallCLI.main(argv=False, gpu='none', values=str(values), apply=True) == 0 + assert cluster.documents[-1][0]['resourceProfiles']['cpu']['imageName'] == 'cpu' + assert 'NVIDIA' not in capsys.readouterr().out + + +def test_malformed_chart_config_does_not_erase_inventory(): + cluster = Cluster(installed=True) + m = manager(cluster) + original = m.run + def run(args, **kwargs): + if 'configmap' in args: + return json.dumps({'data': {'system.yaml': '- invalid\n- mapping'}}) + return original(args, **kwargs) + m.run = run + report = inventory(m, http=HTTP) + assert report['gpu_count'] == 4 + assert report['resource_profiles']['proposed'] + assert report['resource_profiles']['installed'] == {} + assert 'mapping' in report['errors']['profile_yaml'] diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index 0cc31acb..a0cc736a 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -156,7 +156,10 @@ def test_render_model_doc_shape(): assert spec['minReplicas'] == 1 and spec['maxReplicas'] == 1 assert '--tensor-parallel-size=2' in spec['args'] # the gateway request name and vLLM's served name must agree - assert '--served-model-name=qwen' in spec['args'] + assert not any(a.startswith('--served-model-name') for a in spec['args']) + # KubeAI prepends this argument from the Model name before spec.args. + effective = ['--served-model-name=' + doc['metadata']['name'], *spec['args']] + assert effective.count('--served-model-name=qwen') == 1 assert rendered.models == {'qwen': 'grp-a'} assert rendered.request_names == {'grp-a': 'qwen'} @@ -544,7 +547,7 @@ def get(self, url, **kw): checks = be.doctor() gateway = [c for c in checks if c[0].startswith("KubeAI's API")][0] assert gateway[1] is False - assert 'port-forward' in gateway[2] + assert 'infer-stack kube status' in gateway[2] def test_doctor_cli_exit_codes(tmp_path, monkeypatch, capsys): @@ -1158,3 +1161,33 @@ def test_the_host_gateways_unverified_routes_leave_the_apply_incomplete(tmp_path be.gateway.apply = lambda: ApplyResult(routes=False, detail='routes not verified') outcome = be.apply() assert outcome.runtime and not outcome.routes and not outcome.complete + + +def test_kubeai_served_name_override_uses_model_identity_once(): + # Catalog served_name changes Model identity; KubeAI uses that identity. + from infer_stack.leasing import Catalog + catalog = Catalog.from_dict({'models': {'m': {'source': 'hf://org/model'}}, + 'endpoints': {'alias': {'model': 'm', 'engine': 'vllm', 'served_name': 'Custom/Name', + 'runtime': {'resource_profile': 'gpu-single-default'}}}}) + ledger = Ledger(SqliteStore(':memory:')) + result = ledger.acquire('test', catalog.resolve_requests(['alias'])) + rendered = render_models(result.deployments, namespace='default', default_resource_profile=None) + (doc,) = rendered.docs + assert doc['metadata']['name'] == 'custom-name' + effective = ['--served-model-name=' + doc['metadata']['name'], *doc['spec']['args']] + assert effective.count('--served-model-name=custom-name') == 1 + assert rendered.request_names['alias'] == 'custom-name' + + +def test_kubeai_extra_args_preserve_nonidentity_repetitions(): + dep = vllm('grp-a', extra_args=['--dtype=half', '--dtype=bfloat16']) + rendered = render_models([dep], namespace='default', default_resource_profile=None) + assert rendered.docs[0]['spec']['args'][-2:] == ['--dtype=half', '--dtype=bfloat16'] + + +def test_explicit_served_name_extra_argument_is_rejected(): + from infer_stack.leasing import Catalog, CatalogError + with pytest.raises(CatalogError, match='repeats --served-model-name'): + Catalog.from_dict({'models': {'m': {'source': 'hf://org/model'}}, + 'endpoints': {'alias': {'engine': 'vllm', 'model': 'm', 'runtime': { + 'extra_args': ['--served-model-name=incorrect']}}}}) From bc47baaa16de746d379dcd319d8523a0a731c196 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 20:26:52 -0400 Subject: [PATCH 132/141] Secure K3s provisioning and correct Kubernetes readiness Scope K3s bootstrap to local private credentials, verify worker membership and node runtime evidence, tolerate inactive NVIDIA components, and report readiness transitions. Deprecate the separate setup CLI workflow. Co-authored-by: GPT-6.1-Sol --- README.md | 16 ++- dev/journals/codex.md | 37 +++++ dev/kubeai_e2e.sh | 1 + docs/cluster-setup.md | 31 +++- docs/kubeai-backend.md | 29 +++- infer_stack/backends/kubeai.py | 70 +++++++-- infer_stack/cli/commands_kube.py | 108 +++++++------- infer_stack/kube/inspect.py | 34 ++++- infer_stack/kube/k3s.py | 122 ++++++++++++++-- infer_stack/kube/operations.py | 74 +++++++--- infer_stack/leasing/controller.py | 16 ++- scripts/bootstrap_k3s.sh | 2 +- tests/test_cli_kube.py | 7 +- tests/test_kube_readiness.py | 21 ++- tests/test_kube_review.py | 231 ++++++++++++++++++++++++++++++ 15 files changed, 683 insertions(+), 116 deletions(-) create mode 100644 tests/test_kube_review.py diff --git a/README.md b/README.md index f9505297..412a36d9 100644 --- a/README.md +++ b/README.md @@ -621,11 +621,17 @@ Set `kubeai_base_url` to a reachable OpenAI URL for the installed service. `kube bootstrap` defaults to a read-only plan; `--apply` (or `--yes`) authorizes host/cluster changes. K3s is the implemented provisioning provider; inventory, installation and the backend work with any Kubernetes distribution. Bootstrap -preserves working clusters and existing kubeconfigs, installs Helm if missing, +always provisions/reconciles the local K3s server, preserves unrelated selected +contexts and kubeconfigs, installs Helm if missing, and reconciles NVIDIA device plugin **0.17.1** with GPU Feature Discovery. Install the host NVIDIA driver and container toolkit first. K3s discovers the installed runtime on startup; bootstrap restarts local K3s only when runtime -discovery needs repair. It waits for Ready nodes, GPU allocation and GFD labels. +discovery needs repair. It waits for Ready nodes, GPU allocation and GFD labels, and verifies unknown +GPU-node runtime handlers with small node-specific canary pods. Root admin +credentials stay `0600`; a private `0600` copy lives at +`~/.kube/infer-stack-k3s.yaml`. Existing default configs are preserved. Select the +local server explicitly with `export KUBECONFIG=~/.kube/infer-stack-k3s.yaml` +before inventory/install; rerun bootstrap to refresh copied certificates. `kube install` shows inferred Helm values; `--apply` uses Helm upgrade/install and checks readiness afterward. `--dry-run`/`--plan` forces read-only behavior. @@ -634,8 +640,10 @@ Existing named profiles and custom values are preserved; `HF_TOKEN` overrides the chart token using a temporary protected file. Generated public values live at `/generated/kube/kubeai-values.yaml`. -The older `kube setup` and `kube k3s` commands remain available for compatibility -and node join operations. Setup scripts are compatibility wrappers around the +`kube setup` is a deprecated alias of `kube install`; it no longer owns a +separate prerequisite workflow. `kube k3s bootstrap` provisions only the local +server, `join` verifies requested worker membership, and `status` reports local +membership independently of any stale selected admin context. Setup scripts are compatibility wrappers around the package commands. ### Debugging serving failures diff --git a/dev/journals/codex.md b/dev/journals/codex.md index e6d179fb..f2eff3a2 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -191,3 +191,40 @@ both detailed and operational doctor checks with the intended namespace/API URL. Reusable takeaways: prerequisite inventory must precede configuration that consumes it; teardown does not erase persistence ownership; SQLite history rotation should preserve live connections and change epochs transactionally. + +## 2026-10-01 20:12:00 -0400 + +User intent: extend the TUI with efficient Kubernetes node/cluster reporting and +controls, then prioritize the real-host review of the previous setup work before +resuming that UI task. Model/configuration: GPT-6 (Codex), Default collaboration +mode; reasoning effort not exposed. Commit attribution requested by the user is +GPT-6.1-Sol. No sub-agents. TUI work was paused and stashed during this review. + +The review exposed three product boundary mistakes: admin credential access was +made too broad, provider=k3s reused whatever admin context was selected, and an +optional NVIDIA DaemonSet with no targets counted as unhealthy. Root K3s config +now remains 0600; the invoking user gets a private, atomically refreshed 0600 +copy. This accepts certificate refresh maintenance in exchange for preserving +other users' privileges and unrelated kubeconfigs. Explicit K3s provisioning +always scopes reconciliation to that local copy. A worker's local membership is +reported independently from its admin context, and an active agent must match +requested server/name/version before join may return idempotently. + +RuntimeClass existence is a cluster fact, not evidence of node handlers. The +inventory now reports observed nvidia-runtime pod startup separately per node; +unknown handlers block detailed readiness rather than implying success. Explicit +bootstrap/install can run tiny node-specific runtime canaries without reserving +a GPU, retaining completed pods as diagnostic evidence. This avoids taking busy +GPU capacity merely to test a handler. Successful startup proves the observed +runtime invocation, not future driver health. Kube setup is deprecated and routes +to the native install leaf so the shell-visible workflows share policy. Readiness +waits report changed states and final generation verification; KubeAI includes +node/image/replica detail with one pod list for nonfatal diagnosis. + +Review validation: 207 focused tests passed; the full suite before restoring +the TUI passed 1194 tests with 3 skips and 8 cleanup/ResourceWarnings. An +additional startup-detail regression passed in the review module (15 tests). +Type checks, CI flake8, scoped Ruff, shell syntax and actual entrypoint help +checks passed. Remaining real-host checks include private credential access/ +refresh, stale-EKS context isolation, active-worker config inspection, and runtime +canaries on aiq-gpu/namek. No real cluster mutation is part of this work. diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 10014fc6..6ede83d1 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -6,6 +6,7 @@ # infer-stack kube bootstrap --provider=k3s # sudo -v # infer-stack kube bootstrap --provider=k3s --apply +# export KUBECONFIG=~/.kube/infer-stack-k3s.yaml # infer-stack kube install # infer-stack kube install --apply # infer-stack kube doctor diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 7f9f31eb..8a8a0b98 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -122,11 +122,19 @@ infer-stack kube bootstrap --provider=k3s --version= --apply This provider-specific bootstrap plans by default; `--apply` installs/starts a K3s server, waits for its node to become Ready, and makes the K3s kubeconfig -usable without overwriting an existing `~/.kube/config`. +private: the root admin file stays `0600`, and only the invoking user receives +a `0600` copy at `~/.kube/infer-stack-k3s.yaml`. Existing default configs and +`KUBECONFIG` are never replaced. `provider=k3s` always reconciles this local +server through its explicit private kubeconfig, even when EKS or another cluster +is selected. Use `export KUBECONFIG=~/.kube/infer-stack-k3s.yaml` for subsequent +inventory/install. Repeating bootstrap refreshes the copied certificates when +K3s rotates its admin credentials. The older `scripts/bootstrap_k3s.sh` calls +`infer-stack kube k3s bootstrap`, which provisions only the local server. -Then inspect the generic Kubernetes integration: +Then explicitly select the local K3s server and inspect the generic integration: ```bash +export KUBECONFIG=~/.kube/infer-stack-k3s.yaml infer-stack kube inventory infer-stack kube doctor infer-stack kube install @@ -177,6 +185,7 @@ network. Back on the machine operating infer-stack: ```bash +export KUBECONFIG=~/.kube/infer-stack-k3s.yaml infer-stack kube inventory infer-stack kube doctor infer-stack kube install @@ -381,3 +390,21 @@ administrator: This keeps K3s easy to use today without making it part of infer-stack's backend or scheduling architecture. + +### Local worker membership and runtime evidence + +`infer-stack kube k3s status` reports local server/agent membership without +consulting the selected admin context or exposing join tokens. After join, +verify the worker on the control plane with `infer-stack kube node status `. +A stale EKS context on the worker does not mean local membership failed. Active +agents must match requested server/name/version before join can return success. + +A RuntimeClass object is cluster-scoped and does not prove every node handler. +Inventory reports per-node runtime evidence independently. Explicit bootstrap +and install can run small node-specific runtime canaries on GPU nodes; they +reserve no GPUs and retain completed pods in `kube-system`. Unknown handlers +block detailed readiness until verified. Optional NVIDIA components scheduled +onto zero nodes report N/A instead of causing a reinstall/readiness loop. + +`kube setup` is deprecated and delegates to `kube install`; use local K3s +bootstrap for prerequisites and generic install for the selected cluster. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 7d9b44c2..ea239945 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -55,15 +55,23 @@ infer-stack kube status # nodes, GPUs, plugin, pods/services/mana ``` Install host NVIDIA drivers/container toolkit before bootstrap. Inventory and -install are distribution-neutral. Bootstrap preserves working clusters and -existing kubeconfigs; it refuses to replace a selected unreachable context. +install are distribution-neutral. `bootstrap --provider=k3s` always targets +the local K3s server, never an unrelated selected context (reachable or stale). +Root admin credentials remain `0600`. Bootstrap refreshes a private user copy at +`~/.kube/infer-stack-k3s.yaml`, preserving existing default configs. Explicitly +select it with `export KUBECONFIG=~/.kube/infer-stack-k3s.yaml` before inventory/install. Use `--version` to pin K3s; active clusters are never implicitly upgraded. `inventory`, `doctor`, and `status` accept `--json`. Errors belong to individual probes, so a missing KubeAI ConfigMap cannot erase GPU node facts. A cluster with a leftover CRD but no namespace/chart can be diagnosed and repaired with these commands. Kubernetes GPU allocation and GFD readiness are separate from host -GPU visibility. +GPU visibility. A cluster-scoped RuntimeClass is reported separately from node +runtime evidence; missing handler evidence remains unknown, never an all-clear. +Apply bootstrap/install verifies unknown handlers on GPU nodes with small +`busybox:1.37.0` canaries using `runtimeClassName=nvidia`. These reserve no GPU +and retain completed pods in `kube-system` as diagnostic evidence. Inactive +0/0 NVIDIA DaemonSets are N/A, so an optional MPS component does not block readiness. Mutation requires `--apply` (alias `--yes`). Both mutating commands support `--dry-run` (alias `--plan`). Install derives one profile per GPU product with @@ -91,8 +99,9 @@ background port-forward is created. For temporary Compose/KubeAI testing, preserve the Compose authority as described in [cluster setup](cluster-setup.md#temporarily-swap-compose-configured-workstations-into-a-cluster). -`kube setup` remains a compatibility plan/apply workflow, and the setup scripts -forward to the new commands. +`kube setup` is deprecated and delegates to `kube install`; it no longer +reconciles prerequisites independently. The bootstrap compatibility script +invokes the unambiguous `kube k3s bootstrap` local-server command. For explicit CPU development, preserve the existing CPU chart workflow: @@ -242,6 +251,12 @@ infer-stack kube k3s join \ --version= ``` +After join, use `infer-stack kube k3s status` on the worker for local membership. +Joining does not change its admin kubeconfig or grant cluster-admin credentials; +verify it from the control plane with `infer-stack kube node status `. +An already-running agent must match the requested server, node name and version +or join refuses. + The token is read from a file so it does not land in shell history or the installer argv. Between K3s nodes, allow the K3s-required cluster traffic (6443/tcp to the server, the configured Flannel/backend traffic, and kubelet @@ -313,3 +328,7 @@ provides a synthetic second K3s node for development; deployment is its own Model (`-`), so `--dedicated` twice gives two Models behind one alias. The gateway's config changes are shown and approved with the acquire's, before its lease commits. + +Acquire/wait logs changed readiness states once: pending/scheduler reasons, +scheduled node and image, container startup/model loading, replica readiness, +and finally generation verification. It does not log every unchanged poll. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 4000c39a..9d7d4d76 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -217,7 +217,9 @@ def resource_profiles_for_nodes( } if runtime_class_name: profile['runtimeClassName'] = runtime_class_name - profiles[f'nvidia-{dns_slug(product)}'] = profile + slug = dns_slug(product) + name = slug if slug.startswith('nvidia-') else f'nvidia-{slug}' + profiles[name] = profile return profiles @@ -521,6 +523,41 @@ def _requests(pod: dict) -> set[str]: return names +def model_startup_progress(pods: list[dict]) -> str: + """Stable lifecycle text: location, startup reason/image, replica readiness. + + Avoid unstable API error text in the poll loop. Readiness still requires a + generation; container/replica readiness only explains what we are awaiting. + """ + if not pods: + return 'pending — waiting for KubeAI to create a pod' + details = [] + for pod in pods: + spec, status = pod.get('spec', {}), pod.get('status', {}) + node = spec.get('nodeName') + conditions = status.get('conditions') or [] + scheduled = next((c for c in conditions if c.get('type') == 'PodScheduled'), {}) + if not node: + why = scheduled.get('message') or scheduled.get('reason') or 'waiting for scheduler' + details.append(f'pending — {why}') + continue + containers = status.get('containerStatuses') or [] + waiting = [c.get('state', {}).get('waiting', {}).get('reason') for c in containers] + waiting = [w for w in waiting if w] + images = ', '.join(c.get('image', '?') for c in spec.get('containers') or []) + if waiting: + details.append(f"scheduled on {node} — {', '.join(waiting)}; image {images}") + elif not containers: + details.append(f'scheduled on {node} — creating container / pulling image {images}') + elif all(c.get('ready') for c in containers): + details.append(f'scheduled on {node} — replica ready; verifying generation') + else: + details.append(f'scheduled on {node} — container started; model loading') + ready = sum(any(c.get('type') == 'Ready' and c.get('status') == 'True' + for c in p.get('status', {}).get('conditions') or []) for p in pods) + return f'{ready}/{len(pods)} replicas ready; ' + '; '.join(sorted(set(details))) + + class KubeaiBackend(ConvergeScaffold): """Cluster KubeAI backend (converge-style). @@ -560,6 +597,7 @@ def __init__( http = requests self.http = http self.assume_yes = assume_yes + self.progress: Callable[[str], None] | None = None self.last_errors: list[str] = [] self.last_unplaced: set[str] = set() # KubeAI/k8s schedules; there are no host GPU indices to report. @@ -1232,17 +1270,25 @@ def probe_ready(self, deployment: Deployment, endpoint: str) -> Readiness: ) if ok: return Readiness(True, reason) - # A crash-looping engine keeps its pod "Running" between restarts, so - # check every not-ready probe, not only when the Model is missing. - failure = self.startup_failure(deployment) - if failure is not None: - return Readiness(False, failure, fatal=True) - waiting = self._waiting_reason(deployment) - # Room helps only when the scheduler says some node is short of a - # resource; a selector, taint or affinity no node satisfies stays - # unschedulable whatever is stopped (reclaim_candidates picks whom). - return Readiness(False, f'{reason} (pod: {waiting})' if waiting else reason, - needs_room=bool(self._capacity_shortage(deployment))) + # One pod list supplies startup, scheduler and progress facts together. + from ..leasing.diagnosis import diagnose_startup + from ..leasing.residency import residency_from_pods + try: + raw = self._kubectl(['get', 'pods', '-l', f'{MANAGED_LABEL}=true', '-o', 'json']) + residency = residency_from_pods(raw) + units = residency.units(deployment.id) + failure = diagnose_startup(units, + lambda: self.deployment_logs(deployment, tail=200), + replicated=residency.replicated, unit_logs=lambda pod: self._pod_logs(pod, 200)) + if failure is not None: + return Readiness(False, failure, fatal=True) + pods = [p for p in json.loads(raw).get('items', []) + if p.get('metadata', {}).get('labels', {}).get(DEPLOYMENT_LABEL) == deployment.id] + short = any(p.reason == 'Unschedulable' and insufficient_resources(p.message) for p in units) + detail = model_startup_progress(pods) + return Readiness(False, detail, needs_room=short) + except Exception as ex: # diagnostic failure is unknown, never an all-clear + return Readiness(False, f'Cannot inspect Model pods: {ex}; API: {reason}') def connection_info(self): """Where a client reaches these endpoints: the gateway's front door, or diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index e84d23f1..59fd34e1 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -279,13 +279,10 @@ class KubeNodeModalCLI(kw.ModalCLI): class KubeSetupCLI(_PathOverridesMixin): - """Inspect or reconcile the Kubernetes capabilities infer-stack needs. + """Deprecated compatibility alias of kube install. - The default is read-only. ``--apply`` installs/reconciles only components - infer-stack owns: the NVIDIA device-plugin/GFD convenience path (when an - NVIDIA RuntimeClass says the host runtime is ready), and KubeAI with - discovered resource profiles. Existing working GPU integrations and - operator-customized named profiles are preserved. + Use kube bootstrap for local K3s prerequisites. Setup delegates KubeAI + planning/apply to the install command and has no independent orchestration. """ __command__ = 'setup' @@ -313,39 +310,12 @@ class KubeSetupCLI(_PathOverridesMixin): def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) _apply_path_overrides(config) - namespace = _namespace(config) - manager = KubeManager() - try: - operator_values = manager.load_values_file(config.values) - except RuntimeError as ex: - raise SystemExit(str(ex)) from ex - if not config.apply: - try: - plan = manager.plan_setup( - namespace=namespace, release=config.release, gpu=config.gpu, - operator_values=operator_values, - ) - except RuntimeError as ex: - raise SystemExit(str(ex)) from ex - _print_plan(plan) - print('\nNo changes made. Re-run with --apply to reconcile managed components.') - return 1 if plan.failed else 0 - - try: - plan, values_path = manager.apply_setup( - namespace=namespace, - release=config.release, - gpu=config.gpu, - nvidia_plugin_version=config.nvidia_plugin_version, - kubeai_version=config.kubeai_version, - operator_values=operator_values, - ) - except RuntimeError as ex: - raise SystemExit(str(ex)) from ex - _print_plan(plan) - if values_path is not None: - print(f'\nKubeAI values authority: {values_path}') - return 1 if plan.failed else 0 + print('Deprecated: kube setup is an alias of kube install; use kube bootstrap for local K3s prerequisites.') + if config.nvidia_plugin_version != NVIDIA_DEVICE_PLUGIN_VERSION: + raise SystemExit('Use the pinned NVIDIA bootstrap path; setup no longer owns plugin installation') + return KubeInstallCLI.main(argv=False, namespace=_namespace(config), release=config.release, + gpu='none' if config.gpu == 'none' else 'nvidia', + apply=config.apply, values=config['values'], version=config.kubeai_version) class K3sBootstrapCLI(kw.Config): @@ -360,19 +330,20 @@ class K3sBootstrapCLI(kw.Config): @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) - from ..kube.k3s import K3S_NODE_TOKEN, bootstrap + from ..kube.k3s import K3S_NODE_TOKEN, bootstrap, user_kubeconfig try: default_kubeconfig_ready = bootstrap(version=config.version) except RuntimeError as ex: raise SystemExit(str(ex)) from ex print('K3s server is active and all nodes are Ready.') - print('K3s kubeconfig: /etc/rancher/k3s/k3s.yaml') + print('Root admin kubeconfig: /etc/rancher/k3s/k3s.yaml (0600)') + print(f'Private user kubeconfig: {user_kubeconfig()} (0600); rerun bootstrap to refresh certificates') if not default_kubeconfig_ready: print( 'Existing ~/.kube/config was left unchanged. Make sure its ' - 'current context selects this K3s cluster (or set KUBECONFIG) ' - 'before running setup.' + 'current context selects this K3s cluster before installation.' + f' Select it explicitly with: export KUBECONFIG={user_kubeconfig()}' ) print(f'K3s join token: sudo cat {K3S_NODE_TOKEN}') print('Join workers with: infer-stack kube k3s join --server=... --token-file=...') @@ -395,6 +366,8 @@ class K3sJoinCLI(kw.Config): @classmethod def main(cls, argv=True, **kwargs): config = cls.cli(argv=argv, data=kwargs) + import socket + from ..kube.k3s import join try: @@ -406,20 +379,42 @@ def main(cls, argv=True, **kwargs): ) except RuntimeError as ex: raise SystemExit(str(ex)) from ex - print('K3s agent is active.') + print('K3s agent is active; this does not change the selected admin kubeconfig.') + print('Inspect local membership: infer-stack kube k3s status') + print(f'On the control plane: infer-stack kube node status {config.node_name or socket.gethostname().lower()}') return 0 +class K3sStatusCLI(kw.Config): + """Local K3s server/agent membership, independent of selected admin context.""" + __command__ = 'status' + json = kw.Value(False, isflag=True, help='Emit membership facts without credentials.') + + @classmethod + def main(cls, argv=True, **kwargs): + from ..kube.k3s import local_status + config = cls.cli(argv=argv, data=kwargs) + report = local_status() + if config.json: + print(json.dumps(report, indent=2)) + else: + print('local Kubernetes membership (not selected admin context)') + for key, value in report.items(): + print(f' {key}: {value}') + return int(bool(report['errors'])) + + class K3sModalCLI(kw.ModalCLI): """First supported provisioning target: create/join K3s clusters. This namespace is intentionally distribution-specific. Existing clusters - and future provisioning integrations use the same generic ``kube setup``. + and future provisioning integrations use generic inventory/doctor/install. """ __command__ = 'k3s' bootstrap = K3sBootstrapCLI join = K3sJoinCLI + status = K3sStatusCLI def _inventory_for_config(config, manager): @@ -434,7 +429,12 @@ def _inventory_for_config(config, manager): def _print_inventory(report): import yaml - print('cluster') + if report.get('local_membership') is not None: + print('local Kubernetes membership') + for key, value in report['local_membership'].items(): + print(f' {key}: {value}') + print('selected admin cluster') + print(' RuntimeClass objects: ' + ', '.join(report['runtime_classes'] or [])) for key, value in report['cluster'].items(): print(f' {key}: {value}') print(f"host GPUs: {report['host_gpus']}") @@ -443,12 +443,12 @@ def _print_inventory(report): print(f' {key}: {"available" if value else "missing"}') for node in report['nodes']: print(f"node {node['name']}") - for key in ('ready', 'nvidia_runtime_class', 'gpu_count', 'gpu_product', + for key in ('ready', 'nvidia_runtime_verified', 'nvidia_runtime_evidence', 'gpu_count', 'gpu_product', 'gpu_memory_gib', 'gfd_labels'): print(f' {key}: {node[key]}') plugins = report['device_plugin'] print('NVIDIA device plugin: ' + ('unknown' if plugins is None else 'missing' if not plugins else - ', '.join(f"{p['name']} ({p['ready']}/{p['desired']} ready)" for p in plugins))) + ', '.join(f"{p['name']} ({str(p['ready']) + '/' + str(p['desired']) + ' ready' if p['desired'] > 0 else 'N/A: zero scheduled'})" for p in plugins))) kubeai = report['kubeai'] print('KubeAI') print(f" Model CRD: {'installed' if kubeai['crd'] else 'missing' if kubeai['crd'] is False else 'unknown'}") @@ -476,7 +476,10 @@ def _print_inventory(report): summary = f"{status.get('readyReplicas', 0)} ready replicas" print(f' {name}: {summary}') print('resource profiles proposed') - print(yaml.safe_dump(report['resource_profiles']['proposed'], sort_keys=False), end='') + if report['resource_profiles']['proposed']: + print(yaml.safe_dump(report['resource_profiles']['proposed'], sort_keys=False), end='') + else: + print(' none: discover allocatable GPUs with GFD product/memory labels through a reachable admin context') print('resource profiles installed: ' + (', '.join(report['resource_profiles']['installed']) or 'none')) for key, error in report['errors'].items(): print(f' probe {key}: {error}') @@ -588,10 +591,10 @@ def main(cls, argv=True, **kwargs): class KubeBootstrapCLI(KubeInventoryCLI): - """Plan host/cluster GPU prerequisites; --apply converges them with K3s.""" + """Plan a local K3s server and GPU prerequisites; never targets unrelated selected contexts.""" __command__ = 'bootstrap' - provider = kw.Value('k3s', choices=['k3s'], type=str, help='Cluster provisioning provider.') + provider = kw.Value('k3s', choices=['k3s'], type=str, help='Provision local K3s, regardless of selected admin context.') version = kw.Value(None, type=str, help='Optional exact K3s version; no implicit upgrades.') apply = kw.Value(False, isflag=True, alias=['yes'], help='Authorize host/cluster bootstrap changes.') dry_run = kw.Value(False, isflag=True, alias=['plan'], help='Only display the plan.') @@ -621,6 +624,9 @@ def main(cls, argv=True, **kwargs): except RuntimeError as ex: raise SystemExit(f'{ex}. If elevation failed, authenticate with sudo -v and retry.') from ex checks = readiness(report, installation=False) + if not config.json: + from ..kube.k3s import user_kubeconfig + print(f'Local K3s reconciled. For inventory/install: export KUBECONFIG={user_kubeconfig()}') if config.json: print(json.dumps({'inventory': report, 'checks': checks}, indent=2)) else: diff --git a/infer_stack/kube/inspect.py b/infer_stack/kube/inspect.py index afa6ce74..c3753bf1 100644 --- a/infer_stack/kube/inspect.py +++ b/infer_stack/kube/inspect.py @@ -29,7 +29,7 @@ def inventory(manager: KubeManager, *, namespace='kubeai', release='kubeai', report: dict[str, Any] = { 'tools': {name: manager.command_exists(name) for name in ('kubectl', 'helm')}, 'cluster': {'context': '', 'reachable': False, 'distribution': None}, - 'host_gpus': None, 'gpu_count': 0, 'nodes': [], 'runtime_classes': None, 'device_plugin': None, + 'local_membership': None, 'host_gpus': None, 'gpu_count': 0, 'nodes': [], 'runtime_classes': None, 'device_plugin': None, 'kubeai': {'namespace': namespace, 'release_name': release, 'crd': None, 'namespace_exists': None, 'release': None, 'pods': None, 'services': None, 'models': None, @@ -44,6 +44,9 @@ def probe(key, func, default=None): report['errors'][key] = str(ex) return default + if manager.command_exists('k3s'): + from .k3s import local_status + report['local_membership'] = probe('local_membership', lambda: local_status(run=manager.run)) if manager.command_exists('nvidia-smi'): def host_gpus(): text = manager.run(['nvidia-smi', '--query-gpu=index,name,memory.total', @@ -78,8 +81,25 @@ def host_gpus(): report['runtime_classes'] = [i.get('metadata', {}).get('name') for i in runtime.get('items', [])] runtime_ok = 'nvidia' in (report['runtime_classes'] or []) + runtime_pods = probe('node_runtime', lambda: manager.kubectl_json( + ['get', 'pods', '-A', '-o', 'json'])) if runtime_ok else None for row in report['nodes']: - row['nvidia_runtime_class'] = runtime_ok + row['nvidia_runtime_verified'] = None + row['nvidia_runtime_evidence'] = 'No observed pod proves the node runtime handler' + for pod in (runtime_pods or {}).get('items', []): + spec, status = pod.get('spec', {}), pod.get('status', {}) + if spec.get('nodeName') != row['name'] or spec.get('runtimeClassName') != 'nvidia': + continue + started = any(c.get('state', {}).get('running') or c.get('state', {}).get('terminated') + for c in status.get('containerStatuses') or []) + if started: + row['nvidia_runtime_verified'] = True + row['nvidia_runtime_evidence'] = f"nvidia runtime pod {manager._pod_name(pod)} started" + break + messages = ' '.join(str(c.get('message', '')) for c in status.get('conditions') or []) + if 'runtime handler' in messages.lower(): + row['nvidia_runtime_verified'] = False + row['nvidia_runtime_evidence'] = messages ds = probe('device_plugin', lambda: manager.kubectl_json( ['get', 'daemonsets', '-A', '-o', 'json'])) if ds is not None: @@ -91,7 +111,8 @@ def host_gpus(): desired = status.get('desiredNumberScheduled', 0) plugins.append({'name': manager._pod_name(item), 'desired': desired, 'ready': status.get('numberReady', 0), - 'healthy': desired > 0 and status.get('numberReady', 0) == desired}) + 'applicable': desired > 0, + 'healthy': (status.get('numberReady', 0) == desired) if desired > 0 else None}) report['device_plugin'] = plugins report['resource_profiles']['proposed'] = manager.resource_profiles( nodes, runtime_class_name='nvidia' if runtime_ok else None) @@ -161,8 +182,13 @@ def check(name, ok, fix='', detail='', blocked=False): check('NVIDIA runtime', runtime_ok or bool(gpu_rows), 'Install NVIDIA driver/container toolkit on GPU nodes; then infer-stack kube bootstrap', 'RuntimeClass nvidia' if runtime_ok else 'No nvidia RuntimeClass; GPU allocation proves an external integration' if gpu_rows else '') + for row in gpu_rows if runtime_ok else []: + verified = row.get('nvidia_runtime_verified') + check(f"node {row['name']} NVIDIA runtime verified", verified is True, + 'infer-stack kube install --apply (verifies runtime handlers in the selected cluster)', + row.get('nvidia_runtime_evidence', 'Runtime handler unknown')) plugin = report['device_plugin'] - check('NVIDIA device plugin', bool(plugin) and all(p['healthy'] for p in plugin), + check('NVIDIA device plugin', bool(plugin) and any(p['desired'] > 0 for p in plugin) and all(p['healthy'] for p in plugin if p['desired'] > 0), 'infer-stack kube bootstrap', report['errors'].get('device_plugin', '')) if check('nvidia.com/gpu allocatable', bool(gpu_rows), 'infer-stack kube bootstrap'): check('GFD product + memory labels', all(r['gfd_labels'] for r in gpu_rows), diff --git a/infer_stack/kube/k3s.py b/infer_stack/kube/k3s.py index f59708d3..574157e1 100644 --- a/infer_stack/kube/k3s.py +++ b/infer_stack/kube/k3s.py @@ -6,12 +6,16 @@ from __future__ import annotations +import json import os import shutil +import socket import tempfile from pathlib import Path from typing import Callable +import yaml + from .manage import default_run RunFunc = Callable[..., str] @@ -50,20 +54,46 @@ def _ensure_default_kubeconfig_link() -> bool: Existing kubeconfig state is operator authority and is never replaced. """ if os.environ.get('KUBECONFIG'): - return os.environ['KUBECONFIG'] == str(K3S_KUBECONFIG) + return os.environ['KUBECONFIG'] == str(user_kubeconfig()) target = Path.home() / '.kube' / 'config' if target.exists() or target.is_symlink(): if target.is_symlink(): try: - return target.resolve() == K3S_KUBECONFIG + return target.resolve() == user_kubeconfig() except OSError: return False return False - target.parent.mkdir(parents=True, exist_ok=True) - target.symlink_to(K3S_KUBECONFIG) + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + target.symlink_to(user_kubeconfig()) return True +def user_kubeconfig() -> Path: + return Path.home() / '.kube' / 'infer-stack-k3s.yaml' + + +def _provision_user_kubeconfig(run: RunFunc) -> None: + """Refresh an explicitly owned 0600 admin copy; never expose root credentials. + + K3s rotates its admin certificates. Rerunning bootstrap refreshes this copy. + An unrelated default or KUBECONFIG remains untouched. + """ + target = user_kubeconfig() + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + text = run(['sudo', '-n', 'cat', str(K3S_KUBECONFIG)]) + if not text.strip(): + raise RuntimeError('Local K3s kubeconfig is empty; cannot provision user access') + fd, source = tempfile.mkstemp(prefix='.infer-stack-k3s-', dir=target.parent) + try: + with os.fdopen(fd, 'w') as file: + file.write(text) + file.flush() + os.fsync(file.fileno()) + os.replace(source, target) + finally: + Path(source).unlink(missing_ok=True) + + def _ensure_helm(run: RunFunc) -> None: if shutil.which('helm') is not None: return @@ -72,7 +102,7 @@ def _ensure_helm(run: RunFunc) -> None: def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool: - """Install/start a local K3s server and make its kubeconfig readable. + """Install/start a local K3s server and provision restricted user access. NVIDIA drivers/container runtime are intentionally not installed here. K3s detects an already-installed NVIDIA container runtime when it starts; @@ -85,31 +115,31 @@ def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool already_active = _active(run, 'k3s') if already_active and version: installed = run(['k3s', '--version']).splitlines()[0] - if version not in installed: + if installed.split()[2] != version: raise RuntimeError( f'K3s is already active as {installed!r}; requested {version!r}. ' 'Refusing an implicit cluster upgrade; upgrade K3s explicitly.' ) - # Keep this as persistent K3s configuration rather than copying the - # root-owned kubeconfig into ~/.kube (which creates a second authority that - # can drift when K3s rotates credentials). + # The root admin authority stays 0600. Only this invoking user receives + # a private copy; bootstrap refreshes it when credentials rotate. with tempfile.NamedTemporaryFile('w', prefix='infer-stack-k3s-', delete=False) as file: - file.write('write-kubeconfig-mode: "0644"\n') + file.write('write-kubeconfig-mode: "0600"\n') source = file.name try: run(['sudo', '-n', 'mkdir', '-p', '/etc/rancher/k3s/config.yaml.d']) run([ 'sudo', '-n', 'install', '-m', '0644', source, - '/etc/rancher/k3s/config.yaml.d/10-infer-stack-kubeconfig-mode.yaml', + '/etc/rancher/k3s/config.yaml.d/zz-infer-stack-kubeconfig-mode.yaml', ]) finally: Path(source).unlink(missing_ok=True) + run(['sudo', '-n', 'rm', '-f', '/etc/rancher/k3s/config.yaml.d/10-infer-stack-kubeconfig-mode.yaml']) if already_active: # Updating the mode fragment does not require restarting a working # control plane. Repair current access directly; next start uses it too. - run(['sudo', '-n', 'chmod', '0644', str(K3S_KUBECONFIG)]) + run(['sudo', '-n', 'chmod', '0600', str(K3S_KUBECONFIG)]) elif shutil.which('k3s') is not None: run(['sudo', '-n', 'systemctl', 'start', 'k3s']) else: @@ -125,8 +155,10 @@ def bootstrap(*, version: str | None = None, run: RunFunc | None = None) -> bool cmd.extend(['sh', '-']) run(cmd, input_text=installer, env=env) + run(['sudo', '-n', 'chmod', '0600', str(K3S_KUBECONFIG)]) + _provision_user_kubeconfig(run) run([ - 'kubectl', '--kubeconfig', str(K3S_KUBECONFIG), + 'kubectl', '--kubeconfig', str(user_kubeconfig()), 'wait', '--for=condition=Ready', 'node', '--all', '--timeout=180s', ]) default_kubeconfig_ready = _ensure_default_kubeconfig_link() @@ -145,10 +177,19 @@ def join( """Join this host to an existing K3s server without putting its token on argv.""" run = run or default_run _require_local_tool('sudo') + if _active(run, 'k3s'): + raise RuntimeError('This host runs a K3s server; refusing to replace it with an agent') if _active(run, 'k3s-agent'): + membership = local_status(run=run) + if membership['errors']: + raise RuntimeError('Cannot verify active K3s membership: ' + '; '.join(membership['errors'])) + if membership['server'].rstrip('/') != server.rstrip('/'): + raise RuntimeError(f"Active K3s agent server {membership['server']!r} differs from requested {server!r}; stop/reconfigure membership explicitly") + if node_name and membership['node_name'] != node_name: + raise RuntimeError(f"Active K3s agent node name {membership['node_name']!r} differs from requested {node_name!r}") if version: installed = run(['k3s', '--version']).splitlines()[0] - if version not in installed: + if installed.split()[2] != version: raise RuntimeError( f'K3s agent is already active as {installed!r}; requested ' f'{version!r}. Refusing an implicit agent upgrade.' @@ -174,8 +215,59 @@ def join( env['INSTALL_K3S_VERSION'] = version preserve.append('INSTALL_K3S_VERSION') run( - ['sudo', '--preserve-env=' + ','.join(preserve), 'sh', '-'], + ['sudo', '-n', '--preserve-env=' + ','.join(preserve), 'sh', '-'], input_text=installer, env=env, ) run(['systemctl', 'is-active', '--quiet', 'k3s-agent']) + + +def local_status(*, run: RunFunc | None = None) -> dict: + """Local membership independent of selected admin kubeconfig, without secrets. + + Inspect the running agent's actual argv/environment, then K3s config with + its drop-ins. Unknown/unreadable configuration is reported and cannot make + a requested join look idempotent. + """ + run = run or default_run + report = {'agent_active': _active(run, 'k3s-agent'), + 'server_active': _active(run, 'k3s'), 'server': '', + 'node_name': '', 'version': '', 'errors': []} + if not report['agent_active'] and not report['server_active']: + return report + try: + report['version'] = run(['k3s', '--version']).splitlines()[0] + except Exception: + report['errors'].append('Unable to read local K3s version') + if not report['agent_active']: + return report + try: + pid = run(['systemctl', 'show', 'k3s-agent', '--property=MainPID', '--value']).strip() + if not pid.isdigit() or int(pid) == 0: + raise ValueError('agent PID unavailable') + argv = run(['sudo', '-n', 'cat', f'/proc/{pid}/cmdline']).split('\0') + environ = run(['sudo', '-n', 'cat', f'/proc/{pid}/environ']).split('\0') + env = dict(v.split('=', 1) for v in environ if '=' in v) + flags = {} + for i, arg in enumerate(argv): + if arg.startswith('--'): + key, sep, value = arg[2:].partition('=') + flags[key] = value if sep else (argv[i + 1] if i + 1 < len(argv) else '') + config_path = flags.get('config') or env.get('K3S_CONFIG_FILE') or '/etc/rancher/k3s/config.yaml' + # Read only through the privilege seam. The command emits YAML only + # into memory; secrets never enter the status report or diagnostics. + files = json.loads(run(['sudo', '-n', 'python3', '-c', + 'import glob,json,pathlib,sys; p=sys.argv[1]; ' + 'print(json.dumps([pathlib.Path(f).read_text() for f in ([p] if pathlib.Path(p).exists() else []) + sorted(glob.glob(p+".d/*.yaml"))]))', + config_path])) + config = {} + for text in files: + values = yaml.safe_load(text) or {} + config.update(values) + report['server'] = str(flags.get('server') or env.get('K3S_URL') or config.get('server') or '') + report['node_name'] = str(flags.get('node-name') or env.get('K3S_NODE_NAME') or config.get('node-name') or socket.gethostname().lower()) + if not report['server']: + report['errors'].append('Active agent server is unknown') + except Exception: + report['errors'].append('Cannot inspect active agent configuration; authenticate with sudo -v then retry') + return report diff --git a/infer_stack/kube/operations.py b/infer_stack/kube/operations.py index fbc51a64..ff8ccbeb 100644 --- a/infer_stack/kube/operations.py +++ b/infer_stack/kube/operations.py @@ -13,10 +13,7 @@ class K3sProvider: def plan(self, manager: KubeManager, report: dict) -> list[str]: actions = [] - if not report['cluster']['reachable']: - actions.append('Install/start K3s only if no existing cluster/service; establish kubeconfig access') - else: - actions.append('Preserve the selected working cluster and kubeconfig') + actions.append('Provision the local K3s server; preserve unrelated selected context; use private local kubeconfig for all reconciliation') if not report['tools']['helm']: actions.append('Install Helm') if not report.get('gpu_count'): @@ -27,15 +24,9 @@ def plan(self, manager: KubeManager, report: dict) -> list[str]: return actions def apply(self, manager: KubeManager, report: dict, *, version=None, timeout=180) -> dict: - if not report['cluster']['reachable']: - # A configured but broken context is operator authority. Do not silently - # provision a second cluster beneath an unrelated kubeconfig. - if report['cluster']['context']: - raise RuntimeError('Selected context is unreachable; repair/select kubeconfig before bootstrap') - if not k3s.bootstrap(version=version, run=manager.run): - raise RuntimeError('Existing kubeconfig preserved. Select the K3s context, then retry bootstrap') - elif not report['tools']['helm']: - k3s._ensure_helm(manager.run) + # provider=k3s always targets this host's K3s, never the selected EKS/etc. + k3s.bootstrap(version=version, run=manager.run) + manager = self._local_manager(manager) fresh = inventory(manager) if not fresh.get('gpu_count') and (manager.command_exists('k3s') or 'nvidia' not in (fresh['runtime_classes'] or [])): @@ -68,7 +59,8 @@ def apply(self, manager: KubeManager, report: dict, *, version=None, timeout=180 raise RuntimeError('K3s did not detect NVIDIA runtime; ensure nvidia-container-runtime is in the K3s service PATH') manager.run(['kubectl', 'wait', '--for=condition=Ready', 'node', '--all', f'--timeout={timeout}s']) plugin = fresh.get('device_plugin') or [] - if (not plugin or not all(p['healthy'] for p in plugin) + if (not plugin or not any(p['desired'] > 0 for p in plugin) + or not all(p['healthy'] for p in plugin if p['desired'] > 0) or not fresh.get('gpu_count') or not all(r['gfd_labels'] for r in fresh['nodes'] if r['gpu_count'])): if 'nvidia' not in (fresh['runtime_classes'] or []): @@ -77,12 +69,55 @@ def apply(self, manager: KubeManager, report: dict, *, version=None, timeout=180 deadline = time.monotonic() + timeout while True: fresh = inventory(manager) + if fresh.get('gpu_count') and 'nvidia' in (fresh['runtime_classes'] or []): + verify_node_runtimes(manager, fresh, timeout=timeout) + fresh = inventory(manager) if not failed(readiness(fresh, installation=False)): return fresh if time.monotonic() >= deadline: raise RuntimeError('GPU readiness timed out; run infer-stack kube doctor') time.sleep(min(2, max(0, deadline - time.monotonic()))) + @staticmethod + def _local_manager(manager): + def run(args, **kwargs): + if args[0] in {'kubectl', 'helm'}: + args = [args[0], '--kubeconfig', str(k3s.user_kubeconfig()), *args[1:]] + return manager.run(args, **kwargs) + local = KubeManager(run=run) + local.command_exists = manager.command_exists + return local + + +def verify_node_runtimes(manager, report, *, timeout=180): + """Explicit bootstrap canaries test each GPU node's nvidia handler. + + No GPU reservation: allocatable resources are independently checked. A + successful retained Pod supplies inspectable evidence for later inventory. + No hostRuntime claim is inferred from the RuntimeClass object alone. + """ + import hashlib + import json + for node in report['nodes']: + if not node['gpu_count'] or node.get('nvidia_runtime_verified') is True: + continue + name = 'infer-stack-runtime-' + hashlib.sha256(node['name'].encode()).hexdigest()[:12] + existing = manager.kubectl_json(['-n', 'kube-system', 'get', 'pod', name, '--ignore-not-found', '-o', 'json']) + if existing and existing.get('metadata', {}).get('labels', {}).get('infer-stack/runtime-canary') != 'true': + raise RuntimeError(f'Refusing to replace unrelated pod kube-system/{name}') + manager.run(['kubectl', '-n', 'kube-system', 'delete', 'pod', name, '--ignore-not-found', '--wait=true']) + doc = {'apiVersion': 'v1', 'kind': 'Pod', + 'metadata': {'name': name, 'namespace': 'kube-system', + 'labels': {'infer-stack/runtime-canary': 'true'}}, + 'spec': {'nodeName': node['name'], 'runtimeClassName': 'nvidia', + 'restartPolicy': 'Never', 'automountServiceAccountToken': False, + 'containers': [{'name': 'runtime', 'image': 'busybox:1.37.0', 'command': ['/bin/true']}], + 'tolerations': [{'operator': 'Exists'}]}} + manager.run(['kubectl', 'apply', '-f', '-'], input_text=json.dumps(doc)) + manager.run(['kubectl', '-n', 'kube-system', 'wait', '--for=jsonpath={.status.phase}=Succeeded', + f'pod/{name}', f'--timeout={timeout}s']) + + def install_plan(manager: KubeManager, report: dict, *, operator_values=None) -> dict: kubeai = report['kubeai'] @@ -98,9 +133,16 @@ def install(manager: KubeManager, report: dict, *, operator_values=None, checks = readiness(report, installation=False, gpu=gpu) if report['errors'].get('helm_releases'): raise RuntimeError('Cannot inspect existing Helm releases; repair Helm access before installation') + blockers = [c for c in checks if c['status'] == 'fail' and not c['name'].endswith('NVIDIA runtime verified')] + if blockers: + raise RuntimeError('KubeAI prerequisites failed: ' + ', '.join(c['name'] for c in blockers) + + '. Run infer-stack kube doctor / kube bootstrap') + if gpu and 'nvidia' in (report['runtime_classes'] or []): + verify_node_runtimes(manager, report) + report = inventory(manager, namespace=report['kubeai']['namespace'], release=report['kubeai']['release_name'], base_url=report['kubeai']['api']['base_url']) + checks = readiness(report, installation=False, gpu=gpu) if failed(checks): - names = ', '.join(c['name'] for c in checks if c['status'] == 'fail') - raise RuntimeError(f'KubeAI prerequisites failed: {names}. Run infer-stack kube doctor / kube bootstrap') + raise RuntimeError('Node runtime verification incomplete; run infer-stack kube doctor') kubeai = report['kubeai'] manager.install_kubeai(namespace=kubeai['namespace'], release=kubeai['release_name'], resource_profiles=report['resource_profiles']['proposed'] if gpu else {}, diff --git a/infer_stack/leasing/controller.py b/infer_stack/leasing/controller.py index f0783ea8..15986a3e 100644 --- a/infer_stack/leasing/controller.py +++ b/infer_stack/leasing/controller.py @@ -1220,7 +1220,8 @@ def _acquire_profile_candidate(self, *, residency=None) -> dict | None: raise ProfileMismatch( f"the active recovery snapshot uses {stored.get('backend')!r}, " f"but current config selects {invocation.get('backend')!r}; " - 'tear down the old backend before switching' + 'quiesce/tear down the old backend, then archive its history and ' + 'start a new recovery epoch with `infer-stack ledger rotate --yes`' ) drift = profile_drift(stored, invocation) # A runbook whose catalog is already a subset of an explicitly seeded @@ -1558,12 +1559,25 @@ def wait_ready( ] deadline = self.clock() + timeout last_room = float('-inf') + last_details = {} while True: pending = [] failures = [] blocked: dict[str, Deployment] = {} for (g, ep) in pairs: probe = self.backend.probe_ready(g, ep) + detail = 'generation verified' if probe.ready else probe.detail + if detail and last_details.get((g.id, ep)) != detail: + from .._log import logger + message = f'{ep}: {detail}' + logger.info('{}', message) + progress = getattr(self.backend, 'progress', None) + if progress is not None: + try: + progress(message) + except Exception: + pass # presentation must not affect lease readiness + last_details[(g.id, ep)] = detail if probe.ready: continue pending.append((g, ep)) diff --git a/scripts/bootstrap_k3s.sh b/scripts/bootstrap_k3s.sh index c4c00ca0..dcd005cb 100755 --- a/scripts/bootstrap_k3s.sh +++ b/scripts/bootstrap_k3s.sh @@ -4,7 +4,7 @@ set -euo pipefail # Compatibility wrapper. The implementation lives in infer-stack now so the # CLI, tests, and docs share one setup authority. ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ARGS=(kube bootstrap --provider=k3s --apply) +ARGS=(kube k3s bootstrap) if [[ -n "${INSTALL_K3S_VERSION:-}" ]]; then ARGS+=(--version="${INSTALL_K3S_VERSION}") fi diff --git a/tests/test_cli_kube.py b/tests/test_cli_kube.py index 141d83e9..e3730693 100644 --- a/tests/test_cli_kube.py +++ b/tests/test_cli_kube.py @@ -290,7 +290,7 @@ def fake_run(args, **kwargs): return '# installer\n' if args[:3] == ['systemctl', 'is-active', '--quiet']: active_checks += 1 - if active_checks == 1: + if active_checks <= 2: raise RuntimeError('inactive') return 'active\n' return '' @@ -306,7 +306,7 @@ def fake_run(args, **kwargs): assert all(token not in ' '.join(args) for args, _ in calls) install = [item for item in calls if item[0] and item[0][0] == 'sudo'][0] assert install[1]['env']['K3S_TOKEN'] == token - assert 'K3S_TOKEN' in install[0][1] + assert 'K3S_TOKEN' in install[0][2] def test_missing_helm_is_required_when_setup_has_managed_actions(): @@ -422,13 +422,14 @@ def fake_run(args, **kwargs): monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) monkeypatch.setattr(k3s, '_ensure_helm', lambda run: None) + monkeypatch.setattr(k3s, '_provision_user_kubeconfig', lambda run: None) monkeypatch.setattr(k3s, '_ensure_default_kubeconfig_link', lambda: True) ready = k3s.bootstrap(version='v1.34.3+k3s1', run=fake_run) assert ready is True assert not any(args and args[0] == 'curl' for args, _ in calls) assert not any('restart' in args for args, _ in calls) assert any( - args[:3] == ['kubectl', '--kubeconfig', str(k3s.K3S_KUBECONFIG)] + args[:3] == ['kubectl', '--kubeconfig', str(k3s.user_kubeconfig())] for args, _ in calls ) diff --git a/tests/test_kube_readiness.py b/tests/test_kube_readiness.py index 8713e517..0e6784aa 100644 --- a/tests/test_kube_readiness.py +++ b/tests/test_kube_readiness.py @@ -53,6 +53,10 @@ def __call__(self, args, **kwargs): return '{}' if args[0] == 'kubectl' and 'nodes' in args and 'get' in args: return json.dumps({'items': self.nodes}) + if args[:3] == ['kubectl', 'get', 'pods'] and '-A' in args: + return json.dumps({'items': [{'metadata': {'name': 'nvdp-' + n['metadata']['name'], 'namespace': 'nvidia-device-plugin'}, + 'spec': {'nodeName': n['metadata']['name'], 'runtimeClassName': 'nvidia'}, + 'status': {'containerStatuses': [{'state': {'running': {'startedAt': 'now'}}}]}} for n in self.nodes]}) if args[:3] == ['kubectl', 'get', 'runtimeclasses']: return json.dumps({'items': [{'metadata': {'name': 'nvidia'}}]}) if args[:3] == ['kubectl', 'get', 'daemonsets']: @@ -231,18 +235,22 @@ def test_install_requires_prerequisites(): assert not any('upgrade' in c for c in cluster.calls) -def test_bootstrap_working_cluster_idempotent(): +def test_bootstrap_working_cluster_idempotent(monkeypatch): cluster = Cluster() m = manager(cluster) + monkeypatch.setattr('infer_stack.kube.k3s.bootstrap', lambda **kw: True) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda manager: manager)) provider = K3sProvider() for _ in range(2): provider.apply(m, inventory(m), timeout=0) assert not any(c[0] in ('sudo', 'curl', 'systemctl') or 'upgrade' in c for c in cluster.calls) -def test_bootstrap_partial_state_resumes(): +def test_bootstrap_partial_state_resumes(monkeypatch): cluster = Cluster(nodes=[node(count=0, product=None)], plugin=False) m = manager(cluster) + monkeypatch.setattr('infer_stack.kube.k3s.bootstrap', lambda **kw: True) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda manager: manager)) provider = K3sProvider() provider.apply(m, inventory(m), timeout=0) provider.apply(m, inventory(m), timeout=0) @@ -316,6 +324,8 @@ def ensure(run): assert run == cluster m.command_exists = lambda name: name in ('kubectl', 'helm') monkeypatch.setattr('infer_stack.kube.k3s._ensure_helm', ensure) + monkeypatch.setattr('infer_stack.kube.k3s.bootstrap', lambda **kw: ensure(kw['run'])) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda manager: manager)) result = K3sProvider().apply(m, inventory(m), timeout=0) assert result['tools']['helm'] @@ -339,6 +349,7 @@ def run(args, **kwargs): monkeypatch.setattr(k3s.shutil, 'which', lambda name: '/usr/local/bin/k3s' if name == 'k3s' and installed else None) monkeypatch.setattr(k3s, '_ensure_default_kubeconfig_link', lambda: True) monkeypatch.setattr(k3s, '_ensure_helm', lambda run: None) + monkeypatch.setattr(k3s, '_provision_user_kubeconfig', lambda run: None) k3s.bootstrap(run=run) k3s.bootstrap(run=run) assert sum(c[0] == 'curl' for c in calls) == 1 @@ -370,6 +381,8 @@ def run(args, **kwargs): return original(args, **kwargs) m.run = run monkeypatch.setattr('infer_stack.kube.k3s._active', lambda *args: True) + monkeypatch.setattr('infer_stack.kube.k3s.bootstrap', lambda **kw: True) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda manager: manager)) with pytest.raises(RuntimeError, match='differs from local K3s'): K3sProvider().apply(m, inventory(m), timeout=0) assert not any('restart' in c for c in cluster.calls) @@ -379,6 +392,8 @@ def test_k3s_runtimeclass_alone_does_not_prove_runtime_configured(monkeypatch): cluster = Cluster(nodes=[node(count=0)], plugin=False) m = manager(cluster, tools=('kubectl', 'helm', 'k3s')) monkeypatch.setattr('infer_stack.kube.k3s._active', lambda *args: True) + monkeypatch.setattr('infer_stack.kube.k3s.bootstrap', lambda **kw: True) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda manager: manager)) with pytest.raises(RuntimeError, match='nvidia-container-toolkit'): K3sProvider().apply(m, inventory(m), timeout=0) assert not any('upgrade' in c for c in cluster.calls) @@ -398,6 +413,8 @@ def run(args, **kwargs): return original(args, **kwargs) m.run = run monkeypatch.setattr('infer_stack.kube.k3s._active', lambda *args: True) + monkeypatch.setattr('infer_stack.kube.k3s.bootstrap', lambda **kw: True) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda manager: manager)) result = K3sProvider().apply(m, inventory(m), timeout=0) assert result['gpu_count'] == 4 assert not any('restart' in c for c in host_calls) diff --git a/tests/test_kube_review.py b/tests/test_kube_review.py new file mode 100644 index 00000000..4759e7d7 --- /dev/null +++ b/tests/test_kube_review.py @@ -0,0 +1,231 @@ +"""Real-host bootstrap review regressions; no privileged host mutations.""" +import json +from pathlib import Path + +import pytest +from test_kube_readiness import HTTP, Cluster, manager, node + +from infer_stack.kube import k3s +from infer_stack.kube.inspect import failed, inventory, readiness +from infer_stack.kube.manage import KubeManager +from infer_stack.kube.operations import K3sProvider, verify_node_runtimes + + +def test_aiq_optional_mps_zero_zero_is_not_failure(monkeypatch, capsys): + cluster = Cluster(installed=True) + def run(args, **kw): + if args[:3] == ['kubectl', 'get', 'daemonsets']: + return json.dumps({'items': [ + {'metadata': {'name': name}, 'spec': {'template': {'spec': { + 'containers': [{'image': 'nvcr.io/nvidia/k8s-device-plugin:v0.17.1'}]}}}, + 'status': {'desiredNumberScheduled': count, 'numberReady': count}} + for name, count in [('nvdp-nvidia-device-plugin', 1), + ('nvdp-nvidia-device-plugin-gpu-feature-discovery', 1), + ('nvdp-nvidia-device-plugin-mps-control-daemon', 0)]]}) + return cluster(args, **kw) + m = manager(cluster) + m.run = run + report = inventory(m, http=HTTP) + assert not failed(readiness(report)) + assert report['device_plugin'][2]['applicable'] is False + assert report['device_plugin'][2]['healthy'] is None + from infer_stack.cli.commands_kube import _print_inventory + _print_inventory(report) + assert 'mps-control-daemon (N/A: zero scheduled)' in capsys.readouterr().out + monkeypatch.setattr(k3s, 'bootstrap', lambda **kw: True) + monkeypatch.setattr(K3sProvider, '_local_manager', staticmethod(lambda m: m)) + K3sProvider().apply(m, report, timeout=0) + assert not any('upgrade' in args for args in cluster.calls) + + +def test_runtime_class_is_not_a_node_handler(): + cluster = Cluster(nodes=[node('aiq-gpu'), node('namek')]) + m = manager(cluster) + def run(args, **kw): + if args[:3] == ['kubectl', 'get', 'pods']: + return json.dumps({'items': [{'metadata': {'name': 'plugin'}, + 'spec': {'nodeName': 'aiq-gpu', 'runtimeClassName': 'nvidia'}, + 'status': {'containerStatuses': [{'state': {'running': { 'startedAt': 'now'}}}]}}]}) + return cluster(args, **kw) + m.run = run + report = inventory(m, http=HTTP) + assert report['runtime_classes'] == ['nvidia'] + assert report['nodes'][0]['nvidia_runtime_verified'] is True + assert report['nodes'][1]['nvidia_runtime_verified'] is None + assert failed(readiness(report, installation=False)) + assert all('nvidia_runtime_class' not in row for row in report['nodes']) + + +def test_private_user_config_keeps_root_and_default_authorities(tmp_path, monkeypatch): + monkeypatch.setattr(k3s.Path, 'home', lambda: tmp_path) + monkeypatch.delenv('KUBECONFIG', raising=False) + default = tmp_path / '.kube/config' + default.parent.mkdir() + default.write_text('unrelated EKS credentials') + secret = 'cluster-admin-secret' + k3s._provision_user_kubeconfig(lambda args: secret) + copy = k3s.user_kubeconfig() + assert copy.read_text() == secret + assert copy.stat().st_mode & 0o777 == 0o600 + assert not k3s._ensure_default_kubeconfig_link() + assert default.read_text() == 'unrelated EKS credentials' + k3s._provision_user_kubeconfig(lambda args: secret + '-rotated') + assert copy.stat().st_mode & 0o777 == 0o600 + assert copy.read_text().endswith('-rotated') + + +def test_bootstrap_enforces_root_0600_without_restart(monkeypatch): + calls, fragments = [], [] + def run(args, **kw): + calls.append(args) + if args[:2] == ['systemctl', 'is-active'] and args[-1] == 'k3s-agent': + raise RuntimeError('inactive') + if args[:4] == ['sudo', '-n', 'install', '-m']: + fragments.append(Path(args[5]).read_text()) + return '' + monkeypatch.setattr(k3s, '_provision_user_kubeconfig', lambda run: None) + monkeypatch.setattr(k3s, '_ensure_default_kubeconfig_link', lambda: True) + monkeypatch.setattr(k3s, '_ensure_helm', lambda run: None) + monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) + k3s.bootstrap(run=run) + assert fragments == ['write-kubeconfig-mode: "0600"\n'] + assert ['sudo', '-n', 'chmod', '0600', str(k3s.K3S_KUBECONFIG)] in calls + assert not any('restart' in c or ('chmod' in c and '0644' in c) for c in calls) + + +def agent_runner(server='https://old:6443', name='namek', config=None): + calls = [] + def run(args, **kw): + calls.append(args) + if args[:2] == ['systemctl', 'is-active']: + if args[-1] == 'k3s': + raise RuntimeError('inactive server') + return '' + if args[:2] == ['systemctl', 'show']: + return '321' + if args == ['k3s', '--version']: + return 'k3s version v1.34.3+k3s1 (fake)' + if args[-1] == '/proc/321/cmdline': + return '/usr/local/bin/k3s\0agent\0' + if args[-1] == '/proc/321/environ': + return f'K3S_URL={server}\0K3S_NODE_NAME={name}\0K3S_TOKEN=NEVER-PRINT-ME\0' if config is None else '' + if args[:3] == ['sudo', '-n', 'python3']: + return json.dumps(config or []) + raise AssertionError(args) + return run, calls + + +@pytest.mark.parametrize('server,name,version,match', [ + ('https://new:6443', 'namek', None, 'differs from requested'), + ('https://old:6443', 'new-name', None, 'node name'), + ('https://old:6443', 'namek', 'v1.34.3+k3s10', 'implicit agent upgrade'), +]) +def test_active_agent_join_refuses_incompatible_state(monkeypatch, server, name, version, match): + run, calls = agent_runner() + monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) + with pytest.raises(RuntimeError, match=match): + k3s.join(server=server, node_name=name, token_file='unused', version=version, run=run) + assert not any(c[0] == 'curl' for c in calls) + + +def test_active_agent_correct_state_is_idempotent_and_secrets_hidden(monkeypatch): + run, calls = agent_runner() + monkeypatch.setattr(k3s, '_require_local_tool', lambda name: None) + k3s.join(server='https://old:6443/', node_name='namek', token_file='unused', version='v1.34.3+k3s1', run=run) + report = k3s.local_status(run=run) + assert report['server'] == 'https://old:6443' + assert report['node_name'] == 'namek' + assert 'NEVER-PRINT-ME' not in json.dumps(report) + assert not report['errors'] + + +def test_agent_config_dropins_inspected(): + run, _ = agent_runner(config=['server: https://first:6443\nnode-name: a', + 'server: https://second:6443\nnode-name: namek']) + report = k3s.local_status(run=run) + assert report['server'] == 'https://second:6443' + assert report['node_name'] == 'namek' + + +def test_explicit_provider_scopes_every_cluster_mutation_to_local_copy(tmp_path, monkeypatch): + calls = [] + monkeypatch.setenv('KUBECONFIG', str(tmp_path / 'eks.yaml')) + monkeypatch.setattr(k3s, 'user_kubeconfig', lambda: tmp_path / 'local.yaml') + scoped = K3sProvider._local_manager(KubeManager(run=lambda args, **kw: calls.append(args) or '')) + scoped.run(['kubectl', 'wait', 'node']) + scoped.run(['helm', 'upgrade', '--install', 'nvdp']) + assert all(c[1:3] == ['--kubeconfig', str(tmp_path / 'local.yaml')] for c in calls) + assert 'eks' not in str(calls) + + +def test_canary_is_node_scoped_and_does_not_reserve_gpu(): + calls, documents = [], [] + def run(args, **kw): + calls.append(args) + if 'get' in args: + return '{}' + if 'apply' in args: + documents.append(json.loads(kw['input_text'])) + return '' + report = {'nodes': [{'name': 'namek', 'gpu_count': 1, 'nvidia_runtime_verified': None}]} + verify_node_runtimes(KubeManager(run=run), report) + assert documents[0]['spec']['nodeName'] == 'namek' + assert documents[0]['spec']['runtimeClassName'] == 'nvidia' + assert 'resources' not in documents[0]['spec']['containers'][0] + assert documents[0]['spec']['automountServiceAccountToken'] is False + assert any('wait' in c and '--for=jsonpath={.status.phase}=Succeeded' in c for c in calls) + + +def test_profile_vendor_prefix_is_not_duplicated(): + report = inventory(manager(Cluster(nodes=[node(product='NVIDIA-RTX-PRO-6000-Blackwell-Max-Q-Workstation-Edition')])), http=HTTP) + assert list(report['resource_profiles']['proposed']) == ['nvidia-rtx-pro-6000-blackwell-max-q-workstation-edition'] + + +def test_setup_cli_is_deprecated_alias_of_install(monkeypatch): + from infer_stack.cli.commands_kube import KubeInstallCLI, KubeSetupCLI + calls = [] + monkeypatch.setattr(KubeInstallCLI, 'main', lambda **kw: calls.append(kw) or 0) + assert KubeSetupCLI.main(argv=False, apply=True, namespace='default', gpu='none') == 0 + assert calls == [dict(argv=False, apply=True, namespace='default', release='kubeai', gpu='none', values=None, version=None)] + + +def test_controller_reports_only_readiness_transitions(): + from infer_stack.leasing import ( + Catalog, + Controller, + Ledger, + NullBackend, + SqliteStore, + ) + from infer_stack.leasing.backend import Readiness + catalog = Catalog.from_dict({'models': {'q': {'source': 'hf://org/model'}}, + 'endpoints': {'qwen-coder': {'engine': 'vllm', 'model': 'q'}}}) + controller = Controller(Ledger(SqliteStore(':memory:')), NullBackend()) + deployment = controller.acquire('test', catalog.resolve_names(['qwen-coder'])).deployments[0] + transitions = iter([Readiness(False, 'pending'), Readiness(False, 'pending'), + Readiness(False, 'scheduled on namek; pulling image'), + Readiness(False, 'scheduled on namek; pulling image'), Readiness(True, 'ok')]) + controller.backend.probe_ready = lambda *a: next(transitions) + messages = [] + controller.backend.progress = messages.append + controller.sleep = lambda seconds: None + assert controller.wait_ready([deployment], timeout=10).ready + assert len(messages) == 3 + assert messages[-1].endswith('generation verified') + + +def test_startup_details_expose_node_image_and_replica_state(): + from infer_stack.backends.kubeai import model_startup_progress + pod = {'spec': {'containers': [{'image': 'vllm/vllm-openai:v0.11.2'}]}, 'status': {}} + assert 'waiting for scheduler' in model_startup_progress([pod]) + pod['spec']['nodeName'] = 'namek' + assert 'scheduled on namek' in model_startup_progress([pod]) + assert 'vllm/vllm-openai:v0.11.2' in model_startup_progress([pod]) + pod['status']['containerStatuses'] = [{'ready': False, 'state': {'waiting': {'reason': 'ImagePullBackOff'}}}] + assert 'ImagePullBackOff' in model_startup_progress([pod]) + pod['status']['containerStatuses'] = [{'ready': False, 'state': {'running': {'startedAt': 'now'}}}] + assert 'container started; model loading' in model_startup_progress([pod]) + pod['status']['containerStatuses'][0]['ready'] = True + pod['status']['conditions'] = [{'type': 'Ready', 'status': 'True'}] + assert '1/1 replicas ready' in model_startup_progress([pod]) + assert 'verifying generation' in model_startup_progress([pod]) From aad587a790d658c81821e6c44a147c9be385eb1f Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 20:41:24 -0400 Subject: [PATCH 133/141] Add efficient Kubernetes TUI monitoring and controls Cache batched cluster observations, pause hidden polling and log followers, expose node lifecycle previews and confirmed controls, and edit Kubernetes resource profiles without host GPU assignments. Co-authored-by: GPT-6.1-Sol --- README.md | 22 +++ dev/journals/codex.md | 45 +++++ docs/kubeai-backend.md | 24 +++ infer_stack/kube/manage.py | 2 +- infer_stack/kube/monitor.py | 96 ++++++++++ infer_stack/tui.py | 343 ++++++++++++++++++++++++++++++++---- tests/test_tui_kube.py | 328 ++++++++++++++++++++++++++++++++++ 7 files changed, 826 insertions(+), 34 deletions(-) create mode 100644 infer_stack/kube/monitor.py create mode 100644 tests/test_tui_kube.py diff --git a/README.md b/README.md index 412a36d9..88a03cd8 100644 --- a/README.md +++ b/README.md @@ -702,3 +702,25 @@ the timeout is only for a model that is loading. Reading Docker's own signals: `litellm exited with code 137` is a SIGKILL (an OOM kill or a forced replacement), whereas LiteLLM returning HTTP 500 with `Cannot connect to host vllm-*` means LiteLLM is running and its upstream vLLM is not ready yet. + +### Kubernetes dashboard + +`infer-stack tui` provides a **Cluster** tab in the expanded runtime pane on the +KubeAI backend. It shows Ready/scheduling state, GPU allocation and scheduled +requests across namespaces, GFD labels, runtime evidence, managed Models and +KubeAI/NVIDIA pod startup/restart state. GPU requests are scheduling facts, not +live GPU utilization; the system pane explicitly describes the local host. + +Cluster monitoring uses three batched API reads, cached for at least 15 seconds +and polled only while the tab is visible. Slow probes cannot overlap or block +keyboard input; **Refresh cluster** forces a sample, and **Doctor** runs the +shared detailed checks on demand. Pod samples also feed the Instances view. + +Select a node for **Detach node** or **Attach node**. Each previews its workload +and scheduling state and requires confirmation; detach drains workloads and +emptyDir data, while attach confirms local Compose workloads have stopped. +These reuse `kube node detach/attach` and preserve cluster membership. The +Control tab provides **Apply** through the leasing controller and confirmed +**Down** for managed Models and the gateway. Leases remain after Down. The +KubeAI endpoint editor selects resource profiles; Kubernetes chooses node/GPU +placement. diff --git a/dev/journals/codex.md b/dev/journals/codex.md index f2eff3a2..c6db617a 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -228,3 +228,48 @@ Type checks, CI flake8, scoped Ruff, shell syntax and actual entrypoint help checks passed. Remaining real-host checks include private credential access/ refresh, stale-EKS context isolation, active-worker config inspection, and runtime canaries on aiq-gpu/namek. No real cluster mutation is part of this work. + +## 2026-10-01 20:29:02 -0400 + +User intent: resume the Kubernetes TUI work after the bootstrap review and +make logical commits. Model/configuration: GPT-6 (Codex), Default collaboration +mode; reasoning effort not exposed. User-requested commit attribution: +GPT-6.1-Sol. No sub-agents. + +I separated the review corrections from the TUI changes. The dashboard needs +live scheduling facts cheaply; running full inventory/Helm/API diagnosis every +ledger tick would create avoidable latency and subprocess load. kube.monitor +instead gathers three batched lists, retains independent errors, and computes +scheduled GPU requests across namespaces. It explicitly does not call those +requests utilization. Runtime evidence remains observed pod startup, never a +claim inferred from a cluster RuntimeClass. The cached Cluster tab samples at +least 15 seconds apart only while visible; ledger/instance observation has its +own cadence, and non-overlap guards prevent canceled Textual workers from +leaving duplicate subprocesses behind. Existing node GPU parsing and residency +parsing supply tables and cached Instances data. Doctor is deliberate/on-demand. Log followers pause while their tab is hidden; +returning retains backend action output captured in the meantime. Textual +Expanded/Collapsed messages need their own handlers for immediate gating; +relying on the base Toggled message delayed the update until a timer tick. + +Control reuses the existing lifecycle manager: node actions preview affected +workloads, warn about emptyDir loss or Compose ownership handback, confirm, then +recheck before mutation. Apply stays with the controller; Down works for Model +renders without a local gateway file, confirms Model/gateway teardown, retains +leases, and holds the publication lock. Mutations invalidate caches. The endpoint +editor exposes resource_profile rather than physical GPU indices on KubeAI. +Compose behavior and host metrics remain available; local metrics are explicitly +labeled as local on the cluster backend. The tradeoff is bounded polling rather +than persistent Kubernetes watches: this avoids adding a second client/session +lifecycle, and count-based tests pin the request budget. Small retained runtime +canaries prove an observed invocation, not permanent future handler health. + +Validation: 79 existing TUI tests and 11 new Kubernetes TUI tests passed (90 total), +including hidden log streams and captured action output. The +final combined full suite passed 1206 tests with 3 skips and 8 cleanup/ResourceWarnings. +A final N/A representation check passed 49 review/readiness tests. Type checks, +CI flake8, scoped Ruff and diff whitespace checks passed. A read-only headless +TUI run against this VM's actual K3s cluster showed two node rows and four pod +rows with no probe errors; screenshot /tmp/infer-stack-kubernetes-tui.svg. +No real Apply/Down/node/bootstrap/install action was invoked. Maintainer +validation remains the GPU hosts' credential refresh, actual agent membership, +canary startup, and confirmed node handoff with real workloads. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index ea239945..15bc6903 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -332,3 +332,27 @@ provides a synthetic second K3s node for development; Acquire/wait logs changed readiness states once: pending/scheduler reasons, scheduled node and image, container startup/model loading, replica readiness, and finally generation verification. It does not log every unchanged poll. + +## TUI cluster monitoring and control + +Open `infer-stack tui`, expand runtime (`c`), and select **Cluster**. The cached +node table includes Ready, scheduling, GPU allocation/requests, product/memory, +GFD and observed runtime startup evidence. The pod table includes KubeAI and +NVIDIA components, their nodes, startup reasons, readiness, restarts and GPU +requests. **Refresh cluster** requests a new sample; **Doctor** runs the same +inventory/readiness policy as the CLI once, with actionable fixes. + +Three batched list requests supply the cluster view. Automatic sampling occurs +at most every 15 seconds (or the longer configured observation interval), stops +when hidden, and never overlaps a previous sample. Ledger refreshes remain +independent. Partial RBAC/network failures retain other facts and show unknown +values rather than an empty healthy cluster. Scheduled GPU requests count all +namespaces and exclude completed pods; no remote GPU utilization is inferred. + +Node detach/attach previews and confirmation reuse the CLI's ownership-aware +lifecycle operations. Detach warns about eviction and emptyDir loss; attach +requires the operator to stop local Compose GPU work first. Apply reconciles +through the controller even without a local gateway render; Down confirms +managed Model/gateway teardown and retains leases. Existing Acquire, Release, +Evict and pod log controls remain available. Endpoint editing exposes +`runtime.resource_profile` instead of physical host GPU indices. diff --git a/infer_stack/kube/manage.py b/infer_stack/kube/manage.py index 5471d1e8..9562f750 100644 --- a/infer_stack/kube/manage.py +++ b/infer_stack/kube/manage.py @@ -47,7 +47,7 @@ def default_run( env=env, text=True, capture_output=True, - timeout=360 if ('upgrade' in args or 'wait' in args) else 60, + timeout=360 if ('upgrade' in args or 'wait' in args or 'drain' in args) else 60, check=False, ) if proc.returncode: diff --git a/infer_stack/kube/monitor.py b/infer_stack/kube/monitor.py new file mode 100644 index 00000000..f31b5daf --- /dev/null +++ b/infer_stack/kube/monitor.py @@ -0,0 +1,96 @@ +"""Small, read-only live snapshots for interactive cluster monitoring. + +Three list requests per refresh, independent errors, no per-node/per-pod probes, +Helm inspection, host metrics, or API generation. The caller owns the cadence. +""" +from __future__ import annotations + +import time + +from .manage import KubeManager + + +def gpu_request(pod: dict) -> int: + """Scheduler GPU request including init containers and restartable sidecars.""" + spec = pod.get('spec') or {} + + def request(container): + resources = container.get('resources') or {} + requests = resources.get('requests') or {} + limits = resources.get('limits') or {} + return int(requests.get('nvidia.com/gpu', limits.get('nvidia.com/gpu', 0))) + + steady = sum(request(c) for c in spec.get('containers') or []) + sidecars = peak = 0 + for container in spec.get('initContainers') or []: + value = request(container) + if container.get('restartPolicy') == 'Always': + sidecars += value + peak = max(peak, sidecars) + else: + peak = max(peak, sidecars + value) + return max(steady + sidecars, peak) + int((spec.get('overhead') or {}).get('nvidia.com/gpu', 0)) + + +def snapshot(manager: KubeManager, *, namespace: str) -> dict: + report = {'nodes': None, 'pods': None, 'models': None, 'errors': {}, + 'sampled_at': time.monotonic(), 'namespace': namespace} + for key, args in ( + ('nodes', ['get', 'nodes', '-o', 'json']), + ('pods', ['get', 'pods', '-A', '-o', 'json']), + ('models', ['-n', namespace, 'get', 'models.kubeai.org', + '-l', 'infer-stack/managed=true', '-o', 'json']), + ): + try: + result = manager.kubectl_json(args) + if not isinstance(result.get('items'), list): + raise ValueError(f'{key} response has no items list') + report[key] = result['items'] + except Exception as ex: # independent diagnostic boundary + report['errors'][key] = str(ex) + return report + + +def node_rows(report: dict) -> list[tuple]: + requested: dict[str, int] = {} + for pod in report['pods'] or []: + if pod.get('status', {}).get('phase') in {'Succeeded', 'Failed'}: + continue + node = pod.get('spec', {}).get('nodeName', '') + requested[node] = requested.get(node, 0) + gpu_request(pod) + runtime_nodes = {p.get('spec', {}).get('nodeName') for p in report['pods'] or [] + if p.get('spec', {}).get('runtimeClassName') == 'nvidia' and + any(c.get('state', {}).get('running') or c.get('state', {}).get('terminated') + for c in p.get('status', {}).get('containerStatuses') or [])} + return sorted([ + (r['name'], 'yes' if r['ready'] else 'NO', + 'detached' if r['detached_for_compose'] else 'yes' if r['schedulable'] else 'cordoned', + str(r['gpu_count']), str(requested.get(r['name'], 0)) if report['pods'] is not None else '?', + r['gpu_product'] or '-', str(r['gpu_memory_gib'] or '-'), + 'yes' if r['gpu_product'] and r['gpu_memory_gib'] else 'missing', + 'pod started' if r['name'] in runtime_nodes else 'unknown') + for r in KubeManager.node_rows(report['nodes'] or []) + ]) + + +def pod_rows(report: dict) -> list[tuple]: + rows = [] + for pod in report['pods'] or []: + meta, status = pod.get('metadata', {}), pod.get('status', {}) + containers_spec = pod.get('spec', {}).get('containers') or [] + nvidia = any(any(part in c.get('image', '') for part in ('k8s-device-plugin', 'gpu-feature-discovery')) + for c in containers_spec) + if meta.get('namespace') != report['namespace'] and not nvidia: + continue + containers = status.get('containerStatuses') or [] + reasons = [c.get('state', {}).get('waiting', {}).get('reason') for c in containers] + reasons = [r for r in reasons if r] + if not reasons: + reasons = [c.get('reason') for c in status.get('conditions') or [] + if c.get('status') == 'False' and c.get('reason')] + phase = 'deleting' if meta.get('deletionTimestamp') else status.get('phase', '?') + rows.append((KubeManager._pod_name(pod), pod.get('spec', {}).get('nodeName') or '(pending)', + ', '.join(reasons) or phase, + f"{sum(bool(c.get('ready')) for c in containers)}/{len(pod.get('spec', {}).get('containers') or [])}", + str(sum(c.get('restartCount', 0) for c in containers)), str(gpu_request(pod)))) + return sorted(rows) diff --git a/infer_stack/tui.py b/infer_stack/tui.py index b1ac1b74..4fe3c178 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -34,16 +34,17 @@ from pathlib import Path from typing import Any, Callable +from rich.markup import escape as escape_markup from textual import events, work from textual.app import App, ComposeResult, SystemCommand from textual.binding import Binding -from textual.coordinate import Coordinate from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.coordinate import Coordinate from textual.css.query import NoMatches from textual.lazy import Lazy -from textual.widget import Widget from textual.screen import ModalScreen from textual.theme import Theme +from textual.widget import Widget from textual.widgets import ( Button, Collapsible, @@ -60,6 +61,7 @@ ) from . import __version__ +from . import cli_equivalent as cli from .cli.commands_leasing import ( _gpu_label, _lease_ttl, @@ -67,9 +69,6 @@ _running_label, ) from .leasing import DeploymentState, LeaseState -from rich.markup import escape as escape_markup - -from . import cli_equivalent as cli from .log_filter import compact_litellm_tracebacks ALL_SERVICES = '' # the Select value meaning "every service" @@ -122,6 +121,9 @@ '_do_cleanup': 'cleaning up the ledger', '_do_apply': 'applying', '_do_compose': 'docker compose', + '_do_kube_doctor': 'checking Kubernetes readiness', + '_prepare_node_action': 'inspecting node', + '_do_node_action': 'updating node', '_save_endpoint': 'saving endpoint', '_do_suggest': 'inspecting GPUs for a suggestion', '_prepare_endpoint_editor': 'inspecting GPUs', @@ -370,11 +372,13 @@ def __init__( name: str | None = None, entry: dict | None = None, inventory: dict[str, Any] | None = None, + kubeai: bool = False, ): super().__init__() self._models = models self._edit_name = name self._entry = entry or {} + self._kubeai = kubeai self._gpus = sorted( (inventory or {}).get('gpus', []), key=lambda g: int(g.get('index', 0)), @@ -453,7 +457,7 @@ def compose(self) -> ComposeResult: yield Input(value=cur_model or '', placeholder='model name', id='e-model-text') yield Label('engine') - yield Select([('vllm', 'vllm'), ('ollama', 'ollama')], + yield Select([('vllm', 'vllm')] if self._kubeai else [('vllm', 'vllm'), ('ollama', 'ollama')], value=cur_engine, allow_blank=False, id='e-engine') with Vertical(id='vllm-opts'): yield Label('tensor-parallel size (GPUs per replica)') @@ -468,10 +472,17 @@ def compose(self) -> ComposeResult: yield Label('GPU memory utilization (0-1, per GPU)') yield Input(value=self._rt('gpu_memory_utilization'), placeholder='float, e.g. 0.9', id='e-gpu') - yield Label('GPU placement (auto or exact physical indices)') - yield Static(self._gpu_hint(), classes='hint') - yield Input(value=self._gpu_pin_value(), - placeholder='auto, 1, or 0,2', id='e-gpu-pin') + if self._kubeai: + yield Label('KubeAI resource profile') + yield Static('Kubernetes chooses nodes/GPUs. Use an installed profile name ' + '(with optional :GPU-count), or leave blank for configured default.', classes='hint') + yield Input(value=self._rt('resource_profile'), + placeholder='e.g. gpu-single-default:1', id='e-resource-profile') + else: + yield Label('GPU placement (auto or exact physical indices)') + yield Static(self._gpu_hint(), classes='hint') + yield Input(value=self._gpu_pin_value(), + placeholder='auto, 1, or 0,2', id='e-gpu-pin') yield Label('max concurrent sequences') yield Input(value=self._rt('max_num_seqs'), placeholder='int, optional', id='e-seqs') @@ -607,7 +618,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: dict(self._entry.get('placement') or {}) if self._entry.get('engine', 'vllm') == 'vllm' else {} ) - indices = self._parse_gpu_indices(self._v('e-gpu-pin')) + indices = None if self._kubeai else self._parse_gpu_indices(self._v('e-gpu-pin')) required = self._required_gpu_count(tensor_parallel, data_parallel) if indices is not None and len(indices) != required: raise ValueError( @@ -625,6 +636,8 @@ def on_button_pressed(self, event: Button.Pressed) -> None: placement.pop('gpu_indices', None) else: placement['gpu_indices'] = indices + if self._kubeai: + result['resource_profile'] = self._v('e-resource-profile').strip() result.update({ 'tensor_parallel': tensor_parallel, 'data_parallel': data_parallel, @@ -682,16 +695,17 @@ class _ConfirmScreen(ModalScreen): #dialog Button { margin: 0 0 0 2; } """ - def __init__(self, message: str): + def __init__(self, message: str, *, confirm_label: str = 'Remove'): super().__init__() self._message = message + self._confirm_label = confirm_label def compose(self) -> ComposeResult: with Vertical(id='dialog'): yield Label(self._message) with Horizontal(): yield Button('Cancel', id='cancel') - yield Button('Remove', variant='error', id='ok') + yield Button(self._confirm_label, variant='error', id='ok') def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss(event.button.id == 'ok') @@ -776,6 +790,9 @@ class InferStackTUI(App): } #api { padding: 1 2; } + .cluster-actions { height: auto; } + #cluster-nodes { height: 8; min-height: 4; } + #cluster-pods { height: 10; min-height: 4; } #endpoints, #models, #leases-pane, #deployments-pane, #docker, #system, #api { border: round $surface; background: $boost; @@ -884,6 +901,7 @@ def __init__( catalog_path: str | Path | None = None, http: Any = None, exit_after_paint: bool = False, + kube_manager: Any = None, ) -> None: super().__init__() #: Quit once the first frame is drawn (``tui --exit_after_paint``). @@ -893,6 +911,21 @@ def __init__( self.controller = controller self.catalog = catalog self.interval = interval + from .backends.kubeai import KubeaiBackend + self._is_kube = isinstance(controller.backend, KubeaiBackend) + self._kube_manager = kube_manager + self._cluster_visible = False + self._dashboard_visible = True + self._logs_visible = False + self._cluster_at: float | None = None + self._cluster_snapshot: dict | None = None + self._cluster_inflight = False + self._node_names: list[str] = [] + self._node_rows_cache: list[tuple] = [] + self._cluster_pods_cache: list[tuple] = [] + self._instances_at: float | None = None + self._instances_cache: list | None = None + self._instance_warning = '' # Two cadences (see the TUI settings tab): the ledger is cheap in-memory state, so # it drives the visible refresh; ``observe()``/``plan()`` shell out to # docker, so they run on a slower beat and their result is cached between @@ -1111,6 +1144,21 @@ def _compose_dashboard(self) -> ComposeResult: with TabPane('Instances', id='tab-containers'): yield DataTable(id='ps', cursor_type='row', zebra_stripes=True) + if self._is_kube: + with TabPane('Cluster', id='tab-cluster'): + with VerticalScroll(): + yield Static('Open this tab to inspect Kubernetes.', id='cluster-summary') + with Horizontal(classes='cluster-actions'): + yield Button('Refresh cluster', id='btn-cluster-refresh') + yield Button('Doctor', id='btn-kube-doctor') + yield DataTable(id='cluster-nodes', cursor_type='row', zebra_stripes=True) + with Horizontal(classes='cluster-actions'): + yield Button('Detach node', id='btn-node-detach') + yield Button('Attach node', id='btn-node-attach') + yield Static('Detach drains workloads and temporary data; attach restores scheduling. ' + 'Both preview and confirm. Cluster agents keep running.', classes='hint') + yield DataTable(id='cluster-pods', cursor_type='row', zebra_stripes=True) + yield Static('', id='cluster-doctor') with TabPane('Control', id='tab-control'): yield Static( 'Apply brings up what the ledger says should run. ' @@ -1269,6 +1317,16 @@ def on_mount(self) -> None: self.query_one('#compose-path', Static).update( f'rendered: {rendered or "(this backend renders no file)"}' ) + if self._is_kube: + self.query_one('#cluster-nodes', DataTable).add_columns( + 'node', 'Ready', 'scheduling', 'GPUs allocatable', 'GPUs requested', 'product', 'GiB', 'GFD', 'runtime evidence') + self.query_one('#cluster-pods', DataTable).add_columns( + 'pod', 'node', 'phase / reason', 'Ready', 'restarts', 'GPUs requested') + self.query_one('#compose-path', Static).update( + f'Backend: kubeai · recovery snapshot: {(self.controller.ledger.profile() or {}).get("backend", "none")} · ' + f'namespace: {self.controller.backend.namespace} · ' + f'API: {self.controller.backend.base_url} · Kubernetes schedules Models') + self.query_one('#system', Collapsible).title = 'local host (not cluster metrics)' # Capture docker's own chatter (up/down progress on stderr) into the # logs pane instead of letting it bleed onto the full-screen terminal. self._install_quiet_docker() @@ -1687,15 +1745,25 @@ def _collect(self) -> dict[str, Any]: now = time.monotonic() if (self._observed_at is None or (now - self._observed_at) >= self.observe_interval): - self._observed, self._assignments = _placement_view(self.controller) + if (self._is_kube and self._cluster_snapshot is not None + and self._cluster_snapshot['models'] is not None + and self._cluster_at is not None and now - self._cluster_at < self.observe_interval): + self._observed = {m.get('metadata', {}).get('labels', {}).get('infer-stack/deployment') + for m in self._cluster_snapshot['models']} - {None, ''} + self._assignments = {} + else: + self._observed, self._assignments = _placement_view(self.controller) self._observed_at = now data: dict[str, Any] = { 'leases': leases, 'deployments': deployments, 'observed': self._observed, 'assignments': self._assignments, } - if not self._collapsed['docker']: - data['instances'] = self._instance_list() - if not self._collapsed['system']: + if self._dashboard_visible and not self._collapsed['docker']: + if self._instances_at is None or now - self._instances_at >= self.observe_interval: + self._instances_cache = self._instance_list() + self._instances_at = now + data['instances'] = self._instances_cache + if self._dashboard_visible and not self._collapsed['system']: data['gpus'] = self._gpu_rows() data['sysinfo'] = self._system_line() return data @@ -1741,17 +1809,21 @@ def _sync_pane_state(self) -> None: in ``_collect`` reflects reality — Collapsible.Toggled doesn't fire on every path, so don't depend on it alone.""" try: - was_collapsed = self._collapsed['docker'] + was_logs_visible = self._logs_visible self._collapsed['docker'] = \ self.query_one('#docker', Collapsible).collapsed self._collapsed['system'] = \ self.query_one('#system', Collapsible).collapsed self._active_tab = self.query_one('#docker-tabs', TabbedContent).active + dashboard = len(self.screen_stack) == 1 and self.query_one('#top', TabbedContent).active == 'tab-dashboard' + self._dashboard_visible = dashboard + self._cluster_visible = (self._is_kube and dashboard and + not self._collapsed['docker'] and self._active_tab == 'tab-cluster') # Follow logs only while somebody can see them. - if was_collapsed and not self._collapsed['docker']: - if self._log_proc is None: - self._restart_logs(self._log_service) - elif not was_collapsed and self._collapsed['docker']: + self._logs_visible = dashboard and not self._collapsed['docker'] and self._active_tab == 'tab-logs' + if self._logs_visible and not was_logs_visible: + self._resume_visible_logs() + elif was_logs_visible and not self._logs_visible: self._terminate_logs() except Exception: # noqa: BLE001 pass @@ -1821,12 +1893,18 @@ def action_refresh_now(self) -> None: rereads the catalog, edited or not.""" self._cli(cli.command('status')) self._reload_catalog() + self._observed_at = self._instances_at = self._cluster_at = None self.action_refresh() def action_refresh(self) -> None: self._sync_pane_state() # capture pane state on the UI thread first self._reload_catalog_if_changed() - self._refresh_bg() + # Textual exclusive workers cancel delivery, not an in-flight subprocess. + # Never spawn overlapping refreshes when Kubernetes is slow/unreachable. + if not any(w.group == 'refresh' and not w.is_finished for w in self.workers): + self._refresh_bg() + if self._cluster_visible: + self._request_cluster_refresh() def _update_summary(self, leases, deployments, observed) -> None: active = sum(1 for le in leases if str(le.state) == 'active') @@ -2058,7 +2136,28 @@ def _system_line(self) -> str: def _instance_list(self): """What the backend runs (worker thread), or ``None`` if unreadable.""" try: - return list(self.controller.backend.instances()) + backend = self.controller.backend + report = self._cluster_snapshot + self._instance_warning = '' + if (self._is_kube and report is not None and report['pods'] is not None + and self._cluster_at is not None + and time.monotonic() - self._cluster_at < max(15, self.observe_interval)): + import json + + from .leasing.instances import KUBERNETES, from_residency + from .leasing.residency import residency_from_pods + pods = [p for p in report['pods'] + if p.get('metadata', {}).get('namespace') == backend.namespace + and p.get('metadata', {}).get('labels', {}).get('infer-stack/managed') == 'true'] + instances = from_residency(residency_from_pods(json.dumps({'items': pods})), + runtime=KUBERNETES, namespace=backend.namespace) + if backend.gateway is not None: + try: + instances += list(backend.gateway.instances()) + except Exception as ex: + self._instance_warning = f'Gateway instances unavailable: {ex}' + return instances + return list(backend.instances()) except Exception: # noqa: BLE001 - a monitor must never crash return None @@ -2118,7 +2217,8 @@ def _sync_log_services(self) -> None: # begun with an engine that has since left would miss its successor. # The selection did not change, so no Select.Changed will restart it. if (self._log_service == ENGINE_SERVICES - and not self._collapsed['docker']): + and not self._collapsed['docker'] + and self._dashboard_visible and self._active_tab == 'tab-logs'): self._restart_logs(ENGINE_SERVICES) def on_select_changed(self, event: Select.Changed) -> None: @@ -2174,6 +2274,16 @@ def _restart_logs(self, service: str) -> None: log.write(f'— following logs: {label} —') self._stream_logs(target, generation) + def _resume_visible_logs(self): + # A source change deliberately clears the pane; reopening the same + # pane retains backend action output captured while its stream paused. + previous = list(self._log_lines[-LOG_PANE_LINES:]) + self._restart_logs(self._log_service) + if previous: + self.query_one('#logs', RichLog).write('— earlier runtime output —') + self._write_log_lines(previous) + self.query_one('#logs', RichLog).write('— live stream resumed —') + def _resolve_log_target(self, service: str): """(what to hand `docker compose logs`, what to show the user). @@ -2429,6 +2539,9 @@ def on_tabbed_content_tab_activated( ) -> None: # Only the docker sub-tabs gate polling; the top-level Dashboard/Settings # tabs fire this too — ignore those. + if getattr(event.tabbed_content, 'id', None) == 'top': + self.action_refresh() + return if getattr(event.tabbed_content, 'id', None) != 'docker-tabs': return self._active_tab = self.query_one('#docker-tabs', TabbedContent).active @@ -2436,12 +2549,14 @@ def on_tabbed_content_tab_activated( def on_collapsible_toggled(self, event: Collapsible.Toggled) -> None: self._sync_pane_state() - if self._collapsed['docker']: - self._terminate_logs() - elif self._log_proc is None: - self._restart_logs(self._log_service) self.action_refresh() + def on_collapsible_expanded(self, event: Collapsible.Expanded) -> None: + self.on_collapsible_toggled(event) + + def on_collapsible_collapsed(self, event: Collapsible.Collapsed) -> None: + self.on_collapsible_toggled(event) + # -- helpers + actions ------------------------------------------------- #: Passive cursor/relationship hints; not worth logging as events. @@ -2706,6 +2821,10 @@ def on_button_pressed(self, event: Button.Pressed) -> None: 'btn-apply-ui-settings': self._on_apply_ui_settings, 'btn-compose-up': self.action_compose_up, 'btn-compose-down': self.action_compose_down, + 'btn-cluster-refresh': self.action_cluster_refresh, + 'btn-kube-doctor': self.action_kube_doctor, + 'btn-node-detach': lambda: self.action_node_control('detach'), + 'btn-node-attach': lambda: self.action_node_control('attach'), } button = event.button.id or 'button' handler = handlers.get(button) @@ -2805,6 +2924,142 @@ def action_cleanup(self) -> None: self._cli(cli.command('gc', '--forget')) self._do_cleanup() + # -- Kubernetes dashboard --------------------------------------------- + + def _cluster_manager(self): + if self._kube_manager is None: + from .backends.kubeai import _default_kubectl_run + from .kube.manage import KubeManager, default_run + backend = self.controller.backend + # Default dashboard reads have a short bounded deadline. Tests and + # embedding applications retain the backend's injectable runner. + if backend.run is _default_kubectl_run: + def run(args, **kw): + if args[0] == 'kubectl': + args = [args[0], '--request-timeout=5s', *args[1:]] + return default_run(args, **kw) + else: + run = backend.run + self._kube_manager = KubeManager(run=run) + return self._kube_manager + + def _request_cluster_refresh(self, *, force=False): + if not self._is_kube or self._cluster_inflight or any(w.group == 'kube-doctor' and not w.is_finished for w in self.workers): + return + if not force and self._cluster_at is not None and time.monotonic() - self._cluster_at < max(15, self.observe_interval): + if self._cluster_snapshot is not None: + self._render_cluster(self._cluster_snapshot) + return + self._cluster_inflight = True + self._collect_cluster() + + @work(thread=True, group='cluster', exit_on_error=False) + def _collect_cluster(self): + from .kube.monitor import snapshot + try: + report = snapshot(self._cluster_manager(), namespace=self.controller.backend.namespace) + self.call_from_thread(self._receive_cluster, report) + finally: + self._cluster_inflight = False + + def _receive_cluster(self, report): + self._cluster_snapshot = report + self._cluster_at = time.monotonic() + self._render_cluster(report) + + def _render_cluster(self, report): + from .kube.monitor import node_rows, pod_rows + nodes = node_rows(report) + self._node_names = [r[0] for r in nodes] + self._diff_fill(self.screen_stack[0].query_one('#cluster-nodes', DataTable), + nodes or [('(unavailable)' if report['nodes'] is None else '(no nodes)',) + ('-',) * 8], + '_node_rows_cache', id_index=0) + pods = pod_rows(report) + self._diff_fill(self.screen_stack[0].query_one('#cluster-pods', DataTable), + pods or [('(unavailable)' if report['pods'] is None else '(no pods)',) + ('-',) * 5], + '_cluster_pods_cache', id_index=0) + models = report['models'] + model_text = 'unknown' if models is None else ', '.join( + f"{m.get('metadata', {}).get('name')}: {m.get('status', {}).get('readyReplicas', 0)} ready" + for m in models) or 'none' + age = max(0, int(time.monotonic() - report['sampled_at'])) + summary = (f'Namespace: {report["namespace"]} · sample {age}s ago · refresh ≥15s, only while visible\n' + 'GPU requests are scheduled resources across all namespaces, not utilization.\n' + f'Managed Models: {model_text}\nAPI: {self.controller.backend.base_url}') + if report['errors']: + summary += '\n' + '\n'.join(f'{key}: {value}' for key, value in report['errors'].items()) + if self._instance_warning: + summary += '\n' + self._instance_warning + self.screen_stack[0].query_one('#cluster-summary', Static).update(escape_markup(summary)) + + def action_cluster_refresh(self): + self._status('refreshing Kubernetes cluster…') + self._cli(cli.command('kube', 'status')) + self._request_cluster_refresh(force=True) + + def action_kube_doctor(self): + if not self._is_kube: + return + if any(w.group == 'kube-doctor' and not w.is_finished for w in self.workers): + return + self._cli(cli.command('kube', 'doctor', f'--namespace={self.controller.backend.namespace}')) + self._do_kube_doctor() + + @work(thread=True, group='kube-doctor', exit_on_error=False) + def _do_kube_doctor(self): + from .kube.inspect import inventory, readiness + backend = self.controller.backend + report = inventory(self._cluster_manager(), namespace=backend.namespace, + base_url=backend.base_url, http=backend.http) + checks = readiness(report) + text = '\n'.join(f"[{c['status']}] {c['name']} {c['detail']}" + + (f"\n fix: {c['fix']}" if c['fix'] else '') for c in checks) + self.call_from_thread(self._show_cluster_doctor, text) + self.call_from_thread(self.app_log, escape_markup(text)) + + def _show_cluster_doctor(self, text): + self.screen_stack[0].query_one('#cluster-doctor', Static).update(escape_markup(text)) + + def action_node_control(self, verb): + name = self._selected('cluster-nodes', self._node_names) + if not name: + self._refuse('select a cluster node first') + return + self._prepare_node_action(name, verb) + + @work(thread=True, exclusive=True, group='node-plan', exit_on_error=False) + def _prepare_node_action(self, name, verb): + plan = self._cluster_manager().node_lifecycle_plan(name) + message = (f'{verb.title()} node {name}? Ready: {plan.ready}; ' + f'schedulable: {plan.schedulable}; control plane: {plan.control_plane}.\n' + f'Workload pods: {", ".join(plan.workload_pods) or "none"}.\n') + if verb == 'detach': + message += 'Drain evicts workloads and deletes emptyDir data; DaemonSets/static pods remain. ' + else: + message += 'Stop Compose GPU workloads on that host before restoring scheduling. ' + message += 'The cluster agent remains running. State is rechecked before applying.' + self.call_from_thread(self._confirm_node_action, name, verb, message) + + def _confirm_node_action(self, name, verb, message): + self.push_screen(_ConfirmScreen(message, confirm_label=verb.title()), + lambda confirmed: self._start_node_action(name, verb) if confirmed else None) + + def _start_node_action(self, name, verb): + if any(w.group == 'mutate' and not w.is_finished for w in self.workers): + self._refuse('another runtime action is still running; retry node control when it finishes') + return + self._do_node_action(name, verb) + + @work(thread=True, exclusive=True, group='mutate', exit_on_error=False) + def _do_node_action(self, name, verb): + self.call_from_thread(self._cli, cli.command('kube', 'node', verb, name, '--yes')) + manager = self._cluster_manager() + if verb == 'detach': + manager.detach_node_for_compose(name) + else: + manager.attach_node_from_compose(name) + self._after_mutation(f'node {name}: {verb} done') + # -- docker compose control ------------------------------------------- def _compose_target(self): @@ -2813,7 +3068,7 @@ def _compose_target(self): return rendered if rendered is not None and Path(rendered).exists() else None def action_compose_up(self) -> None: - if self._compose_target() is None: + if not self._is_kube and self._compose_target() is None: self._refuse('nothing rendered yet — acquire a model first') return self._status('apply… (output in the Logs tab)') @@ -2834,17 +3089,33 @@ def _do_apply(self) -> None: self._after_mutation(msg) def action_compose_down(self) -> None: - if self._compose_target() is None: + if any(w.group == 'mutate' and not w.is_finished for w in self.workers): + self._refuse('another runtime action is still running; retry Down when it finishes') + return + if not self._is_kube and self._compose_target() is None: self._refuse('nothing rendered yet — nothing to bring down') return self._status('down (bypasses leases; releases nothing)…') self._cli(cli.command('stack', 'down')) + if self._is_kube: + self.push_screen(_ConfirmScreen( + f'Delete all managed KubeAI Models in {self.controller.backend.namespace} ' + 'and stop the gateway? Leases remain; Apply brings them back.', confirm_label='Down'), + lambda confirmed: self._confirmed_down() if confirmed else None) + else: + self._do_down() + + def _confirmed_down(self): + if any(w.group == 'mutate' and not w.is_finished for w in self.workers): + self._refuse('another runtime action is still running; retry Down when it finishes') + return self._do_down() @work(thread=True, exclusive=True, group='mutate') def _do_down(self) -> None: try: - self.controller.backend.down() + with self.controller.publication_lock(): + self.controller.backend.down() msg = 'down done' except Exception as ex: # noqa: BLE001 msg = f'down failed: {_why(ex)}' @@ -2950,7 +3221,7 @@ def action_edit_endpoint(self) -> None: def _prepare_endpoint_editor(self, name: str | None, entry: dict) -> None: try: from .hardware import detect_inventory - inventory = detect_inventory() + inventory = {'gpus': []} if self._is_kube else detect_inventory() except Exception: # noqa: BLE001 - editor remains usable with numeric pins inventory = {'gpu_count': 0, 'gpus': []} try: @@ -2970,6 +3241,7 @@ def _show_endpoint_editor( name=name, entry=entry, inventory=inventory, + kubeai=self._is_kube, ), self._on_add_endpoint, ) @@ -3017,6 +3289,10 @@ def _endpoint_entry(result: dict) -> dict: for rk, ck in keymap.items(): if result.get(rk) is not None: runtime[ck] = result[rk] + if 'resource_profile' in result: + runtime.pop('resource_profile', None) + if result['resource_profile']: + runtime['resource_profile'] = result['resource_profile'] if result.get('gpu_mem') is not None: runtime['gpu_memory_utilization'] = result['gpu_mem'] if result.get('prefix_caching') in ('on', 'off'): @@ -3531,6 +3807,7 @@ def _do_cleanup(self) -> None: def _after_mutation(self, message: str) -> None: self.call_from_thread(self._status, message) + self._observed_at = self._instances_at = self._cluster_at = None self.call_from_thread(self.action_refresh) diff --git a/tests/test_tui_kube.py b/tests/test_tui_kube.py new file mode 100644 index 00000000..5c1be60e --- /dev/null +++ b/tests/test_tui_kube.py @@ -0,0 +1,328 @@ +"""Efficient Kubernetes dashboard and deliberate lifecycle controls.""" +import asyncio +import json +import threading +import time + +import pytest + +pytest.importorskip('textual') + +from test_kube_readiness import Cluster, manager, node +from test_leasing_kubeai import FakeHttp, FakeKubectl +from test_tui import _ctx +from textual.widgets import ( + Button, + Collapsible, + DataTable, + Input, + Select, + Static, + TabbedContent, +) + +from infer_stack.backends.kubeai import KubeaiBackend +from infer_stack.kube.manage import KubeManager +from infer_stack.kube.monitor import gpu_request, node_rows, pod_rows, snapshot +from infer_stack.leasing import Controller, Ledger, SqliteStore +from infer_stack.tui import InferStackTUI, _AddEndpointScreen, _ConfirmScreen + + +def context(tmp_path): + kubectl = FakeKubectl() + backend = KubeaiBackend(state_dir=tmp_path, namespace='default', run=kubectl, + http=FakeHttp(kubectl)) + _, catalog = _ctx() + return Controller(Ledger(SqliteStore(':memory:')), backend), catalog + + +def test_snapshot_budget_and_independent_failure(): + calls = [] + def run(args): + calls.append(args) + if 'nodes' in args: + return json.dumps({'items': [node()]}) + if 'pods' in args: + raise RuntimeError('Forbidden all-namespaces pods') + return json.dumps({'items': [{'metadata': {'name': 'q'}}]}) + report = snapshot(KubeManager(run=run), namespace='default') + assert len(calls) == 3 + assert report['nodes'] and report['models'] + assert report['pods'] is None + assert node_rows(report)[0][4] == '?' + assert node_rows(report)[0][-1] == 'unknown' + assert 'Forbidden' in report['errors']['pods'] + assert not any('helm' in c or 'nvidia-smi' in c for c in calls) + + +def test_gpu_requests_and_pod_diagnosis_include_other_namespace_usage(): + pod = {'metadata': {'name': 'other', 'namespace': 'other'}, + 'spec': {'nodeName': 'gpu-a', 'containers': [{'resources': {'limits': {'nvidia.com/gpu': 2}}}], + 'initContainers': [{'resources': {'requests': {'nvidia.com/gpu': 3}}}]}, + 'status': {'phase': 'Running'}} + assert gpu_request(pod) == 3 + report = {'namespace': 'default', 'nodes': [node()], 'pods': [pod]} + assert node_rows(report)[0][4] == '3' + assert pod_rows(report) == [] + pod['metadata']['namespace'] = 'default' + pod['status']['containerStatuses'] = [{'ready': False, 'restartCount': 4, + 'state': {'waiting': {'reason': 'ImagePullBackOff'}}}] + row = pod_rows(report)[0] + assert row[:3] == ('default/other', 'gpu-a', 'ImagePullBackOff') + assert row[4] == '4' + pod['status']['phase'] = 'Succeeded' + assert node_rows(report)[0][4] == '0' + + +def test_restartable_init_sidecars_count_toward_scheduled_request(): + def container(count): + return {'resources': {'requests': {'nvidia.com/gpu': count}}} + pod = {'spec': {'containers': [container(2)], + 'initContainers': [dict(container(1), restartPolicy='Always'), container(4)]}} + assert gpu_request(pod) == 5 + + +def test_cluster_visible_only_cache_manual_refresh_and_partial_data(tmp_path): + controller, catalog = context(tmp_path) + cluster = Cluster() + m = manager(cluster) + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, kube_manager=m) + async with app.run_test(size=(180, 55)) as pilot: + await pilot.pause() + assert not cluster.calls # no kube inventory at mount/hidden pane + app.query_one('#docker', Collapsible).collapsed = False + app.query_one('#docker-tabs', TabbedContent).active = 'tab-cluster' + await pilot.pause() + await app.workers.wait_for_complete() + assert app.query_one('#cluster-nodes', DataTable).row_count == 1 + count = len(cluster.calls) + assert count == 3 + app.action_refresh() + await app.workers.wait_for_complete() + assert len(cluster.calls) == count + app.action_cluster_refresh() + await app.workers.wait_for_complete() + assert len(cluster.calls) == count + 3 + app._cluster_at = None + app.query_one('#top', TabbedContent).active = 'tab-api' + await pilot.pause() + app.action_refresh() + await app.workers.wait_for_complete() + assert len(cluster.calls) == count + 3 + app.query_one('#top', TabbedContent).active = 'tab-dashboard' + app.query_one('#docker', Collapsible).collapsed = True + await pilot.pause() + app.action_refresh() + await app.workers.wait_for_complete() + assert len(cluster.calls) == count + 3 + report = snapshot(KubeManager(run=lambda args: (_ for _ in ()).throw(RuntimeError('unreachable'))), namespace='default') + app._receive_cluster(report) + assert 'unreachable' in str(app.query_one('#cluster-summary', Static).render()) + assert '(unavailable)' in str(app.query_one('#cluster-nodes', DataTable).get_row_at(0)) + asyncio.run(scenario()) + + +def test_slow_cluster_probe_never_overlaps_or_blocks_ui(tmp_path): + controller, catalog = context(tmp_path) + gate, entered = threading.Event(), threading.Event() + calls = [] + def run(args): + calls.append(args) + entered.set() + gate.wait(5) + return '{"items": []}' + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, + kube_manager=KubeManager(run=run)) + async with app.run_test(size=(180, 55)) as pilot: + app.action_cluster_refresh() + await pilot.pause(0.1) + assert entered.is_set() + for _ in range(4): + app.action_cluster_refresh() + await pilot.press('2') + await pilot.pause() + assert app.query_one('#top', TabbedContent).active == 'tab-api' + assert len(calls) == 1 + gate.set() + await app.workers.wait_for_complete() + assert len(calls) == 3 + try: + asyncio.run(scenario()) + finally: + gate.set() + + +def test_node_control_previews_confirms_and_reuses_manager(tmp_path): + from infer_stack.kube.manage import NodeLifecyclePlan + controller, catalog = context(tmp_path) + calls = [] + m = KubeManager(run=lambda args: '{"items": []}') + m.node_lifecycle_plan = lambda name: calls.append(('plan', name)) or NodeLifecyclePlan( + name, True, True, None, False, workload_pods=['default/q']) + m.detach_node_for_compose = lambda name: calls.append(('detach', name)) + m.attach_node_from_compose = lambda name: calls.append(('attach', name)) + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, kube_manager=m) + async with app.run_test(size=(180, 55)) as pilot: + await pilot.pause() + app._node_names = ['namek'] + app.query_one('#cluster-nodes', DataTable).add_row('namek', *['-'] * 8) + app.action_node_control('detach') + await app.workers.wait_for_complete() + assert isinstance(app.screen, _ConfirmScreen) + assert 'default/q' in app.screen._message + assert 'emptyDir' in app.screen._message + assert calls == [('plan', 'namek')] + app.screen.dismiss(False) + await pilot.pause() + assert calls == [('plan', 'namek')] + app.action_node_control('attach') + await app.workers.wait_for_complete() + assert isinstance(app.screen, _ConfirmScreen) + app.screen.dismiss(True) + await pilot.pause() + await app.workers.wait_for_complete() + assert calls[-1] == ('attach', 'namek') + assert any('infer-stack kube node attach namek --yes' in line for line in app._app_log_lines) + asyncio.run(scenario()) + + +def test_kube_control_without_gateway_render_confirms_down(tmp_path): + controller, catalog = context(tmp_path) + calls = [] + controller.apply_now = lambda: calls.append('apply') or type('Result', (), {'publication_pending': False})() + controller.backend.down = lambda: calls.append('down') + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, + kube_manager=manager(Cluster())) + async with app.run_test() as pilot: + await pilot.pause() + assert app._compose_target() is None + app.action_compose_up() + await app.workers.wait_for_complete() + assert calls == ['apply'] + app.action_compose_down() + assert isinstance(app.screen, _ConfirmScreen) + assert 'managed KubeAI Models in default' in app.screen._message + assert calls == ['apply'] + app.screen.dismiss(True) + await pilot.pause() + await app.workers.wait_for_complete() + assert calls == ['apply', 'down'] + asyncio.run(scenario()) + + +def test_kube_endpoint_editor_uses_resource_profiles(tmp_path): + controller, catalog = context(tmp_path) + results = [] + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None) + async with app.run_test(size=(100, 45)) as pilot: + screen = _AddEndpointScreen(['qc'], kubeai=True, name='q', + entry={'engine': 'vllm', 'model': 'qc', + 'runtime': {'resource_profile': 'gpu-single-default:1'}}) + app.push_screen(screen, results.append) + await pilot.pause() + assert not list(screen.query('#e-gpu-pin')) + assert screen.query_one('#e-engine', Select).value == 'vllm' + screen.query_one('#e-resource-profile', Input).value = 'nvidia-rtx-3090:2' + screen.on_button_pressed(Button.Pressed(Button(id='ok'))) + await pilot.pause() + asyncio.run(scenario()) + entry = InferStackTUI._endpoint_entry(results[0]) + assert entry['runtime']['resource_profile'] == 'nvidia-rtx-3090:2' + assert 'placement' not in entry + + +def test_visible_runtime_instances_share_cadence_and_cluster_pod_data(tmp_path): + controller, catalog = context(tmp_path) + calls = [] + controller.backend.instances = lambda: calls.append('instances') or [] + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, + kube_manager=manager(Cluster())) + async with app.run_test(size=(180, 55)) as pilot: + await pilot.pause() + app.query_one('#docker', Collapsible).collapsed = False + app.query_one('#docker-tabs', TabbedContent).active = 'tab-containers' + await pilot.pause() + await app.workers.wait_for_complete() + app._refresh_now() + app._refresh_now() + assert calls == ['instances'] + report = {'nodes': [], 'models': [], 'errors': {}, 'namespace': 'default', + 'sampled_at': time.monotonic(), + 'pods': [{'metadata': {'name': 'q-pod', 'namespace': 'default', + 'labels': {'infer-stack/managed': 'true', 'infer-stack/deployment': 'g'}}, + 'spec': {'containers': [{'name': 'vllm'}]}, + 'status': {'containerStatuses': [{'state': {'running': {'startedAt': 'now'}}}]}}]} + app._receive_cluster(report) + app._instances_at = None + data = app._collect() + assert [i.name for i in data['instances']] == ['q-pod'] + assert calls == ['instances'] + app._refresh_now() + assert app.query_one('#ps', DataTable).row_count == 1 + asyncio.run(scenario()) + + +def test_refresh_does_not_spawn_duplicate_slow_observation(tmp_path): + controller, catalog = context(tmp_path) + gate = threading.Event() + calls = [] + def observe(): + calls.append('observe') + gate.wait(5) + return set() + controller.backend.observe = observe + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None) + async with app.run_test() as pilot: + await pilot.pause(0.1) + for _ in range(4): + app.action_refresh() + await pilot.press('2') + await pilot.pause() + assert len(calls) == 1 + assert app.query_one('#top', TabbedContent).active == 'tab-api' + gate.set() + await app.workers.wait_for_complete() + try: + asyncio.run(scenario()) + finally: + gate.set() + + +def test_log_followers_pause_while_cluster_or_other_top_tab_is_visible(tmp_path): + controller, catalog = context(tmp_path) + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, + kube_manager=manager(Cluster())) + async with app.run_test(size=(180, 55)) as pilot: + await pilot.pause() + calls = [] + app._restart_logs = lambda svc: calls.append('start') + app._terminate_logs = lambda: calls.append('stop') + app.query_one('#docker', Collapsible).collapsed = False + await pilot.pause() + assert 'start' in calls + calls.clear() + app.query_one('#docker-tabs', TabbedContent).active = 'tab-cluster' + await pilot.pause() + assert calls == ['stop'] + calls.clear() + app._sync_log_services() + app._sync_pane_state() + assert calls == [] + app._write_log_lines(['apply output captured while logs paused']) + app.query_one('#docker-tabs', TabbedContent).active = 'tab-logs' + await pilot.pause() + assert calls == ['start'] + assert 'apply output captured while logs paused' in app._log_lines + calls.clear() + app.query_one('#top', TabbedContent).active = 'tab-api' + await pilot.pause() + assert calls == ['stop'] + asyncio.run(scenario()) From aee7d9792ec80bec5c4e3f32ec8e1ba99fb2ddd7 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 20:58:21 -0400 Subject: [PATCH 134/141] Respect KubeAI Model name limits with stable hashed shortening Budget static and dynamic Model names against the CRD limit of 40 characters while preserving short names and dedicated deployment identities. Cover the unchanged long SmolLM2 e2e identity and matching gateway routes. Co-authored-by: GPT-6.1-Sol --- dev/journals/codex.md | 27 ++++++++++++++++++++++ docs/kubeai-backend.md | 6 +++++ infer_stack/backends/kubeai.py | 20 ++++++++++++---- tests/test_leasing_kubeai.py | 42 ++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/dev/journals/codex.md b/dev/journals/codex.md index c6db617a..56a93562 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -273,3 +273,30 @@ rows with no probe errors; screenshot /tmp/infer-stack-kubernetes-tui.svg. No real Apply/Down/node/bootstrap/install action was invoked. Maintainer validation remains the GPU hosts' credential refresh, actual agent membership, canary startup, and confirmed node handoff with real workloads. + +## 2026-10-01 20:55:21 -0400 + +User intent: correct the reviewed KubeAI 40-character Model-name blocker, +refresh runtime canaries on explicit apply, and make GPU-worker join verification +explicit. Keep the long e2e identity and logical commits. Model/configuration: +GPT-6 (Codex), Default collaboration mode, reasoning effort not exposed; +user-requested attribution GPT-6.1-Sol. No sub-agents. + +The name budget belonged in model_name_for, shared by rendered Models and gateway +routes. Kubernetes DNS validity alone did not imply CRD validity. Short names +retain their existing identities; overlong names reserve an eight-character +SHA-256 digest of the full served identity and the existing deployment tail. +Simple truncation would merge distinct models sharing a prefix. This changes +only names which the existing KubeAI CRD could not accept. The regression keeps +HuggingFaceTB/SmolLM2-135M-Instruct and checks the two dedicated Model/route +identities together. A first test unnecessarily invoked gateway apply with a +fake HTTP implementation that never completed dynamic registration; interrupted +after 73 preceding passes and restricted this rendering regression to its actual +contract. No real workload was created by that test. + +Naming validation: tests/test_leasing_kubeai.py passed all 74 tests, including +short/boundary names, static/dynamic long-prefix collisions, dedicated +separation, and the exact long e2e Model with matching gateway routes. Type +checks, scoped Ruff and whitespace checks passed. The script's long alias is +unchanged. Read-only probes found an existing CPU KubeAI chart and no managed +Models on this VM; its API needs a temporary port-forward before e2e. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 15bc6903..be8b6600 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -79,6 +79,12 @@ GPU requests/limits, a product selector and `runtimeClassName=nvidia` when that RuntimeClass exists. Kubernetes chooses placement; infer-stack assigns no host GPU indices. Models request multiple GPUs via the existing profile/count syntax. +KubeAI Model names are limited to 40 characters. Short served names keep their +DNS slug; longer names use a readable prefix plus a digest of the full name. +Dynamic routes additionally reserve the deployment suffix, keeping dedicated +deployments distinct. Gateway routes use the same canonical name; endpoint +aliases and model source IDs remain unchanged. + ```bash infer-stack kube install --values ./kubeai-values.local.yaml --namespace kubeai infer-stack kube install --values ./kubeai-values.local.yaml --namespace kubeai --apply diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 9d7d4d76..3652575b 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -31,6 +31,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path from typing import Any, Callable @@ -47,6 +48,8 @@ STATE_FILENAME = 'leasing-kubeai-state.json' MANAGED_LABEL = 'infer-stack/managed' DEPLOYMENT_LABEL = 'infer-stack/deployment' +# KubeAI's Model CRD is stricter than Kubernetes's usual DNS label limit. +MODEL_NAME_MAX_LENGTH = 40 DEFAULT_NAMESPACE = 'kubeai' # The standard local access path: `kubectl port-forward svc/kubeai 8000:80`. # An ingress-fronted cluster overrides this via the kubeai_base_url setting. @@ -61,19 +64,26 @@ def model_name_for(served: str, deployment_id: str | None = None) -> str: name clients use through the KubeAI gateway never changes. With ``deployment_id`` (dynamic routing), the deployment's tail is appended, as for a compose service, so same-model ``--dedicated`` deployments are - separate Models; the gateway addresses each by name. + separate Models; the gateway addresses each by name. KubeAI limits Model + names to 40 characters. Overlong slugs retain a readable prefix plus a + digest of the full served name, budgeting separately for a deployment tail. >>> model_name_for('Qwen/Qwen3-8B') 'qwen-qwen3-8b' >>> model_name_for('Qwen/Qwen3-8B', 'grp-0123456789ab') 'qwen-qwen3-8b-01234567' """ - if deployment_id is None: - return dns_slug(served) from ..leasing.naming import deployment_tail - tail = deployment_tail(deployment_id) - return f'{dns_slug(served)[:62 - len(tail)].rstrip("-")}-{tail}' + suffix = f'-{deployment_tail(deployment_id)}' if deployment_id is not None else '' + slug = dns_slug(served) + budget = MODEL_NAME_MAX_LENGTH - len(suffix) + if len(slug) > budget: + # Hash the whole identity, not the retained prefix. Static routes and + # dynamic deployment routes must derive exactly the same shortened CR. + digest = hashlib.sha256(served.encode()).hexdigest()[:8] + slug = f'{slug[:budget - len(digest) - 1].rstrip("-")}-{digest}' + return slug + suffix def model_name(deployment: Deployment, *, unique: bool = False) -> str: diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index a0cc736a..a5e20cce 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -1191,3 +1191,45 @@ def test_explicit_served_name_extra_argument_is_rejected(): Catalog.from_dict({'models': {'m': {'source': 'hf://org/model'}}, 'endpoints': {'alias': {'engine': 'vllm', 'model': 'm', 'runtime': { 'extra_args': ['--served-model-name=incorrect']}}}}) + + +@pytest.mark.parametrize('served,deployment_id,expected', [ + ('short-name', None, 'short-name'), + ('Qwen/Qwen3-8B', 'grp-0123456789ab', 'qwen-qwen3-8b-01234567'), + ('x' * 40, None, 'x' * 40), + ('x' * 31, 'grp-0123456789ab', 'x' * 31 + '-01234567'), +]) +def test_model_name_preserves_names_within_kubeai_limit(served, deployment_id, expected): + from infer_stack.backends.kubeai import model_name_for + assert model_name_for(served, deployment_id) == expected + + +@pytest.mark.parametrize('deployment_id', [None, 'grp-0e0b4141abcd']) +def test_long_model_names_are_bounded_stable_and_collision_resistant(deployment_id): + from infer_stack.backends.kubeai import model_name_for + prefix = 'very-long-served-model-name-' * 4 + names = [model_name_for(prefix + ending, deployment_id) for ending in ('a', 'b')] + assert len(set(names)) == 2 + assert all(len(name) <= 40 and not name.endswith('-') for name in names) + assert names[0] == model_name_for(prefix + 'a', deployment_id) + if deployment_id: + assert names[0].endswith('-0e0b4141') + assert names[0] != model_name_for(prefix + 'a', 'grp-abcdef123456') + + +def test_e2e_long_dynamic_model_name_matches_render_and_gateway(tmp_path): + from infer_stack.backends.kubeai import model_name_for + served = 'HuggingFaceTB/SmolLM2-135M-Instruct' + be, _ = make_front_door_backend(tmp_path, dynamic_routing=True) + deps = [vllm(gid, hf=served, served=served) for gid in + ('grp-0e0b4141abcd', 'grp-abcdef123456')] + for dep in deps: + dep.served = {served: {'served_model_name': served, 'protocol': 'chat'}} + be.converge(deps, apply=False) + docs = list(yaml.safe_load_all(be.models_file.read_text())) + expected = {model_name_for(served, dep.id) for dep in deps} + assert len(expected) == 2 + assert all(len(name) <= 40 for name in expected) + assert {m['metadata']['name'] for m in docs} == expected + routes = json.loads((be.gateway.state_dir / 'litellm_routes.json').read_text()) + assert {r['litellm_params']['model'] for r in routes} == {'openai/' + n for n in expected} From 4a2c80b82d73dc309c37661cbf5502aa105cc864 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 21:10:13 -0400 Subject: [PATCH 135/141] Refresh GPU runtime canaries and clarify worker join readiness Reverify every GPU node on explicit bootstrap/install apply, label retained runtime evidence as historical with timestamps, and warn GPU workers that join membership does not establish cluster-side NVIDIA readiness. Cover repeat apply, broken runtimes, dry-run safety, ownership and join output. Co-authored-by: GPT-6.1-Sol --- dev/journals/codex.md | 48 +++++++++++++++++++++ docs/cluster-setup.md | 14 ++++-- infer_stack/cli/commands_kube.py | 15 ++++++- infer_stack/kube/inspect.py | 17 ++++++-- infer_stack/kube/operations.py | 11 +++-- tests/test_kube_readiness.py | 16 ++++++- tests/test_kube_review.py | 74 ++++++++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 12 deletions(-) diff --git a/dev/journals/codex.md b/dev/journals/codex.md index 56a93562..c946270b 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -300,3 +300,51 @@ separation, and the exact long e2e Model with matching gateway routes. Type checks, scoped Ruff and whitespace checks passed. The script's long alias is unchanged. Read-only probes found an existing CPU KubeAI chart and no managed Models on this VM; its API needs a temporary port-forward before e2e. + +The second correction separates historical observation from explicit verification. +Every bootstrap/install apply replaces infer-stack-owned canaries on all GPU +nodes, even when old pods or currently running workloads supplied evidence. +Bootstrap tracks nodes checked during that invocation to avoid recreating +canaries on every readiness poll. Unrelated same-name pods remain protected; +failed fresh waits abort installation. Inventory retains historical startup +facts with evidence state/timestamp and explicit wording. A successful old +container start is valuable diagnosis, but cannot establish today's handler. + +Worker join now detects local GPU products best-effort after membership is +established and warns that drivers/toolkit and control-plane readiness are +separate. It directs the operator to check the specific worker's allocatable +GPUs, GFD product/memory and fresh runtime canary, so aiq-gpu's four GPUs cannot +mask namek exposing zero. This is an explicit warning, not automated driver +installation or a claim that worker-side admin credentials exist. + +Focused runtime/CLI tests: 72 passed. This includes repeated bootstrap/install +canaries, historical timestamps, later runtime breakage, unrelated pod refusal, +dry-run nonmutation and RTX 3090 join output. Type checks, CI flake8, scoped +Ruff, compileall, shell syntax and diff whitespace checks passed. Actual kwconf +join help and read-only inventory were inspected. Real e2e runs from a frozen +source snapshot with the unchanged script, E2E_MODEL=HuggingFaceTB/SmolLM2-135M-Instruct, +E2E_RESOURCE_PROFILE=cpu, E2E_BASE_URL=http://127.0.0.1:18000/openai/v1, +and all default dynamic/replica/gateway phases enabled. The optional sized, +make-room and remote-node phases remain disabled by their normal defaults; +this VM has no GPU and its old container worker is NotReady. A temporary local +port-forward supplies the installed chart's API. Initial real generation +already passed; the full result is pending. + +Full unit validation: pytest -q passed 1218 tests, 3 skipped, 8 existing cleanup/ +ResourceWarnings (133.24s). Extended Ruff on tests/test_leasing_kubeai.py reports +three import-order findings; checking the parent commit via stdin reproduces +the identical three findings. They predate this correction and are left alone. +CI flake8/type checks and scoped Ruff on changed production/readiness files +remain green. The real CRD has accepted the shortened dynamic long-name Model, +and the gateway has registered its route; generation is still in progress. + +Real e2e completed with exit zero and PASS: kubeai backend end-to-end lifecycle. +The unchanged long identity passed static generation, two keep-warm replicas, +two dynamic dedicated deployments and independent route teardown, dynamic key +rotation, in-cluster NodePort generation/doctor, Secret rotation and gateway +teardown. Log: /tmp/infer-stack-review-e2e.log. Post-run probes confirm zero +managed Models, no infer-stack-gateway Deployment, and no gateway Docker +containers. The temporary port-forward was stopped. The script's cleanup left +root/container-owned temporary PostgreSQL data with a permission warning; +no running database/workload remains. GPU canary freshness and a real RTX 3090 +worker's join/placement/generation still require aiq-gpu/namek validation. diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 8a8a0b98..41c45d9b 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -396,13 +396,21 @@ backend or scheduling architecture. `infer-stack kube k3s status` reports local server/agent membership without consulting the selected admin context or exposing join tokens. After join, verify the worker on the control plane with `infer-stack kube node status `. -A stale EKS context on the worker does not mean local membership failed. Active +A stale EKS context on the worker does not mean local membership failed. +Join establishes membership, not NVIDIA driver/toolkit readiness. If local GPUs +are detected, join warns that control-plane verification is required. Check the +specific worker, rather than relying on another GPU node to make the cluster +look ready: for `namek`, expect Ready, one allocatable GPU, RTX 3090 GFD product, +roughly 24 GiB GFD memory, and a fresh runtime canary from `kube install --apply`. Active agents must match requested server/name/version before join can return success. A RuntimeClass object is cluster-scoped and does not prove every node handler. Inventory reports per-node runtime evidence independently. Explicit bootstrap -and install can run small node-specific runtime canaries on GPU nodes; they -reserve no GPUs and retain completed pods in `kube-system`. Unknown handlers +and install rerun small node-specific runtime canaries on every GPU node on +each explicit apply, including nodes with prior successful evidence. They reserve +no GPUs and retain completed pods in `kube-system`. Inventory labels terminated +pod evidence as historical and includes its observed timestamp; it does not +promise that the handler still works after reconfiguration. Unknown handlers block detailed readiness until verified. Optional NVIDIA components scheduled onto zero nodes report N/A instead of causing a reinstall/readiness loop. diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index 59fd34e1..1d822fef 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -381,6 +381,18 @@ def main(cls, argv=True, **kwargs): raise SystemExit(str(ex)) from ex print('K3s agent is active; this does not change the selected admin kubeconfig.') print('Inspect local membership: infer-stack kube k3s status') + manager = KubeManager() + if manager.command_exists('nvidia-smi'): + try: + products = manager.run(['nvidia-smi', '--query-gpu=name', '--format=csv,noheader']).strip() + except RuntimeError: + print('Local GPU inspection failed; verify driver/runtime preparation and cluster-side GPU availability from the control plane.') + else: + if products: + print('Local GPUs: ' + ', '.join(products.splitlines())) + print('Local GPU detected, but cluster-side GPU availability must be verified from the control plane.') + print('Join does not install NVIDIA drivers/toolkit or prove GPU runtime readiness.') + print('Verify this node is Ready, exposes nvidia.com/gpu, has GFD product/memory labels, and passes a fresh runtime canary via infer-stack kube install --apply.') print(f'On the control plane: infer-stack kube node status {config.node_name or socket.gethostname().lower()}') return 0 @@ -443,7 +455,8 @@ def _print_inventory(report): print(f' {key}: {"available" if value else "missing"}') for node in report['nodes']: print(f"node {node['name']}") - for key in ('ready', 'nvidia_runtime_verified', 'nvidia_runtime_evidence', 'gpu_count', 'gpu_product', + for key in ('ready', 'nvidia_runtime_verified', 'nvidia_runtime_evidence_state', 'nvidia_runtime_observed_at', + 'nvidia_runtime_evidence', 'gpu_count', 'gpu_product', 'gpu_memory_gib', 'gfd_labels'): print(f' {key}: {node[key]}') plugins = report['device_plugin'] diff --git a/infer_stack/kube/inspect.py b/infer_stack/kube/inspect.py index c3753bf1..762fa68a 100644 --- a/infer_stack/kube/inspect.py +++ b/infer_stack/kube/inspect.py @@ -85,20 +85,29 @@ def host_gpus(): ['get', 'pods', '-A', '-o', 'json'])) if runtime_ok else None for row in report['nodes']: row['nvidia_runtime_verified'] = None + row['nvidia_runtime_evidence_state'] = 'unknown' + row['nvidia_runtime_observed_at'] = None row['nvidia_runtime_evidence'] = 'No observed pod proves the node runtime handler' for pod in (runtime_pods or {}).get('items', []): spec, status = pod.get('spec', {}), pod.get('status', {}) if spec.get('nodeName') != row['name'] or spec.get('runtimeClassName') != 'nvidia': continue - started = any(c.get('state', {}).get('running') or c.get('state', {}).get('terminated') - for c in status.get('containerStatuses') or []) - if started: + states = [c.get('state', {}) for c in status.get('containerStatuses') or []] + running = next((s['running'] for s in states if s.get('running')), None) + terminated = next((s['terminated'] for s in states if s.get('terminated')), None) + if running or terminated: row['nvidia_runtime_verified'] = True - row['nvidia_runtime_evidence'] = f"nvidia runtime pod {manager._pod_name(pod)} started" + row['nvidia_runtime_evidence_state'] = 'running' if running else 'historical' + state = running or terminated or {} + row['nvidia_runtime_observed_at'] = state.get('finishedAt') or state.get('startedAt') + row['nvidia_runtime_evidence'] = ( + f"last observed nvidia runtime pod {manager._pod_name(pod)} " + + ('running' if running else 'terminated; historical startup, not a fresh verification')) break messages = ' '.join(str(c.get('message', '')) for c in status.get('conditions') or []) if 'runtime handler' in messages.lower(): row['nvidia_runtime_verified'] = False + row['nvidia_runtime_evidence_state'] = 'failed' row['nvidia_runtime_evidence'] = messages ds = probe('device_plugin', lambda: manager.kubectl_json( ['get', 'daemonsets', '-A', '-o', 'json'])) diff --git a/infer_stack/kube/operations.py b/infer_stack/kube/operations.py index ff8ccbeb..7ba320f8 100644 --- a/infer_stack/kube/operations.py +++ b/infer_stack/kube/operations.py @@ -67,10 +67,14 @@ def apply(self, manager: KubeManager, report: dict, *, version=None, timeout=180 raise RuntimeError('NVIDIA chart remediation requires RuntimeClass nvidia; configure node runtimes first') manager.install_nvidia_device_plugin() deadline = time.monotonic() + timeout + verified_nodes = set() while True: fresh = inventory(manager) if fresh.get('gpu_count') and 'nvidia' in (fresh['runtime_classes'] or []): - verify_node_runtimes(manager, fresh, timeout=timeout) + # Fresh per apply, once per node during this readiness loop. + pending = [n for n in fresh['nodes'] if n['gpu_count'] and n['name'] not in verified_nodes] + verify_node_runtimes(manager, {'nodes': pending}, timeout=timeout) + verified_nodes.update(n['name'] for n in pending) fresh = inventory(manager) if not failed(readiness(fresh, installation=False)): return fresh @@ -93,13 +97,14 @@ def verify_node_runtimes(manager, report, *, timeout=180): """Explicit bootstrap canaries test each GPU node's nvidia handler. No GPU reservation: allocatable resources are independently checked. A - successful retained Pod supplies inspectable evidence for later inventory. + successful retained Pod supplies historical evidence for later inventory. + Always replace our own canary: prior startup cannot verify today's handler. No hostRuntime claim is inferred from the RuntimeClass object alone. """ import hashlib import json for node in report['nodes']: - if not node['gpu_count'] or node.get('nvidia_runtime_verified') is True: + if not node['gpu_count']: continue name = 'infer-stack-runtime-' + hashlib.sha256(node['name'].encode()).hexdigest()[:12] existing = manager.kubectl_json(['-n', 'kube-system', 'get', 'pod', name, '--ignore-not-found', '-o', 'json']) diff --git a/tests/test_kube_readiness.py b/tests/test_kube_readiness.py index 0e6784aa..74b6cee8 100644 --- a/tests/test_kube_readiness.py +++ b/tests/test_kube_readiness.py @@ -42,9 +42,21 @@ def __init__(self, *, nodes=None, installed=False, crd=False, plugin=True, reach self.reachable = reachable self.calls = [] self.documents = [] + self.canaries = {} def __call__(self, args, **kwargs): self.calls.append(args) + if args[:3] == ['kubectl', '-n', 'kube-system'] and 'pod' in args: + name = args[args.index('pod') + 1] + if 'get' in args: + return json.dumps(self.canaries.get(name, {})) + if 'delete' in args: + self.canaries.pop(name, None) + return '' + if args == ['kubectl', 'apply', '-f', '-']: + doc = json.loads(kwargs['input_text']) + self.canaries[doc['metadata']['name']] = doc + return '' if args[:3] == ['kubectl', 'config', 'current-context']: return 'test-context' if args[:2] == ['kubectl', 'version']: @@ -198,7 +210,7 @@ def test_install_values_token_upgrade_dryrun(tmp_path, monkeypatch, capsys): assert KubeInstallCLI.main(argv=['--dry-run', '--json']) == 0 preview = json.loads(capsys.readouterr().out) assert len(preview['values']['resourceProfiles']) == 2 - assert not any('upgrade' in c or 'add' in c for c in cluster.calls) + assert not any('upgrade' in c or 'add' in c or 'apply' in c or 'delete' in c for c in cluster.calls) assert not (tmp_path / 'generated').exists() assert KubeInstallCLI.main(argv=False, apply=True) == 0 assert len(cluster.documents[-1][0]['resourceProfiles']) == 2 @@ -210,6 +222,7 @@ def test_install_values_token_upgrade_dryrun(tmp_path, monkeypatch, capsys): assert KubeInstallCLI.main(argv=False, apply=True) == 0 upgrades = [c for c in cluster.calls if c[:3] == ['helm', 'upgrade', '--install']] assert len(upgrades) == 2 + assert sum(c == ['kubectl', 'apply', '-f', '-'] for c in cluster.calls) == 4 assert '--version' in upgrades[-1] @@ -243,6 +256,7 @@ def test_bootstrap_working_cluster_idempotent(monkeypatch): provider = K3sProvider() for _ in range(2): provider.apply(m, inventory(m), timeout=0) + assert sum(c == ['kubectl', 'apply', '-f', '-'] for c in cluster.calls) == 2 assert not any(c[0] in ('sudo', 'curl', 'systemctl') or 'upgrade' in c for c in cluster.calls) diff --git a/tests/test_kube_review.py b/tests/test_kube_review.py index 4759e7d7..baa16a5d 100644 --- a/tests/test_kube_review.py +++ b/tests/test_kube_review.py @@ -229,3 +229,77 @@ def test_startup_details_expose_node_image_and_replica_state(): pod['status']['conditions'] = [{'type': 'Ready', 'status': 'True'}] assert '1/1 replicas ready' in model_startup_progress([pod]) assert 'verifying generation' in model_startup_progress([pod]) + + +def test_retained_runtime_canary_is_historical_evidence(capsys): + cluster = Cluster() + m = manager(cluster) + def run(args, **kw): + if args[:3] == ['kubectl', 'get', 'pods']: + return json.dumps({'items': [{'metadata': {'name': 'old-canary', 'namespace': 'kube-system'}, + 'spec': {'nodeName': 'gpu-a', 'runtimeClassName': 'nvidia'}, + 'status': {'containerStatuses': [{'state': {'terminated': { + 'exitCode': 0, 'finishedAt': '2026-09-01T00:00:00Z'}}}]}}]}) + return cluster(args, **kw) + m.run = run + report = inventory(m, http=HTTP) + row = report['nodes'][0] + assert row['nvidia_runtime_verified'] is True # observed startup, not a new test + assert row['nvidia_runtime_evidence_state'] == 'historical' + assert row['nvidia_runtime_observed_at'] == '2026-09-01T00:00:00Z' + from infer_stack.cli.commands_kube import _print_inventory + _print_inventory(report) + assert 'historical startup, not a fresh verification' in capsys.readouterr().out + + +def test_runtime_canary_refreshes_all_gpu_nodes_and_detects_new_failure(): + documents, calls = [], [] + def run(args, **kw): + calls.append(args) + if 'get' in args: + return json.dumps({'metadata': {'labels': {'infer-stack/runtime-canary': 'true'}}}) + if 'apply' in args: + documents.append(json.loads(kw['input_text'])) + if 'wait' in args and len(documents) > 2: + raise RuntimeError('runtime handler nvidia is no longer configured') + return '' + report = {'nodes': [ + {'name': name, 'gpu_count': count, 'nvidia_runtime_verified': True} + for name, count in [('aiq-gpu', 4), ('namek', 1), ('cpu-node', 0)]]} + m = KubeManager(run=run) + verify_node_runtimes(m, report) + assert [d['spec']['nodeName'] for d in documents] == ['aiq-gpu', 'namek'] + with pytest.raises(RuntimeError, match='no longer configured'): + verify_node_runtimes(m, report) + assert len(documents) == 3 + assert sum('delete' in c for c in calls) == 3 + + +def test_runtime_canary_never_replaces_unrelated_pod(): + calls = [] + def run(args, **kw): + calls.append(args) + return json.dumps({'metadata': {'labels': {}}}) + report = {'nodes': [{'name': 'namek', 'gpu_count': 1, 'nvidia_runtime_verified': True}]} + with pytest.raises(RuntimeError, match='unrelated pod'): + verify_node_runtimes(KubeManager(run=run), report) + assert len(calls) == 1 and 'get' in calls[0] + + +@pytest.mark.parametrize('products', ['NVIDIA GeForce RTX 3090', '']) +def test_join_reports_local_gpu_separately_from_cluster_availability(monkeypatch, capsys, products): + from infer_stack.cli.commands_kube import K3sJoinCLI + calls = [] + monkeypatch.setattr(k3s, 'join', lambda **kw: calls.append(kw)) + m = KubeManager(run=lambda args: products) + m.command_exists = lambda name: name == 'nvidia-smi' + monkeypatch.setattr('infer_stack.cli.commands_kube.KubeManager', lambda: m) + assert K3sJoinCLI.main(argv=False, server='https://server:6443', token_file='unused', node_name='namek') == 0 + text = capsys.readouterr().out + assert 'infer-stack kube node status namek' in text + assert ('Local GPU detected' in text) == bool(products) + if products: + assert 'RTX 3090' in text + assert 'nvidia.com/gpu' in text and 'GFD product/memory' in text + assert 'fresh runtime canary' in text + assert len(calls) == 1 From 2dc083f0e7041e10727ee4e77f1a97041bdc2096 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 22:47:30 -0400 Subject: [PATCH 136/141] Separate KubeAI declaration, replica readiness and generation evidence Read nested KubeAI replica status through one compatibility helper in CLI and TUI, gate API candidates on replica readiness, label explicit generation evidence, preserve passive polling budgets, and warn on historical runtime evidence. Co-authored-by: GPT-6.1-Sol --- dev/journals/codex.md | 41 ++++++++++ docs/kubeai-backend.md | 17 ++++ infer_stack/backends/kubeai.py | 17 ++++ infer_stack/cli/commands_kube.py | 6 +- infer_stack/kube/inspect.py | 9 ++- infer_stack/kube/monitor.py | 41 +++++++++- infer_stack/kube/operations.py | 2 +- infer_stack/tui.py | 121 ++++++++++++++++++++++------ tests/test_kube_review.py | 31 ++++++++ tests/test_tui_kube.py | 130 +++++++++++++++++++++++++++++-- 10 files changed, 377 insertions(+), 38 deletions(-) diff --git a/dev/journals/codex.md b/dev/journals/codex.md index c946270b..ed7554ce 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -348,3 +348,44 @@ containers. The temporary port-forward was stopped. The script's cleanup left root/container-owned temporary PostgreSQL data with a permission warning; no running database/workload remains. GPU canary freshness and a real RTX 3090 worker's join/placement/generation still require aiq-gpu/namek validation. + +## 2026-10-01 22:29:42 -0400 + +User intent: correct the TUI/status review on top of the already committed name +and canary fixes; secondary review also asks for CLI replica counts and truthful +historical-runtime doctor warnings. Keep efficient monitoring, kwconf and Compose +semantics, and make logical commits. Model/configuration: GPT-6 (Codex), Default +collaboration mode; reasoning effort not exposed. Requested commit attribution: +GPT-6.1-Sol. No sub-agents. + +I kept Model declaration as observation rather than changing the backend's +ownership contract. A shared replica helper understands the actual nested +KubeAI CR status and older flat counters, retaining unknown counts. Model/pod +samples derive declared/scheduled/container-running/replica-ready stages. The +API picker requires positive ready-replica evidence and labels generation as +unverified until an explicit successful request. Last-generation proof belongs +to an endpoint/pool; it does not prove every dedicated deployment or replica +served a request. Proof is scoped to observed Model/pod incarnations and removed +when replacement/restarts, readiness loss or failures invalidate it. No passive +generation probes are introduced. + +The global observation still costs one Model list at the existing cadence; it +previously spent that request on observe() and threw the status away. Fresh +Cluster samples are reused, and its three list reads remain visible-only. Tests +that injected slow observe() now inject the actual Model query seam, with count +budgets for the independent global and Cluster workers. Historical runtime +startup becomes WARN/evidence rather than an ok claim of fresh verification; +unknown/failing handlers still fail and explicit apply still creates new canaries. + +Combined correction validation: 149 backend/CLI/readiness focused tests passed; +97 existing/new TUI tests passed before the three extra editor-refusal cases, +and those four editor cases passed separately. The final full suite passed +1231 tests, 3 skipped, 8 existing cleanup/ResourceWarnings (137.18s). Type check, +CI flake8, scoped Ruff, compileall, shell syntax and whitespace checks passed. +The previously confirmed three import-order Ruff findings in the pre-existing +KubeAI test module remain untouched. Actual kwconf inventory/doctor help and a +read-only headless TUI against the VM cluster passed (two nodes, four pods, no +probe errors). No workload or host mutations occurred in this correction. +The earlier unchanged long-model CPU e2e passed; GPU placement/generation and +fresh worker/runtime tests still need aiq-gpu/namek. Readiness and editor changes +are committed separately so operator-evidence policy is reviewable on its own. diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index be8b6600..97bea86d 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -362,3 +362,20 @@ through the controller even without a local gateway render; Down confirms managed Model/gateway teardown and retains leases. Existing Acquire, Release, Evict and pod log controls remain available. Endpoint editing exposes `runtime.resource_profile` instead of physical host GPU indices. + + +Kubernetes readiness has separate stages: declared Model, scheduled pod, +running container, ready replica, and generation verified by an explicit API +request. CLI inventory and the Cluster summary read KubeAI's +`status.replicas.all/ready`, with fallback for legacy flat counters; missing +counts remain unknown. A CR alone never makes an endpoint eligible for the API +picker. Replica-ready candidates are labeled generation-unverified until +**Send** or **Test all** succeeds. Verification is last-request evidence for the +endpoint, not proof that every replica answered. Replacement, observed restarts, +readiness loss or a failed request invalidates that evidence. + +The global ledger/API observation uses one cached Model-list read, reusing a +fresh Cluster sample when available. Full node/pod monitoring remains limited +to the visible Cluster tab. Passive refresh performs no generation requests. +Doctor reports retained terminated runtime evidence as a non-failing **WARN**, +with `kube install --apply` as the fresh-verification step. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 3652575b..6f467b8f 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -56,6 +56,23 @@ DEFAULT_BASE_URL = 'http://127.0.0.1:8000/openai/v1' +def model_replica_counts(model: dict) -> tuple[int | None, int | None]: + """Observed (all, ready) replicas; missing counts stay unknown. + + KubeAI uses status.replicas.{all,ready}; tolerate older flat counters. + """ + status = model.get('status') or {} + replicas = status.get('replicas') + def count(value): + try: + return max(0, int(value)) if value is not None else None + except (TypeError, ValueError): + return None + if isinstance(replicas, dict): + return count(replicas.get('all')), count(replicas.get('ready', status.get('readyReplicas'))) + return count(replicas), count(status.get('readyReplicas')) + + def model_name_for(served: str, deployment_id: str | None = None) -> str: """Deterministic Model CR name for a served model name: ````. diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index 1d822fef..5372e57c 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -486,7 +486,9 @@ def _print_inventory(report): spec = item.get('spec', {}) summary = f"{spec.get('type', '?')} {spec.get('clusterIP', '')}" else: - summary = f"{status.get('readyReplicas', 0)} ready replicas" + from ..backends.kubeai import model_replica_counts + total, ready = model_replica_counts(item) + summary = f"{ready if ready is not None else '?'}/{total if total is not None else '?'} ready replicas" print(f' {name}: {summary}') print('resource profiles proposed') if report['resource_profiles']['proposed']: @@ -500,7 +502,7 @@ def _print_inventory(report): def _print_checks(checks): for check in checks: - mark = {'ok': 'ok ', 'fail': 'FAIL', 'blocked': 'SKIP'}[check['status']] + mark = {'ok': 'ok ', 'warn': 'WARN', 'fail': 'FAIL', 'blocked': 'SKIP'}[check['status']] print(f"[{mark}] {check['name']} — {check['detail']}") if check['fix']: print(f" fix: {check['fix']}") diff --git a/infer_stack/kube/inspect.py b/infer_stack/kube/inspect.py index 762fa68a..eed33ebc 100644 --- a/infer_stack/kube/inspect.py +++ b/infer_stack/kube/inspect.py @@ -168,9 +168,9 @@ def readiness(report: dict, *, installation=True, gpu=True) -> list[dict]: """Policy over inventory; stop dependent chains at their first blocker.""" checks = [] - def check(name, ok, fix='', detail='', blocked=False): + def check(name, ok, fix='', detail='', blocked=False, warning=False): checks.append({'name': name, 'ok': bool(ok), 'status': 'blocked' if blocked else - ('ok' if ok else 'fail'), 'detail': detail, 'fix': fix if not ok else ''}) + ('warn' if warning else 'ok' if ok else 'fail'), 'detail': detail, 'fix': fix if not ok or warning else ''}) return ok if not check('kubectl available', report['tools']['kubectl'], @@ -193,9 +193,10 @@ def check(name, ok, fix='', detail='', blocked=False): 'RuntimeClass nvidia' if runtime_ok else 'No nvidia RuntimeClass; GPU allocation proves an external integration' if gpu_rows else '') for row in gpu_rows if runtime_ok else []: verified = row.get('nvidia_runtime_verified') - check(f"node {row['name']} NVIDIA runtime verified", verified is True, + check(f"node {row['name']} NVIDIA runtime evidence", verified is True, 'infer-stack kube install --apply (verifies runtime handlers in the selected cluster)', - row.get('nvidia_runtime_evidence', 'Runtime handler unknown')) + row.get('nvidia_runtime_evidence', 'Runtime handler unknown'), + warning=verified is True and row.get('nvidia_runtime_evidence_state') == 'historical') plugin = report['device_plugin'] check('NVIDIA device plugin', bool(plugin) and any(p['desired'] > 0 for p in plugin) and all(p['healthy'] for p in plugin if p['desired'] > 0), 'infer-stack kube bootstrap', report['errors'].get('device_plugin', '')) diff --git a/infer_stack/kube/monitor.py b/infer_stack/kube/monitor.py index f31b5daf..959952b1 100644 --- a/infer_stack/kube/monitor.py +++ b/infer_stack/kube/monitor.py @@ -7,6 +7,7 @@ import time +from ..backends.kubeai import model_replica_counts from .manage import KubeManager @@ -32,7 +33,7 @@ def request(container): return max(steady + sidecars, peak) + int((spec.get('overhead') or {}).get('nvidia.com/gpu', 0)) -def snapshot(manager: KubeManager, *, namespace: str) -> dict: +def snapshot(manager: KubeManager, *, namespace: str, resources=('nodes', 'pods', 'models')) -> dict: report = {'nodes': None, 'pods': None, 'models': None, 'errors': {}, 'sampled_at': time.monotonic(), 'namespace': namespace} for key, args in ( @@ -41,6 +42,8 @@ def snapshot(manager: KubeManager, *, namespace: str) -> dict: ('models', ['-n', namespace, 'get', 'models.kubeai.org', '-l', 'infer-stack/managed=true', '-o', 'json']), ): + if key not in resources: + continue try: result = manager.kubectl_json(args) if not isinstance(result.get('items'), list): @@ -94,3 +97,39 @@ def pod_rows(report: dict) -> list[tuple]: f"{sum(bool(c.get('ready')) for c in containers)}/{len(pod.get('spec', {}).get('containers') or [])}", str(sum(c.get('restartCount', 0) for c in containers)), str(gpu_request(pod)))) return sorted(rows) + + +def model_states(report: dict) -> dict[str, dict]: + """Lifecycle evidence from one sample; declaring a CR proves no readiness.""" + states = {} + for model in report['models'] or []: + meta = model.get('metadata') or {} + gid = (meta.get('labels') or {}).get('infer-stack/deployment') + if not gid: + continue + pods = [p for p in report['pods'] or [] + if p.get('metadata', {}).get('namespace') == report['namespace'] + and p.get('metadata', {}).get('labels', {}).get('infer-stack/deployment') == gid + and not p.get('metadata', {}).get('deletionTimestamp') + and p.get('status', {}).get('phase') not in {'Succeeded', 'Failed'}] + scheduled = any(p.get('spec', {}).get('nodeName') for p in pods) + running = any(c.get('state', {}).get('running') + for p in pods for c in p.get('status', {}).get('containerStatuses') or []) + ready_pods = any(any(c.get('type') == 'Ready' and c.get('status') == 'True' + for c in p.get('status', {}).get('conditions') or []) for p in pods) + total, ready = model_replica_counts(model) + replica_ready = bool(ready and (report['pods'] is None or ready_pods)) + stage = 'replica ready' if replica_ready else 'pod running' if running else 'scheduled' if scheduled else 'declared' + if meta.get('deletionTimestamp'): + stage, replica_ready = 'deleting', False + # Proof is scoped to this incarnation; replacement/restarts invalidate + # an earlier successful API request. A Model-only sample has no pod proof. + identity = (meta.get('uid'), meta.get('generation'), tuple(sorted( + (p.get('metadata', {}).get('uid', ''), tuple( + (c.get('containerID', ''), c.get('restartCount', 0)) + for c in p.get('status', {}).get('containerStatuses') or [])) for p in pods)) + if report['pods'] is not None else None) + states[gid] = {'name': meta.get('name'), 'stage': stage, + 'replica_ready': replica_ready, 'all_replicas': total, + 'ready_replicas': ready, 'identity': identity} + return states diff --git a/infer_stack/kube/operations.py b/infer_stack/kube/operations.py index 7ba320f8..f9396fca 100644 --- a/infer_stack/kube/operations.py +++ b/infer_stack/kube/operations.py @@ -138,7 +138,7 @@ def install(manager: KubeManager, report: dict, *, operator_values=None, checks = readiness(report, installation=False, gpu=gpu) if report['errors'].get('helm_releases'): raise RuntimeError('Cannot inspect existing Helm releases; repair Helm access before installation') - blockers = [c for c in checks if c['status'] == 'fail' and not c['name'].endswith('NVIDIA runtime verified')] + blockers = [c for c in checks if c['status'] == 'fail' and not c['name'].endswith('NVIDIA runtime evidence')] if blockers: raise RuntimeError('KubeAI prerequisites failed: ' + ', '.join(c['name'] for c in blockers) + '. Run infer-stack kube doctor / kube bootstrap') diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 4fe3c178..8e64f67a 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -1010,6 +1010,12 @@ def __init__( # Must match the Collapsible widgets' initial state, or the gate in # _collect polls for a pane that is shut until the first sync. self._collapsed = {'docker': True, 'system': True} + self._kube_states: dict[str, dict] = {} + self._kube_model_error = '' + self._kube_tokens: dict[str, tuple] = {} + self._kube_verified: dict[str, tuple[tuple, str]] = {} + self._api_select_signature = None + self._api_readiness_text = None self._ready_endpoints: list[str] = [] # served endpoint name -> OpenAI surface ('chat' | 'completions'), read # from the live deployment payload so the API tab probes the surface a @@ -1188,6 +1194,8 @@ def _compose_api(self) -> ComposeResult: 'clipboard.', classes='desc', ) yield Static('', id='api-urls', markup=False) + if self._is_kube: + yield Static('', id='api-readiness', markup=False) with Horizontal(id='api-controls'): yield Select([], prompt='model…', id='api-model') yield Button('Send', id='btn-api-send', variant='primary') @@ -1290,7 +1298,7 @@ def on_mount(self) -> None: deployments_tbl = self.query_one('#deployments', DataTable) deployments_tbl.add_column('', width=2, key='sel') # multi-select marker deployments_tbl.add_columns( - 'id', 'engine', 'state', 'running', 'gpus', 'served', + 'id', 'engine', 'state', 'lifecycle' if self._is_kube else 'running', 'gpus', 'served', 'leases', 'held by' ) self.query_one('#ps', DataTable).add_columns( @@ -1664,22 +1672,53 @@ def _cached_label(source: str, hub: Path | None) -> str: return '-' def _sync_api_models(self, names: list[str]) -> None: - """Point the API model selector at the currently-ready endpoints only.""" + """Select running Compose / replica-ready KubeAI endpoints; label proof.""" self._api_models_wanted = names - if names == self._ready_endpoints: + if self._is_kube: + self._update_kube_api_readiness(names) + signature = (tuple(names), tuple(n in self._kube_verified for n in names) if self._is_kube else ()) + if signature == self._api_select_signature: return try: select = self.query_one('#api-model', Select) except NoMatches: return # the API tab is not built yet; filled when it is self._ready_endpoints = names + self._api_select_signature = signature current = None if _select_is_blank(select.value) else select.value - select.set_options([(n, n) for n in names]) + select.set_options([((n + (' (generation verified)' if n in self._kube_verified else + ' (replica ready; generation unverified)')) if self._is_kube else n, n) + for n in names]) if current in names: select.value = current elif names: select.value = names[0] + def _update_kube_api_readiness(self, names): + text = ('Only replica-ready endpoints are offered. Send/Test all verifies generation on demand; ' + 'passive monitoring makes no generation requests.') + if self._kube_verified: + text += '\nLast successful generation: ' + ', '.join( + f'{ep} at {proof[1]}' for ep, proof in sorted(self._kube_verified.items())) + if self._kube_model_error: + text += '\nModel readiness unknown: ' + self._kube_model_error + if text == self._api_readiness_text: + return + try: + self.query_one('#api-readiness', Static).update(text) + self._api_readiness_text = text + except NoMatches: + pass + + def _record_kube_generation(self, model, token, success): + if not self._is_kube: + return + if success and token and self._kube_tokens.get(model) == token: + self._kube_verified[model] = (token, time.strftime('%H:%M:%S')) + else: + self._kube_verified.pop(model, None) + self._sync_api_models(list(self._api_models_wanted)) + def _update_catalog_help(self) -> None: help_ = self.query_one('#catalog-help', Static) if not self._endpoint_names: @@ -1745,11 +1784,18 @@ def _collect(self) -> dict[str, Any]: now = time.monotonic() if (self._observed_at is None or (now - self._observed_at) >= self.observe_interval): - if (self._is_kube and self._cluster_snapshot is not None - and self._cluster_snapshot['models'] is not None - and self._cluster_at is not None and now - self._cluster_at < self.observe_interval): - self._observed = {m.get('metadata', {}).get('labels', {}).get('infer-stack/deployment') - for m in self._cluster_snapshot['models']} - {None, ''} + if self._is_kube: + from .kube.monitor import model_states, snapshot + report = self._cluster_snapshot + if (report is None or report['models'] is None or self._cluster_at is None + or now - self._cluster_at >= self.observe_interval): + # The global ledger/API observation costs one Model list; + # full node/pod monitoring remains gated by the Cluster tab. + report = snapshot(self._cluster_manager(), namespace=self.controller.backend.namespace, + resources=('models',)) + self._kube_states = model_states(report) + self._kube_model_error = report['errors'].get('models', '') + self._observed = set(self._kube_states) self._assignments = {} else: self._observed, self._assignments = _placement_view(self.controller) @@ -1788,17 +1834,26 @@ def _render(self, data: dict[str, Any]) -> None: if 'gpus' in data: self._fill_gpus(data['gpus']) self.query_one('#sysinfo', Static).update(data.get('sysinfo', '')) - # Ready = endpoints served by a deployment that is actually running. + # Compose observes running services. KubeAI eligibility instead needs + # replica evidence; a declared Model is not a ready endpoint. ready: set[str] = set() protocols: dict[str, str] = {} for g in data['deployments']: - if g.id in data['observed']: + if (self._kube_states.get(g.id, {}).get('replica_ready') if self._is_kube + else g.id in data['observed']): ready.update(g.served) for name, payload in g.served.items(): proto = payload.get('protocol') if isinstance(payload, dict) \ else None if proto: protocols[name] = proto + if self._is_kube: + self._kube_tokens = {ep: tuple(sorted( + (g.id, self._kube_states[g.id]['identity']) for g in data['deployments'] + if ep in g.served and self._kube_states.get(g.id, {}).get('replica_ready'))) + for ep in ready} + self._kube_verified = {ep: proof for ep, proof in self._kube_verified.items() + if self._kube_tokens.get(ep) == proof[0]} self._ready_protocols = protocols self._sync_api_models(sorted(ready)) self._update_summary(data['leases'], data['deployments'], data['observed']) @@ -1913,7 +1968,7 @@ def _update_summary(self, leases, deployments, observed) -> None: # paint is ledger-only), the running counts would read as a confident # "0 running" — say we're still observing instead of silently lying. observing = self._observed_at is None - running_label = 'observing…' if observing else f'{running} running' + running_label = 'observing…' if observing else f'{running} declared' if self._is_kube else f'{running} running' # Assign only on change: Textual repaints a pane whenever its border # title is set, even to the same text, and these two panes hold the # lease and deployment tables -- most of the screen, every tick @@ -2013,7 +2068,8 @@ def _fill_deployments(self, deployments, observed, assignments, leases) -> None: ( SELECT_MARK if g.id in self._dep_sel else '', g.id, g.engine, str(g.state), - _running_label(g.id, observed), + (self._kube_states.get(g.id, {}).get('stage', 'unknown' if self._kube_model_error else 'absent') + if self._is_kube else _running_label(g.id, observed)), _gpu_label(g.id, observed, assignments), ','.join(sorted(g.served)) or '-', str(g.demand), ','.join(owners.get(g.id, [])) or '-', @@ -2965,10 +3021,18 @@ def _collect_cluster(self): def _receive_cluster(self, report): self._cluster_snapshot = report self._cluster_at = time.monotonic() + if report['models'] is not None: + from .kube.monitor import model_states + self._kube_states = model_states(report) + self._kube_model_error = '' + self._observed = set(self._kube_states) + self._observed_at = self._cluster_at self._render_cluster(report) + if len(self.screen_stack) == 1: + self._refresh_now() def _render_cluster(self, report): - from .kube.monitor import node_rows, pod_rows + from .kube.monitor import model_states, node_rows, pod_rows nodes = node_rows(report) self._node_names = [r[0] for r in nodes] self._diff_fill(self.screen_stack[0].query_one('#cluster-nodes', DataTable), @@ -2978,10 +3042,12 @@ def _render_cluster(self, report): self._diff_fill(self.screen_stack[0].query_one('#cluster-pods', DataTable), pods or [('(unavailable)' if report['pods'] is None else '(no pods)',) + ('-',) * 5], '_cluster_pods_cache', id_index=0) - models = report['models'] - model_text = 'unknown' if models is None else ', '.join( - f"{m.get('metadata', {}).get('name')}: {m.get('status', {}).get('readyReplicas', 0)} ready" - for m in models) or 'none' + states = model_states(report) + def replica_text(value): + return str(value) if value is not None else '?' + model_text = 'unknown' if report['models'] is None else ', '.join( + f"{r['name']}: {replica_text(r['ready_replicas'])}/{replica_text(r['all_replicas'])} ready; {r['stage']}" + for r in states.values()) or 'none' age = max(0, int(time.monotonic() - report['sampled_at'])) summary = (f'Namespace: {report["namespace"]} · sample {age}s ago · refresh ≥15s, only while visible\n' 'GPU requests are scheduled resources across all namespaces, not utilization.\n' @@ -3562,7 +3628,10 @@ def _api_chat(self, model: str, prompt: str) -> str: resp = self._http_client().post( url, json=body, headers=headers, timeout=120) self._raise_for_body(resp) - return self._completion_text(resp.json()) + data = resp.json() + if self._is_kube and not data.get('choices'): + raise RuntimeError('generation response has no choices') + return self._completion_text(data) def _curl_for(self, model: str, prompt: str, *, reveal_key: bool = False) -> str: """The equivalent ``curl`` for a chat- or text-completion, matching the @@ -3637,7 +3706,8 @@ def _selected_api_model(self) -> str | None: def action_api_send(self) -> None: model = self._selected_api_model() if not model: - self._refuse('no ready models to query (acquire one first)') + self._refuse('no replica-ready endpoints to query (acquire and wait for a ready replica)' if self._is_kube + else 'no ready models to query (acquire one first)') return prompt = (self.query_one('#api-prompt', Input).value.strip() or 'Say hello in one short sentence.') @@ -3648,9 +3718,10 @@ def action_api_send(self) -> None: def action_api_test_all(self) -> None: models = list(self._ready_endpoints) if not models: - self._refuse('no ready models to test (acquire one first)') + self._refuse('no replica-ready endpoints to test (acquire and wait for a ready replica)' if self._is_kube + else 'no ready models to test (acquire one first)') return - self._api_log(f'— testing {len(models)} ready model(s) —') + self._api_log(f'— testing {len(models)} ' + ('replica-ready endpoint(s)' if self._is_kube else 'ready model(s)') + ' —') self._cli(*(cli.command('test', model) for model in models)) self._do_api_test_all(models) @@ -3684,11 +3755,14 @@ def action_open_webui(self) -> None: @work(thread=True, group='api') def _do_api_send(self, model: str, prompt: str) -> None: + token = self._kube_tokens.get(model) try: out = self._api_chat(model, prompt) self.call_from_thread(self._api_log, f' [{model}] {out}') + self.call_from_thread(self._record_kube_generation, model, token, True) except Exception as ex: # noqa: BLE001 self.call_from_thread(self._api_log, f' [{model}] ERROR: {ex}') + self.call_from_thread(self._record_kube_generation, model, token, False) @work(thread=True, group='api') def _do_api_list(self) -> None: @@ -3712,13 +3786,16 @@ def _do_api_test_all(self, models: list[str]) -> None: import time for model in models: + token = self._kube_tokens.get(model) start = time.perf_counter() try: self._api_chat(model, 'Reply with the single word: ok') dt = time.perf_counter() - start self.call_from_thread(self._api_log, f' ✓ {model} ({dt:.1f}s)') + self.call_from_thread(self._record_kube_generation, model, token, True) except Exception as ex: # noqa: BLE001 self.call_from_thread(self._api_log, f' ✗ {model} {ex}') + self.call_from_thread(self._record_kube_generation, model, token, False) self.call_from_thread(self._api_log, '— done —') # Mutations converge the backend (docker up/down, possibly slow) off the UI diff --git a/tests/test_kube_review.py b/tests/test_kube_review.py index baa16a5d..bb47b718 100644 --- a/tests/test_kube_review.py +++ b/tests/test_kube_review.py @@ -303,3 +303,34 @@ def test_join_reports_local_gpu_separately_from_cluster_availability(monkeypatch assert 'nvidia.com/gpu' in text and 'GFD product/memory' in text assert 'fresh runtime canary' in text assert len(calls) == 1 + + +def test_cli_inventory_uses_actual_kubeai_replica_shape_and_unknown_counts(capsys): + from infer_stack.cli.commands_kube import _print_inventory + report = inventory(manager(Cluster(installed=True)), http=HTTP) + report['kubeai']['models'] = [ + {'metadata': {'name': 'q'}, 'status': {'replicas': {'all': 2, 'ready': 1}}}, + {'metadata': {'name': 'old'}, 'status': {'replicas': 3, 'readyReplicas': 2}}, + {'metadata': {'name': 'unknown'}}] + _print_inventory(report) + text = capsys.readouterr().out + assert 'q: 1/2 ready replicas' in text + assert 'old: 2/3 ready replicas' in text + assert 'unknown: ?/? ready replicas' in text + + +def test_doctor_historical_runtime_is_nonfailing_warning_with_fresh_fix(capsys): + from infer_stack.cli.commands_kube import _print_checks + report = inventory(manager(Cluster(installed=True)), http=HTTP) + row = report['nodes'][0] + row['nvidia_runtime_evidence_state'] = 'historical' + row['nvidia_runtime_evidence'] = 'previously observed; not a fresh verification' + checks = readiness(report) + check = next(c for c in checks if c['name'].endswith('NVIDIA runtime evidence')) + assert check['status'] == 'warn' + assert 'infer-stack kube install --apply' in check['fix'] + assert not failed(checks) + _print_checks(checks) + text = capsys.readouterr().out + assert '[WARN] node gpu-a NVIDIA runtime evidence' in text + assert 'previously observed' in text diff --git a/tests/test_tui_kube.py b/tests/test_tui_kube.py index 5c1be60e..45bc59de 100644 --- a/tests/test_tui_kube.py +++ b/tests/test_tui_kube.py @@ -90,14 +90,14 @@ async def scenario(): app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, kube_manager=m) async with app.run_test(size=(180, 55)) as pilot: await pilot.pause() - assert not cluster.calls # no kube inventory at mount/hidden pane + assert len(cluster.calls) == 1 and 'models.kubeai.org' in cluster.calls[0] # one global readiness read; no node/pod inventory app.query_one('#docker', Collapsible).collapsed = False app.query_one('#docker-tabs', TabbedContent).active = 'tab-cluster' await pilot.pause() await app.workers.wait_for_complete() assert app.query_one('#cluster-nodes', DataTable).row_count == 1 count = len(cluster.calls) - assert count == 3 + assert count == 4 # one global Model read plus three Cluster reads app.action_refresh() await app.workers.wait_for_complete() assert len(cluster.calls) == count @@ -144,10 +144,10 @@ async def scenario(): await pilot.press('2') await pilot.pause() assert app.query_one('#top', TabbedContent).active == 'tab-api' - assert len(calls) == 1 + assert len(calls) == 2 # one global Model read and one slow Cluster read gate.set() await app.workers.wait_for_complete() - assert len(calls) == 3 + assert len(calls) == 4 try: asyncio.run(scenario()) finally: @@ -272,11 +272,11 @@ def test_refresh_does_not_spawn_duplicate_slow_observation(tmp_path): controller, catalog = context(tmp_path) gate = threading.Event() calls = [] - def observe(): - calls.append('observe') + def observe(args, **kw): + calls.append(args) gate.wait(5) - return set() - controller.backend.observe = observe + return '{"items": []}' + controller.backend.run = observe async def scenario(): app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None) async with app.run_test() as pilot: @@ -326,3 +326,117 @@ async def scenario(): await pilot.pause() assert calls == ['stop'] asyncio.run(scenario()) + + +@pytest.mark.parametrize('status,expected', [ + ({'replicas': {'all': 2, 'ready': 1}}, (2, 1)), + ({'replicas': {'all': 2, 'ready': 0}, 'readyReplicas': 2}, (2, 0)), + ({'replicas': 2, 'readyReplicas': 1}, (2, 1)), + ({'readyReplicas': 1}, (None, 1)), + ({}, (None, None)), +]) +def test_actual_kubeai_replica_counts_and_compatibility(status, expected): + from infer_stack.backends.kubeai import model_replica_counts + assert model_replica_counts({'status': status}) == expected + + +def lifecycle_report(gid='grp-q'): + return {'nodes': [], 'pods': [], 'namespace': 'default', 'errors': {}, + 'sampled_at': time.monotonic(), 'models': [{ + 'metadata': {'name': 'qwen-coder', 'uid': 'model-1', 'generation': 1, + 'labels': {'infer-stack/deployment': gid}}, + 'status': {'replicas': {'all': 2, 'ready': 0}}}]} + + +def test_kubeai_lifecycle_evidence_and_namespace_scope(): + from infer_stack.kube.monitor import model_states + r = lifecycle_report() + assert model_states(r)['grp-q']['stage'] == 'declared' + pod = {'metadata': {'name': 'vllm', 'namespace': 'default', 'uid': 'pod-1', + 'labels': {'infer-stack/deployment': 'grp-q'}}, + 'spec': {'nodeName': 'namek'}, 'status': {'phase': 'Pending'}} + r['pods'] = [pod] + assert model_states(r)['grp-q']['stage'] == 'scheduled' + pod['status'] = {'phase': 'Running', 'containerStatuses': [ + {'containerID': 'container-1', 'state': {'running': {'startedAt': 'now'}}}]} + assert model_states(r)['grp-q']['stage'] == 'pod running' + r['models'][0]['status']['replicas']['ready'] = 1 + assert not model_states(r)['grp-q']['replica_ready'] # stale CR counter cannot override unready pods + pod['status']['conditions'] = [{'type': 'Ready', 'status': 'True'}] + assert model_states(r)['grp-q']['stage'] == 'replica ready' + before = model_states(r)['grp-q']['identity'] + pod['status']['containerStatuses'][0]['restartCount'] = 1 + assert model_states(r)['grp-q']['identity'] != before + pod['metadata']['namespace'] = 'unrelated' + assert not model_states(r)['grp-q']['replica_ready'] + r['pods'] = None # Model-only polling uses the CR replica count, not guessed pod facts + assert model_states(r)['grp-q']['replica_ready'] + r['models'][0]['metadata']['deletionTimestamp'] = 'now' + assert not model_states(r)['grp-q']['replica_ready'] + + +def test_pending_model_never_offered_as_ready_then_real_generation_is_distinct(tmp_path): + from test_leasing_kubeai import vllm + controller, catalog = context(tmp_path) + deployment = vllm('grp-q', served='qwen-coder') + deployment.served = {'qwen-coder': {'served_model_name': 'qwen-coder', 'protocol': 'chat'}} + controller.ledger.store.insert_deployment(deployment) + requests = [] + class Response: + def raise_for_status(self): + pass + def json(self): + return {'choices': [{'message': {'content': 'ok'}}]} + class HTTP: + def post(self, *args, **kw): + requests.append((args, kw)) + return Response() + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None, + kube_manager=manager(Cluster()), http=HTTP()) + app._litellm = lambda: ('http://gateway/v1', 'key') + async with app.run_test(size=(180, 55)) as pilot: + await pilot.pause() + r = lifecycle_report() + app._receive_cluster(r) + assert app._api_models_wanted == [] + assert app._kube_states['grp-q']['stage'] == 'declared' + assert '0/2 ready' in str(app.query_one('#cluster-summary', Static).render()) + app.query_one('#top', TabbedContent).active = 'tab-api' + await pilot.pause() + assert app._ready_endpoints == [] + r['pods'] = None + r['models'][0]['status']['replicas']['ready'] = 1 + app._receive_cluster(r) + assert app._ready_endpoints == ['qwen-coder'] + assert app._kube_verified == {} + assert '1/2 ready' in str(app.query_one('#cluster-summary', Static).render()) + assert 'generation unverified' in str(app.query_one('#api-model', Select)._options) + assert requests == [] # no generation during passive refresh + app.action_api_send() + await app.workers.wait_for_complete() + await pilot.pause() + assert 'qwen-coder' in app._kube_verified + assert 'Last successful generation' in str(app.query_one('#api-readiness', Static).render()) + assert len(requests) == 1 + old_post = app._http_client().post + def fail(*args, **kwargs): + raise RuntimeError('upstream loading') + app._http_client().post = fail + app.action_api_test_all() + await app.workers.wait_for_complete() + assert not app._kube_verified + app._http_client().post = old_post + app.action_api_test_all() + await app.workers.wait_for_complete() + assert 'qwen-coder' in app._kube_verified + old_token = app._kube_tokens['qwen-coder'] + r['models'][0]['metadata']['uid'] = 'replacement-model' + app._receive_cluster(r) + assert not app._kube_verified + app._record_kube_generation('qwen-coder', old_token, True) + assert not app._kube_verified # late response cannot verify a replacement + r['models'][0]['status']['replicas']['ready'] = 0 + app._receive_cluster(r) + assert app._ready_endpoints == [] + asyncio.run(scenario()) From 51fcad9adfbcbc48be2bfc3062247d5323dd38c9 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 22:48:12 -0400 Subject: [PATCH 137/141] Expose only supported KubeAI endpoint launch controls Keep environment and resource profiles in the KubeAI editor, hide image/command overrides, reject imported unsupported launches and ignored runtime.image values, and distinguish vLLM data parallelism from KubeAI pod replicas. Co-authored-by: GPT-6.1-Sol --- dev/journals/codex.md | 7 +++++++ docs/kubeai-backend.md | 12 ++++++++++++ infer_stack/backends/kubeai.py | 4 ++-- infer_stack/tui.py | 34 +++++++++++++++++++++++----------- tests/test_leasing_kubeai.py | 1 + tests/test_tui_kube.py | 31 +++++++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 13 deletions(-) diff --git a/dev/journals/codex.md b/dev/journals/codex.md index ed7554ce..b00fb5d9 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -377,6 +377,13 @@ budgets for the independent global and Cluster workers. Historical runtime startup becomes WARN/evidence rather than an ok claim of fresh verification; unknown/failing handlers still fail and explicit apply still creates new canaries. +The KubeAI editor hides image/command and retains environment. Existing custom +launches are refused during edit instead of silently erased, and rendering now +rejects image overrides as well as command/mounts. vLLM data parallelism is +explicitly distinguished from independent KubeAI pod replicas. Compose keeps +its image/command controls. I prefer rejection to inventing unsupported CR fields +or pretending an ignored image override changed the realization. + Combined correction validation: 149 backend/CLI/readiness focused tests passed; 97 existing/new TUI tests passed before the three extra editor-refusal cases, and those four editor cases passed separately. The final full suite passed diff --git a/docs/kubeai-backend.md b/docs/kubeai-backend.md index 97bea86d..f21cc58c 100644 --- a/docs/kubeai-backend.md +++ b/docs/kubeai-backend.md @@ -379,3 +379,15 @@ fresh Cluster sample when available. Full node/pod monitoring remains limited to the visible Cluster tab. Passive refresh performs no generation requests. Doctor reports retained terminated runtime evidence as a non-failing **WARN**, with `kube install --apply` as the fresh-verification step. + + +The KubeAI endpoint editor exposes environment and resource profiles, with no +arbitrary container image/command controls. Custom images belong in installed +KubeAI resource/model-server profiles. `runtime.image` overrides are rejected +by rendering instead of ignored; command/mount overrides remain unsupported. +Editing an existing unsupported launch reports an error rather than dropping +those settings. Compose retains its image and command controls. + +The editor's vLLM data-parallel size consumes GPUs inside one serving deployment; +it does not create independent Kubernetes replicas. KubeAI pod replication uses +`runtime.min_replicas` / `runtime.max_replicas`. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index 6f467b8f..b355a240 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -360,10 +360,10 @@ def render_models( from ..leasing.launch import translate_legacy runtime = translate_legacy(deployment.spec.get('runtime', {}) or {}) - custom = [k for k in ('command', 'mounts') if runtime.get(k)] + custom = [k for k in ('image', 'command', 'mounts') if runtime.get(k)] if custom: # A KubeAI Model runs stock vLLM: it has no place for a container - # command or host mounts (env maps onto spec.env). Fail closed, + # image override, command or host mounts (env maps onto spec.env). Fail closed, # never silently serve the stock engine instead. out.unrenderable.add(deployment.id) out.errors.append( diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 8e64f67a..1a45d42d 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -463,7 +463,11 @@ def compose(self) -> ComposeResult: yield Label('tensor-parallel size (GPUs per replica)') yield Input(value=self._rt('tensor_parallel_size'), placeholder='int, e.g. 2', id='e-tp') - yield Label('data-parallel size (replicas across GPUs)') + yield Label('vLLM data-parallel size (within one serving deployment)' if self._kubeai + else 'data-parallel size (replicas across GPUs)') + if self._kubeai: + yield Static('This consumes GPUs within a vLLM deployment; independent KubeAI pod replicas ' + 'use runtime.min_replicas / max_replicas.', classes='hint') yield Input(value=self._rt('data_parallel_size'), placeholder='int, e.g. 2', id='e-dp') yield Label('max model len (context window, tokens)') @@ -493,14 +497,18 @@ def compose(self) -> ComposeResult: yield Input(value=extra_str, placeholder='--dtype=half --enforce-eager', id='e-extra') - yield Label('container image (blank = the default vLLM image)') - yield Input(value=str(rt.get('image') or ''), - placeholder='e.g. vllm/vllm-openai:v0.25.1', id='e-image') - yield Label('container command (replaces `vllm serve MODEL …`; ' - 'blank = stock)') - yield Input(value=command_str, - placeholder="for an image with its own launcher, e.g. single", - id='e-command') + if not self._kubeai: + yield Label('container image (blank = the default vLLM image)') + yield Input(value=str(rt.get('image') or ''), + placeholder='e.g. vllm/vllm-openai:v0.25.1', id='e-image') + yield Label('container command (replaces `vllm serve MODEL …`; ' + 'blank = stock)') + yield Input(value=command_str, + placeholder="for an image with its own launcher, e.g. single", + id='e-command') + else: + yield Static('Container image/launch are configured by KubeAI server/resource profiles, ' + 'not arbitrary endpoint image/command overrides.', classes='hint') yield Label('container environment (KEY=VALUE, space-separated)') yield Static( '{max_model_len} {gpu_memory_utilization} ' @@ -647,11 +655,15 @@ def on_button_pressed(self, event: Button.Pressed) -> None: 'prefix_caching': str( self.query_one('#e-prefix', Select).value or ''), 'extra_args': self._v('e-extra'), - 'image': self._v('e-image').strip(), - 'command': self._v('e-command'), 'env': self._parse_env(self._v('e-env')), 'placement': placement, }) + if not self._kubeai: + result.update(image=self._v('e-image').strip(), command=self._v('e-command')) + else: + from .leasing.launch import translate_legacy + if any(translate_legacy(self._entry.get('runtime') or {}).get(k) for k in ('image', 'command')): + raise ValueError('KubeAI does not support runtime.image/command overrides; remove them in the catalog or use Compose.') else: result.update({ 'host': self._v('e-host'), diff --git a/tests/test_leasing_kubeai.py b/tests/test_leasing_kubeai.py index a5e20cce..58801fec 100644 --- a/tests/test_leasing_kubeai.py +++ b/tests/test_leasing_kubeai.py @@ -195,6 +195,7 @@ def test_render_attention_backend_reaches_cr_env(): @pytest.mark.parametrize('launch', [ {'serve_recipe': 'hyperqwen-3090-single'}, # a legacy entry, translated {'command': ['single']}, + {'image': 'my-vllm:foo'}, {'mounts': {'/cache': 'x/cache'}}, ]) def test_render_refuses_a_custom_container_launch(launch): diff --git a/tests/test_tui_kube.py b/tests/test_tui_kube.py index 45bc59de..7ad80f61 100644 --- a/tests/test_tui_kube.py +++ b/tests/test_tui_kube.py @@ -16,6 +16,7 @@ Collapsible, DataTable, Input, + Label, Select, Static, TabbedContent, @@ -226,6 +227,11 @@ async def scenario(): app.push_screen(screen, results.append) await pilot.pause() assert not list(screen.query('#e-gpu-pin')) + assert not list(screen.query('#e-image')) + assert not list(screen.query('#e-command')) + assert any('within one serving deployment' in str(label.render()) for label in screen.query(Label)) + assert any('runtime.min_replicas' in str(label.render()) for label in screen.query(Static)) + screen.query_one('#e-env', Input).value = 'MODE=fast' assert screen.query_one('#e-engine', Select).value == 'vllm' screen.query_one('#e-resource-profile', Input).value = 'nvidia-rtx-3090:2' screen.on_button_pressed(Button.Pressed(Button(id='ok'))) @@ -233,6 +239,8 @@ async def scenario(): asyncio.run(scenario()) entry = InferStackTUI._endpoint_entry(results[0]) assert entry['runtime']['resource_profile'] == 'nvidia-rtx-3090:2' + assert entry['runtime']['env'] == {'MODE': 'fast'} + assert 'image' not in entry['runtime'] and 'command' not in entry['runtime'] assert 'placement' not in entry @@ -440,3 +448,26 @@ def fail(*args, **kwargs): app._receive_cluster(r) assert app._ready_endpoints == [] asyncio.run(scenario()) + + +@pytest.mark.parametrize('launch', [ + {'image': 'my-vllm:foo'}, {'command': ['custom-server']}, + {'serve_recipe': 'hyperqwen-3090-single'}, +]) +def test_kubeai_editor_refuses_existing_unsupported_launch(tmp_path, launch): + controller, catalog = context(tmp_path) + results, errors = [], [] + async def scenario(): + app = InferStackTUI(controller, catalog, interval=999, proc_factory=lambda svc: None) + async with app.run_test() as pilot: + screen = _AddEndpointScreen(['qc'], kubeai=True, name='q', + entry={'engine': 'vllm', 'model': 'qc', 'runtime': launch}) + screen._error = errors.append + app.push_screen(screen, results.append) + await pilot.pause() + screen.on_button_pressed(Button.Pressed(Button(id='ok'))) + await pilot.pause() + assert app.screen is screen + assert results == [] + assert 'does not support runtime.image/command' in errors[0] + asyncio.run(scenario()) From a3e08fd9ce8e24ebb150aaff001020707dabbcb2 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 23:14:34 -0400 Subject: [PATCH 138/141] Preserve Kubernetes generation evidence across polling and late responses Share unsupported launch policy with the endpoint editor, including mounts; retain pod identity through Model-only samples and scope failed generation evidence to its request incarnation. Co-authored-by: GPT-6.1-Sol --- dev/journals/codex.md | 28 ++++++++++++++++++++++++++++ infer_stack/backends/kubeai.py | 11 ++++++++++- infer_stack/kube/monitor.py | 8 ++++++-- infer_stack/tui.py | 17 ++++++++++------- tests/test_tui_kube.py | 22 ++++++++++++++++++++-- 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/dev/journals/codex.md b/dev/journals/codex.md index b00fb5d9..d6eeae45 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -396,3 +396,31 @@ probe errors). No workload or host mutations occurred in this correction. The earlier unchanged long-model CPU e2e passed; GPU placement/generation and fresh worker/runtime tests still need aiq-gpu/namek. Readiness and editor changes are committed separately so operator-evidence policy is reviewable on its own. + +## 2026-10-01 23:13:37 -0400 + +The user wants easier worker onboarding and GPU acceptance for namek (one +3090), then yardrat (two heterogeneous GPUs) and aiq-gpu2 (four smaller GPUs), +while correcting three focused TUI review findings. Model/configuration: GPT-6 +Codex, Default collaboration mode; reasoning effort is not exposed. Commits +retain the requested GPT-6.1-Sol co-author attribution. + +I separated evidence identity from sampling detail: a Model-only observation +retains known pod incarnation only when Model UID/generation agrees, while +fresh pod observations and readiness loss remain authoritative contradictions. +Old asynchronous failures now invalidate only their own incarnation's proof; +late success likewise cannot alter a replacement. The editor and renderer now +share the stock KubeAI launch capability policy, including mounts and legacy +recipes. This protects evidence from UI tab changes without pretending stale +pod observations are current pod facts. Focused TUI/backend validation: 97 +passed; scoped Ruff passed. A temporary missing local render variable during +refactoring was caught by focused tests and corrected before commit. + +The onboarding design should separate local agent membership from explicit +administrative access and test the named worker rather than total cluster +capacity. Mixed-product nodes need actual device observations: a product label +on a node cannot promise which physical GPU Kubernetes will allocate. I am +building a package-native, reviewed plan/apply path with isolated test resources, +exact placement checks, real generation and ownership-limited cleanup. Driver +installation remains an explicit host prerequisite rather than an implicit OS +migration. Real GPU acceptance will be run by the maintainer, not on this VM. diff --git a/infer_stack/backends/kubeai.py b/infer_stack/backends/kubeai.py index b355a240..48be312d 100644 --- a/infer_stack/backends/kubeai.py +++ b/infer_stack/backends/kubeai.py @@ -50,12 +50,21 @@ DEPLOYMENT_LABEL = 'infer-stack/deployment' # KubeAI's Model CRD is stricter than Kubernetes's usual DNS label limit. MODEL_NAME_MAX_LENGTH = 40 +UNSUPPORTED_RUNTIME_FIELDS = ('image', 'command', 'mounts') DEFAULT_NAMESPACE = 'kubeai' # The standard local access path: `kubectl port-forward svc/kubeai 8000:80`. # An ingress-fronted cluster overrides this via the kubeai_base_url setting. DEFAULT_BASE_URL = 'http://127.0.0.1:8000/openai/v1' +def unsupported_runtime_fields(runtime: dict) -> tuple[str, ...]: + """Shared stock KubeAI launch policy, including legacy launch recipes.""" + from ..leasing.launch import translate_legacy + + translated = translate_legacy(runtime) + return tuple(key for key in UNSUPPORTED_RUNTIME_FIELDS if translated.get(key)) + + def model_replica_counts(model: dict) -> tuple[int | None, int | None]: """Observed (all, ready) replicas; missing counts stay unknown. @@ -360,7 +369,7 @@ def render_models( from ..leasing.launch import translate_legacy runtime = translate_legacy(deployment.spec.get('runtime', {}) or {}) - custom = [k for k in ('image', 'command', 'mounts') if runtime.get(k)] + custom = unsupported_runtime_fields(runtime) if custom: # A KubeAI Model runs stock vLLM: it has no place for a container # image override, command or host mounts (env maps onto spec.env). Fail closed, diff --git a/infer_stack/kube/monitor.py b/infer_stack/kube/monitor.py index 959952b1..e48fa433 100644 --- a/infer_stack/kube/monitor.py +++ b/infer_stack/kube/monitor.py @@ -99,7 +99,7 @@ def pod_rows(report: dict) -> list[tuple]: return sorted(rows) -def model_states(report: dict) -> dict[str, dict]: +def model_states(report: dict, previous: dict | None = None) -> dict[str, dict]: """Lifecycle evidence from one sample; declaring a CR proves no readiness.""" states = {} for model in report['models'] or []: @@ -123,12 +123,16 @@ def model_states(report: dict) -> dict[str, dict]: if meta.get('deletionTimestamp'): stage, replica_ready = 'deleting', False # Proof is scoped to this incarnation; replacement/restarts invalidate - # an earlier successful API request. A Model-only sample has no pod proof. + # an earlier successful API request. Less detailed observations retain + # known pod identity only while the Model incarnation is unchanged. identity = (meta.get('uid'), meta.get('generation'), tuple(sorted( (p.get('metadata', {}).get('uid', ''), tuple( (c.get('containerID', ''), c.get('restartCount', 0)) for c in p.get('status', {}).get('containerStatuses') or [])) for p in pods)) if report['pods'] is not None else None) + old_identity = (previous or {}).get(gid, {}).get('identity') + if report['pods'] is None and old_identity and identity[:2] == old_identity[:2]: + identity = (*identity[:2], old_identity[2]) states[gid] = {'name': meta.get('name'), 'stage': stage, 'replica_ready': replica_ready, 'all_replicas': total, 'ready_replicas': ready, 'identity': identity} diff --git a/infer_stack/tui.py b/infer_stack/tui.py index 1a45d42d..09c473c9 100644 --- a/infer_stack/tui.py +++ b/infer_stack/tui.py @@ -661,9 +661,10 @@ def on_button_pressed(self, event: Button.Pressed) -> None: if not self._kubeai: result.update(image=self._v('e-image').strip(), command=self._v('e-command')) else: - from .leasing.launch import translate_legacy - if any(translate_legacy(self._entry.get('runtime') or {}).get(k) for k in ('image', 'command')): - raise ValueError('KubeAI does not support runtime.image/command overrides; remove them in the catalog or use Compose.') + from .backends.kubeai import unsupported_runtime_fields + unsupported = unsupported_runtime_fields(self._entry.get('runtime') or {}) + if unsupported: + raise ValueError(f'KubeAI does not support runtime.{unsupported[0]}; remove it in the catalog or use Compose.') else: result.update({ 'host': self._v('e-host'), @@ -1727,8 +1728,10 @@ def _record_kube_generation(self, model, token, success): return if success and token and self._kube_tokens.get(model) == token: self._kube_verified[model] = (token, time.strftime('%H:%M:%S')) - else: - self._kube_verified.pop(model, None) + elif not success: + existing = self._kube_verified.get(model) + if existing and existing[0] == token: + self._kube_verified.pop(model, None) self._sync_api_models(list(self._api_models_wanted)) def _update_catalog_help(self) -> None: @@ -1805,7 +1808,7 @@ def _collect(self) -> dict[str, Any]: # full node/pod monitoring remains gated by the Cluster tab. report = snapshot(self._cluster_manager(), namespace=self.controller.backend.namespace, resources=('models',)) - self._kube_states = model_states(report) + self._kube_states = model_states(report, self._kube_states) self._kube_model_error = report['errors'].get('models', '') self._observed = set(self._kube_states) self._assignments = {} @@ -3035,7 +3038,7 @@ def _receive_cluster(self, report): self._cluster_at = time.monotonic() if report['models'] is not None: from .kube.monitor import model_states - self._kube_states = model_states(report) + self._kube_states = model_states(report, self._kube_states) self._kube_model_error = '' self._observed = set(self._kube_states) self._observed_at = self._cluster_at diff --git a/tests/test_tui_kube.py b/tests/test_tui_kube.py index 7ad80f61..9ed8fe8c 100644 --- a/tests/test_tui_kube.py +++ b/tests/test_tui_kube.py @@ -421,12 +421,26 @@ async def scenario(): assert '1/2 ready' in str(app.query_one('#cluster-summary', Static).render()) assert 'generation unverified' in str(app.query_one('#api-model', Select)._options) assert requests == [] # no generation during passive refresh + r['pods'] = [{'metadata': {'namespace': 'default', 'uid': 'pod-1', + 'labels': {'infer-stack/deployment': 'grp-q'}}, + 'spec': {'nodeName': 'namek'}, + 'status': {'conditions': [{'type': 'Ready', 'status': 'True'}], + 'containerStatuses': [{'containerID': 'container-1', 'restartCount': 0}]}}] + app._receive_cluster(r) app.action_api_send() await app.workers.wait_for_complete() await pilot.pause() assert 'qwen-coder' in app._kube_verified assert 'Last successful generation' in str(app.query_one('#api-readiness', Static).render()) assert len(requests) == 1 + proof = app._kube_verified['qwen-coder'] + full_pods = r['pods'] + r['pods'] = None + app._receive_cluster(r) + assert app._kube_verified['qwen-coder'] == proof + r['pods'] = full_pods + app._receive_cluster(r) + assert app._kube_verified['qwen-coder'] == proof old_post = app._http_client().post def fail(*args, **kwargs): raise RuntimeError('upstream loading') @@ -444,6 +458,10 @@ def fail(*args, **kwargs): assert not app._kube_verified app._record_kube_generation('qwen-coder', old_token, True) assert not app._kube_verified # late response cannot verify a replacement + app._record_kube_generation('qwen-coder', app._kube_tokens['qwen-coder'], True) + current_proof = app._kube_verified['qwen-coder'] + app._record_kube_generation('qwen-coder', old_token, False) + assert app._kube_verified['qwen-coder'] == current_proof # late failure cannot erase newer proof r['models'][0]['status']['replicas']['ready'] = 0 app._receive_cluster(r) assert app._ready_endpoints == [] @@ -452,7 +470,7 @@ def fail(*args, **kwargs): @pytest.mark.parametrize('launch', [ {'image': 'my-vllm:foo'}, {'command': ['custom-server']}, - {'serve_recipe': 'hyperqwen-3090-single'}, + {'serve_recipe': 'hyperqwen-3090-single'}, {'mounts': ['/host:/container']}, ]) def test_kubeai_editor_refuses_existing_unsupported_launch(tmp_path, launch): controller, catalog = context(tmp_path) @@ -469,5 +487,5 @@ async def scenario(): await pilot.pause() assert app.screen is screen assert results == [] - assert 'does not support runtime.image/command' in errors[0] + assert 'does not support runtime.' in errors[0] asyncio.run(scenario()) From 34e2c590b85b0a4f0ab20c7f1f590df386669326 Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 23:53:23 -0400 Subject: [PATCH 139/141] Add private K3s onboarding and named GPU worker acceptance Export private resumable join bundles, infer server versions through explicit matching admin access, join and verify the exact worker, then test fresh GPU identities, one-GPU KubeAI placement and real generation. Preserve leases and unrelated Models; reuse Helm helpers and retain node profiles with ownership-scoped interrupted cleanup. Co-authored-by: GPT-6.1-Sol --- dev/journals/codex.md | 44 +++ infer_stack/cli/commands_kube.py | 4 + infer_stack/cli/commands_kube_worker.py | 195 ++++++++++ infer_stack/kube/k3s.py | 63 ++++ infer_stack/kube/worker.py | 475 ++++++++++++++++++++++++ tests/test_kube_worker.py | 467 +++++++++++++++++++++++ 6 files changed, 1248 insertions(+) create mode 100644 infer_stack/cli/commands_kube_worker.py create mode 100644 infer_stack/kube/worker.py create mode 100644 tests/test_kube_worker.py diff --git a/dev/journals/codex.md b/dev/journals/codex.md index d6eeae45..3f3094ec 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -424,3 +424,47 @@ building a package-native, reviewed plan/apply path with isolated test resources exact placement checks, real generation and ownership-limited cleanup. Driver installation remains an explicit host prerequisite rather than an implicit OS migration. Real GPU acceptance will be run by the maintainer, not on this VM. + +The worker path is now `kube k3s export` on the server (private join bundle), +`kube k3s onboard NAME` on the worker (plan/apply), and `kube node test NAME` +for repeat acceptance from any admin host. Export writes 0700/0600 artifacts, +rewrites only the copied server URL, identifies ownership by cluster CA, and +writes its manifest first so interrupted export can resume. Onboard requires +explicit matching admin access and existing host driver/toolkit, infers server +version, reuses membership validation and the existing plugin/chart installers, +then waits for this worker's exact expected resources. Stale EKS context does +not define its target. I intentionally did not add an OS package/driver upgrade +policy to a cluster join command. + +Acceptance reserves all expected GPUs briefly for a fresh NVML device query, +then one GPU for a Model rendered through the existing backend. It asserts the +actual node, request/limit/runtime, replica readiness, serving GPU identity and +real generation response. The temporary Model is outside the lease managed +selector, so no second ledger authority can prune production Models. Stable +node profiles are added through the shared Helm helper with existing version +and values preserved; profiles remain for reuse, while exact owned run resources +are cleaned on failure/success. Interrupted runs have explicit ownership-checked +cleanup, not namespace-wide deletion. Heterogeneous nodes report every UUID, +product and memory; they do not claim generation on every product or GPU-index +selection. Pod startup polling replaces a long opaque wait and emits changes. + +Focused CLI/backend/readiness/TUI validation initially passed 228 cases; worker +acceptance/export tests now cover additional cases. The first full-suite run +passed 1251 with 3 skips and 8 existing ResourceWarnings; final validation follows +after the last export and preservation regressions. Manual root CLI simulations +for namek, yardrat and aiq-gpu2 all passed, with real kwconf parsing/rendering and +fake command/HTTP seams (`/tmp/infer-stack-worker-simulated.log`). A real VM +loopback service forward returned /models HTTP 200 and was stopped on exit; +no worker join, GPU workload, chart upgrade or credential export ran on the VM. + +Final validation: focused suites 233 passed; final full `pytest -q` 1256 passed, +3 skipped, 8 pre-existing cleanup/ResourceWarnings (140.19 seconds), using +`/tmp/infer-stack-venv/bin` (CPython 3.14.6). `ty check infer_stack`, CI flake8 +(E9/F63/F7/F82), scoped Ruff for all changed Python/new worker/TUI tests, +compileall, shell syntax and diff whitespace checks passed. Real help tree and +kube/node/test/k3s/onboard/export help were inspected. Export --plan created no +files. A real read-only VM node-test JSON plan returned exit 1 and correctly +reported Ready but zero allocatable GPUs/missing GFD, rather than using another +node as evidence. The full worker hardware/generation acceptance remains for +namek/aiq-gpu, with heterogeneous generation limited to one allocated GPU per +run. No maintainer design decision is needed to try that documented workflow. diff --git a/infer_stack/cli/commands_kube.py b/infer_stack/cli/commands_kube.py index 5372e57c..bcf09667 100644 --- a/infer_stack/cli/commands_kube.py +++ b/infer_stack/cli/commands_kube.py @@ -15,6 +15,7 @@ from ..kube import KubeManager from ..kube.manage import NVIDIA_DEVICE_PLUGIN_VERSION +from .commands_kube_worker import K3sExportCLI, K3sOnboardCLI, KubeNodeTestCLI from .context import _apply_path_overrides from .options import _PathOverridesMixin @@ -274,6 +275,7 @@ class KubeNodeModalCLI(kw.ModalCLI): __command__ = 'node' status = KubeNodeStatusCLI + test = KubeNodeTestCLI detach = KubeNodeDetachCLI attach = KubeNodeAttachCLI @@ -426,6 +428,8 @@ class K3sModalCLI(kw.ModalCLI): __command__ = 'k3s' bootstrap = K3sBootstrapCLI join = K3sJoinCLI + onboard = K3sOnboardCLI + export = K3sExportCLI status = K3sStatusCLI diff --git a/infer_stack/cli/commands_kube_worker.py b/infer_stack/cli/commands_kube_worker.py new file mode 100644 index 00000000..e89c82c0 --- /dev/null +++ b/infer_stack/cli/commands_kube_worker.py @@ -0,0 +1,195 @@ +"""Reviewable named-worker onboarding and acceptance through kwconf.""" +from __future__ import annotations + +import json +import subprocess +import uuid + +import kwconf as kw + +from ..kube.manage import KubeManager +from ..kube.worker import ( + DEFAULT_MODEL, + acceptance, + cleanup, + onboard, + onboarding_plan, + plan, + private_file, + scoped_manager, +) +from .context import _apply_path_overrides +from .options import _PathOverridesMixin + + +def _show_result(result, machine=False): + if machine: + print(json.dumps(result, indent=2)) + return + print(f"worker: {result.get('node') or result.get('facts', {}).get('node')}") + for key in ('context', 'namespace', 'server', 'version', 'profile_name', 'profile', 'run_id'): + if key in result and result[key] is not None: + print(f' {key}: {result[key]}') + facts = result.get('facts') or {} + if facts: + print(f" Ready: {facts['ready']}; allocatable GPUs: {facts['gpu_count']}; GFD: {facts['product']}, {facts['memory_mib']} MiB") + for message in facts.get('errors') or []: + print(f'[FAIL] {message}') + for device in result.get('devices') or []: + print(f" GPU {device['uuid']}: {device['product']}, {device['memory_mib'] / 1024:g} GiB") + for action in result.get('actions') or []: + print(f'[plan] {action}') + for message in result.get('limitations') or []: + print(f'[info] {message}') + for device in result.get('serving_devices') or []: + print(f" serving GPU: {device['uuid']} ({device['product']})") + if result.get('generation_verified'): + print('[ok] GPU runtime, exact node placement, one-GPU serving replica and real generation') + print('Temporary test resources removed; reusable node profile retained.') + if 'cleanup_run' in result: + print(f"Cleanup {result['cleanup_run']}: {'complete' if result['applied'] else 'planned'}") + + +class KubeNodeTestCLI(_PathOverridesMixin): + """Test this exact worker's GPUs, model placement and real generation. + + Plans by default. Apply briefly reserves ALL expected GPUs for a device + query, then ONE GPU for a temporary serving replica. Leaves a reusable + node profile, deletes only this test's resources, and preserves leases. + NVIDIA drivers/container toolkit must already be installed on the worker. + """ + + __command__ = 'test' + node = kw.Value(None, type=str, position=1, required=True, help='Exact Kubernetes node name.') + expected_gpus = kw.Value(None, type=int, help='Required exact count, e.g. 1 for namek, 2 for yardrat, 4 for aiq-gpu2.') + kubeconfig = kw.Value(None, type=str, help='Explicit admin kubeconfig; otherwise the selected context.') + namespace = kw.Value('kubeai', type=str, help='Existing KubeAI namespace (use default on aiq-gpu).') + release = kw.Value('kubeai', type=str, help='Existing KubeAI Helm release.') + resource_profile = kw.Value(None, type=str, help='Optional installed one-GPU base profile to constrain to this node.') + base_url = kw.Value(None, type=str, help='Direct KubeAI OpenAI URL; default: temporary loopback service forwarding.') + model = kw.Value(DEFAULT_MODEL, type=str, help='Hugging Face model for real generation.') + timeout = kw.Value(900, type=int, help='GPU/startup wait budget in seconds (first image pull can take minutes).') + run_id = kw.Value(None, type=str, help='Acceptance run ID; generated unless supplied. Required for explicit cleanup.') + cleanup = kw.Value(False, isflag=True, help='Remove only a named interrupted test run; needs --run-id --apply.') + apply = kw.Value(False, isflag=True, alias=['yes'], help='Authorize the printed test/installation changes.') + dry_run = kw.Value(False, isflag=True, alias=['plan'], help='Only display the plan, even with --apply.') + json = kw.Value(False, isflag=True, help='Emit structured plan/result; progress goes to stderr.') + + @classmethod + def main(cls, argv=True, **kwargs): + import sys + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + manager = scoped_manager(KubeManager(), config.kubeconfig) + def progress(message): + print(message, file=sys.stderr if config.json else sys.stdout, flush=True) + try: + if config.cleanup: + if not config.run_id: + raise RuntimeError('--cleanup requires --run-id; no broad cleanup is performed') + if config.apply and not config.dry_run: + cleanup(manager, node=config.node, namespace=config.namespace, run_id=config.run_id) + result = {'cleanup_run': config.run_id, 'node': config.node, + 'applied': bool(config.apply and not config.dry_run)} + else: + if config.expected_gpus is None: + raise RuntimeError('--expected-gpus is required; cluster totals cannot prove this worker is configured') + result = plan(manager, node=config.node, namespace=config.namespace, + expected_gpus=config.expected_gpus, resource_profile=config.resource_profile) + if config.apply and not config.dry_run: + result = acceptance(manager, node=config.node, namespace=config.namespace, + release=config.release, expected_gpus=config.expected_gpus, + base_url=config.base_url, kubeconfig=config.kubeconfig, + run_id=config.run_id or uuid.uuid4().hex[:12], + timeout=config.timeout, model=config.model, + resource_profile=config.resource_profile, progress=progress) + _show_result(result, config.json) + if not config.apply or config.dry_run: + print('No changes made. Review the target node and rerun with --apply.', file=sys.stderr if config.json else sys.stdout) + facts = result.get('facts') + return int(isinstance(facts, dict) and bool(facts.get('errors'))) + except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as ex: + raise SystemExit(f'worker test: {ex}') from ex + + +class K3sOnboardCLI(KubeNodeTestCLI): + """Join this GPU host and test it through explicit control-plane access. + + Plans by default. Checks host driver/runtime and admin target, infers the + server's K3s version, joins/resumes membership, then verifies every local + device plus node-specific model placement/generation. No kubeconfig or + catalog/ledger is replaced. Driver/toolkit installation is a prerequisite. + """ + + __command__ = 'onboard' + kubeconfig = kw.Value(None, type=str, required=True, help='Private admin kubeconfig selecting the SAME --server; never inferred from stale context.') + server = kw.Value(None, type=str, required=True, help='Existing K3s server URL, e.g. https://aiq-gpu:6443.') + token_file = kw.Value(None, type=str, required=True, help='Private file containing the join token; never pass token text.') + version = kw.Value(None, type=str, help='Exact K3s version; otherwise infer from this cluster control plane.') + + @classmethod + def main(cls, argv=True, **kwargs): + import sys + config = cls.cli(argv=argv, data=kwargs) + _apply_path_overrides(config) + local = KubeManager() + cluster = scoped_manager(local, config.kubeconfig) + def progress(message): + print(message, file=sys.stderr if config.json else sys.stdout, flush=True) + try: + if config.cleanup: + raise RuntimeError('Use infer-stack kube node test NODE --cleanup --run-id=... --apply for interrupted acceptance resources') + private_file(config.kubeconfig, 'Admin kubeconfig') + private_file(config.token_file, 'Join token') + prepared = onboarding_plan(local, cluster, server=config.server, node=config.node, version=config.version) + if config.expected_gpus is not None and config.expected_gpus != prepared['expected_gpus']: + raise RuntimeError('--expected-gpus differs from locally detected hardware') + result = prepared + if config.apply and not config.dry_run: + result = onboard(local, cluster, prepared, token_file=config.token_file, + namespace=config.namespace, release=config.release, + base_url=config.base_url, kubeconfig=config.kubeconfig, + run_id=config.run_id or uuid.uuid4().hex[:12], timeout=config.timeout, + model=config.model, resource_profile=config.resource_profile, + progress=progress) + _show_result(result, config.json) + if not config.apply or config.dry_run: + print('No changes made. Authenticate with sudo -v, review this plan, then rerun with --apply.', file=sys.stderr if config.json else sys.stdout) + return 0 + except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as ex: + raise SystemExit(f'worker onboarding: {ex}') from ex + + +class K3sExportCLI(kw.Config): + """Export private token/admin kubeconfig files for reviewed worker onboarding. + + Run on the K3s server. Plans by default; --apply creates a mode-0700 bundle + with mode-0600 files and rewrites only its copy's server URL. Existing owned + bundles can refresh; unrelated directories/clusters are refused. + """ + + __command__ = 'export' + server = kw.Value(None, type=str, required=True, help='Reachable HTTPS server URL as workers should use it.') + directory = kw.Value(None, type=str, required=True, help='Private destination directory for token/kubeconfig/manifest.') + apply = kw.Value(False, isflag=True, alias=['yes'], help='Authorize writing/refreshing the private join bundle.') + dry_run = kw.Value(False, isflag=True, alias=['plan'], help='Only display paths; never read/write credentials.') + + @classmethod + def main(cls, argv=True, **kwargs): + from pathlib import Path + + from ..kube.k3s import export_join_info + config = cls.cli(argv=argv, data=kwargs) + try: + directory = Path(config.directory).expanduser() + if config.apply and not config.dry_run: + result = export_join_info(server=config.server, directory=directory) + print(json.dumps(result, indent=2)) + print('Private join bundle exported. Transfer it securely to the worker; then run kube k3s onboard NAME using these files. Keep admin credentials private.') + else: + print(f'[plan] Export private join bundle for {config.server} into {directory} (directory 0700, files 0600).') + print('[plan] Files: token, kubeconfig.yaml (reachable server URL), manifest.json. Original kubeconfigs remain unchanged.') + print('No credentials read or written. Authenticate with sudo -v, then rerun with --apply.') + return 0 + except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as ex: + raise SystemExit(f'join export: {ex}') from ex diff --git a/infer_stack/kube/k3s.py b/infer_stack/kube/k3s.py index 574157e1..06e686dd 100644 --- a/infer_stack/kube/k3s.py +++ b/infer_stack/kube/k3s.py @@ -271,3 +271,66 @@ def local_status(*, run: RunFunc | None = None) -> dict: except Exception: report['errors'].append('Cannot inspect active agent configuration; authenticate with sudo -v then retry') return report + + +def export_join_info(*, server: str, directory: str | Path, run: RunFunc | None = None) -> dict: + """Export a private join bundle; refresh only a bundle owned by this cluster. + + Token and rewritten admin kubeconfig never appear on stdout. The original + root/default configs are unchanged. The manifest is written first so an + interrupted export can safely complete on retry. + """ + import hashlib + + run = run or default_run + if not server.startswith('https://'): + raise RuntimeError('Join server must use https:// with a reachable LAN/VPN address') + if not _active(run, 'k3s'): + raise RuntimeError('Export must run on the local K3s server; an agent/selected remote context is not sufficient') + config = yaml.safe_load(run(['sudo', '-n', 'cat', str(K3S_KUBECONFIG)])) + if not isinstance(config, dict): + raise RuntimeError('Local K3s kubeconfig must contain a mapping') + token = run(['sudo', '-n', 'cat', str(K3S_NODE_TOKEN)]).strip() + if not token: + raise RuntimeError('Local K3s node token is empty') + contexts = {c['name']: c['context'] for c in config.get('contexts') or []} + selected = contexts.get(config.get('current-context'), {}).get('cluster') + clusters = [c for c in config.get('clusters') or [] if c.get('name') == selected] + if len(clusters) != 1 or not clusters[0].get('cluster', {}).get('certificate-authority-data'): + raise RuntimeError('Local K3s kubeconfig has no unique selected cluster with embedded CA data') + ca_data = clusters[0]['cluster']['certificate-authority-data'] + if not isinstance(ca_data, str): + raise RuntimeError('Local K3s CA data must be a string') + fingerprint = hashlib.sha256(ca_data.encode()).hexdigest() + clusters[0]['cluster']['server'] = server.rstrip('/') + target = Path(directory).expanduser() + marker = target / 'manifest.json' + metadata = {'kind': 'infer-stack-k3s-join', 'cluster_ca_sha256': fingerprint, 'server': server.rstrip('/')} + if target.exists(): + if target.is_symlink() or not target.is_dir() or target.stat().st_uid != os.getuid(): + raise RuntimeError('Join export directory must be a directory owned by this user, not a symlink') + if target.stat().st_mode & 0o077: + raise RuntimeError(f'Join export directory must be private (chmod 700): {target}') + if any(target.iterdir()): + if not marker.is_file(): + raise RuntimeError('Refusing to overwrite a nonempty directory without an infer-stack join manifest') + previous = json.loads(marker.read_text()) + if previous.get('kind') != metadata['kind'] or previous.get('cluster_ca_sha256') != fingerprint: + raise RuntimeError('Join export belongs to a different cluster; choose a new directory') + else: + target.mkdir(parents=True, mode=0o700) + documents = {'manifest.json': json.dumps(metadata, indent=2) + '\n', + 'kubeconfig.yaml': yaml.safe_dump(config), 'token': token + '\n'} + for name, content in documents.items(): + # NamedTemporaryFile creates mode 0600 before any secret is written. + with tempfile.NamedTemporaryFile('w', dir=target, prefix='.join-', delete=False) as temporary: + pending = Path(temporary.name) + try: + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(pending, target / name) + finally: + pending.unlink(missing_ok=True) + return {'server': metadata['server'], 'directory': str(target), + 'kubeconfig': str(target / 'kubeconfig.yaml'), 'token_file': str(target / 'token')} diff --git a/infer_stack/kube/worker.py b/infer_stack/kube/worker.py new file mode 100644 index 00000000..a83cb537 --- /dev/null +++ b/infer_stack/kube/worker.py @@ -0,0 +1,475 @@ +"""Named-worker acceptance, separate from the normal lease authority. + +Tests reserve the target node's GPUs briefly, then create one isolated KubeAI +Model. They never converge/prune the application's managed Model set or ledger. +""" +from __future__ import annotations + +import copy +import csv +import hashlib +import io +import json +import re +import selectors +import shlex +import subprocess +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import requests + +from ..backends.kubeai import ( + GPU_MEMORY_LABEL, + GPU_PRODUCT_LABEL, + model_replica_counts, + render_models, +) +from ..leasing.models import Deployment +from .manage import KubeManager + +RUN_LABEL = 'infer-stack/acceptance-run' +NODE_LABEL = 'infer-stack/acceptance-node' +DEFAULT_MODEL = 'HuggingFaceTB/SmolLM2-135M-Instruct' +GPU_PROBE_IMAGE = 'nvidia/cuda:12.4.1-base-ubuntu22.04' + + +def scoped_manager(manager: KubeManager, kubeconfig: str | Path | None) -> KubeManager: + """Scope both Helm and kubectl without changing the operator's environment.""" + if not kubeconfig: + return manager + path = str(Path(kubeconfig).expanduser().resolve()) + def run(args, **kwargs): + if args[0] in {'kubectl', 'helm'}: + args = [args[0], '--kubeconfig', path, *args[1:]] + return manager.run(args, **kwargs) + class ScopedManager(KubeManager): + def command_exists(self, name: str) -> bool: + return manager.command_exists(name) + return ScopedManager(run=run) + + +def devices(text: str) -> list[dict]: + """Preserve each device's identity/product/memory, including mixed nodes.""" + rows = [] + for row in csv.reader(io.StringIO(text)): + if row: + uuid, product, memory = (value.strip() for value in row) + rows.append({'uuid': uuid, 'product': product, 'memory_mib': int(float(memory))}) + if not rows or len({row['uuid'] for row in rows}) != len(rows): + raise RuntimeError('GPU query returned no devices or duplicate device identities') + return sorted(rows, key=lambda row: row['uuid']) + + +def local_devices(manager: KubeManager) -> list[dict]: + return devices(manager.run(['nvidia-smi', '--query-gpu=uuid,name,memory.total', + '--format=csv,noheader,nounits'])) + + +def worker_facts(manager: KubeManager, node: str, expected_gpus: int) -> dict: + if expected_gpus < 1: + raise RuntimeError('expected GPUs must be positive') + raw = manager.node(node) + labels = raw.get('metadata', {}).get('labels') or {} + count = int(raw.get('status', {}).get('allocatable', {}).get('nvidia.com/gpu', 0)) + facts = {'node': node, 'ready': manager._node_ready(raw), 'gpu_count': count, + 'product': labels.get(GPU_PRODUCT_LABEL), 'memory_mib': labels.get(GPU_MEMORY_LABEL), + 'hostname': labels.get('kubernetes.io/hostname'), + 'schedulable': not raw.get('spec', {}).get('unschedulable', False)} + facts['errors'] = [] + if not facts['ready']: + facts['errors'].append(f'node {node} is not Ready') + if not facts['schedulable']: + facts['errors'].append(f'node {node} is cordoned; use infer-stack kube node attach {node} when appropriate') + if count != expected_gpus: + facts['errors'].append(f'node {node} exposes {count} GPUs, expected {expected_gpus}; check its NVIDIA runtime/device-plugin') + if not facts['product'] or not facts['memory_mib']: + facts['errors'].append(f'node {node} is missing GFD product/memory labels') + if not facts['hostname']: + facts['errors'].append(f'node {node} has no kubernetes.io/hostname label') + return facts + + +def worker_profile(manager: KubeManager, facts: dict, *, namespace: str, + resource_profile: str | None = None) -> tuple[str, dict, bool]: + """Add a stable, node-constrained profile without rewriting operator profiles.""" + config = manager.kubectl_json(['-n', namespace, 'get', 'configmap', 'kubeai-config', '-o', 'json']) + import yaml + system = yaml.safe_load(config.get('data', {}).get('system.yaml') or '{}') + installed = (system or {}).get('resourceProfiles') or {} + if resource_profile: + if ':' in resource_profile or resource_profile not in installed: + raise RuntimeError('resource-profile must name an installed profile (without a :count suffix)') + profile = copy.deepcopy(installed[resource_profile]) + else: + profile: dict[str, Any] = {'imageName': 'nvidia-gpu', 'runtimeClassName': 'nvidia', + 'requests': {'nvidia.com/gpu': '1'}, 'limits': {'nvidia.com/gpu': '1'}} + for field in ('requests', 'limits'): + if str(profile.get(field, {}).get('nvidia.com/gpu')) != '1': + raise RuntimeError('Worker acceptance requires a base profile requesting/limiting one NVIDIA GPU') + if profile.get('runtimeClassName') != 'nvidia': + raise RuntimeError('Worker acceptance requires runtimeClassName nvidia') + selector = profile.setdefault('nodeSelector', {}) + labels = manager.node(facts['node']).get('metadata', {}).get('labels') or {} + if any(labels.get(k) != v for k, v in selector.items()): + raise RuntimeError('Base resource profile selectors do not match the requested node') + selector['kubernetes.io/hostname'] = facts['hostname'] + digest = hashlib.sha256(json.dumps([facts['node'], resource_profile], sort_keys=True).encode()).hexdigest()[:12] + name = 'infer-stack-node-' + digest + if name in installed and installed[name] != profile: + raise RuntimeError(f'Existing profile {name} differs; refusing to overwrite it') + return name, profile, name not in installed + + +def _names(node: str, run_id: str) -> tuple[str, str]: + if not re.fullmatch(r'[a-z0-9]{8,20}', run_id): + raise RuntimeError('run-id must contain 8-20 lowercase letters/digits') + if not re.fullmatch(r'[a-z0-9](?:[-.a-z0-9]*[a-z0-9])?', node) or len(node) > 63: + raise RuntimeError('Node name must fit a Kubernetes label value (at most 63 characters)') + return f'infer-stack-gpu-{run_id}', f'infer-stack-e2e-{run_id}' + + +def cleanup(manager: KubeManager, *, node: str, namespace: str, run_id: str) -> None: + """Explicit retry/cleanup deletes only exact, matching acceptance resources.""" + pod_name, model_name = _names(node, run_id) + for kind, name in [('models.kubeai.org', model_name), ('pod', pod_name)]: + existing = manager.kubectl_json(['-n', namespace, 'get', kind, name, + '--ignore-not-found', '-o', 'json']) + if not existing: + continue + labels = existing.get('metadata', {}).get('labels') or {} + if labels.get(RUN_LABEL) != run_id or labels.get(NODE_LABEL) != node: + raise RuntimeError(f'Refusing to delete unrelated {kind}/{name}') + manager.run(['kubectl', '-n', namespace, 'delete', kind, name, '--wait=true', '--timeout=180s']) + # Model deletion is asynchronous; wait for its backing Pods to release GPUs. + manager.run(['kubectl', '-n', namespace, 'wait', '--for=delete', 'pods', + '-l', f'{RUN_LABEL}={run_id},{NODE_LABEL}={node}', '--timeout=180s']) + manager.run(['kubectl', '-n', namespace, 'wait', '--for=delete', 'pods', + '-l', f'model={model_name}', '--timeout=180s']) + + +def plan(manager: KubeManager, *, node: str, namespace: str, expected_gpus: int, + resource_profile: str | None = None) -> dict: + facts = worker_facts(manager, node, expected_gpus) + result = {'facts': facts, 'actions': [ + f'Fresh NVIDIA runtime/device query reserving all {expected_gpus} GPUs on {node} temporarily', + f'Create one temporary KubeAI Model on {node}; verify GPU requests, replica readiness and real generation', + 'Delete only this run\'s Model/Pod; retain its reusable node profile; leave catalog/ledger/gateway untouched'], + 'context': manager.current_context(), 'namespace': namespace, 'profile_name': None, 'profile': None, 'needs_profile': False} + if facts['errors']: + return result + name, profile, missing = worker_profile(manager, facts, namespace=namespace, resource_profile=resource_profile) + result.update(profile_name=name, profile=profile, needs_profile=missing) + if missing: + result['actions'].insert(0, f'Add node profile {name} through the existing KubeAI Helm installer (preserve installed chart version/values)') + result['profile'] = profile + return result + + +def wait_gpu_probe(manager: KubeManager, *, node: str, namespace: str, name: str, + timeout: int, progress=print, sleep=time.sleep, clock=time.monotonic): + """Bounded fresh launch with changing startup details, not a silent long wait.""" + from .monitor import pod_rows + + deadline, last = clock() + timeout, None + while True: + pod = manager.kubectl_json(['-n', namespace, 'get', 'pod', name, '-o', 'json']) + phase = pod.get('status', {}).get('phase') + rows = pod_rows({'namespace': namespace, 'pods': [pod]}) + detail = rows[0][2] if rows else str(phase or 'pending') + if pod.get('spec', {}).get('nodeName') not in {None, node}: + raise RuntimeError('GPU probe landed on another node') + if phase == 'Succeeded': + return + if phase == 'Failed': + raise RuntimeError(f'Fresh GPU runtime probe failed: {detail}') + if detail != last: + progress(f'{node}: GPU runtime probe {detail}') + last = detail + if clock() >= deadline: + raise RuntimeError(f'Fresh GPU runtime probe timed out: {detail}') + sleep(min(5, max(0, deadline - clock()))) + + +def acceptance(manager: KubeManager, *, node: str, namespace: str, release: str, + expected_gpus: int, base_url: str | None, run_id: str, timeout=900, + kubeconfig: str | None = None, + resource_profile: str | None = None, model=DEFAULT_MODEL, + expected_devices: list[dict] | None = None, http=requests, + progress=print, sleep=time.sleep, clock=time.monotonic) -> dict: + """Fresh GPU launch + node-specific KubeAI generation with bounded cleanup.""" + pod_name, model_name = _names(node, run_id) + scope = f' --kubeconfig={shlex.quote(str(kubeconfig))}' if kubeconfig else '' + progress(f'Acceptance run {run_id} on {node}; cleanup: infer-stack kube node test {node} --run-id={run_id} --cleanup --apply --namespace={namespace}{scope}') + prepared = plan(manager, node=node, namespace=namespace, expected_gpus=expected_gpus, resource_profile=resource_profile) + if prepared['facts']['errors']: + raise RuntimeError('; '.join(prepared['facts']['errors'])) + for kind, name in [('pod', pod_name), ('models.kubeai.org', model_name)]: + if manager.kubectl_json(['-n', namespace, 'get', kind, name, '--ignore-not-found', '-o', 'json']): + raise RuntimeError(f'Acceptance resource {kind}/{name} already exists; use explicit --cleanup --run-id={run_id} before retrying') + if prepared['needs_profile']: + found = manager.kubeai_release(release) + if not found or found.get('namespace') != namespace: + raise RuntimeError('Cannot add node profile to an externally managed/mismatched chart; use the configured Helm release') + manager.install_kubeai(namespace=namespace, release=release, + resource_profiles={prepared['profile_name']: prepared['profile']}, + version=manager._release_chart_version(found)) + # The controller config must reflect the Helm rollout before testing. + name, _, missing = worker_profile(manager, prepared['facts'], namespace=namespace, resource_profile=resource_profile) + if missing or name != prepared['profile_name']: + raise RuntimeError('KubeAI did not load the node resource profile after installation') + labels = {RUN_LABEL: run_id, NODE_LABEL: node} + pod = {'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {'name': pod_name, 'namespace': namespace, 'labels': labels}, + 'spec': {'nodeSelector': {'kubernetes.io/hostname': prepared['facts']['hostname']}, + 'runtimeClassName': 'nvidia', 'restartPolicy': 'Never', 'automountServiceAccountToken': False, + 'containers': [{'name': 'gpu', 'image': GPU_PROBE_IMAGE, + 'command': ['nvidia-smi', '--query-gpu=uuid,name,memory.total', '--format=csv,noheader,nounits'], + 'resources': {'requests': {'nvidia.com/gpu': str(expected_gpus)}, + 'limits': {'nvidia.com/gpu': str(expected_gpus)}}}]}} + deployment = Deployment(id=f'test-{run_id}', compat_key='acceptance', engine='vllm', + sharing='dedicated', capacity={}, state='live', created_at=0, updated_at=0, + spec={'engine': 'vllm', 'hf_model_id': model, + 'runtime': {'resource_profile': prepared['profile_name'], 'max_model_len': 2048, + 'gpu_memory_utilization': 0.5}, + 'topology': {'tensor_parallel_size': 1}}, + served={model_name: {'served_model_name': model_name, 'protocol': 'chat'}}) + rendered = render_models([deployment], namespace=namespace, default_resource_profile=None) + if rendered.errors: + raise RuntimeError('; '.join(rendered.errors)) + doc = rendered.docs[0] + # Deliberately outside the normal lease authority's managed selector. + doc['metadata']['labels'] = labels + created = False + result = {'context': prepared['context'], 'namespace': namespace, 'run_id': run_id, 'node': node, 'profile': prepared['profile_name'], 'model': model_name} + try: + # Atomic create refuses occupied names, even an interrupted test. + manager.run(['kubectl', 'create', '-f', '-'], input_text=json.dumps(pod)) + created = True + progress(f'{node}: launching fresh runtime/GPU probe ({expected_gpus} GPUs)') + wait_gpu_probe(manager, node=node, namespace=namespace, name=pod_name, timeout=timeout, + progress=progress, sleep=sleep, clock=clock) + actual = manager.kubectl_json(['-n', namespace, 'get', 'pod', pod_name, '-o', 'json']) + if actual.get('spec', {}).get('nodeName') != node: + raise RuntimeError('GPU probe landed on another node; refusing an aggregate-cluster success') + observed = devices(manager.run(['kubectl', '-n', namespace, 'logs', pod_name])) + if len(observed) != expected_gpus or (expected_devices is not None and observed != sorted(expected_devices, key=lambda row: row['uuid'])): + raise RuntimeError(f'Runtime GPU identities differ from expected hardware: {observed}') + product_names = {d['product'] for d in observed} + if len(product_names) == 1: + def normalize(value): + return re.sub('[^a-z0-9]', '', str(value).lower()) + if normalize(prepared['facts']['product']) != normalize(observed[0]['product']): + raise RuntimeError('GFD product label disagrees with freshly observed GPU product') + if any(abs(int(prepared['facts']['memory_mib']) - d['memory_mib']) > 512 for d in observed): + raise RuntimeError('GFD memory label disagrees with freshly observed GPU memory') + else: + result['limitations'] = ['Heterogeneous node: GFD labels are node-level; this test verifies every device, but generation uses one Kubernetes-allocated GPU, not every product.'] + result['devices'] = observed + result['gfd'] = {'product': prepared['facts']['product'], 'memory_mib': prepared['facts']['memory_mib']} + manager.run(['kubectl', '-n', namespace, 'delete', 'pod', pod_name, '--wait=true', '--timeout=180s']) + manager.run(['kubectl', 'create', '-f', '-'], input_text=json.dumps(doc)) + deadline, last = clock() + timeout, None + while True: + cr = manager.kubectl_json(['-n', namespace, 'get', 'models.kubeai.org', model_name, '-o', 'json']) + pods = manager.kubectl_json(['-n', namespace, 'get', 'pods', '-l', f'model={model_name}', '-o', 'json']).get('items') or [] + active = [p for p in pods if not p.get('metadata', {}).get('deletionTimestamp') + and p.get('status', {}).get('phase') not in {'Succeeded', 'Failed'}] + state = 'waiting for scheduler' + for p in active: + spec = p.get('spec') or {} + scheduled = spec.get('nodeName') + if scheduled and scheduled != node: + raise RuntimeError(f'Model scheduled on {scheduled}, expected {node}') + containers = spec.get('containers') or [] + requested = sum(int(c.get('resources', {}).get('requests', {}).get('nvidia.com/gpu', 0)) for c in containers) + limited = sum(int(c.get('resources', {}).get('limits', {}).get('nvidia.com/gpu', 0)) for c in containers) + if scheduled and (requested != 1 or limited != 1 or spec.get('runtimeClassName') != 'nvidia'): + raise RuntimeError('Model pod does not request/limit exactly one GPU with NVIDIA runtime') + state = f'scheduled on {scheduled}' if scheduled else state + for c in p.get('status', {}).get('containerStatuses') or []: + waiting = c.get('state', {}).get('waiting') or {} + if waiting: + state += ': ' + str(waiting.get('reason', 'starting')) + ' ' + str(waiting.get('message', '')) + if c.get('state', {}).get('running'): + state += ': container running; model loading' + ready_pods = [p for p in active if p.get('spec', {}).get('nodeName') == node + and any(c.get('type') == 'Ready' and c.get('status') == 'True' + for c in p.get('status', {}).get('conditions') or [])] + _, ready = model_replica_counts(cr) + if ready and ready_pods: + state = f'{node}: replica ready; verifying generation' + if state != last: + progress(state) + last = state + serving_devices = devices(manager.run(['kubectl', '-n', namespace, 'exec', + ready_pods[0]['metadata']['name'], '--', + 'nvidia-smi', '--query-gpu=uuid,name,memory.total', + '--format=csv,noheader,nounits'])) + if len(serving_devices) != 1 or serving_devices[0] not in observed: + raise RuntimeError("Serving replica must expose exactly one of this worker's verified GPUs") + result['serving_devices'] = serving_devices + with api_access(namespace=namespace, kubeconfig=kubeconfig, base_url=base_url) as base: + try: + response = http.post(base.rstrip('/') + '/chat/completions', + json={'model': model_name, 'messages': [{'role': 'user', 'content': 'Reply with ready.'}], 'max_tokens': 8}, timeout=45) + response.raise_for_status() + if not response.json().get('choices'): + raise RuntimeError('Generation response contains no choices') + except requests.RequestException as ex: + raise RuntimeError(f'Worker generation failed: {ex}') from ex + result['generation_verified'] = True + progress(f'{node}: real generation verified') + return result + if state != last: + progress(f'{model_name}: {state}') + last = state + if clock() >= deadline: + raise RuntimeError(f'Worker acceptance timed out: {state}') + sleep(min(5, max(0, deadline - clock()))) + finally: + if created: + cleanup(manager, node=node, namespace=namespace, run_id=run_id) + + +@contextmanager +def api_access(*, namespace: str, kubeconfig: str | None = None, base_url: str | None = None): + """Temporary loopback-only service forwarding; no gateway/config changes.""" + if base_url: + yield base_url + return + args = ['kubectl'] + if kubeconfig: + args += ['--kubeconfig', str(Path(kubeconfig).expanduser().resolve())] + args += ['-n', namespace, 'port-forward', '--address=127.0.0.1', 'service/kubeai', ':80'] + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + assert proc.stdout is not None + with selectors.DefaultSelector() as selector: + selector.register(proc.stdout, selectors.EVENT_READ) + deadline, lines = time.monotonic() + 30, [] + while time.monotonic() < deadline: + if not selector.select(timeout=1): + continue + line = proc.stdout.readline() + lines.append(line.strip()) + match = re.search(r'Forwarding from 127\.0\.0\.1:(\d+)', line) + if match: + yield f'http://127.0.0.1:{match[1]}/openai/v1' + return + if not line: + break + raise RuntimeError('Could not forward KubeAI API: ' + '; '.join(lines)) + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + if proc.stdout is not None: + proc.stdout.close() + + +def private_file(path: str | Path, purpose: str) -> Path: + """Onboarding credentials must be user-private; never print their contents.""" + source = Path(path).expanduser() + if not source.is_file(): + raise RuntimeError(f'{purpose} file is missing: {source}') + if source.stat().st_mode & 0o077: + raise RuntimeError(f'{purpose} file must be private (chmod 600): {source}') + return source + + +def onboarding_plan(local: KubeManager, cluster: KubeManager, *, server: str, + node: str, version: str | None = None) -> dict: + """Verify host prerequisites and the explicit administrative cluster target.""" + _names(node, 'onboardcheck') + if not server.startswith('https://'): + raise RuntimeError('K3s server must use an https:// URL') + for tool in ('nvidia-smi', 'kubectl', 'helm'): + if not local.command_exists(tool): + raise RuntimeError(f'{tool} is required before onboarding; prepare host driver and admin tooling first') + hardware = local_devices(local) + if not local.command_exists('nvidia-container-runtime'): + raise RuntimeError('Install NVIDIA container toolkit on this GPU worker first; nvidia-container-runtime is missing. No agent changes made.') + selected = cluster.kubectl_json(['config', 'view', '--minify', '-o', 'json']) + servers = [c.get('cluster', {}).get('server', '').rstrip('/') for c in selected.get('clusters') or []] + if servers != [server.rstrip('/')]: + raise RuntimeError(f'Explicit admin kubeconfig server {servers} differs from requested join server {server!r}') + if not cluster.cluster_reachable()[0]: + raise RuntimeError('Explicit admin kubeconfig cannot reach the requested cluster') + nodes = cluster.nodes() + versions = {n.get('status', {}).get('nodeInfo', {}).get('kubeletVersion') + for n in nodes if cluster._node_control_plane(n)} + if not version: + if len(versions) != 1 or not next(iter(versions), '') or 'k3s' not in str(next(iter(versions))): + raise RuntimeError('Cannot infer one K3s server version; pass --version explicitly') + version = next(iter(versions)) + if versions and version not in versions: + raise RuntimeError(f'Requested K3s version {version} differs from control-plane versions {sorted(str(v) for v in versions)}') + return {'node': node, 'server': server, 'version': version, 'devices': hardware, + 'expected_gpus': len(hardware), 'actions': [ + 'Verify existing agent membership or install/start a K3s agent using the private token file', + 'Verify local containerd NVIDIA handler; restart this agent only if discovery needs it', + 'Ensure cluster NVIDIA device plugin/GFD via existing installer if absent', + f'Wait specifically for {node}: Ready, {len(hardware)} GPUs, product/memory labels', + 'Ensure KubeAI via the shared installer if absent; run node-specific GPU/generation acceptance']} + + +def onboard(local: KubeManager, cluster: KubeManager, prepared: dict, *, token_file: str, + namespace: str, release: str, base_url: str | None, run_id: str, + kubeconfig: str | None = None, + timeout=900, model=DEFAULT_MODEL, resource_profile=None, + progress=print, sleep=time.sleep, clock=time.monotonic, http=requests) -> dict: + """Join, wait for the exact worker, then use the same targeted acceptance.""" + from . import k3s + from .inspect import inventory + from .operations import install + + node = prepared['node'] + k3s.join(server=prepared['server'], token_file=token_file, node_name=node, + version=prepared['version'], run=local.run) + config_path = '/var/lib/rancher/k3s/agent/etc/containerd/config.toml' + runtime_config = local.run(['sudo', '-n', 'cat', config_path]) + if 'nvidia-container-runtime' not in runtime_config: + progress(f'{node}: restarting local agent for NVIDIA runtime discovery') + local.run(['sudo', '-n', 'systemctl', 'restart', 'k3s-agent']) + discovery_deadline = clock() + min(timeout, 60) + while True: + runtime_config = local.run(['sudo', '-n', 'cat', config_path]) + if 'nvidia-container-runtime' in runtime_config: + break + if clock() >= discovery_deadline: + raise RuntimeError('Local K3s agent has no NVIDIA handler; ensure nvidia-container-runtime is in its service PATH') + sleep(2) + report = inventory(cluster, namespace=namespace, release=release, base_url=base_url or 'http://127.0.0.1:8000/openai/v1') + if report.get('errors', {}).get('device_plugin'): + raise RuntimeError('Cannot inspect NVIDIA components; repair admin access before reconciling the plugin') + if not report['device_plugin']: + if 'nvidia' not in (report['runtime_classes'] or []): + raise RuntimeError('Cluster requires RuntimeClass nvidia before GPU plugin setup; use infer-stack kube bootstrap on the server') + cluster.install_nvidia_device_plugin() + deadline, last = clock() + timeout, None + while True: + try: + facts = worker_facts(cluster, node, prepared['expected_gpus']) + detail = '; '.join(facts['errors']) + except RuntimeError as ex: + detail = str(ex) + if not detail: + break + if detail != last: + progress(f'{node}: {detail}') + last = detail + if clock() >= deadline: + raise RuntimeError(f'{node}: worker GPU readiness timed out: {detail}. Other nodes cannot satisfy this test.') + sleep(min(5, max(0, deadline - clock()))) + report = inventory(cluster, namespace=namespace, release=release, base_url=base_url or 'http://127.0.0.1:8000/openai/v1') + if not report['kubeai']['crd'] or not report['kubeai']['namespace_exists']: + install(cluster, report) + return acceptance(cluster, node=node, namespace=namespace, release=release, + expected_gpus=prepared['expected_gpus'], expected_devices=prepared['devices'], + base_url=base_url, kubeconfig=kubeconfig, run_id=run_id, timeout=timeout, model=model, + resource_profile=resource_profile, progress=progress, sleep=sleep, clock=clock, http=http) diff --git a/tests/test_kube_worker.py b/tests/test_kube_worker.py new file mode 100644 index 00000000..f0ece71d --- /dev/null +++ b/tests/test_kube_worker.py @@ -0,0 +1,467 @@ +"""Named GPU workers: deterministic plan, join, device/placement/generation tests.""" +import copy +import json +from types import SimpleNamespace + +import pytest +import yaml +from test_kube_readiness import node + +from infer_stack.cli.commands_kube_worker import K3sOnboardCLI, KubeNodeTestCLI +from infer_stack.kube import k3s +from infer_stack.kube.manage import KubeManager +from infer_stack.kube.worker import ( + DEFAULT_MODEL, + NODE_LABEL, + RUN_LABEL, + acceptance, + cleanup, + devices, + onboard, + onboarding_plan, + plan, + scoped_manager, +) + + +class Worker: + """Commands traverse real rendering and operations; only the cluster is fake.""" + def __init__(self, name='namek', count=1, products=None): + self.name, self.count = name, count + self.devices = [{'uuid': f'GPU-{i}', 'product': p, 'memory_mib': 24576 - (i * 8192 if products and len(set(products)) > 1 else 0)} + for i, p in enumerate(products or ['NVIDIA RTX 3090'] * count)] + self.nodes = [node('aiq-gpu', count=4), node(name, count=count)] + self.nodes[1]['metadata']['labels']['nvidia.com/gpu.product'] = self.devices[0]['product'].replace(' ', '-') if self.devices else 'NVIDIA-RTX-3090' + for n in self.nodes: + n['metadata']['labels']['kubernetes.io/hostname'] = n['metadata']['name'] + self.nodes[0]['metadata']['labels']['node-role.kubernetes.io/control-plane'] = 'true' + self.profiles, self.resources, self.calls, self.upgrades = {}, {}, [], [] + self.landing, self.model_gpu_count, self.model_ready = name, 1, 1 + self.handler = True + self.generation_calls = [] + self.manager = KubeManager(run=self.run) + self.manager.command_exists = lambda name: True + self.manager.install_kubeai = self.install + self.http = SimpleNamespace(post=self.post) + + def install(self, **kw): + self.upgrades.append(kw) + self.profiles.update(copy.deepcopy(kw['resource_profiles'])) + + def post(self, url, **kw): + self.generation_calls.append((url, kw)) + return SimpleNamespace(raise_for_status=lambda: None, + json=lambda: {'choices': [{'message': {'content': 'ready'}}]}) + + def run(self, args, **kw): + self.calls.append(args) + if args[0] == 'nvidia-smi': + return '\n'.join(f"{d['uuid']}, {d['product']}, {d['memory_mib']}" for d in self.devices) + if args[0] == 'sudo' and 'cat' in args: + return 'nvidia-container-runtime' if self.handler else 'no handler' + if args[0] == 'sudo' and 'restart' in args: + self.handler = True + return '' + if args == ['kubectl', 'config', 'current-context']: + return 'aiq-gpu' + if args[:3] == ['kubectl', 'config', 'view']: + return json.dumps({'clusters': [{'cluster': {'server': 'https://aiq-gpu:6443'}}]}) + if args[:2] == ['kubectl', 'version']: + return '{}' + if args == ['kubectl', 'get', 'nodes', '-o', 'json']: + return json.dumps({'items': self.nodes}) + if args[:3] == ['kubectl', 'get', 'node']: + return json.dumps(next(n for n in self.nodes if n['metadata']['name'] == args[3])) + if args[0] == 'kubectl' and 'configmap' in args: + return json.dumps({'data': {'system.yaml': yaml.safe_dump({'resourceProfiles': self.profiles})}}) + if args[:2] == ['helm', 'list']: + return json.dumps([{'name': 'kubeai', 'namespace': 'default', 'chart': 'kubeai-0.23.2'}]) + if args[:2] == ['kubectl', 'create']: + doc = json.loads(kw['input_text']) + key = (doc['kind'], doc['metadata']['name']) + if key in self.resources: + raise RuntimeError('AlreadyExists') + self.resources[key] = doc + return '' + if args[:3] == ['kubectl', '-n', 'default']: + verb = args[3] + if verb == 'wait': + return '' + if verb == 'exec': + return f"{self.devices[0]['uuid']}, {self.devices[0]['product']}, {self.devices[0]['memory_mib']}" + if verb == 'logs': + return self.run(['nvidia-smi']) + if verb in {'get', 'delete'}: + kind, name = args[4:6] + kind = {'models.kubeai.org': 'Model', 'pod': 'Pod'}.get(kind, kind) + if kind == 'pods': + model_name = args[args.index('-l') + 1].split('=', 1)[1] + cr = self.resources[('Model', model_name)] + return json.dumps({'items': [{'metadata': {'name': 'model-pod', 'namespace': 'default', + 'labels': dict(cr['metadata']['labels'], model=cr['metadata']['name'])}, + 'spec': {'nodeName': self.landing, 'runtimeClassName': 'nvidia', + 'containers': [{'resources': {'requests': {'nvidia.com/gpu': str(self.model_gpu_count)}, + 'limits': {'nvidia.com/gpu': str(self.model_gpu_count)}}}]}, + 'status': {'phase': 'Running', 'conditions': [{'type': 'Ready', 'status': 'True'}]}}]}) + if verb == 'delete': + self.resources.pop((kind, name), None) + return '' + doc = copy.deepcopy(self.resources.get((kind, name), {})) + if doc and kind == 'Pod': + doc['spec']['nodeName'] = self.name + doc['status'] = {'phase': 'Succeeded'} + if doc and kind == 'Model': + doc['status'] = {'replicas': {'all': 1, 'ready': self.model_ready}} + return json.dumps(doc) + raise AssertionError(args) + + +def run_test(worker, **kwargs): + return acceptance(worker.manager, node=worker.name, namespace='default', release='kubeai', + expected_gpus=worker.count, base_url='http://kubeai/openai/v1', + run_id=kwargs.pop('run_id', 'abcdef123456'), http=worker.http, + expected_devices=worker.devices, **kwargs) + + +@pytest.mark.parametrize('name,products', [ + ('namek', ['NVIDIA RTX 3090']), + ('yardrat', ['NVIDIA RTX A6000', 'NVIDIA RTX 4080']), + ('aiq-gpu2', ['NVIDIA RTX 4070'] * 4), +]) +def test_named_worker_real_rendering_device_inventory_and_cleanup(name, products): + worker = Worker(name, len(products), products) + out = run_test(worker) + assert out['node'] == name and out['generation_verified'] + assert out['devices'] == worker.devices + assert out['serving_devices'] == worker.devices[:1] + assert out['profile'] in worker.profiles # deliberate reusable profile retained + assert worker.resources == {} + assert worker.upgrades[0]['version'] == '0.23.2' + profile = worker.profiles[out['profile']] + assert profile['nodeSelector'] == {'kubernetes.io/hostname': name} + assert profile['limits']['nvidia.com/gpu'] == '1' + assert worker.generation_calls[0][1]['json']['model'] == out['model'] + assert not any('apply' in args or 'prune' in args for args in worker.calls) + run_test(worker, run_id='123456abcdef') + assert len(worker.upgrades) == 1 # repeats preserve profile/version + + +def test_plan_is_read_only_and_aggregate_does_not_mask_broken_worker(): + w = Worker(count=0) + result = plan(w.manager, node='namek', namespace='default', expected_gpus=1) + assert 'exposes 0 GPUs' in '; '.join(result['facts']['errors']) + assert w.nodes[0]['status']['allocatable']['nvidia.com/gpu'] == '4' + assert not w.resources and not w.upgrades + with pytest.raises(RuntimeError, match='expected 1'): + acceptance(w.manager, node='namek', namespace='default', release='kubeai', expected_gpus=1, + base_url='unused', run_id='abcdefgh') + assert not any('create' in c for c in w.calls) + + +@pytest.mark.parametrize('change,match', [('landing', 'expected namek'), ('model_gpu_count', 'exactly one GPU')]) +def test_wrong_placement_or_gpu_request_fails_and_cleans(change, match): + w = Worker() + setattr(w, change, 'aiq-gpu' if change == 'landing' else 2) + with pytest.raises(RuntimeError, match=match): + run_test(w) + assert not w.resources and not w.generation_calls + + +def test_generation_failure_and_device_mismatch_cleanup(): + w = Worker() + w.http.post = lambda *a, **kw: SimpleNamespace(raise_for_status=lambda: None, json=lambda: {}) + with pytest.raises(RuntimeError, match='no choices'): + run_test(w) + assert not w.resources + w.devices.append(dict(w.devices[0], uuid='GPU-extra')) + with pytest.raises(RuntimeError, match='identities differ'): + run_test(w) + assert not w.resources + + +def test_pending_model_timeout_cleanup_reports_wait_state(): + w = Worker() + w.model_ready = 0 + with pytest.raises(RuntimeError, match='timed out'): + run_test(w, timeout=0) + assert not w.resources + + +def test_explicit_interrupted_run_cleanup_refuses_unrelated_and_retries_safely(): + w = Worker() + name = 'infer-stack-e2e-abcdefgh' + w.resources[('Model', name)] = {'metadata': {'labels': {}}} + with pytest.raises(RuntimeError, match='unrelated'): + cleanup(w.manager, node='namek', namespace='default', run_id='abcdefgh') + assert ('Model', name) in w.resources + w.resources[('Model', name)]['metadata']['labels'] = {RUN_LABEL: 'abcdefgh', NODE_LABEL: 'namek'} + cleanup(w.manager, node='namek', namespace='default', run_id='abcdefgh') + cleanup(w.manager, node='namek', namespace='default', run_id='abcdefgh') + assert not w.resources + run_test(w, run_id='abcdefgh') + + +def test_profile_constraints_preserve_base_and_refuse_operator_collision(): + w = Worker() + w.profiles['custom'] = {'imageName': 'custom-vllm', 'runtimeClassName': 'nvidia', + 'requests': {'nvidia.com/gpu': '1'}, 'limits': {'nvidia.com/gpu': '1'}, + 'nodeSelector': {'nvidia.com/gpu.product': 'NVIDIA-RTX-3090'}} + before = copy.deepcopy(w.profiles) + p = plan(w.manager, node='namek', namespace='default', expected_gpus=1, resource_profile='custom') + assert w.profiles == before + assert p['profile']['imageName'] == 'custom-vllm' + assert p['profile']['nodeSelector']['kubernetes.io/hostname'] == 'namek' + w.profiles[p['profile_name']] = {} + with pytest.raises(RuntimeError, match='overwrite'): + plan(w.manager, node='namek', namespace='default', expected_gpus=1, resource_profile='custom') + + +def test_scoping_never_replaces_stale_context(tmp_path, monkeypatch): + monkeypatch.setenv('KUBECONFIG', '/stale/eks') + w = Worker() + m = scoped_manager(w.manager, tmp_path / 'admin') + for args in [['kubectl', 'version'], ['helm', 'list', '-A', '-o', 'json'], ['nvidia-smi']]: + # Wrapper construction only: real fake expects commands without prefix. + w.manager.run = lambda args, **kw: w.calls.append(args) or '{}' + m.run(args) + assert w.calls[0][1:3] == ['--kubeconfig', str(tmp_path / 'admin')] + assert w.calls[1][1:3] == ['--kubeconfig', str(tmp_path / 'admin')] + assert w.calls[2] == ['nvidia-smi'] + + +def test_onboarding_plan_infers_version_and_preserves_heterogeneous_hardware(): + w = Worker('yardrat', 2, ['RTX A6000', 'RTX 4080']) + prepared = onboarding_plan(w.manager, w.manager, server='https://aiq-gpu:6443', node='yardrat') + assert prepared['version'] == 'v1.34.3+k3s1' + assert prepared['expected_gpus'] == 2 + assert prepared['devices'] == w.devices + assert not w.upgrades and not w.resources + with pytest.raises(RuntimeError, match='differs'): + onboarding_plan(w.manager, w.manager, server='https://wrong:6443', node='yardrat') + w.manager.command_exists = lambda name: name != 'nvidia-container-runtime' + with pytest.raises(RuntimeError, match='container toolkit'): + onboarding_plan(w.manager, w.manager, server='https://aiq-gpu:6443', node='yardrat') + + +def test_onboard_integration_join_runtime_discovery_then_exact_worker_test(monkeypatch): + w = Worker() + prepared = onboarding_plan(w.manager, w.manager, server='https://aiq-gpu:6443', node='namek') + joins = [] + monkeypatch.setattr(k3s, 'join', lambda **kw: joins.append(kw)) + from infer_stack.kube import inspect + monkeypatch.setattr(inspect, 'inventory', lambda *a, **kw: { + 'device_plugin': [{'name': 'nvdp'}], 'kubeai': {'crd': True, 'namespace_exists': True}}) + w.handler = False + out = onboard(w.manager, w.manager, prepared, token_file='private-file', namespace='default', + release='kubeai', base_url='http://kubeai/openai/v1', run_id='onboard12345', http=w.http) + assert joins[0]['node_name'] == 'namek' and joins[0]['version'] == prepared['version'] + assert ['sudo', '-n', 'systemctl', 'restart', 'k3s-agent'] in w.calls + assert out['generation_verified'] and not w.resources + + +def test_cli_registration_plan_apply_and_cleanup(tmp_path, monkeypatch, capsys): + from infer_stack.cli import ManageCLI + from infer_stack.cli import commands_kube_worker as cli + w = Worker() + monkeypatch.setattr(cli, 'KubeManager', lambda: w.manager) + assert KubeNodeTestCLI.main(argv=['namek', '--expected-gpus=1', '--namespace=default', '--json']) == 0 + assert json.loads(capsys.readouterr().out)['profile']['nodeSelector']['kubernetes.io/hostname'] == 'namek' + assert not w.upgrades and not w.resources + import requests + monkeypatch.setattr(requests, 'post', w.post) + assert KubeNodeTestCLI.main(argv=False, node='namek', expected_gpus=1, namespace='default', apply=True, base_url='http://kubeai/openai/v1') == 0 + assert ManageCLI.main(argv=['kube', 'node', 'test', 'namek', '--expected-gpus=1', '--namespace=default', '--apply', '--base-url=http://kubeai/openai/v1']) in (None, 0) + assert not w.resources + with pytest.raises(SystemExit, match='run-id'): + KubeNodeTestCLI.main(argv=False, node='namek', cleanup=True) + ManageCLI.main(argv=['help', 'tree']) + tree = capsys.readouterr().out + assert 'onboard' in tree and 'test' in tree + for cls in (K3sOnboardCLI, KubeNodeTestCLI): + with pytest.raises(SystemExit) as exc: + cls.main(argv=['--help']) + assert exc.value.code == 0 + assert DEFAULT_MODEL == 'HuggingFaceTB/SmolLM2-135M-Instruct' + + +def test_device_parser_rejects_missing_or_duplicate_identity(): + with pytest.raises(RuntimeError): + devices('') + with pytest.raises(RuntimeError): + devices('GPU-a,3090,24576\nGPU-a,4080,16384') + + +def test_onboard_does_not_accept_existing_other_node_gpus(monkeypatch): + from infer_stack.kube import inspect + w = Worker() + prepared = onboarding_plan(w.manager, w.manager, server='https://aiq-gpu:6443', node='namek') + w.nodes[1]['status']['allocatable']['nvidia.com/gpu'] = '0' + monkeypatch.setattr(k3s, 'join', lambda **kw: None) + monkeypatch.setattr(inspect, 'inventory', lambda *a, **kw: {'device_plugin': [{'name': 'nvdp'}]}) + with pytest.raises(RuntimeError, match='Other nodes cannot satisfy'): + onboard(w.manager, w.manager, prepared, token_file='private', namespace='default', + release='kubeai', base_url='http://kubeai/openai/v1', run_id='badworker123', timeout=0, http=w.http) + assert not w.resources and not w.generation_calls + + +def test_onboard_uses_shared_plugin_and_kubeai_installers(monkeypatch): + from infer_stack.kube import inspect, operations + w = Worker() + prepared = onboarding_plan(w.manager, w.manager, server='https://aiq-gpu:6443', node='namek') + plugin_calls, chart_calls = [], [] + monkeypatch.setattr(k3s, 'join', lambda **kw: None) + monkeypatch.setattr(inspect, 'inventory', lambda *a, **kw: { + 'device_plugin': [], 'runtime_classes': ['nvidia'], + 'kubeai': {'crd': False, 'namespace_exists': False}}) + monkeypatch.setattr(w.manager, 'install_nvidia_device_plugin', lambda: plugin_calls.append(True)) + monkeypatch.setattr(operations, 'install', lambda *a, **kw: chart_calls.append(a)) + result = onboard(w.manager, w.manager, prepared, token_file='private', namespace='default', + release='kubeai', base_url='http://kubeai/openai/v1', run_id='setupworker1', http=w.http) + assert plugin_calls == [True] and len(chart_calls) == 1 + assert result['generation_verified'] and not w.resources + + +def test_onboard_kwconf_plan_and_apply_from_root(tmp_path, monkeypatch, capsys): + import requests + + from infer_stack.cli import ManageCLI + from infer_stack.cli import commands_kube_worker as cli + from infer_stack.kube import inspect + w = Worker() + token, admin = tmp_path / 'token', tmp_path / 'admin' + for file in (token, admin): + file.write_text('private credentials') + file.chmod(0o600) + monkeypatch.setattr(cli, 'KubeManager', lambda: w.manager) + monkeypatch.setattr(cli, 'scoped_manager', lambda manager, path: manager) + monkeypatch.setattr(requests, 'post', w.post) + joins = [] + monkeypatch.setattr(k3s, 'join', lambda **kw: joins.append(kw)) + monkeypatch.setattr(inspect, 'inventory', lambda *a, **kw: { + 'device_plugin': [{'name': 'nvdp'}], 'kubeai': {'crd': True, 'namespace_exists': True}}) + args = ['kube', 'k3s', 'onboard', 'namek', '--server=https://aiq-gpu:6443', + f'--kubeconfig={admin}', f'--token-file={token}', '--namespace=default', + '--base-url=http://kubeai/openai/v1', '--json'] + ManageCLI.main(argv=args) + assert json.loads(capsys.readouterr().out)['expected_gpus'] == 1 + assert not joins and not w.upgrades and not w.resources + ManageCLI.main(argv=args + ['--apply']) + out = capsys.readouterr() + assert json.loads(out.out)['generation_verified'] + assert 'real generation verified' in out.err + assert len(joins) == 1 and not w.resources + token.chmod(0o644) + with pytest.raises(SystemExit, match='private'): + K3sOnboardCLI.main(argv=False, node='namek', server='https://aiq-gpu:6443', + kubeconfig=str(admin), token_file=str(token)) + + +def test_real_helm_operation_preserves_values_and_chart_version(tmp_path, monkeypatch): + w = Worker() + from infer_stack.kube import manage + monkeypatch.setattr(manage, 'data_root', lambda: tmp_path) + del w.manager.install_kubeai # use the actual shared operation, not our fixture shortcut + original = w.manager.run + values = {'resourceProfiles': {}, 'modelServers': {'vllm': {'images': {'nvidia-gpu': 'my-image'}}}} + commands = [] + def run(args, **kw): + commands.append(args) + if args[:3] == ['helm', 'get', 'values']: + return yaml.safe_dump(values) + if args[:2] == ['helm', 'repo']: + return '' + if args[:3] == ['helm', 'upgrade', '--install']: + from pathlib import Path + document = yaml.safe_load(Path(args[args.index('-f') + 1]).read_text()) + assert document['modelServers'] == values['modelServers'] + w.profiles.update(document['resourceProfiles']) + return '' + return original(args, **kw) + w.manager.run = run + result = run_test(w) + cmd = next(c for c in commands if c[:3] == ['helm', 'upgrade', '--install']) + assert cmd[3:5] == ['kubeai', 'kubeai/kubeai'] + assert cmd[cmd.index('--version') + 1] == '0.23.2' + assert cmd[cmd.index('-n') + 1] == 'default' + assert result['generation_verified'] + + +def test_targeted_acceptance_preserves_unrelated_managed_models(): + w = Worker() + old = {'apiVersion': 'kubeai.org/v1', 'kind': 'Model', + 'metadata': {'name': 'important-production-model', 'labels': {'infer-stack/managed': 'true'}}, + 'spec': {'resourceProfile': 'custom:1'}} + w.resources[('Model', old['metadata']['name'])] = copy.deepcopy(old) + run_test(w) + assert w.resources == {('Model', old['metadata']['name']): old} + + +def test_gpu_probe_failure_is_fresh_and_cleans_own_resource(): + w = Worker() + original = w.manager.run + def run(args, **kw): + data = original(args, **kw) + if args[:5] == ['kubectl', '-n', 'default', 'get', 'pod']: + doc = json.loads(data) + if doc: + doc['status'] = {'phase': 'Failed'} + return json.dumps(doc) + return data + w.manager.run = run + with pytest.raises(RuntimeError, match='Fresh GPU runtime probe failed'): + run_test(w) + assert not w.resources and not w.generation_calls + + +def test_join_bundle_private_refresh_and_interrupted_retry(tmp_path): + original = {'current-context': 'default', 'contexts': [{'name': 'default', 'context': {'cluster': 'default'}}], + 'clusters': [{'name': 'default', 'cluster': {'server': 'https://127.0.0.1:6443', + 'certificate-authority-data': 'public-CA'}}], + 'users': [{'name': 'default', 'user': {'client-key-data': 'ADMIN-PRIVATE-KEY'}}]} + calls = [] + def run(args): + calls.append(args) + if 'is-active' in args: + return '' + return yaml.safe_dump(original) if args[-1] == str(k3s.K3S_KUBECONFIG) else 'SECRET-TOKEN' + path = tmp_path / 'bundle' + result = k3s.export_join_info(server='https://aiq-gpu:6443', directory=path, run=run) + assert path.stat().st_mode & 0o777 == 0o700 + assert all(p.stat().st_mode & 0o777 == 0o600 for p in path.iterdir()) + assert (path / 'token').read_text().strip() == 'SECRET-TOKEN' + assert yaml.safe_load((path / 'kubeconfig.yaml').read_text())['clusters'][0]['cluster']['server'] == result['server'] + assert original['clusters'][0]['cluster']['server'] == 'https://127.0.0.1:6443' + assert 'SECRET-TOKEN' not in json.dumps(result) and 'ADMIN-PRIVATE-KEY' not in json.dumps(result) + (path / 'token').unlink() # interrupted write resumes from owned manifest + k3s.export_join_info(server='https://aiq-gpu:6443', directory=path, run=run) + assert (path / 'token').exists() + original['clusters'][0]['cluster']['certificate-authority-data'] = 'different-CA' + with pytest.raises(RuntimeError, match='different cluster'): + k3s.export_join_info(server='https://aiq-gpu:6443', directory=path, run=run) + + +def test_join_bundle_refuses_unrelated_directory_and_agent(tmp_path): + path = tmp_path / 'bundle' + path.mkdir(mode=0o700) + (path / 'important').write_text('preserve me') + def run(args): + if 'is-active' in args: + return '' + if args[-1] == str(k3s.K3S_NODE_TOKEN): + return 'token' + return yaml.safe_dump({'current-context': 'default', 'contexts': [{'name': 'default', 'context': {'cluster': 'default'}}], + 'clusters': [{'name': 'default', 'cluster': {'certificate-authority-data': 'CA'}}]}) + with pytest.raises(RuntimeError, match='nonempty'): + k3s.export_join_info(server='https://aiq-gpu:6443', directory=path, run=run) + assert (path / 'important').read_text() == 'preserve me' + def inactive(args): + raise RuntimeError('inactive') + with pytest.raises(RuntimeError, match='local K3s server'): + k3s.export_join_info(server='https://aiq-gpu:6443', directory=path, run=inactive) + + +def test_join_bundle_kwconf_plan_reads_no_credentials(tmp_path, monkeypatch, capsys): + from infer_stack.cli import ManageCLI + monkeypatch.setattr(k3s, 'export_join_info', lambda **kw: pytest.fail('plan read credentials')) + ManageCLI.main(argv=['kube', 'k3s', 'export', '--server=https://aiq-gpu:6443', f'--directory={tmp_path}', '--plan', '--apply']) + assert 'No credentials read or written' in capsys.readouterr().out From 97244dc5025abcc233d9afc400741a517143753c Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Thu, 1 Oct 2026 23:53:40 -0400 Subject: [PATCH 140/141] Document private worker onboarding and scoped GPU acceptance Show aiq-gpu to namek export/join/test commands, repeat tests for mixed yardrat and four-GPU workers, ownership-limited interruption cleanup, host prerequisites and precise generation coverage. Co-authored-by: GPT-6.1-Sol --- README.md | 18 +++++++ dev/journals/codex.md | 8 ++++ dev/kubeai_e2e.sh | 4 ++ docs/cluster-setup.md | 106 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) diff --git a/README.md b/README.md index 88a03cd8..f434435c 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ infer-stack catalog suggest --apply # seed catalog.yaml from this host's GPUs infer-stack catalog show # what can be acquired infer-stack kube inventory # Kubernetes/GPU/KubeAI facts, before or after install infer-stack kube doctor # detailed readiness + actionable fixes +infer-stack kube node test NODE --expected-gpus=1 # plan worker GPU + real generation acceptance infer-stack kube node detach NODE # preview handing a cluster GPU host to Compose infer-stack acquire # lease, render, bring up, wait for a real generation infer-stack access --env-file e.env # reach it, managed or external; leases only what runs here @@ -703,6 +704,23 @@ signals: `litellm exited with code 137` is a SIGKILL (an OOM kill or a forced replacement), whereas LiteLLM returning HTTP 500 with `Cannot connect to host vllm-*` means LiteLLM is running and its upstream vLLM is not ready yet. +### Join and accept GPU workers + +`infer-stack kube k3s onboard namek --server=... --token-file=... --kubeconfig=...` +plans a local GPU worker join and node-specific acceptance; add `--apply` to +execute it. Host NVIDIA drivers/toolkit are prerequisites. It infers the server +version, verifies every locally detected device, then checks actual one-GPU +KubeAI placement and generation on this worker. Admin credentials are explicit +private files, separate from any selected EKS context. + +For an already joined worker, run `infer-stack kube node test namek +--expected-gpus=1 --namespace=default --apply` from the control plane. The same +surface accepts two-device `yardrat` and four-device `aiq-gpu2`; mixed products +are reported separately. Tests preserve leases and unrelated Models, remove +their temporary resources, and retain a reusable node profile. See the +[worker onboarding instructions](docs/cluster-setup.md#join-and-test-a-gpu-worker-in-one-reviewed-operation) +for plan/apply, credentials, interrupted cleanup and heterogeneous-node limits. + ### Kubernetes dashboard `infer-stack tui` provides a **Cluster** tab in the expanded runtime pane on the diff --git a/dev/journals/codex.md b/dev/journals/codex.md index 3f3094ec..47d67238 100644 --- a/dev/journals/codex.md +++ b/dev/journals/codex.md @@ -468,3 +468,11 @@ reported Ready but zero allocatable GPUs/missing GFD, rather than using another node as evidence. The full worker hardware/generation acceptance remains for namek/aiq-gpu, with heterogeneous generation limited to one allocated GPU per run. No maintainer design decision is needed to try that documented workflow. + +The documentation is committed separately from implementation: README links the +worker recipe, cluster setup gives the exact private export -> secure transfer +-> onboard plan/apply sequence plus repeat node tests and interrupted cleanup, +and the broader dev e2e comments distinguish worker acceptance from its +namespace-wide lifecycle assumptions. This preserves one Python installer +implementation while making the operator's path concrete for all three planned +worker shapes. diff --git a/dev/kubeai_e2e.sh b/dev/kubeai_e2e.sh index 6ede83d1..e8e66335 100755 --- a/dev/kubeai_e2e.sh +++ b/dev/kubeai_e2e.sh @@ -16,6 +16,10 @@ # dev/e2e_tests/kubeai-cpu-values.yaml, whose `cpu` profile runs real vLLM on # CPU (needs AVX-512), and run with E2E_RESOURCE_PROFILE=cpu. # +# For an individual joining GPU worker, prefer the scoped acceptance harness: +# infer-stack kube k3s onboard NODE --server=... --token-file=... --kubeconfig=... --namespace=default +# infer-stack kube node test NODE --expected-gpus=1 --namespace=default --apply +# This broader lifecycle/gateway suite expects a quiescent test namespace. # Then: ./dev/kubeai_e2e.sh # # Knobs (env): diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 41c45d9b..976e0f0b 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -180,6 +180,112 @@ to reach the K3s API on the server, and inter-node traffic required by the configured K3s networking backend must be allowed on the trusted cluster network. +### Join and test a GPU worker in one reviewed operation + +On **aiq-gpu**, prepare the bundle without printing credentials: + +```bash +infer-stack kube k3s export \ + --server=https://:6443 \ + --directory=~/.private/namek-join +sudo -v +infer-stack kube k3s export \ + --server=https://:6443 \ + --directory=~/.private/namek-join --apply +``` + +The directory is `0700`; `token`, `kubeconfig.yaml` and `manifest.json` are +`0600`. Export rewrites the server URL in this copy, leaves original kubeconfigs +unchanged, and can refresh an owned bundle after interruption/certificate +rotation. It refuses unrelated directories or a bundle for another cluster. +Securely transfer the directory to **namek** using your normal transport, +retaining those permissions. + +On `namek`, with the NVIDIA driver and container toolkit already installed, +use the private bundle files. The kubeconfig's selected server must be +**the same URL** passed to `--server`, including the reachable LAN/VPN address +instead of `127.0.0.1`. Do not overwrite an existing EKS/default kubeconfig. +The onboarding command requires `kubectl` and Helm on the invoking machine. +These credentials grant administrative access; keep the separate files private. + +Preview on `namek`: + +```bash +infer-stack kube k3s onboard namek \ + --server=https://:6443 \ + --token-file=~/.private/namek-join/token \ + --kubeconfig=~/.private/namek-join/kubeconfig.yaml \ + --namespace=default +``` + +Then run the same command with `--apply` after authenticating with `sudo -v`. +It detects local GPU count/products/memory/UUIDs and infers the control plane's +K3s version, verifies existing membership or installs the agent, checks local +containerd NVIDIA discovery, waits for **namek itself** to become Ready with the +expected GPU resources and GFD labels, and runs real GPU/model acceptance. +It uses the existing NVIDIA/KubeAI installers if those components are absent. +Host drivers/toolkit remain prerequisites: a missing toolkit fails before any +agent changes. An existing agent on a different server is refused. Repeating +onboarding rechecks membership and launches a fresh acceptance test. + +Acceptance briefly reserves **all GPUs on the named node** for a fresh runtime +query, then uses **one GPU** for a small KubeAI Model and real generation. +Allow those GPUs to become available first; the test does not evict workloads. +It asserts actual pod placement on the requested node, exactly one GPU +request/limit, NVIDIA runtime, nested KubeAI replica readiness, and an actual +OpenAI generation response. The serving pod's device UUID/product are reported. +Other cluster nodes cannot satisfy the test. The default model is the public +`HuggingFaceTB/SmolLM2-135M-Instruct`; first image/model downloads may take +minutes (`--timeout=900` by default). State changes are printed while waiting. +No manually started port-forward or Docker gateway is needed: the test opens +and closes a temporary, loopback-only KubeAI service forward. + +For an already joined worker, repeat acceptance from **aiq-gpu** or another +administrative host, without distributing admin credentials to the worker: + +```bash +infer-stack kube node test namek --expected-gpus=1 --namespace=default +infer-stack kube node test namek --expected-gpus=1 --namespace=default --apply + +# The same acceptance surface handles later workers: +infer-stack kube node test yardrat --expected-gpus=2 --namespace=default --apply +infer-stack kube node test aiq-gpu2 --expected-gpus=4 --namespace=default --apply +``` + +`--kubeconfig=...` scopes all administrative commands to an explicit file; +otherwise `node test` uses the selected context shown in its plan. +`--resource-profile=` retains custom image and +resource settings and adds the node's hostname constraint. With no override, +the normal NVIDIA model-server image profile is used. The test adds a stable +node-specific KubeAI resource profile through the existing Helm upgrade helper, +preserving installed chart version and values. That reusable profile remains +installed and is printed for subsequent catalog use; it is not a host GPU index. +The initial profile addition can roll KubeAI's controller, as the plan states. + +For heterogeneous `yardrat`, the device query reports both GPU products and +memories independently. GFD selectors describe nodes, and Kubernetes' generic +`nvidia.com/gpu` allocation does not select a particular physical product within +a mixed node. The generation check verifies one allocated GPU and reports which +one; it does **not** claim generation on every GPU/product or multi-GPU tensor +parallelism. The other devices receive the fresh runtime/device query. + +Temporary Models and probe Pods have their own acceptance-run labels and are +outside infer-stack's normal managed lease set. Catalog, recovery ledger, +gateway and unrelated Models are preserved. Each run prints its cleanup command; +normal success/failure deletes only that run's resources. After process/host +interruption, use the printed ID (and the same namespace/kubeconfig): + +```bash +infer-stack kube node test namek --namespace=default \ + --run-id= --cleanup --apply +``` + +Cleanup verifies ownership before deletion and is safe to repeat. An occupied +run ID is refused until explicit cleanup. Plans and results also support +`--json`; progress goes to stderr. The broader `dev/kubeai_e2e.sh` remains the +control-plane lifecycle/gateway suite and should run in a quiescent test namespace; +use the targeted test above to accept individual joining workers. + ### 5. Reconcile newly visible hardware Back on the machine operating infer-stack: From 2c44ed3b8ff65e24ae95da5df8a02346e0e71de5 Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 2 Oct 2026 11:40:35 -0400 Subject: [PATCH 141/141] Fix GPU holder scan races Co-authored-by: GPT-5.6 Sol --- dev/journals/gpt-5.6-sol.md | 12 +++++ dev/lessons/lessons.md | 13 ++++++ infer_stack/gpu_doctor.py | 92 +++++++++++++++++++++++++++++++------ tests/test_gpu_doctor.py | 77 ++++++++++++++++++++++++++++++- 4 files changed, 179 insertions(+), 15 deletions(-) diff --git a/dev/journals/gpt-5.6-sol.md b/dev/journals/gpt-5.6-sol.md index 45fcc21e..cdec8463 100644 --- a/dev/journals/gpt-5.6-sol.md +++ b/dev/journals/gpt-5.6-sol.md @@ -207,3 +207,15 @@ A second issue was backend recovery state. The existing leasing design intention The resulting two-machine loop is: quiesce local Compose on A/B, create/join Kubernetes once, run KubeAI from its separate authority/data root, release KubeAI workloads, detach A/B for Compose, use their old Compose installations, quiesce Compose, attach A/B, and resume the same KubeAI authority. There is no second `kube k3s join` because detach preserves membership and agent state. A K3s server that is also the sole control-plane node can be detached from workload scheduling without stopping the server; cluster workloads may become Pending when all workers are cordoned, but the control plane remains available to attach them again. Validation in this extracted/offline checkout: `tests/test_cli_kube.py` passes 18/18 with the same minimal ubelt stub used for the Kubernetes work; the tests cover reversible detach/attach, retained DaemonSet/static pods, refusal to force-delete unmanaged pods, refusal to undo foreign cordons, interrupted-detach retry, control-plane handling, and node inventory state. Four existing KubeAI doctor/sizing tests also pass. A minimal external kwconf/ubelt shape stub imports the full CLI tree and verifies `ManageCLI.kube -> KubeModalCLI.node -> status/detach/attach`. `python -m compileall -q infer_stack tests` passes. The container still lacks the real ubelt/kwconf and ruff packages, so the real parser/help path and repository lint suite should run in the normal development environment before merge. + +## 2026-10-02 11:19:47 -0400 + +Summary of user intent: fix the Compose GPU doctor false-negative observed on aiq-gpu after handing the node back from Kubernetes. The live reproducer showed cached non-interactive sudo working, while GNU find over /proc returned status 1 because one fd disappeared during traversal even though it produced 52 valid NVIDIA-device holder records. Those holders were nvidia-persistenced, the Kubernetes NVIDIA device plugin, and nvtop; none represented a CUDA compute workload. + +Model: GPT-5.6 Sol. Configuration: tool-enabled reasoning session. + +I kept the fix inside the GPU diagnostic rather than weakening backend admission. The privileged holder scan now verifies sudo independently, runs the /proc traversal with a stable C locale, and accepts a nonzero find exit only when every reported error is the expected live-/proc race (a /proc path vanished with ENOENT). Valid stdout is retained. Real permission/execution errors still fail closed. A second race was handled at the pid-detail stage: if the process disappears before cmdline/cgroup can be read, that stale holder is dropped rather than formatted as a current conflict. + +The holder classification now distinguishes passive/system observers from unexpected holders. nvidia-persistenced, nvidia-device-plugin, gpu-feature-discovery, and nvtop are informational: they can hold NVIDIA character devices (and may block a low-level reset) but do not by themselves reserve GPU memory or make a Compose/vLLM placement unsafe. Unknown holders still fail the check and preserve the container/pod attribution. The reset-specific note is emitted only when nvidia-persistenced is actually present. + +Validation in this extracted checkout: python compileall passes. The normal pytest entrypoint cannot collect because ubelt is absent from the container, so I ran a dependency-light direct harness that reproduces the exact sudo=0/find=1+ENOENT+valid-stdout case and the observed three-process holder set; both pass. Focused pytest regressions were added for the transient /proc race, real find errors, and passive holder classification. The overlay is direct-overwrite and intentionally limited to the diagnostic, its tests, this journal, and the reusable /proc lesson. diff --git a/dev/lessons/lessons.md b/dev/lessons/lessons.md index e5a1c392..d4da0b26 100644 --- a/dev/lessons/lessons.md +++ b/dev/lessons/lessons.md @@ -166,3 +166,16 @@ evidence; prefer append-only; supersede incorrect entries with a new one. until killed; removing the inner lock made it finish in 4 s. - **Applies when:** adding a gateway/backend helper that writes state under the state dir and may be reached from a render or converge. + +- **Lesson:** A traversal of live `/proc` can return useful stdout and exit 1 + because a PID or fd disappears between directory enumeration and `stat`. + Treating any nonzero `find /proc` status as total scan failure discards valid + observations and can be misreported as a sudo/authentication problem. Check + privilege separately, tolerate only the expected `/proc/...: No such file or + directory` race, and fail closed on other errors. + - **Evidence / MWE:** `tests/test_gpu_doctor.py::test_proc_find_race_keeps_valid_holder_results`; + live aiq-gpu reproduction on 2026-10-02 produced 52 valid NVIDIA-device + holder records while one vanished fd made GNU find exit 1. + - **Applies when:** inspecting process/fd state under live `/proc`, especially + privileged diagnostics that must distinguish incomplete observation from + authentication failure. diff --git a/infer_stack/gpu_doctor.py b/infer_stack/gpu_doctor.py index 69754684..5ee89e4e 100644 --- a/infer_stack/gpu_doctor.py +++ b/infer_stack/gpu_doctor.py @@ -30,6 +30,7 @@ """ from __future__ import annotations +import os import re import subprocess from dataclasses import dataclass @@ -41,8 +42,16 @@ #: More than this allocated means somebody's memory is on the card, so high #: utilization is explained and there is nothing to report. _IDLE_MEM_MIB = 512 -#: Holders that are expected on a healthy machine, matched against argv[0]. -_EXPECTED_HOLDERS = ('nvidia-persistenced',) +#: Passive/system holders that are expected on a healthy workstation. These +#: processes may keep NVIDIA character devices open, and therefore can block a +#: low-level GPU reset, but they do not by themselves reserve GPU memory or make +#: the device unavailable to Compose/vLLM workloads. +_EXPECTED_HOLDERS = ( + 'nvidia-persistenced', + 'nvidia-device-plugin', + 'gpu-feature-discovery', + 'nvtop', +) @dataclass @@ -65,9 +74,17 @@ class Holder: cmdline: str = '' cgroup: str = '' + @property + def process_name(self) -> str: + """Basename of argv[0], or a placeholder after a /proc race.""" + words = self.cmdline.split() + if not words: + return '' + return words[0].rsplit('/', 1)[-1] + @property def expected(self) -> bool: - return any(name in self.cmdline for name in _EXPECTED_HOLDERS) + return self.process_name in _EXPECTED_HOLDERS @property def where(self) -> str: @@ -155,11 +172,52 @@ def device_holders(use_sudo: bool): """ if not use_sudo: return None - text = _run(['sudo', '-n', 'find', '/proc', '-maxdepth', '3', - '-path', '*/fd/*', '-lname', '/dev/nvidia*', - '-printf', '%h %l\n'], timeout=30) - if text is None: + + # Check elevation separately so a racy /proc traversal cannot be + # misreported as "sudo is unavailable". ``find /proc`` legitimately + # returns 1 if a process/fd disappears between readdir and stat; stdout + # from the rest of the traversal is still valid and must not be discarded. + try: + sudo_check = subprocess.run( + ['sudo', '-n', 'true'], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if sudo_check.returncode != 0: return None + + try: + out = subprocess.run( + ['sudo', '-n', 'find', '/proc', '-maxdepth', '3', + '-path', '*/fd/*', '-lname', '/dev/nvidia*', + '-printf', '%h %l\n'], + capture_output=True, + text=True, + timeout=30, + check=False, + env={**os.environ, 'LC_ALL': 'C'}, + ) + except (OSError, subprocess.SubprocessError): + return None + + if out.returncode != 0: + errors = [ + line.strip() + for line in out.stderr.splitlines() + if line.strip() + ] + benign_proc_race = bool(errors) and all( + '/proc/' in line and 'No such file or directory' in line + for line in errors + ) + if not benign_proc_race: + return None + + text = out.stdout holders: dict[int, Holder] = {} for line in text.strip().splitlines(): parts = line.split() @@ -171,9 +229,14 @@ def device_holders(use_sudo: bool): pid = int(m.group(1)) if pid in holders: continue - holders[pid] = Holder(pid, parts[1], - _proc_text(pid, 'cmdline').replace('\0', ' ').strip(), - _proc_text(pid, 'cgroup').strip()) + cmdline = _proc_text(pid, 'cmdline').replace('\0', ' ').strip() + cgroup = _proc_text(pid, 'cgroup').strip() + # The pid can disappear after find observed its fd. At that point the + # holder itself is gone too, so do not turn a stale observation into a + # current conflict (or crash later while formatting an empty argv). + if not cmdline and not cgroup: + continue + holders[pid] = Holder(pid, parts[1], cmdline, cgroup) return sorted(holders.values(), key=lambda h: h.pid) @@ -221,21 +284,22 @@ def gpu_checks(use_sudo: bool = False, *, _sample=None, _apps=None, unexpected = [h for h in holders if not h.expected] expected = [h for h in holders if h.expected] if not unexpected: - detail = 'only expected holders' + detail = 'only passive/system holders' if expected: - detail += f" ({', '.join(sorted({h.cmdline.split()[0].split('/')[-1] for h in expected}))})" + names = ', '.join(sorted({h.process_name for h in expected})) + detail += f' ({names})' yield ('device holders', True, detail) else: lines = [] for h in unexpected: where = f' in {h.where}' if h.where else '' - lines.append(f'pid {h.pid} {h.cmdline.split()[0][:40]}{where}') + lines.append(f'pid {h.pid} {h.process_name[:40]}{where}') yield ('device holders', False, '; '.join(lines) + ' — these hold a device, so `nvidia-smi -r` ' 'will refuse. Stop the owning container or pod rather than ' 'resetting.') - if expected: + if any(h.process_name == 'nvidia-persistenced' for h in expected): yield ('reset would be blocked', True, 'nvidia-persistenced holds every device by design. `nvidia-smi ' '-pm 0` turns the mode off but leaves the daemon holding its ' diff --git a/tests/test_gpu_doctor.py b/tests/test_gpu_doctor.py index 7871b330..ca43e232 100644 --- a/tests/test_gpu_doctor.py +++ b/tests/test_gpu_doctor.py @@ -7,6 +7,9 @@ reported "nothing holds it" while ``nvidia-smi -r`` said ``In use by another client``, and a reset was recommended on that basis. """ +from types import SimpleNamespace +from unittest import mock + from infer_stack.gpu_doctor import GpuSample, Holder, gpu_checks @@ -67,12 +70,30 @@ def test_persistenced_alone_is_expected(): assert '-pm 0' in res['reset would be blocked'][1] +def test_passive_monitor_and_device_plugin_are_expected(): + """Passive observers should not make a healthy Compose host fail doctor.""" + holders = [ + Holder(10857, '/dev/nvidia0', 'nvidia-device-plugin', + '0::/kubepods.slice/cri-containerd-deadbeef.scope'), + Holder(1932476, '/dev/nvidia0', 'nvtop', '/user.slice'), + Holder(2646, '/dev/nvidia0', + '/usr/bin/nvidia-persistenced --user nvidia-persistenced', '/'), + ] + res = _checks(_sample=lambda: [GpuSample(0, 0, 2)], _apps=lambda: [], + _holders=lambda: holders) + ok, detail = res['device holders'] + assert ok is True + assert 'nvidia-device-plugin' in detail + assert 'nvtop' in detail + assert 'nvidia-persistenced' in detail + + def test_a_kubernetes_pod_holding_a_device_is_named(): """`pid 9030` is useless; the pod it lives in is the answer.""" cg = ('0::/kubepods.slice/kubepods-besteffort.slice/' 'kubepods-besteffort-pod00397bb3_dbeb_412e_8a60_731e5d34b4a7.slice/' 'cri-containerd-4471688cb2a43ab78441da05bebe492e712d64fedbcfcc02c176d5e16273c8f9.scope') - h = Holder(9030, '/dev/nvidia0', 'gpu-feature-discovery', cg) + h = Holder(9030, '/dev/nvidia0', 'python -m rogue-gpu-prober', cg) assert 'kubernetes pod' in h.where res = _checks(_sample=lambda: [GpuSample(0, 100, 2)], _apps=lambda: [], _holders=lambda: [h]) @@ -120,3 +141,57 @@ def test_requested_sudo_unavailable_has_actionable_hint(): assert not ok assert 'sudo -v' in detail assert 'Re-run with --sudo' not in detail + + +def test_proc_find_race_keeps_valid_holder_results(): + """GNU find may return 1 when an fd vanishes while walking live /proc. + + That is a normal race, not a failed privileged scan. Preserve the valid + stdout instead of turning the whole holder check into "not checked". + """ + from infer_stack import gpu_doctor + + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(cmd) + if cmd == ['sudo', '-n', 'true']: + return SimpleNamespace(returncode=0, stdout='', stderr='') + if cmd[:4] == ['sudo', '-n', 'find', '/proc']: + return SimpleNamespace( + returncode=1, + stdout='/proc/10857/fd /dev/nvidia0\n', + stderr=("find: '/proc/2960321/fd/6': " + 'No such file or directory\n'), + ) + raise AssertionError(cmd) + + with mock.patch.object( + gpu_doctor.subprocess, 'run', side_effect=fake_run + ), \ + mock.patch.object(gpu_doctor, '_proc_text', side_effect=[ + 'nvidia-device-plugin', + '0::/kubepods.slice/cri-containerd-deadbeef.scope', + ]): + holders = gpu_doctor.device_holders(use_sudo=True) + + assert holders is not None + assert [h.pid for h in holders] == [10857] + assert holders[0].cmdline == 'nvidia-device-plugin' + assert calls[0] == ['sudo', '-n', 'true'] + + +def test_proc_find_real_error_still_fails_closed(): + """Only disappearing /proc entries are tolerated; real errors are not.""" + from infer_stack import gpu_doctor + + def fake_run(cmd, **kwargs): + if cmd == ['sudo', '-n', 'true']: + return SimpleNamespace(returncode=0, stdout='', stderr='') + if cmd[:4] == ['sudo', '-n', 'find', '/proc']: + return SimpleNamespace(returncode=1, stdout='', + stderr='find: /proc: Permission denied\n') + raise AssertionError(cmd) + + with mock.patch.object(gpu_doctor.subprocess, 'run', side_effect=fake_run): + assert gpu_doctor.device_holders(use_sudo=True) is None