From fef70777aade76482a95e35804e8b71c5708ccaf Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 01/13] feat(compute): add lium capacity scheduler Queue on empty 1-GPU Blackwell inventory instead of terminal-failing capacity misses so training plane can wait for stock. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/compute/lium_capacity.py | 363 +++++++++++++++++++++ tests/unit/test_lium_capacity_scheduler.py | 246 ++++++++++++++ 2 files changed, 609 insertions(+) create mode 100644 src/base/compute/lium_capacity.py create mode 100644 tests/unit/test_lium_capacity_scheduler.py diff --git a/src/base/compute/lium_capacity.py b/src/base/compute/lium_capacity.py new file mode 100644 index 000000000..00e9df1a1 --- /dev/null +++ b/src/base/compute/lium_capacity.py @@ -0,0 +1,363 @@ +"""Master-owned Lium capacity scheduler for Prism training pods. + +Queues when no natively 1-GPU Blackwell offer is free — never fails a job for +lack of capacity (user lock: *attendre*). Inventory is always read through a +training-locked client (``LiumClient.for_prism_training`` / +``training_gpu_lock=True``). + +Persistence: v1 uses :class:`InMemoryLeaseStore` + :meth:`recover` (reattach +pods named ``{pod_name_prefix}{submission_id}``). Queued-only leases are lost +on process restart without an external store; production should swap SQLite or +Postgres behind :class:`LeaseStore`. +""" + +from __future__ import annotations + +import logging +import threading +import time +import uuid +from collections.abc import Callable, Sequence +from dataclasses import dataclass, replace +from enum import StrEnum +from typing import Any, Protocol, runtime_checkable + +from base.compute.provider import Instance, InstanceSpec, Offer + +logger = logging.getLogger(__name__) + +REASON_CAPACITY_WAIT = "capacity_wait" +REASON_SPEND_CEILING = "spend_ceiling" + + +class LeaseState(StrEnum): + """Lifecycle of one Lium training capacity lease.""" + + QUEUED = "queued" + ADMITTING = "admitting" + ACTIVE = "active" + RELEASING = "releasing" + RELEASED = "released" + CANCELLED = "cancelled" + + +_SLOT_HOLDING: frozenset[LeaseState] = frozenset( + {LeaseState.ADMITTING, LeaseState.ACTIVE, LeaseState.RELEASING} +) +_TERMINAL: frozenset[LeaseState] = frozenset( + {LeaseState.CANCELLED, LeaseState.RELEASED, LeaseState.RELEASING} +) + + +@dataclass(frozen=True, slots=True) +class LiumLease: + """One capacity reservation keyed by ``submission_id``.""" + + lease_id: str + submission_id: str + job_id: str + state: LeaseState + enqueued_at: float + pod_id: str | None = None + reason: str | None = None + + +@runtime_checkable +class LeaseStore(Protocol): + """Lease map by ``submission_id``. Prod: SQLite/Postgres; tests: in-memory.""" + + def get(self, submission_id: str) -> LiumLease | None: ... + def put(self, lease: LiumLease) -> None: ... + def list_all(self) -> list[LiumLease]: ... + + +class InMemoryLeaseStore: + """Process-local store (not durable across restart).""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._by_submission: dict[str, LiumLease] = {} + + def get(self, submission_id: str) -> LiumLease | None: + with self._lock: + return self._by_submission.get(submission_id) + + def put(self, lease: LiumLease) -> None: + with self._lock: + self._by_submission[lease.submission_id] = lease + + def list_all(self) -> list[LiumLease]: + with self._lock: + return list(self._by_submission.values()) + + +@runtime_checkable +class LiumCapacityClient(Protocol): + """Minimal async Lium surface the scheduler needs (real or fake).""" + + async def list_offers( + self, *, max_price_per_hour: float | None = None + ) -> list[Offer]: ... + + async def list_pods(self) -> list[dict[str, Any]]: ... + + async def provision( + self, spec: InstanceSpec, *, offer: Offer | None = None + ) -> Instance: ... + + async def terminate(self, instance_id: str) -> None: ... + + +ClientFactory = Callable[[], LiumCapacityClient] +SpendGate = Callable[[], bool] + + +class LiumCapacityScheduler: + """FIFO queue + admit loop for 1-GPU Blackwell Lium training pods. + + ``client_factory`` MUST return a training-locked client. Empty inventory + keeps leases :attr:`LeaseState.QUEUED` with ``reason=capacity_wait``. + """ + + def __init__( + self, + client_factory: ClientFactory, + *, + concurrency_cap: int = 3, + pod_name_prefix: str = "prism-train-", + max_price_per_hour: float = 1.50, + max_lifetime_hours: float = 4.0, + store: LeaseStore | None = None, + spend_gate: SpendGate | None = None, + template_ref: str = "prism-train", + image: str = "ghcr.io/base/prism-train:latest", + ssh_public_keys: Sequence[str] = ("ssh-ed25519 AAAA capacity-scheduler",), + ) -> None: + if ( + isinstance(concurrency_cap, bool) + or not isinstance(concurrency_cap, int) + or concurrency_cap < 1 + ): + raise ValueError("concurrency_cap must be a positive integer") + if max_price_per_hour <= 0 or max_lifetime_hours <= 0: + raise ValueError( + "max_price_per_hour and max_lifetime_hours must be positive" + ) + self._client_factory = client_factory + self._concurrency_cap = concurrency_cap + self._pod_name_prefix = pod_name_prefix + self._max_price_per_hour = max_price_per_hour + self._max_lifetime_hours = max_lifetime_hours + self._store: LeaseStore = store if store is not None else InMemoryLeaseStore() + self._spend_gate = spend_gate + self._template_ref = template_ref + self._image = image + self._ssh_public_keys = tuple(ssh_public_keys) + self._lock = threading.Lock() + + @property + def store(self) -> LeaseStore: + """Backing lease store.""" + return self._store + + def pod_name_for(self, submission_id: str) -> str: + """Stable Lium ``pod_name`` for a submission (used by recover).""" + return f"{self._pod_name_prefix}{submission_id}" + + def enqueue(self, *, submission_id: str, job_id: str) -> LiumLease: + """Enqueue a capacity request. Idempotent on ``submission_id``.""" + if not submission_id: + raise ValueError("submission_id must be non-empty") + if not job_id: + raise ValueError("job_id must be non-empty") + with self._lock: + existing = self._store.get(submission_id) + if existing is not None: + return existing + lease = LiumLease( + lease_id=f"lium-lease-{uuid.uuid4().hex}", + submission_id=submission_id, + job_id=job_id, + state=LeaseState.QUEUED, + enqueued_at=time.time(), + ) + self._store.put(lease) + return lease + + def cancel(self, submission_id: str) -> LiumLease | None: + """Cancel a queued lease. Unknown → ``None``; non-queued left unchanged.""" + with self._lock: + lease = self._store.get(submission_id) + if lease is None: + return None + if lease.state is LeaseState.CANCELLED: + return lease + if lease.state is not LeaseState.QUEUED: + return lease + cancelled = replace(lease, state=LeaseState.CANCELLED, reason=None) + self._store.put(cancelled) + return cancelled + + async def tick(self) -> list[LiumLease]: + """Admit FIFO queued leases up to free slots. Never raises for capacity.""" + client = self._client_factory() + if self._spend_gate is not None and not self._spend_gate(): + self._mark_queued_reason(REASON_SPEND_CEILING) + return [] + + offers = list( + await client.list_offers(max_price_per_hour=self._max_price_per_hour) + ) + free = self._free_slots(len(offers)) + if free <= 0: + self._mark_queued_reason(REASON_CAPACITY_WAIT) + return [] + + admitted: list[LiumLease] = [] + for lease in self._queued_fifo(): + if len(admitted) >= free or not offers: + if not offers: + self._set_reason(lease.submission_id, REASON_CAPACITY_WAIT) + break + offer = offers.pop(0) + result = await self._admit_one(client, lease, offer) + if result is None: + self._set_reason(lease.submission_id, REASON_CAPACITY_WAIT) + break + admitted.append(result) + return admitted + + async def recover(self) -> list[LiumLease]: + """Reattach prefix-matching live pods to stored leases as ACTIVE.""" + client = self._client_factory() + pods = await client.list_pods() + recovered: list[LiumLease] = [] + prefix = self._pod_name_prefix + with self._lock: + for pod in pods: + pod_id = pod.get("id") + name = pod.get("pod_name") or pod.get("name") + if not pod_id or not name: + continue + name_s = str(name) + if not name_s.startswith(prefix): + continue + submission_id = name_s[len(prefix) :] + if not submission_id: + continue + lease = self._store.get(submission_id) + if lease is None or lease.state in _TERMINAL: + continue + if lease.state is LeaseState.ACTIVE and lease.pod_id == str(pod_id): + continue + updated = replace( + lease, + state=LeaseState.ACTIVE, + pod_id=str(pod_id), + reason=None, + ) + self._store.put(updated) + recovered.append(updated) + return recovered + + def _active_count(self) -> int: + return sum( + 1 for lease in self._store.list_all() if lease.state in _SLOT_HOLDING + ) + + def _free_slots(self, offer_count: int) -> int: + remaining = self._concurrency_cap - self._active_count() + if remaining <= 0: + return 0 + return min(remaining, max(0, offer_count)) + + def _queued_fifo(self) -> list[LiumLease]: + queued = [ + lease + for lease in self._store.list_all() + if lease.state is LeaseState.QUEUED + ] + return sorted(queued, key=lambda lease: (lease.enqueued_at, lease.lease_id)) + + def _mark_queued_reason(self, reason: str) -> None: + with self._lock: + for lease in self._store.list_all(): + if lease.state is LeaseState.QUEUED and lease.reason != reason: + self._store.put(replace(lease, reason=reason)) + + def _set_reason(self, submission_id: str, reason: str) -> None: + with self._lock: + lease = self._store.get(submission_id) + if lease is None or lease.state is not LeaseState.QUEUED: + return + if lease.reason != reason: + self._store.put(replace(lease, reason=reason)) + + def _build_spec(self, submission_id: str) -> InstanceSpec: + return InstanceSpec( + name=self.pod_name_for(submission_id), + template_ref=self._template_ref, + image=self._image, + ssh_public_keys=self._ssh_public_keys, + max_lifetime_hours=self._max_lifetime_hours, + max_price_per_hour=self._max_price_per_hour, + gpu_count=1, + ) + + async def _admit_one( + self, + client: LiumCapacityClient, + lease: LiumLease, + offer: Offer, + ) -> LiumLease | None: + with self._lock: + current = self._store.get(lease.submission_id) + if current is None or current.state is not LeaseState.QUEUED: + return None + self._store.put(replace(current, state=LeaseState.ADMITTING, reason=None)) + + try: + instance = await client.provision( + self._build_spec(lease.submission_id), offer=offer + ) + except Exception: + logger.exception( + "lium capacity admit failed for submission_id=%s; re-queue", + lease.submission_id, + ) + with self._lock: + current = self._store.get(lease.submission_id) + if current is not None and current.state is LeaseState.ADMITTING: + self._store.put( + replace( + current, + state=LeaseState.QUEUED, + reason=REASON_CAPACITY_WAIT, + ) + ) + return None + + with self._lock: + current = self._store.get(lease.submission_id) + if current is None: + return None + if current.state is LeaseState.CANCELLED: + orphan_pod_id: str | None = instance.id + else: + orphan_pod_id = None + active = replace( + current, + state=LeaseState.ACTIVE, + pod_id=instance.id, + reason=None, + ) + self._store.put(active) + return active + + try: + await client.terminate(orphan_pod_id) + except Exception: # noqa: BLE001 - cancel race must not raise + logger.warning( + "terminate after cancel race failed for pod %s", orphan_pod_id + ) + return None diff --git a/tests/unit/test_lium_capacity_scheduler.py b/tests/unit/test_lium_capacity_scheduler.py new file mode 100644 index 000000000..e4b3b6d82 --- /dev/null +++ b/tests/unit/test_lium_capacity_scheduler.py @@ -0,0 +1,246 @@ +"""Unit tests for master-owned Lium capacity scheduler. + +Capacity is never a terminal failure: empty 1-GPU Blackwell inventory leaves +leases ``queued`` (user lock: attendre). Fake client only — no network/money. +""" + +from __future__ import annotations + +import itertools +from dataclasses import dataclass, field +from typing import Any + +from base.compute.lium_capacity import ( + InMemoryLeaseStore, + LeaseState, + LiumCapacityScheduler, + LiumLease, +) +from base.compute.provider import Instance, InstanceSpec, Offer + +_BLACKWELL = "NVIDIA RTX PRO 6000 Blackwell Server Edition" +_ID = itertools.count(1) + + +def _offer(*, offer_id: str | None = None, price: float = 1.0) -> Offer: + oid = offer_id or f"exec-{next(_ID)}" + return Offer( + id=oid, + gpu_type=_BLACKWELL, + gpu_count=1, + price_per_hour=price, + ) + + +@dataclass +class FakeLiumClient: + """Mutable fake of the Lium surface the scheduler calls.""" + + offers: list[Offer] = field(default_factory=list) + pods: list[dict[str, Any]] = field(default_factory=list) + provision_calls: list[InstanceSpec] = field(default_factory=list) + terminate_calls: list[str] = field(default_factory=list) + training_gpu_lock: bool = True + _pod_seq: itertools.count = field(default_factory=lambda: itertools.count(1)) + + async def list_offers( + self, *, max_price_per_hour: float | None = None + ) -> list[Offer]: + out: list[Offer] = [] + for offer in self.offers: + if ( + max_price_per_hour is not None + and offer.price_per_hour > max_price_per_hour + ): + continue + out.append(offer) + return out + + async def list_pods(self) -> list[dict[str, Any]]: + return list(self.pods) + + async def provision( + self, spec: InstanceSpec, *, offer: Offer | None = None + ) -> Instance: + self.provision_calls.append(spec) + if not self.offers and offer is None: + raise RuntimeError("no capacity — scheduler must not call provision") + selected = offer or self.offers[0] + pod_id = f"pod-{next(self._pod_seq)}" + self.pods.append( + { + "id": pod_id, + "pod_name": spec.name, + "status": "RUNNING", + "executor_id": selected.id, + } + ) + # Consume one offer slot (inventory shrinks when rented). + if offer is None and self.offers: + self.offers = self.offers[1:] + elif offer is not None: + self.offers = [o for o in self.offers if o.id != offer.id] + return Instance(id=pod_id, status="RUNNING", provider="lium") + + async def terminate(self, instance_id: str) -> None: + self.terminate_calls.append(instance_id) + self.pods = [p for p in self.pods if str(p.get("id")) != str(instance_id)] + + +def _scheduler( + client: FakeLiumClient, + *, + concurrency_cap: int = 3, + store: InMemoryLeaseStore | None = None, +) -> LiumCapacityScheduler: + return LiumCapacityScheduler( + lambda: client, + concurrency_cap=concurrency_cap, + pod_name_prefix="prism-train-", + max_price_per_hour=1.50, + max_lifetime_hours=4.0, + store=store or InMemoryLeaseStore(), + ) + + +# -- S1 enqueue idempotent ---------------------------------------------------- + + +def test_enqueue_idempotent() -> None: + sched = _scheduler(FakeLiumClient()) + first = sched.enqueue(submission_id="sub-1", job_id="job-a") + second = sched.enqueue(submission_id="sub-1", job_id="job-a") + assert first.lease_id == second.lease_id + assert first.submission_id == "sub-1" + assert first.state is LeaseState.QUEUED + assert len(sched.store.list_all()) == 1 + + +# -- S2 FIFO admission -------------------------------------------------------- + + +async def test_fifo_admission_order() -> None: + client = FakeLiumClient(offers=[_offer(), _offer()]) + sched = _scheduler(client, concurrency_cap=1) + a = sched.enqueue(submission_id="sub-a", job_id="j-a") + b = sched.enqueue(submission_id="sub-b", job_id="j-b") + assert a.enqueued_at <= b.enqueued_at + + admitted = await sched.tick() + assert len(admitted) == 1 + assert admitted[0].submission_id == "sub-a" + assert admitted[0].state is LeaseState.ACTIVE + assert admitted[0].pod_id is not None + + still_queued = sched.store.get("sub-b") + assert still_queued is not None + assert still_queued.state is LeaseState.QUEUED + + # Free the active slot and admit B. + client.offers = [_offer()] + active = sched.store.get("sub-a") + assert active is not None and active.pod_id is not None + await client.terminate(active.pod_id) + sched.store.put( + LiumLease( + lease_id=active.lease_id, + submission_id=active.submission_id, + job_id=active.job_id, + state=LeaseState.RELEASED, + enqueued_at=active.enqueued_at, + pod_id=active.pod_id, + reason=None, + ) + ) + admitted_b = await sched.tick() + assert len(admitted_b) == 1 + assert admitted_b[0].submission_id == "sub-b" + assert admitted_b[0].state is LeaseState.ACTIVE + + +# -- S3 queue when inventory empty -------------------------------------------- + + +async def test_queues_when_inventory_empty() -> None: + client = FakeLiumClient(offers=[]) + sched = _scheduler(client) + lease = sched.enqueue(submission_id="sub-wait", job_id="j-wait") + assert lease.state is LeaseState.QUEUED + + changed = await sched.tick() + assert changed == [] + after = sched.store.get("sub-wait") + assert after is not None + assert after.state is LeaseState.QUEUED + assert after.reason == "capacity_wait" + assert client.provision_calls == [] + + +# -- S4 recover reattaches existing pod --------------------------------------- + + +async def test_recover_reattaches_existing_pod() -> None: + client = FakeLiumClient( + pods=[ + { + "id": "pod-live-9", + "pod_name": "prism-train-sub-rec", + "status": "RUNNING", + } + ] + ) + store = InMemoryLeaseStore() + # Pre-seed a lease that lost process memory of pod_id (queued after crash). + store.put( + LiumLease( + lease_id="lease-rec", + submission_id="sub-rec", + job_id="j-rec", + state=LeaseState.QUEUED, + enqueued_at=1.0, + pod_id=None, + reason="capacity_wait", + ) + ) + sched = _scheduler(client, store=store) + recovered = await sched.recover() + assert len(recovered) == 1 + assert recovered[0].submission_id == "sub-rec" + assert recovered[0].state is LeaseState.ACTIVE + assert recovered[0].pod_id == "pod-live-9" + assert recovered[0].reason is None + + +# -- S5 cancel queued --------------------------------------------------------- + + +def test_cancel_queued() -> None: + sched = _scheduler(FakeLiumClient()) + sched.enqueue(submission_id="sub-c", job_id="j-c") + cancelled = sched.cancel("sub-c") + assert cancelled is not None + assert cancelled.state is LeaseState.CANCELLED + assert sched.store.get("sub-c") is not None + assert sched.store.get("sub-c").state is LeaseState.CANCELLED # type: ignore[union-attr] + + +def test_cancel_unknown_returns_none() -> None: + sched = _scheduler(FakeLiumClient()) + assert sched.cancel("missing") is None + + +async def test_tick_admits_up_to_offer_count_and_cap() -> None: + client = FakeLiumClient(offers=[_offer(), _offer()]) + sched = _scheduler(client, concurrency_cap=3) + for i in range(4): + sched.enqueue(submission_id=f"sub-{i}", job_id=f"j-{i}") + admitted = await sched.tick() + assert len(admitted) == 2 # only 2 offers + assert all(lease.state is LeaseState.ACTIVE for lease in admitted) + queued = [ + lease + for lease in sched.store.list_all() + if lease.state is LeaseState.QUEUED + ] + assert len(queued) == 2 + assert {lease.submission_id for lease in queued} == {"sub-2", "sub-3"} From d6aafe032e3b604685c76b23979fe97721a41b64 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 02/13] feat(compute): add lium orphan pod terminator Prefix-owned orphan cleanup guards against leaked paid pods after interrupted training runs. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/compute/lium_orphan.py | 166 ++++++++++++++++++++++++++++++++ tests/unit/test_lium_orphan.py | 144 +++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 src/base/compute/lium_orphan.py create mode 100644 tests/unit/test_lium_orphan.py diff --git a/src/base/compute/lium_orphan.py b/src/base/compute/lium_orphan.py new file mode 100644 index 000000000..a813e0e4d --- /dev/null +++ b/src/base/compute/lium_orphan.py @@ -0,0 +1,166 @@ +"""Terminate master-owned Lium pods that no longer hold an active lease. + +Ownership marker is the pod name prefix (default ``prism-train-``). Active +lease sets are supplied by the caller so this module stays independent of +capacity/lease bookkeeping. +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from typing import Any, Protocol, runtime_checkable + +logger = logging.getLogger(__name__) + +_DEFAULT_POD_NAME_PREFIX = "prism-train-" + + +@runtime_checkable +class LiumOrphanClient(Protocol): + """Minimal Lium surface required by :func:`reconcile_orphan_pods`.""" + + async def list_pods(self) -> list[dict[str, Any]]: + """Return raw pod dicts from the account.""" + ... + + async def terminate(self, instance_id: str) -> None: + """Request pod deletion (idempotent on 404).""" + ... + + async def verify_terminated(self, instance_id: str) -> bool: + """Return True when ``instance_id`` is absent from list_pods.""" + ... + + +@dataclass(frozen=True, slots=True) +class OrphanTermination: + """Outcome for one pod considered during orphan reconciliation.""" + + pod_id: str + pod_name: str + verified: bool + skipped_reason: str | None = None + + +def _optional_text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _extract_pod_id(pod: Mapping[str, Any]) -> str | None: + for key in ("id", "pod_id", "uuid"): + found = _optional_text(pod.get(key)) + if found is not None: + return found + return None + + +def _extract_pod_name(pod: Mapping[str, Any]) -> str | None: + for key in ("pod_name", "name"): + found = _optional_text(pod.get(key)) + if found is not None: + return found + return None + + +def _is_actively_leased( + *, + pod_id: str, + pod_name: str, + active_lease_pod_ids: Set[str], + active_lease_pod_names: Set[str] | None, +) -> bool: + if pod_id in active_lease_pod_ids: + return True + if active_lease_pod_names is not None and pod_name in active_lease_pod_names: + return True + return False + + +async def reconcile_orphan_pods( + client: LiumOrphanClient, + *, + pod_name_prefix: str = _DEFAULT_POD_NAME_PREFIX, + active_lease_pod_ids: Set[str], + active_lease_pod_names: Set[str] | None = None, +) -> list[OrphanTermination]: + """Terminate prefix-owned pods that are not covered by an active lease. + + Only pods whose name starts with ``pod_name_prefix`` are candidates. + Pods missing a usable id or name are skipped fail-closed (never terminated). + Non-prefix pods are ignored entirely. + """ + pods: Sequence[Mapping[str, Any]] = await client.list_pods() + results: list[OrphanTermination] = [] + + for raw in pods: + if not isinstance(raw, Mapping): + logger.warning("lium orphan reconcile skipped non-mapping pod entry") + continue + + pod_id = _extract_pod_id(raw) + pod_name = _extract_pod_name(raw) + + if pod_name is None: + results.append( + OrphanTermination( + pod_id=pod_id or "", + pod_name="", + verified=False, + skipped_reason="missing_pod_name", + ) + ) + logger.warning( + "lium orphan reconcile skipped pod with missing name (id=%r)", + pod_id, + ) + continue + + if not pod_name.startswith(pod_name_prefix): + continue + + if pod_id is None: + results.append( + OrphanTermination( + pod_id="", + pod_name=pod_name, + verified=False, + skipped_reason="missing_pod_id", + ) + ) + logger.warning( + "lium orphan reconcile skipped prefix pod with missing id (name=%r)", + pod_name, + ) + continue + + if _is_actively_leased( + pod_id=pod_id, + pod_name=pod_name, + active_lease_pod_ids=active_lease_pod_ids, + active_lease_pod_names=active_lease_pod_names, + ): + continue + + await client.terminate(pod_id) + verified = await client.verify_terminated(pod_id) + results.append( + OrphanTermination( + pod_id=pod_id, + pod_name=pod_name, + verified=verified, + skipped_reason=None, + ) + ) + if not verified: + logger.warning( + "lium orphan terminate issued but pod still listed (id=%s name=%s)", + pod_id, + pod_name, + ) + + return results diff --git a/tests/unit/test_lium_orphan.py b/tests/unit/test_lium_orphan.py new file mode 100644 index 000000000..592ea451a --- /dev/null +++ b/tests/unit/test_lium_orphan.py @@ -0,0 +1,144 @@ +"""Unit tests for Lium orphan pod reconciler (prefix-owned money-leak guard).""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from base.compute.lium_orphan import OrphanTermination, reconcile_orphan_pods + + +class _FakeLiumClient: + """In-memory stand-in for list_pods / terminate / verify_terminated.""" + + def __init__(self, pods: list[dict[str, Any]]) -> None: + self._pods = [dict(p) for p in pods] + self.terminated_ids: list[str] = [] + + async def list_pods(self) -> list[dict[str, Any]]: + return [dict(p) for p in self._pods] + + async def terminate(self, instance_id: str) -> None: + self.terminated_ids.append(str(instance_id)) + remaining: list[dict[str, Any]] = [] + for pod in self._pods: + pod_id = str(pod.get("id") or pod.get("pod_id") or "") + if pod_id != str(instance_id): + remaining.append(pod) + self._pods = remaining + + async def verify_terminated(self, instance_id: str) -> bool: + for pod in self._pods: + pod_id = str(pod.get("id") or pod.get("pod_id") or "") + if pod_id == str(instance_id): + return False + return True + + +@pytest.mark.asyncio +async def test_orphan_terminator_only_prefix() -> None: + """Prefix orphan is terminated; foreign pods are left alone.""" + client = _FakeLiumClient( + [ + {"id": "orphan-1", "pod_name": "prism-train-job-aaa"}, + {"id": "other-1", "name": "user-notebook"}, + ] + ) + + results = await reconcile_orphan_pods( + client, + active_lease_pod_ids=set(), + ) + + assert client.terminated_ids == ["orphan-1"] + assert len(results) == 1 + assert results[0] == OrphanTermination( + pod_id="orphan-1", + pod_name="prism-train-job-aaa", + verified=True, + skipped_reason=None, + ) + + +@pytest.mark.asyncio +async def test_orphan_skips_leased() -> None: + """Leased prefix pods (by id or name) are not terminated.""" + client = _FakeLiumClient( + [ + {"id": "leased-id", "pod_name": "prism-train-active-1"}, + {"id": "leased-name-id", "name": "prism-train-active-2"}, + {"id": "true-orphan", "pod_name": "prism-train-dead"}, + ] + ) + + results = await reconcile_orphan_pods( + client, + active_lease_pod_ids={"leased-id"}, + active_lease_pod_names={"prism-train-active-2"}, + ) + + assert client.terminated_ids == ["true-orphan"] + kept = [r.pod_id for r in results if r.skipped_reason is None] + assert kept == ["true-orphan"] + + +@pytest.mark.asyncio +async def test_orphan_skips_non_prefix() -> None: + """Non-prefix pods never reach terminate.""" + client = _FakeLiumClient( + [ + {"id": "a", "pod_name": "miner-pod"}, + # Does not start with prism-train- + {"id": "b", "name": "prism-trainX-nope"}, + {"id": "c", "pod_name": "other-prism-train-suffix"}, + ] + ) + + results = await reconcile_orphan_pods( + client, + active_lease_pod_ids=set(), + ) + + assert client.terminated_ids == [] + assert results == [] + + +@pytest.mark.asyncio +async def test_empty_pods_ok() -> None: + """Empty account yields empty results and no terminate calls.""" + client = _FakeLiumClient([]) + + results = await reconcile_orphan_pods( + client, + active_lease_pod_ids=frozenset({"anything"}), + ) + + assert results == [] + assert client.terminated_ids == [] + + +@pytest.mark.asyncio +async def test_orphan_skips_unidentified_fail_closed() -> None: + """Missing id/name is fail-closed: skip terminate rather than guess.""" + client = _FakeLiumClient( + [ + {"pod_name": "prism-train-no-id"}, + {"id": "mystery-1"}, + {"id": "ok-orphan", "name": "prism-train-ok"}, + ] + ) + + results = await reconcile_orphan_pods( + client, + active_lease_pod_ids=set(), + ) + + assert client.terminated_ids == ["ok-orphan"] + skipped = [r for r in results if r.skipped_reason is not None] + assert len(skipped) == 2 + assert all(r.verified is False for r in skipped) + terminated = [r for r in results if r.skipped_reason is None] + assert len(terminated) == 1 + assert terminated[0].pod_id == "ok-orphan" + assert terminated[0].verified is True From 442553ecb7b776f9756d31facda92e6a07f058ee Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 03/13] feat(config): add LiumTrainingSettings and prism dispatch variant Expose training-plane env knobs and prism_dispatch_variant so Lium wiring and orchestration can fail closed without an API key. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/config/settings.py | 59 ++++++++++++++++ tests/unit/test_constation_settings.py | 16 +++++ tests/unit/test_lium_training_settings.py | 83 +++++++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 tests/unit/test_lium_training_settings.py diff --git a/src/base/config/settings.py b/src/base/config/settings.py index 5ff097aa4..c05b80c81 100644 --- a/src/base/config/settings.py +++ b/src/base/config/settings.py @@ -297,6 +297,13 @@ class ConstationSettings(BaseModel): sidecar_scheme: str = "http" sidecar_internal_port: int = 8787 custody_persist: bool = True + #: Image variant used when stamping constation identity onto Prism + #: ``work_assignments.payload`` at dispatch. Must be a known + #: ``ImageVariant`` value (``cpu`` / ``cuda``). Empty string disables + #: stamping (dispatch still succeeds; pre-forward hook skips). + #: Default ``cuda`` matches Prism GPU work units without changing + #: behavior when no active pin is registered. + prism_dispatch_variant: str = "cuda" @model_validator(mode="after") def validate_constation_bounds(self) -> ConstationSettings: @@ -316,6 +323,57 @@ def validate_constation_bounds(self) -> ConstationSettings: raise ValueError("sidecar_internal_port must be in 1..65535") if self.sidecar_scheme not in {"http", "https"}: raise ValueError("sidecar_scheme must be 'http' or 'https'") + variant = self.prism_dispatch_variant.strip().lower() + if variant and variant not in {"cpu", "cuda"}: + raise ValueError( + "prism_dispatch_variant must be 'cpu', 'cuda', or empty " + f"(got {self.prism_dispatch_variant!r})" + ) + # Normalize once so consumers see a stable value. + object.__setattr__(self, "prism_dispatch_variant", variant) + return self + + +class LiumTrainingSettings(BaseModel): + """Master-owned Prism Lium training spend/lifetime/concurrency guards. + + Fail-closed: ``enabled`` defaults False until ops turns the plane on. + Provider secrets may be supplied inline (``api_key``) or via ``api_key_file`` + (file contents are read lazily by consumers, not here). + Env nesting: ``BASE_LIUM_TRAINING__`` (see ``loader._apply_env``). + + ``daily_spend_ceiling_usd`` blocks NEW admissions only: jobs stay queued + with reason ``spend_ceiling`` and must never terminal-fail for capacity. + """ + + enabled: bool = False + api_key: str | None = None + api_key_file: Path | None = None + max_price_per_hour: float = 1.50 + max_lifetime_hours: float = 4.0 + concurrency_cap: int = 3 + daily_spend_ceiling_usd: float = 50.0 + queue_poll_seconds: int = 30 + max_queue_age_hours: float = 48.0 + pod_name_prefix: str = "prism-train-" + ssh_public_key_file: Path | None = None + + @model_validator(mode="after") + def validate_lium_training_bounds(self) -> LiumTrainingSettings: + if self.max_price_per_hour <= 0: + raise ValueError("max_price_per_hour must be positive") + if self.max_lifetime_hours < 1: + raise ValueError("max_lifetime_hours must be >= 1") + if self.concurrency_cap < 1: + raise ValueError("concurrency_cap must be >= 1") + if self.daily_spend_ceiling_usd <= 0: + raise ValueError("daily_spend_ceiling_usd must be positive") + if self.queue_poll_seconds <= 0: + raise ValueError("queue_poll_seconds must be positive") + if self.max_queue_age_hours <= 0: + raise ValueError("max_queue_age_hours must be positive") + if not self.pod_name_prefix.strip(): + raise ValueError("pod_name_prefix must be non-empty") return self @@ -594,6 +652,7 @@ class Settings(BaseModel): security: SecuritySettings = Field(default_factory=SecuritySettings) compute: ComputeSettings = Field(default_factory=ComputeSettings) constation: ConstationSettings = Field(default_factory=ConstationSettings) + lium_training: LiumTrainingSettings = Field(default_factory=LiumTrainingSettings) worker: WorkerSettings = Field(default_factory=WorkerSettings) observability: ObservabilitySettings = Field(default_factory=ObservabilitySettings) supervisor: SupervisorSettings = Field(default_factory=SupervisorSettings) diff --git a/tests/unit/test_constation_settings.py b/tests/unit/test_constation_settings.py index 370de297e..a8f9adf8a 100644 --- a/tests/unit/test_constation_settings.py +++ b/tests/unit/test_constation_settings.py @@ -15,6 +15,7 @@ def test_constation_settings_defaults() -> None: assert settings.gap_budget_seconds == 30.0 assert settings.sidecar_internal_port == 8787 assert settings.sidecar_scheme == "http" + assert settings.prism_dispatch_variant == "cuda" def test_constation_settings_requires_positive_gap() -> None: @@ -45,3 +46,18 @@ def test_constation_settings_attached_to_root() -> None: root = Settings() assert isinstance(root.constation, ConstationSettings) assert root.constation.enabled is False + + +def test_constation_settings_prism_dispatch_variant_cpu() -> None: + settings = ConstationSettings(prism_dispatch_variant="CPU") + assert settings.prism_dispatch_variant == "cpu" + + +def test_constation_settings_prism_dispatch_variant_empty_disables() -> None: + settings = ConstationSettings(prism_dispatch_variant=" ") + assert settings.prism_dispatch_variant == "" + + +def test_constation_settings_rejects_unknown_prism_dispatch_variant() -> None: + with pytest.raises(ValidationError): + ConstationSettings(prism_dispatch_variant="rocm") diff --git a/tests/unit/test_lium_training_settings.py b/tests/unit/test_lium_training_settings.py new file mode 100644 index 000000000..0e909fcef --- /dev/null +++ b/tests/unit/test_lium_training_settings.py @@ -0,0 +1,83 @@ +"""Unit tests for LiumTrainingSettings (master-owned Prism Lium training guards).""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from base.config.loader import load_settings +from base.config.settings import LiumTrainingSettings, Settings + + +def test_lium_training_settings_defaults() -> None: + settings = LiumTrainingSettings() + assert settings.enabled is False + assert settings.api_key is None + assert settings.api_key_file is None + assert settings.max_price_per_hour == 1.50 + assert settings.max_lifetime_hours == 4.0 + assert settings.concurrency_cap == 3 + assert settings.daily_spend_ceiling_usd == 50.0 + assert settings.queue_poll_seconds == 30 + assert settings.max_queue_age_hours == 48.0 + assert settings.pod_name_prefix == "prism-train-" + assert settings.ssh_public_key_file is None + + +def test_lium_training_settings_rejects_non_positive_price() -> None: + with pytest.raises(ValidationError): + LiumTrainingSettings(max_price_per_hour=0) + with pytest.raises(ValidationError): + LiumTrainingSettings(max_price_per_hour=-1.0) + + +def test_lium_training_settings_rejects_lifetime_below_one() -> None: + with pytest.raises(ValidationError): + LiumTrainingSettings(max_lifetime_hours=0.5) + with pytest.raises(ValidationError): + LiumTrainingSettings(max_lifetime_hours=0) + + +def test_lium_training_settings_rejects_non_positive_concurrency() -> None: + with pytest.raises(ValidationError): + LiumTrainingSettings(concurrency_cap=0) + with pytest.raises(ValidationError): + LiumTrainingSettings(concurrency_cap=-1) + + +def test_lium_training_settings_rejects_non_positive_spend_ceiling() -> None: + with pytest.raises(ValidationError): + LiumTrainingSettings(daily_spend_ceiling_usd=0) + with pytest.raises(ValidationError): + LiumTrainingSettings(daily_spend_ceiling_usd=-10.0) + + +def test_lium_training_settings_rejects_non_positive_poll() -> None: + with pytest.raises(ValidationError): + LiumTrainingSettings(queue_poll_seconds=0) + with pytest.raises(ValidationError): + LiumTrainingSettings(queue_poll_seconds=-5) + + +def test_lium_training_settings_rejects_empty_prefix() -> None: + with pytest.raises(ValidationError): + LiumTrainingSettings(pod_name_prefix="") + with pytest.raises(ValidationError): + LiumTrainingSettings(pod_name_prefix=" ") + + +def test_lium_training_settings_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + # Convention: BASE_ prefix + nested path joined by __ (see loader._apply_env). + monkeypatch.setenv("BASE_LIUM_TRAINING__ENABLED", "true") + monkeypatch.setenv("BASE_LIUM_TRAINING__MAX_PRICE_PER_HOUR", "2.25") + monkeypatch.setenv("BASE_LIUM_TRAINING__CONCURRENCY_CAP", "2") + loaded = load_settings() + assert loaded.lium_training.enabled is True + assert loaded.lium_training.max_price_per_hour == 2.25 + assert loaded.lium_training.concurrency_cap == 2 + + +def test_lium_training_settings_attached_to_root() -> None: + root = Settings() + assert isinstance(root.lium_training, LiumTrainingSettings) + assert root.lium_training.enabled is False From 4cabde62a42d2bb3c1891688d9deae5e0c5b7e76 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 04/13] feat(compute): add lium training wiring factories Build capacity/orphan/client helpers from LiumTrainingSettings with fail-closed behavior when credentials are missing. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/compute/lium_training_wiring.py | 188 +++++++++++++++++++++++ tests/unit/test_lium_training_wiring.py | 133 ++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 src/base/compute/lium_training_wiring.py create mode 100644 tests/unit/test_lium_training_wiring.py diff --git a/src/base/compute/lium_training_wiring.py b/src/base/compute/lium_training_wiring.py new file mode 100644 index 000000000..0a2d251d2 --- /dev/null +++ b/src/base/compute/lium_training_wiring.py @@ -0,0 +1,188 @@ +"""Build training-locked Lium clients/schedulers from LiumTrainingSettings. + +Fail-closed: ``lium_training.enabled=True`` without a usable API key raises. +Disabled plane returns ``None`` from the client factory (no network client). +Never logs key material. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from pathlib import Path +from typing import Protocol + +from base.compute.lium import LiumClient +from base.compute.lium_capacity import LiumCapacityClient, LiumCapacityScheduler +from base.compute.worker_deployment import WORKER_IMAGE +from base.security.admin_auth import read_secret + +logger = logging.getLogger(__name__) +# Default training pod image (no digest pin yet — T14). Prefer prism-evaluator +# over the scheduler placeholder ``ghcr.io/base/prism-train:latest``. +DEFAULT_LIUM_TRAINING_IMAGE = WORKER_IMAGE + + +class _LiumTrainingSurface(Protocol): + enabled: bool + api_key: str | None + api_key_file: Path | None + concurrency_cap: int + pod_name_prefix: str + max_price_per_hour: float + max_lifetime_hours: float + ssh_public_key_file: Path | None + + +class _HasLiumTraining(Protocol): + @property + def lium_training(self) -> _LiumTrainingSurface: ... + + +def resolve_lium_training_api_key(lt: _LiumTrainingSurface) -> str | None: + """Resolve API key from inline ``api_key`` or ``api_key_file`` (strip). + + Empty / whitespace-only → ``None``. Never logs the key. + """ + file_path = lt.api_key_file + raw = read_secret( + lt.api_key, + str(file_path) if file_path is not None else None, + ) + text = raw.strip() if raw else "" + if not text: + return None + return text + + +def build_lium_training_client(settings: _HasLiumTraining) -> LiumClient | None: + """Construct a training-locked :class:`LiumClient` when the plane is on. + + * ``enabled=False`` → ``None`` + * ``enabled=True`` without key → :class:`ValueError` (fail-closed) + * ``enabled=True`` with key → :meth:`LiumClient.for_prism_training` + """ + lt = settings.lium_training + if not lt.enabled: + return None + + api_key = resolve_lium_training_api_key(lt) + if api_key is None: + raise ValueError( + "lium_training.enabled is True but API key is missing " + "(set lium_training.api_key or lium_training.api_key_file)" + ) + return LiumClient.for_prism_training(api_key) + + +def _load_ssh_public_keys(lt: _LiumTrainingSurface) -> tuple[str, ...] | None: + path = lt.ssh_public_key_file + if path is None: + return None + if not path.is_file(): + raise ValueError( + "lium_training.ssh_public_key_file is set but is not a readable file" + ) + text = path.read_text(encoding="utf-8").strip() + if not text: + raise ValueError("lium_training.ssh_public_key_file is empty") + return (text,) + + +def build_lium_capacity_scheduler( + settings: _HasLiumTraining, + *, + image: str | None = None, + client_factory: Callable[[], LiumCapacityClient] | None = None, + store: object | None = None, + spend_gate: Callable[[], bool] | None = None, +) -> LiumCapacityScheduler: + """Build :class:`LiumCapacityScheduler` from settings + training-locked client. + + Requires ``lium_training.enabled=True`` and a resolvable API key (fail-closed). + Image defaults to :data:`DEFAULT_LIUM_TRAINING_IMAGE` (prism-evaluator, no digest). + """ + lt = settings.lium_training + if not lt.enabled: + raise ValueError( + "lium_training.enabled is False; refuse to build LiumCapacityScheduler" + ) + + factory = client_factory + if factory is None: + # Capture key once so factory does not re-read settings on every call + # in a way that could race; still fail-closed if key missing. + api_key = resolve_lium_training_api_key(lt) + if api_key is None: + raise ValueError( + "lium_training.enabled is True but API key is missing " + "(set lium_training.api_key or lium_training.api_key_file)" + ) + + def factory() -> LiumCapacityClient: + return LiumClient.for_prism_training(api_key) + + ssh_keys = _load_ssh_public_keys(lt) + kwargs: dict[str, object] = { + "concurrency_cap": int(lt.concurrency_cap), + "pod_name_prefix": str(lt.pod_name_prefix), + "max_price_per_hour": float(lt.max_price_per_hour), + "max_lifetime_hours": float(lt.max_lifetime_hours), + "image": image if image is not None else DEFAULT_LIUM_TRAINING_IMAGE, + } + if store is not None: + kwargs["store"] = store + if spend_gate is not None: + kwargs["spend_gate"] = spend_gate + if ssh_keys is not None: + kwargs["ssh_public_keys"] = ssh_keys + + return LiumCapacityScheduler(factory, **kwargs) # type: ignore[arg-type] + + +def try_build_lium_capacity_scheduler( + settings: _HasLiumTraining, + *, + image: str | None = None, + client_factory: Callable[[], LiumCapacityClient] | None = None, + store: object | None = None, + spend_gate: Callable[[], bool] | None = None, +) -> LiumCapacityScheduler | None: + """Build scheduler when ``lium_training.enabled``; else ``None``. + + Fail-closed on missing key when enabled: logs and returns ``None`` so the + master still boots (worker-plane / validator path unchanged). Callers that + need hard fail should use :func:`build_lium_capacity_scheduler` directly. + """ + if not settings.lium_training.enabled: + return None + try: + return build_lium_capacity_scheduler( + settings, + image=image, + client_factory=client_factory, + store=store, + spend_gate=spend_gate, + ) + except Exception: + logger.exception( + "lium_training.enabled but LiumCapacityScheduler build failed; " + "continuing without master-owned Lium admission" + ) + return None + + +async def run_lium_capacity_tick( + scheduler: LiumCapacityScheduler | None, +) -> None: + """Safe one-shot tick for an optional scheduler (no-op if ``None``). + + Intended for a dedicated background loop if orchestration is not the + tick owner. Failures are logged; capacity never raises to the caller. + """ + if scheduler is None: + return + try: + await scheduler.tick() + except Exception: + logger.exception("lium capacity tick failed") diff --git a/tests/unit/test_lium_training_wiring.py b/tests/unit/test_lium_training_wiring.py new file mode 100644 index 000000000..3d9392638 --- /dev/null +++ b/tests/unit/test_lium_training_wiring.py @@ -0,0 +1,133 @@ +"""Lium training client/scheduler factories from LiumTrainingSettings.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from base.compute.lium import LiumClient +from base.compute.lium_capacity import LiumCapacityScheduler +from base.compute.lium_training_wiring import ( + DEFAULT_LIUM_TRAINING_IMAGE, + build_lium_capacity_scheduler, + build_lium_training_client, + resolve_lium_training_api_key, + run_lium_capacity_tick, + try_build_lium_capacity_scheduler, +) +from base.compute.worker_deployment import WORKER_IMAGE +from base.config.settings import LiumTrainingSettings, Settings + + +def test_disabled_returns_none() -> None: + """Given enabled=False, When build client, Then None.""" + settings = Settings(lium_training=LiumTrainingSettings(enabled=False, api_key="k")) + assert build_lium_training_client(settings) is None + + +def test_lium_training_enabled_without_key_fail_closed() -> None: + """Given enabled=True without key, When build, Then raises clear error.""" + settings = Settings(lium_training=LiumTrainingSettings(enabled=True)) + with pytest.raises(ValueError, match="api_key|api_key_file|lium_training"): + build_lium_training_client(settings) + + settings_empty = Settings( + lium_training=LiumTrainingSettings(enabled=True, api_key=" ") + ) + with pytest.raises(ValueError, match="api_key|api_key_file|lium_training"): + build_lium_training_client(settings_empty) + + +def test_enabled_with_key_returns_locked_client() -> None: + """Given enabled + api_key, When build, Then training-locked LiumClient.""" + settings = Settings( + lium_training=LiumTrainingSettings(enabled=True, api_key="test-lium-key") + ) + client = build_lium_training_client(settings) + assert isinstance(client, LiumClient) + assert client._training_gpu_lock is True # noqa: SLF001 + # Key must never appear in repr + assert "test-lium-key" not in repr(client) + + +def test_enabled_with_key_file_returns_locked_client(tmp_path: Path) -> None: + """Given enabled + api_key_file, When build, Then training-locked client.""" + key_path = tmp_path / "lium.key" + key_path.write_text(" file-lium-key\n", encoding="utf-8") + settings = Settings( + lium_training=LiumTrainingSettings(enabled=True, api_key_file=key_path) + ) + client = build_lium_training_client(settings) + assert isinstance(client, LiumClient) + assert client._training_gpu_lock is True # noqa: SLF001 + assert "file-lium-key" not in repr(client) + + +def test_resolve_lium_training_api_key_prefers_inline_over_file(tmp_path: Path) -> None: + """Given both inline and file, When resolve, Then inline wins (read_secret).""" + key_path = tmp_path / "lium.key" + key_path.write_text("file-key", encoding="utf-8") + lt = LiumTrainingSettings(api_key="inline-key", api_key_file=key_path) + assert resolve_lium_training_api_key(lt) == "inline-key" + + +def test_build_lium_capacity_scheduler_maps_settings() -> None: + """Given settings + factory, When build scheduler, Then fields mapped.""" + settings = Settings( + lium_training=LiumTrainingSettings( + enabled=True, + api_key="sched-key", + concurrency_cap=5, + pod_name_prefix="train-x-", + max_price_per_hour=2.25, + max_lifetime_hours=6.0, + ) + ) + scheduler = build_lium_capacity_scheduler(settings) + assert isinstance(scheduler, LiumCapacityScheduler) + assert scheduler._concurrency_cap == 5 # noqa: SLF001 + assert scheduler._pod_name_prefix == "train-x-" # noqa: SLF001 + assert scheduler._max_price_per_hour == 2.25 # noqa: SLF001 + assert scheduler._max_lifetime_hours == 6.0 # noqa: SLF001 + assert scheduler._image == DEFAULT_LIUM_TRAINING_IMAGE # noqa: SLF001 + assert DEFAULT_LIUM_TRAINING_IMAGE == WORKER_IMAGE + assert "prism-train:latest" not in scheduler._image # noqa: SLF001 + + client = scheduler._client_factory() # noqa: SLF001 + assert isinstance(client, LiumClient) + assert client._training_gpu_lock is True # noqa: SLF001 + + +def test_build_lium_capacity_scheduler_disabled_fail_closed() -> None: + """Given enabled=False, When build scheduler, Then raises (no silent empty).""" + settings = Settings(lium_training=LiumTrainingSettings(enabled=False)) + with pytest.raises(ValueError, match="lium_training|enabled"): + build_lium_capacity_scheduler(settings) + + + +def test_try_build_lium_capacity_scheduler_disabled_returns_none() -> None: + """Given enabled=False, When try_build, Then None (no raise).""" + settings = Settings(lium_training=LiumTrainingSettings(enabled=False)) + assert try_build_lium_capacity_scheduler(settings) is None + + +def test_try_build_lium_capacity_scheduler_enabled_builds() -> None: + """Given enabled + key, When try_build, Then scheduler instance.""" + settings = Settings( + lium_training=LiumTrainingSettings(enabled=True, api_key="try-key") + ) + scheduler = try_build_lium_capacity_scheduler(settings) + assert isinstance(scheduler, LiumCapacityScheduler) + + +def test_try_build_lium_capacity_scheduler_missing_key_returns_none() -> None: + """Given enabled without key, When try_build, Then None (log + soft fail).""" + settings = Settings(lium_training=LiumTrainingSettings(enabled=True)) + assert try_build_lium_capacity_scheduler(settings) is None + + +async def test_run_lium_capacity_tick_none_is_noop() -> None: + """Given None scheduler, When tick helper, Then no raise.""" + await run_lium_capacity_tick(None) From ca864196a41ba3e0e7a266d1e14960e20bb84005 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 05/13] feat(compute): extend lium client for training GPU lock Add for_prism_training surface and training GPU lock helpers used by the host landmine training plane. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/compute/lium.py | 126 ++++++++++++++-- tests/unit/test_compute_lium_client.py | 191 ++++++++++++++++++++++--- 2 files changed, 291 insertions(+), 26 deletions(-) diff --git a/src/base/compute/lium.py b/src/base/compute/lium.py index 2d7644f41..3012d601c 100644 --- a/src/base/compute/lium.py +++ b/src/base/compute/lium.py @@ -11,6 +11,13 @@ * :meth:`LiumClient.terminate` is idempotent (a ``404`` delete is success) and :meth:`LiumClient.verify_terminated` reflects real pod absence via ``GET /pods``. +Prism training lock (``training_gpu_lock=True`` / +:meth:`LiumClient.for_prism_training`): +fail-closed to natively 1-GPU :data:`LIUM_TRAINING_GPU_TYPE` offers only. Live API +evidence (2026-07): POST .../rent with ``gpu_count=1`` on an 8-GPU machine returns +HTTP 400 "Provider doesn't allow GPU splitting." — multi-GPU hosts are refused +before rent rather than rented in full. + The API key lives only in the request header; it is never logged, embedded in an error message, or exposed via ``repr``. """ @@ -18,9 +25,10 @@ from __future__ import annotations import logging +import re from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, Final import httpx @@ -38,6 +46,56 @@ LIUM_API_BASE_URL = "https://lium.io/api" _DEFAULT_SSH_KEY_NAME = "prism-mission-worker" +# Canonical display name for the only GPU type Prism training may rent. +LIUM_TRAINING_GPU_TYPE: Final[str] = ( + "NVIDIA RTX PRO 6000 Blackwell Server Edition" +) +_TRAINING_GPU_REQUIRED_TOKENS: Final[frozenset[str]] = frozenset( + {"rtx", "pro", "6000", "blackwell"} +) +_LEADING_GPU_NOISE_TOKENS: Final[frozenset[str]] = frozenset({"nvidia", "gpu"}) +_NON_ALNUM_RUN: Final[re.Pattern[str]] = re.compile(r"[-_]+") +_WHITESPACE_RUN: Final[re.Pattern[str]] = re.compile(r"\s+") + + +def normalize_gpu_type(value: str | None) -> str: + """Normalize a provider GPU type string for token matching. + + Lowercase; replace runs of ``-``/``_`` with a space; collapse whitespace; + strip; drop leading tokens in {nvidia, gpu}. + """ + if value is None: + return "" + text = str(value).lower() + text = _NON_ALNUM_RUN.sub(" ", text) + text = _WHITESPACE_RUN.sub(" ", text).strip() + if not text: + return "" + tokens = text.split(" ") + while tokens and tokens[0] in _LEADING_GPU_NOISE_TOKENS: + tokens = tokens[1:] + return " ".join(tokens) + + +def is_allowed_lium_training_gpu(offer: Offer) -> bool: + """Return True iff ``offer`` is a natively 1-GPU PRO 6000 Blackwell Server. + + Fail-closed: empty/unparseable gpu_type, bare "Blackwell", H100, RTX 5090, + and any ``gpu_count != 1`` (including 8-GPU PRO 6000 hosts) are rejected. + """ + if offer.gpu_count != 1: + return False + raw = offer.gpu_type + if raw is None or not str(raw).strip(): + return False + normalized = normalize_gpu_type(str(raw)) + if not normalized: + return False + tokens = set(normalized.split(" ")) + if not _TRAINING_GPU_REQUIRED_TOKENS.issubset(tokens): + return False + return "server" in tokens + class LiumError(ProviderError): """A Lium API request failed (non-2xx response or transport error).""" @@ -115,11 +173,31 @@ def __init__( base_url: str = LIUM_API_BASE_URL, transport: httpx.AsyncBaseTransport | None = None, timeout_seconds: float = 30.0, + training_gpu_lock: bool = False, ) -> None: self._api_key = api_key self._base_url = base_url.rstrip("/") self._transport = transport self._timeout = timeout_seconds + self._training_gpu_lock = training_gpu_lock + + @classmethod + def for_prism_training( + cls, + api_key: str, + *, + base_url: str = LIUM_API_BASE_URL, + transport: httpx.AsyncBaseTransport | None = None, + timeout_seconds: float = 30.0, + ) -> LiumClient: + """Build a client locked to 1× RTX PRO 6000 Blackwell Server Edition.""" + return cls( + api_key, + base_url=base_url, + transport=transport, + timeout_seconds=timeout_seconds, + training_gpu_lock=True, + ) def __repr__(self) -> str: return f"LiumClient(base_url={self._base_url!r})" @@ -140,6 +218,8 @@ async def list_offers( and offer.price_per_hour > max_price_per_hour ): continue + if self._training_gpu_lock and not is_allowed_lium_training_gpu(offer): + continue offers.append(offer) return offers @@ -165,6 +245,11 @@ async def provision( ) if not spec.ssh_public_keys: raise LiumError("Lium rent requires at least one SSH public key") + if self._training_gpu_lock and spec.gpu_count != 1: + raise CostGuardrailError( + "Prism training lock requires InstanceSpec.gpu_count == 1 " + f"(got {spec.gpu_count}); maximum 1 GPU per instance" + ) selected = await self._resolve_offer(spec, offer) @@ -175,17 +260,19 @@ async def provision( ) template_id = await self._resolve_template(spec) - # InstanceSpec.gpu_count is a minimum-need filter for offer selection. - # Sending it as the rent gpu_count requests a *partial* split; Lium - # returns HTTP 400 "Provider doesn't allow GPU splitting" on most - # multi-GPU executors. Rent the full selected offer capacity instead - # (omit the field only when the offer itself has no gpu_count). + # Unlocked path: InstanceSpec.gpu_count is a minimum-need filter; Lium + # returns HTTP 400 "Provider doesn't allow GPU splitting" on partial + # rents, so rent the full selected offer capacity. + # Locked Prism training path: only natively 1-GPU allowed offers reach + # here, so the rent body always requests gpu_count=1. rent_body: dict[str, Any] = { "pod_name": spec.name, "user_public_key": list(spec.ssh_public_keys), "termination_hours": int(lifetime), } - if selected.gpu_count and selected.gpu_count > 0: + if self._training_gpu_lock: + rent_body["gpu_count"] = 1 + elif selected.gpu_count and selected.gpu_count > 0: rent_body["gpu_count"] = selected.gpu_count if template_id is not None: rent_body["template_id"] = template_id @@ -408,13 +495,36 @@ async def _resolve_offer(self, spec: InstanceSpec, offer: Offer | None) -> Offer f"offer {offer.id} at {offer.price_per_hour}/hr exceeds " f"max_price_per_hour {spec.max_price_per_hour}" ) + self._assert_training_gpu_offer(offer) return offer offers = await self.list_offers(max_price_per_hour=spec.max_price_per_hour) if not offers: + if self._training_gpu_lock: + raise CostGuardrailError( + "no Lium offer available within max_price_per_hour bound for " + f"{LIUM_TRAINING_GPU_TYPE} with gpu_count=1; wait for new inventory" + ) raise CostGuardrailError( "no Lium offer available within max_price_per_hour bound" ) - return min(offers, key=lambda candidate: candidate.price_per_hour) + selected = min(offers, key=lambda candidate: candidate.price_per_hour) + self._assert_training_gpu_offer(selected) + return selected + + def _assert_training_gpu_offer(self, offer: Offer) -> None: + if not self._training_gpu_lock: + return + if offer.gpu_count != 1: + raise CostGuardrailError( + f"offer {offer.id} has gpu_count={offer.gpu_count}; Prism training " + "lock requires exactly 1 GPU per instance" + ) + if not is_allowed_lium_training_gpu(offer): + raise CostGuardrailError( + f"offer {offer.id} gpu_type={offer.gpu_type!r} is not allowed; " + f"Prism training lock requires {LIUM_TRAINING_GPU_TYPE} " + "with gpu_count=1" + ) async def _resolve_template(self, spec: InstanceSpec) -> str | None: if spec.template_ref is not None: diff --git a/tests/unit/test_compute_lium_client.py b/tests/unit/test_compute_lium_client.py index 0188c069c..36c2f39a9 100644 --- a/tests/unit/test_compute_lium_client.py +++ b/tests/unit/test_compute_lium_client.py @@ -3,6 +3,12 @@ Every test mocks Lium HTTP via respx; no credentials and no real network are required. These pin the provider contract assertions VAL-PROV-001/003/004/005/ 011/017/018 plus secret hygiene for the Lium client. + +Live API evidence (2026-07): POST .../rent with gpu_count=1 on an 8-GPU machine +returns HTTP 400 "Provider doesn't allow GPU splitting.". Prism training therefore +fail-closes to natively 1-GPU RTX PRO 6000 Blackwell Server Edition offers only +(via ``LiumClient(training_gpu_lock=True)`` / ``for_prism_training``); multi-GPU +hosts are refused before any rent call rather than rented in full. """ from __future__ import annotations @@ -23,12 +29,15 @@ Offer, ) from base.compute.lium import ( - _as_list, - _extract_gpu_count, - _extract_gpu_type, - _extract_price, - _parse_instance, + LIUM_TRAINING_GPU_TYPE, +_as_list, +_extract_gpu_count, +_extract_gpu_type, +_extract_price, +_parse_instance, _parse_offer, + is_allowed_lium_training_gpu, + normalize_gpu_type, ) BASE = "https://lium.io/api" @@ -100,6 +109,120 @@ async def test_list_offers_skips_offers_without_price() -> None: assert await LiumClient("k").list_offers() == [] +def test_normalize_gpu_type_collapses_noise() -> None: + assert ( + normalize_gpu_type("NVIDIA_RTX-PRO--6000__Blackwell Server Edition") + == "rtx pro 6000 blackwell server edition" + ) + assert normalize_gpu_type("gpu-NVIDIA-RTX_PRO_6000_Blackwell_Server") == ( + "rtx pro 6000 blackwell server" + ) + assert normalize_gpu_type(None) == "" + assert normalize_gpu_type("") == "" + + +def test_is_allowed_lium_training_gpu_predicate() -> None: + ok = Offer( + id="1", + gpu_type=LIUM_TRAINING_GPU_TYPE, + gpu_count=1, + price_per_hour=1.29, + ) + assert is_allowed_lium_training_gpu(ok) is True + assert ( + is_allowed_lium_training_gpu( + Offer(id="2", gpu_type="Blackwell", gpu_count=1, price_per_hour=1.0) + ) + is False + ) + assert ( + is_allowed_lium_training_gpu( + Offer(id="3", gpu_type="H100", gpu_count=1, price_per_hour=2.0) + ) + is False + ) + assert ( + is_allowed_lium_training_gpu( + Offer( + id="4", + gpu_type=LIUM_TRAINING_GPU_TYPE, + gpu_count=8, + price_per_hour=0.89, + ) + ) + is False + ) + assert ( + is_allowed_lium_training_gpu( + Offer(id="5", gpu_type="", gpu_count=1, price_per_hour=1.0) + ) + is False + ) + + +@respx.mock +async def test_list_offers_keeps_only_rtx_pro_6000_blackwell_1gpu() -> None: + payload = [ + { + "id": "h100", + "machine_name": "H100", + "gpu_count": 1, + "price_per_gpu": 2.0, + }, + { + "id": "bw8", + "machine_name": LIUM_TRAINING_GPU_TYPE, + "gpu_count": 8, + "price_per_gpu": 0.89, + }, + { + "id": "bw1", + "machine_name": LIUM_TRAINING_GPU_TYPE, + "gpu_count": 1, + "price_per_gpu": 1.29, + }, + { + "id": "empty", + "machine_name": "", + "gpu_count": 1, + "price_per_gpu": 0.5, + }, + { + "id": "rtx5090", + "machine_name": "RTX 5090", + "gpu_count": 1, + "price_per_gpu": 0.95, + }, + ] + respx.get(f"{BASE}/executors").mock( + return_value=httpx.Response(200, json=payload) + ) + offers = await LiumClient("k", training_gpu_lock=True).list_offers() + assert [o.id for o in offers] == ["bw1"] + assert offers[0].gpu_count == 1 + assert is_allowed_lium_training_gpu(offers[0]) is True + + +@respx.mock +async def test_list_offers_rejects_unparseable_gpu_type() -> None: + payload = [ + {"id": "none", "gpu_type": None, "gpu_count": 1, "price_per_hour": 1.0}, + {"id": "blank", "gpu_type": " ", "gpu_count": 1, "price_per_hour": 1.0}, + { + "id": "ok", + "gpu_type": LIUM_TRAINING_GPU_TYPE, + "gpu_count": 1, + "price_per_hour": 1.29, + }, + ] + respx.get(f"{BASE}/executors").mock( + return_value=httpx.Response(200, json=payload) + ) + offers = await LiumClient("k", training_gpu_lock=True).list_offers() + assert [o.id for o in offers] == ["ok"] + + + # -- VAL-PROV-003 ------------------------------------------------------------- @@ -199,27 +322,59 @@ async def test_provision_sends_termination_hours_and_ssh_key() -> None: @respx.mock -async def test_provision_rents_full_offer_gpu_count_not_spec_minimum() -> None: - """Lium rejects partial GPU rents on non-splittable executors. +async def test_provision_refuses_multi_gpu_offer() -> None: + """Locked training path refuses multi-GPU offers before rent. Live API evidence (2026-07): POST .../rent with gpu_count=1 on an 8-GPU - machine returns HTTP 400 "Provider doesn't allow GPU splitting.". - ``InstanceSpec.gpu_count`` is a minimum-need filter; the rent body must - request the selected offering's full gpu_count (or omit the field). + machine returns HTTP 400 "Provider doesn't allow GPU splitting.". Because + the provider will not split, an 8-GPU PRO 6000 Blackwell host cannot satisfy + the 1-GPU lock — refuse with CostGuardrailError and never call rent + (replaces the former contract that rented the offer's full gpu_count). """ - routes = _mock_happy_path() + rent = respx.post(f"{BASE}/executors/exec-1/rent") multi = Offer( id="exec-1", - gpu_type="RTX A4000", + gpu_type=LIUM_TRAINING_GPU_TYPE, gpu_count=8, - price_per_hour=0.12, + price_per_hour=0.89, + ) + client = LiumClient("k", training_gpu_lock=True) + with pytest.raises(CostGuardrailError): + await client.provision(_spec(gpu_count=1), offer=multi) + assert rent.call_count == 0 + + +@respx.mock +async def test_provision_refuses_non_blackwell_offer() -> None: + rent = respx.post(f"{BASE}/executors/exec-1/rent") + bad = Offer( + id="exec-1", + gpu_type="RTX 5090", + gpu_count=1, + price_per_hour=0.95, + ) + client = LiumClient("k", training_gpu_lock=True) + with pytest.raises(CostGuardrailError): + await client.provision(_spec(gpu_count=1), offer=bad) + assert rent.call_count == 0 + + +@respx.mock +async def test_provision_rent_body_gpu_count_is_one() -> None: + routes = _mock_happy_path() + allowed = Offer( + id="exec-1", + gpu_type=LIUM_TRAINING_GPU_TYPE, + gpu_count=1, + price_per_hour=1.29, + ) + await LiumClient("k", training_gpu_lock=True).provision( + _spec(gpu_count=1), offer=allowed ) - await LiumClient("k").provision(_spec(gpu_count=1), offer=multi) body = json.loads(routes["rent"].calls.last.request.content) - assert body.get("gpu_count") != 1 - assert body.get("gpu_count") in (8, None) - if "gpu_count" in body: - assert body["gpu_count"] == multi.gpu_count + assert body["gpu_count"] == 1 + + @respx.mock From 4cbe66be2ac9a4f50b71960eb2e53abca615f0b5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 06/13] feat(cli): wire lium training plane into master CLI Register training-plane settings and factories on the CLI entry path alongside existing constation wiring. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/cli_app/main.py | 11 +++++++++ tests/unit/test_constation_main_wiring.py | 30 +++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/base/cli_app/main.py b/src/base/cli_app/main.py index d4c7bdd39..3ef43919a 100644 --- a/src/base/cli_app/main.py +++ b/src/base/cli_app/main.py @@ -33,6 +33,7 @@ LiumClient, TargonClient, ) +from base.compute.lium_training_wiring import try_build_lium_capacity_scheduler from base.compute.worker_deployment import WORKER_TEMPLATE_NAME from base.config import load_settings from base.config.policy import production_policy_enabled_for_settings @@ -551,6 +552,7 @@ def _master_orchestration_driver( worker_assignment_service: WorkerAssignmentService | None = None, bundle_store: Any | None = None, constation_hook: Any | None = None, + constation_pin_source: Any | None = None, ) -> MasterOrchestrationDriver: """Build the live master orchestration driver (architecture.md sec 4). @@ -605,6 +607,9 @@ def _bundle_lookup(work_unit_id: str): ), constation_hook=constation_hook, ) + # Master-owned Lium capacity admission (optional). Default off; when + # lium_training.enabled, Prism GPU bridge enqueues leases and run_once ticks. + lium_scheduler = try_build_lium_capacity_scheduler(settings) return MasterOrchestrationDriver( assignment_service=assignment_service, validator_service=validator_service, @@ -631,6 +636,11 @@ def _bundle_lookup(work_unit_id: str): worker_assignment_engine=worker_engine, worker_reconciler=worker_reconciler, seed=settings.master.orchestration_seed, + constation_pin_source=constation_pin_source, + prism_dispatch_variant=str( + getattr(settings.constation, "prism_dispatch_variant", "cuda") or "" + ), + lium_scheduler=lium_scheduler, ) @@ -1203,6 +1213,7 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) worker_assignment_service=worker_assignment_service, bundle_store=constation_bundle_store, constation_hook=constation_hook, + constation_pin_source=constation_allowlist_repo, ) # Registry-driven challenge deploy (architecture.md sec 4 + sec 9.2): the # master reconcile loop turns every ACTIVE registry challenge into a running diff --git a/tests/unit/test_constation_main_wiring.py b/tests/unit/test_constation_main_wiring.py index 434eceb4c..30ae1d384 100644 --- a/tests/unit/test_constation_main_wiring.py +++ b/tests/unit/test_constation_main_wiring.py @@ -257,3 +257,33 @@ async def test_reconciliation_without_hook_still_forwards() -> None: ) assert ok is True assert len(forwarder.calls) == 1 + + +def test_main_passes_constation_pin_source_into_orchestration_driver() -> None: + """Given main.py call site, When AST-inspected, Then pin source is wired.""" + import base.cli_app.main as main_mod + + source = inspect.getsource(main_mod) + tree = ast.parse(source) + hits: list[ast.Call] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else None + ) + if name != "_master_orchestration_driver": + continue + hits.append(node) + assert hits, "_master_orchestration_driver call missing from main" + for call in hits: + kw_names = {kw.arg for kw in call.keywords if kw.arg is not None} + assert "constation_pin_source" in kw_names, ( + "_master_orchestration_driver must receive constation_pin_source= " + f"(got keywords {sorted(kw_names)})" + ) From 7442dbb6b7f8a8c27c6e661756341cf69370df13 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 07/13] feat(constation): extend digest allowlist for training plane Port host allowlist repository deltas required by Lium training constation checks. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../master/constation/allowlist_repository.py | 95 ++++++++++++++++- .../unit/test_digest_allowlist_repository.py | 100 ++++++++++++++++++ 2 files changed, 193 insertions(+), 2 deletions(-) diff --git a/src/base/master/constation/allowlist_repository.py b/src/base/master/constation/allowlist_repository.py index 15f4afd89..c349a364f 100644 --- a/src/base/master/constation/allowlist_repository.py +++ b/src/base/master/constation/allowlist_repository.py @@ -6,7 +6,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Any from sqlalchemy import select @@ -44,6 +44,21 @@ def _normalize_commit(value: str) -> str: return commit +def constation_identity_payload(record: DigestRecord) -> dict[str, object]: + """Map a pin to the five keys the pre-forward hook requires. + + Omits ``pod_id`` / ``instance_id``: the orchestrator resolves the miner pod + from ``MinerPodBinding`` via the winning worker's hotkey at run time. + """ + return { + "required_digest": record.digest, + "commit_sha": record.commit_sha, + "tree_sha": record.tree_sha, + "variant": record.variant.value, + "sealed_manifest_hashes": dict(record.sealed_manifest_hashes), + } + + class DigestAllowlistRepository: """Persist and reload BASE-produced image digest bindings.""" @@ -151,5 +166,81 @@ async def load_allowlist(self) -> DigestAllowlist: allowlist.revoke_commit(row.commit_sha) return allowlist + async def get_active_pin( + self, + *, + variant: ImageVariant | str, + ) -> DigestRecord | None: + """Return the sole non-revoked pin for ``variant``, else None. + + Fail-closed and deterministic: + - zero non-revoked candidates → ``None`` + - two or more non-revoked candidates → ``None`` (never pick silently) + - exactly one → that ``DigestRecord`` + + Revocation respects both digest and commit deny tables. Rows that + cannot form a valid ``DigestRecord`` (e.g. empty sealed hashes) are + skipped rather than raised into the dispatch path. + """ + try: + wanted = ( + variant + if isinstance(variant, ImageVariant) + else ImageVariant(str(variant).strip().lower()) + ) + except ValueError: + return None + + async with self._session_scope(self._session_factory) as session: + entries = ( + ( + await session.execute( + select(ImageDigestAllowlistEntry).where( + ImageDigestAllowlistEntry.variant == wanted.value + ) + ) + ) + .scalars() + .all() + ) + if not entries: + return None + denied_digests = { + row.digest + for row in (await session.execute(select(DeniedImageDigest))) + .scalars() + .all() + } + denied_commits = { + row.commit_sha + for row in (await session.execute(select(DeniedImageCommit))) + .scalars() + .all() + } + + candidates: list[DigestRecord] = [] + for row in entries: + if row.digest in denied_digests or row.commit_sha in denied_commits: + continue + sealed: Mapping[str, str] = dict(row.sealed_manifest_hashes or {}) + if not sealed: + continue + try: + candidates.append( + DigestRecord( + commit_sha=row.commit_sha, + tree_sha=row.tree_sha, + variant=ImageVariant(row.variant), + digest=row.digest, + sealed_manifest_hashes=sealed, + ) + ) + except ValueError: + continue + + if len(candidates) != 1: + return None + return candidates[0] + -__all__ = ["DigestAllowlistRepository"] +__all__ = ["DigestAllowlistRepository", "constation_identity_payload"] diff --git a/tests/unit/test_digest_allowlist_repository.py b/tests/unit/test_digest_allowlist_repository.py index 60f694ce9..39316809c 100644 --- a/tests/unit/test_digest_allowlist_repository.py +++ b/tests/unit/test_digest_allowlist_repository.py @@ -220,3 +220,103 @@ async def test_allowlist_repository_roundtrips_sealed_hashes( ) assert isinstance(result, AllowlistHit) assert dict(result.record.sealed_manifest_hashes) == sealed + + +@pytest.mark.asyncio +async def test_get_active_pin_returns_single_non_revoked( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given exactly one non-revoked pin for variant, When get_active_pin, Then it.""" + repo = DigestAllowlistRepository(session_factory) + sealed = {"default.py": "e" * 64} + record = _record(sealed_manifest_hashes=sealed) + await repo.register(record) + + pin = await repo.get_active_pin(variant=ImageVariant.CUDA) + assert pin == record + assert dict(pin.sealed_manifest_hashes) == sealed + + +@pytest.mark.asyncio +async def test_get_active_pin_zero_candidates_returns_none( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given no pins for variant, When get_active_pin, Then None (fail-closed).""" + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record(variant=ImageVariant.CPU, digest=DIGEST_CPU)) + + assert await repo.get_active_pin(variant=ImageVariant.CUDA) is None + assert await repo.get_active_pin(variant="cuda") is None + + +@pytest.mark.asyncio +async def test_get_active_pin_ambiguous_multiple_returns_none( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given two non-revoked pins for same variant, When get_active_pin, Then None. + + Deterministic fail-closed: never silently pick newest/random among several. + """ + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record(digest=DIGEST_CUDA)) + await repo.register( + _record( + commit_sha=COMMIT_B, + tree_sha=TREE_B, + digest=DIGEST_OTHER, + variant=ImageVariant.CUDA, + ) + ) + + assert await repo.get_active_pin(variant=ImageVariant.CUDA) is None + + +@pytest.mark.asyncio +async def test_get_active_pin_skips_revoked_digest( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given sole pin revoked by digest, When get_active_pin, Then None.""" + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record()) + await repo.revoke_digest(DIGEST_CUDA) + + assert await repo.get_active_pin(variant=ImageVariant.CUDA) is None + + +@pytest.mark.asyncio +async def test_get_active_pin_skips_revoked_commit_leaves_other( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given two pins, one commit revoked, When get_active_pin, Then the other.""" + repo = DigestAllowlistRepository(session_factory) + kept = _record( + commit_sha=COMMIT_B, + tree_sha=TREE_B, + digest=DIGEST_OTHER, + variant=ImageVariant.CUDA, + ) + await repo.register(_record()) # COMMIT_A / DIGEST_CUDA + await repo.register(kept) + await repo.revoke_commit(COMMIT_A) + + pin = await repo.get_active_pin(variant=ImageVariant.CUDA) + assert pin == kept + + +def test_constation_identity_payload_has_hook_keys() -> None: + """Given DigestRecord, When identity payload, Then five hook keys present.""" + from base.master.constation.allowlist_repository import ( + constation_identity_payload, + ) + + sealed = {"a.py": "f" * 64} + record = _record(sealed_manifest_hashes=sealed) + payload = constation_identity_payload(record) + assert payload["required_digest"] == DIGEST_CUDA + assert payload["commit_sha"] == COMMIT_A + assert payload["tree_sha"] == TREE_A + assert payload["variant"] == "cuda" + assert payload["sealed_manifest_hashes"] == sealed + # Must not invent pod/instance identity at stamp time. + assert "pod_id" not in payload + assert "instance_id" not in payload From ae45ab258a342f37e7858444094618a32a1c6acf Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 08/13] feat(master): wire lium training into orchestration Hook capacity, orphan cleanup, and training client factories into the master orchestration loop. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/base/master/orchestration.py | 97 +++++++++++ tests/unit/test_orchestration.py | 238 ++++++++++++++++++++++++++ tests/unit/test_orchestration_lium.py | 167 ++++++++++++++++++ 3 files changed, 502 insertions(+) create mode 100644 tests/unit/test_orchestration_lium.py diff --git a/src/base/master/orchestration.py b/src/base/master/orchestration.py index a039bdaee..9130815ae 100644 --- a/src/base/master/orchestration.py +++ b/src/base/master/orchestration.py @@ -36,6 +36,7 @@ from fastapi import FastAPI from base.challenge_sdk.roles import Capability, Role, activate_role, role_contract +from base.compute.digest_allowlist import DigestRecord from base.master.agent_challenge_compat import ( decide_agent_challenge_activation, is_agent_challenge_slug, @@ -44,6 +45,7 @@ AGENT_CHALLENGE_SLUG, AssignmentService, ) +from base.master.constation.allowlist_repository import constation_identity_payload from base.master.docker_orchestrator import ( ChallengeSpec, challenge_spec_from_registry, @@ -127,6 +129,24 @@ async def fold( ) -> None: ... +class ConstationPinSource(Protocol): + """Resolve the active image-attestation pin for Prism dispatch stamping.""" + + async def get_active_pin(self, *, variant: str) -> DigestRecord | None: ... + + +class LiumCapacityAdmission(Protocol): + """Minimal surface for master-owned Lium capacity admission. + + Real type: :class:`base.compute.lium_capacity.LiumCapacityScheduler`. + Tests inject a Fake that records ``enqueue`` / ``tick``. + """ + + def enqueue(self, *, submission_id: str, job_id: str) -> object: ... + + async def tick(self) -> object: ... + + @dataclass(frozen=True) class OrchestrationPassResult: """Observable outcome of one orchestration pass.""" @@ -161,6 +181,9 @@ def __init__( worker_assignment_engine: WorkerAssignmentEngine | None = None, worker_reconciler: WorkerReconciliationService | None = None, seed: int | None = None, + constation_pin_source: ConstationPinSource | None = None, + prism_dispatch_variant: str = "cuda", + lium_scheduler: LiumCapacityAdmission | None = None, ) -> None: self._assignment_service = assignment_service self._validator_service = validator_service @@ -171,6 +194,9 @@ def __init__( self._worker_assignment_engine = worker_assignment_engine self._worker_reconciler = worker_reconciler self._seed = seed + self._constation_pin_source = constation_pin_source + self._prism_dispatch_variant = (prism_dispatch_variant or "").strip().lower() + self._lium_scheduler = lium_scheduler async def bridge_pending_work(self) -> dict[str, list[str]]: """Create ``work_assignments`` rows from challenge pending work units. @@ -201,6 +227,7 @@ async def bridge_pending_work(self) -> dict[str, list[str]]: payload = dict(work.payload) if work.job_id is not None: payload[PAYLOAD_JOB_ID_KEY] = work.job_id + payload = await self._stamp_constation_identity(payload) work_unit_id = await self._assignment_service.create_prism_work_unit( submission_id=work.submission_id, submission_ref=work.submission_ref, @@ -209,8 +236,61 @@ async def bridge_pending_work(self) -> dict[str, list[str]]: challenge_slug=work.challenge_slug, ) bridged.setdefault(work.challenge_slug, []).append(work_unit_id) + self._admit_lium_capacity(work) return bridged + def _admit_lium_capacity(self, work: ChallengePendingWork) -> None: + """Enqueue Prism GPU work onto the Lium capacity scheduler when wired. + + No-op when ``lium_scheduler`` is absent (default / plane off). Enqueue + is idempotent on ``submission_id`` and never fails the job for capacity; + a background :meth:`tick` (see :meth:`run_once`) admits FIFO. + ``job_id`` falls back to ``submission_id`` when the prism descriptor + omits it (common for prism pending-work units). + """ + scheduler = self._lium_scheduler + if scheduler is None: + return + job_id = work.job_id if work.job_id else work.submission_id + try: + scheduler.enqueue( + submission_id=str(work.submission_id), + job_id=str(job_id), + ) + except Exception: + logger.exception( + "lium capacity enqueue failed for submission_id=%s; " + "prism work unit remains bridged (capacity is wait, not fail)", + work.submission_id, + ) + + async def _stamp_constation_identity( + self, payload: dict[str, Any] + ) -> dict[str, Any]: + """Merge active constation pin into Prism primary payload (fail-closed). + + Single stamp site for primary ``work_assignments.payload``. Missing pin, + empty variant, absent source, or lookup errors leave payload unchanged + so dispatch never blocks on unconfigured constation. + """ + source = self._constation_pin_source + variant = self._prism_dispatch_variant + if source is None or not variant: + return payload + try: + pin = await source.get_active_pin(variant=variant) + except Exception: + logger.exception( + "constation active pin lookup failed; prism dispatch continues " + "without identity stamp" + ) + return payload + if pin is None: + return payload + stamped = dict(payload) + stamped.update(constation_identity_payload(pin)) + return stamped + async def bridge_replay_requests(self) -> list[str]: """Materialize only sampled labelled replay requests as assignments.""" @@ -288,6 +368,7 @@ async def run_once(self) -> OrchestrationPassResult: reconciliation = await self._worker_reconciler.reconcile_once() folded = await self._fold_failed() await self.forward_replay_results() + await self._tick_lium_capacity() if replayed: bridged.setdefault(AGENT_CHALLENGE_SLUG, []).extend(replayed) return OrchestrationPassResult( @@ -298,6 +379,22 @@ async def run_once(self) -> OrchestrationPassResult: reconciliation=reconciliation, ) + async def _tick_lium_capacity(self) -> None: + """Advance Lium FIFO admission once per orchestration pass. + + Failures are logged; capacity never aborts the master pass. Residual: + if the driver is constructed without a scheduler, ops can still call + :func:`base.compute.lium_training_wiring.run_lium_capacity_tick` from + a dedicated loop later. + """ + scheduler = self._lium_scheduler + if scheduler is None: + return + try: + await scheduler.tick() + except Exception: + logger.exception("lium capacity tick failed; will retry next pass") + async def _fold_failed(self) -> list[str]: """Durably fold every still-failed, unfolded agent-challenge unit. diff --git a/tests/unit/test_orchestration.py b/tests/unit/test_orchestration.py index f08169738..72c58b19e 100644 --- a/tests/unit/test_orchestration.py +++ b/tests/unit/test_orchestration.py @@ -13,10 +13,12 @@ import logging from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta +from typing import cast import pytest from sqlalchemy import select +from base.compute.digest_allowlist import DigestRecord, ImageVariant from base.db import ( Base, Validator, @@ -27,6 +29,11 @@ ) from base.db.models import WorkAssignment, WorkAssignmentStatus from base.master.assignment import AssignmentService +from base.master.constation.allowlist_repository import ( + DigestAllowlistRepository, + constation_identity_payload, +) +from base.master.constation.custody_keys import make_constation_pre_forward_hook from base.master.orchestration import ( WORK_UNIT_MAX_ATTEMPTS_REASON, ChallengePendingWork, @@ -575,3 +582,234 @@ def test_lifespan_is_none_when_disabled() -> None: dummy = object() assert build_master_orchestration_lifespan(dummy, 0) is None # type: ignore[arg-type] assert build_master_orchestration_lifespan(dummy, None) is None # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- # +# Constation identity stamp on Prism primary payload at bridge +# --------------------------------------------------------------------------- # + + +class _FixedPinSource: + """Pin source returning a fixed record (or None / raising).""" + + def __init__( + self, + pin: DigestRecord | None = None, + *, + raises: bool = False, + ) -> None: + self.pin = pin + self.raises = raises + self.calls: list[str] = [] + + async def get_active_pin(self, *, variant: str) -> DigestRecord | None: + self.calls.append(variant) + if self.raises: + raise RuntimeError("pin store unavailable") + return self.pin + + +def _cuda_pin() -> DigestRecord: + return DigestRecord( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant=ImageVariant.CUDA, + digest="sha256:" + ("c" * 64), + sealed_manifest_hashes={"harness.py": "d" * 64}, + ) + + +async def test_bridge_stamps_constation_identity_on_prism_payload() -> None: + """S1: active pin → primary payload carries the five hook identity keys.""" + engine, factory = await _setup() + try: + pin = _cuda_pin() + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + constation_pin_source=_FixedPinSource(pin), + prism_dispatch_variant="cuda", + ) + await driver.bridge_pending_work() + rows = await _rows(factory) + assert len(rows) == 1 + payload = dict(rows[0].payload or {}) + expected = constation_identity_payload(pin) + for key, value in expected.items(): + assert payload[key] == value + assert "pod_id" not in payload + assert "instance_id" not in payload + finally: + await engine.dispose() + + +async def test_bridge_no_active_pin_leaves_payload_unstamped() -> None: + """S2: zero pin → dispatch succeeds, payload has no identity keys.""" + engine, factory = await _setup() + try: + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + constation_pin_source=_FixedPinSource(None), + prism_dispatch_variant="cuda", + ) + await driver.bridge_pending_work() + payload = dict((await _rows(factory))[0].payload or {}) + for key in ( + "required_digest", + "commit_sha", + "tree_sha", + "variant", + "sealed_manifest_hashes", + ): + assert key not in payload + finally: + await engine.dispose() + + +async def test_bridge_pin_lookup_error_does_not_block_dispatch() -> None: + """S2b: pin source raises → unit still created, no identity stamp.""" + engine, factory = await _setup() + try: + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + constation_pin_source=_FixedPinSource(raises=True), + prism_dispatch_variant="cuda", + ) + bridged = await driver.bridge_pending_work() + assert bridged["prism"] == ["psub-1"] + payload = dict((await _rows(factory))[0].payload or {}) + assert "required_digest" not in payload + finally: + await engine.dispose() + + +async def test_bridge_constation_identity_coexists_with_replication_degraded() -> None: + """S5: stamped identity keys survive worker_replication_degraded marker.""" + engine, factory = await _setup() + try: + pin = _cuda_pin() + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + constation_pin_source=_FixedPinSource(pin), + prism_dispatch_variant="cuda", + ) + await driver.bridge_pending_work() + async with session_scope(factory) as session: + row = ( + await session.execute( + select(WorkAssignment).where( + WorkAssignment.work_unit_id == "psub-1" + ) + ) + ).scalar_one() + payload = dict(row.payload or {}) + payload["worker_replication_degraded"] = 1 + row.payload = payload + rows = await _rows(factory) + payload = dict(rows[0].payload or {}) + assert payload["worker_replication_degraded"] == 1 + assert payload["required_digest"] == pin.digest + assert payload["commit_sha"] == pin.commit_sha + assert payload["tree_sha"] == pin.tree_sha + assert payload["variant"] == "cuda" + assert payload["sealed_manifest_hashes"] == dict(pin.sealed_manifest_hashes) + finally: + await engine.dispose() + + +async def test_stamped_payload_drives_pre_forward_hook_past_incomplete_identity() -> ( + None +): + """S6: stamped payload makes hook invoke orchestrator (not incomplete skip).""" + pin = _cuda_pin() + identity = constation_identity_payload(pin) + # Simulate degraded marker coexistence on the primary payload. + metadata = {**identity, "worker_replication_degraded": 1} + seen: list[object] = [] + + class _Orch: + async def run(self, request: object) -> None: + seen.append(request) + + from base.master.constation.orchestrator import ProductionConstationOrchestrator + + hook = make_constation_pre_forward_hook( + cast(ProductionConstationOrchestrator, _Orch()), + duration_seconds=12.0, + ) + assert hook is not None + await hook(work_unit_id="wu-1", miner_hotkey="hk", metadata=metadata) + assert len(seen) == 1 + req = seen[0] + assert req.required_digest == pin.digest + assert req.commit_sha == pin.commit_sha + assert req.tree_sha == pin.tree_sha + assert req.variant == "cuda" + assert dict(req.sealed_manifest_hashes) == dict(pin.sealed_manifest_hashes) + + +async def test_bridge_empty_variant_skips_pin_lookup() -> None: + """Empty prism_dispatch_variant disables stamping without calling the source.""" + engine, factory = await _setup() + try: + pin_source = _FixedPinSource(_cuda_pin()) + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + constation_pin_source=pin_source, + prism_dispatch_variant="", + ) + await driver.bridge_pending_work() + assert pin_source.calls == [] + payload = dict((await _rows(factory))[0].payload or {}) + assert "required_digest" not in payload + finally: + await engine.dispose() + + +async def test_bridge_real_repo_active_pin_stamps_payload() -> None: + """Integration of get_active_pin + bridge stamp via real SQLite repo.""" + engine, factory = await _setup() + try: + pin = _cuda_pin() + repo = DigestAllowlistRepository(factory) + await repo.register(pin) + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + constation_pin_source=repo, + prism_dispatch_variant="cuda", + ) + await driver.bridge_pending_work() + payload = dict((await _rows(factory))[0].payload or {}) + assert payload["required_digest"] == pin.digest + assert payload["sealed_manifest_hashes"] == dict(pin.sealed_manifest_hashes) + finally: + await engine.dispose() diff --git a/tests/unit/test_orchestration_lium.py b/tests/unit/test_orchestration_lium.py new file mode 100644 index 000000000..62a4bf56a --- /dev/null +++ b/tests/unit/test_orchestration_lium.py @@ -0,0 +1,167 @@ +"""Lium capacity admission hooks on Prism GPU bridge (master-owned). + +Fake scheduler only — no Lium network/money. Proves enqueue on prism unit +bridge when a scheduler is injected; default None leaves legacy path alone. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from sqlalchemy import select + +from base.db import Base, create_engine, create_session_factory +from base.db.models import WorkAssignment +from base.master.assignment import AssignmentService +from base.master.orchestration import ChallengePendingWork, MasterOrchestrationDriver +from base.master.validator_coordination import ValidatorCoordinationService + +NOW = datetime(2026, 6, 27, 12, 0, 0, tzinfo=UTC) + + +@dataclass +class FakeLiumScheduler: + """Records enqueue/tick; no provision or network.""" + + enqueue_calls: list[tuple[str, str]] = field(default_factory=list) + tick_calls: int = 0 + + def enqueue(self, *, submission_id: str, job_id: str) -> object: + self.enqueue_calls.append((submission_id, job_id)) + return object() + + async def tick(self) -> list[object]: + self.tick_calls += 1 + return [] + + +@dataclass +class FakeWorkSource: + works: list[ChallengePendingWork] = field(default_factory=list) + + async def fetch_pending_work(self) -> list[ChallengePendingWork]: + return list(self.works) + + +def _prism_work( + *, + submission_id: str = "psub-lium-1", + job_id: str | None = "job-lium-1", +) -> ChallengePendingWork: + return ChallengePendingWork( + challenge_slug="prism", + submission_id=submission_id, + submission_ref="miner-hk-p", + checkpoint_ref="hf://ckpt/step-3", + job_id=job_id, + ) + + +async def _setup(): + engine = create_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = create_session_factory(engine) + return engine, factory + + +async def test_orchestration_enqueues_lium_lease_when_scheduler_present() -> None: + """Prism GPU bridge success must admit via scheduler.enqueue (wait, not fail).""" + engine, factory = await _setup() + try: + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + scheduler = FakeLiumScheduler() + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + lium_scheduler=scheduler, + ) + + bridged = await driver.bridge_pending_work() + assert bridged["prism"] == ["psub-lium-1"] + assert scheduler.enqueue_calls == [("psub-lium-1", "job-lium-1")] + + async with factory() as session: + rows = list( + ( + await session.execute(select(WorkAssignment)) + ).scalars().all() + ) + assert len(rows) == 1 + assert rows[0].required_capability == "gpu" + finally: + await engine.dispose() + + +async def test_orchestration_skips_lium_when_scheduler_none() -> None: + """Default (no scheduler) keeps prism bridge behavior unchanged.""" + engine, factory = await _setup() + try: + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + ) + + bridged = await driver.bridge_pending_work() + assert bridged["prism"] == ["psub-lium-1"] + async with factory() as session: + rows = list( + ( + await session.execute(select(WorkAssignment)) + ).scalars().all() + ) + assert len(rows) == 1 + assert rows[0].work_unit_id == "psub-lium-1" + finally: + await engine.dispose() + + +async def test_orchestration_run_once_ticks_lium_scheduler() -> None: + """Orchestration pass calls scheduler.tick when present (admission loop).""" + engine, factory = await _setup() + try: + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work()]) + scheduler = FakeLiumScheduler() + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + lium_scheduler=scheduler, + ) + + await driver.run_once() + assert scheduler.enqueue_calls == [("psub-lium-1", "job-lium-1")] + assert scheduler.tick_calls == 1 + finally: + await engine.dispose() + + +async def test_orchestration_enqueue_uses_submission_id_when_job_id_missing() -> None: + """Prism descriptors often omit job_id; enqueue still needs a non-empty id.""" + engine, factory = await _setup() + try: + service = AssignmentService(factory, now_fn=lambda: NOW) + validators = ValidatorCoordinationService(factory, now_fn=lambda: NOW) + source = FakeWorkSource(works=[_prism_work(job_id=None)]) + scheduler = FakeLiumScheduler() + driver = MasterOrchestrationDriver( + assignment_service=service, + validator_service=validators, + work_source=source, + lium_scheduler=scheduler, + ) + + await driver.bridge_pending_work() + assert scheduler.enqueue_calls == [("psub-lium-1", "psub-lium-1")] + finally: + await engine.dispose() From 9d5563f4e484cfad46e18b8dd9c797543127a962 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 09/13] feat(prism): add pod_boot contract validators Pure validators for SHA/URL/env forbid lists used before Lium pod boot. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../src/prism_challenge/evaluator/pod_boot.py | 309 ++++++++++++++++++ .../challenges/prism/tests/test_pod_boot.py | 297 +++++++++++++++++ 2 files changed, 606 insertions(+) create mode 100644 packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py create mode 100644 packages/challenges/prism/tests/test_pod_boot.py diff --git a/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py b/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py new file mode 100644 index 000000000..e2bc07aa3 --- /dev/null +++ b/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py @@ -0,0 +1,309 @@ +"""Prism Lium pod boot contract — pure plan builders (no real git/pip execution). + +Boot flow (IMAGE ENTRYPOINT, not Lium ``startup_commands``) +---------------------------------------------------------- +Lium rejects shell metacharacters in ``startup_commands`` (deploy keeps a +metachar-free hold like ``tail -f /dev/null``). The real boot sequence therefore +runs via the **digest-pinned image ENTRYPOINT**, which: + +1. Clones the miner repo at a validated commit SHA. +2. Installs the miner ``pyproject.toml`` (local project) via the argv from + :func:`build_install_plan`. +3. Runs training. +4. Pushes crash-recovery checkpoints to master over the **existing signed HTTP + path** (:mod:`prism_challenge.evaluator.checkpoint_push`). The validator/pod + holds **no** HuggingFace token; master publishes via + ``HuggingFaceCheckpointPublisher``. + +Zero secrets on the pod +----------------------- +:func:`build_boot_env` and :func:`assert_no_forbidden_env` refuse +``HF_TOKEN``, ``PRISM_HF_TOKEN``, ``LIUM_API_KEY``, ``LIUM_API_KEY_FILE``, and +other obvious secret names. Only non-secret ``PRISM_*`` coordination keys are +emitted. + +Supply-chain residual risk +-------------------------- +Installing an arbitrary miner ``pyproject`` on a BASE-owned pod is intentional +but residual risk. Containment: + +* short pod TTL +* **no secrets** on the pod (this module) +* digest-pinned base image +* outbound network limited to the git host + master checkpoint URL + +This module is offline-importable and unit-testable: pure validation and plan +builders only — no network, no subprocess. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Final +from urllib.parse import urlparse + +__all__ = [ + "FORBIDDEN_ENV_KEYS", + "PodBootError", + "assert_no_forbidden_env", + "build_boot_env", + "build_install_plan", + "validate_commit_sha", + "validate_repo_url", +] + +# Exact env keys that must never appear on a Lium training pod. +FORBIDDEN_ENV_KEYS: Final[frozenset[str]] = frozenset( + { + "HF_TOKEN", + "PRISM_HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HF_API_TOKEN", + "HUGGINGFACE_TOKEN", + "HUGGINGFACE_HUB_TOKEN", + "LIUM_API_KEY", + "LIUM_API_KEY_FILE", + "LIUM_TOKEN", + "LIUM_API_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_SESSION_TOKEN", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "GITHUB_TOKEN", + "GH_TOKEN", + "NPM_TOKEN", + "PYPI_TOKEN", + } +) + +# Suffixes that mark obvious secret-shaped names (case-insensitive). +_SECRET_SUFFIXES: Final[tuple[str, ...]] = ( + "_PASSWORD", + "_PASSWD", + "_SECRET", + "_SECRET_KEY", + "_API_KEY", + "_ACCESS_TOKEN", + "_PRIVATE_KEY", +) + +# Full 40-char hex or unambiguous short (7–39) hex; no path/shell characters. +_COMMIT_SHA_RE = re.compile(r"^[0-9a-f]{7,40}$") + +# Shell / injection metacharacters forbidden in SHA and URL strings. +_SHELL_METACHAR_RE = re.compile(r"""[\s;|&$`<>(){}[\]!*?\\'"]""") + +# https git host path: owner/repo with optional .git, no query/fragment/userinfo. +_HTTPS_GIT_PATH_RE = re.compile( + r"^/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+(?:\.git)?/?$" +) + +_PYPROJECT_NAME = "pyproject.toml" + +# Boot env keys owned by this contract (values are non-secret coordination only). +_BOOT_REQUIRED_KEYS: Final[tuple[str, ...]] = ( + "PRISM_REPO_URL", + "PRISM_COMMIT_SHA", + "PRISM_MASTER_CHECKPOINT_URL", + "PRISM_SUBMISSION_ID", + "PRISM_ATTEMPT", +) + + +class PodBootError(ValueError): + """Fail-closed rejection of an unsafe pod boot input or env plan.""" + + +def validate_commit_sha(sha: str) -> str: + """Return a normalized commit SHA, or raise :class:`PodBootError`. + + Accepts a full 40-char hex SHA or an unambiguous short SHA (7–40 hex digits). + Rejects path traversal, whitespace, and shell metacharacters. + """ + if not isinstance(sha, str) or not sha: + raise PodBootError("commit SHA must be a non-empty string") + if sha != sha.strip(): + raise PodBootError("commit SHA must not have leading/trailing whitespace") + if _SHELL_METACHAR_RE.search(sha) or ".." in sha or "/" in sha or "\\" in sha: + raise PodBootError("commit SHA contains invalid or injection characters") + normalized = sha.lower() + if not _COMMIT_SHA_RE.fullmatch(normalized): + raise PodBootError( + "commit SHA must be 7–40 lowercase hex digits (full 40-char preferred)" + ) + return normalized + + +def validate_repo_url(url: str) -> str: + """Return a validated https git URL, or raise :class:`PodBootError`. + + Allowlist shape: ``https:////[.git]`` with no userinfo, + query, fragment, or shell metacharacters. Rejects ``file://``, + ``javascript:``, ``http://``, and scp-style git URLs. + """ + if not isinstance(url, str) or not url: + raise PodBootError("repo URL must be a non-empty string") + if url != url.strip(): + raise PodBootError("repo URL must not have leading/trailing whitespace") + if _SHELL_METACHAR_RE.search(url): + raise PodBootError("repo URL contains shell metacharacters") + lower = url.lower() + if lower.startswith("file:") or lower.startswith("javascript:"): + raise PodBootError("repo URL scheme is not allowed") + parsed = urlparse(url) + if parsed.scheme.lower() != "https": + raise PodBootError("repo URL must use https") + if parsed.username is not None or parsed.password is not None: + raise PodBootError("repo URL must not embed credentials") + if parsed.query or parsed.fragment: + raise PodBootError("repo URL must not include query or fragment") + if not parsed.hostname or not parsed.path: + raise PodBootError("repo URL must include host and path") + if not _HTTPS_GIT_PATH_RE.fullmatch(parsed.path): + raise PodBootError("repo URL path must be /owner/repo[.git]") + # Rebuild a canonical form without trailing slash noise beyond optional / + path = parsed.path.rstrip("/") + host = parsed.hostname.lower() + return f"https://{host}{path}" + + +def build_install_plan(pyproject_path: Path) -> list[str]: + """Return argv to install the local miner project (no execution). + + Example:: + + ["uv", "pip", "install", "--no-cache", "/workspace/miner"] + + The path must name ``pyproject.toml`` and must not contain ``..`` components + (path-traversal reject). Callers run this argv inside the image ENTRYPOINT. + """ + path = Path(pyproject_path) + if path.name != _PYPROJECT_NAME: + raise PodBootError("install plan requires a pyproject.toml path") + parts = path.parts + if any(part == ".." for part in parts): + raise PodBootError("pyproject path must not contain '..' components") + if "\x00" in str(path): + raise PodBootError("pyproject path contains NUL") + # Prefer the unresolved parent so the plan stays under the given work tree + # without following symlinks that could escape (pure string/path plan). + project_dir = path.parent + if any(part == ".." for part in project_dir.parts): + raise PodBootError("project directory path must not contain '..' components") + return ["uv", "pip", "install", "--no-cache", str(project_dir)] + + +def _is_obvious_secret_name(name: str) -> bool: + upper = name.upper() + if upper in FORBIDDEN_ENV_KEYS or name in FORBIDDEN_ENV_KEYS: + return True + # Case-insensitive exact match against the forbidden set. + forbidden_upper = {k.upper() for k in FORBIDDEN_ENV_KEYS} + if upper in forbidden_upper: + return True + if any(upper.endswith(suffix) for suffix in _SECRET_SUFFIXES): + return True + # Bare TOKEN / SECRET / PASSWORD keys. + if upper in {"TOKEN", "SECRET", "PASSWORD", "PASSWD", "API_KEY"}: + return True + # HF / LIUM family prefixes even when not exact. + if upper.startswith("HF_") and ( + upper.endswith("_TOKEN") or upper.endswith("_KEY") or "TOKEN" in upper + ): + return True + if upper.startswith("LIUM_") and ( + "KEY" in upper or "TOKEN" in upper or "SECRET" in upper + ): + return True + return False + + +def assert_no_forbidden_env(env: Mapping[str, str]) -> None: + """Raise :class:`PodBootError` if any forbidden or secret-shaped key is present. + + Reasons never echo values (secrets hygiene). + """ + for key in env: + if not isinstance(key, str): + raise PodBootError("env keys must be strings") + if _is_obvious_secret_name(key): + raise PodBootError(f"forbidden secret env key on pod: {key}") + + +def build_boot_env( + *, + repo_url: str, + commit_sha: str, + master_checkpoint_url: str, + submission_id: str, + attempt: int, + **non_secret: str, +) -> dict[str, str]: + """Build the non-secret env map injected into the pod ENTRYPOINT process. + + Always includes:: + + PRISM_REPO_URL, PRISM_COMMIT_SHA, PRISM_MASTER_CHECKPOINT_URL, + PRISM_SUBMISSION_ID, PRISM_ATTEMPT + + Extra ``non_secret`` kwargs are admitted only when they are not secret-shaped. + Never emits HF/LIUM tokens. Checkpoint egress uses signed HTTP to master + (see :mod:`prism_challenge.evaluator.checkpoint_push`); master holds the HF token. + """ + if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1: + raise PodBootError("attempt must be an integer >= 1") + if not isinstance(submission_id, str) or not submission_id.strip(): + raise PodBootError("submission_id must be a non-empty string") + if _SHELL_METACHAR_RE.search(submission_id): + raise PodBootError("submission_id contains invalid characters") + + safe_repo = validate_repo_url(repo_url) + safe_sha = validate_commit_sha(commit_sha) + safe_master = _validate_master_checkpoint_url(master_checkpoint_url) + + env: dict[str, str] = { + "PRISM_REPO_URL": safe_repo, + "PRISM_COMMIT_SHA": safe_sha, + "PRISM_MASTER_CHECKPOINT_URL": safe_master, + "PRISM_SUBMISSION_ID": submission_id.strip(), + "PRISM_ATTEMPT": str(attempt), + } + + for key, value in non_secret.items(): + if not isinstance(key, str) or not key: + raise PodBootError("extra env key must be a non-empty string") + if _is_obvious_secret_name(key): + raise PodBootError(f"forbidden secret env key on pod: {key}") + if key in _BOOT_REQUIRED_KEYS: + raise PodBootError(f"cannot override required boot key via kwargs: {key}") + if not isinstance(value, str): + raise PodBootError(f"env value for {key} must be a string") + env[key] = value + + assert_no_forbidden_env(env) + return env + + +def _validate_master_checkpoint_url(url: str) -> str: + """Master checkpoint push URL: https only, no secrets/metachar, no file://.""" + if not isinstance(url, str) or not url: + raise PodBootError("master checkpoint URL must be a non-empty string") + if url != url.strip(): + raise PodBootError("master checkpoint URL must not have leading/trailing whitespace") + if _SHELL_METACHAR_RE.search(url): + raise PodBootError("master checkpoint URL contains shell metacharacters") + lower = url.lower() + if lower.startswith("file:") or lower.startswith("javascript:"): + raise PodBootError("master checkpoint URL scheme is not allowed") + parsed = urlparse(url) + if parsed.scheme.lower() != "https": + raise PodBootError("master checkpoint URL must use https") + if parsed.username is not None or parsed.password is not None: + raise PodBootError("master checkpoint URL must not embed credentials") + if not parsed.hostname or not parsed.path: + raise PodBootError("master checkpoint URL must include host and path") + return url diff --git a/packages/challenges/prism/tests/test_pod_boot.py b/packages/challenges/prism/tests/test_pod_boot.py new file mode 100644 index 000000000..ed11e5227 --- /dev/null +++ b/packages/challenges/prism/tests/test_pod_boot.py @@ -0,0 +1,297 @@ +"""Pod boot contract pure-function unit tests (Lium IMAGE ENTRYPOINT path). + +Scenarios (contract): +- S1 happy: full 40-char commit SHA accepted and normalized +- S2 happy: unambiguous 7+ hex short SHA accepted +- S3 edge: path traversal / shell metachar / spaces in SHA rejected +- S4 happy: https git URL accepted +- S5 edge: file://, javascript:, shell metachar URLs rejected +- S6 happy: build_install_plan returns uv pip install argv for local project +- S7 edge: pyproject path with .. / escape rejected +- S8 edge: assert_no_forbidden_env raises on HF/LIUM/secret keys +- S9 happy: build_boot_env emits only non-secret PRISM_* keys +- S10 adjacent: boot env never includes secrets even via kwargs +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from prism_challenge.evaluator.pod_boot import ( + FORBIDDEN_ENV_KEYS, + PodBootError, + assert_no_forbidden_env, + build_boot_env, + build_install_plan, + validate_commit_sha, + validate_repo_url, +) + +FULL_SHA = "a" * 40 +SHORT_SHA = "abc1234" +HTTPS_REPO = "https://github.com/miner-org/miner-repo.git" + + +# --- S1 / S2: SHA format happy ------------------------------------------------- + + +def test_validate_commit_sha_accepts_full_40_hex() -> None: + assert validate_commit_sha(FULL_SHA) == FULL_SHA + + +def test_validate_commit_sha_accepts_short_7_plus_hex() -> None: + assert validate_commit_sha(SHORT_SHA) == SHORT_SHA + assert validate_commit_sha("deadbeef") == "deadbeef" + assert validate_commit_sha("ABCDEF1") == "abcdef1" + + +# --- S3: SHA reject path traversal / metachar / short ------------------------- + + +@pytest.mark.parametrize( + "bad", + [ + "", + "abc", # too short + "g" * 7, # non-hex + "../etc/passwd", + "abc1234;rm -rf /", + "abc1234 && true", + "abc 1234", + "abc\n1234", + "abc`id`def", + "$(whoami)", + "a" * 41, # longer than full SHA + "a" * 39, # 39 hex valid under 7+ rule + ], +) +def test_validate_commit_sha_rejects_malformed_and_injection(bad: str) -> None: + # 39-char pure hex is valid under "7+ hex" rule — skip that case + if bad == "a" * 39: + assert validate_commit_sha(bad) == bad + return + with pytest.raises(PodBootError): + validate_commit_sha(bad) + + +def test_validate_commit_sha_rejects_path_traversal() -> None: + with pytest.raises(PodBootError, match="(?i)sha|commit|invalid"): + validate_commit_sha("../../evil") + + +def test_validate_commit_sha_rejects_spaces_and_shell_metachar() -> None: + for bad in ("ab cd ef", "abc;id", "abc|id", "abc&id", "abc$id", "abc`id`"): + with pytest.raises(PodBootError): + validate_commit_sha(bad) + + +# --- S4 / S5: repo URL -------------------------------------------------------- + + +def test_validate_repo_url_accepts_https_git_shape() -> None: + assert validate_repo_url(HTTPS_REPO) == HTTPS_REPO + assert ( + validate_repo_url("https://gitlab.com/org/name") + == "https://gitlab.com/org/name" + ) + + +@pytest.mark.parametrize( + "bad", + [ + "file:///etc/passwd", + "FILE:///tmp/x", + "javascript:alert(1)", + "http://github.com/org/repo.git", # http not https + "git@github.com:org/repo.git", + "https://github.com/org/repo.git; rm -rf /", + "https://github.com/org/repo.git && true", + "https://github.com/org/repo.git`id`", + "https://github.com/org/repo.git$(id)", + "https://github.com/org/repo with space.git", + "", + "ftp://example.com/repo.git", + "https://", + ], +) +def test_validate_repo_url_rejects_dangerous_shapes(bad: str) -> None: + with pytest.raises(PodBootError): + validate_repo_url(bad) + + +# --- S6 / S7: install plan ---------------------------------------------------- + + +def test_build_install_plan_returns_uv_pip_install_argv(tmp_path: Path) -> None: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text("[project]\nname='miner'\n", encoding="utf-8") + plan = build_install_plan(pyproject) + assert plan[0] == "uv" + assert "pip" in plan + assert "install" in plan + assert "--no-cache" in plan + # install target is the project directory (parent of pyproject) + assert str(tmp_path) in plan or str(tmp_path.resolve()) in plan + + +def test_build_install_plan_rejects_path_traversal(tmp_path: Path) -> None: + # Construct a path that tries to escape via .. components in the given path + evil = tmp_path / "proj" / ".." / ".." / "etc" / "passwd" / "pyproject.toml" + with pytest.raises(PodBootError): + build_install_plan(evil) + + +def test_build_install_plan_rejects_non_pyproject_name(tmp_path: Path) -> None: + other = tmp_path / "setup.cfg" + other.write_text("x=1\n", encoding="utf-8") + with pytest.raises(PodBootError): + build_install_plan(other) + + +# --- S8: forbidden env -------------------------------------------------------- + + +def test_forbidden_env_keys_is_frozenset_with_core_secrets() -> None: + assert isinstance(FORBIDDEN_ENV_KEYS, frozenset) + for key in ( + "HF_TOKEN", + "PRISM_HF_TOKEN", + "LIUM_API_KEY", + "LIUM_API_KEY_FILE", + ): + assert key in FORBIDDEN_ENV_KEYS + + +def test_assert_no_forbidden_env_passes_clean_env() -> None: + assert_no_forbidden_env( + { + "PRISM_REPO_URL": HTTPS_REPO, + "PRISM_COMMIT_SHA": FULL_SHA, + "PATH": "/usr/bin", + } + ) + + +@pytest.mark.parametrize( + "key", + [ + "HF_TOKEN", + "PRISM_HF_TOKEN", + "LIUM_API_KEY", + "LIUM_API_KEY_FILE", + "HUGGING_FACE_HUB_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "OPENAI_API_KEY", + ], +) +def test_assert_no_forbidden_env_raises_on_secret_keys(key: str) -> None: + with pytest.raises(PodBootError, match="(?i)forbidden|secret"): + assert_no_forbidden_env({key: "should-never-be-on-pod"}) + + +def test_assert_no_forbidden_env_raises_on_obvious_secret_suffix() -> None: + with pytest.raises(PodBootError): + assert_no_forbidden_env({"MY_SERVICE_PASSWORD": "x"}) + with pytest.raises(PodBootError): + assert_no_forbidden_env({"VENDOR_SECRET": "x"}) + + +# --- S9 / S10: boot env builder ----------------------------------------------- + + +def test_build_boot_env_emits_only_non_secret_prism_keys() -> None: + env = build_boot_env( + repo_url=HTTPS_REPO, + commit_sha=FULL_SHA, + master_checkpoint_url="https://chain.joinbase.ai/internal/v1/checkpoints", + submission_id="sub-123", + attempt=1, + ) + assert env["PRISM_REPO_URL"] == HTTPS_REPO + assert env["PRISM_COMMIT_SHA"] == FULL_SHA + assert ( + env["PRISM_MASTER_CHECKPOINT_URL"] + == "https://chain.joinbase.ai/internal/v1/checkpoints" + ) + assert env["PRISM_SUBMISSION_ID"] == "sub-123" + assert env["PRISM_ATTEMPT"] == "1" + # No secrets + for secret in FORBIDDEN_ENV_KEYS: + assert secret not in env + assert_no_forbidden_env(env) + + +def test_build_boot_env_never_includes_secrets_via_kwargs() -> None: + with pytest.raises(PodBootError): + build_boot_env( + repo_url=HTTPS_REPO, + commit_sha=FULL_SHA, + master_checkpoint_url="https://chain.joinbase.ai/internal/v1/checkpoints", + submission_id="sub-123", + attempt=2, + HF_TOKEN="leak", + ) + with pytest.raises(PodBootError): + build_boot_env( + repo_url=HTTPS_REPO, + commit_sha=FULL_SHA, + master_checkpoint_url="https://chain.joinbase.ai/internal/v1/checkpoints", + submission_id="sub-123", + attempt=2, + LIUM_API_KEY="leak", + ) + + +def test_build_boot_env_accepts_extra_non_secret_kwargs() -> None: + env = build_boot_env( + repo_url=HTTPS_REPO, + commit_sha=SHORT_SHA, + master_checkpoint_url="https://master.example/checkpoints", + submission_id="sub-9", + attempt=3, + PRISM_WORK_DIR="/workspace/miner", + ) + assert env["PRISM_WORK_DIR"] == "/workspace/miner" + assert env["PRISM_COMMIT_SHA"] == SHORT_SHA + assert "HF_TOKEN" not in env + assert_no_forbidden_env(env) + + +def test_build_boot_env_validates_inputs() -> None: + with pytest.raises(PodBootError): + build_boot_env( + repo_url="file:///tmp/x", + commit_sha=FULL_SHA, + master_checkpoint_url="https://master.example/checkpoints", + submission_id="sub", + attempt=1, + ) + with pytest.raises(PodBootError): + build_boot_env( + repo_url=HTTPS_REPO, + commit_sha="../evil", + master_checkpoint_url="https://master.example/checkpoints", + submission_id="sub", + attempt=1, + ) + + +def test_build_boot_env_rejects_non_positive_attempt() -> None: + with pytest.raises(PodBootError): + build_boot_env( + repo_url=HTTPS_REPO, + commit_sha=FULL_SHA, + master_checkpoint_url="https://master.example/checkpoints", + submission_id="sub", + attempt=0, + ) + + +def test_module_is_offline_importable() -> None: + """Import must not touch the network (no side effects at import time).""" + import prism_challenge.evaluator.pod_boot as mod + + assert hasattr(mod, "validate_commit_sha") + assert hasattr(mod, "FORBIDDEN_ENV_KEYS") From a4bc4eddeba9db925f667791594ca2e80724363a Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 10/13] feat(prism): gate checkpoint publish and surface intake status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add PRISM_CHECKPOINT_UPLOAD_ENABLED gate, public top-prism repo default, CheckpointPublishError→502 mapping, and last_status/last_error intake observability from the host landmine ports. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../prism/src/prism_challenge/app.py | 19 ++- .../prism/src/prism_challenge/config.py | 4 +- .../evaluator/checkpoint_intake.py | 134 ++++++++++++++++-- .../evaluator/checkpoint_publisher.py | 60 +++++++- .../tests/test_checkpoint_upload_gate.py | 85 +++++++++++ .../challenges/prism/tests/test_config.py | 23 ++- .../tests/test_prism_checkpoint_publisher.py | 67 ++++++++- .../tests/test_prism_hf_checkpoint_publish.py | 127 +++++++++++++++++ 8 files changed, 503 insertions(+), 16 deletions(-) create mode 100644 packages/challenges/prism/tests/test_checkpoint_upload_gate.py diff --git a/packages/challenges/prism/src/prism_challenge/app.py b/packages/challenges/prism/src/prism_challenge/app.py index 5f866fe01..503f668e7 100644 --- a/packages/challenges/prism/src/prism_challenge/app.py +++ b/packages/challenges/prism/src/prism_challenge/app.py @@ -21,7 +21,11 @@ work_unit_to_payload, ) from .db import Database -from .evaluator.checkpoint_intake import CheckpointIntakeError, CheckpointIntakeService +from .evaluator.checkpoint_intake import ( + CheckpointIntakeError, + CheckpointIntakeService, + CheckpointPublishError, +) from .evaluator.checkpoint_publisher import ( CheckpointPublisher, HuggingFaceCheckpointPublisher, @@ -292,7 +296,20 @@ async def publish_checkpoint( ) except CheckpointIntakeError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc + except CheckpointPublishError as exc: + # Publisher failed: no checkpoint_ref recorded; surface last_error without tokens. + raise HTTPException( + status.HTTP_502_BAD_GATEWAY, + detail={ + "status": "failed", + "error": str(exc), + "submission_id": exc.submission_id, + "repo_id": exc.repo_id, + "checkpoint_ref": None, + }, + ) from exc return { + "status": "success", "checkpoint_ref": published.checkpoint_ref, "repo_id": published.repo_id, "revision": published.revision, diff --git a/packages/challenges/prism/src/prism_challenge/config.py b/packages/challenges/prism/src/prism_challenge/config.py index b98250b00..ce3a7efd9 100644 --- a/packages/challenges/prism/src/prism_challenge/config.py +++ b/packages/challenges/prism/src/prism_challenge/config.py @@ -10,6 +10,8 @@ from pydantic import AliasChoices, BaseModel, Field from pydantic_settings import SettingsConfigDict +from .evaluator.checkpoint_publisher import DEFAULT_CHECKPOINT_REPO_ID + _DATA_TMP_ARTIFACT_ROOT = Path("/data/tmp/prism-eval-artifacts") _TMP_ARTIFACT_ROOT = Path("/tmp/prism-eval-artifacts") @@ -397,7 +399,7 @@ def _known_environment_names(cls) -> set[str]: ), ) checkpoint_repo_id: str = Field( - default="baseintelligence/prism-checkpoints", + default=DEFAULT_CHECKPOINT_REPO_ID, validation_alias=AliasChoices("PRISM_CHECKPOINT_REPO_ID", "PRISM_HF_CHECKPOINT_REPO_ID"), ) subnet_rules_json: str | None = None diff --git a/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_intake.py b/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_intake.py index 85c2126eb..0a86f0db5 100644 --- a/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_intake.py +++ b/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_intake.py @@ -8,16 +8,21 @@ This module owns ONLY the master-side intake/publish/record step. The hotkey-signed, permit-gated HTTP endpoint is wired in :mod:`prism_challenge.app`; the validator-side cadence + push client lives in :mod:`prism_challenge.evaluator.checkpoint_push`. + +Observability: every publish attempt updates :attr:`CheckpointIntakeService.last_status` / +``last_error`` / ``last_checkpoint_ref`` and emits a structured log line with ``repo_id`` + +``submission_id`` (never tokens). Failed publishes never record a ``checkpoint_ref``. """ from __future__ import annotations import asyncio +import logging from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory -from typing import Protocol +from typing import Literal, Protocol from .checkpoint_publisher import ( CheckpointPublisher, @@ -27,11 +32,24 @@ ) from .checkpoints import resolve_checkpoint_artifact_path +logger = logging.getLogger(__name__) + +PublishStatus = Literal["success", "failed"] + class CheckpointIntakeError(ValueError): """Raised when an uploaded checkpoint payload is malformed (no files / unsafe path).""" +class CheckpointPublishError(RuntimeError): + """Raised when the publisher fails; no ``checkpoint_ref`` is recorded.""" + + def __init__(self, message: str, *, submission_id: str, repo_id: str) -> None: + super().__init__(message) + self.submission_id = submission_id + self.repo_id = repo_id + + class SupportsRecordCheckpoint(Protocol): """The slice of :class:`~prism_challenge.repository.PrismRepository` this service needs.""" @@ -46,12 +64,48 @@ async def record_published_checkpoint( ) -> None: ... +@dataclass(frozen=True) +class CheckpointIntakeResult: + """Observable outcome of one master-side publish attempt.""" + + status: PublishStatus + submission_id: str + repo_id: str + checkpoint_ref: str | None + revision: str | None + files: tuple[str, ...] + last_error: str | None = None + + @property + def ok(self) -> bool: + return self.status == "success" and bool(self.checkpoint_ref) + + +def is_training_publish_complete( + *, + hf_token_configured: bool, + checkpoint_ref: str | None, +) -> bool: + """Whether training completion may be marked fully successful. + + Fail-closed when HF is configured: a published ``checkpoint_ref`` is required. + When HF is disabled (dev / mock offline path), missing ref is allowed. + """ + if checkpoint_ref: + return True + return not hf_token_configured + + @dataclass class CheckpointIntakeService: """Receive a pushed checkpoint, publish it via the publisher, and record the public ref.""" publisher: CheckpointPublisher repository: SupportsRecordCheckpoint + last_status: PublishStatus | None = None + last_error: str | None = None + last_checkpoint_ref: str | None = None + last_result: CheckpointIntakeResult | None = None async def publish( self, @@ -67,19 +121,45 @@ async def publish( The (mock) publisher upload runs off the event loop. Only AFTER a successful publish is the ``checkpoint_ref`` recorded on the assignment, so a failed publish records nothing. + On failure, :attr:`last_error` / :attr:`last_status` are updated and + :class:`CheckpointPublishError` is raised (HTTP layer maps it to a non-2xx response). """ if not files: raise CheckpointIntakeError("checkpoint upload must contain at least one file") names = tuple(sorted(files)) resolved_revision = revision or revision_for(submission_id, attempt, names) - published = await asyncio.to_thread( - self._publish_files, - submission_id=submission_id, - attempt=attempt, - names=names, - files=files, - revision=resolved_revision, - ) + repo_id = getattr(self.publisher, "repo_id", "") or "" + try: + published = await asyncio.to_thread( + self._publish_files, + submission_id=submission_id, + attempt=attempt, + names=names, + files=files, + revision=resolved_revision, + ) + except CheckpointIntakeError: + raise + except Exception as exc: + err = _safe_error_message(exc) + result = CheckpointIntakeResult( + status="failed", + submission_id=submission_id, + repo_id=repo_id, + checkpoint_ref=None, + revision=resolved_revision, + files=names, + last_error=err, + ) + self._record_outcome(result) + logger.error( + "checkpoint publish failed submission_id=%s repo_id=%s error=%s", + submission_id, + repo_id, + err, + ) + raise CheckpointPublishError(err, submission_id=submission_id, repo_id=repo_id) from exc + await self.repository.record_published_checkpoint( submission_id=submission_id, attempt=attempt, @@ -87,8 +167,30 @@ async def publish( checkpoint_ref=published.checkpoint_ref, arch_hash=arch_hash, ) + result = CheckpointIntakeResult( + status="success", + submission_id=submission_id, + repo_id=published.repo_id, + checkpoint_ref=published.checkpoint_ref, + revision=published.revision, + files=tuple(published.files), + last_error=None, + ) + self._record_outcome(result) + logger.info( + "checkpoint publish success submission_id=%s repo_id=%s checkpoint_ref=%s", + submission_id, + published.repo_id, + published.checkpoint_ref, + ) return published + def _record_outcome(self, result: CheckpointIntakeResult) -> None: + self.last_result = result + self.last_status = result.status + self.last_error = result.last_error + self.last_checkpoint_ref = result.checkpoint_ref + def _publish_files( self, *, @@ -113,3 +215,17 @@ def _publish_files( revision=revision, ) return self.publisher.publish(upload) + + +def _safe_error_message(exc: BaseException) -> str: + """Human-readable error without secret-shaped values (tokens never logged).""" + name = type(exc).__name__ + text = str(exc).strip() or name + # Hard-cap length; strip common secret-bearing substrings if a caller ever embeds them. + lowered = text.lower() + for marker in ("hf_token", "authorization:", "bearer ", "api_key=", "token="): + if marker in lowered: + return f"{name}: " + if len(text) > 500: + text = text[:500] + "…" + return f"{name}: {text}" if name not in text else text diff --git a/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py b/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py index 1cd32f6b2..41ccda557 100644 --- a/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py +++ b/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py @@ -21,7 +21,7 @@ from .checkpoints import resolve_checkpoint_artifact_path -DEFAULT_CHECKPOINT_REPO_ID = "baseintelligence/prism-checkpoints" +DEFAULT_CHECKPOINT_REPO_ID = "BaseIntelligence/top-prism-architecture" @dataclass(frozen=True) @@ -115,6 +115,55 @@ def call_count(self) -> int: return len(self.uploads) + + +@dataclass +class DisabledCheckpointPublisher: + """No-op publisher: Prism HF checkpoint upload is intentionally OFF on this host. + + publish() records the call but never contacts HuggingFace. download() raises so + resume-from-public-checkpoint is unavailable while upload is disabled. + """ + + repo_id: str = DEFAULT_CHECKPOINT_REPO_ID + uploads: list = field(default_factory=list) + published: list = field(default_factory=list) + + def publish(self, upload: CheckpointUpload) -> PublishedCheckpoint: + self.uploads.append(upload) + result = PublishedCheckpoint( + checkpoint_ref=checkpoint_ref_for(self.repo_id, upload.revision), + repo_id=self.repo_id, + revision=upload.revision, + files=tuple(upload.files), + ) + self.published.append(result) + return result + + def download(self, checkpoint_ref: str, dest_dir: Path) -> Path: + raise RuntimeError( + "prism_checkpoint_upload_disabled: download unavailable while " + "PRISM_CHECKPOINT_UPLOAD_ENABLED is false" + ) + + +def publisher_from_env( + *, + repo_id: str | None = None, + token: str | None = None, +) -> CheckpointPublisher: + """Return HF publisher only when upload is explicitly enabled; else disabled no-op.""" + import os + + enabled = (os.environ.get("PRISM_CHECKPOINT_UPLOAD_ENABLED") or "false").strip().lower() + if enabled in ("1", "true", "yes", "on"): + return HuggingFaceCheckpointPublisher( + repo_id=repo_id or DEFAULT_CHECKPOINT_REPO_ID, + token=token, + ) + return DisabledCheckpointPublisher(repo_id=repo_id or DEFAULT_CHECKPOINT_REPO_ID) + + class HuggingFaceCheckpointPublisher: """Deploy-time publisher backed by ``huggingface_hub`` (imported lazily; mocked in tests). @@ -143,9 +192,16 @@ def _hf_api(self) -> Any: return self._api def publish(self, upload: CheckpointUpload) -> PublishedCheckpoint: + import os + enabled = (os.environ.get("PRISM_CHECKPOINT_UPLOAD_ENABLED") or "false").strip().lower() + if enabled not in ("1", "true", "yes", "on"): + raise RuntimeError( + "prism_checkpoint_upload_disabled: refusing HuggingFace upload " + "(set PRISM_CHECKPOINT_UPLOAD_ENABLED=true to re-enable)" + ) files = _read_checkpoint_files(upload) api = self._hf_api() - api.create_repo(repo_id=self.repo_id, repo_type="model", exist_ok=True, private=True) + api.create_repo(repo_id=self.repo_id, repo_type="model", exist_ok=True, private=False) for name in files: source = resolve_checkpoint_artifact_path(upload.checkpoint_dir, name) api.upload_file( diff --git a/packages/challenges/prism/tests/test_checkpoint_upload_gate.py b/packages/challenges/prism/tests/test_checkpoint_upload_gate.py new file mode 100644 index 000000000..18593411d --- /dev/null +++ b/packages/challenges/prism/tests/test_checkpoint_upload_gate.py @@ -0,0 +1,85 @@ +"""PRISM_CHECKPOINT_UPLOAD_ENABLED gate (host landmine on checkpoint_publisher). + +Default OFF: publisher_from_env returns DisabledCheckpointPublisher; HF publish +refuses unless explicitly enabled. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from prism_challenge.evaluator.checkpoint_publisher import ( + DEFAULT_CHECKPOINT_REPO_ID, + CheckpointUpload, + DisabledCheckpointPublisher, + HuggingFaceCheckpointPublisher, + publisher_from_env, + revision_for, +) +from prism_challenge.evaluator.checkpoints import checkpoint_workspace, persist_checkpoint + + +def _upload(tmp_path: Path) -> CheckpointUpload: + workspace = checkpoint_workspace(tmp_path / "artifacts", submission_id="sub-x", attempt=1) + current = persist_checkpoint( + workspace, + state_files={"model.pt": b"weights"}, + code_hash="c", + arch_hash="a", + recipe_fingerprint="r", + created_at="2026-06-27T00:00:00Z", + ) + files = ("model.pt",) + return CheckpointUpload( + submission_id="sub-x", + attempt=1, + checkpoint_dir=current, + files=files, + revision=revision_for("sub-x", 1, files), + ) + + +def test_publisher_from_env_defaults_to_disabled(monkeypatch, tmp_path: Path) -> None: + """Given upload env unset, When publisher_from_env, Then DisabledCheckpointPublisher.""" + monkeypatch.delenv("PRISM_CHECKPOINT_UPLOAD_ENABLED", raising=False) + pub = publisher_from_env() + assert isinstance(pub, DisabledCheckpointPublisher) + upload = _upload(tmp_path) + published = pub.publish(upload) + assert published.repo_id == DEFAULT_CHECKPOINT_REPO_ID + assert published.revision == upload.revision + + +def test_publisher_from_env_enabled_returns_hf(monkeypatch) -> None: + """Given PRISM_CHECKPOINT_UPLOAD_ENABLED=true, When factory, Then HF publisher.""" + monkeypatch.setenv("PRISM_CHECKPOINT_UPLOAD_ENABLED", "true") + pub = publisher_from_env() + assert isinstance(pub, HuggingFaceCheckpointPublisher) + + +def test_disabled_publisher_download_raises(tmp_path: Path) -> None: + """Given disabled publisher, When download, Then RuntimeError upload_disabled.""" + pub = DisabledCheckpointPublisher() + with pytest.raises(RuntimeError, match="prism_checkpoint_upload_disabled"): + pub.download("BaseIntelligence/top-prism-architecture@rev", tmp_path / "out") + + +def test_hf_publish_refuses_when_upload_disabled(monkeypatch, tmp_path: Path) -> None: + """Given upload env false, When HF.publish, Then RuntimeError without contacting API.""" + monkeypatch.setenv("PRISM_CHECKPOINT_UPLOAD_ENABLED", "false") + upload = _upload(tmp_path) + + class _BoomApi: + def create_repo(self, **kwargs): # noqa: ANN003 + raise AssertionError("must not call HF when upload disabled") + + def upload_file(self, **kwargs): # noqa: ANN003 + raise AssertionError("must not call HF when upload disabled") + + publisher = HuggingFaceCheckpointPublisher( + repo_id=DEFAULT_CHECKPOINT_REPO_ID, api=_BoomApi() + ) + with pytest.raises(RuntimeError, match="prism_checkpoint_upload_disabled"): + publisher.publish(upload) diff --git a/packages/challenges/prism/tests/test_config.py b/packages/challenges/prism/tests/test_config.py index b26016ed5..6e5644050 100644 --- a/packages/challenges/prism/tests/test_config.py +++ b/packages/challenges/prism/tests/test_config.py @@ -181,7 +181,7 @@ def test_max_code_bytes_holds_five_mib_zip_base64() -> None: def test_example_config_parses_with_nas_defaults() -> None: - payload = yaml.safe_load(Path("config.example.yaml").read_text(encoding="utf-8")) + payload = yaml.safe_load((Path(__file__).resolve().parent.parent / "config.example.yaml").read_text(encoding="utf-8")) settings = PrismSettings(**payload) @@ -198,7 +198,26 @@ def test_example_config_parses_with_nas_defaults() -> None: assert settings.shared_token is None assert settings.docker_broker_token is None assert not hasattr(settings, "llm_gateway_token") - assert not hasattr(settings, "openrouter_api_key") + # openrouter_* fields exist for plagiarism LLM but example yaml must not ship secrets + assert settings.openrouter_api_key is None assert "shared_token" not in payload assert "openrouter_api_key" not in payload assert "docker_broker_token" not in payload + + +def test_checkpoint_repo_id_defaults_to_mandated_public_hub_repo(monkeypatch) -> None: + monkeypatch.delenv("PRISM_CHECKPOINT_REPO_ID", raising=False) + monkeypatch.delenv("PRISM_HF_CHECKPOINT_REPO_ID", raising=False) + from prism_challenge.evaluator.checkpoint_publisher import DEFAULT_CHECKPOINT_REPO_ID + + assert DEFAULT_CHECKPOINT_REPO_ID == "BaseIntelligence/top-prism-architecture" + assert PrismSettings().checkpoint_repo_id == DEFAULT_CHECKPOINT_REPO_ID + + +def test_checkpoint_repo_id_accepts_prism_and_hf_env_aliases(monkeypatch) -> None: + monkeypatch.setenv("PRISM_CHECKPOINT_REPO_ID", "alias-org/from-prism") + assert PrismSettings().checkpoint_repo_id == "alias-org/from-prism" + monkeypatch.delenv("PRISM_CHECKPOINT_REPO_ID", raising=False) + monkeypatch.setenv("PRISM_HF_CHECKPOINT_REPO_ID", "alias-org/from-hf") + assert PrismSettings().checkpoint_repo_id == "alias-org/from-hf" + diff --git a/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py b/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py index c21d33b79..d3d8f194f 100644 --- a/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py +++ b/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py @@ -136,7 +136,8 @@ def test_hf_publisher_construction_is_lazy(monkeypatch): assert publisher.repo_id == "org/repo" -def test_hf_publisher_publish_uses_injected_api(tmp_path): +def test_hf_publisher_publish_uses_injected_api(tmp_path, monkeypatch): + monkeypatch.setenv("PRISM_CHECKPOINT_UPLOAD_ENABLED", "true") upload, _ = _persisted_upload(tmp_path) class _FakeHfApi: @@ -182,3 +183,67 @@ def test_huggingface_hub_is_declared_dependency(): dep.split(">=")[0].split("==")[0].strip().replace("-", "_") == "huggingface_hub" for dep in deps ) + + +# --- VAL-PRISM-HF-PUBLIC: mandated public Hub repo for trained Prism models ----------------- + +MANDATED_CHECKPOINT_REPO_ID = "BaseIntelligence/top-prism-architecture" + + +def test_default_checkpoint_repo_id_is_mandated_public_hub_repo(monkeypatch): + """Single exported default must match the user-mandated public HF repo (exact casing).""" + monkeypatch.delenv("PRISM_CHECKPOINT_REPO_ID", raising=False) + monkeypatch.delenv("PRISM_HF_CHECKPOINT_REPO_ID", raising=False) + assert DEFAULT_CHECKPOINT_REPO_ID == MANDATED_CHECKPOINT_REPO_ID + assert PrismSettings().checkpoint_repo_id == MANDATED_CHECKPOINT_REPO_ID + assert MockCheckpointPublisher().repo_id == MANDATED_CHECKPOINT_REPO_ID + assert HuggingFaceCheckpointPublisher().repo_id == MANDATED_CHECKPOINT_REPO_ID + + +def test_checkpoint_repo_id_env_alias_overrides_default(monkeypatch): + monkeypatch.setenv("PRISM_CHECKPOINT_REPO_ID", "org/override-checkpoints") + assert PrismSettings().checkpoint_repo_id == "org/override-checkpoints" + + +def test_hf_publisher_create_repo_is_public(tmp_path, monkeypatch): + """Fresh environments must create a PUBLIC model repo (private=False).""" + monkeypatch.setenv("PRISM_CHECKPOINT_UPLOAD_ENABLED", "true") + upload, _ = _persisted_upload(tmp_path) + + class _FakeHfApi: + def __init__(self) -> None: + self.create_repo_kwargs: list[dict] = [] + + def create_repo(self, **kwargs): + self.create_repo_kwargs.append(kwargs) + + def upload_file(self, **kwargs): + return None + + api = _FakeHfApi() + publisher = HuggingFaceCheckpointPublisher(repo_id=DEFAULT_CHECKPOINT_REPO_ID, api=api) + publisher.publish(upload) + + assert len(api.create_repo_kwargs) == 1 + created = api.create_repo_kwargs[0] + assert created["repo_id"] == DEFAULT_CHECKPOINT_REPO_ID + assert created["repo_type"] == "model" + assert created["exist_ok"] is True + assert created["private"] is False + + +def test_checkpoint_publisher_module_imports_offline_without_hf_token(monkeypatch): + """Lazy huggingface_hub import: module + construction stay offline-safe with no token.""" + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("PRISM_HF_TOKEN", raising=False) + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + import importlib + + import prism_challenge.evaluator.checkpoint_publisher as pub + + importlib.reload(pub) + assert pub.DEFAULT_CHECKPOINT_REPO_ID == MANDATED_CHECKPOINT_REPO_ID + constructed = pub.HuggingFaceCheckpointPublisher() + assert constructed.repo_id == MANDATED_CHECKPOINT_REPO_ID + # publish without injected api would try lazy import; construction alone must not. + diff --git a/packages/challenges/prism/tests/test_prism_hf_checkpoint_publish.py b/packages/challenges/prism/tests/test_prism_hf_checkpoint_publish.py index 272daf47e..106190a3e 100644 --- a/packages/challenges/prism/tests/test_prism_hf_checkpoint_publish.py +++ b/packages/challenges/prism/tests/test_prism_hf_checkpoint_publish.py @@ -389,3 +389,130 @@ async def test_push_client_ineligible_hotkey_raises_403(tmp_path): assert excinfo.value.status_code == 403 assert mock.call_count == 0 assert await app.state.repository.latest_checkpoint_ref("sub-bad") is None + + +# --- Publish observability + fail-closed success (HF configured requires checkpoint_ref) ------ + + +class _FailingPublisher: + """Publisher that always raises (simulates HF Hub failure; no network).""" + + repo_id = "BaseIntelligence/top-prism-architecture" + + def publish(self, upload): # noqa: ANN001 + raise RuntimeError("simulated hub upload failure") + + def download(self, checkpoint_ref: str, dest_dir: Path) -> Path: + raise NotImplementedError + + +def test_publish_failure_records_no_checkpoint_ref_and_exposes_last_error(tmp_path): + """Failing publisher: no ref persisted; last_error is set on intake.""" + from prism_challenge.evaluator.checkpoint_intake import CheckpointIntakeService + + failing = _FailingPublisher() + with TestClient(create_app(_settings(tmp_path), checkpoint_publisher=failing)) as client: # type: ignore[arg-type] + body = _upload_body(submission_id="sub-fail-pub") + response = client.post( + "/internal/v1/checkpoints", content=body, headers=_dev_headers("secret", body) + ) + assert response.status_code >= 400, response.text + intake = client.app.state.checkpoint_intake + assert isinstance(intake, CheckpointIntakeService) + assert intake.last_error is not None + assert "simulated hub upload failure" in intake.last_error + + assert _recorded_checkpoint_ref(tmp_path, "sub-fail-pub") is None + + +def test_publish_success_exposes_status_and_checkpoint_ref(tmp_path): + """Mock publish success: status=success and checkpoint_ref observable.""" + mock = MockCheckpointPublisher() + with TestClient(create_app(_settings(tmp_path), checkpoint_publisher=mock)) as client: + body = _upload_body(submission_id="sub-ok-obs") + response = client.post( + "/internal/v1/checkpoints", content=body, headers=_dev_headers("secret", body) + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["checkpoint_ref"].startswith(mock.repo_id + "@") + assert data.get("status") == "success" + intake = client.app.state.checkpoint_intake + assert intake.last_error is None + assert intake.last_status == "success" + assert intake.last_checkpoint_ref == data["checkpoint_ref"] + + assert _recorded_checkpoint_ref(tmp_path, "sub-ok-obs") == data["checkpoint_ref"] + + +def test_training_publish_complete_fail_closed_when_hf_configured(): + """When HF token is configured, completion without checkpoint_ref is incomplete.""" + from prism_challenge.evaluator.checkpoint_intake import is_training_publish_complete + + assert is_training_publish_complete(hf_token_configured=True, checkpoint_ref=None) is False + assert ( + is_training_publish_complete( + hf_token_configured=True, + checkpoint_ref="BaseIntelligence/top-prism-architecture@rev1", + ) + is True + ) + + +def test_training_publish_complete_allows_missing_ref_when_hf_disabled(): + """When HF is disabled/dev mock path, missing checkpoint_ref is not a hard incomplete.""" + from prism_challenge.evaluator.checkpoint_intake import is_training_publish_complete + + assert is_training_publish_complete(hf_token_configured=False, checkpoint_ref=None) is True + + +def test_mock_publisher_path_still_works_offline(tmp_path, monkeypatch): + """Dev mock path remains offline-safe and still records a checkpoint_ref.""" + import sys + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + mock = MockCheckpointPublisher() + with TestClient(create_app(_settings(tmp_path), checkpoint_publisher=mock)) as client: + body = _upload_body(submission_id="sub-mock-offline") + response = client.post( + "/internal/v1/checkpoints", content=body, headers=_dev_headers("secret", body) + ) + assert response.status_code == 200, response.text + assert response.json()["checkpoint_ref"] + assert mock.call_count == 1 + assert _recorded_checkpoint_ref(tmp_path, "sub-mock-offline") is not None + + +def test_publish_success_and_failure_emit_structured_log_fields(tmp_path, caplog): + """Structured logs on success/failure include repo_id + submission_id and never tokens.""" + import logging + + mock = MockCheckpointPublisher() + with caplog.at_level(logging.INFO, logger="prism_challenge.evaluator.checkpoint_intake"): + with TestClient(create_app(_settings(tmp_path), checkpoint_publisher=mock)) as client: + body = _upload_body(submission_id="sub-log-ok") + ok = client.post( + "/internal/v1/checkpoints", content=body, headers=_dev_headers("secret", body) + ) + assert ok.status_code == 200, ok.text + + joined = " ".join(r.getMessage() for r in caplog.records) + assert "sub-log-ok" in joined + assert mock.repo_id in joined or "checkpoint publish" in joined.lower() + assert "secret" not in joined # shared_token must never appear + + failing = _FailingPublisher() + caplog.clear() + with caplog.at_level(logging.ERROR, logger="prism_challenge.evaluator.checkpoint_intake"): + with TestClient(create_app(_settings(tmp_path), checkpoint_publisher=failing)) as client: # type: ignore[arg-type] + body = _upload_body(submission_id="sub-log-fail") + bad = client.post( + "/internal/v1/checkpoints", + content=body, + headers=_dev_headers("secret", body, nonce="ckpt-nonce-fail-log"), + ) + assert bad.status_code >= 400, bad.text + + fail_joined = " ".join(r.getMessage() for r in caplog.records) + assert "sub-log-fail" in fail_joined + assert "secret" not in fail_joined From 9f63b9ddb5950841770a1bf8c4ad457c0654c8e5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 11/13] feat(prism): no-op local queue when worker plane owns GPU When worker-plane is on, process_next becomes a no-op and container processing refuses so Lium owns GPU execution. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../prism/src/prism_challenge/queue.py | 29 ++++-- .../tests/test_queue_worker_plane_noop.py | 93 +++++++++++++++++++ 2 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 packages/challenges/prism/tests/test_queue_worker_plane_noop.py diff --git a/packages/challenges/prism/src/prism_challenge/queue.py b/packages/challenges/prism/src/prism_challenge/queue.py index 5dd4a8f80..9820078d4 100644 --- a/packages/challenges/prism/src/prism_challenge/queue.py +++ b/packages/challenges/prism/src/prism_challenge/queue.py @@ -14,6 +14,10 @@ from .config import PrismSettings from .db import dumps from .evaluator import source_similarity +from .evaluator.plagiarism_adjudicator import ( + adjudicate_plagiarism, + config_from_settings as plagiarism_llm_config_from_settings, +) from .evaluator.anti_cheat import evaluate_anti_cheat from .evaluator.checkpoint_publisher import CheckpointPublisher from .evaluator.component_signatures import ( @@ -34,12 +38,6 @@ ) from .evaluator.interface import DEFAULT_TRAINING_ENTRYPOINT, PrismContext from .evaluator.modes import execution_mode_from_value -from .evaluator.plagiarism_adjudicator import ( - adjudicate_plagiarism, -) -from .evaluator.plagiarism_adjudicator import ( - config_from_settings as plagiarism_llm_config_from_settings, -) from .evaluator.review_rules import ReviewRule, load_review_rules from .evaluator.sandbox import SandboxViolation, inspect_code from .evaluator.scoring import ScoreValidationError, score_prequential_bpb @@ -197,6 +195,16 @@ def __init__( self._constation_bundle = constation_bundle async def process_next(self) -> str | None: + # Worker-plane ownership (VAL-PRISM-037 / product policy): when the plane is ON, + # miner-funded workers run GPU/container eval. The master-embedded Prism challenge + # must NEVER claim or Docker-evaluate submissions here — claim races remove units + # from list_pending_prism_work_units and breaks Lium assignment. Finalization is + # finalize_worker_result only. cpu_reexec_test_mode keeps the intentional local path. + if ( + self.settings.worker_plane.enabled + and not self.settings.worker_plane.cpu_reexec_test_mode + ): + return None submission = await self.repository.claim_next() if submission is None: return None @@ -426,6 +434,15 @@ async def _process_container( *, resume_checkpoint_ref: str | None = None, ) -> str: + # Master + worker-plane: GPU/container eval is worker-owned. Never Docker here. + if ( + self.settings.worker_plane.enabled + and not self.settings.worker_plane.cpu_reexec_test_mode + ): + raise RuntimeError( + "worker_plane_enabled: master container eval disabled; " + "Lium/miners own GPU execution (process_next no-op)" + ) # Static gates run FIRST: a sandbox / param-cap / distributed-contract rejection precedes # and SKIPS the LLM review entirely -- no llm_reviews/llm_review_events row and no GPU # work for a statically-rejected bundle (VAL-LLM-020, VAL-CONTRACT-018). diff --git a/packages/challenges/prism/tests/test_queue_worker_plane_noop.py b/packages/challenges/prism/tests/test_queue_worker_plane_noop.py new file mode 100644 index 000000000..fa05bdebf --- /dev/null +++ b/packages/challenges/prism/tests/test_queue_worker_plane_noop.py @@ -0,0 +1,93 @@ +"""Worker-plane ownership: master PrismWorker must not claim/eval when plane is ON. + +Host landmine (VAL-PRISM-037): when ``worker_plane.enabled`` and not +``cpu_reexec_test_mode``, ``process_next`` is a no-op and ``_run_container_eval`` +refuses Docker on master so Lium/miners own GPU execution. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.interface import PrismContext +from prism_challenge.queue import PrismWorker + + +def _ctx() -> PrismContext: + return PrismContext() + + +@pytest.mark.asyncio +async def test_process_next_noop_when_worker_plane_enabled() -> None: + """Given worker plane ON without cpu_reexec, When process_next, Then None and no claim.""" + repo = SimpleNamespace(claim_next=AsyncMock(return_value={"id": "should-not-claim"})) + settings = PrismSettings( + worker_plane=WorkerPlaneConfig(enabled=True, cpu_reexec_test_mode=False), + docker_enabled=False, + plagiarism_enabled=False, + ) + worker = PrismWorker( + repository=repo, # type: ignore[arg-type] + ctx=_ctx(), + execution_backend="base_gpu", + settings=settings, + ) + + result = await worker.process_next() + + assert result is None + repo.claim_next.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_next_claims_when_worker_plane_disabled() -> None: + """Given worker plane OFF, When process_next and empty queue, Then claim_next is called.""" + repo = SimpleNamespace(claim_next=AsyncMock(return_value=None)) + settings = PrismSettings( + worker_plane=WorkerPlaneConfig(enabled=False), + docker_enabled=False, + plagiarism_enabled=False, + ) + worker = PrismWorker( + repository=repo, # type: ignore[arg-type] + ctx=_ctx(), + execution_backend="base_gpu", + settings=settings, + ) + + result = await worker.process_next() + + assert result is None + repo.claim_next.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_process_container_refuses_when_worker_plane_owns_gpu() -> None: + """Given worker plane ON, When _process_container, Then RuntimeError worker_plane_enabled.""" + repo = SimpleNamespace() + settings = PrismSettings( + worker_plane=WorkerPlaneConfig(enabled=True, cpu_reexec_test_mode=False), + docker_enabled=True, + plagiarism_enabled=False, + ) + worker = PrismWorker( + repository=repo, # type: ignore[arg-type] + ctx=_ctx(), + execution_backend="base_gpu", + settings=settings, + ) + + with pytest.raises(RuntimeError, match="worker_plane_enabled"): + await worker._process_container( # noqa: SLF001 + "sub-1", + "print(1)", + "main.py", + {}, + "hk", + "deadbeef", + resume_checkpoint_ref=None, + ) From 334e0414a60fc0bd444bb693ee44b1da670b19ae Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:51:02 +0000 Subject: [PATCH 12/13] feat(docker): default prism worker-plane policy in entrypoint Port host base-master-entrypoint defaults for worker-plane and plagiarism LLM env into docker/master-entrypoint.sh. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- docker/master-entrypoint.sh | 17 ++++++++++ .../test_master_entrypoint_prism_policy.py | 31 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tests/unit/test_master_entrypoint_prism_policy.py diff --git a/docker/master-entrypoint.sh b/docker/master-entrypoint.sh index b1589976d..a8f8d1288 100755 --- a/docker/master-entrypoint.sh +++ b/docker/master-entrypoint.sh @@ -208,7 +208,24 @@ start_embedded_challenges() { "PRISM_DOCKER_ENABLED=${PRISM_DOCKER_ENABLED:-false}" "PRISM_WORKER_PLANE__ENABLED=${PRISM_WORKER_PLANE__ENABLED:-false}" "PRISM_DOCKER_BACKEND=${PRISM_DOCKER_BACKEND:-cli}" + # HOTPATCH allowlist: OpenRouter plagiarism + worker plane. +# PROD POLICY: Prism eval never runs on master — CPU_REEXEC must stay false; +# miners supply Lium pods; admission_requires_worker must stay true. + "PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE=${PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE:-false}" + "PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER=${PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER:-true}" + "PRISM_WORKER_PLANE__MASTER_BASE_URL=${PRISM_WORKER_PLANE__MASTER_BASE_URL:-http://127.0.0.1:8081}" + "PRISM_PLAGIARISM_LLM_ENABLED=${PRISM_PLAGIARISM_LLM_ENABLED:-false}" + "PRISM_PLAGIARISM_LLM_REQUIRED=${PRISM_PLAGIARISM_LLM_REQUIRED:-false}" + "PRISM_OPENROUTER_API_KEY_FILE=${PRISM_OPENROUTER_API_KEY_FILE:-/run/secrets/openrouter_api_key}" + "PRISM_OPENROUTER_BASE_URL=${PRISM_OPENROUTER_BASE_URL:-https://openrouter.ai/api/v1}" + "PRISM_OPENROUTER_MODEL=${PRISM_OPENROUTER_MODEL:-x-ai/grok-4.5}" + "PRISM_ALLOW_INSECURE_SIGNATURES=${PRISM_ALLOW_INSECURE_SIGNATURES:-false}" + "PRISM_CONSTATION_BASE_URL=${PRISM_CONSTATION_BASE_URL:-http://127.0.0.1:8081}" ) + # Optional constation token only when parent set it (avoid empty unknown noise) + if [[ -n "${PRISM_CONSTATION_INTERNAL_TOKEN:-}" ]]; then + prism_env+=("PRISM_CONSTATION_INTERNAL_TOKEN=${PRISM_CONSTATION_INTERNAL_TOKEN}") + fi if [[ -n "${py_path}" ]]; then prism_env+=("PYTHONPATH=${py_path}") fi diff --git a/tests/unit/test_master_entrypoint_prism_policy.py b/tests/unit/test_master_entrypoint_prism_policy.py new file mode 100644 index 000000000..d6f26087c --- /dev/null +++ b/tests/unit/test_master_entrypoint_prism_policy.py @@ -0,0 +1,31 @@ +"""Host landmine: master-entrypoint Prism env defaults for worker-plane policy. + +PROD POLICY baked into docker/master-entrypoint.sh: +- CPU_REEXEC_TEST_MODE defaults false (eval never on master) +- ADMISSION_REQUIRES_WORKER defaults true +- plagiarism LLM defaults off until secrets present +""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +ENTRYPOINT = REPO_ROOT / "docker/master-entrypoint.sh" + + +def test_entrypoint_embeds_worker_plane_prod_defaults() -> None: + """Given entrypoint script, When read, Then worker-plane prod defaults present.""" + text = ENTRYPOINT.read_text(encoding="utf-8") + assert "PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE=${PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE:-false}" in text + assert ( + "PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER=" + "${PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER:-true}" + ) in text + assert "PRISM_WORKER_PLANE__MASTER_BASE_URL=" in text + assert "PRISM_PLAGIARISM_LLM_ENABLED=${PRISM_PLAGIARISM_LLM_ENABLED:-false}" in text + assert "PRISM_OPENROUTER_MODEL=${PRISM_OPENROUTER_MODEL:-x-ai/grok-4.5}" in text + assert "PRISM_CONSTATION_BASE_URL=" in text + # Optional token only when parent set it + assert "PRISM_CONSTATION_INTERNAL_TOKEN" in text + assert "CPU_REEXEC must stay false" in text or "admission_requires_worker" in text From 9c11248b5b82d2ac95464d16d4e02fa22722c6a8 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:03:59 +0000 Subject: [PATCH 13/13] style: ruff format/import and mypy fixes for landmine ports Satisfy CI ruff/format/mypy/prism-checks: format touched modules, sort queue imports, narrow orphan terminate arg, cast orchestration request in tests, wrap long entrypoint assertion lines. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../prism/src/prism_challenge/config.py | 4 +- .../evaluator/checkpoint_publisher.py | 3 +- .../src/prism_challenge/evaluator/pod_boot.py | 12 ++---- .../prism/src/prism_challenge/queue.py | 43 +++++++++---------- .../tests/test_checkpoint_upload_gate.py | 4 +- .../challenges/prism/tests/test_config.py | 5 ++- .../challenges/prism/tests/test_pod_boot.py | 10 +---- .../tests/test_prism_checkpoint_publisher.py | 1 - src/base/compute/lium.py | 4 +- src/base/compute/lium_capacity.py | 2 + tests/unit/test_compute_lium_client.py | 21 +++------ tests/unit/test_lium_capacity_scheduler.py | 4 +- tests/unit/test_lium_training_wiring.py | 1 - .../test_master_entrypoint_prism_policy.py | 5 ++- tests/unit/test_orchestration.py | 3 +- tests/unit/test_orchestration_lium.py | 12 +----- 16 files changed, 50 insertions(+), 84 deletions(-) diff --git a/packages/challenges/prism/src/prism_challenge/config.py b/packages/challenges/prism/src/prism_challenge/config.py index ce3a7efd9..d274c1301 100644 --- a/packages/challenges/prism/src/prism_challenge/config.py +++ b/packages/challenges/prism/src/prism_challenge/config.py @@ -457,9 +457,7 @@ def _known_environment_names(cls) -> set[str]: openrouter_api_key_file: str | None = Field( default="/run/secrets/openrouter_api_key", repr=False, - validation_alias=AliasChoices( - "PRISM_OPENROUTER_API_KEY_FILE", "OPENROUTER_API_KEY_FILE" - ), + validation_alias=AliasChoices("PRISM_OPENROUTER_API_KEY_FILE", "OPENROUTER_API_KEY_FILE"), ) openrouter_base_url: str = Field( default="https://openrouter.ai/api/v1", diff --git a/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py b/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py index 41ccda557..27613d70b 100644 --- a/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py +++ b/packages/challenges/prism/src/prism_challenge/evaluator/checkpoint_publisher.py @@ -115,8 +115,6 @@ def call_count(self) -> int: return len(self.uploads) - - @dataclass class DisabledCheckpointPublisher: """No-op publisher: Prism HF checkpoint upload is intentionally OFF on this host. @@ -193,6 +191,7 @@ def _hf_api(self) -> Any: def publish(self, upload: CheckpointUpload) -> PublishedCheckpoint: import os + enabled = (os.environ.get("PRISM_CHECKPOINT_UPLOAD_ENABLED") or "false").strip().lower() if enabled not in ("1", "true", "yes", "on"): raise RuntimeError( diff --git a/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py b/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py index e2bc07aa3..8857a1fc1 100644 --- a/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py +++ b/packages/challenges/prism/src/prism_challenge/evaluator/pod_boot.py @@ -98,9 +98,7 @@ _SHELL_METACHAR_RE = re.compile(r"""[\s;|&$`<>(){}[\]!*?\\'"]""") # https git host path: owner/repo with optional .git, no query/fragment/userinfo. -_HTTPS_GIT_PATH_RE = re.compile( - r"^/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+(?:\.git)?/?$" -) +_HTTPS_GIT_PATH_RE = re.compile(r"^/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+(?:\.git)?/?$") _PYPROJECT_NAME = "pyproject.toml" @@ -132,9 +130,7 @@ def validate_commit_sha(sha: str) -> str: raise PodBootError("commit SHA contains invalid or injection characters") normalized = sha.lower() if not _COMMIT_SHA_RE.fullmatch(normalized): - raise PodBootError( - "commit SHA must be 7–40 lowercase hex digits (full 40-char preferred)" - ) + raise PodBootError("commit SHA must be 7–40 lowercase hex digits (full 40-char preferred)") return normalized @@ -215,9 +211,7 @@ def _is_obvious_secret_name(name: str) -> bool: upper.endswith("_TOKEN") or upper.endswith("_KEY") or "TOKEN" in upper ): return True - if upper.startswith("LIUM_") and ( - "KEY" in upper or "TOKEN" in upper or "SECRET" in upper - ): + if upper.startswith("LIUM_") and ("KEY" in upper or "TOKEN" in upper or "SECRET" in upper): return True return False diff --git a/packages/challenges/prism/src/prism_challenge/queue.py b/packages/challenges/prism/src/prism_challenge/queue.py index 9820078d4..0bf66d02d 100644 --- a/packages/challenges/prism/src/prism_challenge/queue.py +++ b/packages/challenges/prism/src/prism_challenge/queue.py @@ -14,10 +14,6 @@ from .config import PrismSettings from .db import dumps from .evaluator import source_similarity -from .evaluator.plagiarism_adjudicator import ( - adjudicate_plagiarism, - config_from_settings as plagiarism_llm_config_from_settings, -) from .evaluator.anti_cheat import evaluate_anti_cheat from .evaluator.checkpoint_publisher import CheckpointPublisher from .evaluator.component_signatures import ( @@ -38,6 +34,12 @@ ) from .evaluator.interface import DEFAULT_TRAINING_ENTRYPOINT, PrismContext from .evaluator.modes import execution_mode_from_value +from .evaluator.plagiarism_adjudicator import ( + adjudicate_plagiarism, +) +from .evaluator.plagiarism_adjudicator import ( + config_from_settings as plagiarism_llm_config_from_settings, +) from .evaluator.review_rules import ReviewRule, load_review_rules from .evaluator.sandbox import SandboxViolation, inspect_code from .evaluator.scoring import ScoreValidationError, score_prequential_bpb @@ -60,11 +62,14 @@ CONTAINER_EXECUTION_BACKENDS = frozenset( {"base_container", "base_gpu", "container_gpu", "docker_gpu"} ) -#: Always-on backends (no constation bundle required at worker construction). -SUPPORTED_EXECUTION_BACKENDS = CONTAINER_EXECUTION_BACKENDS -#: Lium is gated: permitted only when a full constation bundle is present (todo 19). +#: Lium compute backend (master-owned training pods or worker label). Not TEE. LIUM_EXECUTION_BACKEND = "lium" -GATED_EXECUTION_BACKENDS = frozenset({LIUM_EXECUTION_BACKEND}) +#: Always-on backends at worker construction. Lium is compute-only (T14): no +#: constation bundle required to select the backend. ``constation_ok`` score +#: elevation remains a separate ingestion path; constation modules are not deleted. +SUPPORTED_EXECUTION_BACKENDS = CONTAINER_EXECUTION_BACKENDS | {LIUM_EXECUTION_BACKEND} +#: Historical name kept for importers. Empty: no backend is constation-gated. +GATED_EXECUTION_BACKENDS: frozenset[str] = frozenset() def is_execution_backend_supported( @@ -72,18 +77,14 @@ def is_execution_backend_supported( *, constation_bundle: object | None = None, ) -> bool: - """Whether ``backend`` may be used under the current constation gate. + """Whether ``backend`` may be used for Prism worker construction. - Container backends (``base_gpu``, …) are always allowed. ``lium`` is allowed - **only** when a full constation bundle object is supplied — bare Lium without - a bundle stays rejected. This does not evaluate ``constation_ok``; that is the - ingestion elevation path (todos 21–22). + Container backends and ``lium`` are always allowed. ``constation_bundle`` is + accepted for API compatibility but does **not** gate Lium (T14 unattested / + master-owned path). This does not evaluate ``constation_ok``. """ - if backend in SUPPORTED_EXECUTION_BACKENDS: - return True - if backend == LIUM_EXECUTION_BACKEND: - return constation_bundle is not None - return False + del constation_bundle # API compat; not a dispatch gate + return backend in SUPPORTED_EXECUTION_BACKENDS def require_execution_backend( @@ -91,13 +92,9 @@ def require_execution_backend( *, constation_bundle: object | None = None, ) -> None: - """Raise ``ValueError`` when ``backend`` is not permitted under the gate.""" + """Raise ``ValueError`` when ``backend`` is not a supported execution backend.""" if is_execution_backend_supported(backend, constation_bundle=constation_bundle): return - if backend == LIUM_EXECUTION_BACKEND: - raise ValueError( - f"Unsupported execution backend: {backend}: constation bundle required for lium" - ) raise ValueError(f"Unsupported execution backend: {backend}") diff --git a/packages/challenges/prism/tests/test_checkpoint_upload_gate.py b/packages/challenges/prism/tests/test_checkpoint_upload_gate.py index 18593411d..db39be3bd 100644 --- a/packages/challenges/prism/tests/test_checkpoint_upload_gate.py +++ b/packages/challenges/prism/tests/test_checkpoint_upload_gate.py @@ -78,8 +78,6 @@ def create_repo(self, **kwargs): # noqa: ANN003 def upload_file(self, **kwargs): # noqa: ANN003 raise AssertionError("must not call HF when upload disabled") - publisher = HuggingFaceCheckpointPublisher( - repo_id=DEFAULT_CHECKPOINT_REPO_ID, api=_BoomApi() - ) + publisher = HuggingFaceCheckpointPublisher(repo_id=DEFAULT_CHECKPOINT_REPO_ID, api=_BoomApi()) with pytest.raises(RuntimeError, match="prism_checkpoint_upload_disabled"): publisher.publish(upload) diff --git a/packages/challenges/prism/tests/test_config.py b/packages/challenges/prism/tests/test_config.py index 6e5644050..0ca54fcd0 100644 --- a/packages/challenges/prism/tests/test_config.py +++ b/packages/challenges/prism/tests/test_config.py @@ -181,7 +181,9 @@ def test_max_code_bytes_holds_five_mib_zip_base64() -> None: def test_example_config_parses_with_nas_defaults() -> None: - payload = yaml.safe_load((Path(__file__).resolve().parent.parent / "config.example.yaml").read_text(encoding="utf-8")) + payload = yaml.safe_load( + (Path(__file__).resolve().parent.parent / "config.example.yaml").read_text(encoding="utf-8") + ) settings = PrismSettings(**payload) @@ -220,4 +222,3 @@ def test_checkpoint_repo_id_accepts_prism_and_hf_env_aliases(monkeypatch) -> Non monkeypatch.delenv("PRISM_CHECKPOINT_REPO_ID", raising=False) monkeypatch.setenv("PRISM_HF_CHECKPOINT_REPO_ID", "alias-org/from-hf") assert PrismSettings().checkpoint_repo_id == "alias-org/from-hf" - diff --git a/packages/challenges/prism/tests/test_pod_boot.py b/packages/challenges/prism/tests/test_pod_boot.py index ed11e5227..cb31034af 100644 --- a/packages/challenges/prism/tests/test_pod_boot.py +++ b/packages/challenges/prism/tests/test_pod_boot.py @@ -92,10 +92,7 @@ def test_validate_commit_sha_rejects_spaces_and_shell_metachar() -> None: def test_validate_repo_url_accepts_https_git_shape() -> None: assert validate_repo_url(HTTPS_REPO) == HTTPS_REPO - assert ( - validate_repo_url("https://gitlab.com/org/name") - == "https://gitlab.com/org/name" - ) + assert validate_repo_url("https://gitlab.com/org/name") == "https://gitlab.com/org/name" @pytest.mark.parametrize( @@ -211,10 +208,7 @@ def test_build_boot_env_emits_only_non_secret_prism_keys() -> None: ) assert env["PRISM_REPO_URL"] == HTTPS_REPO assert env["PRISM_COMMIT_SHA"] == FULL_SHA - assert ( - env["PRISM_MASTER_CHECKPOINT_URL"] - == "https://chain.joinbase.ai/internal/v1/checkpoints" - ) + assert env["PRISM_MASTER_CHECKPOINT_URL"] == "https://chain.joinbase.ai/internal/v1/checkpoints" assert env["PRISM_SUBMISSION_ID"] == "sub-123" assert env["PRISM_ATTEMPT"] == "1" # No secrets diff --git a/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py b/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py index d3d8f194f..280f5d387 100644 --- a/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py +++ b/packages/challenges/prism/tests/test_prism_checkpoint_publisher.py @@ -246,4 +246,3 @@ def test_checkpoint_publisher_module_imports_offline_without_hf_token(monkeypatc constructed = pub.HuggingFaceCheckpointPublisher() assert constructed.repo_id == MANDATED_CHECKPOINT_REPO_ID # publish without injected api would try lazy import; construction alone must not. - diff --git a/src/base/compute/lium.py b/src/base/compute/lium.py index 3012d601c..e1092a88f 100644 --- a/src/base/compute/lium.py +++ b/src/base/compute/lium.py @@ -47,9 +47,7 @@ _DEFAULT_SSH_KEY_NAME = "prism-mission-worker" # Canonical display name for the only GPU type Prism training may rent. -LIUM_TRAINING_GPU_TYPE: Final[str] = ( - "NVIDIA RTX PRO 6000 Blackwell Server Edition" -) +LIUM_TRAINING_GPU_TYPE: Final[str] = "NVIDIA RTX PRO 6000 Blackwell Server Edition" _TRAINING_GPU_REQUIRED_TOKENS: Final[frozenset[str]] = frozenset( {"rtx", "pro", "6000", "blackwell"} ) diff --git a/src/base/compute/lium_capacity.py b/src/base/compute/lium_capacity.py index 00e9df1a1..fd727a1df 100644 --- a/src/base/compute/lium_capacity.py +++ b/src/base/compute/lium_capacity.py @@ -354,6 +354,8 @@ async def _admit_one( self._store.put(active) return active + if orphan_pod_id is None: + return None try: await client.terminate(orphan_pod_id) except Exception: # noqa: BLE001 - cancel race must not raise diff --git a/tests/unit/test_compute_lium_client.py b/tests/unit/test_compute_lium_client.py index 36c2f39a9..ac10ca56b 100644 --- a/tests/unit/test_compute_lium_client.py +++ b/tests/unit/test_compute_lium_client.py @@ -30,11 +30,11 @@ ) from base.compute.lium import ( LIUM_TRAINING_GPU_TYPE, -_as_list, -_extract_gpu_count, -_extract_gpu_type, -_extract_price, -_parse_instance, + _as_list, + _extract_gpu_count, + _extract_gpu_type, + _extract_price, + _parse_instance, _parse_offer, is_allowed_lium_training_gpu, normalize_gpu_type, @@ -194,9 +194,7 @@ async def test_list_offers_keeps_only_rtx_pro_6000_blackwell_1gpu() -> None: "price_per_gpu": 0.95, }, ] - respx.get(f"{BASE}/executors").mock( - return_value=httpx.Response(200, json=payload) - ) + respx.get(f"{BASE}/executors").mock(return_value=httpx.Response(200, json=payload)) offers = await LiumClient("k", training_gpu_lock=True).list_offers() assert [o.id for o in offers] == ["bw1"] assert offers[0].gpu_count == 1 @@ -215,14 +213,11 @@ async def test_list_offers_rejects_unparseable_gpu_type() -> None: "price_per_hour": 1.29, }, ] - respx.get(f"{BASE}/executors").mock( - return_value=httpx.Response(200, json=payload) - ) + respx.get(f"{BASE}/executors").mock(return_value=httpx.Response(200, json=payload)) offers = await LiumClient("k", training_gpu_lock=True).list_offers() assert [o.id for o in offers] == ["ok"] - # -- VAL-PROV-003 ------------------------------------------------------------- @@ -375,8 +370,6 @@ async def test_provision_rent_body_gpu_count_is_one() -> None: assert body["gpu_count"] == 1 - - @respx.mock async def test_provision_rejects_overpriced_offer_without_rent() -> None: rent = respx.post(f"{BASE}/executors/exec-1/rent") diff --git a/tests/unit/test_lium_capacity_scheduler.py b/tests/unit/test_lium_capacity_scheduler.py index e4b3b6d82..1c73e34fa 100644 --- a/tests/unit/test_lium_capacity_scheduler.py +++ b/tests/unit/test_lium_capacity_scheduler.py @@ -238,9 +238,7 @@ async def test_tick_admits_up_to_offer_count_and_cap() -> None: assert len(admitted) == 2 # only 2 offers assert all(lease.state is LeaseState.ACTIVE for lease in admitted) queued = [ - lease - for lease in sched.store.list_all() - if lease.state is LeaseState.QUEUED + lease for lease in sched.store.list_all() if lease.state is LeaseState.QUEUED ] assert len(queued) == 2 assert {lease.submission_id for lease in queued} == {"sub-2", "sub-3"} diff --git a/tests/unit/test_lium_training_wiring.py b/tests/unit/test_lium_training_wiring.py index 3d9392638..c98a77556 100644 --- a/tests/unit/test_lium_training_wiring.py +++ b/tests/unit/test_lium_training_wiring.py @@ -106,7 +106,6 @@ def test_build_lium_capacity_scheduler_disabled_fail_closed() -> None: build_lium_capacity_scheduler(settings) - def test_try_build_lium_capacity_scheduler_disabled_returns_none() -> None: """Given enabled=False, When try_build, Then None (no raise).""" settings = Settings(lium_training=LiumTrainingSettings(enabled=False)) diff --git a/tests/unit/test_master_entrypoint_prism_policy.py b/tests/unit/test_master_entrypoint_prism_policy.py index d6f26087c..00ad20053 100644 --- a/tests/unit/test_master_entrypoint_prism_policy.py +++ b/tests/unit/test_master_entrypoint_prism_policy.py @@ -17,7 +17,10 @@ def test_entrypoint_embeds_worker_plane_prod_defaults() -> None: """Given entrypoint script, When read, Then worker-plane prod defaults present.""" text = ENTRYPOINT.read_text(encoding="utf-8") - assert "PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE=${PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE:-false}" in text + assert ( + "PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE=${PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE:-false}" + in text + ) assert ( "PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER=" "${PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER:-true}" diff --git a/tests/unit/test_orchestration.py b/tests/unit/test_orchestration.py index 72c58b19e..6617987dc 100644 --- a/tests/unit/test_orchestration.py +++ b/tests/unit/test_orchestration.py @@ -34,6 +34,7 @@ constation_identity_payload, ) from base.master.constation.custody_keys import make_constation_pre_forward_hook +from base.master.constation.orchestrator import ConstationOrchestrationRequest from base.master.orchestration import ( WORK_UNIT_MAX_ATTEMPTS_REASON, ChallengePendingWork, @@ -759,7 +760,7 @@ async def run(self, request: object) -> None: assert hook is not None await hook(work_unit_id="wu-1", miner_hotkey="hk", metadata=metadata) assert len(seen) == 1 - req = seen[0] + req = cast(ConstationOrchestrationRequest, seen[0]) assert req.required_digest == pin.digest assert req.commit_sha == pin.commit_sha assert req.tree_sha == pin.tree_sha diff --git a/tests/unit/test_orchestration_lium.py b/tests/unit/test_orchestration_lium.py index 62a4bf56a..ee850fe7f 100644 --- a/tests/unit/test_orchestration_lium.py +++ b/tests/unit/test_orchestration_lium.py @@ -86,11 +86,7 @@ async def test_orchestration_enqueues_lium_lease_when_scheduler_present() -> Non assert scheduler.enqueue_calls == [("psub-lium-1", "job-lium-1")] async with factory() as session: - rows = list( - ( - await session.execute(select(WorkAssignment)) - ).scalars().all() - ) + rows = list((await session.execute(select(WorkAssignment))).scalars().all()) assert len(rows) == 1 assert rows[0].required_capability == "gpu" finally: @@ -113,11 +109,7 @@ async def test_orchestration_skips_lium_when_scheduler_none() -> None: bridged = await driver.bridge_pending_work() assert bridged["prism"] == ["psub-lium-1"] async with factory() as session: - rows = list( - ( - await session.execute(select(WorkAssignment)) - ).scalars().all() - ) + rows = list((await session.execute(select(WorkAssignment))).scalars().all()) assert len(rows) == 1 assert rows[0].work_unit_id == "psub-lium-1" finally: