Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion api/config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import aioboto3
import json
import yaml
from dataclasses import dataclass
from dataclasses import dataclass, field as dataclass_field
from api.safe_redis import SafeRedis
from functools import cached_property, lru_cache
import redis.asyncio as redis
Expand Down Expand Up @@ -65,6 +65,16 @@ class TeeMeasurementConfig:
expected_gpus: List[str]
gpu_count: Optional[int] = None
rc: bool = False # release candidate / in-test: attestable but unpublished
# rc authorization allowlists (both required non-empty when rc is True; ignored for published
# measurements). The two gate modes use different primitives, matched to their environment:
# * authorized_hotkeys -- miner hotkeys allowed on the register/runtime (userspace,
# get_current_user-authenticated) paths.
# * authorized_signing_keys -- operator RSA *public* keys (PEM) allowed on the boot/provision
# (initramfs) paths, where the VM signs the nonce with `openssl dgst -sha256 -sign` and no
# sr25519/substrate signer is available.
# See api.server.util.authorize_rc_measurement.
authorized_hotkeys: List[str] = dataclass_field(default_factory=list)
authorized_signing_keys: List[str] = dataclass_field(default_factory=list)

@property
def boot_rtmrs(self) -> Dict[str, str]:
Expand Down Expand Up @@ -430,6 +440,48 @@ def _hex96(value: str, field_name: str, owner: str) -> str:
runtime_rtmr3 = _hex96(version_config["runtime_rtmr3"], "runtime RTMR3", version)
rc = bool(version_config.get("rc", False))

# rc authorization allowlists. Meaningful only for rc measurements (published ones lock
# down identical guest software for everyone, so operator identity is irrelevant there):
# authorized_hotkeys -> register/runtime paths, authorized_signing_keys -> boot/provision.
authorized_hotkeys = [
str(hk).strip()
for hk in (version_config.get("authorized_hotkeys") or [])
if str(hk).strip()
]
authorized_signing_keys = [
str(k).strip()
for k in (version_config.get("authorized_signing_keys") or [])
if str(k).strip()
]

# Load-time invariant: an rc measurement with no allowlist would be usable by anyone who
# can build the same image -- exactly the exposure rc gating exists to prevent. A full VM
# lifecycle hits BOTH gate modes, so require both allowlists non-empty, and require every
# signing key to parse as a PEM public key (a bad key can't verify anything, so treating
# it as usable would be a silent hole). Drop the whole version (all hardware variants) and
# log loudly rather than raise, so one misconfigured rc entry can't take down attestation
# for published VMs; a dropped entry never matches any quote, so affected VMs fail closed.
drop_reason = None
if rc and not authorized_hotkeys:
drop_reason = "'authorized_hotkeys' allowlist is empty"
elif rc and not authorized_signing_keys:
drop_reason = "'authorized_signing_keys' allowlist is empty"
elif rc:
for pem in authorized_signing_keys:
try:
serialization.load_pem_public_key(pem.encode())
except Exception as e:
drop_reason = f"an 'authorized_signing_keys' entry is not a valid PEM public key ({e})"
break
if drop_reason:
logger.error(
f"Refusing to load rc measurement version '{version}': {drop_reason}. rc "
"measurements MUST declare non-empty, valid 'authorized_hotkeys' and "
"'authorized_signing_keys'. This version is UNUSABLE until fixed; VMs on it "
"will fail attestation."
)
continue

hardware = version_config.get("hardware") or []
if not hardware:
raise ValueError(
Expand Down Expand Up @@ -461,6 +513,8 @@ def _hex96(value: str, field_name: str, owner: str) -> str:
expected_gpus=[gpu.lower() for gpu in hw["expected_gpus"]],
gpu_count=gpu_count,
rc=rc,
authorized_hotkeys=authorized_hotkeys,
authorized_signing_keys=authorized_signing_keys,
)
)

Expand Down
5 changes: 5 additions & 0 deletions api/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ class ServerHealthStatus(str, Enum):
COLDKEY_HEADER = "X-Chutes-Coldkey"
SIGNATURE_HEADER = "X-Chutes-Signature"
NONCE_HEADER = "X-Chutes-Nonce"
# RSA operator signature over the quote nonce, used only by the initramfs (signed) attestation
# path where sr25519 is unavailable. Distinct from SIGNATURE_HEADER so the rc gate can tell the
# signed proof from the sr25519 request signature by header presence alone (see
# extract_attestation_auth / authorize_rc_measurement).
OPERATOR_SIGNATURE_HEADER = "X-Operator-Signature"
AUTHORIZATION_HEADER = "Authorization"
PURPOSE_HEADER = "X-Chutes-Purpose"
MINER_HEADER = "X-Chutes-Miner"
Expand Down
25 changes: 25 additions & 0 deletions api/migrations/20260806120000_vm_boot_records.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- migrate:up
-- Broaden the boot-attestation table into the pre-server "vm boot record": one row per boot
-- (append, preserving all existing attestation history), each capturing that boot's full initramfs
-- lifecycle -- the boot quote AND, once /provision runs, the runtime quote + the per-boot VM root
-- CA recorded in fully-measured initramfs (before the miner registers the server via POST /servers).
--
-- In place (no data loss): rename the table, rename quote_data -> boot_quote, and add the new
-- columns. The boot vs provision distinction is which quote column is set, so no phase discriminator
-- is needed. Existing indexes (idx_boot_*) carry over with the table.
ALTER TABLE boot_attestations RENAME TO vm_boot_records;
ALTER TABLE vm_boot_records RENAME COLUMN quote_data TO boot_quote;
ALTER TABLE vm_boot_records ADD COLUMN provision_quote TEXT;
ALTER TABLE vm_boot_records ADD COLUMN vm_root_ca_cert TEXT;
-- The luks_quote_nonce minted at /boot/attestation and consumed at /provision -- ties the two
-- calls of one boot to the same row (deterministic, not by timestamp).
ALTER TABLE vm_boot_records ADD COLUMN provision_nonce TEXT;
ALTER TABLE vm_boot_records ADD COLUMN updated_at TIMESTAMPTZ;

-- migrate:down
ALTER TABLE vm_boot_records DROP COLUMN updated_at;
ALTER TABLE vm_boot_records DROP COLUMN provision_nonce;
ALTER TABLE vm_boot_records DROP COLUMN vm_root_ca_cert;
ALTER TABLE vm_boot_records DROP COLUMN provision_quote;
ALTER TABLE vm_boot_records RENAME COLUMN boot_quote TO quote_data;
ALTER TABLE vm_boot_records RENAME TO boot_attestations;
33 changes: 22 additions & 11 deletions api/miner/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import orjson as json
from decimal import Decimal
from sqlalchemy.ext.asyncio import AsyncSession
from fastapi import APIRouter, Depends, Header, status, HTTPException, Response, Request
from fastapi import APIRouter, Depends, Header, status, HTTPException, Request
from starlette.responses import StreamingResponse
from sqlalchemy import text
from sqlalchemy.future import select
Expand All @@ -25,7 +25,16 @@
from api.image.schemas import Image
from api.instance.schemas import Instance, LaunchConfig
from api.server.schemas import Server
from api.miner.schemas import MinerServersResponse
from api.miner.schemas import (
ActiveInstance,
MinerChute,
MinerInventoryEntry,
MinerMetagraphNode,
MinerScoresResponse,
MinerServersResponse,
MinerStatsResponse,
UniqueChuteHistoryEntry,
)
from api.job.schemas import Job
from api.invocation.util import gather_metrics
from api.user.service import get_current_user
Expand Down Expand Up @@ -243,7 +252,7 @@ async def release_job(
)


@router.get("/inventory")
@router.get("/inventory", responses={200: {"model": list[MinerInventoryEntry]}})
async def get_full_inventory(
hotkey: str | None = Header(None, alias=HOTKEY_HEADER),
session: AsyncSession = Depends(get_db_session),
Expand Down Expand Up @@ -281,7 +290,7 @@ async def _stream():
return StreamingResponse(_stream())


@router.get("/active_instances/")
@router.get("/active_instances/", responses={200: {"model": list[ActiveInstance]}})
async def list_active_instances(
_: User = Depends(get_current_user(purpose="miner", registered_to=settings.netuid)),
session: AsyncSession = Depends(get_db_session),
Expand Down Expand Up @@ -317,7 +326,7 @@ async def list_active_instances(
]


@router.get("/chutes/{chute_id}/{version}")
@router.get("/chutes/{chute_id}/{version}", responses={200: {"model": MinerChute}})
async def get_chute(
chute_id: str,
version: str,
Expand All @@ -341,13 +350,13 @@ async def get_chute(
return await model_to_dict(chute)


@router.get("/stats")
@router.get("/stats", responses={200: {"model": MinerStatsResponse}})
async def get_stats(
miner_hotkey: Optional[str] = None,
session: AsyncSession = Depends(get_db_session),
per_chute: Optional[bool] = False,
request: Request = None,
) -> Response:
):
"""
Get miner stats over different intervals based on instance data (matching actual scoring).

Expand Down Expand Up @@ -570,7 +579,7 @@ def _filter_by_key(mstats):
return _filter_by_key(results)


@router.get("/scores")
@router.get("/scores", responses={200: {"model": MinerScoresResponse}})
async def get_scores(hotkey: Optional[str] = None, request: Request = None):
cache_key = "get_scores"
rv = None
Expand All @@ -593,7 +602,9 @@ async def get_scores(hotkey: Optional[str] = None, request: Request = None):
return rv


@router.get("/unique_chute_history/{hotkey}")
@router.get(
"/unique_chute_history/{hotkey}", responses={200: {"model": list[UniqueChuteHistoryEntry]}}
)
async def unique_chute_history(hotkey: str, request: Request = None):
if not await settings.redis_client.get(f"miner_exists:{hotkey}"):
async with get_session(readonly=True) as session:
Expand All @@ -614,7 +625,7 @@ async def unique_chute_history(hotkey: str, request: Request = None):
)


@router.get("/thrash_cooldowns")
@router.get("/thrash_cooldowns", responses={200: {"model": list[Any]}})
async def get_thrash_cooldowns(
hotkey: str | None = Header(None, alias=HOTKEY_HEADER),
session: AsyncSession = Depends(get_db_session),
Expand All @@ -626,7 +637,7 @@ async def get_thrash_cooldowns(
return []


@router.get("/metagraph")
@router.get("/metagraph", responses={200: {"model": list[MinerMetagraphNode]}})
async def get_metagraph():
async with get_session(readonly=True) as session:
return (
Expand Down
Loading
Loading