diff --git a/.env.validator.example b/.env.validator.example index 1b28fb2..ba30e5b 100644 --- a/.env.validator.example +++ b/.env.validator.example @@ -126,3 +126,34 @@ EIREL_VALIDATOR_MAX_PARALLEL=2 # thinking: full reasoning + tool use allowed, 10-minute hard ceiling. EIREL_MINER_INSTANT_LATENCY_BUDGET_SECONDS=120 EIREL_MINER_THINKING_LATENCY_BUDGET_SECONDS=600 + +# ============================================================================= +# Decentralized execution (EIREL_EXEC_MODE=local) — OPTIONAL +# +# Only needed if you run miners on YOUR OWN cluster instead of through the +# owner proxy. Requires the local stack overlay: +# docker compose -f docker-compose.validator.yml -f docker-compose.validator.local.yml up -d +# Leave EIREL_EXEC_MODE unset (or =owner) to use the owner proxy (default). +# +# You GENERATE these secrets yourself (random 32+ bytes); they must match across +# the engine, provider-proxy and all four tool services, and never enter a +# miner pod. The validator pays its own eval LLM/tool spend. +# ============================================================================= +# EIREL_EXEC_MODE=local +# EIREL_OWNER_HOTKEY= # pin the subnet owner's ss58 (roster verify) +# EIREL_TOOL_JOB_SIGNING_KEY= # per-job HMAC key (engine mints, proxy/tools verify) +# EIREL_PROVIDER_PROXY_MASTER_TOKEN= +# EIREL_TOOL_LEDGER_TOKEN= # tools -> ledger-sink write auth +# EIREL_LOCAL_LEDGER_TOKEN= # engine -> ledger-sink read auth (defaults to TOOL_LEDGER_TOKEN) +# REDIS_PASSWORD= +# REDIS_URL=redis://:CHANGEME@redis:6379/0 +# SERVER_A_HOST_IP= # validator host IP miner k3s pods NAT to +# EIREL_PROVIDER_CHUTES_API_KEY= # YOUR LLM spend during eval +# EIREL_BRAVE_SEARCH_API_KEY= # (or SERPER/TAVILY) for web_search +# EIREL_RAG_EMBEDDING_API_KEY= # for the rag tool +# EIREL_RUN_BUDGET_USD=30.0 # per-miner-run LLM budget cap the proxy enforces +# Validator k3s the runtime manager deploys miners into: +# EIREL_VALIDATOR_KUBECONFIG_PATH= +# EIREL_VALIDATOR_RUNTIME_NAMESPACE=eirel-miners +# EIREL_VALIDATOR_RUNTIME_SYSTEM_NAMESPACE=eirel-system +# EIREL_VALIDATOR_RUNTIME_IMAGE=eirel-miner-runtime:latest diff --git a/control_plane/owner_api/_helpers.py b/control_plane/owner_api/_helpers.py index 71f9830..750782c 100644 --- a/control_plane/owner_api/_helpers.py +++ b/control_plane/owner_api/_helpers.py @@ -187,10 +187,10 @@ def _strip_sensitive_task_metadata( The validator-claim path passes ``strip_expected_output=False`` — validators run inside the operator's stack and need - ``expected_output.answer`` / ``must_not_claim`` to compute the - multi-metric per-task score. Miners never see ``expected_output`` - regardless; that strip happens upstream when the task is dispatched - to the miner pod. + ``expected_output`` (e.g. ``required_tool``) plus the oracle-derived + ``expected_claims`` to compute the multi-metric per-task score. + Miners never see ``expected_output`` regardless; that strip happens + upstream when the task is dispatched to the miner pod. """ task_metadata = dict(task_dict.get("metadata") or {}) for key in _SENSITIVE_TASK_METADATA_KEYS: diff --git a/control_plane/owner_api/app.py b/control_plane/owner_api/app.py index 86d9e33..2e200f8 100644 --- a/control_plane/owner_api/app.py +++ b/control_plane/owner_api/app.py @@ -15,7 +15,9 @@ import os import uvicorn from starlette.types import ASGIApp, Message, Receive, Scope, Send -from fastapi import FastAPI +from dataclasses import dataclass + +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware logging.basicConfig( @@ -48,6 +50,7 @@ internal, internal_eval, checkpoints, + roster, ) from shared.common.request_context import RequestIdMiddleware from shared.common.security import create_replay_protector @@ -254,6 +257,40 @@ async def _claim_expiry_sweeper_loop(app: FastAPI) -> None: await asyncio.sleep(interval) +# --------------------------------------------------------------------------- +# Ledger-only role (validator-local tool-call attestation sink) +# --------------------------------------------------------------------------- + +# Resolved once at import so router registration (module scope) can gate on it. +_OWNER_API_ROLE = os.getenv("EIREL_OWNER_API_ROLE", "full").strip().lower() +_LEDGER_ROLE = _OWNER_API_ROLE == "ledger" + + +@dataclass +class _LedgerServices: + """Minimal ``app.state.services`` stand-in for the ledger role — the only + attributes the tool-call ledger + eval-feedback handlers touch are ``db`` + and ``settings``. Avoids constructing the full ManagedOwnerServices (runtime + manager, scheduler, fee verifier) which a validator must not run.""" + + db: Database + settings: object + + +async def _ledger_read_auth(request: Request) -> str: + """Read-auth for the ledger role. The validator reads its OWN local sink, so + instead of the metagraph ``validator_dependency`` (which needs chain access) + we accept a shared bearer token. When no token is configured (dev), allow.""" + expected = (os.getenv("EIREL_LOCAL_LEDGER_TOKEN") or os.getenv("EIREL_TOOL_LEDGER_TOKEN") or "").strip() + if not expected: + return "local-ledger" + header = request.headers.get("Authorization", "") + token = header[7:].strip() if header.lower().startswith("bearer ") else "" + if token != expected: + raise HTTPException(status_code=401, detail="invalid local ledger token") + return "local-ledger" + + # --------------------------------------------------------------------------- # Lifespan # --------------------------------------------------------------------------- @@ -262,6 +299,21 @@ async def _claim_expiry_sweeper_loop(app: FastAPI) -> None: @asynccontextmanager async def lifespan(app: FastAPI): settings = get_settings() + if settings.owner_api_role == "ledger": + # Stripped startup: just the DB (small local store — SQLite ok). No + # runtime manager, no scheduler loops, no chain. create_all() gives the + # current schema directly, so the heavy production migration set is + # skipped. + db = Database(settings.database_url) + db.create_all() + app.state.services = _LedgerServices(db=db, settings=settings) + logger.info( + "owner-api LEDGER role: tool-call attestation sink only (db=%s)", + settings.database_url, + ) + yield + db.engine.dispose() + return db = Database(settings.database_url) db.create_all() run_migrations(db.engine) @@ -410,18 +462,37 @@ async def lifespan(app: FastAPI): "total_applied_count": 0, "last_result": None, } - await app.state.services.refresh_runtime_node_inventory() - await app.state.services.reconcile_all_active_deployments() - refresh_task = asyncio.create_task(_runtime_capacity_refresh_loop(app)) - remediation_task = asyncio.create_task(_runtime_remediation_loop(app)) - sweeper_task = asyncio.create_task(_claim_expiry_sweeper_loop(app)) - health_task = asyncio.create_task(_container_health_loop(app)) - reaper_task = asyncio.create_task(_retired_runtime_reaper_loop(app)) - retry_task = asyncio.create_task(_pending_capacity_retry_loop(app)) + # In decentralized execution the OWNER no longer manages miner runtimes — + # validators run submissions on their own clusters. An owner-api in that + # mode serves registry/roster/claim/archive but must NOT run the runtime + # reconcile/health/reaper loops (they would demote deployments it isn't + # actually hosting). Gated off by default so the owner-hosted path is + # byte-for-byte unchanged. + _disable_runtime_mgmt = os.getenv( + "EIREL_OWNER_DISABLE_RUNTIME_RECONCILE", "" + ).strip().lower() in ("1", "true", "yes", "on") + loop_tasks: list[asyncio.Task] = [] + if _disable_runtime_mgmt: + logger.warning( + "EIREL_OWNER_DISABLE_RUNTIME_RECONCILE set — skipping runtime " + "node inventory, reconcile, and the runtime management loops " + "(decentralized-execution owner-api: registry/roster/claim only)" + ) + else: + await app.state.services.refresh_runtime_node_inventory() + await app.state.services.reconcile_all_active_deployments() + loop_tasks = [ + asyncio.create_task(_runtime_capacity_refresh_loop(app)), + asyncio.create_task(_runtime_remediation_loop(app)), + asyncio.create_task(_claim_expiry_sweeper_loop(app)), + asyncio.create_task(_container_health_loop(app)), + asyncio.create_task(_retired_runtime_reaper_loop(app)), + asyncio.create_task(_pending_capacity_retry_loop(app)), + ] yield - for task in (refresh_task, remediation_task, sweeper_task, health_task, reaper_task, retry_task): + for task in loop_tasks: task.cancel() - for task in (refresh_task, remediation_task, sweeper_task, health_task, reaper_task, retry_task): + for task in loop_tasks: try: await asyncio.wait_for(task, timeout=5.0) except (asyncio.CancelledError, asyncio.TimeoutError): @@ -457,24 +528,37 @@ async def lifespan(app: FastAPI): allow_headers=["*"], ) -app.include_router(health.router) -app.include_router(submissions.router) -app.include_router(deployments.router) -app.include_router(runs.router) -app.include_router(scoring.router) -app.include_router(evaluation_tasks.router) -app.include_router(serving.router) -app.include_router(workflows.router) -app.include_router(operator.router) -app.include_router(runtime.router) -app.include_router(validators.router) -app.include_router(internal.router) -app.include_router(internal_eval.router) -app.include_router(checkpoints.router) -app.include_router(dashboard.router) - -from control_plane.owner_api.routers import admin as admin_router -app.include_router(admin_router.router) +if _LEDGER_ROLE: + # Validator-local sink: only the tool-call attestation ledger + a trivial + # healthz. Read-auth is the local bearer token, not the metagraph dependency. + from control_plane.owner_api.dependencies import validator_dependency + + @app.get("/healthz") + async def _ledger_healthz() -> dict[str, str]: + return {"status": "ok", "role": "ledger"} + + app.include_router(internal_eval.router) + app.dependency_overrides[validator_dependency] = _ledger_read_auth +else: + app.include_router(health.router) + app.include_router(submissions.router) + app.include_router(deployments.router) + app.include_router(runs.router) + app.include_router(scoring.router) + app.include_router(evaluation_tasks.router) + app.include_router(serving.router) + app.include_router(workflows.router) + app.include_router(operator.router) + app.include_router(runtime.router) + app.include_router(validators.router) + app.include_router(internal.router) + app.include_router(internal_eval.router) + app.include_router(checkpoints.router) + app.include_router(roster.router) + app.include_router(dashboard.router) + + from control_plane.owner_api.routers import admin as admin_router + app.include_router(admin_router.router) def main() -> None: diff --git a/control_plane/owner_api/dashboard/cache.py b/control_plane/owner_api/dashboard/cache.py index 0a92f9e..aaa022a 100644 --- a/control_plane/owner_api/dashboard/cache.py +++ b/control_plane/owner_api/dashboard/cache.py @@ -1,15 +1,35 @@ from __future__ import annotations import time +from collections import OrderedDict from typing import Any, Generic, TypeVar T = TypeVar("T") +# Dashboard cache keys vary with run_id, miner hotkey, and pagination, so the +# keyspace grows over the lifetime of the process. ``get()`` only evicts the +# single key it is asked for, so without an active sweep and a hard cap, +# expired entries for keys that are never fetched again are never reclaimed — +# a slow leak that climbs the owner-api pod to its memory limit and OOMKills it +# after a few days. The periodic sweep reclaims lapsed entries in bulk; the cap +# bounds worst-case memory regardless of how fast the keyspace churns. +_DEFAULT_MAX_ENTRIES = 512 +_SWEEP_INTERVAL_SECONDS = 30.0 + class TTLCache(Generic[T]): - def __init__(self, default_ttl_seconds: float = 30.0) -> None: + def __init__( + self, + default_ttl_seconds: float = 30.0, + *, + max_entries: int = _DEFAULT_MAX_ENTRIES, + ) -> None: self._default_ttl = default_ttl_seconds - self._store: dict[tuple[Any, ...], tuple[float, T]] = {} + self._max_entries = max(1, max_entries) + # Insertion-ordered so that, once the cap is hit and a sweep has not + # freed enough room, we can evict the oldest entry deterministically. + self._store: OrderedDict[tuple[Any, ...], tuple[float, T]] = OrderedDict() + self._last_sweep = time.monotonic() def get(self, key: tuple[Any, ...]) -> T | None: entry = self._store.get(key) @@ -22,8 +42,27 @@ def get(self, key: tuple[Any, ...]) -> T | None: return value def set(self, key: tuple[Any, ...], value: T, ttl: float | None = None) -> None: - expires_at = time.monotonic() + (ttl if ttl is not None else self._default_ttl) + now = time.monotonic() + # Periodic full sweep: reclaim every lapsed entry, not just the one a + # caller happens to read back. This is what stops a churning keyspace + # (new run_ids, new hotkeys, paginated views) from accumulating dead + # entries indefinitely. + if now - self._last_sweep >= _SWEEP_INTERVAL_SECONDS: + self._sweep(now) + expires_at = now + (ttl if ttl is not None else self._default_ttl) self._store[key] = (expires_at, value) + self._store.move_to_end(key) + # Hard backstop between sweeps: never retain more than the cap, evicting + # oldest-first so the live working set survives. + while len(self._store) > self._max_entries: + self._store.popitem(last=False) + + def _sweep(self, now: float) -> None: + expired = [k for k, (expires_at, _) in self._store.items() if expires_at < now] + for k in expired: + self._store.pop(k, None) + self._last_sweep = now def clear(self) -> None: self._store.clear() + self._last_sweep = time.monotonic() diff --git a/control_plane/owner_api/dashboard/queries.py b/control_plane/owner_api/dashboard/queries.py index 7ee547a..ee1cd23 100644 --- a/control_plane/owner_api/dashboard/queries.py +++ b/control_plane/owner_api/dashboard/queries.py @@ -3,7 +3,7 @@ from collections import defaultdict from typing import Any -from sqlalchemy import case, func, select +from sqlalchemy import case, exists, func, select from sqlalchemy.orm import Session from eirel.groups import LAUNCH_FAMILIES, ensure_family_id @@ -137,8 +137,8 @@ def _effective_verdict(verdict: str | None, final_task_score: float | None) -> s Rule: a task is in its pass bucket (matches / partially_matches / not_applicable) only if the composite gates (tool_attestation, - hallucination_knockout, grounded_gate, safety_gate, cost_attestation, - safety_attestation) didn't zero it. When ``final_task_score == 0`` + grounded_gate, safety_gate, cost_attestation, safety_attestation) + didn't zero it. When ``final_task_score == 0`` and the verdict was a pass type, the row becomes ``gate_knockout`` so the displayed bucket counts and ``mean_agreement`` reflect the same reality as the canonical score. @@ -348,9 +348,22 @@ def fetch_overview(session: Session, *, services: Any) -> OverviewResponse: # received → pending_capacity → building → deployed_for_eval → retired # build_failed is a terminal error branch. _QUEUED = ("received", "pending_capacity", "building") - _EVALUATING = ("deployed_for_eval",) queued = sum(int(sub_counts.get(s, 0)) for s in _QUEUED) - evaluating = sum(int(sub_counts.get(s, 0)) for s in _EVALUATING) + # "Evaluating" = deployed-for-eval submissions that aren't yet on the + # leaderboard. A submission keeps the sticky ``deployed_for_eval`` status + # even after it's been scored (and after the capacity manager parks its + # pod in standby), so the raw status count double-counts ranked miners. + # Exclude any submission that already has a DeploymentScoreRecord — those + # belong on the leaderboard, not the evaluating tally. + evaluating = int(session.scalar( + select(func.count(ManagedMinerSubmission.id)) + .where(ManagedMinerSubmission.status == "deployed_for_eval") + .where( + ~exists().where( + DeploymentScoreRecord.submission_id == ManagedMinerSubmission.id + ) + ) + ) or 0) retired = int(sub_counts.get("retired", 0)) build_failed = int(sub_counts.get("build_failed", 0)) completed = retired + build_failed diff --git a/control_plane/owner_api/evaluation/general_chat_scoring.py b/control_plane/owner_api/evaluation/general_chat_scoring.py index 8e6cee3..533bd9f 100644 --- a/control_plane/owner_api/evaluation/general_chat_scoring.py +++ b/control_plane/owner_api/evaluation/general_chat_scoring.py @@ -9,9 +9,9 @@ error → not counted in the denominator Gate-aware bucketing: a task whose multi-metric composite was zeroed -by a gate (tool_attestation / hallucination_knockout / grounded_gate / -safety_gate / cost_attestation / safety_attestation) is reclassified -OUT of its pass bucket and into ``gate_knockout``. The ``mean_agreement`` +by a gate (tool_attestation / grounded_gate / safety_gate / +cost_attestation / safety_attestation) is reclassified OUT of its +pass bucket and into ``gate_knockout``. The ``mean_agreement`` treats those rows as contributing 0 to the numerator — so the displayed verdict counts and the agreement mean reflect the same reality as the canonical multi-metric score. diff --git a/control_plane/owner_api/routers/internal_eval.py b/control_plane/owner_api/routers/internal_eval.py index 6498db7..08fd09a 100644 --- a/control_plane/owner_api/routers/internal_eval.py +++ b/control_plane/owner_api/routers/internal_eval.py @@ -66,7 +66,11 @@ def eval_feedback_enabled() -> bool: class ToolCallLogWriteRequest(BaseModel): - job_id: str = Field(min_length=1, max_length=64) + # 128 (not 64): the local-exec cost tag is + # ``task-eval={task_id};deployment={deployment_id}`` — a real task_id plus a + # 36-char deployment UUID runs ~80-98 chars, so a 64 cap 422s every + # validator-local tool-call write and silently empties tool attestation. + job_id: str = Field(min_length=1, max_length=128) tool_name: str = Field(min_length=1, max_length=64) args_hash: str = Field(default="", max_length=64) args_json: dict[str, Any] = Field(default_factory=dict) @@ -148,7 +152,9 @@ async def write_tool_call( ) async def read_job_ledger( request: Request, - job_id: str = Query(min_length=1, max_length=64), + # 128 to match ToolCallLogWriteRequest.job_id — the local-exec cost tag + # (task-eval={task_id};deployment={deployment_id}) overruns a 64 cap. + job_id: str = Query(min_length=1, max_length=128), validator_hotkey: str = Depends(validator_dependency), ) -> JobLedgerResponse: del validator_hotkey # auth-only — registered active validators may read diff --git a/control_plane/owner_api/routers/roster.py b/control_plane/owner_api/routers/roster.py new file mode 100644 index 0000000..a64c95f --- /dev/null +++ b/control_plane/owner_api/routers/roster.py @@ -0,0 +1,178 @@ +"""Decentralized-execution distribution endpoints (Phase 0). + +Two read-only routes the owner serves so validators can run miner submissions +on their own infrastructure without trusting the owner for response/latency/cost +(only for *who competes* and *what code*, both cryptographically committed): + + GET /v1/runs/{run_id}/roster?family_id=... (validator hotkey) + The owner-signed roster for a frozen run/family snapshot: each member's + hotkey + submission_id + archive_sha256, plus the task-bundle ref. + Validators verify the owner signature and pin the committed hashes. + + GET /v1/runs/{run_id}/submissions/{submission_id}/archive (validator hotkey) + The raw submission archive bytes. The validator hashes them and refuses + to run unless the hash matches the roster's committed archive_sha256. + +Both are purely additive — they read existing tables (EpochTargetSnapshot, +ManagedMinerSubmission, SubmissionArtifact) and do not touch the live +claim/evaluate/aggregate path. Auth is the same registered-active-validator +signature gate used by the tool-call ledger. + +Note: serving archive bytes to validators is intentional — in decentralized +execution every validator runs the code, so submission code is no longer +owner-confidential. +""" +from __future__ import annotations + +import logging +import os +from datetime import UTC, datetime + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from sqlalchemy import select +from starlette.responses import Response + +from control_plane.owner_api.dependencies import validator_dependency +from control_plane.owner_api.managed import ManagedOwnerServices +from shared.common.bittensor_signing import LoadedSigner, load_signer +from shared.common.models import ( + EpochTargetSnapshot, + ManagedMinerSubmission, + SubmissionArtifact, +) +from shared.common.roster import build_roster, sign_roster + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["roster"]) + +_OWNER_SIGNER: LoadedSigner | None = None + + +def _owner_signer(settings) -> LoadedSigner: + """Cached owner signer. Prefers ``EIREL_OWNER_MNEMONIC``; falls back to the + owner wallet files already used to resolve ``owner_hotkey_ss58``. 503 when + no signing key is available (the owner can't sign rosters without it).""" + global _OWNER_SIGNER + if _OWNER_SIGNER is None: + try: + _OWNER_SIGNER = load_signer( + mnemonic=os.getenv("EIREL_OWNER_MNEMONIC") or None, + wallet_name=getattr(settings, "owner_wallet_name", None) or None, + hotkey_name=getattr(settings, "owner_hotkey_name", None) or None, + wallet_path=getattr(settings, "owner_wallet_path", None) or None, + ) + except Exception as exc: # noqa: BLE001 — surface as a clean 503 + raise HTTPException( + status_code=503, + detail=( + "owner signing key not configured " + "(set EIREL_OWNER_MNEMONIC or the owner wallet)" + ), + ) from exc + return _OWNER_SIGNER + + +def _reset_owner_signer() -> None: + """Test hook — drop the cached signer so a test can install its own key.""" + global _OWNER_SIGNER + _OWNER_SIGNER = None + + +def _task_bundle_ref(run_id: str, family_id: str) -> str: + template = os.getenv( + "EIREL_EVAL_POOL_KEY_TEMPLATE", "{family_id}/pool-run-{run_id}.json" + ) + return template.format(family_id=family_id, run_id=run_id) + + +def _member_integrity(session, snapshot: EpochTargetSnapshot) -> list[dict]: + """Map the snapshot's frozen members to integrity records, joining each + member's submission_id to its committed ``archive_sha256``. Members whose + submission row is missing are skipped (they can't be run anyway).""" + members: list[dict] = [] + for member in (snapshot.members_json or []): + meta = (member or {}).get("metadata") or {} + submission_id = meta.get("submission_id") + if not submission_id: + continue + submission = session.get(ManagedMinerSubmission, submission_id) + if submission is None: + continue + members.append( + { + "hotkey": member.get("hotkey") or submission.miner_hotkey, + "submission_id": submission_id, + "deployment_id": meta.get("deployment_id") or "", + "archive_sha256": submission.archive_sha256, + # The miner's parsed manifest (entry module, runtime port, + # resources) — carried under the owner signature so a local-exec + # validator can build the pod without a separate unsigned fetch. + "manifest_json": submission.manifest_json or {}, + } + ) + return members + + +@router.get("/v1/runs/{run_id}/roster") +async def get_run_roster( + request: Request, + run_id: str, + family_id: str = Query(min_length=1, max_length=64), + validator_hotkey: str = Depends(validator_dependency), +) -> dict: + del validator_hotkey # auth-only — registered active validators may read + services: ManagedOwnerServices = request.app.state.services + signer = _owner_signer(services.settings) + with services.db.sessionmaker() as session: + snapshot = session.execute( + select(EpochTargetSnapshot).where( + EpochTargetSnapshot.run_id == run_id, + EpochTargetSnapshot.family_id == family_id, + ) + ).scalar_one_or_none() + if snapshot is None: + raise HTTPException( + status_code=404, + detail="no frozen snapshot for this run/family yet", + ) + members = _member_integrity(session, snapshot) + roster = build_roster( + run_id=run_id, + family_id=family_id, + benchmark_version=snapshot.benchmark_version, + rubric_version=snapshot.rubric_version, + judge_model=snapshot.judge_model, + task_bundle_ref=_task_bundle_ref(run_id, family_id), + members=members, + timestamp=datetime.now(UTC).isoformat(), + ) + return sign_roster(signer.keypair, roster) + + +@router.get("/v1/runs/{run_id}/submissions/{submission_id}/archive") +async def get_submission_archive( + request: Request, + run_id: str, + submission_id: str, + validator_hotkey: str = Depends(validator_dependency), +) -> Response: + del run_id, validator_hotkey # auth-only; submission_id is the key + services: ManagedOwnerServices = request.app.state.services + with services.db.sessionmaker() as session: + submission = session.get(ManagedMinerSubmission, submission_id) + if submission is None: + raise HTTPException(status_code=404, detail="submission not found") + artifact = session.get(SubmissionArtifact, submission.artifact_id) + if artifact is None: + raise HTTPException(status_code=404, detail="submission artifact not found") + data = bytes(artifact.archive_bytes) + archive_sha256 = submission.archive_sha256 + return Response( + content=data, + media_type="application/gzip", + headers={ + "X-Archive-Sha256": archive_sha256, + "Content-Disposition": f'attachment; filename="{submission_id}.tar.gz"', + }, + ) diff --git a/control_plane/owner_api/routers/runtime.py b/control_plane/owner_api/routers/runtime.py index d880f6a..21d92e6 100644 --- a/control_plane/owner_api/routers/runtime.py +++ b/control_plane/owner_api/routers/runtime.py @@ -14,7 +14,7 @@ from shared.common.manifest import SubmissionManifest from shared.common.models import ManagedDeployment, ManagedMinerSubmission, ServingDeployment -from shared.common.job_token import generate_job_token +from shared.common.job_attribution import build_cost_tag, job_request_headers from eirel.schemas import AgentInvocationRequest, AgentInvocationResponse from control_plane.owner_api.dependencies import ( ensure_candidate_runtime_available, @@ -67,17 +67,14 @@ def _build_cost_tag(*, deployment_id: str, payload: dict[str, Any]) -> str: """Stable per-task tag used as the provider-proxy job_id. - Validator-issued requests carry ``turn_id`` (slim 0.3.0 contract); - consumer-chat-api requests do too. Falls back to ``task_id`` - (legacy 0.2.x) for older callers, then to deployment_id for any - request that has neither (e.g. orchestrator-internal probes — - those keep the deployment-sticky semantics they had before). + Thin payload-based wrapper over :func:`shared.common.job_attribution.build_cost_tag` + so owner-api and the decentralized-exec validator derive the identical tag. """ - turn_id = payload.get("turn_id") or payload.get("task_id") or "" - turn_id = str(turn_id).strip() - if turn_id: - return f"task-eval={turn_id};deployment={deployment_id}" - return f"miner-{deployment_id}" + return build_cost_tag( + deployment_id, + turn_id=str(payload.get("turn_id") or ""), + task_id=str(payload.get("task_id") or ""), + ) async def _fetch_proxy_cost(job_id: str) -> dict[str, Any] | None: @@ -528,16 +525,13 @@ async def _proxy_stream_lines_to_pod( non-``done`` events pass through byte-for-byte to the caller. """ url = f"{pod_endpoint.rstrip('/')}/v1/agent/infer/stream" - headers: dict[str, str] = {} - if cost_tag: - headers["X-Eirel-Job-Id"] = cost_tag - # Per-job tool credential bound to this exact job tag. The miner - # forwards it to the tool services; it cannot authenticate as any - # other miner's job because it never holds the signing key. - if _TOOL_JOB_SIGNING_KEY: - headers["X-Eirel-Job-Token"] = generate_job_token( - _TOOL_JOB_SIGNING_KEY, cost_tag, - ) + # Per-job id + tool credential bound to this exact job tag. The miner + # forwards the token to the tool services; it cannot authenticate as any + # other miner's job because it never holds the signing key. Same helper the + # decentralized-exec validator uses, so the tag/token are byte-identical. + headers: dict[str, str] = job_request_headers( + cost_tag=cost_tag, signing_key=_TOOL_JOB_SIGNING_KEY, + ) pending_done: dict[str, Any] | None = None trace_buffer: list[dict[str, Any]] = [] timeout = ( diff --git a/deploy/k8s/validator-stack/README.md b/deploy/k8s/validator-stack/README.md new file mode 100644 index 0000000..99e7090 --- /dev/null +++ b/deploy/k8s/validator-stack/README.md @@ -0,0 +1,64 @@ +# Validator-local execution stack (`EIREL_EXEC_MODE=local`) + +The services a validator runs in its **own** k3s so it can execute miner +submissions itself instead of through the owner proxy — making the +stake-weighted-median consensus aggregate first-hand observations rather than one +forgeable owner oracle. Apply this only if you run `EIREL_EXEC_MODE=local`. + +## What's here + +| Service | Port | Role | +|---|---|---| +| `provider-proxy` | 8092 | the validator's own metered LLM gateway (its own API keys) | +| `web-search` / `url-fetch` / `rag` / `sandbox` | 8085/8087/8088/8091 | tool services miners call directly | +| `ledger-sink` | 8000 | owner-api in `EIREL_OWNER_API_ROLE=ledger` — the local tool-call attestation sink | +| `redis` | 6379 | budget ledger + rate-limit store | + +All run in namespace **`eirel-system`** (labeled `name: eirel-system`). The miner +NetworkPolicy pure-pod egress branch matches these by `namespaceSelector {name: +eirel-system}` + `podSelector {app: }`, so miner pods in `eirel-miners` +reach them and nothing else (no open internet — LLM traffic is proxy-only). + +## Deploy + +```bash +# 1. Create the secret from the template (generate tokens with `openssl rand -hex 32`) +cp secret.example.yaml secret.yaml && $EDITOR secret.yaml +kubectl apply -f secret.yaml + +# 2. Apply the stack +kubectl apply -k . + +# 3. Point the validator-engine at it (env on the engine pod/compose): +# EIREL_EXEC_MODE=local +# EIREL_OWNER_HOTKEY= +# EIREL_TOOL_JOB_SIGNING_KEY= +# EIREL_LOCAL_PROVIDER_PROXY_URL=http://provider-proxy.eirel-system.svc.cluster.local:8092 +# EIREL_LOCAL_PROVIDER_PROXY_TOKEN= +# EIREL_LOCAL_LEDGER_URL=http://ledger-sink.eirel-system.svc.cluster.local:8000 +# EIREL_LOCAL_LEDGER_TOKEN= +# EIREL_VALIDATOR_RUNTIME_SYSTEM_NAMESPACE=eirel-system +# EIREL_VALIDATOR_RUNTIME_NAMESPACE=eirel-miners +# # Tool URLs the runtime manager injects into miner pods: +# EIREL_WEB_SEARCH_TOOL_URL=http://web-search-tool-service.eirel-system.svc.cluster.local:8085 +# EIREL_URL_FETCH_TOOL_URL=http://url-fetch-tool-service.eirel-system.svc.cluster.local:8087 +# EIREL_RAG_TOOL_URL=http://rag-tool-service.eirel-system.svc.cluster.local:8088 +# EIREL_SANDBOX_TOOL_SERVICE_URL=http://sandbox-tool-service.eirel-system.svc.cluster.local:8091 +``` + +## Notes / trust model + +- The signing key + tokens are **validator-generated** and identical across the + engine, proxy and tools; they never enter a miner pod. The miner only ever + forwards a per-job HMAC token it can't forge. +- The validator pays all eval LLM/tool spend; `EIREL_RUN_BUDGET_USD` (proxy) + caps per-miner-run LLM cost. +- `ledger-sink` uses SQLite on a **PVC** (`pvc.yaml`, default StorageClass — k3s + ships local-path) so the attestation ledger survives a pod restart mid-run. If + your cluster has no default StorageClass, swap the volume back to an `emptyDir` + (loss fail-closes attestation to empty, which is conservative). +- Residual owner trust after this: *who competes* + *what tasks* (both committed + in the owner-signed roster / hash-pinned archives). That's the commit-reveal + frontier, not this stack. +- For a host-NAT topology (proxy/tools in docker-compose, miners in k3s) use + `docker-compose.validator.local.yml` + set `SERVER_A_HOST_IP` instead. diff --git a/deploy/k8s/validator-stack/configmap.yaml b/deploy/k8s/validator-stack/configmap.yaml new file mode 100644 index 0000000..c78ecf8 --- /dev/null +++ b/deploy/k8s/validator-stack/configmap.yaml @@ -0,0 +1,18 @@ +# Non-secret config shared by the validator-local stack (envFrom on every pod). +# Secrets (signing keys, provider API keys, REDIS_URL) live in the Secret. +apiVersion: v1 +kind: ConfigMap +metadata: + name: validator-stack-config + namespace: eirel-system +data: + # provider-proxy + EIREL_RUN_BUDGET_USD: "30.0" + EIREL_PROVIDER_CHUTES_BASE_URL: "https://llm.chutes.ai/v1/chat/completions" + EIREL_PROVIDER_PROXY_ALLOW_MOCK: "false" + # tool services → validator-local attestation sink (closes the loop locally) + OWNER_API_URL: "http://ledger-sink.eirel-system.svc.cluster.local:8000" + EIREL_WEB_SEARCH_TOOL_BACKEND: "catalog" + EIREL_WEB_SEARCH_TOOL_DEFAULT_MAX_REQUESTS: "10000" + EIREL_RAG_EMBEDDING_BASE_URL: "https://api.openai.com/v1" + EIREL_RAG_EMBEDDING_MODEL: "text-embedding-3-small" diff --git a/deploy/k8s/validator-stack/deployments.yaml b/deploy/k8s/validator-stack/deployments.yaml new file mode 100644 index 0000000..6ac092e --- /dev/null +++ b/deploy/k8s/validator-stack/deployments.yaml @@ -0,0 +1,273 @@ +# Validator-local stack (EIREL_EXEC_MODE=local), all in the eirel-system +# namespace so the miner NetworkPolicy pure-pod egress branch +# (namespaceSelector {name: eirel-system} + podSelector {app: ...}) routes +# miner pods to the provider-proxy and the four tool services. +# +# image tag is pinned by the kustomization (images: transformer). +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7 + args: ["redis-server", "--requirepass", "$(REDIS_PASSWORD)"] + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: validator-stack-secrets + key: REDIS_PASSWORD + ports: + - containerPort: 6379 + name: redis +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: provider-proxy + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: provider-proxy + template: + metadata: + labels: + app: provider-proxy + spec: + containers: + - name: provider-proxy + image: rendixnetwork/eirel-ai:latest + command: ["eirel-provider-proxy"] + envFrom: + - configMapRef: + name: validator-stack-config + - secretRef: + name: validator-stack-secrets + ports: + - containerPort: 8092 + name: http + readinessProbe: + httpGet: { path: /healthz, port: 8092 } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /healthz, port: 8092 } + initialDelaySeconds: 15 + periodSeconds: 20 +--- +# owner-api in ledger-only role — the validator-local tool-call attestation sink. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ledger-sink + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: ledger-sink + template: + metadata: + labels: + app: ledger-sink + spec: + containers: + - name: ledger-sink + image: rendixnetwork/eirel-ai:latest + command: ["owner-api"] + envFrom: + - secretRef: + name: validator-stack-secrets + env: + - name: EIREL_OWNER_API_ROLE + value: "ledger" + # SQLite on a PVC (pvc.yaml) so the attestation ledger survives a + # pod restart mid-run. (Replace the claim with an emptyDir if your + # cluster has no default StorageClass — loss fail-closes attestation.) + - name: DATABASE_URL + value: "sqlite+aiosqlite:////data/ledger.db" + ports: + - containerPort: 8000 + name: http + readinessProbe: + httpGet: { path: /healthz, port: 8000 } + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: ledger-data + mountPath: /data + volumes: + - name: ledger-data + persistentVolumeClaim: + claimName: ledger-data +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web-search-tool-service + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: web-search-tool-service + template: + metadata: + labels: + app: web-search-tool-service + spec: + containers: + - name: web-search-tool-service + image: rendixnetwork/eirel-ai:latest + command: ["web-search-tool-service"] + envFrom: + - configMapRef: + name: validator-stack-config + - secretRef: + name: validator-stack-secrets + ports: + - containerPort: 8085 + name: http + readinessProbe: + httpGet: { path: /healthz, port: 8085 } + initialDelaySeconds: 5 + periodSeconds: 10 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: url-fetch-tool-service + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: url-fetch-tool-service + template: + metadata: + labels: + app: url-fetch-tool-service + spec: + containers: + - name: url-fetch-tool-service + image: rendixnetwork/eirel-ai:latest + command: ["url-fetch-tool-service"] + envFrom: + - configMapRef: + name: validator-stack-config + - secretRef: + name: validator-stack-secrets + env: + - name: EIREL_URL_FETCH_PORT + value: "8087" + ports: + - containerPort: 8087 + name: http + readinessProbe: + httpGet: { path: /healthz, port: 8087 } + initialDelaySeconds: 5 + periodSeconds: 10 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rag-tool-service + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: rag-tool-service + template: + metadata: + labels: + app: rag-tool-service + spec: + containers: + - name: rag-tool-service + image: rendixnetwork/eirel-ai:latest + command: ["rag-tool-service"] + envFrom: + - configMapRef: + name: validator-stack-config + - secretRef: + name: validator-stack-secrets + env: + - name: EIREL_RAG_TOOL_PORT + value: "8088" + ports: + - containerPort: 8088 + name: http + readinessProbe: + httpGet: { path: /healthz, port: 8088 } + initialDelaySeconds: 5 + periodSeconds: 10 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sandbox-tool-service + namespace: eirel-system +spec: + replicas: 1 + selector: + matchLabels: + app: sandbox-tool-service + template: + metadata: + labels: + app: sandbox-tool-service + spec: + containers: + - name: sandbox-tool-service + image: rendixnetwork/eirel-ai:latest + command: ["sandbox-tool-service"] + envFrom: + - configMapRef: + name: validator-stack-config + - secretRef: + name: validator-stack-secrets + env: + - name: EIREL_SANDBOX_TOOL_PORT + value: "8091" + ports: + - containerPort: 8091 + name: http + # Same hardening as the owner stack: unprivileged, no caps, no new + # privs, read-only rootfs + scratch tmpfs, memory capped. + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + limits: + memory: "512Mi" + readinessProbe: + httpGet: { path: /healthz, port: 8091 } + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + - name: scratch + emptyDir: + sizeLimit: "64Mi" diff --git a/deploy/k8s/validator-stack/kustomization.yaml b/deploy/k8s/validator-stack/kustomization.yaml new file mode 100644 index 0000000..eb433e7 --- /dev/null +++ b/deploy/k8s/validator-stack/kustomization.yaml @@ -0,0 +1,13 @@ +# Validator-local execution stack (EIREL_EXEC_MODE=local). +# secret.yaml is intentionally NOT listed — create it from secret.example.yaml +# out-of-band (it holds the validator's generated tokens + provider keys). +resources: + - namespace.yaml + - configmap.yaml + - pvc.yaml + - services.yaml + - deployments.yaml + +images: + - name: rendixnetwork/eirel-ai + newTag: latest diff --git a/deploy/k8s/validator-stack/namespace.yaml b/deploy/k8s/validator-stack/namespace.yaml new file mode 100644 index 0000000..1c7c971 --- /dev/null +++ b/deploy/k8s/validator-stack/namespace.yaml @@ -0,0 +1,21 @@ +# Namespaces for a validator running decentralized execution (EIREL_EXEC_MODE=local). +# +# The miner NetworkPolicy (infra/miner_runtime/_k8s_helpers.py, pure-pod branch) +# matches the provider-proxy/tool pods via namespaceSelector {name: } +# + podSelector {app: ...}. The `name` LABEL on the namespace is what that +# selector matches — it must be present (a namespace's metadata.name is not a +# label), so set it explicitly here. Keep these in sync with the runtime +# manager's EIREL_VALIDATOR_RUNTIME_SYSTEM_NAMESPACE / _NAMESPACE. +apiVersion: v1 +kind: Namespace +metadata: + name: eirel-system + labels: + name: eirel-system +--- +apiVersion: v1 +kind: Namespace +metadata: + name: eirel-miners + labels: + name: eirel-miners diff --git a/deploy/k8s/validator-stack/pvc.yaml b/deploy/k8s/validator-stack/pvc.yaml new file mode 100644 index 0000000..2694e19 --- /dev/null +++ b/deploy/k8s/validator-stack/pvc.yaml @@ -0,0 +1,15 @@ +# Durable store for the ledger-sink's SQLite attestation DB, so a pod restart +# mid-run doesn't zero the tool-call ledger (which would fail-close attestation +# to empty for in-flight tasks). Uses the cluster default StorageClass +# (k3s ships local-path). 1Gi is ample for tool-call rows. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ledger-data + namespace: eirel-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/deploy/k8s/validator-stack/secret.example.yaml b/deploy/k8s/validator-stack/secret.example.yaml new file mode 100644 index 0000000..f6d1df5 --- /dev/null +++ b/deploy/k8s/validator-stack/secret.example.yaml @@ -0,0 +1,25 @@ +# TEMPLATE — copy to secret.yaml, fill in, and `kubectl apply` (do NOT commit +# the filled copy). The validator GENERATES the tokens itself (random 32+ bytes, +# e.g. `openssl rand -hex 32`); they must be IDENTICAL across the engine, +# provider-proxy and all four tool services, and never enter a miner pod. +apiVersion: v1 +kind: Secret +metadata: + name: validator-stack-secrets + namespace: eirel-system +type: Opaque +stringData: + # redis (password embedded in the URL the proxy/tools read) + REDIS_PASSWORD: "CHANGE_ME" + REDIS_URL: "redis://:CHANGE_ME@redis.eirel-system.svc.cluster.local:6379/0" + # per-job HMAC key: engine mints X-Eirel-Job-Token, proxy/tools verify + EIREL_TOOL_JOB_SIGNING_KEY: "CHANGE_ME" + # provider-proxy master token (engine presents it for /v1/jobs/*/cost) + EIREL_PROVIDER_PROXY_MASTER_TOKEN: "CHANGE_ME" + # tools → ledger-sink write auth; engine → ledger-sink read auth + EIREL_TOOL_LEDGER_TOKEN: "CHANGE_ME" + EIREL_LOCAL_LEDGER_TOKEN: "CHANGE_ME" + # the validator's OWN provider/tool API keys — it pays all eval spend + EIREL_PROVIDER_CHUTES_API_KEY: "CHANGE_ME" + EIREL_BRAVE_SEARCH_API_KEY: "" + EIREL_RAG_EMBEDDING_API_KEY: "" diff --git a/deploy/k8s/validator-stack/services.yaml b/deploy/k8s/validator-stack/services.yaml new file mode 100644 index 0000000..c073626 --- /dev/null +++ b/deploy/k8s/validator-stack/services.yaml @@ -0,0 +1,93 @@ +# ClusterIP services for the validator-local stack. Names match the DNS the +# runtime manager injects into miner pods (EIREL_PROVIDER_PROXY_URL, +# EIREL_*_TOOL_URL) and the ledger OWNER_API_URL. +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: eirel-system +spec: + selector: + app: redis + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: provider-proxy + namespace: eirel-system +spec: + selector: + app: provider-proxy + ports: + - name: http + port: 8092 + targetPort: 8092 +--- +apiVersion: v1 +kind: Service +metadata: + name: ledger-sink + namespace: eirel-system +spec: + selector: + app: ledger-sink + ports: + - name: http + port: 8000 + targetPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: web-search-tool-service + namespace: eirel-system +spec: + selector: + app: web-search-tool-service + ports: + - name: http + port: 8085 + targetPort: 8085 +--- +apiVersion: v1 +kind: Service +metadata: + name: url-fetch-tool-service + namespace: eirel-system +spec: + selector: + app: url-fetch-tool-service + ports: + - name: http + port: 8087 + targetPort: 8087 +--- +apiVersion: v1 +kind: Service +metadata: + name: rag-tool-service + namespace: eirel-system +spec: + selector: + app: rag-tool-service + ports: + - name: http + port: 8088 + targetPort: 8088 +--- +apiVersion: v1 +kind: Service +metadata: + name: sandbox-tool-service + namespace: eirel-system +spec: + selector: + app: sandbox-tool-service + ports: + - name: http + port: 8091 + targetPort: 8091 diff --git a/docker-compose.validator.local.yml b/docker-compose.validator.local.yml new file mode 100644 index 0000000..b3357a1 --- /dev/null +++ b/docker-compose.validator.local.yml @@ -0,0 +1,185 @@ +# ----------------------------------------------------------------------------- +# Decentralized-execution overlay for the validator stack (EIREL_EXEC_MODE=local) +# +# Apply ON TOP of docker-compose.validator.yml: +# docker compose -f docker-compose.validator.yml -f docker-compose.validator.local.yml up -d +# +# It adds the services a validator must run locally to execute miner +# submissions itself instead of through the owner proxy: +# - redis : budget ledger + rate-limit store for the proxy/tools +# - provider-proxy : the validator's own metered LLM gateway (its own keys) +# - 4 tool services : web_search / url_fetch / rag / sandbox (validator-paid) +# - ledger-sink : owner-api in EIREL_OWNER_API_ROLE=ledger — the local +# tool-call attestation sink the tools write to and the +# engine reads back (closes the attestation loop locally) +# +# Miner pods run in the validator's OWN k3s (KubernetesMinerRuntimeManager) and +# reach these over host-NAT: set SERVER_A_HOST_IP and DO leave SERVER_A_HOST_IP +# set in the runtime-manager env so the NetworkPolicy host_ip egress branch +# allows miner→proxy/tools on host ports 18092/18085/18087/18088/18091. +# (For an all-in-k3s topology, deploy these as pods labeled app=provider-proxy +# etc. in the validator's system_namespace instead; the pure-pod egress branch +# routes to them.) +# +# The validator GENERATES its own secrets (EIREL_TOOL_JOB_SIGNING_KEY, +# EIREL_PROVIDER_PROXY_MASTER_TOKEN, EIREL_TOOL_LEDGER_TOKEN) and supplies its +# own provider API keys. See .env.validator.example (local-exec section). +# ----------------------------------------------------------------------------- + +x-eirel-ai-local: &eirel-ai-local + image: ${DOCKERHUB_NAMESPACE:-rendixnetwork}/eirel-ai:${EIREL_AI_IMAGE_TAG:-latest} + working_dir: /app/eirel-ai + env_file: .env.validator + restart: unless-stopped + +services: + redis: + image: redis:7 + command: + - redis-server + - --requirepass + - ${REDIS_PASSWORD:?REDIS_PASSWORD must be set in .env.validator} + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set in .env.validator}:${REDIS_HOST_PORT:-16379}:6379" + restart: unless-stopped + + provider-proxy: + <<: *eirel-ai-local + command: ["eirel-provider-proxy"] + environment: + REDIS_URL: ${REDIS_URL} + EIREL_PROVIDER_PROXY_MASTER_TOKEN: ${EIREL_PROVIDER_PROXY_MASTER_TOKEN:?set EIREL_PROVIDER_PROXY_MASTER_TOKEN} + EIREL_PROVIDER_PROXY_ALLOW_MOCK: ${EIREL_PROVIDER_PROXY_ALLOW_MOCK:-false} + EIREL_RUN_BUDGET_USD: ${EIREL_RUN_BUDGET_USD:-30.0} + EIREL_PROVIDER_CHUTES_API_KEY: ${EIREL_PROVIDER_CHUTES_API_KEY:-} + EIREL_PROVIDER_CHUTES_BASE_URL: ${EIREL_PROVIDER_CHUTES_BASE_URL:-https://llm.chutes.ai/v1/chat/completions} + depends_on: + - redis + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set}:${PROVIDER_PROXY_HOST_PORT:-18092}:8092" + + # owner-api in ledger-only role: the validator-local tool-call attestation + # sink. Tools POST rows here (OWNER_API_URL) and the engine reads them back + # (EIREL_LOCAL_LEDGER_URL). SQLite store — no postgres needed. + ledger-sink: + <<: *eirel-ai-local + command: ["owner-api"] + environment: + EIREL_OWNER_API_ROLE: "ledger" + DATABASE_URL: ${EIREL_LEDGER_DATABASE_URL:-sqlite+aiosqlite:////data/ledger.db} + EIREL_TOOL_LEDGER_TOKEN: ${EIREL_TOOL_LEDGER_TOKEN:?set EIREL_TOOL_LEDGER_TOKEN} + EIREL_LOCAL_LEDGER_TOKEN: ${EIREL_LOCAL_LEDGER_TOKEN:-${EIREL_TOOL_LEDGER_TOKEN}} + volumes: + - eirel-ledger-data:/data + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set}:${LEDGER_SINK_HOST_PORT:-18000}:8000" + + web-search-tool-service: + <<: *eirel-ai-local + command: ["web-search-tool-service"] + environment: + EIREL_WEB_SEARCH_TOOL_BACKEND: ${EIREL_WEB_SEARCH_TOOL_BACKEND:-catalog} + EIREL_WEB_SEARCH_TOOL_API_TOKEN: ${EIREL_WEB_SEARCH_TOOL_API_TOKEN:-} + EIREL_BRAVE_SEARCH_API_KEY: ${EIREL_BRAVE_SEARCH_API_KEY:-} + EIREL_SERPER_API_KEY: ${EIREL_SERPER_API_KEY:-} + EIREL_TAVILY_API_KEY: ${EIREL_TAVILY_API_KEY:-} + EIREL_WEB_SEARCH_MOCK: ${EIREL_WEB_SEARCH_MOCK:-false} + EIREL_WEB_SEARCH_TOOL_DEFAULT_MAX_REQUESTS: ${EIREL_WEB_SEARCH_TOOL_DEFAULT_MAX_REQUESTS:-10000} + EIREL_TOOL_JOB_SIGNING_KEY: ${EIREL_TOOL_JOB_SIGNING_KEY:-} + EIREL_TOOL_LEDGER_TOKEN: ${EIREL_TOOL_LEDGER_TOKEN:-} + OWNER_API_URL: ${EIREL_LOCAL_LEDGER_URL:-http://ledger-sink:8000} + REDIS_URL: ${REDIS_URL} + depends_on: + - redis + - ledger-sink + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set}:${WEB_SEARCH_TOOL_SERVICE_HOST_PORT:-18085}:8085" + + url-fetch-tool-service: + <<: *eirel-ai-local + command: ["url-fetch-tool-service"] + environment: + EIREL_URL_FETCH_API_TOKEN: ${EIREL_URL_FETCH_API_TOKEN:-} + EIREL_URL_FETCH_PORT: 8087 + EIREL_URL_FETCH_DEFAULT_MAX_REQUESTS: ${EIREL_URL_FETCH_DEFAULT_MAX_REQUESTS:-10000} + EIREL_TOOL_JOB_SIGNING_KEY: ${EIREL_TOOL_JOB_SIGNING_KEY:-} + EIREL_TOOL_LEDGER_TOKEN: ${EIREL_TOOL_LEDGER_TOKEN:-} + OWNER_API_URL: ${EIREL_LOCAL_LEDGER_URL:-http://ledger-sink:8000} + REDIS_URL: ${REDIS_URL} + depends_on: + - redis + - ledger-sink + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set}:${URL_FETCH_TOOL_SERVICE_HOST_PORT:-18087}:8087" + + rag-tool-service: + <<: *eirel-ai-local + command: ["rag-tool-service"] + environment: + EIREL_RAG_TOOL_API_TOKEN: ${EIREL_RAG_TOOL_API_TOKEN:-} + EIREL_RAG_TOOL_PORT: 8088 + EIREL_RAG_EMBEDDING_BASE_URL: ${EIREL_RAG_EMBEDDING_BASE_URL:-https://api.openai.com/v1} + EIREL_RAG_EMBEDDING_API_KEY: ${EIREL_RAG_EMBEDDING_API_KEY:-${OPENAI_API_KEY:-}} + EIREL_RAG_EMBEDDING_MODEL: ${EIREL_RAG_EMBEDDING_MODEL:-text-embedding-3-small} + EIREL_TOOL_JOB_SIGNING_KEY: ${EIREL_TOOL_JOB_SIGNING_KEY:-} + EIREL_TOOL_LEDGER_TOKEN: ${EIREL_TOOL_LEDGER_TOKEN:-} + OWNER_API_URL: ${EIREL_LOCAL_LEDGER_URL:-http://ledger-sink:8000} + REDIS_URL: ${REDIS_URL} + depends_on: + - redis + - ledger-sink + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set}:${RAG_TOOL_SERVICE_HOST_PORT:-18088}:8088" + + sandbox-tool-service: + <<: *eirel-ai-local + command: ["sandbox-tool-service"] + # Same hardening as the owner stack: unprivileged, no caps, no new privs, + # read-only fs + scratch tmpfs, memory/pids capped. + user: "65534:65534" + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp:size=64m,mode=1777 + environment: + EIREL_SANDBOX_TOOL_API_TOKEN: ${EIREL_SANDBOX_TOOL_API_TOKEN:-} + EIREL_SANDBOX_TOOL_PORT: 8091 + EIREL_TOOL_JOB_SIGNING_KEY: ${EIREL_TOOL_JOB_SIGNING_KEY:-} + EIREL_TOOL_LEDGER_TOKEN: ${EIREL_TOOL_LEDGER_TOKEN:-} + OWNER_API_URL: ${EIREL_LOCAL_LEDGER_URL:-http://ledger-sink:8000} + REDIS_URL: ${REDIS_URL} + mem_limit: 512m + pids_limit: 100 + depends_on: + - redis + - ledger-sink + ports: + - "${SERVER_A_HOST_IP:?SERVER_A_HOST_IP must be set}:${SANDBOX_TOOL_SERVICE_HOST_PORT:-18091}:8091" + + # Merge local-exec env into the base validator-engine service. + validator-engine: + environment: + EIREL_EXEC_MODE: "local" + EIREL_OWNER_HOTKEY: ${EIREL_OWNER_HOTKEY:?pin the owner ss58 for roster verification} + EIREL_TOOL_JOB_SIGNING_KEY: ${EIREL_TOOL_JOB_SIGNING_KEY:?set EIREL_TOOL_JOB_SIGNING_KEY} + EIREL_LOCAL_PROVIDER_PROXY_URL: ${EIREL_LOCAL_PROVIDER_PROXY_URL:-http://provider-proxy:8092} + EIREL_LOCAL_PROVIDER_PROXY_TOKEN: ${EIREL_PROVIDER_PROXY_MASTER_TOKEN:-} + EIREL_LOCAL_LEDGER_URL: ${EIREL_LOCAL_LEDGER_URL:-http://ledger-sink:8000} + EIREL_LOCAL_LEDGER_TOKEN: ${EIREL_LOCAL_LEDGER_TOKEN:-${EIREL_TOOL_LEDGER_TOKEN:-}} + # k3s the validator runs miners in (host-NAT egress to the stack above). + EIREL_VALIDATOR_KUBECONFIG_PATH: ${EIREL_VALIDATOR_KUBECONFIG_PATH:-} + EIREL_VALIDATOR_RUNTIME_NAMESPACE: ${EIREL_VALIDATOR_RUNTIME_NAMESPACE:-eirel-miners} + EIREL_VALIDATOR_RUNTIME_SYSTEM_NAMESPACE: ${EIREL_VALIDATOR_RUNTIME_SYSTEM_NAMESPACE:-eirel-system} + EIREL_VALIDATOR_RUNTIME_IMAGE: ${EIREL_VALIDATOR_RUNTIME_IMAGE:-eirel-miner-runtime:latest} + EIREL_VALIDATOR_RUNTIME_SERVICE_DOMAIN: ${EIREL_VALIDATOR_RUNTIME_SERVICE_DOMAIN:-svc.cluster.local} + depends_on: + provider-proxy: + condition: service_started + ledger-sink: + condition: service_started + +volumes: + eirel-ledger-data: diff --git a/infra/miner_runtime/_k8s_helpers.py b/infra/miner_runtime/_k8s_helpers.py index 5f5aca8..4a326a5 100644 --- a/infra/miner_runtime/_k8s_helpers.py +++ b/infra/miner_runtime/_k8s_helpers.py @@ -294,6 +294,19 @@ def _build_k8s_service(*, name: str, port: int) -> dict[str, Any]: } +# Tool services a miner pod may call directly (their URLs are injected into the +# pod env). In the pure-pod egress branch these need explicit allow rules +# alongside the provider-proxy — otherwise miners can reach the LLM gateway but +# not web_search/url_fetch/rag/sandbox, silently breaking retrieval tasks. The +# host_ip branch already NATs these via host ports. +_TOOL_SERVICE_EGRESS: tuple[tuple[str, int], ...] = ( + ("web-search-tool-service", 8085), + ("url-fetch-tool-service", 8087), + ("rag-tool-service", 8088), + ("sandbox-tool-service", 8091), +) + + def _build_network_policy( *, name: str, @@ -317,13 +330,16 @@ def _build_network_policy( ] egress_rules: list[dict[str, Any]] = [] if host_ip: + # Host-NAT ports the validator/owner binds the stack to: + # 18092 provider-proxy, 18085 web_search, 18087 url_fetch, + # 18088 rag, 18091 sandbox (matches the *_HOST_PORT compose bindings). egress_rules.append({ "to": [{"ipBlock": {"cidr": f"{host_ip}/32"}}], "ports": [ {"port": 18092, "protocol": "TCP"}, {"port": 18085, "protocol": "TCP"}, - {"port": 18086, "protocol": "TCP"}, {"port": 18087, "protocol": "TCP"}, + {"port": 18088, "protocol": "TCP"}, {"port": 18091, "protocol": "TCP"}, ], }) @@ -341,6 +357,22 @@ def _build_network_policy( ], "ports": [{"port": 8092}], }) + # Direct egress to each tool service (in-cluster, same namespace). One + # rule per tool so each is reachable only on its own port. + for tool_app, tool_port in _TOOL_SERVICE_EGRESS: + egress_rules.append({ + "to": [ + { + "namespaceSelector": { + "matchLabels": {"name": system_namespace}, + }, + "podSelector": { + "matchLabels": {"app": tool_app}, + }, + }, + ], + "ports": [{"port": tool_port}], + }) egress_rules.append({ "to": [ { diff --git a/shared/common/config.py b/shared/common/config.py index 8203205..f9359af 100644 --- a/shared/common/config.py +++ b/shared/common/config.py @@ -119,6 +119,13 @@ class Settings: launch_mode: str = field( default_factory=lambda: os.getenv("LAUNCH_MODE", "development") ) + # owner-api process role. "full" (default) = the complete owner-api. + # "ledger" = a stripped tool-call attestation sink a decentralized-exec + # validator runs locally (only health + /v1/internal/eval/* against a small + # local store; no runtime manager / scheduler / background loops). + owner_api_role: str = field( + default_factory=lambda: os.getenv("EIREL_OWNER_API_ROLE", "full").strip().lower() + ) use_redis_pool: bool = field( default_factory=lambda: _bool_env("USE_REDIS_POOL", False) ) diff --git a/shared/common/job_attribution.py b/shared/common/job_attribution.py new file mode 100644 index 0000000..f71f7fb --- /dev/null +++ b/shared/common/job_attribution.py @@ -0,0 +1,44 @@ +"""Per-job cost attribution: the stable job tag + the headers stamped onto a +miner invocation so the provider-proxy/tool services attribute and authorize the +miner's LLM/tool spend. + +Today only owner-api (``routers/runtime.py``) builds these, because it is the +sole thing that invokes miners. In decentralized execution the *validator* +invokes miners directly through its own provider-proxy, so it must stamp the +identical tag + token. This module is the single source of that logic, imported +by both sides — drift here would mean a validator's proxy can't attribute the +cost it just paid for. + +The per-job ``X-Eirel-Job-Token`` is an HMAC the miner forwards but never can +forge (it doesn't hold the signing key), so it can only authenticate as its own +job. The per-run budget header (``X-Eirel-Run-Budget-Usd``) is NOT stamped here — +the miner SDK sets it from the pod's ``EIREL_RUN_BUDGET_USD`` env (injected by +the runtime manager), not by the invoker. +""" +from __future__ import annotations + +from shared.common.job_token import generate_job_token + + +def build_cost_tag(deployment_id: str, *, turn_id: str = "", task_id: str = "") -> str: + """Stable per-task provider-proxy job_id. + + Prefers ``turn_id`` (slim contract), then ``task_id`` (legacy), else falls + back to a deployment-sticky tag for requests that carry neither. Identical + to owner-api's historical ``_build_cost_tag``. + """ + tag = (str(turn_id).strip() or str(task_id).strip()) + if tag: + return f"task-eval={tag};deployment={deployment_id}" + return f"miner-{deployment_id}" + + +def job_request_headers(*, cost_tag: str, signing_key: str) -> dict[str, str]: + """Headers to stamp on a miner invocation: the job id and (when a signing + key is configured) the per-job HMAC token the tool services verify.""" + if not cost_tag: + return {} + headers = {"X-Eirel-Job-Id": cost_tag} + if signing_key: + headers["X-Eirel-Job-Token"] = generate_job_token(signing_key, cost_tag) + return headers diff --git a/shared/common/roster.py b/shared/common/roster.py new file mode 100644 index 0000000..f8482e7 --- /dev/null +++ b/shared/common/roster.py @@ -0,0 +1,137 @@ +"""Owner-signed run roster — the anti-equivocation backbone for decentralized +execution. + +When validators run miner submissions on their *own* infrastructure (instead of +through the owner's HTTP proxy), they must agree on exactly two things the owner +still controls: *which* submissions compete in a run, and *what code* each one +is. A malicious owner could otherwise hand validator A submission X and +validator B a tampered X'. The roster closes that: the owner serves a single +document — per run/family, listing each member's ``archive_sha256`` — signed by +the owner hotkey. Every validator fetches it, verifies the signature against the +*known* owner hotkey, and refuses to run any archive whose bytes don't hash to +the committed value. The owner cannot show two validators different rosters +without producing two conflicting signed statements. + +This module is the pure sign/verify core, imported by BOTH sides: owner-api +signs (``sign_roster``); the validator verifies (``verify_roster``) and looks up +the committed hash before executing (``archive_hash_for``). Signing/verification +use the same ``bittensor.Keypair`` primitives as the score-gossip layer. +""" +from __future__ import annotations + +import json +from typing import Any + + +def canonical_roster_bytes(roster: dict[str, Any]) -> bytes: + """Deterministic serialization of the roster body that gets signed. + + Sorted keys + compact separators so the exact same bytes are produced on + the signing side and every verifying side regardless of dict ordering. + """ + return json.dumps( + roster, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + + +def build_roster( + *, + run_id: str, + family_id: str, + benchmark_version: str | None, + rubric_version: str | None, + judge_model: str | None, + task_bundle_ref: str, + members: list[dict[str, Any]], + timestamp: str, +) -> dict[str, Any]: + """Assemble the canonical roster body. + + ``members`` is normalized to exactly ``{hotkey, submission_id, + deployment_id, archive_sha256, manifest_json}`` per entry and **sorted** (by + hotkey then submission_id) so the signed bytes don't depend on DB row order. + + ``manifest_json`` rides inside the signed body so the validator can build the + pod (entry module, runtime port, resources) from tamper-evident config — the + owner can't hand one validator a different manifest than another. + """ + norm = sorted( + ( + { + "hotkey": str(m["hotkey"]), + "submission_id": str(m["submission_id"]), + "deployment_id": str(m.get("deployment_id") or ""), + "archive_sha256": str(m["archive_sha256"]), + "manifest_json": m.get("manifest_json") or {}, + } + for m in members + ), + key=lambda m: (m["hotkey"], m["submission_id"]), + ) + return { + "run_id": str(run_id), + "family_id": str(family_id), + "benchmark_version": str(benchmark_version or ""), + "rubric_version": str(rubric_version or ""), + "judge_model": str(judge_model or ""), + "task_bundle_ref": str(task_bundle_ref or ""), + "members": norm, + "timestamp": str(timestamp), + } + + +def sign_roster(keypair: Any, roster: dict[str, Any]) -> dict[str, Any]: + """Wrap ``roster`` with the owner hotkey + a signature over its canonical + bytes. The returned envelope is what the roster endpoint serves.""" + signature = keypair.sign(canonical_roster_bytes(roster)) + return { + "roster": roster, + "owner_hotkey": keypair.ss58_address, + "signature": "0x" + signature.hex(), + } + + +def verify_roster( + signed: dict[str, Any], *, expected_owner_hotkey: str | None = None +) -> bool: + """True iff the signature is valid for ``owner_hotkey`` over the roster body. + + When ``expected_owner_hotkey`` is given (the validator's pinned owner + identity), the served hotkey must equal it — otherwise an attacker could + serve a roster validly signed by *some other* key. ``bittensor`` is imported + lazily so this module stays importable in pure-logic test contexts. + """ + try: + roster = signed["roster"] + owner_hotkey = str(signed["owner_hotkey"]) + sig_hex = str(signed["signature"]) + except (KeyError, TypeError): + return False + if not isinstance(roster, dict): + return False + if expected_owner_hotkey is not None and owner_hotkey != str(expected_owner_hotkey): + return False + if sig_hex.startswith("0x"): + sig_hex = sig_hex[2:] + try: + signature = bytes.fromhex(sig_hex) + except ValueError: + return False + try: + import bittensor as bt + + keypair = bt.Keypair(ss58_address=owner_hotkey) + return bool(keypair.verify(canonical_roster_bytes(roster), signature)) + except Exception: + return False + + +def archive_hash_for(signed: dict[str, Any], submission_id: str) -> str | None: + """The committed ``archive_sha256`` for a submission in a (verified) roster, + or ``None`` if it isn't a member. Call ``verify_roster`` first.""" + roster = signed.get("roster") if isinstance(signed, dict) else None + members = (roster or {}).get("members") or [] + for member in members: + if str(member.get("submission_id")) == str(submission_id): + return str(member.get("archive_sha256")) + return None diff --git a/shared/core/judge_client.py b/shared/core/judge_client.py index c85ea8e..2c4d1fe 100644 --- a/shared/core/judge_client.py +++ b/shared/core/judge_client.py @@ -98,7 +98,6 @@ def judge_eval( *, bundle: dict[str, Any], expected_answer: str, - must_not_claim: list[str] | None = None, required_tool: str | None = None, oracle_source: str = "deterministic", ) -> dict[str, Any]: @@ -106,10 +105,10 @@ def judge_eval( The validator engine builds ``bundle`` (a JudgeInputBundle dict with task-shape fields + candidate response in ``answers[0]``) - and passes per-call extras alongside. ``expected_answer`` + - ``must_not_claim`` come from the validator's per-task in-memory - cache populated by the oracle/reconciler enrichment phase (or - from ``task.expected_output`` for ``deterministic`` items). + and passes per-call extras alongside. ``expected_answer`` comes + from the validator's per-task in-memory cache populated by the + oracle/reconciler enrichment phase (or from the deterministic + grader gold for ``deterministic`` items). ``oracle_source`` ∈ {``three_oracle``, ``deterministic``} signals which outcomes the judge is allowed to return — @@ -121,7 +120,6 @@ def judge_eval( body: dict[str, Any] = { "bundle": bundle, "expected_answer": expected_answer, - "must_not_claim": list(must_not_claim or []), "required_tool": required_tool, "oracle_source": oracle_source, } @@ -190,7 +188,6 @@ def judge_eval_composite( outcome: str, failure_mode: str | None = None, candidate_response: str = "", - must_not_claim: list[str] | None = None, required_tool: str | None = None, ledger_tools: list[str] | None = None, latency_ms: int = 0, @@ -214,7 +211,6 @@ def judge_eval_composite( "outcome": outcome, "failure_mode": failure_mode, "candidate_response": candidate_response, - "must_not_claim": list(must_not_claim or []), "required_tool": required_tool, "ledger_tools": list(ledger_tools or []), "latency_ms": int(max(0, latency_ms)), diff --git a/shared/scoring/multi_metric.py b/shared/scoring/multi_metric.py index 0d3c8eb..96d29da 100644 --- a/shared/scoring/multi_metric.py +++ b/shared/scoring/multi_metric.py @@ -42,7 +42,6 @@ "rag_required": "rag_required", "multi_turn_agentic_memory": "memory_required", "compute_or_orchestrate": "sandbox_required", - "abstention_probe": "no_tool", } diff --git a/tests/common/test_job_attribution.py b/tests/common/test_job_attribution.py new file mode 100644 index 0000000..b614a25 --- /dev/null +++ b/tests/common/test_job_attribution.py @@ -0,0 +1,38 @@ +"""Tests for the shared per-job cost-attribution helper. + +The property that matters: the owner (routers/runtime.py) and the +decentralized-exec validator derive the SAME tag + token for the same job, so a +validator's own provider-proxy can attribute the spend it just paid for. +""" +from __future__ import annotations + +from shared.common.job_attribution import build_cost_tag, job_request_headers +from shared.common.job_token import generate_job_token + + +def test_build_cost_tag_prefers_turn_then_task(): + assert build_cost_tag("dep-1", turn_id="t-9") == "task-eval=t-9;deployment=dep-1" + assert build_cost_tag("dep-1", task_id="task-9") == "task-eval=task-9;deployment=dep-1" + # turn_id wins when both present + assert build_cost_tag("dep-1", turn_id="t-9", task_id="task-9") == "task-eval=t-9;deployment=dep-1" + # neither → deployment-sticky fallback + assert build_cost_tag("dep-1") == "miner-dep-1" + + +def test_job_request_headers_match_manual_mint(): + key = "validator-local-signing-key" + tag = build_cost_tag("dep-1", turn_id="t-9") + headers = job_request_headers(cost_tag=tag, signing_key=key) + assert headers["X-Eirel-Job-Id"] == tag + # The token equals what a verifier (proxy/tool) computes for the same tag. + assert headers["X-Eirel-Job-Token"] == generate_job_token(key, tag) + + +def test_job_request_headers_no_token_without_key(): + headers = job_request_headers(cost_tag="task-eval=t;deployment=d", signing_key="") + assert headers == {"X-Eirel-Job-Id": "task-eval=t;deployment=d"} + assert "X-Eirel-Job-Token" not in headers + + +def test_job_request_headers_empty_tag(): + assert job_request_headers(cost_tag="", signing_key="k") == {} diff --git a/tests/common/test_roster.py b/tests/common/test_roster.py new file mode 100644 index 0000000..322e2ad --- /dev/null +++ b/tests/common/test_roster.py @@ -0,0 +1,126 @@ +"""Tests for the owner-signed run roster (anti-equivocation core). + +The property under test: a validator that pins the owner hotkey can detect ANY +tampering with which submissions compete or what code each one is — a changed +archive hash, an added/removed member, a wrong owner identity, or a forged +signature all fail verification. +""" +from __future__ import annotations + +import bittensor as bt + +from shared.common.roster import ( + archive_hash_for, + build_roster, + canonical_roster_bytes, + sign_roster, + verify_roster, +) + + +def _kp(): + return bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + + +def _roster(members): + return build_roster( + run_id="run-9", + family_id="general_chat", + benchmark_version="hivemind_v1", + rubric_version="governance-v1", + judge_model="local-rubric-judge", + task_bundle_ref="general_chat/pool-run-run-9.json", + members=members, + timestamp="2026-06-24T00:00:00+00:00", + ) + + +_MEMBERS = [ + {"hotkey": "5Bminer", "submission_id": "sub-b", "deployment_id": "dep-b", + "archive_sha256": "b" * 64, "manifest_json": {"entry_module": "app", "port": 8080}}, + {"hotkey": "5Aminer", "submission_id": "sub-a", "deployment_id": "dep-a", + "archive_sha256": "a" * 64, "manifest_json": {"entry_module": "app", "port": 9090}}, +] + + +def test_build_roster_sorts_members_deterministically(): + # Same members, opposite input order → identical canonical bytes. + r1 = _roster(_MEMBERS) + r2 = _roster(list(reversed(_MEMBERS))) + assert canonical_roster_bytes(r1) == canonical_roster_bytes(r2) + assert [m["hotkey"] for m in r1["members"]] == ["5Aminer", "5Bminer"] + + +def test_sign_verify_roundtrip(): + kp = _kp() + signed = sign_roster(kp, _roster(_MEMBERS)) + assert signed["owner_hotkey"] == kp.ss58_address + assert verify_roster(signed) is True + assert verify_roster(signed, expected_owner_hotkey=kp.ss58_address) is True + + +def test_verify_fails_on_tampered_archive_hash(): + kp = _kp() + signed = sign_roster(kp, _roster(_MEMBERS)) + # Owner (or a MITM) swaps one member's committed code hash. + signed["roster"]["members"][0]["archive_sha256"] = "f" * 64 + assert verify_roster(signed) is False + + +def test_manifest_is_signed_and_tamper_evident(): + kp = _kp() + roster = _roster(_MEMBERS) + # Manifest is carried per member (so the validator can build the pod). + assert roster["members"][0]["manifest_json"] == {"entry_module": "app", "port": 9090} + signed = sign_roster(kp, roster) + # Swapping the entry module (e.g. to hijack what code runs) breaks the sig. + signed["roster"]["members"][0]["manifest_json"]["entry_module"] = "evil" + assert verify_roster(signed) is False + + +def test_verify_fails_on_added_or_removed_member(): + kp = _kp() + signed = sign_roster(kp, _roster(_MEMBERS)) + signed["roster"]["members"].append( + {"hotkey": "5Cevil", "submission_id": "sub-c", "deployment_id": "", "archive_sha256": "c" * 64} + ) + assert verify_roster(signed) is False + + signed2 = sign_roster(kp, _roster(_MEMBERS)) + signed2["roster"]["members"].pop() + assert verify_roster(signed2) is False + + +def test_verify_fails_on_wrong_pinned_owner(): + kp = _kp() + other = _kp() + signed = sign_roster(kp, _roster(_MEMBERS)) + # Validator pinned a different owner hotkey than the one that signed. + assert verify_roster(signed, expected_owner_hotkey=other.ss58_address) is False + + +def test_verify_fails_on_forged_signature(): + # Attacker signs with their own key but claims the real owner's hotkey. + owner = _kp() + attacker = _kp() + signed = sign_roster(attacker, _roster(_MEMBERS)) + signed["owner_hotkey"] = owner.ss58_address # lie about who signed it + assert verify_roster(signed) is False + assert verify_roster(signed, expected_owner_hotkey=owner.ss58_address) is False + + +def test_verify_fails_on_malformed_envelope(): + assert verify_roster({}) is False + assert verify_roster({"roster": "not-a-dict", "owner_hotkey": "x", "signature": "0x00"}) is False + kp = _kp() + signed = sign_roster(kp, _roster(_MEMBERS)) + signed["signature"] = "0xnothex" + assert verify_roster(signed) is False + + +def test_archive_hash_for_lookup(): + kp = _kp() + signed = sign_roster(kp, _roster(_MEMBERS)) + assert archive_hash_for(signed, "sub-a") == "a" * 64 + assert archive_hash_for(signed, "sub-b") == "b" * 64 + assert archive_hash_for(signed, "sub-missing") is None diff --git a/tests/owner_api/test_capacity_pool_eval.py b/tests/owner_api/test_capacity_pool_eval.py index cb6382d..656097d 100644 --- a/tests/owner_api/test_capacity_pool_eval.py +++ b/tests/owner_api/test_capacity_pool_eval.py @@ -112,14 +112,12 @@ def test_migration_alters_preexisting_db_and_is_idempotent(): def test_reconciled_payload_round_trips(): original = ReconciledOracle( expected_claims=["a", "b"], - must_not_claim=["x"], oracle_status="consensus", vendor_costs={"openai": 0.0021}, vendor_citations={"openai": ["http://e.g/1"]}, ) restored = _reconciled_from_payload(_reconciled_to_payload(original)) assert restored.expected_claims == ["a", "b"] - assert restored.must_not_claim == ["x"] assert restored.oracle_status == "consensus" assert restored.vendor_costs == {"openai": 0.0021} diff --git a/tests/owner_api/test_dataset_loader_oracle_source_tag.py b/tests/owner_api/test_dataset_loader_oracle_source_tag.py index e43c964..382e26d 100644 --- a/tests/owner_api/test_dataset_loader_oracle_source_tag.py +++ b/tests/owner_api/test_dataset_loader_oracle_source_tag.py @@ -160,17 +160,16 @@ def test_bundle_with_mixed_oracle_sources(): def test_expected_output_unchanged_alongside_oracle_source(): """Adding oracle_source to the task model must not affect expected_output parsing — it carries the deterministic answer + - must_not_claim floor for items that don't need three_oracle - enrichment.""" + required_tool for items that don't need three_oracle enrichment.""" task = FamilyEvaluationTask.model_validate( _task_dict( oracle_source="deterministic", expected_output={ "answer": "Paris", - "must_not_claim": ["London", "Berlin"], + "required_tool": "web_search", }, ), ) assert task.oracle_source == "deterministic" assert task.expected_output["answer"] == "Paris" - assert task.expected_output["must_not_claim"] == ["London", "Berlin"] + assert task.expected_output["required_tool"] == "web_search" diff --git a/tests/owner_api/test_eval_feedback_endpoint.py b/tests/owner_api/test_eval_feedback_endpoint.py index 0ad1792..1168514 100644 --- a/tests/owner_api/test_eval_feedback_endpoint.py +++ b/tests/owner_api/test_eval_feedback_endpoint.py @@ -106,14 +106,14 @@ def test_upsert_carries_failure_mode_and_knockouts(tmp_path): eval_failure_mode="wrong_fact", eval_guidance="check the date", composite_score=0.0, - eval_knockout_reasons=["hallucination_knockout=0"], + eval_knockout_reasons=["grounded_correctness_below_floor: 0.30 < 0.60"], ) with db.sessionmaker() as session: row = session.query(EvalFeedback).one() assert row.outcome == "wrong" assert row.failure_mode == "wrong_fact" - assert row.knockout_reasons_json == ["hallucination_knockout=0"] + assert row.knockout_reasons_json == ["grounded_correctness_below_floor: 0.30 < 0.60"] def test_upsert_idempotent_on_run_miner_task_key(tmp_path): diff --git a/tests/owner_api/test_k8s_helpers.py b/tests/owner_api/test_k8s_helpers.py index e8ab46e..397e8c3 100644 --- a/tests/owner_api/test_k8s_helpers.py +++ b/tests/owner_api/test_k8s_helpers.py @@ -256,13 +256,63 @@ def test_manifest_emits_network_policy_when_emit_network_policy_true(): ingress_from = np["spec"]["ingress"][0]["from"] assert ingress_from[0]["namespaceSelector"] == {"matchLabels": {"name": "eirel-system"}} assert ingress_from[1]["namespaceSelector"] == {"matchLabels": {"name": "eirel-control-plane"}} - egress_dns = np["spec"]["egress"][1] - assert egress_dns["ports"] == [{"port": 53, "protocol": "UDP"}] + # DNS egress must be present (found by content — rule ordering may change + # as tool-service egress rules are added). + dns_rules = [r for r in np["spec"]["egress"] if r.get("ports") == [{"port": 53, "protocol": "UDP"}]] + assert len(dns_rules) == 1 # -- _build_network_policy (k8s backend) ----------- +def test_build_network_policy_pure_pod_allows_tool_services(): + """Pure-pod egress (no host_ip) must reach the provider-proxy AND every tool + service in the system namespace — otherwise retrieval/sandbox tasks break + because miners can't reach web_search/url_fetch/rag/sandbox.""" + np = _build_network_policy( + name="miner-abc", + submission_id="abc", + system_namespace="eirel-system", + control_plane_namespace="eirel-control-plane", + port=8080, + ) + + def _allowed(app_label): + for rule in np["spec"]["egress"]: + for peer in rule.get("to", []) or []: + if (peer.get("podSelector") or {}).get("matchLabels", {}).get("app") == app_label: + ns = (peer.get("namespaceSelector") or {}).get("matchLabels", {}).get("name") + return ns, [p["port"] for p in rule["ports"]] + return None + + assert _allowed("provider-proxy") == ("eirel-system", [8092]) + assert _allowed("web-search-tool-service") == ("eirel-system", [8085]) + assert _allowed("url-fetch-tool-service") == ("eirel-system", [8087]) + assert _allowed("rag-tool-service") == ("eirel-system", [8088]) + assert _allowed("sandbox-tool-service") == ("eirel-system", [8091]) + + +def test_build_network_policy_host_ip_branch_unchanged(): + """When host_ip is set, egress is the host-NAT ipBlock branch (no in-cluster + pod selectors) — the tool-service pod rules belong only to the pure-pod path.""" + np = _build_network_policy( + name="miner-abc", + submission_id="abc", + system_namespace="eirel-system", + port=8080, + host_ip="10.1.2.3", + ) + ip_rules = [ + r for r in np["spec"]["egress"] + if any((peer.get("ipBlock") or {}).get("cidr") == "10.1.2.3/32" for peer in r.get("to", [])) + ] + assert len(ip_rules) == 1 + # No in-cluster tool-service podSelector rules in the host_ip branch. + for rule in np["spec"]["egress"]: + for peer in rule.get("to", []) or []: + assert (peer.get("podSelector") or {}).get("matchLabels", {}).get("app") != "web-search-tool-service" + + def test_build_network_policy_includes_system_and_control_plane_namespaces(): np = _build_network_policy( name="miner-abc", diff --git a/tests/owner_api/test_ledger_role.py b/tests/owner_api/test_ledger_role.py new file mode 100644 index 0000000..262a3a7 --- /dev/null +++ b/tests/owner_api/test_ledger_role.py @@ -0,0 +1,45 @@ +"""Tests for the ledger-only owner-api role (validator-local attestation sink). + +Importing control_plane.owner_api.app in the default role only registers routes +(no DB/k8s), so the local read-auth helper can be unit-tested in-process. Full +app construction in each role is smoke-checked in isolated subprocesses (see +test_ledger_role_subprocess.py-style asserts run from the verification step). +""" +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from control_plane.owner_api.app import _ledger_read_auth + + +def _req(authorization: str | None = None) -> SimpleNamespace: + headers = {"Authorization": authorization} if authorization is not None else {} + return SimpleNamespace(headers=headers) + + +def test_ledger_read_auth_allows_when_no_token(monkeypatch): + monkeypatch.delenv("EIREL_LOCAL_LEDGER_TOKEN", raising=False) + monkeypatch.delenv("EIREL_TOOL_LEDGER_TOKEN", raising=False) + assert asyncio.run(_ledger_read_auth(_req())) == "local-ledger" + + +def test_ledger_read_auth_accepts_matching_bearer(monkeypatch): + monkeypatch.setenv("EIREL_LOCAL_LEDGER_TOKEN", "s3cret") + assert asyncio.run(_ledger_read_auth(_req("Bearer s3cret"))) == "local-ledger" + + +def test_ledger_read_auth_rejects_wrong_or_missing_bearer(monkeypatch): + monkeypatch.setenv("EIREL_LOCAL_LEDGER_TOKEN", "s3cret") + for bad in (None, "Bearer nope", "s3cret", ""): + with pytest.raises(Exception) as exc: + asyncio.run(_ledger_read_auth(_req(bad))) + assert getattr(exc.value, "status_code", None) == 401 + + +def test_ledger_read_auth_falls_back_to_tool_ledger_token(monkeypatch): + monkeypatch.delenv("EIREL_LOCAL_LEDGER_TOKEN", raising=False) + monkeypatch.setenv("EIREL_TOOL_LEDGER_TOKEN", "ledger-tok") + assert asyncio.run(_ledger_read_auth(_req("Bearer ledger-tok"))) == "local-ledger" diff --git a/tests/owner_api/test_roster_endpoint.py b/tests/owner_api/test_roster_endpoint.py new file mode 100644 index 0000000..e0ba533 --- /dev/null +++ b/tests/owner_api/test_roster_endpoint.py @@ -0,0 +1,125 @@ +"""Tests for the decentralized-execution distribution endpoints: + + * GET /v1/runs/{run_id}/roster (owner-signed) + * GET /v1/runs/{run_id}/submissions/{submission_id}/archive (raw bytes) + +Handler-level (no TestClient): a real SQLite DB is seeded with a snapshot + +submission + artifact, and the route functions are called directly with the +auth dependency injected — matching tests/owner_api/test_eval_job_ledger_endpoint.py. +""" +from __future__ import annotations + +import hashlib +from types import SimpleNamespace + +import bittensor as bt +import pytest + +from shared.common.database import Database +from shared.common.models import ( + EpochTargetSnapshot, + ManagedMinerSubmission, + SubmissionArtifact, +) +from shared.common.roster import verify_roster, archive_hash_for +from control_plane.owner_api.routers import roster as roster_router +from control_plane.owner_api.routers.roster import ( + get_run_roster, + get_submission_archive, +) + + +_ARCHIVE = b"\x1f\x8b\x08 fake-gzip-archive-bytes" +_SHA = hashlib.sha256(_ARCHIVE).hexdigest() + + +@pytest.fixture() +def owner_kp(monkeypatch): + mnemonic = bt.Keypair.generate_mnemonic() + kp = bt.Keypair.create_from_mnemonic(mnemonic) + monkeypatch.setenv("EIREL_OWNER_MNEMONIC", mnemonic) + roster_router._reset_owner_signer() + yield kp + roster_router._reset_owner_signer() + + +def _seed(tmp_path) -> Database: + db = Database(f"sqlite+aiosqlite:///{tmp_path / 'roster.db'}") + db.create_all() + with db.sessionmaker() as session: + session.add(SubmissionArtifact( + id="art-1", archive_bytes=_ARCHIVE, sha256=_SHA, + size_bytes=len(_ARCHIVE), manifest_json={"entry": "app:app"}, + )) + session.add(ManagedMinerSubmission( + id="sub-1", miner_hotkey="5Hminer", submission_seq=1, + family_id="general_chat", status="received", artifact_id="art-1", + manifest_json={"entry": "app:app"}, archive_sha256=_SHA, + submission_block=100, + )) + session.add(EpochTargetSnapshot( + id="snap-1", run_id="run-9", family_id="general_chat", + benchmark_version="hivemind_v1", rubric_version="governance-v1", + judge_model="local-rubric-judge", status="open", + frozen_validator_stakes_json={}, + members_json=[{ + "hotkey": "5Hminer", + "metadata": {"submission_id": "sub-1", "deployment_id": "dep-1"}, + }], + )) + session.commit() + return db + + +def _request(db) -> SimpleNamespace: + services = SimpleNamespace(db=db, settings=SimpleNamespace()) + return SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(services=services))) + + +async def test_roster_is_owner_signed_and_carries_archive_hash(tmp_path, owner_kp): + db = _seed(tmp_path) + signed = await get_run_roster( + _request(db), run_id="run-9", family_id="general_chat", + validator_hotkey="5Vvalidator", + ) + # Signed by the configured owner key, and a pinned validator verifies it. + assert signed["owner_hotkey"] == owner_kp.ss58_address + assert verify_roster(signed, expected_owner_hotkey=owner_kp.ss58_address) is True + # The committed code hash for the member is the submission's archive_sha256. + assert archive_hash_for(signed, "sub-1") == _SHA + assert signed["roster"]["task_bundle_ref"] == "general_chat/pool-run-run-9.json" + assert signed["roster"]["members"][0]["hotkey"] == "5Hminer" + # Manifest rides inside the signed roster so the validator can build the pod. + assert signed["roster"]["members"][0]["manifest_json"] == {"entry": "app:app"} + + +async def test_roster_404_when_no_snapshot(tmp_path, owner_kp): + db = _seed(tmp_path) + with pytest.raises(Exception) as exc: + await get_run_roster( + _request(db), run_id="run-404", family_id="general_chat", + validator_hotkey="5Vvalidator", + ) + assert getattr(exc.value, "status_code", None) == 404 + + +async def test_archive_endpoint_returns_verifiable_bytes(tmp_path, owner_kp): + db = _seed(tmp_path) + resp = await get_submission_archive( + _request(db), run_id="run-9", submission_id="sub-1", + validator_hotkey="5Vvalidator", + ) + assert resp.body == _ARCHIVE + # The header hash lets the validator confirm against the roster commitment. + assert resp.headers["X-Archive-Sha256"] == _SHA + assert hashlib.sha256(resp.body).hexdigest() == _SHA + + +async def test_archive_404_for_unknown_submission(tmp_path, owner_kp): + db = _seed(tmp_path) + with pytest.raises(Exception) as exc: + await get_submission_archive( + _request(db), run_id="run-9", submission_id="nope", + validator_hotkey="5Vvalidator", + ) + assert getattr(exc.value, "status_code", None) == 404 diff --git a/tests/validator/calibration/test_reconciler_agreement_gate.py b/tests/validator/calibration/test_reconciler_agreement_gate.py index db51117..7bca939 100644 --- a/tests/validator/calibration/test_reconciler_agreement_gate.py +++ b/tests/validator/calibration/test_reconciler_agreement_gate.py @@ -2,8 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable - import pytest from validation.validator.calibration.reconciler_agreement import ( @@ -31,14 +29,12 @@ async def reconcile( *, prompt: str, groundings: list[OracleGrounding], - must_not_claim_floor: Iterable[str] = (), ) -> ReconciledOracle: if not self._queue: raise RuntimeError("scripted reconciler exhausted") claims = self._queue.pop(0) return ReconciledOracle( expected_claims=claims, - must_not_claim=list(must_not_claim_floor), oracle_status="consensus", consensus_claims=claims, ) diff --git a/tests/validator/consensus/__init__.py b/tests/validator/consensus/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/validator/consensus/test_aggregate.py b/tests/validator/consensus/test_aggregate.py new file mode 100644 index 0000000..b8feb7b --- /dev/null +++ b/tests/validator/consensus/test_aggregate.py @@ -0,0 +1,155 @@ +"""Consensus-critical tests for stake-weighted aggregation + winner selection. + +These guard the two properties decentralized weight-setting rests on: + 1. >50%-honest-stake Byzantine tolerance (median, not mean). + 2. Determinism — identical inputs yield the identical winner on every validator. +""" +from __future__ import annotations + +from validation.validator.consensus.aggregate import ( + ValidatorReport, + aggregate_scores, + participating_stake, + select_winner, + stake_weighted_median, + winner_weight_vector, +) + + +# ── stake_weighted_median ──────────────────────────────────────────────────── + +def test_median_single_sample(): + assert stake_weighted_median([(0.7, 100.0)]) == 0.7 + + +def test_median_empty_or_zero_stake_is_none(): + assert stake_weighted_median([]) is None + assert stake_weighted_median([(0.9, 0.0), (0.1, -5.0)]) is None + + +def test_median_crosses_at_half_stake(): + # cumulative stake: 0.1@30, 0.5@30(=60), 0.9@40 → half=50 → 0.5 + assert stake_weighted_median([(0.9, 40.0), (0.1, 30.0), (0.5, 30.0)]) == 0.5 + + +def test_median_ignores_stake_below_the_crossing(): + # one tiny-stake outlier at 0.0 and a tiny one at 1.0 can't move a + # majority-stake cluster at 0.6 + samples = [(0.0, 1.0), (0.6, 100.0), (1.0, 1.0)] + assert stake_weighted_median(samples) == 0.6 + + +# ── Byzantine tolerance: <50% dishonest stake cannot move the winner ────────── + +def test_median_tolerates_dishonest_minority_stake(): + # 60% honest stake says miner is good (0.8); 40% dishonest says bad (0.0). + # Median lands on the honest value. + honest = [(0.8, 30.0), (0.8, 30.0)] # 60 stake + liars = [(0.0, 40.0)] # 40 stake + assert stake_weighted_median(honest + liars) == 0.8 + + +def test_mean_would_have_been_swung_but_median_is_not(): + # Same data: the mean would be dragged to ~0.48, flipping a ranking; + # the median holds at the honest 0.8. This is the whole point. + samples = [(0.8, 30.0), (0.8, 30.0), (0.0, 40.0)] + mean = sum(s * w for s, w in samples) / sum(w for _, w in samples) + assert mean < 0.5 + assert stake_weighted_median(samples) == 0.8 + + +def test_dishonest_majority_stake_does_move_it(): + # >50% dishonest stake CAN move the median — this is the documented + # security boundary (honest stake must be >50%), asserted explicitly. + samples = [(0.8, 40.0), (0.0, 60.0)] + assert stake_weighted_median(samples) == 0.0 + + +# ── aggregate_scores ───────────────────────────────────────────────────────── + +def test_aggregate_per_miner_median_over_union(): + reports = [ + ValidatorReport("vA", 50.0, {"m1": 0.9, "m2": 0.2}), + ValidatorReport("vB", 30.0, {"m1": 0.8, "m2": 0.3}), + ValidatorReport("vC", 20.0, {"m1": 0.1}), # only scored m1 + ] + out = aggregate_scores(reports) + # m1 samples: (0.9,50),(0.8,30),(0.1,20) sorted→0.1@20,0.8@50,0.9@100; half=50 →0.8 + assert out["m1"] == 0.8 + # m2 samples: (0.2,50),(0.3,30) sorted→0.2@50,0.3@80; half=40 →0.2 + assert out["m2"] == 0.2 + + +def test_aggregate_drops_zero_stake_validators(): + reports = [ + ValidatorReport("vA", 0.0, {"m1": 0.0}), # zero stake → ignored + ValidatorReport("vB", 100.0, {"m1": 0.9}), + ] + assert aggregate_scores(reports) == {"m1": 0.9} + + +def test_participating_stake(): + reports = [ + ValidatorReport("vA", 50.0, {"m1": 0.5}), + ValidatorReport("vB", 0.0, {"m1": 0.5}), + ValidatorReport("vC", 30.0, {"m1": 0.5}), + ] + assert participating_stake(reports) == 80.0 + + +# ── select_winner (dethrone-by-margin) ─────────────────────────────────────── + +def test_winner_no_incumbent_picks_top(): + assert select_winner({"a": 0.5, "b": 0.7}, incumbent=None, margin=0.05) == "b" + + +def test_incumbent_holds_within_margin(): + # challenger a=0.72 vs incumbent b=0.70 → 0.02 < 0.05 margin → b holds + assert select_winner({"a": 0.72, "b": 0.70}, incumbent="b", margin=0.05) == "b" + + +def test_challenger_unseats_when_clearing_margin(): + # challenger a=0.76 vs incumbent b=0.70 → 0.06 >= 0.05 → a wins + assert select_winner({"a": 0.76, "b": 0.70}, incumbent="b", margin=0.05) == "a" + + +def test_incumbent_on_top_keeps_crown(): + assert select_winner({"a": 0.9, "b": 0.4}, incumbent="a", margin=0.05) == "a" + + +def test_incumbent_absent_falls_back_to_top(): + assert select_winner({"a": 0.9, "b": 0.4}, incumbent="gone", margin=0.05) == "a" + + +def test_winner_empty_consensus_is_none(): + assert select_winner({}, incumbent="a", margin=0.05) is None + + +def test_winner_tiebreak_is_deterministic(): + # exact tie → highest score, then lexicographically smallest hotkey + assert select_winner({"b": 0.8, "a": 0.8}, incumbent=None, margin=0.05) == "a" + + +# ── winner_weight_vector ───────────────────────────────────────────────────── + +def test_weight_vector_winner_take_all(): + assert winner_weight_vector("a", ["a", "b", "c"], family_weight=1.0) == {"a": 1.0} + + +def test_weight_vector_empty_when_no_winner(): + assert winner_weight_vector(None, ["a", "b"]) == {} + assert winner_weight_vector("ghost", ["a", "b"]) == {} # winner not a known miner + + +# ── end-to-end determinism: two validators, same inputs → same winner ──────── + +def test_two_validators_same_inputs_same_winner(): + reports = [ + ValidatorReport("vA", 40.0, {"m1": 0.81, "m2": 0.79}), + ValidatorReport("vB", 35.0, {"m1": 0.80, "m2": 0.78}), + ValidatorReport("vC", 25.0, {"m1": 0.82, "m2": 0.77}), + ] + # Aggregate independently (order shuffled to prove order-independence). + out1 = select_winner(aggregate_scores(reports), incumbent=None, margin=0.0) + out2 = select_winner(aggregate_scores(list(reversed(reports))), incumbent=None, margin=0.0) + assert out1 == out2 == "m1" diff --git a/tests/validator/consensus/test_archive.py b/tests/validator/consensus/test_archive.py new file mode 100644 index 0000000..396462f --- /dev/null +++ b/tests/validator/consensus/test_archive.py @@ -0,0 +1,60 @@ +"""Tests for the durable signed score archive.""" +from __future__ import annotations + +import bittensor as bt + +from validation.validator.consensus.archive import ScoreArchive +from validation.validator.consensus.commitment import ( + build_commitment_payload, + verify_scores_against_commitment, +) +from validation.validator.consensus.gossip import sign_scores, verify_report + + +def test_write_read_roundtrip(tmp_path): + arch = ScoreArchive(tmp_path) + report = {"run_id": "run-9", "family_id": "general_chat", "scores": {"m": 0.7}, + "validator_hotkey": "5V", "timestamp": "t", "signature": "0xab"} + assert arch.write(report) is True + assert arch.read("run-9") == report + assert arch.read("run-404") is None + assert arch.runs() == ["run-9"] + + +def test_write_rejects_missing_run_id(tmp_path): + arch = ScoreArchive(tmp_path) + assert arch.write({"scores": {}}) is False + assert arch.runs() == [] + + +def test_run_id_is_sanitized_no_path_escape(tmp_path): + arch = ScoreArchive(tmp_path / "archive") + # A crafted run_id must not write outside the archive directory. + arch.write({"run_id": "../../etc/run", "scores": {}}) + # Nothing escaped the archive dir; the parent has no stray files. + assert not (tmp_path / "etc").exists() + written = list((tmp_path / "archive").glob("*.json")) + assert len(written) == 1 + assert written[0].parent == (tmp_path / "archive") + + +def test_archived_report_is_verifiable_and_matches_commitment(tmp_path): + """The audit property: the archived SIGNED report is the pre-image of the + on-chain commitment — its signature verifies and its scores hash to the + committed digest.""" + kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + scores = {"5Ha": 0.83, "5Hb": 0.41} + report = sign_scores(kp, run_id="run-9", family_id="general_chat", + scores=scores, timestamp="2026-06-24T00:00:00Z") + arch = ScoreArchive(tmp_path) + arch.write(report) + + fetched = arch.read("run-9") + # 1. signature verifies (authenticity survives the disk round-trip) + assert verify_report(fetched) is True + # 2. its scores hash to what the validator would commit on-chain + committed = build_commitment_payload(run_id="run-9", family_id="general_chat", scores=scores) + assert verify_scores_against_commitment( + run_id="run-9", family_id="general_chat", + scores=fetched["scores"], committed_payload=committed, + ) is True diff --git a/tests/validator/consensus/test_commitment.py b/tests/validator/consensus/test_commitment.py new file mode 100644 index 0000000..0b63fd5 --- /dev/null +++ b/tests/validator/consensus/test_commitment.py @@ -0,0 +1,95 @@ +"""Tests for on-chain score-hash commitment (anti-equivocation core).""" +from __future__ import annotations + +from validation.validator.consensus.commitment import ( + build_commitment_payload, + commit_score_hash, + fetch_all_commitments, + parse_commitment_payload, + score_commitment_hash, + verify_scores_against_commitment, +) + + +_SCORES = {"5Ha": 0.81, "5Hb": 0.42} + + +def test_hash_is_deterministic_and_order_independent(): + h1 = score_commitment_hash(run_id="run-9", family_id="general_chat", scores={"a": 0.5, "b": 0.6}) + h2 = score_commitment_hash(run_id="run-9", family_id="general_chat", scores={"b": 0.6, "a": 0.5}) + assert h1 == h2 and len(h1) == 64 + + +def test_hash_changes_with_scores_run_or_family(): + base = score_commitment_hash(run_id="run-9", family_id="general_chat", scores=_SCORES) + assert base != score_commitment_hash(run_id="run-9", family_id="general_chat", scores={"5Ha": 0.80, "5Hb": 0.42}) + assert base != score_commitment_hash(run_id="run-10", family_id="general_chat", scores=_SCORES) + assert base != score_commitment_hash(run_id="run-9", family_id="coding", scores=_SCORES) + + +def test_payload_build_parse_roundtrip(): + payload = build_commitment_payload(run_id="run-9", family_id="general_chat", scores=_SCORES) + assert payload.startswith("eirel/scores/v1:run-9:") + parsed = parse_commitment_payload(payload) + assert parsed is not None + run_id, digest = parsed + assert run_id == "run-9" + assert digest == score_commitment_hash(run_id="run-9", family_id="general_chat", scores=_SCORES) + + +def test_parse_rejects_malformed(): + assert parse_commitment_payload(None) is None + assert parse_commitment_payload("") is None + assert parse_commitment_payload("not-ours:run-9:" + "a" * 64) is None + assert parse_commitment_payload("eirel/scores/v1:run-9:tooshort") is None + assert parse_commitment_payload("eirel/scores/v1:" + "a" * 64) is None # missing run_id + + +def test_verify_matches_committed_scores(): + payload = build_commitment_payload(run_id="run-9", family_id="general_chat", scores=_SCORES) + assert verify_scores_against_commitment( + run_id="run-9", family_id="general_chat", scores=_SCORES, committed_payload=payload, + ) is True + + +def test_verify_rejects_equivocation_wrong_run_and_missing(): + payload = build_commitment_payload(run_id="run-9", family_id="general_chat", scores=_SCORES) + # equivocation: different scores served than committed + assert verify_scores_against_commitment( + run_id="run-9", family_id="general_chat", + scores={"5Ha": 0.10, "5Hb": 0.99}, committed_payload=payload, + ) is False + # commitment is for a different run + assert verify_scores_against_commitment( + run_id="run-10", family_id="general_chat", scores=_SCORES, committed_payload=payload, + ) is False + # no commitment at all + assert verify_scores_against_commitment( + run_id="run-9", family_id="general_chat", scores=_SCORES, committed_payload=None, + ) is False + + +class _FakeSubtensor: + def __init__(self, all_commitments=None): + self.committed = [] + self._all = all_commitments or {} + + def commit(self, *, wallet, netuid, data): + self.committed.append((netuid, data)) + return True + + def get_all_commitments(self, netuid, block=None): + return self._all + + +def test_commit_score_hash_publishes_payload(): + st = _FakeSubtensor() + ok = commit_score_hash(st, object(), netuid=7, run_id="run-9", family_id="general_chat", scores=_SCORES) + assert ok is True + assert st.committed == [(7, build_commitment_payload(run_id="run-9", family_id="general_chat", scores=_SCORES))] + + +def test_fetch_all_commitments_normalizes(): + st = _FakeSubtensor(all_commitments={"5Hpeer": "eirel/scores/v1:run-9:" + "a" * 64}) + out = fetch_all_commitments(st, netuid=7) + assert out == {"5Hpeer": "eirel/scores/v1:run-9:" + "a" * 64} diff --git a/tests/validator/consensus/test_discovery.py b/tests/validator/consensus/test_discovery.py new file mode 100644 index 0000000..9e927e4 --- /dev/null +++ b/tests/validator/consensus/test_discovery.py @@ -0,0 +1,78 @@ +"""Tests for metagraph-based peer discovery (no chain connection).""" +from __future__ import annotations + +from dataclasses import dataclass + +from validation.validator.consensus.discovery import ( + ValidatorInfo, + discover_validators, + stake_by_hotkey, +) + + +@dataclass +class FakeAxon: + ip: str | int + port: int + + +class FakeMetagraph: + """Duck-typed stand-in for a synced bittensor metagraph.""" + + def __init__(self, hotkeys, stake, validator_permit, axons): + self.hotkeys = hotkeys + self.S = stake + self.validator_permit = validator_permit + self.axons = axons + + +def test_discover_filters_permit_and_stake(): + mg = FakeMetagraph( + hotkeys=["vA", "vB", "miner", "vC"], + stake=[10000.0, 8000.0, 50.0, 200.0], + validator_permit=[True, True, False, True], # miner has no permit + axons=[FakeAxon("1.1.1.1", 8010), FakeAxon("2.2.2.2", 8010), + FakeAxon("3.3.3.3", 8010), FakeAxon("4.4.4.4", 8010)], + ) + vals = discover_validators(mg, min_stake=1000.0) + # vC has permit but stake 200 < 1000 → excluded; miner has no permit + assert [v.hotkey for v in vals] == ["vA", "vB"] + assert vals[0].uid == 0 and vals[0].stake == 10000.0 + + +def test_servable_and_endpoint(): + serving = ValidatorInfo("vA", 0, 9000.0, "10.0.0.5", 8010) + assert serving.servable + assert serving.endpoint() == "http://10.0.0.5:8010" + + not_serving = ValidatorInfo("vB", 1, 9000.0, "0.0.0.0", 0) + assert not not_serving.servable + + +def test_ipv6_endpoint_brackets(): + v = ValidatorInfo("vA", 0, 1.0, "2606:4700::1111", 8010) + assert v.endpoint() == "http://[2606:4700::1111]:8010" + + +def test_int_ip_is_decoded(): + # bittensor sometimes stores axon ip as an int + mg = FakeMetagraph( + hotkeys=["vA"], stake=[5000.0], validator_permit=[True], + axons=[FakeAxon(ip=int.from_bytes(bytes([1, 2, 3, 4]), "big"), port=8010)], + ) + [v] = discover_validators(mg) + assert v.ip == "1.2.3.4" and v.servable + + +def test_missing_permit_treated_as_permitted(): + # lite metagraph without validator_permit → don't drop everyone + mg = FakeMetagraph(hotkeys=["vA", "vB"], stake=[1.0, 2.0], + validator_permit=None, axons=None) + vals = discover_validators(mg) + assert {v.hotkey for v in vals} == {"vA", "vB"} + assert all(not v.servable for v in vals) # no axons → not servable + + +def test_stake_by_hotkey_snapshot(): + vals = [ValidatorInfo("vA", 0, 50.0, "", 0), ValidatorInfo("vB", 1, 30.0, "", 0)] + assert stake_by_hotkey(vals) == {"vA": 50.0, "vB": 30.0} diff --git a/tests/validator/consensus/test_divergence.py b/tests/validator/consensus/test_divergence.py new file mode 100644 index 0000000..cc4dbaa --- /dev/null +++ b/tests/validator/consensus/test_divergence.py @@ -0,0 +1,80 @@ +"""Tests for shadow-mode divergence metrics + the running tracker.""" +from __future__ import annotations + +import pytest + +from validation.validator.consensus.divergence import ( + DivergenceTracker, + spearman_rank_correlation, + top1_agreement, + topk_overlap, +) + + +def test_top1_agreement(): + assert top1_agreement("a", "a") is True + assert top1_agreement("a", "b") is False + assert top1_agreement(None, None) is True # neither side picked + assert top1_agreement("a", None) is False # one-sided + + +def test_spearman_identical_and_reversed(): + a = {"x": 0.9, "y": 0.5, "z": 0.1} + assert spearman_rank_correlation(a, a) == pytest.approx(1.0) + rev = {"x": 0.1, "y": 0.5, "z": 0.9} + assert spearman_rank_correlation(a, rev) == pytest.approx(-1.0) + + +def test_spearman_needs_two_common_keys(): + assert spearman_rank_correlation({"x": 1.0}, {"x": 1.0}) is None + assert spearman_rank_correlation({"x": 1.0}, {"y": 1.0}) is None + + +def test_spearman_handles_ties_no_variance(): + # All tied on one side → no rank variance → treated as agreement (1.0). + a = {"x": 0.5, "y": 0.5, "z": 0.5} + b = {"x": 0.9, "y": 0.5, "z": 0.1} + assert spearman_rank_correlation(a, b) == pytest.approx(1.0) + + +def test_topk_overlap(): + a = {"x": 0.9, "y": 0.5, "z": 0.1, "w": 0.05} + b = {"x": 0.8, "y": 0.6, "z": 0.2, "w": 0.9} # top-2 = {x,w} vs a's {x,y} + assert topk_overlap(a, a, 2) == pytest.approx(1.0) + assert topk_overlap(a, b, 2) == pytest.approx(1 / 3) # share {x}, union {x,y,w} + + +def test_tracker_agreement_rate_and_summary(): + t = DivergenceTracker(window=10) + assert t.agreement_rate() is None + t.record(run_id="r1", local_winner="a", owner_winner="a") + t.record(run_id="r2", local_winner="a", owner_winner="b") + t.record(run_id="r3", local_winner="c", owner_winner="c") + assert t.n_runs() == 3 + assert t.agreement_rate() == pytest.approx(2 / 3) + assert "winner_agreement=67%" in t.summary() + + +def test_tracker_dedupes_rearmed_run(): + t = DivergenceTracker(window=10) + t.record(run_id="r1", local_winner="a", owner_winner="b") # disagree + t.record(run_id="r1", local_winner="a", owner_winner="a") # re-armed → agree + assert t.n_runs() == 1 + assert t.agreement_rate() == pytest.approx(1.0) + + +def test_tracker_window_bounds_memory(): + t = DivergenceTracker(window=3) + for i in range(5): + t.record(run_id=f"r{i}", local_winner="a", owner_winner="a") + assert t.n_runs() == 3 # only the last 3 kept + + +def test_tracker_records_rank_correlation_when_both_maps_present(): + t = DivergenceTracker() + t.record( + run_id="r1", local_winner="x", owner_winner="x", + local_scores={"x": 0.9, "y": 0.5, "z": 0.1}, + owner_scores={"x": 0.8, "y": 0.6, "z": 0.2}, + ) + assert t.mean_rank_correlation() == pytest.approx(1.0) diff --git a/tests/validator/consensus/test_divergence_persist.py b/tests/validator/consensus/test_divergence_persist.py new file mode 100644 index 0000000..289b6ea --- /dev/null +++ b/tests/validator/consensus/test_divergence_persist.py @@ -0,0 +1,66 @@ +"""Persistence + signing of the shadow-mode divergence gate evidence.""" +from __future__ import annotations + +import bittensor as bt + +from validation.validator.consensus.divergence import ( + DivergenceTracker, + sign_divergence, + verify_divergence, +) + + +def _populate() -> DivergenceTracker: + t = DivergenceTracker(window=50) + t.record(run_id="run-7", local_winner="A", owner_winner="A", + local_scores={"A": 0.9, "B": 0.4}, owner_scores={"A": 1.0, "B": 0.0}) + t.record(run_id="run-8", local_winner="A", owner_winner="B") # disagreement + return t + + +def test_persist_load_roundtrip(tmp_path): + t = _populate() + path = tmp_path / "div.json" + t.persist(path) + + restored = DivergenceTracker(window=50) + restored.load(path) + assert restored.n_runs() == 2 + assert restored.agreement_rate() == t.agreement_rate() == 0.5 + assert [r.run_id for r in restored.ordered_records()] == ["run-7", "run-8"] + + +def test_load_missing_is_noop(tmp_path): + t = DivergenceTracker(window=50) + t.load(tmp_path / "nope.json") # must not raise + assert t.n_runs() == 0 + + +def test_owner_scores_light_up_rank_correlation(): + t = DivergenceTracker(window=50) + t.record(run_id="r", local_winner="A", owner_winner="A", + local_scores={"A": 0.9, "B": 0.4, "C": 0.1}, + owner_scores={"A": 1.0, "B": 0.5, "C": 0.2}) + # ≥2 common miners → rank ρ populates (perfect agreement here → 1.0) + assert t.mean_rank_correlation() == 1.0 + + +def test_sign_and_verify_summary_roundtrip(): + kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + signed = sign_divergence(kp, _populate(), weight_mode="shadow", timestamp="t") + assert signed["validator_hotkey"] == kp.ss58_address + assert signed["summary"]["n_runs"] == 2 + assert signed["summary"]["winner_agreement"] == 0.5 + assert verify_divergence(signed) is True + assert verify_divergence(signed, expected_hotkey=kp.ss58_address) is True + + +def test_verify_rejects_tampered_summary_or_wrong_hotkey(): + kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + other = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + signed = sign_divergence(kp, _populate(), weight_mode="shadow", timestamp="t") + # tamper the agreement rate after signing + tampered = {**signed, "summary": {**signed["summary"], "winner_agreement": 1.0}} + assert verify_divergence(tampered) is False + # pinned to a different owner hotkey + assert verify_divergence(signed, expected_hotkey=other.ss58_address) is False diff --git a/tests/validator/consensus/test_gossip.py b/tests/validator/consensus/test_gossip.py new file mode 100644 index 0000000..a6465d2 --- /dev/null +++ b/tests/validator/consensus/test_gossip.py @@ -0,0 +1,201 @@ +"""Tests for signed score gossip — authenticity + metagraph identity binding.""" +from __future__ import annotations + +import asyncio + +import bittensor as bt +import pytest + +from validation.validator.consensus.aggregate import ValidatorReport +from validation.validator.consensus.discovery import ValidatorInfo +from validation.validator.consensus.gossip import ( + collect_reports, + consensus_scores_path, + fetch_peer, + sign_scores, + verify_report, +) + + +@pytest.fixture +def keypair(): + return bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + + +# ── sign / verify ──────────────────────────────────────────────────────────── + +def test_sign_then_verify_roundtrip(keypair): + report = sign_scores( + keypair, run_id="run-9", family_id="general_chat", + scores={"m1": 0.8, "m2": 0.30000000000000004}, timestamp="2026-06-24T00:00:00Z", + ) + assert report["validator_hotkey"] == keypair.ss58_address + assert verify_report(report) is True + + +def test_verify_rejects_tampered_scores(keypair): + report = sign_scores(keypair, run_id="run-9", family_id="f", + scores={"m1": 0.5}, timestamp="t") + report["scores"]["m1"] = 0.99 # forge a higher score after signing + assert verify_report(report) is False + + +def test_verify_rejects_wrong_hotkey(keypair): + other = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + report = sign_scores(keypair, run_id="r", family_id="f", scores={"m": 0.5}, timestamp="t") + report["validator_hotkey"] = other.ss58_address # claim someone else signed + assert verify_report(report) is False + + +def test_verify_rejects_garbage(): + assert verify_report({}) is False + assert verify_report({"validator_hotkey": "x", "signature": "0xzz", + "run_id": "r", "family_id": "f", "timestamp": "t", + "scores": {}}) is False + + +# ── fetch_peer (fake HTTP client) ──────────────────────────────────────────── + +class _Resp: + def __init__(self, status, payload): + self.status_code = status + self._payload = payload + + def json(self): + return self._payload + + +class _Client: + def __init__(self, resp): + self._resp = resp + self.calls = [] + + async def get(self, url, timeout=None): + self.calls.append(url) + if isinstance(self._resp, Exception): + raise self._resp + return self._resp + + +def _signed(keypair, run_id="run-9", family_id="general_chat", scores=None): + return sign_scores(keypair, run_id=run_id, family_id=family_id, + scores=scores or {"m1": 0.7}, timestamp="t") + + +def test_fetch_peer_happy_path(keypair): + report = _signed(keypair) + vinfo = ValidatorInfo(keypair.ss58_address, 3, 5000.0, "10.0.0.2", 8010) + client = _Client(_Resp(200, report)) + out = asyncio.run(fetch_peer(client, vinfo, run_id="run-9", family_id="general_chat")) + assert isinstance(out, ValidatorReport) + assert out.hotkey == keypair.ss58_address and out.stake == 5000.0 + assert out.scores == {"m1": 0.7} + assert client.calls == ["http://10.0.0.2:8010" + consensus_scores_path("run-9")] + + +def test_fetch_peer_rejects_hotkey_mismatch(keypair): + # axon claims hotkey X but the served report is signed by keypair → reject + report = _signed(keypair) + vinfo = ValidatorInfo("5SomeOtherValidatorHotkey", 3, 5000.0, "10.0.0.2", 8010) + out = asyncio.run(fetch_peer(_Client(_Resp(200, report)), vinfo, + run_id="run-9", family_id="general_chat")) + assert out is None + + +def test_fetch_peer_rejects_wrong_run(keypair): + report = _signed(keypair, run_id="run-8") + vinfo = ValidatorInfo(keypair.ss58_address, 3, 5000.0, "10.0.0.2", 8010) + out = asyncio.run(fetch_peer(_Client(_Resp(200, report)), vinfo, + run_id="run-9", family_id="general_chat")) + assert out is None + + +def test_fetch_peer_handles_non200_and_errors(keypair): + vinfo = ValidatorInfo(keypair.ss58_address, 3, 5000.0, "10.0.0.2", 8010) + assert asyncio.run(fetch_peer(_Client(_Resp(503, {})), vinfo, + run_id="r", family_id="f")) is None + assert asyncio.run(fetch_peer(_Client(RuntimeError("conn refused")), vinfo, + run_id="r", family_id="f")) is None + + +def test_fetch_peer_skips_unservable(keypair): + vinfo = ValidatorInfo(keypair.ss58_address, 3, 5000.0, "0.0.0.0", 0) + # should not even attempt the call + client = _Client(_Resp(200, _signed(keypair))) + assert asyncio.run(fetch_peer(client, vinfo, run_id="r", family_id="f")) is None + assert client.calls == [] + + +# ── collect_reports: self (local) + verified peers ─────────────────────────── + +def test_collect_reports_self_plus_peers(keypair): + peer_kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + peer_report = _signed(peer_kp, scores={"m1": 0.6, "m2": 0.4}) + validators = [ + ValidatorInfo(keypair.ss58_address, 0, 7000.0, "0.0.0.0", 0), # self (no axon) + ValidatorInfo(peer_kp.ss58_address, 1, 3000.0, "10.0.0.9", 8010), # peer + ] + client = _Client(_Resp(200, peer_report)) + reports = asyncio.run(collect_reports( + client=client, validators=validators, + self_hotkey=keypair.ss58_address, self_scores={"m1": 0.9, "m2": 0.1}, + run_id="run-9", family_id="general_chat", + )) + by_hk = {r.hotkey: r for r in reports} + assert by_hk[keypair.ss58_address].stake == 7000.0 # self stake from metagraph + assert by_hk[keypair.ss58_address].scores == {"m1": 0.9, "m2": 0.1} # self from local + assert by_hk[peer_kp.ss58_address].stake == 3000.0 # peer stake from metagraph + assert by_hk[peer_kp.ss58_address].scores == {"m1": 0.6, "m2": 0.4} # peer from gossip + + +# ── anti-equivocation: scores must match the peer's on-chain commitment ─────── + +def test_fetch_peer_rejects_equivocation_when_commitment_required(keypair): + from validation.validator.consensus.commitment import build_commitment_payload + # Peer COMMITTED scores {m1: 0.7} but SERVES {m1: 0.99} (validly signed) → + # with enforcement on, the mismatch vs its on-chain commitment drops it. + served = _signed(keypair, scores={"m1": 0.99}) + committed = build_commitment_payload(run_id="run-9", family_id="general_chat", scores={"m1": 0.7}) + vinfo = ValidatorInfo(keypair.ss58_address, 3, 5000.0, "10.0.0.2", 8010) + client = _Client(_Resp(200, served)) + # rejected under enforcement + assert asyncio.run(fetch_peer( + client, vinfo, run_id="run-9", family_id="general_chat", + committed_payload=committed, require_commitment=True, + )) is None + # but accepted when enforcement is off (rollout) — signature still valid + out = asyncio.run(fetch_peer( + _Client(_Resp(200, served)), vinfo, run_id="run-9", family_id="general_chat", + committed_payload=committed, require_commitment=False, + )) + assert isinstance(out, ValidatorReport) + + +def test_fetch_peer_accepts_when_scores_match_commitment(keypair): + from validation.validator.consensus.commitment import build_commitment_payload + served = _signed(keypair, scores={"m1": 0.7}) + committed = build_commitment_payload(run_id="run-9", family_id="general_chat", scores={"m1": 0.7}) + vinfo = ValidatorInfo(keypair.ss58_address, 3, 5000.0, "10.0.0.2", 8010) + out = asyncio.run(fetch_peer( + _Client(_Resp(200, served)), vinfo, run_id="run-9", family_id="general_chat", + committed_payload=committed, require_commitment=True, + )) + assert isinstance(out, ValidatorReport) and out.scores == {"m1": 0.7} + + +def test_collect_reports_drops_uncommitted_peer_under_enforcement(keypair): + # Peer has NO on-chain commitment → dropped under enforcement; self survives. + peer_kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + validators = [ + ValidatorInfo(keypair.ss58_address, 0, 7000.0, "0.0.0.0", 0), # self + ValidatorInfo(peer_kp.ss58_address, 1, 3000.0, "10.0.0.9", 8010), # peer (uncommitted) + ] + client = _Client(_Resp(200, _signed(peer_kp, scores={"m1": 0.6}))) + reports = asyncio.run(collect_reports( + client=client, validators=validators, + self_hotkey=keypair.ss58_address, self_scores={"m1": 0.9}, + run_id="run-9", family_id="general_chat", + commitments={}, # no commitments known + require_commitment=True, + )) + assert [r.hotkey for r in reports] == [keypair.ss58_address] # only self diff --git a/tests/validator/consensus/test_score_store.py b/tests/validator/consensus/test_score_store.py new file mode 100644 index 0000000..ba76982 --- /dev/null +++ b/tests/validator/consensus/test_score_store.py @@ -0,0 +1,83 @@ +"""Tests for the validator-local score store.""" +from __future__ import annotations + +import pytest + +from validation.validator.consensus.score_store import ScoreStore, _coalesce_score + + +def test_coalesce_prefers_final_task_score(): + assert _coalesce_score({"final_task_score": 0.7, "agreement_score": 0.3}) == 0.7 + assert _coalesce_score({"agreement_score": 0.3}) == 0.3 + assert _coalesce_score({"final_task_score": 0.0}) == 0.0 # 0.0 is a real score + assert _coalesce_score({"verdict": "error"}) is None + + +def test_record_and_score_map_means_over_tasks(): + s = ScoreStore() + s.record("run-1", "general_chat", [ + {"miner_hotkey": "m1", "final_task_score": 0.8}, + {"miner_hotkey": "m2", "final_task_score": 0.2}, + ]) + s.record("run-1", "general_chat", [ + {"miner_hotkey": "m1", "final_task_score": 0.6}, # m1 mean → 0.7 + {"miner_hotkey": "m2", "agreement_score": 0.4}, # m2 mean → 0.3 + ]) + assert s.score_map("run-1", "general_chat") == pytest.approx({"m1": 0.7, "m2": 0.3}) + + +def test_record_skips_missing_hotkey_and_score(): + s = ScoreStore() + n = s.record("run-1", "f", [ + {"miner_hotkey": "", "final_task_score": 0.9}, # no hotkey + {"miner_hotkey": "m1"}, # no score + {"miner_hotkey": "m2", "final_task_score": 0.5}, # ok + ]) + assert n == 1 + assert s.score_map("run-1", "f") == {"m2": 0.5} + + +def test_record_no_op_on_empty_inputs(): + s = ScoreStore() + assert s.record("", "f", [{"miner_hotkey": "m"}]) == 0 + assert s.record("run", "", [{"miner_hotkey": "m"}]) == 0 + assert s.record("run", "f", []) == 0 + + +def test_score_map_unknown_run_is_empty(): + assert ScoreStore().score_map("nope", "f") == {} + + +def test_prune_drops_runs_not_kept(): + s = ScoreStore() + s.record("run-1", "f", [{"miner_hotkey": "m", "final_task_score": 0.5}]) + s.record("run-2", "f", [{"miner_hotkey": "m", "final_task_score": 0.5}]) + dropped = s.prune(keep={"run-2"}) + assert dropped == 1 + assert s.score_map("run-1", "f") == {} + assert s.score_map("run-2", "f") == {"m": 0.5} + + +def test_persist_and_load_roundtrip(tmp_path): + s = ScoreStore() + s.record("run-1", "general_chat", [ + {"miner_hotkey": "m1", "final_task_score": 0.8}, + {"miner_hotkey": "m1", "final_task_score": 0.6}, + {"miner_hotkey": "m2", "final_task_score": 0.2}, + ]) + path = tmp_path / "scores.json" + s.persist(path) + + s2 = ScoreStore() + s2.load(path) + assert s2.score_map("run-1", "general_chat") == pytest.approx({"m1": 0.7, "m2": 0.2}) + + +def test_load_missing_or_corrupt_file_is_safe(tmp_path): + s = ScoreStore() + s.load(tmp_path / "absent.json") # no raise + assert s.runs() == [] + bad = tmp_path / "bad.json" + bad.write_text("{not json", encoding="utf-8") + s.load(bad) # no raise + assert s.runs() == [] diff --git a/tests/validator/consensus/test_weights.py b/tests/validator/consensus/test_weights.py new file mode 100644 index 0000000..0f7106a --- /dev/null +++ b/tests/validator/consensus/test_weights.py @@ -0,0 +1,138 @@ +"""Tests for the consensus orchestration (compute_consensus + incumbent).""" +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +import bittensor as bt +import pytest + +from validation.validator.consensus.gossip import sign_scores +from validation.validator.consensus.weights import compute_consensus, current_incumbent + + +@dataclass +class FakeAxon: + ip: str + port: int + + +class FakeMetagraph: + def __init__(self, hotkeys, stake, permits, axons, incentive=None): + self.hotkeys = hotkeys + self.S = stake + self.validator_permit = permits + self.axons = axons + self.I = incentive + + +class _Resp: + def __init__(self, status, payload): + self.status_code = status + self._payload = payload + + def json(self): + return self._payload + + +class _RoutingClient: + """Returns a per-URL canned response (keyed by host).""" + + def __init__(self, by_host: dict[str, object]): + self._by_host = by_host + + async def get(self, url, timeout=None): + for host, resp in self._by_host.items(): + if host in url: + if isinstance(resp, Exception): + raise resp + return resp + return _Resp(404, {}) + + +def test_current_incumbent_is_max_incentive(): + mg = FakeMetagraph( + hotkeys=["v", "minerA", "minerB"], stake=[1, 0, 0], + permits=[True, False, False], axons=None, incentive=[0.0, 0.2, 0.7], + ) + assert current_incumbent(mg) == "minerB" + + +def test_current_incumbent_none_when_no_incentive(): + mg = FakeMetagraph(["a"], [1], [True], None, incentive=None) + assert current_incumbent(mg) is None + mg2 = FakeMetagraph(["a"], [1], [True], None, incentive=[0.0]) + assert current_incumbent(mg2) is None # all-zero incentive → no incumbent + + +def test_compute_consensus_quorum_and_winner(): + self_kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + peer_kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + + mg = FakeMetagraph( + hotkeys=[self_kp.ss58_address, peer_kp.ss58_address, "minerA", "minerB"], + stake=[6000.0, 4000.0, 0.0, 0.0], + permits=[True, True, False, False], + axons=[FakeAxon("0.0.0.0", 0), # self: no axon (uses local) + FakeAxon("10.0.0.2", 8010), # peer + FakeAxon("1.1.1.1", 1), FakeAxon("2.2.2.2", 1)], + incentive=[0.0, 0.0, 0.0, 0.0], + ) + # peer's signed report + peer_report = sign_scores(peer_kp, run_id="run-9", family_id="general_chat", + scores={"minerA": 0.82, "minerB": 0.40}, timestamp="t") + client = _RoutingClient({"10.0.0.2": _Resp(200, peer_report)}) + + result = asyncio.run(compute_consensus( + metagraph=mg, http_client=client, self_hotkey=self_kp.ss58_address, + run_id="run-9", family_id="general_chat", + local_scores={"minerA": 0.80, "minerB": 0.42}, + min_stake=0.0, margin=0.05, quorum_fraction=0.5, + )) + assert result.quorum_met is True # self(6000)+peer(4000)=10000 of 10000 + assert result.n_reports == 2 + # stake-weighted median per miner: minerA {0.80@6000, 0.82@4000} → 0.80; + # minerB {0.42@6000, 0.40@4000} → 0.42 → winner minerA + assert result.winner == "minerA" + assert result.weights_by_hotkey == {"minerA": 1.0} + + +def test_compute_consensus_no_quorum_holds(): + self_kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + mg = FakeMetagraph( + hotkeys=[self_kp.ss58_address, "peerHotkey", "minerA"], + stake=[2000.0, 8000.0, 0.0], # self only 2000 of 10000; peer unreachable + permits=[True, True, False], + axons=[FakeAxon("0.0.0.0", 0), FakeAxon("10.0.0.5", 8010), FakeAxon("1.1.1.1", 1)], + incentive=[0, 0, 0], + ) + client = _RoutingClient({"10.0.0.5": ConnectionError("down")}) # peer unreachable + result = asyncio.run(compute_consensus( + metagraph=mg, http_client=client, self_hotkey=self_kp.ss58_address, + run_id="run-9", family_id="general_chat", local_scores={"minerA": 0.9}, + quorum_fraction=0.5, + )) + # only self's 2000 stake (20%) < 50% quorum → hold, no winner + assert result.quorum_met is False + assert result.winner is None + assert result.weights_by_hotkey == {} + + +def test_compute_consensus_incumbent_hysteresis(): + self_kp = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + mg = FakeMetagraph( + hotkeys=[self_kp.ss58_address, "minerA", "minerB"], + stake=[10000.0, 0.0, 0.0], + permits=[True, False, False], + axons=[FakeAxon("0.0.0.0", 0), FakeAxon("1.1.1.1", 1), FakeAxon("2.2.2.2", 1)], + incentive=[0.0, 0.0, 0.6], # minerB is the incumbent + ) + client = _RoutingClient({}) + # challenger minerA only 0.03 above incumbent minerB → within margin → B holds + result = asyncio.run(compute_consensus( + metagraph=mg, http_client=client, self_hotkey=self_kp.ss58_address, + run_id="run-9", family_id="general_chat", + local_scores={"minerA": 0.71, "minerB": 0.68}, margin=0.05, + )) + assert result.incumbent == "minerB" + assert result.winner == "minerB" # hysteresis held the incumbent diff --git a/tests/validator/local_exec/__init__.py b/tests/validator/local_exec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/validator/local_exec/test_runtime_controller.py b/tests/validator/local_exec/test_runtime_controller.py new file mode 100644 index 0000000..d9cb9c0 --- /dev/null +++ b/tests/validator/local_exec/test_runtime_controller.py @@ -0,0 +1,209 @@ +"""Tests for LocalRunController — the fetch→verify→hash→deploy lifecycle, with +fake manager + http client + signer (no real cluster).""" +from __future__ import annotations + +import asyncio +import hashlib + +import bittensor as bt +import pytest + +from shared.common.roster import build_roster, sign_roster +from validation.validator.local_exec.runtime_controller import LocalRunController + + +_MANIFEST = {"agent": {"name": "m"}, "family_id": "general_chat"} + + +def _archive(tag: bytes) -> bytes: + return b"\x1f\x8b\x08 archive-" + tag + + +def _sha(b: bytes) -> str: + return hashlib.sha256(b).hexdigest() + + +class _Resp: + def __init__(self, *, json_body=None, content=b""): + self._json = json_body + self.content = content + + def raise_for_status(self): + return None + + def json(self): + return self._json + + +class _FakeHttp: + """Routes roster vs archive URLs to canned responses; counts calls.""" + + def __init__(self, *, roster_signed, archives: dict[str, bytes]): + self._roster = roster_signed + self._archives = archives + self.roster_calls = 0 + self.archive_calls = 0 + + async def get(self, url, headers=None): + if "/roster" in url: + self.roster_calls += 1 + return _Resp(json_body=self._roster) + for sub_id, data in self._archives.items(): + if f"/submissions/{sub_id}/archive" in url: + self.archive_calls += 1 + return _Resp(content=data) + return _Resp(content=b"") + + +class _FakeHandle: + def __init__(self, submission_id): + self.endpoint_url = f"http://miner-{submission_id}.local:8080" + + +class _FakeManager: + def __init__(self): + self.deployed: list[str] = [] + self.stopped: list[str] = [] + self.reconciled_with = None + + async def ensure_runtime(self, *, submission_id, deployment_id, archive_bytes, manifest, + provider_proxy_url, provider_proxy_token, + requested_cpu_millis, requested_memory_bytes): + self.deployed.append(submission_id) + return _FakeHandle(submission_id) + + async def stop_runtime(self, submission_id, *, reason): + self.stopped.append(submission_id) + + async def reconcile_active_submissions(self, ids): + self.reconciled_with = set(ids) + + +class _FakeSigner: + def signed_headers(self, method, path, body_hash): + return {"X-Hotkey": "5Vval", "X-Signature": "0xsig", "X-Timestamp": "t"} + + +def _signed_roster(owner_kp, members): + roster = build_roster( + run_id="run-9", family_id="general_chat", benchmark_version="b", + rubric_version="r", judge_model="j", task_bundle_ref="ref", + members=members, timestamp="t", + ) + return sign_roster(owner_kp, roster) + + +def _member(hotkey, sub_id, archive_bytes): + return {"hotkey": hotkey, "submission_id": sub_id, "deployment_id": f"dep-{sub_id}", + "archive_sha256": _sha(archive_bytes), "manifest_json": _MANIFEST} + + +def _controller(owner_kp, http, manager): + return LocalRunController( + manager=manager, http_client=http, signer=_FakeSigner(), + owner_hotkey=owner_kp.ss58_address, + provider_proxy_url="http://provider-proxy:8092", provider_proxy_token="tok", + ) + + +def test_ensure_run_deploys_verified_members(): + owner = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + a, b = _archive(b"A"), _archive(b"B") + signed = _signed_roster(owner, [_member("5Ha", "sub-a", a), _member("5Hb", "sub-b", b)]) + http = _FakeHttp(roster_signed=signed, archives={"sub-a": a, "sub-b": b}) + mgr = _FakeManager() + ctl = _controller(owner, http, mgr) + + res = asyncio.run(ctl.ensure_run( + run_id="run-9", family_id="general_chat", owner_url="http://owner", + wanted_hotkeys={"5Ha", "5Hb"}, + )) + assert res.endpoint_for("5Ha") == "http://miner-sub-a.local:8080" + assert res.deployment_id_for("5Hb") == "dep-sub-b" + assert set(mgr.deployed) == {"sub-a", "sub-b"} + assert res.failed == {} + + +def test_archive_hash_mismatch_is_refused_not_deployed(): + owner = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + a = _archive(b"A") + signed = _signed_roster(owner, [_member("5Ha", "sub-a", a)]) + # Serve DIFFERENT bytes than the roster committed. + http = _FakeHttp(roster_signed=signed, archives={"sub-a": _archive(b"TAMPERED")}) + mgr = _FakeManager() + res = asyncio.run(_controller(owner, http, mgr).ensure_run( + run_id="run-9", family_id="general_chat", owner_url="http://owner", + wanted_hotkeys={"5Ha"}, + )) + assert mgr.deployed == [] + assert "5Ha" in res.failed and "hash mismatch" in res.failed["5Ha"] + + +def test_wanted_hotkeys_bounds_footprint_to_claim_intersection(): + owner = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + a, b = _archive(b"A"), _archive(b"B") + signed = _signed_roster(owner, [_member("5Ha", "sub-a", a), _member("5Hb", "sub-b", b)]) + http = _FakeHttp(roster_signed=signed, archives={"sub-a": a, "sub-b": b}) + mgr = _FakeManager() + # Only asked to score 5Ha → only sub-a is booted. + res = asyncio.run(_controller(owner, http, mgr).ensure_run( + run_id="run-9", family_id="general_chat", owner_url="http://owner", + wanted_hotkeys={"5Ha"}, + )) + assert mgr.deployed == ["sub-a"] + assert res.endpoint_for("5Hb") is None + + +def test_unverifiable_roster_refuses_whole_run(): + owner = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + attacker = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + a = _archive(b"A") + # Roster signed by attacker but the controller pins the real owner hotkey. + signed = _signed_roster(attacker, [_member("5Ha", "sub-a", a)]) + http = _FakeHttp(roster_signed=signed, archives={"sub-a": a}) + mgr = _FakeManager() + ctl = _controller(owner, http, mgr) # owner_hotkey = real owner + with pytest.raises(RuntimeError): + asyncio.run(ctl.ensure_run( + run_id="run-9", family_id="general_chat", owner_url="http://owner", + wanted_hotkeys={"5Ha"}, + )) + assert mgr.deployed == [] + + +def test_ensure_run_is_memoized_per_run(): + owner = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + a = _archive(b"A") + signed = _signed_roster(owner, [_member("5Ha", "sub-a", a)]) + http = _FakeHttp(roster_signed=signed, archives={"sub-a": a}) + mgr = _FakeManager() + ctl = _controller(owner, http, mgr) + + async def _run(): + r1 = await ctl.ensure_run(run_id="run-9", family_id="general_chat", + owner_url="http://owner", wanted_hotkeys={"5Ha"}) + r2 = await ctl.ensure_run(run_id="run-9", family_id="general_chat", + owner_url="http://owner", wanted_hotkeys={"5Ha"}) + return r1, r2 + + r1, r2 = asyncio.run(_run()) + assert r1 is r2 + assert http.roster_calls == 1 # second call served from cache + assert mgr.deployed == ["sub-a"] + + +def test_teardown_stops_each_submission(): + owner = bt.Keypair.create_from_mnemonic(bt.Keypair.generate_mnemonic()) + a = _archive(b"A") + signed = _signed_roster(owner, [_member("5Ha", "sub-a", a)]) + http = _FakeHttp(roster_signed=signed, archives={"sub-a": a}) + mgr = _FakeManager() + ctl = _controller(owner, http, mgr) + + async def _run(): + await ctl.ensure_run(run_id="run-9", family_id="general_chat", + owner_url="http://owner", wanted_hotkeys={"5Ha"}) + await ctl.teardown_run("run-9") + + asyncio.run(_run()) + assert mgr.stopped == ["sub-a"] diff --git a/tests/validator/test_engine_oracle_helpers.py b/tests/validator/test_engine_oracle_helpers.py index ea74889..01b0202 100644 --- a/tests/validator/test_engine_oracle_helpers.py +++ b/tests/validator/test_engine_oracle_helpers.py @@ -18,10 +18,12 @@ import httpx import pytest +import validation.validator.engine as engine from validation.validator.engine import ( _build_oracle_layer, _enrich_task_oracle, _fetch_ledger_tools, + _should_use_cached_baseline, ) from validation.validator.oracles.base import OracleGrounding from validation.validator.reconciler import ReconciledOracle, Reconciler @@ -30,6 +32,22 @@ pytestmark = pytest.mark.asyncio +@pytest.mark.parametrize("exec_mode,cached,expected", [ + ("owner", {"response_text": "ref"}, True), # owner mode: cache is a fine optimization + ("owner", None, False), # nothing to use + ("owner", {}, False), # empty payload + ("local", {"response_text": "ref"}, False), # decentralized: NEVER trust owner's baseline + ("local", None, False), +]) +async def test_should_use_cached_baseline_blocks_owner_grading_in_local_mode( + monkeypatch, exec_mode, cached, expected, +): + # In local exec the grading reference must be validator-independent, so the + # owner-supplied cached baseline is rejected even when present. + monkeypatch.setattr(engine, "_EXEC_MODE", exec_mode) + assert _should_use_cached_baseline(cached) is expected + + def _stub_signer(token: str = "hk-test-validator") -> SimpleNamespace: """Minimal signer stub. Returns a fixed Authorization header so the test handlers can confirm the request was hotkey-signed.""" @@ -61,11 +79,10 @@ async def test_deterministic_path_wraps_pool_answer(): """``oracle_source=deterministic`` (or unset) builds a ReconciledOracle from the task's pre-baked expected_output — no LLM calls, no fanout, no reconciler needed.""" - task = _task(answer="4", must_not_claim=["five"]) + task = _task(answer="4") rec = await _enrich_task_oracle(task, fanout=None, reconciler=None) assert rec.oracle_status == "deterministic" assert rec.expected_claims == ["4"] - assert rec.must_not_claim == ["five"] async def test_unset_oracle_source_treated_as_deterministic(): @@ -78,11 +95,10 @@ async def test_three_oracle_without_layer_falls_back_to_disputed(): """Operator forgot to configure oracle creds — three_oracle items surface as ``disputed`` with the template floor preserved instead of crashing the run.""" - task = _task(oracle_source="three_oracle", must_not_claim=["never X"]) + task = _task(oracle_source="three_oracle") rec = await _enrich_task_oracle(task, fanout=None, reconciler=None) assert rec.oracle_status == "disputed" assert rec.expected_claims == [] - assert rec.must_not_claim == ["never X"] assert rec.disagreement_note == "oracle_layer_not_configured" @@ -101,21 +117,19 @@ async def run(self, ctx): ] class _StubReconciler: - async def reconcile(self, *, prompt, groundings, must_not_claim_floor): + async def reconcile(self, *, prompt, groundings): return ReconciledOracle( expected_claims=["Paris is the capital"], - must_not_claim=list(must_not_claim_floor), oracle_status="consensus", consensus_claims=["Paris is the capital"], ) - task = _task(oracle_source="three_oracle", must_not_claim=["never London"]) + task = _task(oracle_source="three_oracle") rec = await _enrich_task_oracle( task, fanout=_StubFanout(), reconciler=_StubReconciler(), ) assert rec.oracle_status == "consensus" assert rec.expected_claims == ["Paris is the capital"] - assert rec.must_not_claim == ["never London"] async def test_three_oracle_passes_web_search_flag_to_context(): @@ -137,10 +151,9 @@ async def run(self, ctx): ] class _PassThroughReconciler: - async def reconcile(self, *, prompt, groundings, must_not_claim_floor): + async def reconcile(self, *, prompt, groundings): return ReconciledOracle( expected_claims=["x"], - must_not_claim=list(must_not_claim_floor), oracle_status="consensus", ) diff --git a/tests/validator/test_judge_client_eval.py b/tests/validator/test_judge_client_eval.py index 16317e2..afb623c 100644 --- a/tests/validator/test_judge_client_eval.py +++ b/tests/validator/test_judge_client_eval.py @@ -41,7 +41,6 @@ def _handler(request: httpx.Request) -> httpx.Response: "answers": ["The answer is 4."], }, expected_answer="4", - must_not_claim=["five", "six"], oracle_source="three_oracle", ) assert captured["url"] == "http://mock/v1/judge/eval" @@ -49,7 +48,7 @@ def _handler(request: httpx.Request) -> httpx.Response: assert body["bundle"]["question"] == "What is 2+2?" assert body["bundle"]["answers"] == ["The answer is 4."] assert body["expected_answer"] == "4" - assert body["must_not_claim"] == ["five", "six"] + assert "must_not_claim" not in body assert body["oracle_source"] == "three_oracle" assert result["outcome"] == "correct" @@ -98,7 +97,6 @@ def _handler(request: httpx.Request) -> httpx.Response: "outcome_score": 1.0, "tool_attestation_factor": 1.0, "efficiency_factor": 1.0, - "hallucination_knockout": 1.0, "cost_attestation_knockout": 1.0, "knockout_reason": None, }, @@ -108,7 +106,6 @@ def _handler(request: httpx.Request) -> httpx.Response: result = client.judge_eval_composite( outcome="correct", candidate_response="ok", - must_not_claim=[], required_tool="web_search", ledger_tools=["web_search"], latency_ms=200, diff --git a/tests/validator/test_reconciler.py b/tests/validator/test_reconciler.py index d7bbf60..2026f2f 100644 --- a/tests/validator/test_reconciler.py +++ b/tests/validator/test_reconciler.py @@ -5,9 +5,8 @@ expected_claims = consensus + majority. * 2 successful (1 down) → reconciler still runs. * <2 successful → skip reconciler call, return disputed. - * Reconciler error → disputed with template floor preserved. + * Reconciler error → disputed. * Reconciler returns malformed JSON → disputed. - * Additive must_not_claim: floor + extras, deduplicated. * from_deterministic constructor for non-three_oracle items. * Telemetry fields populated (vendor_status, latency, cost). """ @@ -85,7 +84,6 @@ def handler(req: httpx.Request) -> httpx.Response: assert result.oracle_status == "consensus" assert result.expected_claims == ["Paris is the capital of France"] - assert result.must_not_claim == [] assert result.vendor_status == {"openai": "ok", "gemini": "ok", "grok": "ok"} assert result.reconciler_cost_usd == pytest.approx(0.0007) @@ -162,43 +160,8 @@ def handler(req: httpx.Request) -> httpx.Response: assert result.disagreement_note == "openai and gemini disagree on X" -async def test_must_not_claim_is_additive(): - """Reconciler extras + template floor are unioned, dedup preserves - floor order so the floor ends up first in the final list.""" - - def handler(req: httpx.Request) -> httpx.Response: - return _ok_response({ - "consensus_claims": ["X is the answer"], - "majority_claims": [], - "minority_claims": [], - "must_not_claim_extras": [ - "claim about Y is wrong", - "do not assert Z", - ], - "oracle_status": "consensus", - "disagreement_note": None, - }) - - client = OpenAICompatibleClient(_cfg(), transport=httpx.MockTransport(handler)) - rec = Reconciler(client=client) - result = await rec.reconcile( - prompt="What's the answer?", - groundings=_three_groundings(), - must_not_claim_floor=["never claim A", "do not assert Z"], # 'do not assert Z' overlaps - ) - await rec.aclose() - - # Floor entries first (preserving order), then non-overlapping extras. - assert result.must_not_claim == [ - "never claim A", - "do not assert Z", - "claim about Y is wrong", - ] - - async def test_reconciler_skipped_when_fewer_than_two_oracles_succeeded(): - """1 ok + 2 errors → skip reconciler call, return disputed with - template floor.""" + """1 ok + 2 errors → skip reconciler call, return disputed.""" calls = {"n": 0} def handler(req: httpx.Request) -> httpx.Response: @@ -214,14 +177,12 @@ def handler(req: httpx.Request) -> httpx.Response: OracleGrounding(vendor="gemini", status="error", error_msg="boom"), OracleGrounding(vendor="grok", status="error", error_msg="boom"), ], - must_not_claim_floor=["never X"], ) await rec.aclose() assert calls["n"] == 0 # reconciler call NOT made assert result.oracle_status == "disputed" assert result.expected_claims == [] - assert result.must_not_claim == ["never X"] assert result.vendor_status == {"openai": "ok", "gemini": "error", "grok": "error"} assert "1/3 oracles" in (result.disagreement_note or "") @@ -243,9 +204,9 @@ async def test_reconciler_all_oracles_failed_returns_disputed(): assert result.expected_claims == [] -async def test_reconciler_error_falls_back_to_disputed_with_floor(): +async def test_reconciler_error_falls_back_to_disputed(): """Provider 5xx after retries → reconciler returns disputed with - expected_claims=[] and must_not_claim=template_floor only.""" + expected_claims=[].""" def handler(req: httpx.Request) -> httpx.Response: return httpx.Response(503, json={"error": "always 503"}) @@ -258,13 +219,11 @@ def handler(req: httpx.Request) -> httpx.Response: result = await rec.reconcile( prompt="...", groundings=_three_groundings(), - must_not_claim_floor=["floor item"], ) await rec.aclose() assert result.oracle_status == "disputed" assert result.expected_claims == [] - assert result.must_not_claim == ["floor item"] assert "reconciler_error" in (result.disagreement_note or "") @@ -280,12 +239,11 @@ def handler(req: httpx.Request) -> httpx.Response: result = await rec.reconcile( prompt="...", groundings=_three_groundings(), - must_not_claim_floor=["floor"], ) await rec.aclose() assert result.oracle_status == "disputed" - assert result.must_not_claim == ["floor"] + assert result.expected_claims == [] assert "malformed_json" in (result.disagreement_note or "") @@ -310,13 +268,9 @@ def handler(req: httpx.Request) -> httpx.Response: async def test_from_deterministic_factory(): """Pool kinds with built-in graders skip the reconciler entirely.""" - rec = ReconciledOracle.from_deterministic( - answer="Paris", - must_not_claim_floor=["never claim London"], - ) + rec = ReconciledOracle.from_deterministic(answer="Paris") assert rec.oracle_status == "deterministic" assert rec.expected_claims == ["Paris"] - assert rec.must_not_claim == ["never claim London"] assert rec.consensus_claims == ["Paris"] assert rec.vendor_status == {} diff --git a/tests/validator/test_validator_metrics.py b/tests/validator/test_validator_metrics.py index 2190492..ad4af81 100644 --- a/tests/validator/test_validator_metrics.py +++ b/tests/validator/test_validator_metrics.py @@ -87,7 +87,6 @@ def test_record_composite_score_clamps_to_unit_range(): def test_record_composite_knockout_per_factor(): factors = ( "tool_attestation", - "hallucination_knockout", "cost_attestation_knockout", "outcome_zero", ) diff --git a/validation/validator/calibration/reconciler_agreement.py b/validation/validator/calibration/reconciler_agreement.py index 759ddad..a64714b 100644 --- a/validation/validator/calibration/reconciler_agreement.py +++ b/validation/validator/calibration/reconciler_agreement.py @@ -42,7 +42,6 @@ async def reconcile( *, prompt: str, groundings: list[OracleGrounding], - must_not_claim_floor: Iterable[str] = (), ) -> ReconciledOracle: ... @@ -58,12 +57,6 @@ class ReconcilerAgreementFixture: prompt: str groundings: list[OracleGrounding] golden_claims: list[str] - must_not_claim_floor: list[str] = None # type: ignore[assignment] - - def __post_init__(self) -> None: - # Hack: dataclass with default mutable list — convert None → []. - if self.must_not_claim_floor is None: - object.__setattr__(self, "must_not_claim_floor", []) def claim_set_jaccard(a: Iterable[str], b: Iterable[str]) -> float: @@ -115,7 +108,6 @@ async def measure_reconciler_agreement( reconciled = await reconciler.reconcile( prompt=fixture.prompt, groundings=fixture.groundings, - must_not_claim_floor=fixture.must_not_claim_floor, ) agreement = claim_set_jaccard( reconciled.expected_claims, fixture.golden_claims, diff --git a/validation/validator/consensus/__init__.py b/validation/validator/consensus/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/validation/validator/consensus/aggregate.py b/validation/validator/consensus/aggregate.py new file mode 100644 index 0000000..44a3fb4 --- /dev/null +++ b/validation/validator/consensus/aggregate.py @@ -0,0 +1,134 @@ +"""Deterministic consensus aggregation — stake-weighted median + winner-take-all. + +This is the consensus-critical core of decentralized weight-setting. Every +honest validator runs these *pure* functions over the same gossiped inputs (the +per-validator score maps + the metagraph stake snapshot at the epoch block) and +must arrive at the *identical* winner — that identity is what makes the on-chain +Yuma vote unanimous instead of a copied owner pick. + +Design invariants (do not break — they are what convergence rests on): + * Pure & deterministic: no clocks, no randomness, no network. Same inputs → + same output, byte-for-byte, on every validator. + * Stake-weighted MEDIAN, never mean: the winner is correct as long as >50% of + the participating stake is honest, regardless of who the dishonest minority + is. A mean would let one high-stake liar drag the result. + * Total order on ties: rank by (score desc, hotkey asc) so two validators with + identical scores never pick different winners. +""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ValidatorReport: + """One validator's gossiped, signature-verified scores for the run. + + ``stake`` is that validator's stake from the epoch-block metagraph snapshot + (the same snapshot on every aggregator). ``scores`` maps miner hotkey → + composite in [0, 1]; a miner absent from the map simply wasn't scored by + this validator and contributes nothing to that miner's median. + """ + + hotkey: str + stake: float + scores: dict[str, float] + + +def stake_weighted_median(samples: list[tuple[float, float]]) -> float | None: + """Return the score at which cumulative stake first reaches half of total. + + ``samples`` is a list of ``(score, stake)``. Zero/negative-stake samples are + dropped. Returns ``None`` when no positive-stake sample exists. + + Lower-median convention: with an even split we take the lower of the two + central values, which is deterministic and conservative (a challenger needs + genuine majority support, not a knife-edge, to move the median up). + """ + pairs = sorted(((s, w) for s, w in samples if w > 0.0), key=lambda p: p[0]) + if not pairs: + return None + half = sum(w for _, w in pairs) / 2.0 + cumulative = 0.0 + for score, weight in pairs: + cumulative += weight + # ``>=`` with the sorted-ascending walk yields the lower median on an + # exact even split — deterministic across validators. + if cumulative >= half: + return score + return pairs[-1][0] # unreachable (cumulative reaches the full total) + + +def aggregate_scores(reports: list[ValidatorReport]) -> dict[str, float]: + """Per-miner stake-weighted median across all validator reports. + + The miner set is the union over every report. For each miner, only the + validators that actually scored it (and have positive stake) contribute to + its median — so a miner scored by few validators is judged by those few, + weighted by their stake. + """ + miners: set[str] = set() + for report in reports: + miners.update(report.scores.keys()) + + consensus: dict[str, float] = {} + for miner in sorted(miners): # sorted → deterministic iteration + samples = [ + (report.scores[miner], report.stake) + for report in reports + if miner in report.scores and report.stake > 0.0 + ] + median = stake_weighted_median(samples) + if median is not None: + consensus[miner] = median + return consensus + + +def participating_stake(reports: list[ValidatorReport]) -> float: + """Total positive stake represented by the gathered reports (for quorum).""" + return sum(r.stake for r in reports if r.stake > 0.0) + + +def select_winner( + consensus: dict[str, float], + *, + incumbent: str | None, + margin: float, +) -> str | None: + """Winner-take-all with a dethrone-by-``margin`` hysteresis. + + The incumbent (the miner currently holding the on-chain weight) keeps the + crown unless a challenger's consensus score beats the incumbent's by at + least ``margin``. The margin must exceed typical inter-validator score + variance, or near-ties flap the winner; the hysteresis is what makes + winner-take-all stable under decentralized scoring. + + Returns ``None`` only when ``consensus`` is empty. + """ + if not consensus: + return None + + # Total order: highest score first, hotkey as the deterministic tie-break. + top_hotkey, top_score = sorted(consensus.items(), key=lambda kv: (-kv[1], kv[0]))[0] + + if incumbent is None or incumbent not in consensus: + return top_hotkey # no defensible incumbent → the ranked top wins + if top_hotkey == incumbent: + return incumbent # incumbent already on top + # A challenger unseats the incumbent only by clearing the margin. + if top_score >= consensus[incumbent] + margin: + return top_hotkey + return incumbent + + +def winner_weight_vector( + winner: str | None, + miner_hotkeys: list[str], + *, + family_weight: float = 1.0, +) -> dict[str, float]: + """Winner-take-all vector: ``{winner: family_weight}``. Empty when there is + no winner (caller burns the unallocated remainder to UID 0).""" + if winner is None or winner not in set(miner_hotkeys): + return {} + return {winner: family_weight} diff --git a/validation/validator/consensus/archive.py b/validation/validator/consensus/archive.py new file mode 100644 index 0000000..9e2f856 --- /dev/null +++ b/validation/validator/consensus/archive.py @@ -0,0 +1,80 @@ +"""Durable signed score archive — the off-chain pre-image for the on-chain commitment. + +When a validator finalizes (and commits) its scores for a run, it writes the +canonical SIGNED report — ``{run_id, family_id, validator_hotkey, scores, +timestamp, signature}`` — to a per-run file. This is the durable, tamper-evident +record an external auditor replays: verify the signature, recompute the +commitment hash (commitment.py), and check it equals the validator's on-chain +commitment for the run. Unlike the in-memory ``STORE``, it survives restarts and +persists after the run completes, so the audit answer to "what did validator V +sign for run R?" is always available. +""" +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any + +# run_id reaches us from owner-api; sanitize before using it as a filename so a +# crafted value can't write outside the archive directory. +_SAFE_RUN = re.compile(r"[^A-Za-z0-9._-]") + + +def _safe_run_id(run_id: str) -> str: + return _SAFE_RUN.sub("_", str(run_id))[:128] + + +class ScoreArchive: + """Per-run signed score reports on disk (one JSON file per run_id).""" + + def __init__(self, directory: str | os.PathLike[str]) -> None: + self._dir = Path(directory) + + def _path(self, run_id: str) -> Path: + return self._dir / f"{_safe_run_id(run_id)}.json" + + def write(self, signed_report: dict[str, Any]) -> bool: + """Atomically persist one signed report (keyed on its run_id). Returns + False (never raises) on a missing run_id or write error — archiving is + best-effort and must not break the weight loop.""" + run_id = str((signed_report or {}).get("run_id") or "").strip() + if not run_id: + return False + try: + self._dir.mkdir(parents=True, exist_ok=True) + path = self._path(run_id) + tmp = path.with_suffix(".tmp") + tmp.write_text(json.dumps(signed_report), encoding="utf-8") + tmp.replace(path) # atomic swap + return True + except OSError: + return False + + def read(self, run_id: str) -> dict[str, Any] | None: + """The persisted signed report for a run, or None if not archived.""" + path = self._path(run_id) + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + + def runs(self) -> list[str]: + if not self._dir.exists(): + return [] + return sorted(p.stem for p in self._dir.glob("*.json")) + + +def _default_dir() -> str: + base = os.getenv("EIREL_SCORE_ARCHIVE_DIR") + if base: + return base + return str(Path(os.getenv("EIREL_DATA_DIR", ".eirel")) / "score-archive") + + +# Process-local singleton shared by the weight loop (writes) and the +# /v1/consensus/archive endpoint (reads) — same process, same directory. +ARCHIVE = ScoreArchive(_default_dir()) diff --git a/validation/validator/consensus/commitment.py b/validation/validator/consensus/commitment.py new file mode 100644 index 0000000..8bd3ceb --- /dev/null +++ b/validation/validator/consensus/commitment.py @@ -0,0 +1,94 @@ +"""On-chain score-hash commitment — makes validator equivocation provable. + +A validator commits ``H(run_id, family_id, its score map)`` on-chain (the +Bittensor commitments pallet) for the run it scored. A peer that fetches that +validator's gossiped scores recomputes the hash and checks it against the +on-chain commitment: a validator that serves two *different* score maps for one +run can match at most one on-chain commitment, so the other is provably +equivocated and is dropped. The commit extrinsic also lands in block history, +giving an external auditor a tamper-evident record of what each validator +committed per run. + +This module is the pure hash/payload/verify core (unit-testable) plus thin, +duck-typed wrappers over ``subtensor.commit`` / ``subtensor.get_all_commitments`` +(so callers can inject a fake). The hash is computed over the SAME score +representation gossip signs, and is timestamp-free so it's reproducible from the +served scores regardless of when they were fetched. +""" +from __future__ import annotations + +import hashlib +import json +from typing import Any + +# Versioned prefix so a reader can tell an EIREL score commitment from any other +# use of the hotkey's single commitment slot, and migrate the format later. +COMMITMENT_PREFIX = "eirel/scores/v1" + + +def _canonical(run_id: str, family_id: str, scores: dict[str, float]) -> bytes: + clean = {str(k): float(v) for k, v in scores.items()} + body = { + "run_id": str(run_id), + "family_id": str(family_id), + "scores": {k: clean[k] for k in sorted(clean)}, + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def score_commitment_hash(*, run_id: str, family_id: str, scores: dict[str, float]) -> str: + """Deterministic hex sha256 over (run_id, family_id, sorted scores). Both the + committer (its STORE score map) and a verifier (the received report's scores) + produce the same digest when the validator is honest.""" + return hashlib.sha256(_canonical(run_id, family_id, scores)).hexdigest() + + +def build_commitment_payload(*, run_id: str, family_id: str, scores: dict[str, float]) -> str: + """The on-chain string: ``eirel/scores/v1::`` (~87 bytes, + well within the commitment field limit).""" + digest = score_commitment_hash(run_id=run_id, family_id=family_id, scores=scores) + return f"{COMMITMENT_PREFIX}:{run_id}:{digest}" + + +def parse_commitment_payload(payload: str | None) -> tuple[str, str] | None: + """``(run_id, sha256hex)`` from a committed payload, or ``None`` if it isn't a + well-formed EIREL score commitment.""" + if not payload or not str(payload).startswith(COMMITMENT_PREFIX + ":"): + return None + rest = str(payload)[len(COMMITMENT_PREFIX) + 1:] + run_id, sep, digest = rest.rpartition(":") + if not sep or len(digest) != 64 or not run_id: + return None + return run_id, digest + + +def verify_scores_against_commitment( + *, run_id: str, family_id: str, scores: dict[str, float], committed_payload: str | None +) -> bool: + """True iff ``scores`` hash to the committed digest for this run. False on a + missing/malformed commitment, a wrong-run commitment, or a hash mismatch + (equivocation / uncommitted scores).""" + parsed = parse_commitment_payload(committed_payload) + if parsed is None: + return False + committed_run, committed_hash = parsed + if committed_run != str(run_id): + return False + return score_commitment_hash(run_id=run_id, family_id=family_id, scores=scores) == committed_hash + + +# ── thin chain wrappers (duck-typed subtensor; raise on failure, caller wraps) ─ + +def commit_score_hash( + subtensor: Any, wallet: Any, *, netuid: int, run_id: str, family_id: str, scores: dict[str, float] +) -> bool: + """Publish this validator's score-hash commitment on-chain for ``run_id``.""" + payload = build_commitment_payload(run_id=run_id, family_id=family_id, scores=scores) + return bool(subtensor.commit(wallet=wallet, netuid=int(netuid), data=payload)) + + +def fetch_all_commitments(subtensor: Any, *, netuid: int) -> dict[str, str]: + """``{hotkey: committed_payload}`` for the subnet — one chain call, used to + verify every peer's gossiped scores against its commitment.""" + result = subtensor.get_all_commitments(int(netuid)) + return {str(k): str(v) for k, v in (result or {}).items()} diff --git a/validation/validator/consensus/discovery.py b/validation/validator/consensus/discovery.py new file mode 100644 index 0000000..b4e4fbd --- /dev/null +++ b/validation/validator/consensus/discovery.py @@ -0,0 +1,106 @@ +"""Peer discovery from the metagraph — NOT the owner-api. + +Routing discovery through the owner would re-introduce the exact centralization +we're removing (the owner could censor, inject, or partition validators). Every +input here comes from on-chain metagraph state instead: + + * the validator set → neurons with ``validator_permit`` (and ≥ min stake) + * the consensus weights → each validator's ``S`` (stake) + * the gossip endpoints → each validator's serving ``axon`` (ip / port) + +The extraction is a pure, duck-typed function over a synced ``metagraph`` object +so it unit-tests without a chain connection. +""" +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class ValidatorInfo: + hotkey: str + uid: int + stake: float + ip: str + port: int + + @property + def servable(self) -> bool: + """True when the axon advertises a reachable endpoint we can fetch from.""" + return bool(self.ip) and self.ip not in ("0.0.0.0", "::") and self.port > 0 + + def endpoint(self) -> str: + host = f"[{self.ip}]" if ":" in self.ip else self.ip # bracket IPv6 + return f"http://{host}:{self.port}" + + +def _at(seq: Any, idx: int, default: Any) -> Any: + """Index numpy/torch/list defensively; ``default`` on miss or ``None`` seq.""" + if seq is None: + return default + try: + return seq[idx] + except (IndexError, KeyError, TypeError): + return default + + +def _int_to_ip(value: int) -> str: + try: + return str(ipaddress.ip_address(int(value))) + except (ValueError, TypeError): + return "" + + +def _axon_ip_port(axon: Any) -> tuple[str, int]: + if axon is None: + return "", 0 + ip = getattr(axon, "ip", "") + if isinstance(ip, int): + ip = _int_to_ip(ip) + try: + port = int(getattr(axon, "port", 0) or 0) + except (ValueError, TypeError): + port = 0 + return str(ip or ""), port + + +def discover_validators( + metagraph: Any, + *, + min_stake: float = 0.0, +) -> list[ValidatorInfo]: + """Return every permitted validator with stake ≥ ``min_stake``. + + Includes *this* validator too — the caller separates self (scored locally) + from peers (fetched). ``validator_permit`` missing (e.g. a lite metagraph in + a test) is treated as "permitted" so the function degrades gracefully. + Result is ordered by uid for deterministic iteration. + """ + hotkeys = list(getattr(metagraph, "hotkeys", []) or []) + stake = getattr(metagraph, "S", None) + permits = getattr(metagraph, "validator_permit", None) + axons = getattr(metagraph, "axons", None) + + out: list[ValidatorInfo] = [] + for uid, raw_hotkey in enumerate(hotkeys): + hotkey = str(raw_hotkey).strip() + if not hotkey: + continue + if not bool(_at(permits, uid, True)): + continue + try: + stake_val = float(_at(stake, uid, 0.0) or 0.0) + except (ValueError, TypeError): + stake_val = 0.0 + if stake_val < min_stake: + continue + ip, port = _axon_ip_port(_at(axons, uid, None)) + out.append(ValidatorInfo(hotkey=hotkey, uid=uid, stake=stake_val, ip=ip, port=port)) + return out + + +def stake_by_hotkey(validators: list[ValidatorInfo]) -> dict[str, float]: + """Snapshot ``{hotkey: stake}`` — the stake weights every aggregator uses.""" + return {v.hotkey: v.stake for v in validators} diff --git a/validation/validator/consensus/divergence.py b/validation/validator/consensus/divergence.py new file mode 100644 index 0000000..ecd0479 --- /dev/null +++ b/validation/validator/consensus/divergence.py @@ -0,0 +1,287 @@ +"""Shadow-mode divergence metrics — the quantitative gate for going decentralized. + +In shadow mode a validator computes the stake-weighted-median consensus winner +but still sets the owner's pick on-chain. The question that gates the flip to +``EIREL_WEIGHT_MODE=decentralized`` is: *how often, and how closely, does the +consensus agree with the owner?* A one-off "winner diverged" log line can't +answer that — you need it accumulated across runs. + +This module is the pure measurement core: + * ``top1_agreement`` — did consensus pick the same winner as the owner? + * ``spearman_rank_correlation`` / ``topk_overlap`` — full-ranking agreement + when both score maps are available (owner exposes per-miner scores). + * ``DivergenceTracker`` — accumulates per-run records and reports the running + agreement rate + mean rank correlation, so an operator can gate on e.g. + "winner agreement ≥ 0.95 over the last 20 runs" before flipping. + +All pure / in-memory; the engine's shadow branch records into a process-global +tracker and logs its summary each cycle. +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +def top1_agreement(local_winner: str | None, owner_winner: str | None) -> bool: + """Did consensus and the owner pick the same winner? Two ``None``s (no pick + either side) count as agreement; one-sided ``None`` is disagreement.""" + return local_winner == owner_winner + + +def _ranks(score_map: dict[str, float], keys: list[str]) -> list[float]: + """Average ranks (1 = highest score) for ``keys`` under ``score_map``, + tie-corrected so equal scores share the mean of their rank positions.""" + ordered = sorted(keys, key=lambda k: score_map[k], reverse=True) + ranks: dict[str, float] = {} + i = 0 + while i < len(ordered): + j = i + while j + 1 < len(ordered) and score_map[ordered[j + 1]] == score_map[ordered[i]]: + j += 1 + avg = (i + j) / 2.0 + 1.0 # 1-based average rank for the tie group + for t in range(i, j + 1): + ranks[ordered[t]] = avg + i = j + 1 + return [ranks[k] for k in keys] + + +def spearman_rank_correlation( + a: dict[str, float], b: dict[str, float] +) -> float | None: + """Spearman ρ over the miners both maps score. ``None`` when fewer than two + miners are in common (correlation undefined). ``1.0`` for identical rankings, + ``-1.0`` for fully reversed.""" + common = sorted(set(a) & set(b)) + n = len(common) + if n < 2: + return None + ra, rb = _ranks(a, common), _ranks(b, common) + mean_a = sum(ra) / n + mean_b = sum(rb) / n + da = [x - mean_a for x in ra] + db = [x - mean_b for x in rb] + num = sum(x * y for x, y in zip(da, db)) + den = (sum(x * x for x in da) * sum(y * y for y in db)) ** 0.5 + if den == 0.0: + # No rank variance on at least one side (e.g. all tied) — treat as + # perfect agreement, since neither ranking distinguishes anyone. + return 1.0 + return num / den + + +def topk_overlap(a: dict[str, float], b: dict[str, float], k: int) -> float: + """Fraction of the top-``k`` miners shared between the two maps (Jaccard-ish + over the top-k sets). ``1.0`` = identical top-k, ``0.0`` = disjoint.""" + if k <= 0: + return 1.0 + top = lambda m: {x for x, _ in sorted(m.items(), key=lambda kv: kv[1], reverse=True)[:k]} + ta, tb = top(a), top(b) + if not ta and not tb: + return 1.0 + return len(ta & tb) / max(1, len(ta | tb)) + + +@dataclass +class DivergenceRecord: + run_id: str + local_winner: str | None + owner_winner: str | None + agreed: bool + rank_correlation: float | None = None + + +@dataclass +class DivergenceTracker: + """Accumulates per-run divergence records, keeping the most recent + ``window`` for a running summary. One record per (run_id) — re-recording a + run replaces its prior entry, so re-armed continuous-pool runs don't double + count.""" + + window: int = 50 + _records: dict[str, DivergenceRecord] = field(default_factory=dict) + _order: list[str] = field(default_factory=list) + + def record( + self, + *, + run_id: str, + local_winner: str | None, + owner_winner: str | None, + local_scores: dict[str, float] | None = None, + owner_scores: dict[str, float] | None = None, + ) -> DivergenceRecord: + rank_corr = ( + spearman_rank_correlation(local_scores, owner_scores) + if local_scores and owner_scores else None + ) + rec = DivergenceRecord( + run_id=run_id, + local_winner=local_winner, + owner_winner=owner_winner, + agreed=top1_agreement(local_winner, owner_winner), + rank_correlation=rank_corr, + ) + if run_id not in self._records: + self._order.append(run_id) + self._records[run_id] = rec + # Bound memory to the window (drop oldest run ids). + while len(self._order) > self.window: + self._records.pop(self._order.pop(0), None) + return rec + + def n_runs(self) -> int: + return len(self._records) + + def agreement_rate(self) -> float | None: + """Fraction of recorded runs where consensus matched the owner winner. + ``None`` when nothing has been recorded yet.""" + if not self._records: + return None + agreed = sum(1 for r in self._records.values() if r.agreed) + return agreed / len(self._records) + + def mean_rank_correlation(self) -> float | None: + vals = [r.rank_correlation for r in self._records.values() if r.rank_correlation is not None] + return sum(vals) / len(vals) if vals else None + + def summary(self) -> str: + rate = self.agreement_rate() + rho = self.mean_rank_correlation() + rate_s = "n/a" if rate is None else f"{rate:.0%}" + rho_s = "n/a" if rho is None else f"{rho:.3f}" + return ( + f"divergence over {self.n_runs()} run(s): " + f"winner_agreement={rate_s} mean_rank_rho={rho_s}" + ) + + def ordered_records(self) -> list[DivergenceRecord]: + return [self._records[rid] for rid in self._order if rid in self._records] + + # ── durability: survive restarts so the gate window is a real history ───── + + def to_dict(self) -> dict[str, Any]: + return { + "window": self.window, + "order": list(self._order), + "records": { + rid: { + "run_id": r.run_id, "local_winner": r.local_winner, + "owner_winner": r.owner_winner, "agreed": r.agreed, + "rank_correlation": r.rank_correlation, + } + for rid, r in self._records.items() + }, + } + + def load_dict(self, data: dict[str, Any]) -> None: + self.window = int(data.get("window", self.window)) + self._records = { + str(rid): DivergenceRecord( + run_id=str(r.get("run_id") or rid), + local_winner=r.get("local_winner"), + owner_winner=r.get("owner_winner"), + agreed=bool(r.get("agreed")), + rank_correlation=r.get("rank_correlation"), + ) + for rid, r in (data.get("records") or {}).items() + } + order = data.get("order") or list(self._records.keys()) + self._order = [str(x) for x in order if str(x) in self._records] + + def persist(self, path: str | os.PathLike[str]) -> None: + Path(path).parent.mkdir(parents=True, exist_ok=True) + tmp = Path(path).with_suffix(".tmp") + tmp.write_text(json.dumps(self.to_dict()), encoding="utf-8") + tmp.replace(path) # atomic swap + + def load(self, path: str | os.PathLike[str]) -> None: + p = Path(path) + if not p.exists(): + return + try: + self.load_dict(json.loads(p.read_text(encoding="utf-8"))) + except (json.JSONDecodeError, OSError, TypeError, ValueError): + return + + +def divergence_path() -> str: + explicit = os.getenv("EIREL_DIVERGENCE_PATH") + if explicit: + return explicit + return str(Path(os.getenv("EIREL_DATA_DIR", ".eirel")) / "shadow-divergence.json") + + +def _signing_bytes(body: dict[str, Any]) -> bytes: + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def divergence_summary_body( + tracker: DivergenceTracker, *, validator_hotkey: str, weight_mode: str, timestamp: str +) -> dict[str, Any]: + """The summary payload that gets signed — running stats + the per-run records + behind them, so a verifier can both check the signature and recompute the + aggregate from the records.""" + return { + "validator_hotkey": str(validator_hotkey), + "weight_mode": str(weight_mode), + "window": tracker.window, + "n_runs": tracker.n_runs(), + "winner_agreement": tracker.agreement_rate(), + "mean_rank_rho": tracker.mean_rank_correlation(), + "records": [ + { + "run_id": r.run_id, "local_winner": r.local_winner, + "owner_winner": r.owner_winner, "agreed": r.agreed, + "rank_correlation": r.rank_correlation, + } + for r in tracker.ordered_records() + ], + "timestamp": str(timestamp), + } + + +def sign_divergence( + keypair: Any, tracker: DivergenceTracker, *, weight_mode: str, timestamp: str +) -> dict[str, Any]: + """Hotkey-signed divergence summary — the tamper-evident gate evidence a peer + or auditor fetches to confirm a validator's shadow-mode agreement before the + network flips to decentralized.""" + body = divergence_summary_body( + tracker, validator_hotkey=keypair.ss58_address, weight_mode=weight_mode, timestamp=timestamp, + ) + signature = keypair.sign(_signing_bytes(body)) + return {"summary": body, "validator_hotkey": keypair.ss58_address, "signature": "0x" + signature.hex()} + + +def verify_divergence(signed: dict[str, Any], *, expected_hotkey: str | None = None) -> bool: + """True iff the signature is valid for ``validator_hotkey`` over the summary. + Lazy ``bittensor`` import keeps this module cheap to import in tests.""" + try: + body = signed["summary"] + hotkey = str(signed["validator_hotkey"]) + sig_hex = str(signed["signature"]) + except (KeyError, TypeError): + return False + if not isinstance(body, dict) or body.get("validator_hotkey") != hotkey: + return False + if expected_hotkey is not None and hotkey != str(expected_hotkey): + return False + try: + signature = bytes.fromhex(sig_hex[2:] if sig_hex.startswith("0x") else sig_hex) + except ValueError: + return False + try: + import bittensor as bt + + return bool(bt.Keypair(ss58_address=hotkey).verify(_signing_bytes(body), signature)) + except Exception: # noqa: BLE001 + return False + + +# Process-local singleton: the weight loop records + persists into it, the +# /v1/consensus/divergence endpoint signs + serves it. +TRACKER = DivergenceTracker(window=int(os.getenv("EIREL_SHADOW_DIVERGENCE_WINDOW", "50"))) diff --git a/validation/validator/consensus/gossip.py b/validation/validator/consensus/gossip.py new file mode 100644 index 0000000..03f277d --- /dev/null +++ b/validation/validator/consensus/gossip.py @@ -0,0 +1,195 @@ +"""Signed gossip of per-validator score maps, and verification on receipt. + +Each validator serves its own score map signed by its hotkey; peers fetch it, +verify the signature against the *metagraph* hotkey for that axon, and only then +fold it into the stake-weighted median. Two distinct checks must both pass for a +peer's scores to count: + + 1. **Authenticity** — the signature is valid for the claimed ``validator_hotkey`` + over the report's canonical bytes (``verify_report``). + 2. **Identity binding** — that claimed hotkey equals the metagraph hotkey of the + axon we fetched from (``fetch_peer``), so a node can't serve scores under + another validator's name. + +The signed canonical message also pins ``run_id``/``family_id``, so a report +can't be replayed across runs. **Equivocation** — serving different scores to +different peers within a run — is caught by an optional third check +(``require_commitment``): the report's scores must hash to the peer's on-chain +commitment for this run (commitment.py). A validator can match at most one +on-chain commitment per run, so a second, conflicting score map is rejected. +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any + +from validation.validator.consensus.aggregate import ValidatorReport +from validation.validator.consensus.commitment import verify_scores_against_commitment +from validation.validator.consensus.discovery import ValidatorInfo + +_SCORES_PATH = "/v1/consensus/scores" + + +def consensus_scores_path(run_id: str) -> str: + return f"{_SCORES_PATH}/{run_id}" + + +def _canonical_message( + run_id: str, family_id: str, hotkey: str, scores: dict[str, float], timestamp: str +) -> bytes: + """Deterministic bytes both signer and verifier agree on. Scores are keyed + in sorted order; ``json.dumps`` round-trips Python floats exactly, so the + receiver re-derives identical bytes from the parsed report.""" + body = { + "run_id": str(run_id), + "family_id": str(family_id), + "validator_hotkey": str(hotkey), + "timestamp": str(timestamp), + "scores": {k: scores[k] for k in sorted(scores)}, + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sign_scores( + keypair: Any, + *, + run_id: str, + family_id: str, + scores: dict[str, float], + timestamp: str, +) -> dict[str, Any]: + """Build this validator's signed score report (the wire payload).""" + clean = {str(k): float(v) for k, v in scores.items()} + msg = _canonical_message(run_id, family_id, keypair.ss58_address, clean, timestamp) + signature = keypair.sign(msg) + return { + "run_id": str(run_id), + "family_id": str(family_id), + "validator_hotkey": keypair.ss58_address, + "timestamp": str(timestamp), + "scores": clean, + "signature": "0x" + signature.hex(), + } + + +def verify_report(report: dict[str, Any]) -> bool: + """True iff the signature is valid for ``report['validator_hotkey']`` over + the report's canonical content. Does *not* assert the hotkey is an authorized + validator — the caller cross-checks that against the metagraph.""" + try: + hotkey = str(report["validator_hotkey"]) + sig_hex = str(report["signature"]) + signature = bytes.fromhex(sig_hex[2:] if sig_hex.startswith("0x") else sig_hex) + scores = {str(k): float(v) for k, v in (report.get("scores") or {}).items()} + msg = _canonical_message( + report["run_id"], report["family_id"], hotkey, scores, report["timestamp"] + ) + except (KeyError, ValueError, TypeError): + return False + import bittensor as bt # lazy — keeps importing this module cheap for tests + + try: + return bool(bt.Keypair(ss58_address=hotkey).verify(msg, signature)) + except Exception: # noqa: BLE001 — malformed key/sig → reject + return False + + +def report_to_validator_report(report: dict[str, Any], *, stake: float) -> ValidatorReport: + scores = {str(k): float(v) for k, v in (report.get("scores") or {}).items()} + return ValidatorReport( + hotkey=str(report["validator_hotkey"]), stake=float(stake), scores=scores + ) + + +async def fetch_peer( + client: Any, + vinfo: ValidatorInfo, + *, + run_id: str, + family_id: str, + timeout: float = 8.0, + committed_payload: str | None = None, + require_commitment: bool = False, +) -> ValidatorReport | None: + """Fetch + fully validate one peer's report. Any failure returns ``None`` so + the peer is silently dropped from the round (liveness handled by quorum). + + When ``require_commitment`` is set, the report's scores must also hash to the + peer's on-chain ``committed_payload`` for this run (anti-equivocation) — a + mismatch or missing commitment drops the peer.""" + if not vinfo.servable: + return None + url = vinfo.endpoint() + consensus_scores_path(run_id) + try: + resp = await client.get(url, timeout=timeout) + if getattr(resp, "status_code", None) != 200: + return None + report = resp.json() + except Exception: # noqa: BLE001 — unreachable / malformed peer → drop + return None + if not isinstance(report, dict): + return None + # Identity binding: served hotkey must be THIS axon's metagraph hotkey. + if str(report.get("validator_hotkey")) != vinfo.hotkey: + return None + if str(report.get("run_id")) != str(run_id) or str(report.get("family_id")) != str(family_id): + return None + if not verify_report(report): + return None + if require_commitment: + scores = {str(k): float(v) for k, v in (report.get("scores") or {}).items()} + if not verify_scores_against_commitment( + run_id=run_id, family_id=family_id, scores=scores, committed_payload=committed_payload, + ): + return None # equivocated / uncommitted scores → drop + return report_to_validator_report(report, stake=vinfo.stake) + + +async def collect_reports( + *, + client: Any, + validators: list[ValidatorInfo], + self_hotkey: str, + self_scores: dict[str, float], + run_id: str, + family_id: str, + per_peer_timeout: float = 8.0, + commitments: dict[str, str] | None = None, + require_commitment: bool = False, +) -> list[ValidatorReport]: + """This validator's own report (from local scores, no network) + every + peer's verified report. Stake for each comes from the metagraph snapshot in + ``validators`` — so a peer can report scores but not its own weight. + + ``commitments`` maps hotkey → on-chain committed payload; with + ``require_commitment`` it enforces that each peer's scores match its + commitment (anti-equivocation). Self is never commitment-checked.""" + stake_of = {v.hotkey: v.stake for v in validators} + commitments = commitments or {} + reports: list[ValidatorReport] = [] + + if self_scores: + reports.append( + ValidatorReport( + hotkey=self_hotkey, + stake=float(stake_of.get(self_hotkey, 0.0)), + scores=dict(self_scores), + ) + ) + + peers = [v for v in validators if v.hotkey != self_hotkey and v.servable] + if peers: + fetched = await asyncio.gather( + *( + fetch_peer( + client, v, run_id=run_id, family_id=family_id, timeout=per_peer_timeout, + committed_payload=commitments.get(v.hotkey), + require_commitment=require_commitment, + ) + for v in peers + ), + return_exceptions=True, + ) + reports.extend(r for r in fetched if isinstance(r, ValidatorReport)) + return reports diff --git a/validation/validator/consensus/score_store.py b/validation/validator/consensus/score_store.py new file mode 100644 index 0000000..6a29b0c --- /dev/null +++ b/validation/validator/consensus/score_store.py @@ -0,0 +1,146 @@ +"""Validator-local store of this validator's own per-miner scores for a run. + +The validator computes per-(task, miner) composites in ``_judge_miner`` and +today POSTs them straight to the owner-api and forgets them — per-run +aggregation lives on the owner. Decentralized consensus needs each validator to +*retain* its own scores so it can (a) gossip them to peers and (b) fold peers' +scores into a stake-weighted median itself. This module is that retained store. + +It deliberately mirrors the owner's aggregation formula — +``avg(coalesce(final_task_score, agreement_score))`` over a miner's rows +(see ``_aggregate_miner_from_results``) — so a validator's local ``score_map`` +is the same per-miner number the system already trusts, just computed locally +instead of server-side. + +Process-local and thread-safe (``record`` may be called from parallel +``asyncio.to_thread`` judge tasks). Optional JSON persistence makes it survive a +mid-run restart; without it a restarted validator simply re-scores or sits out +the round, which the >50%-stake quorum tolerates. +""" +from __future__ import annotations + +import json +import os +import threading +from pathlib import Path +from typing import Any + + +def _coalesce_score(result: dict[str, Any]) -> float | None: + """final_task_score if present, else agreement_score — matching the owner's + ``coalesce(final_task_score, agreement_score)``. None when neither is set.""" + for key in ("final_task_score", "agreement_score"): + value = result.get(key) + if isinstance(value, (int, float)): + return float(value) + return None + + +class ScoreStore: + """Accumulates ``(run_id, family_id) → {miner_hotkey: [task scores]}`` and + collapses to one composite per miner on read.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + # (run_id, family_id) -> miner_hotkey -> list of per-task composites + self._runs: dict[tuple[str, str], dict[str, list[float]]] = {} + + def record(self, run_id: str, family_id: str, results: list[dict[str, Any]]) -> int: + """Record one task's per-miner results (the ``miner_results`` list the + validator already builds). Returns how many scores were captured.""" + if not run_id or not family_id or not results: + return 0 + captured = 0 + with self._lock: + family = self._runs.setdefault((run_id, family_id), {}) + for result in results: + hotkey = str(result.get("miner_hotkey") or "").strip() + if not hotkey: + continue + score = _coalesce_score(result) + if score is None: + continue + family.setdefault(hotkey, []).append(score) + captured += 1 + return captured + + def score_map(self, run_id: str, family_id: str) -> dict[str, float]: + """``{miner_hotkey: mean composite}`` for the run — this validator's + opinion, ready to gossip and aggregate.""" + with self._lock: + family = self._runs.get((run_id, family_id)) + if not family: + return {} + return { + hotkey: sum(scores) / len(scores) + for hotkey, scores in family.items() + if scores + } + + def runs(self) -> list[tuple[str, str]]: + with self._lock: + return list(self._runs.keys()) + + def busiest_run(self) -> tuple[str, str] | None: + """The ``(run_id, family_id)`` this validator has scored the most miners + for — a stable heuristic for "the run I just finished," which is the one + to set weights on. ``None`` when nothing has been scored yet.""" + with self._lock: + if not self._runs: + return None + return max(self._runs.items(), key=lambda kv: len(kv[1]))[0] + + def prune(self, keep: set[str]) -> int: + """Drop runs whose run_id isn't in ``keep`` (bound memory across runs). + Returns the number of (run, family) entries dropped.""" + with self._lock: + stale = [key for key in self._runs if key[0] not in keep] + for key in stale: + del self._runs[key] + return len(stale) + + # ── optional durability (survive a mid-run restart) ────────────────────── + + def persist(self, path: str | os.PathLike[str]) -> None: + with self._lock: + snapshot = { + f"{run_id}\t{family_id}": family + for (run_id, family_id), family in self._runs.items() + } + tmp = Path(path).with_suffix(".tmp") + tmp.write_text(json.dumps(snapshot), encoding="utf-8") + tmp.replace(path) # atomic swap + + def load(self, path: str | os.PathLike[str]) -> None: + p = Path(path) + if not p.exists(): + return + try: + snapshot = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return + restored: dict[tuple[str, str], dict[str, list[float]]] = {} + for key, family in (snapshot or {}).items(): + run_id, _, family_id = str(key).partition("\t") + if run_id and family_id and isinstance(family, dict): + restored[(run_id, family_id)] = { + str(hk): [float(x) for x in v] + for hk, v in family.items() + if isinstance(v, list) + } + with self._lock: + self._runs = restored + + +def store_path() -> str: + """Disk path for STORE persistence — shared by the eval loop (persist) and + startup (load) so a restarted validator still serves the scores it gathered. + """ + explicit = os.getenv("EIREL_SCORE_STORE_PATH") + if explicit: + return explicit + return str(Path(os.getenv("EIREL_DATA_DIR", ".eirel")) / "score-store.json") + + +# Process-local singleton the eval loop records into and the weight loop reads. +STORE = ScoreStore() diff --git a/validation/validator/consensus/weights.py b/validation/validator/consensus/weights.py new file mode 100644 index 0000000..2916313 --- /dev/null +++ b/validation/validator/consensus/weights.py @@ -0,0 +1,125 @@ +"""Consensus orchestration: metagraph + local scores + peer gossip → winner. + +This is what the weight-setting loop calls instead of ``GET /v1/weights``. It +ties the pure pieces together and returns a ``weights_by_hotkey`` map in the +*same shape* the owner endpoint returned — so the loop's downstream (uid +resolution, burn-to-UID-0, ``set_weights``, on-chain verify) is reused unchanged. + +The single network step is the peer gossip; everything else is deterministic +over the epoch-block metagraph snapshot, so honest validators that gathered the +same reports land on the same winner. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from validation.validator.consensus.aggregate import ( + aggregate_scores, + participating_stake, + select_winner, +) +from validation.validator.consensus.discovery import discover_validators +from validation.validator.consensus.gossip import collect_reports + + +@dataclass(slots=True) +class ConsensusResult: + run_id: str + family_id: str + winner: str | None + incumbent: str | None + weights_by_hotkey: dict[str, float] + consensus_scores: dict[str, float] = field(default_factory=dict) + n_reports: int = 0 + participating_stake: float = 0.0 + total_stake: float = 0.0 + quorum_met: bool = False + + def summary(self) -> str: + return ( + f"run={self.run_id} winner={self.winner} incumbent={self.incumbent} " + f"reports={self.n_reports} stake={self.participating_stake:.0f}/" + f"{self.total_stake:.0f} quorum={'yes' if self.quorum_met else 'NO'}" + ) + + +def current_incumbent(metagraph: Any) -> str | None: + """The miner currently holding the crown = the highest-incentive neuron. + + Incentive (``metagraph.I``) is what a *miner* earns from consensus weight, so + its argmax is the current winner — a single on-chain fact every validator + reads identically, which is what the dethrone-margin hysteresis needs. + """ + hotkeys = list(getattr(metagraph, "hotkeys", []) or []) + incentive = getattr(metagraph, "I", None) + if incentive is None or not hotkeys: + return None + best_uid, best_val = None, 0.0 + for uid in range(len(hotkeys)): + try: + val = float(incentive[uid]) + except (IndexError, TypeError, ValueError): + continue + if val > best_val: + best_val, best_uid = val, uid + return str(hotkeys[best_uid]) if best_uid is not None else None + + +async def compute_consensus( + *, + metagraph: Any, + http_client: Any, + self_hotkey: str, + run_id: str, + family_id: str, + local_scores: dict[str, float], + min_stake: float = 0.0, + margin: float = 0.05, + quorum_fraction: float = 0.5, + per_peer_timeout: float = 8.0, + commitments: dict[str, str] | None = None, + require_commitment: bool = False, +) -> ConsensusResult: + """Gather verified validator reports, stake-weighted-median them, and pick the + winner under the dethrone margin. Returns ``weights_by_hotkey={winner: 1.0}`` + when quorum is met, else an empty map (caller holds / burns). + + ``commitments`` (hotkey → on-chain committed payload) with + ``require_commitment`` rejects peers whose gossiped scores don't match their + on-chain commitment for this run (anti-equivocation).""" + validators = discover_validators(metagraph, min_stake=min_stake) + total_stake = sum(v.stake for v in validators) + + reports = await collect_reports( + client=http_client, + validators=validators, + self_hotkey=self_hotkey, + self_scores=local_scores, + run_id=run_id, + family_id=family_id, + per_peer_timeout=per_peer_timeout, + commitments=commitments, + require_commitment=require_commitment, + ) + part = participating_stake(reports) + quorum_met = total_stake > 0.0 and part >= quorum_fraction * total_stake + + incumbent = current_incumbent(metagraph) + consensus = aggregate_scores(reports) + + winner = select_winner(consensus, incumbent=incumbent, margin=margin) if quorum_met else None + weights = {winner: 1.0} if winner else {} + + return ConsensusResult( + run_id=run_id, + family_id=family_id, + winner=winner, + incumbent=incumbent, + weights_by_hotkey=weights, + consensus_scores=consensus, + n_reports=len(reports), + participating_stake=part, + total_stake=total_stake, + quorum_met=quorum_met, + ) diff --git a/validation/validator/engine.py b/validation/validator/engine.py index c734a88..dec65b7 100644 --- a/validation/validator/engine.py +++ b/validation/validator/engine.py @@ -4,6 +4,7 @@ import os import secrets import time +from datetime import UTC, datetime from dataclasses import dataclass, fields as _dc_fields from typing import Any from urllib.parse import urlparse, urlunparse @@ -13,6 +14,7 @@ logger = logging.getLogger(__name__) from shared.common.bittensor_signing import load_signer +from shared.common.job_attribution import build_cost_tag, job_request_headers from shared.common.security import sha256_hex from eirel.groups import ensure_active_family_id from shared.core.evaluation_models import MinerBenchmarkTarget @@ -87,9 +89,9 @@ def _reconciled_to_payload(reconciled: "ReconciledOracle") -> dict[str, Any]: Embedded under ``_reconciled`` in the submitted ``baseline_response`` so a later pool cycle reconstructs the *exact* reconciled oracle — - identical expected_claims/must_not_claim/vendor data — making the - cached judging input byte-identical to the first compute (the - invariant that keeps scoring batch-independent). + identical expected_claims/vendor data — making the cached judging + input byte-identical to the first compute (the invariant that keeps + scoring batch-independent). """ return {f.name: getattr(reconciled, f.name) for f in _dc_fields(reconciled)} @@ -215,15 +217,12 @@ async def _enrich_task_oracle( """ expected_output = getattr(task_obj, "expected_output", None) or {} answer = str(expected_output.get("answer") or "").strip() - must_not_claim_floor = list(expected_output.get("must_not_claim") or []) oracle_source = getattr(task_obj, "oracle_source", None) if oracle_source != "three_oracle": # Deterministic path. Optionally fetch a single-vendor LLM # reference for the pairwise judge. - base = ReconciledOracle.from_deterministic( - answer=answer, must_not_claim_floor=must_not_claim_floor, - ) + base = ReconciledOracle.from_deterministic(answer=answer) if fanout is None or pairwise_vendor not in fanout.vendors: return base context = OracleContext( @@ -245,7 +244,6 @@ async def _enrich_task_oracle( # build a new instance because ReconciledOracle is frozen. return ReconciledOracle( expected_claims=base.expected_claims, - must_not_claim=base.must_not_claim, oracle_status="deterministic", disagreement_note=base.disagreement_note, consensus_claims=base.consensus_claims, @@ -264,12 +262,11 @@ async def _enrich_task_oracle( if fanout is None or reconciler is None: logger.warning( "three_oracle task %s but oracle layer not configured; " - "falling back to disputed with template floor", + "falling back to disputed", getattr(task_obj, "task_id", "?"), ) return ReconciledOracle( expected_claims=[], - must_not_claim=must_not_claim_floor, oracle_status="disputed", disagreement_note="oracle_layer_not_configured", ) @@ -286,12 +283,11 @@ async def _enrich_task_oracle( return await reconciler.reconcile( prompt=context.prompt, groundings=groundings, - must_not_claim_floor=must_not_claim_floor, ) async def _fetch_ledger_tools( - job_id: str, *, owner_url: str, signer, + job_id: str, *, owner_url: str, signer, extra_headers: dict[str, str] | None = None, ) -> list[str]: """Fetch tool names invoked under ``job_id`` from the orchestrator ledger. Returns the deduped list in arrival order. @@ -299,22 +295,22 @@ async def _fetch_ledger_tools( Authenticates with the validator's hotkey signature — the owner-api gates ``/v1/internal/eval/job_ledger`` on ``validator_dependency``, so any registered active validator can read the ledger for any job - they're scoring. Missing job_id → empty list. Network errors also - return [] (fail-safe: composite's tool_attestation factor will be 0 + they're scoring. In decentralized-exec mode the caller passes + ``extra_headers`` (a bearer token) and ``owner_url`` of the validator's + OWN local ledger sink instead. Missing job_id → empty list. Network errors + also return [] (fail-safe: composite's tool_attestation factor will be 0 for required_tool tasks if the ledger is unreachable). """ if not job_id: return [] path = f"/v1/internal/eval/job_ledger?job_id={job_id}" url = f"{owner_url}{path}" + headers = extra_headers if extra_headers is not None else _signed_headers( + signer=signer, method="GET", path=path, body=b"", + ) try: async with httpx.AsyncClient(timeout=10.0) as client: - resp = await client.get( - url, - headers=_signed_headers( - signer=signer, method="GET", path=path, body=b"", - ), - ) + resp = await client.get(url, headers=headers) if resp.status_code != 200: logger.warning( "ledger fetch returned %d for job_id=%s: %s", @@ -337,6 +333,30 @@ async def _fetch_ledger_tools( return tool_names +async def _fetch_local_proxy_cost(job_id: str) -> dict[str, Any]: + """Decentralized-exec cost lookup against THIS validator's own provider-proxy + (mirrors owner-api's ``_fetch_proxy_cost``). In local mode the owner no longer + injects ``proxy_cost_usd`` into the stream, so the validator reads its own + proxy ledger. Fail-soft: any error → zero cost (cost is a low-weight factor).""" + if not job_id: + return {"llm_cost_usd": 0.0, "tool_cost_usd": 0.0, "absent": True} + url = f"{_LOCAL_PROVIDER_PROXY_URL}/v1/jobs/{job_id}/cost" + headers = ( + {"Authorization": f"Bearer {_LOCAL_PROVIDER_PROXY_TOKEN}"} + if _LOCAL_PROVIDER_PROXY_TOKEN else {} + ) + try: + async with httpx.AsyncClient(timeout=5.0) as client: + resp = await client.get(url, headers=headers) + if resp.status_code == 404: + return {"llm_cost_usd": 0.0, "tool_cost_usd": 0.0, "absent": True} + resp.raise_for_status() + return resp.json() + except Exception as exc: # noqa: BLE001 + logger.warning("local proxy_cost lookup failed for job=%s: %s", job_id, exc) + return {"llm_cost_usd": 0.0, "tool_cost_usd": 0.0, "absent": True} + + def _hydrate_agent_inputs(task_payload: dict[str, Any]) -> dict[str, Any]: """Fold top-level task fields into the ``inputs`` dict sent to the miner. @@ -403,7 +423,11 @@ def _load_validator_signer(): def _signed_headers(*, signer, method: str, path: str, body: bytes) -> dict[str, str]: - return signer.signed_headers(method, path, sha256_hex(body)) + # The owner-api verifies the signature over ``request.url.path`` — the path + # WITHOUT the query string (shared/common/security.authenticate_request). So + # sign the bare path; signing the query too (e.g. ``?job_id=``/``?family_id=``) + # yields a signature the server can't reproduce → 401. + return signer.signed_headers(method, path.split("?", 1)[0], sha256_hex(body)) def _pairwise_miner_score(*, winner: str, miner_position: str) -> float: @@ -675,6 +699,177 @@ async def run_validator_loop() -> None: _WEIGHT_SET_INTERVAL_SECONDS = _WEIGHT_SET_INTERVAL_BLOCKS * 12 _MIN_STAKE_TO_SET_WEIGHTS = int(os.getenv("EIREL_MIN_STAKE_TO_SET_WEIGHTS", "5000")) +# Decentralized weight-setting mode (the migration switch): +# owner — copy the owner-api /v1/weights pick (current behavior) +# shadow — ALSO compute the consensus winner from gossiped validator +# scores and LOG the divergence, but still set the owner pick +# (zero-risk Phase-1 measurement of inter-validator agreement) +# decentralized — set the stake-weighted-median consensus winner; the owner +# endpoint is out of the weight path entirely +_WEIGHT_MODE = os.getenv("EIREL_WEIGHT_MODE", "owner").strip().lower() +_CONSENSUS_MIN_STAKE = float(os.getenv("EIREL_CONSENSUS_MIN_STAKE", "1000")) +_CONSENSUS_MARGIN = float(os.getenv("EIREL_CONSENSUS_DOMINANCE_MARGIN", "0.05")) +_CONSENSUS_QUORUM = float(os.getenv("EIREL_CONSENSUS_QUORUM_FRACTION", "0.5")) +_CONSENSUS_PEER_TIMEOUT = float(os.getenv("EIREL_CONSENSUS_PEER_TIMEOUT_S", "8.0")) +# Anti-equivocation: commit H(score map) on-chain per run (default on in +# shadow/decentralized — builds the auditable trail), and optionally REJECT +# peers whose gossiped scores don't match their on-chain commitment. Enforcement +# defaults off so it can be flipped on once validators are committing (same +# rollout philosophy as owner→shadow→decentralized). +_COMMIT_SCORES = os.getenv("EIREL_COMMIT_SCORES", "true").strip().lower() not in ("0", "false", "no") +_REQUIRE_COMMITMENT = os.getenv("EIREL_CONSENSUS_REQUIRE_COMMITMENT", "false").strip().lower() in ("1", "true", "yes") +_COMMITTED_RUNS: set[str] = set() # run_ids this process has already committed +_ARCHIVED_RUNS: set[str] = set() # run_ids this process has already archived + +# Decentralized EXECUTION mode (orthogonal to _WEIGHT_MODE): +# owner — invoke miners through the owner-api proxy (current behavior) +# local — deploy + run miners on THIS validator's own cluster, measuring +# response/latency/cost/tool-use first-hand (no owner in the data path) +_EXEC_MODE = os.getenv("EIREL_EXEC_MODE", "owner").strip().lower() +# Validator's own job-token signing key (it both mints the per-job token at +# invocation and verifies it in its own provider-proxy/tool services). +_TOOL_JOB_SIGNING_KEY = os.getenv("EIREL_TOOL_JOB_SIGNING_KEY", "") +_LOCAL_LEDGER_URL = os.getenv("EIREL_LOCAL_LEDGER_URL", "").rstrip("/") +_LOCAL_LEDGER_TOKEN = (os.getenv("EIREL_LOCAL_LEDGER_TOKEN") or _TOOL_JOB_SIGNING_KEY or "") +_LOCAL_PROVIDER_PROXY_URL = os.getenv("EIREL_LOCAL_PROVIDER_PROXY_URL", "http://provider-proxy:8092").rstrip("/") +_LOCAL_PROVIDER_PROXY_TOKEN = os.getenv("EIREL_LOCAL_PROVIDER_PROXY_TOKEN", "") + +_LOCAL_RUN_CONTROLLER = None + +# Running shadow-mode divergence (consensus winner vs owner pick) across runs — +# the quantitative gate for flipping EIREL_WEIGHT_MODE to decentralized. Shared +# singleton: the weight loop records + persists, the /v1/consensus/divergence +# endpoint signs + serves the same tracker. +from validation.validator.consensus.divergence import ( + TRACKER as _SHADOW_DIVERGENCE, + divergence_path as _divergence_path, +) + + +def _local_exec_enabled() -> bool: + return _EXEC_MODE == "local" + + +def _should_use_cached_baseline(cached_baseline: Any) -> bool: + """Whether to reuse the owner-supplied cached oracle baseline as the grading + reference (pairwise comparator + expected_claims). + + In decentralized execution the grading reference must be validator- + INDEPENDENT — otherwise the owner could shape what counts as "correct" via + the baseline it attaches to the claim, a scoring lever that survives moving + miner execution validator-side. So under EIREL_EXEC_MODE=local we NEVER + trust the owner's cached baseline; each validator recomputes its own oracle + (the stake-weighted-median consensus tolerates per-validator oracle + variance). In owner mode the cache is a fine cross-cycle cost optimization. + """ + return bool(cached_baseline) and not _local_exec_enabled() + + +def _get_run_controller(signer): + """Process-singleton LocalRunController, built lazily so ``owner`` mode never + imports kubernetes or touches the cluster.""" + global _LOCAL_RUN_CONTROLLER + if _LOCAL_RUN_CONTROLLER is None: + from validation.validator.local_exec.runtime_controller import build_local_run_controller + _LOCAL_RUN_CONTROLLER = build_local_run_controller(signer) + return _LOCAL_RUN_CONTROLLER + + +def _archive_signed_scores(signer: Any, *, run_id: str, family_id: str, local_scores: dict[str, float]) -> None: + """Write the canonical SIGNED score report to the durable archive (once per + run) — the off-chain pre-image for the on-chain commitment, so an external + auditor can replay what this validator signed for the run. Best-effort.""" + if run_id in _ARCHIVED_RUNS: + return + try: + from validation.validator.consensus.archive import ARCHIVE + from validation.validator.consensus.gossip import sign_scores + report = sign_scores( + signer.keypair, run_id=run_id, family_id=family_id, + scores=local_scores, timestamp=datetime.now(UTC).isoformat(), + ) + if ARCHIVE.write(report): + _ARCHIVED_RUNS.add(run_id) + logger.info("consensus: archived signed score report for run=%s", run_id) + except Exception as exc: # noqa: BLE001 — archiving is best-effort + logger.warning("consensus: archiving signed scores failed run=%s: %s", run_id, exc) + + +def _commit_and_fetch_commitments( + *, subtensor: Any, wallet: Any, signer: Any, netuid: int, run_id: str, family_id: str, + local_scores: dict[str, float], +) -> dict[str, str]: + """Publish this validator's score-hash commitment on-chain + archive the + signed pre-image (both once per run, same scores so they're mutually + consistent), and return ``{hotkey: committed_payload}`` for all validators. + Best-effort: any chain/disk error logs and returns whatever was read (or {}).""" + from validation.validator.consensus.commitment import ( + commit_score_hash, fetch_all_commitments, + ) + # Archive the signed pre-image first (survives even if the chain is down). + _archive_signed_scores(signer, run_id=run_id, family_id=family_id, local_scores=local_scores) + if _COMMIT_SCORES and subtensor is not None and wallet is not None and run_id not in _COMMITTED_RUNS: + try: + commit_score_hash( + subtensor, wallet, netuid=netuid, run_id=run_id, + family_id=family_id, scores=local_scores, + ) + _COMMITTED_RUNS.add(run_id) + logger.info("consensus: committed score hash on-chain for run=%s", run_id) + except Exception as exc: # noqa: BLE001 — commit is best-effort + logger.warning("consensus: on-chain score commit failed run=%s: %s", run_id, exc) + if subtensor is None: + return {} + try: + return fetch_all_commitments(subtensor, netuid=netuid) + except Exception as exc: # noqa: BLE001 + logger.warning("consensus: reading commitments failed: %s", exc) + return {} + + +async def _consensus_weights_for_current_run( + metagraph: Any, signer: Any, *, subtensor: Any = None, wallet: Any = None, netuid: int = 0, +): + """Gather verified validator score maps over the metagraph snapshot and + return the stake-weighted-median consensus result, or ``None`` if this + validator hasn't scored a run yet. Pure read of ``metagraph`` + the local + score store + peer gossip; never raises into the weight loop.""" + from validation.validator.consensus.score_store import STORE + from validation.validator.consensus.weights import compute_consensus + + run = STORE.busiest_run() + if run is None: + return None + run_id, family_id = run + local_scores = STORE.score_map(run_id, family_id) + if not local_scores: + return None + # Commit our score hash on-chain + read everyone's commitments (anti- + # equivocation). Best-effort; never blocks consensus. + commitments = _commit_and_fetch_commitments( + subtensor=subtensor, wallet=wallet, signer=signer, netuid=netuid, + run_id=run_id, family_id=family_id, local_scores=local_scores, + ) + try: + async with httpx.AsyncClient() as client: + return await compute_consensus( + metagraph=metagraph, + http_client=client, + self_hotkey=signer.hotkey, + run_id=run_id, + family_id=family_id, + local_scores=local_scores, + min_stake=_CONSENSUS_MIN_STAKE, + margin=_CONSENSUS_MARGIN, + quorum_fraction=_CONSENSUS_QUORUM, + per_peer_timeout=_CONSENSUS_PEER_TIMEOUT, + commitments=commitments, + require_commitment=_REQUIRE_COMMITMENT, + ) + except Exception as exc: # noqa: BLE001 — consensus failure must not skip the window + logger.warning("consensus: computation failed (%s)", exc) + return None + async def run_weight_setting_loop() -> None: """Background loop: poll owner-api for weights, set on-chain every cycle. @@ -805,6 +1000,73 @@ async def run_weight_setting_loop() -> None: continue uid_by_hotkey = {str(hk): uid for uid, hk in enumerate(metagraph.hotkeys)} + # ── Decentralized weight-setting (shadow / decentralized modes) ─── + # Compute the stake-weighted-median consensus winner from gossiped + # validator score maps over THIS metagraph snapshot. In shadow mode + # we only log how it diverges from the owner pick; in decentralized + # mode it replaces the owner pick and the owner endpoint leaves the + # weight path entirely. + if _WEIGHT_MODE in ("shadow", "decentralized"): + consensus = await _consensus_weights_for_current_run( + metagraph, signer, subtensor=subtensor, wallet=wallet, netuid=netuid, + ) + if consensus is not None: + logger.info("consensus: %s", consensus.summary()) + owner_winner = ( + max(weights_by_hotkey, key=weights_by_hotkey.get) + if weights_by_hotkey else None + ) + if _WEIGHT_MODE == "shadow": + # Accumulate winner-agreement across runs so the operator + # has a quantitative gate for flipping to decentralized, + # not just a per-cycle diverged/agreed line. + _SHADOW_DIVERGENCE.record( + run_id=str(consensus.run_id or ""), + local_winner=consensus.winner, + owner_winner=owner_winner, + local_scores=consensus.consensus_scores, + # Owner per-miner weights (≥2 entries → rank ρ + # populates; sparse/WTA → stays n/a, no overclaim). + owner_scores=weights_by_hotkey, + ) + # Persist so the gate window is a durable history, not + # reset on restart (best-effort). + try: + _SHADOW_DIVERGENCE.persist(_divergence_path()) + except Exception: # noqa: BLE001 + pass + if consensus.winner != owner_winner: + logger.warning( + "weight-setting SHADOW DIVERGENCE: consensus=%s owner=%s " + "(setting owner pick — flip EIREL_WEIGHT_MODE=decentralized " + "once divergence is acceptably low)", + consensus.winner, owner_winner, + ) + logger.info("weight-setting shadow %s", _SHADOW_DIVERGENCE.summary()) + elif consensus.quorum_met and consensus.weights_by_hotkey: + weights_by_hotkey = consensus.weights_by_hotkey + current_run_id = consensus.run_id + elif consensus.incumbent: + # Partition / insufficient quorum: hold the on-chain + # incumbent rather than fall back to the owner — a missed + # round must never re-centralize the pick. + weights_by_hotkey = {consensus.incumbent: 1.0} + current_run_id = consensus.run_id + logger.warning( + "weight-setting: no consensus quorum — holding incumbent %s", + consensus.incumbent, + ) + else: + weights_by_hotkey = {} # nobody to hold → burn to UID 0 + logger.warning( + "weight-setting: no consensus quorum and no incumbent — burning", + ) + elif _WEIGHT_MODE == "decentralized": + logger.info( + "weight-setting: no local scores for consensus yet — burning to UID 0", + ) + weights_by_hotkey = {} + # 3. Build UID weight vector uid_weights: dict[int, float] = {} total_assigned = 0.0 @@ -1068,16 +1330,17 @@ async def run_distributed_benchmarks( _PAIRWISE_BANNER_LOGGED["done"] = True # Oracle enrichment layer — runs at task-claim time, produces - # ``ReconciledOracle`` per task with ``expected_claims`` + - # ``must_not_claim`` for the EvalJudge to score against. Wired - # only when the validator has API keys for ≥2 oracles + Chutes - # reconciler; otherwise three_oracle items fall back to disputed. + # ``ReconciledOracle`` per task with ``expected_claims`` for the + # EvalJudge to score against. Wired only when the validator has API + # keys for ≥2 oracles + Chutes reconciler; otherwise three_oracle + # items fall back to disputed. oracle_fanout, reconciler = _build_oracle_layer() total_claimed = 0 total_submitted = 0 total_failed = 0 total_baseline_failed = 0 + _booted_run_ids: set[str] = set() # local-exec runs deployed this invocation _benchmark_started_at = time.perf_counter() _metrics.benchmark_runs_started_total.labels(family=family_id).inc() _benchmark_outcome = "success" @@ -1094,13 +1357,31 @@ async def run_distributed_benchmarks( ) async def _invoke_one_miner( - miner: dict[str, Any], task_obj: Any, task_id: str, + miner: dict[str, Any], task_obj: Any, task_id: str, local_run=None, ) -> tuple[str, BenchmarkTaskRun]: + if _local_exec_enabled(): + # Decentralized exec: invoke the locally-deployed pod directly and + # stamp the job id + per-job token so the validator's own + # provider-proxy/tools attribute and authorize the spend. + endpoint = local_run.endpoint_for(miner["hotkey"]) if local_run else None + if not endpoint: + reason = (local_run.failed.get(miner["hotkey"]) if local_run else None) or "local_runtime_unavailable" + return miner["hotkey"], BenchmarkTaskRun( + task_id=task_id, family_id=family_id, + prompt=task_obj.prompt, expected_output=task_obj.expected_output, + response={}, status="failed", error=reason, metadata={}, + ) + deployment_id = local_run.deployment_id_for(miner["hotkey"]) or miner["hotkey"] + cost_tag = build_cost_tag(deployment_id, task_id=task_id) + auth_headers = job_request_headers(cost_tag=cost_tag, signing_key=_TOOL_JOB_SIGNING_KEY) + else: + endpoint = miner["endpoint"] + auth_headers = miner.get("auth_headers", {}) miner_target = MinerBenchmarkTarget( hotkey=miner["hotkey"], - endpoint=miner["endpoint"], + endpoint=endpoint, stake=0, - metadata={"auth_headers": miner.get("auth_headers", {})}, + metadata={"auth_headers": auth_headers}, ) try: run = await _invoke_task( @@ -1123,6 +1404,25 @@ async def _evaluate_task(task_claim: dict[str, Any]) -> str: miners = task_claim.get("miners") or [] if not miners: return "ok" # nothing to evaluate + + # Decentralized exec: lazily deploy (roster ∩ claimed miners) into this + # validator's own cluster on the first task of the run. ensure_run is + # memoized, so only the first task pays the boot cost. On failure we run + # with local_run=None → every miner scores as error (same as owner-mode + # never-ready), never silently falling back to the owner proxy. + local_run = None + if _local_exec_enabled(): + _run_id_for_exec = str(task_claim.get("_run_id") or "") + try: + local_run = await _get_run_controller(signer).ensure_run( + run_id=_run_id_for_exec, + family_id=family_id, + owner_url=owner_url, + wanted_hotkeys={str(m["hotkey"]) for m in miners}, + ) + _booted_run_ids.add(_run_id_for_exec) + except Exception as exc: # noqa: BLE001 + logger.warning("local-exec ensure_run failed run=%s: %s", _run_id_for_exec, exc) task_started_at = time.perf_counter() class _TaskProxy: @@ -1187,7 +1487,7 @@ class _TaskProxy: # cycle's judging input identical (batch-independent scoring) # at flat oracle cost. cached_baseline = task_claim.get("cached_baseline") - using_cached_baseline = bool(cached_baseline) + using_cached_baseline = _should_use_cached_baseline(cached_baseline) # Oracle enrichment runs in parallel with miner dispatch. # three_oracle items → 3 frontier-LLM calls + 1 reconciler. @@ -1214,7 +1514,7 @@ class _TaskProxy: ) miner_tasks = [ - asyncio.create_task(_invoke_one_miner(m, task_obj, task_id)) for m in miners + asyncio.create_task(_invoke_one_miner(m, task_obj, task_id, local_run)) for m in miners ] try: @@ -1267,9 +1567,6 @@ class _TaskProxy: ) reconciled_for_baseline = ReconciledOracle( expected_claims=[], - must_not_claim=list( - (task_obj.expected_output or {}).get("must_not_claim") or [] - ), oracle_status="disputed", disagreement_note=f"enrichment_crashed: {enrich_exc!r}", ) @@ -1400,6 +1697,15 @@ async def _judge_miner(miner_run: tuple[str, BenchmarkTaskRun]) -> dict[str, Any # the submit endpoint for storage. response_meta = (run.response or {}).get("metadata") or {} proxy_cost_usd = float(response_meta.get("proxy_cost_usd") or 0.0) + # Decentralized exec: the owner didn't inject cost into the stream — + # read it from THIS validator's own provider-proxy ledger, keyed on + # the same job tag the validator stamped at invocation. + _local_cost_tag = "" + if _local_exec_enabled(): + _dep = (local_run.deployment_id_for(miner_hotkey) if local_run else None) or miner_hotkey + _local_cost_tag = build_cost_tag(_dep, task_id=task_id) + _local_cost = await _fetch_local_proxy_cost(_local_cost_tag) + proxy_cost_usd = float(_local_cost.get("llm_cost_usd") or 0.0) if run.status != "completed": return { "miner_hotkey": miner_hotkey, @@ -1496,11 +1802,20 @@ async def _judge_miner(miner_run: tuple[str, BenchmarkTaskRun]) -> dict[str, Any expected_output = ( getattr(task_obj, "expected_output", {}) or {} ) - expected_answer = str(expected_output.get("answer") or "").strip() - must_not_claim_text = ( - "; ".join(str(x) for x in (expected_output.get("must_not_claim") or [])) + # All ground truth comes from the reconciled oracle — + # ``expected_claims`` holds oracle-generated consensus + + # majority claims for three_oracle tasks, or the pool's + # deterministic grader gold (via ``from_deterministic``) + # for deterministic tasks. There is no static + # ``expected_output.answer`` / ``must_not_claim`` plumbed + # into scoring any more. + expected_claims_text = "\n".join( + reconciled_for_task.expected_claims + ) + reconciled_answer = ( + reconciled_for_task.expected_claims[0] + if reconciled_for_task.expected_claims else "" ) - constraints = must_not_claim_text or None # Deterministic dimensions miner_response_payload = run.response or {} @@ -1523,22 +1838,27 @@ async def _judge_miner(miner_run: tuple[str, BenchmarkTaskRun]) -> dict[str, Any miner_latency_seconds=miner_latency, mode_budget_seconds=latency_budget, proxy_cost_usd=proxy_cost_usd, - cost_budget_usd=None, # per-task cost cap not - # yet wired; latency leg - # alone is informative. + # cost_budget_usd=None → the cost ramp is skipped; only + # the latency leg scores. DO NOT wire a cost budget here + # without normalizing under EIREL_EXEC_MODE=local: each + # validator pays its own provider prices, so a raw + # proxy_cost_usd/budget ratio would inject per-validator + # price variance into the stake-weighted-median consensus. + cost_budget_usd=None, ) if "latency_cost" in applicable else None ) # Sandbox tasks: simple deterministic exact-match for - # computation_correctness when the bundle ships an - # ``expected_output.answer``. Substring match against - # the candidate response keeps it forgiving for - # whitespace / formatting drift. + # computation_correctness against the reconciled gold + # (``expected_claims[0]`` — the deterministic grader's + # pre-baked answer). Substring match against the + # candidate response keeps it forgiving for whitespace / + # formatting drift. computation_correctness_score: float | None = None if "computation_correctness" in applicable: - if expected_answer: - ans_norm = expected_answer.strip().lower() + if reconciled_answer: + ans_norm = reconciled_answer.strip().lower() cand_norm = (miner_answer or "").strip().lower() computation_correctness_score = ( 1.0 if ans_norm and ans_norm in cand_norm else 0.0 @@ -1563,13 +1883,17 @@ async def _judge_miner(miner_run: tuple[str, BenchmarkTaskRun]) -> dict[str, Any "question": pairwise_prompt, "answers": [miner_answer], } - if constraints: - multi_bundle["constraints"] = constraints try: multi_resp = await asyncio.to_thread( judge_client.judge_multi, bundle=multi_bundle, - expected_answer=expected_answer or None, + # grounded_correctness anchors on the + # oracle-generated expected_claims, not a + # static answer. ``None`` only when the + # oracle produced no claims (e.g. disputed), + # in which case the rubric falls back to + # the candidate's own citations. + expected_answer=expected_claims_text or None, candidate_citations=citations_list, applicable_metrics=outer_dims, ) @@ -1609,23 +1933,16 @@ def _multi_score(name: str) -> float | None: # ── EvalJudge + composite (the new ranking signal) ───── # Use the cached reconciled-oracle output (expected - # claims + must_not_claim) as the judge's reference. - # For three_oracle tasks: validator-side reconciler - # produced the expected_claims at task-claim time; for - # deterministic tasks: from_deterministic wrapped the - # pool's pre-baked answer. + # claims) as the judge's reference. For three_oracle + # tasks: validator-side reconciler produced the + # expected_claims at task-claim time; for deterministic + # tasks: from_deterministic wrapped the pool's pre-baked + # answer. # # The composite multiplicatively combines outcome # (correct/partial/wrong/...) × tool_attestation × - # efficiency × hallucination_knockout × - # cost_attestation_knockout. final_task_score is the - # composite, replacing the legacy weighted-sum. - expected_claims_text = "\n".join( - reconciled_for_task.expected_claims - ) - must_not_claim_for_judge = list( - reconciled_for_task.must_not_claim - ) + # efficiency × cost_attestation_knockout. final_task_score + # is the composite, replacing the legacy weighted-sum. eval_required_tool = ( str(expected_output.get("required_tool") or "").strip() or None @@ -1642,10 +1959,6 @@ def _multi_score(name: str) -> float | None: "question": judge_prompt, "answers": [miner_answer], } - if must_not_claim_for_judge: - eval_bundle["constraints"] = "; ".join( - must_not_claim_for_judge - ) eval_outcome_str: str = "wrong" eval_failure_mode: str | None = None @@ -1655,8 +1968,7 @@ def _multi_score(name: str) -> float | None: eval_resp = await asyncio.to_thread( judge_client.judge_eval, bundle=eval_bundle, - expected_answer=expected_claims_text or expected_answer or "(no expected answer)", - must_not_claim=must_not_claim_for_judge, + expected_answer=expected_claims_text or "(no expected answer)", required_tool=eval_required_tool, oracle_source=oracle_source_for_judge, ) @@ -1682,12 +1994,26 @@ def _multi_score(name: str) -> float | None: # miner can set that to another miner's tag to inherit # their attested tool calls. Absent tag → empty ledger # (fail closed) rather than trusting a miner value. - miner_job_id = str(response_meta.get("proxy_cost_tag") or "") - ledger_tools = await _fetch_ledger_tools( - miner_job_id, - owner_url=owner_url, - signer=signer, - ) + if _local_exec_enabled(): + # Local sink: key on the tag the validator itself stamped + # (never a miner echo); auth with the local bearer token. + miner_job_id = _local_cost_tag + ledger_tools = await _fetch_ledger_tools( + miner_job_id, + owner_url=_LOCAL_LEDGER_URL, + signer=signer, + extra_headers=( + {"Authorization": f"Bearer {_LOCAL_LEDGER_TOKEN}"} + if _LOCAL_LEDGER_TOKEN else {} + ), + ) + else: + miner_job_id = str(response_meta.get("proxy_cost_tag") or "") + ledger_tools = await _fetch_ledger_tools( + miner_job_id, + owner_url=owner_url, + signer=signer, + ) composite_resp: dict[str, Any] = {} try: @@ -1696,7 +2022,6 @@ def _multi_score(name: str) -> float | None: outcome=eval_outcome_str, failure_mode=eval_failure_mode, candidate_response=miner_answer, - must_not_claim=must_not_claim_for_judge, required_tool=eval_required_tool, ledger_tools=ledger_tools, latency_ms=int(max(0.0, miner_latency) * 1000), @@ -1705,6 +2030,12 @@ def _multi_score(name: str) -> float | None: int(latency_budget * 1000) if latency_budget else None ), + # cost_budget/floor=None → the eiretes composite gets + # cost_usd but no threshold, so cost can't knockout + # (cost is display-only). DO NOT wire either under + # EIREL_EXEC_MODE=local without per-validator cost + # normalization — raw dollars vary by each validator's + # provider prices and would pollute the median. cost_budget_usd=None, cost_floor_usd=None, # Outer-dimension gates: ``grounded`` ≥ 0.60, @@ -1898,6 +2229,16 @@ def _cost_usd(payload: dict[str, Any] | None) -> float: judge_results = await asyncio.gather(*[_judge_miner(r) for r in resolved_runs]) + # Retain our own per-miner scores locally so this validator can gossip + # and stake-weighted-median-aggregate them for DECENTRALIZED weight + # setting — otherwise the owner-api is the only place these scores ever + # land. Best-effort: never let score capture break evaluation/submission. + try: + from validation.validator.consensus.score_store import STORE + STORE.record(str(task_claim.get("_run_id") or ""), family_id, judge_results) + except Exception: # noqa: BLE001 — capture is advisory, not load-bearing + pass + # Verdict tally — one-line summary so an operator scrolling the # validator log can see "task X scored 2/3 matches" at a glance, # without having to query the DB. @@ -2014,6 +2355,13 @@ async def _bounded_evaluate(task_claim: dict[str, Any]) -> str: if not tasks: break + # Tag each task with the claim's run_id so the per-task evaluator + # files its local scores under the right run for consensus. + claimed_run_id = str(claim_data.get("run_id") or run_id or "") + for _t in tasks: + if isinstance(_t, dict): + _t.setdefault("_run_id", claimed_run_id) + total_claimed += len(tasks) results = await asyncio.gather( @@ -2027,8 +2375,25 @@ async def _bounded_evaluate(task_claim: dict[str, Any]) -> str: total_baseline_failed += 1 else: total_failed += 1 + + # Persist the score store after each batch so a restart still serves + # the scores gathered so far (the gossip endpoint reads STORE). + try: + from validation.validator.consensus.score_store import STORE, store_path + STORE.persist(store_path()) + except Exception: # noqa: BLE001 — persistence is best-effort + pass finally: judge_client.close() + # Decentralized exec: tear down the pods this validator booted for the + # run(s) it just drained, so we don't leak miner pods between cycles. + if _local_exec_enabled() and _booted_run_ids: + controller = _get_run_controller(signer) + for _rid in _booted_run_ids: + try: + await controller.teardown_run(_rid) + except Exception as exc: # noqa: BLE001 + logger.warning("local-exec teardown failed run=%s: %s", _rid, exc) if total_claimed > 0 and (total_failed + total_baseline_failed) == total_claimed: _benchmark_outcome = "failed" diff --git a/validation/validator/eval_config.py b/validation/validator/eval_config.py index 5ce33b1..413266a 100644 --- a/validation/validator/eval_config.py +++ b/validation/validator/eval_config.py @@ -7,8 +7,7 @@ for items tagged ``oracle_source: three_oracle``. Each validator independently produces its own ground truth. * **Reconciler** (Chutes-hosted ``zai-org/GLM-5.1-TEE``) — synthesizes - the 3 oracle answers into ``expected_claims`` + ``must_not_claim`` - extras for the judge. + the 3 oracle answers into ``expected_claims`` for the judge. This module is the single source of truth for the env vars these dependencies consume. Importing from here keeps the providers / oracle diff --git a/validation/validator/local_exec/__init__.py b/validation/validator/local_exec/__init__.py new file mode 100644 index 0000000..03efabf --- /dev/null +++ b/validation/validator/local_exec/__init__.py @@ -0,0 +1,6 @@ +"""Validator-side local execution of miner submissions (EIREL_EXEC_MODE=local). + +Lets a validator run miner submissions on its OWN cluster — pulling owner-signed +rosters + hash-pinned archives — instead of invoking miners through the owner +proxy, so it observes response/latency/cost/tool-use first-hand. +""" diff --git a/validation/validator/local_exec/runtime_controller.py b/validation/validator/local_exec/runtime_controller.py new file mode 100644 index 0000000..6b2c1f7 --- /dev/null +++ b/validation/validator/local_exec/runtime_controller.py @@ -0,0 +1,240 @@ +"""LocalRunController — deploys miner submissions into the validator's own +cluster for decentralized execution. + +Per run, it: fetches the owner-signed roster, verifies the owner signature +(anti-equivocation), and for each member the validator is actually asked to +score (roster ∩ claimed hotkeys — bounds the footprint), pulls the submission +archive, checks its bytes hash to the roster-committed ``archive_sha256`` +(refuses tampered code), and deploys it via the reused +``KubernetesMinerRuntimeManager`` pointed at the validator-local provider-proxy. +It exposes ``{hotkey -> local endpoint}`` so the engine can redirect invocation, +and tears the pods down when the run drains. + +The class is dependency-injected (manager, http client, signer) so the +fetch→verify→hash→deploy logic is unit-testable without a real cluster; +``build_local_run_controller`` wires the real components from env. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +from dataclasses import dataclass, field +from typing import Any + +from shared.common.manifest import SubmissionManifest +from shared.common.roster import verify_roster + +logger = logging.getLogger(__name__) + + +@dataclass +class RunEndpoints: + """Per-run result the engine consumes: where each miner runs locally, the + deployment id used for cost-tag attribution, and members that failed to + build/boot/verify (scored as ``error``, like today's never-ready path).""" + + run_id: str + endpoint_by_hotkey: dict[str, str] = field(default_factory=dict) + deployment_id_by_hotkey: dict[str, str] = field(default_factory=dict) + failed: dict[str, str] = field(default_factory=dict) + submission_ids: set[str] = field(default_factory=set) + + def endpoint_for(self, hotkey: str) -> str | None: + return self.endpoint_by_hotkey.get(hotkey) + + def deployment_id_for(self, hotkey: str) -> str | None: + return self.deployment_id_by_hotkey.get(hotkey) + + +def _sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +class LocalRunController: + def __init__( + self, + *, + manager: Any, + http_client: Any, + signer: Any, + owner_hotkey: str | None, + provider_proxy_url: str, + provider_proxy_token: str, + cpu_millis: int = 1000, + memory_bytes: int = 2 * 1024 * 1024 * 1024, + ) -> None: + self._manager = manager + self._http = http_client + self._signer = signer + self._owner_hotkey = owner_hotkey or None + self._provider_proxy_url = provider_proxy_url + self._provider_proxy_token = provider_proxy_token + self._cpu_millis = cpu_millis + self._memory_bytes = memory_bytes + self._runs: dict[str, RunEndpoints] = {} + self._build_locks: dict[str, asyncio.Lock] = {} + self._guard = asyncio.Lock() + + # ── public lifecycle ───────────────────────────────────────────────────── + + async def ensure_run( + self, *, run_id: str, family_id: str, owner_url: str, wanted_hotkeys: set[str] | None + ) -> RunEndpoints: + """Idempotent per run_id: fetch+verify roster, deploy the wanted members, + return the endpoint map. Concurrent first-task callers await the same + build (per-run lock).""" + async with self._guard: + lock = self._build_locks.setdefault(run_id, asyncio.Lock()) + async with lock: + cached = self._runs.get(run_id) + if cached is not None: + return cached + state = await self._build_run( + run_id=run_id, family_id=family_id, owner_url=owner_url, + wanted_hotkeys=wanted_hotkeys, + ) + self._runs[run_id] = state + return state + + async def teardown_run(self, run_id: str) -> None: + state = self._runs.pop(run_id, None) + if state is None: + return + for submission_id in state.submission_ids: + try: + await self._manager.stop_runtime(submission_id, reason="run_complete") + except Exception as exc: # noqa: BLE001 — teardown is best-effort + logger.warning("local-exec: stop_runtime failed for %s: %s", submission_id, exc) + async with self._guard: + self._build_locks.pop(run_id, None) + + async def reconcile(self, active_run_ids: set[str]) -> None: + """GC pods from runs no longer active (crash recovery / leaked pods).""" + active_submissions: set[str] = set() + for run_id in active_run_ids: + state = self._runs.get(run_id) + if state is not None: + active_submissions |= state.submission_ids + try: + await self._manager.reconcile_active_submissions(active_submissions) + except Exception as exc: # noqa: BLE001 + logger.warning("local-exec: reconcile failed: %s", exc) + + # ── internals ──────────────────────────────────────────────────────────── + + async def _build_run( + self, *, run_id: str, family_id: str, owner_url: str, wanted_hotkeys: set[str] | None + ) -> RunEndpoints: + state = RunEndpoints(run_id=run_id) + signed = await self._fetch_roster(owner_url, run_id, family_id) + if not verify_roster(signed, expected_owner_hotkey=self._owner_hotkey): + # An unverifiable roster means we can't trust who/what to run — refuse + # the whole run rather than execute unattested code. + raise RuntimeError( + f"roster signature verification failed for run={run_id} " + f"(expected owner {self._owner_hotkey})" + ) + members = (signed.get("roster") or {}).get("members") or [] + for member in members: + hotkey = str(member.get("hotkey") or "") + if not hotkey: + continue + if wanted_hotkeys is not None and hotkey not in wanted_hotkeys: + continue # only boot what this validator was asked to score + await self._deploy_member(state, owner_url, run_id, member) + logger.info( + "local-exec run=%s deployed=%d failed=%d", + run_id, len(state.endpoint_by_hotkey), len(state.failed), + ) + return state + + async def _deploy_member( + self, state: RunEndpoints, owner_url: str, run_id: str, member: dict[str, Any] + ) -> None: + hotkey = str(member["hotkey"]) + submission_id = str(member["submission_id"]) + deployment_id = str(member.get("deployment_id") or "") or submission_id + committed_sha = str(member.get("archive_sha256") or "") + try: + archive = await self._fetch_archive(owner_url, run_id, submission_id) + actual_sha = _sha256_hex(archive) + if actual_sha != committed_sha: + state.failed[hotkey] = "archive hash mismatch vs signed roster" + logger.warning( + "local-exec run=%s sub=%s REFUSED: archive %s != committed %s", + run_id, submission_id, actual_sha[:12], committed_sha[:12], + ) + return + manifest = SubmissionManifest.model_validate(member.get("manifest_json") or {}) + handle = await self._manager.ensure_runtime( + submission_id=submission_id, + deployment_id=deployment_id, # == submission_id so runtime_handle keys line up + archive_bytes=archive, + manifest=manifest, + provider_proxy_url=self._provider_proxy_url, + provider_proxy_token=self._provider_proxy_token, + requested_cpu_millis=self._cpu_millis, + requested_memory_bytes=self._memory_bytes, + ) + state.endpoint_by_hotkey[hotkey] = handle.endpoint_url + state.deployment_id_by_hotkey[hotkey] = deployment_id + state.submission_ids.add(submission_id) + except Exception as exc: # noqa: BLE001 — one bad submission must not abort the run + state.failed[hotkey] = str(exc) + logger.warning("local-exec run=%s sub=%s deploy failed: %s", run_id, submission_id, exc) + + async def _fetch_roster(self, owner_url: str, run_id: str, family_id: str) -> dict: + path = f"/v1/runs/{run_id}/roster?family_id={family_id}" + resp = await self._http.get( + f"{owner_url.rstrip('/')}{path}", headers=self._signed_get(path) + ) + resp.raise_for_status() + return resp.json() + + async def _fetch_archive(self, owner_url: str, run_id: str, submission_id: str) -> bytes: + path = f"/v1/runs/{run_id}/submissions/{submission_id}/archive" + resp = await self._http.get( + f"{owner_url.rstrip('/')}{path}", headers=self._signed_get(path) + ) + resp.raise_for_status() + return resp.content + + def _signed_get(self, path: str) -> dict[str, str]: + # Same hotkey-signed header scheme the engine uses for owner-api reads. + # The owner verifies the signature over ``request.url.path`` (no query), + # so sign the bare path — signing ``?family_id=`` too yields a 401. + return self._signer.signed_headers("GET", path.split("?", 1)[0], _sha256_hex(b"")) + + +def build_local_run_controller(signer: Any) -> LocalRunController: + """Wire the real components from env (used by the engine in local exec).""" + import httpx + + from infra.miner_runtime.runtime_manager import KubernetesMinerRuntimeManager + + manager = KubernetesMinerRuntimeManager( + kubeconfig_path=os.getenv("EIREL_VALIDATOR_KUBECONFIG_PATH") or None, + namespace=os.getenv("EIREL_VALIDATOR_RUNTIME_NAMESPACE", "eirel-miners"), + system_namespace=os.getenv("EIREL_VALIDATOR_RUNTIME_SYSTEM_NAMESPACE", "eirel-system"), + control_plane_namespace=os.getenv( + "EIREL_VALIDATOR_RUNTIME_CONTROL_PLANE_NAMESPACE", "eirel-system" + ), + runtime_image=os.getenv("EIREL_VALIDATOR_RUNTIME_IMAGE", "eirel-miner-runtime:latest"), + shared_secret_name=os.getenv("EIREL_VALIDATOR_RUNTIME_SHARED_SECRET_NAME", "eirel-runtime-shared"), + service_domain=os.getenv("EIREL_VALIDATOR_RUNTIME_SERVICE_DOMAIN", "svc.cluster.local"), + health_timeout_seconds=float(os.getenv("EIREL_VALIDATOR_RUNTIME_HEALTH_TIMEOUT_SECONDS", "300")), + ) + # Long read timeout — archives can be up to the 200MB submission cap. + http_client = httpx.AsyncClient(timeout=httpx.Timeout(120.0, read=300.0)) + return LocalRunController( + manager=manager, + http_client=http_client, + signer=signer, + owner_hotkey=os.getenv("EIREL_OWNER_HOTKEY") or None, + provider_proxy_url=os.getenv("EIREL_LOCAL_PROVIDER_PROXY_URL", "http://provider-proxy:8092"), + provider_proxy_token=os.getenv("EIREL_LOCAL_PROVIDER_PROXY_TOKEN", ""), + cpu_millis=int(os.getenv("EIREL_VALIDATOR_RUNTIME_SUBMISSION_CPU_MILLIS", "1000")), + memory_bytes=int(os.getenv("EIREL_VALIDATOR_RUNTIME_SUBMISSION_MEMORY_MB", "2048")) * 1024 * 1024, + ) diff --git a/validation/validator/main.py b/validation/validator/main.py index 278b7bb..7409fee 100644 --- a/validation/validator/main.py +++ b/validation/validator/main.py @@ -3,6 +3,7 @@ import asyncio import logging import os +from datetime import UTC, datetime import uvicorn from contextlib import asynccontextmanager @@ -13,6 +14,11 @@ from validation.validator.engine import run_distributed_benchmarks, run_validator_loop, run_weight_setting_loop from validation.validator import metrics as _metrics +from validation.validator.consensus.gossip import sign_scores +from validation.validator.consensus.archive import ARCHIVE +from validation.validator.consensus.divergence import TRACKER, divergence_path, sign_divergence +from validation.validator.consensus.score_store import STORE, store_path +from shared.common.bittensor_signing import load_signer logger = logging.getLogger(__name__) @@ -53,6 +59,18 @@ class DistributedBenchmarkRequest(BaseModel): @asynccontextmanager async def _lifespan(app: FastAPI): """Start evaluation and weight-setting background loops on startup.""" + # Restore persisted scores so a restarted validator still serves the scores + # it gathered before the restart (the gossip endpoint reads STORE). + try: + STORE.load(store_path()) + except Exception: # noqa: BLE001 — best-effort + logger.warning("could not load persisted score store", exc_info=True) + # Restore the shadow-mode divergence history so the gate window is a durable + # record across restarts (and the endpoint reflects it immediately). + try: + TRACKER.load(divergence_path()) + except Exception: # noqa: BLE001 — best-effort + logger.warning("could not load persisted divergence tracker", exc_info=True) tasks: list[asyncio.Task] = [] if _AUTO_LOOP_ENABLED: logger.info("validator auto-loop enabled — starting evaluation + weight-setting loops") @@ -86,6 +104,64 @@ async def metrics() -> Response: ) +_SIGNER = None + + +def _signer(): + """Cached validator signer — loaded once, reused for every signed payload.""" + global _SIGNER + if _SIGNER is None: + _SIGNER = load_signer( + mnemonic=os.getenv("EIREL_VALIDATOR_MNEMONIC") or None, + wallet_name=os.getenv("EIREL_VALIDATOR_WALLET_NAME") or None, + hotkey_name=os.getenv("EIREL_VALIDATOR_HOTKEY_NAME") or None, + wallet_path=os.getenv("EIREL_VALIDATOR_WALLET_PATH") or None, + ) + return _SIGNER + + +@app.get("/v1/consensus/scores/{run_id}") +async def consensus_scores(run_id: str, family_id: str | None = None) -> dict: + """Serve THIS validator's per-miner score map for ``run_id``, signed by its + hotkey. Peers fetch this to aggregate; the signature + the on-chain axon + identity are what let them trust it. + + Once the validator has finalized + committed the run, the DURABLE archived + signed report is returned (a fixed signature whose hash matches the on-chain + commitment) — so the served record is stable and survives restarts. Before + that, it's signed live from the in-memory store. Empty scores are returned + (signed) when this validator hasn't scored the run. + """ + archived = ARCHIVE.read(run_id) + if archived is not None: + return archived + fam = family_id + if fam is None: + fam = next((f for rid, f in STORE.runs() if rid == run_id), None) + fam = fam or os.getenv("EIREL_FAMILY_ID", "general_chat") + return sign_scores( + _signer().keypair, + run_id=run_id, + family_id=fam, + scores=STORE.score_map(run_id, fam), + timestamp=datetime.now(UTC).isoformat(), + ) + + +@app.get("/v1/consensus/divergence") +async def consensus_divergence() -> dict: + """THIS validator's hotkey-signed shadow-mode divergence summary — the + provable gate evidence (running winner-agreement + rank correlation, plus + the per-run records behind it) a peer/auditor checks before the network + flips EIREL_WEIGHT_MODE to decentralized.""" + return sign_divergence( + _signer().keypair, + TRACKER, + weight_mode=os.getenv("EIREL_WEIGHT_MODE", "owner"), + timestamp=datetime.now(UTC).isoformat(), + ) + + @app.post("/v1/distributed-benchmark-runs") async def distributed_benchmark_runs(payload: DistributedBenchmarkRequest) -> dict: return await run_distributed_benchmarks( diff --git a/validation/validator/metrics.py b/validation/validator/metrics.py index f939324..8ecc48e 100644 --- a/validation/validator/metrics.py +++ b/validation/validator/metrics.py @@ -184,8 +184,7 @@ def record_composite_knockout(family: str, knockout_factor: str) -> None: """Bump the per-knockout counter when composite is zeroed. ``knockout_factor`` ∈ {``tool_attestation`` / - ``hallucination_knockout`` / ``cost_attestation_knockout`` / - ``outcome_zero``}. + ``cost_attestation_knockout`` / ``outcome_zero``}. """ composite_knockouts_total.labels( family=family, knockout_factor=knockout_factor, diff --git a/validation/validator/reconciler.py b/validation/validator/reconciler.py index c15f8a2..3687710 100644 --- a/validation/validator/reconciler.py +++ b/validation/validator/reconciler.py @@ -9,15 +9,13 @@ supporting-vendor list per claim. * ``minority_claims`` — facts only 1 oracle supports (dropped from ``expected_claims`` but kept for telemetry). - * ``must_not_claim_extras`` — atomic claims any oracle explicitly - contradicts; ADDITIVE to the template-time floor. * ``oracle_status`` — ``consensus`` | ``majority`` | ``disputed``. * ``disagreement_note`` — one-line summary. Failure path: malformed JSON, LLM error, or fewer than 2 successful oracles → ``ReconciledOracle(oracle_status="disputed", -expected_claims=[], must_not_claim=template_floor)``. Items still go -to judging; ``disputed`` outcomes contribute 0.5. +expected_claims=[])``. Items still go to judging; ``disputed`` +outcomes contribute 0.5. """ from __future__ import annotations @@ -55,15 +53,13 @@ class ReconciledOracle: """Output of the reconciler — consumed by ``_judge_miner``. ``expected_claims`` is the set the judge scores satisfaction - against (consensus + majority claims combined). ``must_not_claim`` - is the union of template floor + reconciler extras. Telemetry + against (consensus + majority claims combined). Telemetry fields (``consensus_claims`` / ``majority_claims`` / ``minority_claims`` / ``vendor_status``) are persisted on the TaskMinerResult row for per-vendor agreement-rate analysis. """ expected_claims: list[str] = field(default_factory=list) - must_not_claim: list[str] = field(default_factory=list) oracle_status: OracleConsensusStatus = "disputed" disagreement_note: str | None = None consensus_claims: list[str] = field(default_factory=list) @@ -98,21 +94,17 @@ def from_deterministic( cls, *, answer: str, - must_not_claim_floor: Iterable[str] = (), ) -> "ReconciledOracle": """Build a ``ReconciledOracle`` for ``oracle_source=deterministic`` items. The pool's built-in grader (live_endpoint, sandbox_python, span F1, regex) is the truth; no oracle/reconciler call needed. ``answer`` is the pool's pre-baked gold; the judge consumes it - as the single member of ``expected_claims``. ``must_not_claim`` - carries only the template floor (no reconciler extras possible - without an LLM call). + as the single member of ``expected_claims``. """ cleaned_answer = (answer or "").strip() return cls( expected_claims=[cleaned_answer] if cleaned_answer else [], - must_not_claim=_dedupe_preserving_order(must_not_claim_floor), oracle_status="deterministic", consensus_claims=[cleaned_answer] if cleaned_answer else [], disagreement_note=None, @@ -135,9 +127,6 @@ def from_deterministic( (list of vendor names). * minority_claims — atomic claim only 1 oracle supports. Objects \ with ``claim`` and ``supporting_oracle`` (single vendor). - * must_not_claim_extras — atomic claims any oracle EXPLICITLY \ - contradicts another's. Plain strings, phrased as the false claim \ - (NOT the correction). Then set ``oracle_status``: * "consensus" — every claim has ALL-oracle support (every claim is \ @@ -158,7 +147,7 @@ def from_deterministic( "additionalProperties": False, "required": [ "consensus_claims", "majority_claims", "minority_claims", - "must_not_claim_extras", "oracle_status", + "oracle_status", ], "properties": { "consensus_claims": { @@ -190,9 +179,6 @@ def from_deterministic( }, }, }, - "must_not_claim_extras": { - "type": "array", "items": {"type": "string"}, - }, "oracle_status": { "type": "string", "enum": ["consensus", "majority", "disputed"], @@ -225,15 +211,8 @@ async def reconcile( *, prompt: str, groundings: list[OracleGrounding], - must_not_claim_floor: Iterable[str] = (), ) -> ReconciledOracle: - """Reconcile a list of oracle groundings into ``expected_claims``. - - ``must_not_claim_floor`` carries any template-time forbidden - claims; the final ``must_not_claim`` is their union with the - reconciler-derived extras. - """ - floor_list = _dedupe_preserving_order(must_not_claim_floor) + """Reconcile a list of oracle groundings into ``expected_claims``.""" ok = successful_groundings(groundings) statuses = vendor_status_map(groundings) # Capture each vendor's citations regardless of whether they @@ -265,10 +244,9 @@ async def reconcile( if len(ok) < 2: # 0-1 oracle survived: not enough signal for plurality - # voting. Fail-safe → disputed with template floor only. + # voting. Fail-safe → disputed. return ReconciledOracle( expected_claims=[], - must_not_claim=floor_list, oracle_status="disputed", disagreement_note=( f"only {len(ok)}/{len(groundings)} oracles " @@ -294,7 +272,6 @@ async def reconcile( ) return ReconciledOracle( expected_claims=[], - must_not_claim=floor_list, oracle_status="disputed", disagreement_note=f"reconciler_error: {exc}", vendor_status=statuses, @@ -312,7 +289,6 @@ async def reconcile( ) return ReconciledOracle( expected_claims=[], - must_not_claim=floor_list, oracle_status="disputed", disagreement_note=f"reconciler_malformed_json: {exc}", vendor_status=statuses, @@ -326,7 +302,6 @@ async def reconcile( consensus = _string_list(parsed.get("consensus_claims")) majority = _claim_list(parsed.get("majority_claims"), "supporting_oracles") minority = _claim_list(parsed.get("minority_claims"), "supporting_oracle") - extras = _string_list(parsed.get("must_not_claim_extras")) status = parsed.get("oracle_status") if status not in ("consensus", "majority", "disputed"): _logger.warning( @@ -349,13 +324,8 @@ async def reconcile( expected_claims.append(text) expected_claims = _dedupe_preserving_order(expected_claims) - # ``must_not_claim`` = union(template_floor, reconciler_extras), - # deduplicated preserving floor order so floor stays first. - must_not_claim = _dedupe_preserving_order(list(floor_list) + extras) - return ReconciledOracle( expected_claims=expected_claims, - must_not_claim=must_not_claim, oracle_status=status, disagreement_note=note, consensus_claims=consensus,