From 9648cfb2c157ad0b0163eabe6426bf0fded865c5 Mon Sep 17 00:00:00 2001 From: fstandhartinger Date: Thu, 30 Jul 2026 19:59:09 +0200 Subject: [PATCH] Add the two endpoints Hugging Face needs to list Chutes as an inference provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HF's provider registration has exactly two hard requirements we don't meet today. Both are covered here, additively — no existing response field changes shape and no existing caller has to be touched. 1. /v1/models pricing aliases. HF reads `pricing.input` / `pricing.output` (USD per million tokens) to power their provider comparison table and the `:cheapest` routing policy. We emit `pricing.prompt` / `pricing.completion` (OpenRouter naming). Both key pairs are now present with identical values. Also backfills `context_length` from `max_model_len` for the two models that don't report it (Mistral-Nemo, Nemotron-3-Nano-Omni), since HF wants it on every listed model. 2. POST /partners/huggingface/billing. HF polls this once a minute with the request IDs it routed to us and bills its users whatever we report. Costs come from the per-invocation billed balance already persisted in `invocations.metrics->>'b'`, so there is no second pricing path to keep in sync — HF users are charged exactly the standard Chutes rate. Scoped to the authenticated user, same bearer auth as inference. Two details in there are load-bearing rather than stylistic. Unknown request IDs are omitted rather than returned as 0: HF replaces its placeholder with whatever we return and never asks again, so a 0 for an invocation that just hasn't been written yet would bill it as free forever. An entirely unknown batch returns `{"requests": null}`, HF's documented retry signal. Both give us the ~30 minutes of retries HF allows. Also mirrors X-Chutes-InvocationID onto an `Inference-Id` response header in build_response_headers, and exposes it via CORS. HF accepts a custom header name, so this is optional — but it costs one line at the single choke point and makes the integration zero-config on their side. Co-Authored-By: Claude Opus 5 (1M context) --- api/chute/util.py | 14 +++- api/invocation/util.py | 8 +++ api/main.py | 2 + api/partners/__init__.py | 0 api/partners/router.py | 107 ++++++++++++++++++++++++++++++ charts/templates/api-ingress.yaml | 2 +- 6 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 api/partners/__init__.py create mode 100644 api/partners/router.py diff --git a/api/chute/util.py b/api/chute/util.py index 879405cd..81ac6a0d 100644 --- a/api/chute/util.py +++ b/api/chute/util.py @@ -2172,12 +2172,24 @@ async def get_and_store_llm_details(chute_id: str): model_info["price"] = price model_info["confidential_compute"] = chute.tee - # OpenRouter format. + # OpenRouter format, plus the Hugging Face aliases. HF's + # /v1/models contract asks for `pricing.input` / `pricing.output` + # (USD per million tokens) and powers their provider comparison + # table and the `:cheapest` routing policy. Same numbers, extra + # keys only — nothing that reads `prompt`/`completion` changes. model_info["pricing"] = { "prompt": per_million_in, "completion": per_million_out, "input_cache_read": input_cache_read, + "input": per_million_in, + "output": per_million_out, } + + # HF also requires `context_length` on every listed model. Most + # instances report it already; fall back to max_model_len for the + # few that don't rather than leaving the field absent. + if not model_info.get("context_length") and model_info.get("max_model_len"): + model_info["context_length"] = model_info["max_model_len"] if chute.llm_detail and isinstance(chute.llm_detail.overrides, dict): model_info.update( { diff --git a/api/invocation/util.py b/api/invocation/util.py index 85a834a4..482913b4 100644 --- a/api/invocation/util.py +++ b/api/invocation/util.py @@ -376,6 +376,14 @@ def build_response_headers(request, base_headers=None): Build response headers dict with quota, rate limit, and invoice billing info. """ headers = dict(base_headers or {}) + + # Mirror the invocation ID onto `Inference-Id`. Hugging Face uses this header + # to key the billing callback in api/partners/router.py; it is the name they + # suggest to providers who don't already have one. Purely additive — + # X-Chutes-InvocationID stays exactly as it is. + if "X-Chutes-InvocationID" in headers and "Inference-Id" not in headers: + headers["Inference-Id"] = headers["X-Chutes-InvocationID"] + if getattr(request.state, "quota_total", None) is not None: headers["X-Chutes-Quota-Total"] = str(int(request.state.quota_total)) headers["X-Chutes-Quota-Used"] = str(int(request.state.quota_used)) diff --git a/api/main.py b/api/main.py index c7c74d88..c5567dce 100644 --- a/api/main.py +++ b/api/main.py @@ -42,6 +42,7 @@ from api.e2e.router import router as e2e_router from api.encrypted_logs.router import router as encrypted_logs_router from api.model_alias.router import router as model_alias_router +from api.partners.router import router as partners_router from api.chute.util import chute_id_by_slug from api.database import Base, engine, get_session from api.config import settings @@ -173,6 +174,7 @@ async def lifespan(_: FastAPI): encrypted_logs_router, prefix="/encrypted_logs", tags=["Encrypted Logs"] ) default_router.include_router(model_alias_router, prefix="/model_aliases", tags=["Model Aliases"]) +default_router.include_router(partners_router, prefix="/partners", tags=["Partners"]) # Do not use app for this, else middleware picks it up diff --git a/api/partners/__init__.py b/api/partners/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/partners/router.py b/api/partners/router.py new file mode 100644 index 00000000..6ea0dbd8 --- /dev/null +++ b/api/partners/router.py @@ -0,0 +1,107 @@ +""" +Partner integration endpoints. + +Currently only Hugging Face, which requires providers to expose a billing +callback so HF can charge its users the provider's real per-request cost: +https://huggingface.co/docs/inference-providers/en/register-as-a-provider#4-billing +""" + +import math +from typing import Optional + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from api.database import get_db_session +from api.user.schemas import User +from api.user.service import get_current_user + +router = APIRouter() + +# HF sends batches of up to 10,000 request IDs per call. +MAX_REQUEST_IDS = 10000 + +# HF gives up on a request roughly 30 minutes after it was served, so there is no +# point scanning further back than that (and it keeps the index range tight). +LOOKBACK_INTERVAL = "2 hours" + +NANO_USD = 1_000_000_000 + + +class BillingRequest(BaseModel): + requestIds: list[str] = Field(default_factory=list) + + +class BillingEntry(BaseModel): + requestId: str + costNanoUsd: int + + +class BillingResponse(BaseModel): + requests: Optional[list[BillingEntry]] + + +@router.post("/huggingface/billing", response_model=BillingResponse) +async def huggingface_billing( + body: BillingRequest, + db: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_current_user()), +): + """ + Return the cost, in nano-USD, of previously served invocations. + + Hugging Face polls this once a minute with the IDs it routed to us, using the + same bearer auth as inference. Costs are read from the per-invocation billed + balance already persisted in `invocations.metrics->>'b'` (USD), so this + reports exactly what the user was charged — no separate pricing path to keep + in sync. + + Two behaviours are load-bearing and come straight from HF's spec: + + * Unknown IDs are OMITTED, never returned as 0. HF replaces its placeholder + cost with whatever we return and never asks again, so answering 0 for an + invocation that simply hasn't been written yet would bill it as free, + permanently. Omitted IDs get retried every minute for ~30 minutes. + * A batch we know nothing about returns `{"requests": null}`, HF's documented + "no data yet, retry later" signal. + + Scoped to the authenticated user so one partner key can never read another + account's costs. + """ + request_ids = list(dict.fromkeys(body.requestIds or []))[:MAX_REQUEST_IDS] + if not request_ids: + return BillingResponse(requests=None) + + result = await db.execute( + text(f""" +SELECT + invocation_id, + (metrics->>'b')::float AS billed_usd +FROM invocations +WHERE + invocation_id = ANY(:request_ids) + AND user_id = :user_id + AND completed_at >= NOW() - INTERVAL '{LOOKBACK_INTERVAL}' + AND metrics->>'b' IS NOT NULL +"""), + {"request_ids": request_ids, "user_id": current_user.user_id}, + ) + + entries = [] + for row in result.fetchall(): + if row.billed_usd is None or row.billed_usd < 0: + # Unparseable or negative: omit so HF retries rather than locking in + # a value it would otherwise treat as final. + continue + # HF requires a non-negative integer and rounds non-integers up, so do it + # here explicitly. 0 is valid and means "served for free". + entries.append( + BillingEntry( + requestId=row.invocation_id, + costNanoUsd=math.ceil(row.billed_usd * NANO_USD), + ) + ) + + return BillingResponse(requests=entries or None) diff --git a/charts/templates/api-ingress.yaml b/charts/templates/api-ingress.yaml index 08123d21..08a2637c 100644 --- a/charts/templates/api-ingress.yaml +++ b/charts/templates/api-ingress.yaml @@ -8,7 +8,7 @@ metadata: nginx.ingress.kubernetes.io/cors-allow-credentials: "false" nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization,X-Chute-Id,X-Instance-Id,X-E2E-Nonce,X-E2E-Stream,X-E2E-Path,X-Enable-Thinking,X-Provider,X-Provider-Version,X-Title" nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, OPTIONS, PUT, DELETE, PATCH" - nginx.ingress.kubernetes.io/cors-expose-headers: "X-Chutes-Quota-Total,X-Chutes-Quota-Used,X-Chutes-Quota-Remaining,X-Chutes-RL-User,X-Chutes-RL-Chute,X-Chutes-InvocationID" + nginx.ingress.kubernetes.io/cors-expose-headers: "X-Chutes-Quota-Total,X-Chutes-Quota-Used,X-Chutes-Quota-Remaining,X-Chutes-RL-User,X-Chutes-RL-Chute,X-Chutes-InvocationID,Inference-Id" nginx.ingress.kubernetes.io/cors-max-age: "1728000" nginx.ingress.kubernetes.io/proxy-body-size: "0" nginx.ingress.kubernetes.io/proxy-buffering: "off"