diff --git a/.design/python-sdk.md b/.design/python-sdk.md new file mode 100644 index 000000000..bb5e12f34 --- /dev/null +++ b/.design/python-sdk.md @@ -0,0 +1,629 @@ +# Python SDK (`tingly`) — design + +> Audience: tingly-box contributors touching the SDK seam (`sdk/python/`), the +> `/api/v1/sdk/session` endpoint, or the `experiment` scenario. + +Diagram: `.design/python-sdk.pencil.md` — two simple pencil graphs: the one +idea, and a request start to finish. + +## Why + +tb is a capable personal-intelligence gateway, but extending or experimenting +on top of it meant either editing the Go backend or hand-rolling HTTP calls +with the right base URL, token and scenario path. There was no fast seam for +"I have an idea, let me try it against my box in ten lines". + +`tingly` is that seam: a pip module where the user writes only their own logic +(prompt, retrieval, agent loop) and **reuses the gateway's power** — provider +routing, tier/fallback, guard rails, quota, logging — for free. + +## Scope (current milestone) + +The near-term target is deliberately narrow — connect, send a message, and +have a plugin work end-to-end including forwarding to another tb rule and +back. All three are done and verified live (`examples/e2e_run.sh`, real `tb` +binary, no mocks): + +1. `tingly.connect()` reaches tb and mints a session (Layer 1). +2. `tb.ask(...)` / `tb.anthropic.messages.create(...)` send a message through + tb's pipeline and get a real answer back (Layer 1). +3. A `tingly.Plugin` runs, tb routes a model to it, and the plugin's handler + calls `plugin.llm.ask(...)` / `plugin.use(scenario).ask(...)` to forward + into any other rule and get the result back before answering (Layer 2 + + Layer 3). + +**Protocol scope, deliberately narrowed:** Anthropic is primary everywhere in +the SDK; OpenAI chat completions is a real secondary path — kept, not +removed, but not what new work defaults to. Concretely: + +- `Client.ask()` tries the Anthropic transport first when a scenario supports + both (flipped from OpenAI-first — see "Two-token model" / Request flow). +- The plugin's own HTTP server answers `POST /v1/messages` (Anthropic, + primary) and `POST /v1/chat/completions` (OpenAI, secondary) — both real, + sharing one handler and one normalized `ChatRequest`; only the response + shaping differs per route. +- New plugins register with `api_style="anthropic"` by default + (`Plugin(api_style=...)` overrides it, per-plugin); the wire-level default + at `POST /api/v2/plugins` itself (a caller that omits the field entirely) + stays `"openai"`, for back-compat with anything hitting the endpoint + directly. + +Out of scope for now, unchanged from before: the tb-side plugin sub-process +supervisor, the `/plugins//*` reverse-proxy mount, and the lifecycle UI +(see Open follow-ups). None of the three milestone points above need them — +a plugin author starts their own process today, same as any local dev server. + +## Architecture (one idea, not three layers) + +There is a single concept: + +> **tb is a hub of rules. A rule's upstream can be a plugin. A plugin can +> originate calls against any other rule.** + +A client request matches a **rule** (as today). That rule's upstream is **plugin +code** instead of a provider — the only new thing. The plugin does its custom +work and, for any LLM work, calls **back into tb against any other rule / model / +provider** you have configured. tb stays the single router; a plugin is just a +graph node that happens to be user code and can also originate edges. + +``` + ┌──────────────────── tingly-box (the hub) ───────────────────┐ + clients │ │ + ┌─────────────┐ req │ rule A ──upstream──► PLUGIN CODE (your logic) │ + │ Claude Code │──────►│ (model=plugin/x) │ │ + │ Cursor │ │ │ calls back into tb: │ + │ tb UI │ │ rule B ◄──────────────────┤ use("…").ask(model="…") │ + │ tingly.ask()│ │ (→ Anthropic real) │ │ + └─────────────┘ │ rule C ◄──────────────────┤ (another model / provider) │ + │ rule D ◄──────────────────┘ (even another plugin) │ + │ │ │ + │ ▼ every edge gets: guard rails · routing/tiers · │ + │ failover · quota · logging │ + └──────────────────────────────────────────────────────────────┘ + │ + ▼ real upstreams (Anthropic / OpenAI / local …) +``` + +Everything else in this document is *how* that relationship is implemented with +today's pieces — three verbs for the one rule⇄plugin relationship: + +| verb | what it is | SDK surface | +|------|------------|-------------| +| **connect** | a plugin (or experiment) *consumes* a rule | `tingly.connect()` / `plugin.use(scenario).ask(model=…)` | +| **serve** | a plugin *is* a rule's upstream | `tingly.Plugin` (Anthropic-primary server) | +| **register**| point a rule's upstream at the plugin | `register_with_tb()` → tb provider + rule | + +The historical "Layer 1/2/3" headings below map exactly to connect / serve / +register. They are an implementation tour, not three separate products. + +### tb-side: a plugin is a normal, tagged provider (implemented) + +**Design history, briefly, because it's instructive.** Three earlier +iterations over-built this: first a persisted "plugin provider kind" with its +own DB column and a distinct registration endpoint; then a full ephemeral +service-discovery layer (in-memory registry, per-instance lease, heartbeat +thread, TTL expiry, a `Config` hook consulted on every provider lookup) built +to avoid leaving a stale DB row behind when a plugin process stopped; then, +even after that was cut down to an idempotent upsert, the handler methods +still lived directly on `*Server` in `internal/server/*.go`, coupling plugin +registration — a self-contained concern whose only dependency is `*config.Config` +— into the same file/struct as every other server concern. All three were +fixed. The circuit-breaker point: + +**tb already has liveness detection** — every `(rule, service)` +pair is covered by the existing per-service circuit breaker +(`internal/loadbalance/breaker.go`). A dead plugin's first failed request trips +it exactly like a dead real provider; traffic tier-fails-over automatically +when a fallback tier is configured. Lease/heartbeat/TTL was reinventing that +mechanism — distributed-service-discovery machinery for a problem tb doesn't +have (a personal, single-operator box, not a multi-tenant cluster). See +`git log` on this file's directory for the removed designs if useful as a +cautionary reference. + +**What shipped instead — the minimal version, in its own module:** + +- Lives in `internal/server/module/plugin/` (`handler.go` / `types.go` / + `routes.go`), matching the same pattern every other server concern already + uses (`module/provider`, `module/rule`, `module/providertemplate`, …): + a `Handler` struct constructed with only the dependencies it needs + (`NewHandler(cfg *config.Config)` — nothing else, since registration is just + provider + rule creation), and a `RegisterRoutes(group, handler)` mounted + from `server_webui_api.go`. Plugin logic no longer lives as methods on the + giant `*Server` struct. +- A plugin is an ordinary provider (`APIStyle=openai|anthropic`, `api_key`/ + `no_key`) carrying the tag `"plugin"` in the existing, generic + `Provider.Tags` field. `Provider.IsPlugin()` checks for that tag. No new + struct, no new DB column — Tags already round-trips through the provider + store unconditionally. +- **`POST /api/v2/plugins`** is an **idempotent upsert by name**: register once + at startup (and again on every restart) and it updates the same provider + instead of duplicating it. When `scenario` is given it also idempotently + ensures the rule. Request carries `api_style` (`openai`|`anthropic`, empty + → `openai` at the wire level; the SDK's own default is `anthropic`, see + Scope above) — this is what tells tb which of the plugin's two routes to + call. Response: `{provider_uuid, model_id, scenario, rule_uuid, ready, note}`. +- **A real, non-obvious fix underneath this:** `Provider.GetAccessToken()` + returned `""` for a no-key provider, and the vendored `anthropic-sdk-go` + treats an empty API key as "go look for ambient credentials" — it does its + own discovery (env vars, `anthropic auth login` profile, …) and errors + loudly when none exist, instead of just sending an empty/absent header the + way the OpenAI client does. This was invisible while every plugin was + `APIStyle=openai`; it surfaced immediately once Anthropic became the + default and broke the very first live end-to-end run. Fixed by + `ai.NoKeySentinelToken` (`ai/provider.go`): when `AuthType=api_key`, + `Token==""` and `NoKeyRequired=true`, `GetAccessToken()` returns that + sentinel instead of `""` — a real (if meaningless) value the SDK is happy + to send as the header, which the plugin's own auth check (`api_key=""` → + accept anything) ignores. General fix, not plugin-specific: any + no-key-required Anthropic-style provider benefits. +- **`GET /api/v2/plugins`** lists plugin-tagged providers, deriving the display + model id from the rule(s) bound to each (no extra field needed). +- **Retiring a plugin** is the same as retiring any other provider: delete it + in the tb UI. There is no separate lifecycle to reason about. + +**Active configuration** (SDK): `tingly.configure(url=, admin_token_env=)` / +`Connection` inject the tb target + credentials at runtime (secrets by env +reference), top-precedence in `config.resolve()` — for containers / CI / remote +where there is no `~/.tingly-box`. This part was cheap and answers a real need, +so it stayed. `Plugin.serve(register=True, scenario=…, tb=Connection(...))` +registers once at startup — no background thread, no lease to manage. + +Verified end-to-end (`examples/e2e_run.sh`, real `tb` binary, no network/keys): +the plugin registers once as an `api_style=anthropic` provider; a client's +OpenAI-shaped `chat/completions` call to `model=plugin/rag-demo` routes +through tb, which forwards it to the plugin as `POST /v1/messages?beta=true` +(tb's real Anthropic client — the `?beta=true` query string is why the +server routes on path only, ignoring the query); the plugin's handler calls +`plugin.use("experiment").ask(..., model="echo-model")`, itself now an +Anthropic-transport call, which tb routes to a `vmodel` provider and back; +the composed answer returns to the plugin, which tb reshapes back to +OpenAI `chat.completion` for the original caller. Killing the plugin leaves +its provider listed (same as any provider) and the next request fails with a +plain connection error (add a tier-1 fallback to see failover instead); +restarting the plugin upserts the same provider, no duplicate. + + +## Shape + +``` +sdk/python/ + tingly/ + client.py # Layer 1: Client + connect() ← consume tb + discovery.py # probe gateway + POST /sdk/session + config.py # (base_url, admin_token) resolution precedence + scenarios.py # scenario + transport constants + transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb + helpers/ # usage + guardrails views + plugin/ # Layer 2: be an AI server tb routes to + core.py # Plugin class (@plugin.chat, .llm, .serve, api_style) + server.py # stdlib HTTP server: /v1/messages (primary) + /v1/chat/completions (secondary), + SSE + types.py # ChatRequest / Message (from_anthropic_body / from_openai_body) + manifest.py # tingly.toml read/write + register.py # one-shot, idempotent register with tb + cli.py # `tingly doctor` + `tingly plugin {init,run}` + errors.py # TinglyError hierarchy +``` + +## Request flow + +``` +connect(scenario="experiment") + │ + ├─ config.resolve() args → env → ~/.tingly-box/sdk.json → config.json → localhost + ├─ discovery.probe_version() GET /api/v1/info/version (liveness) + ├─ discovery.create_session() POST /api/v1/sdk/session (admin token → model token) + └─ Client(session, gateway_url, admin_token) + .openai → openai.OpenAI(base_url = scenario_root + "/v1") + .anthropic → anthropic.Anthropic(base_url = scenario_root) + .ask() → Anthropic first when the scenario supports both, else OpenAI + .usage → GET /api/v1/requests (admin token) + .guardrails → GET /api/v1/guardrails/config (admin token) +``` + +## How it works (pencil) + +Two phases. **Provisioning** happens once in `connect()` (admin token, dashed +lines). **Inference** happens on every call (model token, solid lines) and +reuses the exact same gateway pipeline as any other tb client — the SDK adds no +new path through the box. + +``` + YOUR PYTHON tingly-box GATEWAY UPSTREAMS + ┌───────────────────────┐ ┌──────────────────────────────────┐ ┌───────────────┐ + │ import tingly │ │ │ │ Anthropic │ + │ tb = tingly.connect() │ │ /api/v1/... (admin auth) │ │ OpenAI │ + │ │ │ /tingly/:scn (model auth) │ │ Deepseek │ + └───────────┬───────────┘ └──────────────────────────────────┘ │ vLLM / local │ + │ └───────▲───────┘ + ── PROVISION (once, admin token) ─────────────────────────────────────────────────┊──────── + │ ┊ + config.resolve() ┊ + args→env→sdk.json→config.json→localhost ┊ + │ ┊ + │ GET /api/v1/info/version (liveness) ┄┄┄┄┄┄┄┄►┐ ┊ + │ POST /api/v1/sdk/session {scenario,name} ┄┄┄►│ CreateSDKSession ┊ + │ Authorization: Bearer │ · validate scenario in registry + │ │ · transport = openai|anthropic|both + │ ◄┄┄┄ {base_url, token=, │ · ready/services from active rule + │ transport, ready, services} ┄┄┄┄┄┄┄┄┄┄┄┄┄┘ + ▼ + Client ── builds lazily ──► openai.OpenAI(base_url = root+"/v1", api_key=ModelToken) + anthropic.Anthropic(base_url = root, api_key=ModelToken) + + ══ INFERENCE (every call, model token) ═══════════════════════════════════════════════════ + │ + tb.ask("...", model="auto") + tb.openai.chat.completions.create(...) + tb.anthropic.messages.create(...) + │ POST /tingly/experiment/v1/chat/completions ┌─────────────────────────┐ + │ POST /tingly/experiment/v1/messages │ the SAME pipeline as │ + │ Authorization: Bearer ─────────►│ any other tb client │ + │ │ │ + │ │ scenario → rule resolve │ + │ │ guard rails (in/out) │ + │ │ smart routing / tiers │ + │ │ circuit-breaker failover│──► pick + │ │ quota + usage logging │ upstream + │ │ protocol transform │──────────► + │ ◄──────────── response (+ usage recorded) ──────└─────────────────────────┘ (solid) + ▼ + tb.usage.this_session() GET /api/v1/requests (admin token, read-back) + tb.guardrails.status() GET /api/v1/guardrails/config (admin token, read-back) +``` + +Key reading of the graph: + +- The SDK never talks to providers directly — the rightmost column is reachable + **only** through the gateway box in the middle. That is the whole point: the + experiment inherits routing/fallback/guard-rails/quota for free. +- Provisioning (dashed) uses the **admin** token and the `/api/v1/*` control + plane; inference (solid) uses the **model** token and the `/tingly/:scenario` + data plane. Different tokens, different surfaces. +- The inference box is *unchanged* tb internals — the SDK contributes the new + `experiment` scenario and the one provisioning endpoint, nothing in the hot + path. + +## Two-token model + +- **Admin token** (tb's `UserToken`): authorizes `POST /sdk/session`. Resolved + from `TINGLY_BOX_TOKEN` / `sdk.json` / `config.json:UserToken`. Provisioning + requires admin rights. +- **Model token** (tb's `ModelToken`): returned *by* the session, used as the + bearer for the actual LLM calls. The OpenAI/Anthropic clients carry this, not + the admin token. + +In v0.1 the session returns the existing long-lived model token (same as +`tbclient.GetConnectionConfig` / `GetClaudeCodeEnv` already do). Short-lived +scoped tokens (`expires_at`) are the obvious follow-up — the response field is +already present and `omitempty`. + +## Gateway seam: `POST /api/v1/sdk/session` + +Handler: `internal/server/sdk_session.go` (`CreateSDKSession`), registered in +`webui_api.go` under the authenticated `apiV1` group (so it needs the admin +token). + +Request `{ scenario, name }` → response +`{ base_url, token, scenario, transport, ready, services, expires_at? }`. + +- `base_url` is the scenario root `http://host:port/tingly/`. Bind + host `0.0.0.0`/`::` is rewritten to `127.0.0.1` so it's client-usable. +- `transport` is `openai`|`anthropic`|`both`, collapsed from the scenario + descriptor's `SupportedTransport`. +- `ready`/`services` report whether an active rule with ≥1 service is bound, so + `tingly doctor` can tell the user the next action instead of failing opaquely. +- Unknown / non-bindable scenario → 404 with `valid_scenarios` in the body. + +No new routes were needed for the LLM calls themselves: `/tingly/:scenario` and +`/tingly/:scenario/v1` are already dynamic, so `experiment` flows through the +existing mixin endpoints (`chat/completions`, `messages`, `responses`, …). + +## The `experiment` scenario + +Added to `internal/typ/type.go` (`ScenarioExperiment = "experiment"`) and the +descriptor registry (`scenario_registry.go`): OpenAI + Anthropic transports, +rule-bindable, path-usable, profile-capable. It exists so SDK traffic has its +own isolated rule instead of polluting `claude_code` / `openai` rules — and so +users can name parallel experiments via profiles (`experiment:p1`). + +## UX-principles alignment + +- **No mode picker.** `connect()` is identical in dev and (future) hosted + contexts; the environment decides discovery, not the user. +- **Smart defaults.** `scenario="experiment"`, `model="auto"`. +- **Concrete values.** `usage.this_session()` returns token counts, not aliases. +- **Diagnostics traverse the real path.** `tingly doctor` runs the actual + discover → session → live round-trip; green = user code will run. +- **Surface the artifact for the next action.** `ready=false` and + `GuardrailBlockedError(policy_id, reason)` tell the user exactly what to fix. + +## Testing + +- Python: `sdk/python/tests/` — config precedence, discovery/session (respx + mocked gateway), transport URL shaping, client transport routing. Integration + tests that need a live tb are marked `@needs_tb` and skipped by default. +- Go: `internal/server/sdk_session_test.go` freezes the response JSON field + names (contract with the SDK) and the transport-label logic; + `internal/typ/scenario_registry_test.go` pins the experiment descriptor. + +## ⚠️ Naming collision to resolve before the lifecycle UI + +The frontend already has a deliberately-unified name **"Plugins"** for a +completely different concept — per-rule feature flags (`smart_compact`, +`vision_proxy_service`, `clean_header`, `session_affinity`, …), surfaced via +`RulePluginsCard` / `FlagCatalogDialog` / `PluginFeatures` (see +`.design/rule-flags.md` §"统一命名:Plugins"). That unification itself +resolved an earlier "Plugin" / "Rule Extensions" mixed-usage collision — this +exact kind of debt has already been paid down once in this codebase. + +The SDK's `tingly.Plugin` / `POST /api/v2/plugins` / the `"plugin"` provider +tag names an unrelated concept — *external code acting as an upstream* — but +reuses the same word. Per `.design/ux-principles.md` §3 ("一个词在产品中只能指 +一件事"), this has to be split before it becomes user-visible. It's silent +today only because there's no lifecycle UI yet (see follow-up 4 below) — the +moment that ships, both meanings of "Plugin" appear in the same product, on +adjacent surfaces of the same rule (its flags card vs. its upstream +binding). Rename one side **now**, while the surface is still an API + +Python class and not yet UI copy — cheaper than renaming after users have +`tingly.toml` manifests and muscle memory. This SDK side is the newer, +smaller-footprint concept, so it's the one that should move; candidates: +`upstream plugin`, `connector`, `extension provider`. Needs a product +decision, not a unilateral rename — flagged here rather than acted on. + +## Open follow-ups + +The Scope milestone above (connect + send + plugin round-trip, Anthropic +primary / OpenAI secondary) is **done**. What's left, roughly in priority +order — 1–3 are backend/SDK-only (no naming exposure, safe to build +regardless of the rename above); 4 is blocked on that decision for its UI +portion specifically (the supervisor + reverse-proxy mount are not): + +1. Layer 2 tb-side remainder — Python side is **done** (`tingly.Plugin`, + manifest, dual-protocol server, `register`); still missing: a sub-process + supervisor that boots plugins from their manifest (reuse + `agentboot/process`), a `/plugins//*` reverse-proxy mount, and the + install/enable/logs/disable lifecycle UI. The first two are ordinary + backend work; the UI is the piece that should wait on the naming decision + above. See the "Layer 2" section below. +2. Scoped short-lived session tokens (`expires_at` + refresh on 401). +3. Dedicated `GET /api/v1/sdk/usage?session=` so usage doesn't scan + `/api/v1/requests`. +4. Async client (`AsyncClient`, `aask`) — transports already have async builders. + +Layer 3 (expose a plugin as a model tb can route to) needed no new work of its +own — it's already fully supported as provider-as-upstream (see "Layer 3" +below) — so it's not listed as a follow-up. + +## Layer 2: write an AI server (`tingly.Plugin`) + +A plugin is an upstream tb can call two ways — **Anthropic Messages +(primary)** and **OpenAI chat completions (secondary)**, both real, both +always served regardless of what registration advertises. The author writes +one chat handler, and `serve()` runs the HTTP server. The whole surface is +one class. + +```python +from tingly import Plugin + +plugin = Plugin(name="my-rag") # model_id defaults to "plugin/my-rag" + +@plugin.chat +def handle(req): # req: ChatRequest + docs = retrieve(req.last_user_text()) + return plugin.llm.ask( # ← calls BACK into tb (Layer 1) + f"Using {docs}, answer: {req.last_user_text()}", model="auto" + ) + +if __name__ == "__main__": + plugin.serve() # http://127.0.0.1:8765/v1 +``` + +### How it works (pencil) + +Two things to read here: the **anatomy** of a plugin (left), and the **request +lifecycle** when tb routes a model to it (the numbered loop). Note the loop is a +cycle — the plugin's handler calls back *into* tb (steps 4–6), so tb is both the +caller (step 3) and the upstream-for-the-plugin (step 5). + +``` + A PLUGIN (one Python process) tingly-box GATEWAY + ┌───────────────────────────────────┐ ┌──────────────────────────────┐ + │ Plugin(name="my-rag") │ │ │ + │ │ │ provider: │ + │ @plugin.chat │ │ name=my-rag │ + │ def handle(req): ... │ │ api_base=http://…:8765/v1 │ + │ │ │ │ api_style=anthropic (dflt) │ + │ │ returns str | iter[str] │ │ model=plugin/my-rag │ + │ ▼ │ │ │ + │ serve() → stdlib HTTP server │ │ rule: plugin/my-rag → ↑ │ + │ POST /v1/messages ◄────┼─ (3) POST /v1/messages?beta=true ──┘ ▲ │ + │ (primary, api_style match) │ (model=plugin/my-rag) │ │ + │ POST /v1/chat/completions ◄────┼─ (3') POST /v1/chat/completions ─────┘ │ + │ (secondary, if api_style=openai) (6) answer │ + │ GET /v1/models │ │ + │ GET /health │ │ + │ · buffered → message / chat.completion │ + │ · stream → SSE (message_* events / chat.completion.chunk) ── (7) ───┘ + │ │ │ + │ plugin.llm (lazy Layer-1 client)│ + │ │ │ + └────────┼──────────────────────────┘ + │ (4) plugin.llm.ask("…", model="auto") + │ = tingly.connect(scenario="experiment") → POST /tingly/experiment/v1/messages + ▼ + ┌──────────────────────────────────────────────────────────────┐ + │ tingly-box pipeline (SAME as any client — see Layer 1 graph) │ + │ scenario→rule · guard rails · routing/tiers · failover · │ + │ quota · logging · transform ─────────────────────────► (5) real upstream + └──────────────────────────────────────────────────────────────┘ (Anthropic/ + OpenAI/…) + + request lifecycle: + (1) client sends model="plugin/my-rag" to tb, any protocol ── see Layer 3 graph + (2) tb resolves rule → provider my-rag (api_base = plugin, api_style picks the route) + (3) tb calls the PLUGIN on whichever route matches provider.api_style + (Anthropic /v1/messages by default; /v1/chat/completions if registered openai) + (4) handler runs; calls plugin.llm.ask(...) ── back INTO tb, Anthropic-first + (5) tb routes that call to a real upstream, applies guard rails/quota/… + (6) generated text returns to the handler + (7) handler's str/iterator → response/SSE shaped for whichever route was hit → back to tb → back to client +``` + +Key reading: + +- **One process, two roles.** As a *server* the plugin answers tb on + `:8765/v1`; as a *client* (`plugin.llm`) it consumes tb via Layer 1. Same + gateway, both directions. +- **The author writes only step 4's body.** Everything else — wire parsing, + response/SSE shaping (steps 3 & 7), discovery/session (step 4's connect), + routing/guard-rails (step 5) — is the SDK and the gateway. +- **Guard rails apply twice, correctly:** once on the inbound call to the + plugin (step 3, via the provider/rule), and again on the plugin's own LLM call + (step 5). Neither is wired by the author. + +Design choices: + +- **No framework dependency.** The server is `http.server.ThreadingHTTPServer` + (stdlib), so a plugin is one `pip install tingly` away. It always serves + both `POST /v1/messages` (Anthropic, buffered **and** real SSE) and + `POST /v1/chat/completions` (OpenAI, same), plus `GET /v1/models`, + `GET /health` — which route tb actually uses is a registration choice + (`api_style`), not a server capability limit. +- **Handler contract is minimal and protocol-agnostic.** Return a `str` + (buffered) or an iterator of `str` (streamed); the server shapes it into + `message`/SSE `message_*` events for the Anthropic route or + `chat.completion`/`chat.completion.chunk` for the OpenAI route, whichever + was hit. The author never touches wire format either way. + `ChatRequest.from_anthropic_body` folds Anthropic's top-level `system` + field into a leading `role="system"` message so `req.system_text()` / + `req.last_user_text()` work the same regardless of which route the caller + used. +- **`plugin.llm` is a lazy Layer-1 client.** The plugin reuses the gateway for + its own generation instead of hard-coding a provider/key — the recursion in + the Layer 3 graph. Its own `ask()` calls try Anthropic first (see Scope). +- **`tingly.toml` manifest** (`manifest.py`) declares name / model_id / + entrypoint / transport (`anthropic` by default, tracks `Plugin.api_style`) / + port, so a future tb-side supervisor can install and run the plugin. + `tingly plugin init` scaffolds a module + manifest. +- **Optional token auth.** `Plugin(api_key=...)` enforces a bearer token so only + tb (carrying the matching provider token) can call it — checked once, + ahead of both routes. + +CLI: + +``` +tingly plugin init my-rag # scaffold my_rag_plugin.py + tingly.toml +tingly plugin run my_rag_plugin.py # serve it AND register with tb +``` + +`run` (via `Plugin.serve()`) registers with `POST /api/v2/plugins` on startup — +an idempotent upsert by name that creates/updates the provider *and* the rule +(when `scenario` is set on the constructor) in one call. There is no separate +`register` command: a one-shot register with nothing keeping it alive would be +meaningless once ephemeral lifecycle was cut (see the "tb-side" section above). + +**Not yet built (tb-side):** a sub-process supervisor that boots plugins from +their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy +mount, and the install/enable/logs/disable lifecycle UI. The Python side and the +provider wiring are complete; those are the remaining backend pieces. + +### Example plugins (`sdk/python/examples/`) + +Three, each a different real-world shape of "plugin composes the box by +calling back into other rules" — not toys picked at random, each maps onto a +pattern already in wide use: + +- **`rag_plugin.py`** — one call back into tb for generation over retrieved + context. The baseline shape. +- **`critic_plugin.py`** (`model="plugin/critic"`) — cross-model critique: + forwards the artifact-to-review to a *different* rule/model, returns a + structured `{verdict, issues, suggestion}`. Chosen over self-critique + deliberately: Huang et al. (ICLR 2024) found LLMs can't reliably + self-correct without external feedback, so a plugin reviewing with a + different model is the robust variant, not a stylistic choice. This is the + pattern behind [Zen MCP](https://github.com/jray2123/zen-mcp-server) and + [Consult7](https://github.com/szeider/consult7) (both real MCP servers + coding agents use today to consult another model mid-task) and behind + aider's architect/editor split. Named "critic" deliberately, not "advisor" + — tb already has an unrelated, in-process `advisor` MCP tool + (`internal/mcp/runtime/advisor_virtual.go` + the response-hook machinery in + `internal/server/servertool/`); reusing that name for an architecturally + different thing (plugin-as-upstream calling back into the gateway, vs. a + direct in-process upstream call) would be the same collision already + flagged above for "Plugin" vs. rule-flag "Plugins" — same fix, applied + before it started rather than after. +- **`fusion_plugin.py`** (`model="plugin/fusion"`) — multi-model consensus: + polls a panel of rules/models concurrently (`ThreadPoolExecutor`), skips + the judge call when the panel already agrees, otherwise a judge call + synthesizes. Mirrors Consult7's 2026 Fusion feature (a panel of frontier + models answers in parallel, a judge model merges). The clearest + illustration of the architecture line at the top of this document — a + plugin can freely originate calls against *any* number of other rules, not + just one. + +Every example plugin has unit tests (`tests/test_example_plugins.py`) that +monkeypatch `plugin.use` to a fake client and pin the decision logic — +JSON-verdict formatting and graceful degradation on non-JSON (critic); +judge-skipped-on-agreement vs. judge-called-on-disagreement (fusion) — +without needing a live tb. + +Deliberately not built (yet): quota-aware / deterministic-dispatch routing +(picking a candidate rule by remaining quota, forcing a specific service +within a rule). It's a real, well-scoped follow-up — not out of scope +forever, just not needed for the current milestone above, and speculative +infrastructure built ahead of a real consumer tends to become exactly the +kind of thing that needs redesigning once an actual use shows up. Revisit +if/when a concrete need for it appears. + +## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) + +Layer 1 points the **data-flow into** tb: the plugin is a *consumer*. For tb to +*select* a plugin as a model, the flow inverts — the plugin becomes a +*producer*, an HTTP upstream tb calls out to. That is Layer 2's `Plugin.serve()`. + +There are two distinct "virtual model" notions; only one fits an out-of-process +Python plugin: + +| | in-process `vmodel` (`AuthType=virtual`) | provider-as-upstream | +|---|---|---| +| what | Go code implementing `openai.VirtualModel` / `anthropic.VirtualModel`, compiled in (`ai/provider.go:IsVirtual` → `virtualModelService`) | a normal provider whose `api_base` is an external HTTP server speaking `/v1/chat/completions` or `/v1/messages` | +| lives | inside tb's process | out-of-process, any language | +| Python plugin fit | ✗ (needs a Go shim forwarding to Python) | ✓ natural route | + +So a Python plugin is selected by registering it as a **provider/upstream**, not +via the in-process `vmodel` package. + +``` + ANY tb client tingly-box GATEWAY UPSTREAMS + ┌──────────────┐ ┌──────────────────────────────┐ + │ Claude Code │ │ HandleOpenAIChatCompletions │ tier 1 ┌──────────────┐ + │ Cursor │ model │ scenario → rule resolve │ ┌───────► │ Anthropic / │ + │ tb UI ├───────►│ guard rails (in/out) │ │ fallback│ OpenAI (real)│ + │ tingly.ask() │ "plugin│ smart routing / TIERS ──────┼──┤ └──────────────┘ + └──────────────┘ /my-rag"│ circuit-breaker failover │ │ tier 0 ┌──────────────┐ + │ quota + usage logging │ └───────► │ my-rag PLUGIN│ ◄─ Layer 2 + │ provider.api_base = plugin │ POST │ POST /v1/ │ Plugin.serve() + │ provider.api_style picks route│ /v1/msgs │ messages │ + └────────────────────────────────┘ (dflt) └──────┬───────┘ + ctx.llm.ask() ┄┄┄┘ (plugin may + back INTO tb for its own LLM calls) +``` + +Wiring (no new gateway hot-path code — it's just a provider): + +1. **Plugin serves** both `POST /v1/messages` and `POST /v1/chat/completions` + (Layer 2 `Plugin.serve()`). +2. **Register**: `POST /api/v2/plugins {name:"my-rag", endpoint:"http://127.0.0.1:/v1", + model_id:"plugin/my-rag", scenario:"experiment", api_style:"anthropic"}` + creates a *normal* provider (not `AuthType=virtual`, tagged `"plugin"`, + `APIStyle` set from `api_style`) — this is exactly what `Plugin.serve()` + does on startup, with `api_style` defaulting to `"anthropic"`. +3. That same call **binds the rule/service**: model `plugin/my-rag` → that provider. +4. Now `model:"plugin/my-rag"` from any client resolves through the same + dispatcher as every other model. Put the plugin in tier 0 and a real model in + tier 1 and tb fails over automatically when the plugin is down. + +The deeper option — a true in-process `AuthType=virtual` vmodel — means writing +a small Go adapter implementing `openai.VirtualModel` that forwards to the Python +process. Only worth it to bundle the plugin with no separate port; +provider-as-upstream is simpler and already fully supported. diff --git a/.design/python-sdk.pencil.md b/.design/python-sdk.pencil.md new file mode 100644 index 000000000..19f856235 --- /dev/null +++ b/.design/python-sdk.pencil.md @@ -0,0 +1,51 @@ +# Python SDK (`tingly`) — Pencil Graph + +Visual companion to `python-sdk.md`. Two pictures. For exact endpoints / +field names / file:line references, that doc is the source of truth — this +page is just the shape of things. + +Contents: + +- The one idea +- A request, start to finish + +## The one idea + +``` + client tingly-box real upstream + ┌────────┐ model=x ┌──────────────────────┐ + │ any app │────────────►│ rule x → PLUGIN CODE │ + └────────┘ │ │ │ + │ │ calls back:│ + │ rule y ◄──┘ use(y) │ + └─────┬──────────────────┘ + ▼ + Anthropic / OpenAI / local … +``` + +A plugin is just a rule whose upstream happens to be your code. It can call +*back* into any other rule to get its own answer — same gateway, same guard +rails / quota / logging, both directions. + +## A request, start to finish + +``` + 1. connect() admin token ──► mint a session ──► model token + + 2. tb.ask("...") model token ──► tb picks a rule ──► picks a service + │ + ▼ + real model answers + + 3. if that model IS a plugin: + tb calls the plugin instead of a real model (step 2, inbound) + the plugin's handler does step 2 AGAIN, on its own, to get ITS answer + the plugin's answer becomes tb's answer to the original caller +``` + +Steps 1 and 2 are all of Layer 1 (`Client`). Step 3 is Layer 2 (`Plugin`) — +same request, plugin just sits in the middle and calls back once. +`critic_plugin.py` (ask a different model to review), `fusion_plugin.py` +(ask several, then a judge), and `rag_plugin.py` (ask one, with retrieved +context) are all step 3 with different logic in the handler — no new +mechanism. diff --git a/ai/provider.go b/ai/provider.go index 26a576940..3fd7fac74 100644 --- a/ai/provider.go +++ b/ai/provider.go @@ -53,6 +53,14 @@ type VModelDetail struct { LatencyProfile string `json:"latency_profile,omitempty"` } +// PluginTag is the Provider.Tags value that marks a provider as backed by +// external plugin code. A plugin provider is otherwise an ordinary OpenAI HTTP +// upstream (APIStyle=openai, api_key / no_key) — there is NO routing change; +// reusing the existing generic Tags field means plugin identity needs no new +// persisted column. Distinct from AuthTypeVirtual (the in-process vmodel +// path): a plugin runs out-of-process and is reached over HTTP. +const PluginTag = "plugin" + // CredentialBundle holds the credential fields for multi-field auth types // (AWS SigV4, Azure, GCP Vertex). Fields is a generic, schema-validated // key/value map so new credential shapes can be added as data rather than new @@ -253,6 +261,21 @@ func (p *Provider) IsVirtual() bool { return p != nil && p.AuthType == AuthTypeVirtual } +// IsPlugin reports whether this provider is backed by external plugin code +// (carries the PluginTag). Plugin providers route as ordinary OpenAI HTTP +// upstreams; this is metadata for UI grouping only. +func (p *Provider) IsPlugin() bool { + if p == nil { + return false + } + for _, tag := range p.Tags { + if tag == PluginTag { + return true + } + } + return false +} + // IsBuiltin reports whether this provider was seeded by the system and is // therefore protected from deletion/mutation. func (p *Provider) IsBuiltin() bool { @@ -337,6 +360,16 @@ func (p *Provider) ResolveEndpoint(clientStyle APIStyle) (string, APIStyle) { // value is never transmitted. const VModelSentinelToken = "EMPTY" +// NoKeySentinelToken satisfies the same non-empty-APIKey check for real +// outbound HTTP providers that genuinely take no key (NoKeyRequired=true; +// e.g. a local plugin process). Unlike VModelSentinelToken this value IS +// transmitted, as an Authorization/x-api-key header the receiving side is +// expected to ignore. It exists because some client SDKs (anthropic-sdk-go) +// treat an empty API key as "look for ambient credentials" and fail loudly +// when none are found, instead of just sending an empty/absent header the +// way the OpenAI client does. +const NoKeySentinelToken = "tingly-no-key" + // GetAccessToken returns the access token based on auth type func (p *Provider) GetAccessToken() string { switch p.AuthType { @@ -348,6 +381,9 @@ func (p *Provider) GetAccessToken() string { return VModelSentinelToken case AuthTypeAPIKey, "": // Default to api_key for backward compatibility + if p.Token == "" && p.NoKeyRequired { + return NoKeySentinelToken + } return p.Token } return "" diff --git a/ai/provider_test.go b/ai/provider_test.go index 97f4e8128..6f9b9f34f 100644 --- a/ai/provider_test.go +++ b/ai/provider_test.go @@ -373,6 +373,24 @@ func TestProvider_GetAccessToken(t *testing.T) { }, want: "", }, + { + name: "API key auth, no key required, empty token -> sentinel", + provider: &Provider{ + AuthType: AuthTypeAPIKey, + NoKeyRequired: true, + Token: "", + }, + want: NoKeySentinelToken, + }, + { + name: "API key auth, no key required, but a real token is still preferred", + provider: &Provider{ + AuthType: AuthTypeAPIKey, + NoKeyRequired: true, + Token: "sk-real", + }, + want: "sk-real", + }, } for _, tt := range tests { diff --git a/ai/quota/manager.go b/ai/quota/manager.go index 16bca2cea..85adc4a24 100644 --- a/ai/quota/manager.go +++ b/ai/quota/manager.go @@ -127,12 +127,15 @@ func (m *Manager) RefreshProvider(ctx context.Context, providerUUID string) (*Pr } // GetQuota returns cached quota data and refreshes it when expired. +// +// A not-found store lookup returns ErrUsageNotFound UNWRAPPED — callers +// (e.g. the provider-quota HTTP handlers) compare against that sentinel +// with == to treat "no data yet" as a skip rather than an error; wrapping it +// here (as this used to do, via fmt.Errorf) silently broke that comparison +// and turned every "no quota data" provider into a hard error upstream. func (m *Manager) GetQuota(ctx context.Context, providerUUID string) (*ProviderUsage, error) { usage, err := m.store.Get(ctx, providerUUID) if err != nil { - if err == ErrUsageNotFound { - return nil, fmt.Errorf("quota not found for provider: %s", providerUUID) - } return nil, err } @@ -146,15 +149,9 @@ func (m *Manager) GetQuota(ctx context.Context, providerUUID string) (*ProviderU } // GetQuotaNoCache returns the latest quota data stored in the database. +// See GetQuota above for why ErrUsageNotFound must reach the caller unwrapped. func (m *Manager) GetQuotaNoCache(ctx context.Context, providerUUID string) (*ProviderUsage, error) { - usage, err := m.store.Get(ctx, providerUUID) - if err != nil { - if err == ErrUsageNotFound { - return nil, fmt.Errorf("quota not found for provider: %s", providerUUID) - } - return nil, err - } - return usage, nil + return m.store.Get(ctx, providerUUID) } // ListQuota returns quota data for all providers. diff --git a/ai/quota/manager_test.go b/ai/quota/manager_test.go index 4b72fa6ea..4b6e52f36 100644 --- a/ai/quota/manager_test.go +++ b/ai/quota/manager_test.go @@ -57,6 +57,25 @@ func (f *concurrencyTestFetcher) Fetch(_ context.Context, provider *typ.Provider return &ProviderUsage{ProviderUUID: provider.UUID}, nil } +// TestGetQuota_NotFoundIsUnwrapped locks in that GetQuota (and +// GetQuotaNoCache) return ErrUsageNotFound identically to what the store +// returned — not a re-wrapped error. Callers such as the provider-quota +// batch handler compare with == against this sentinel to skip providers +// with no quota data instead of failing the whole request; a wrapped error +// silently breaks that comparison (this was a real bug: BatchGetQuota 500'd +// for any provider with no quota data, e.g. a vmodel/local provider, instead +// of just omitting it from the result). +func TestGetQuota_NotFoundIsUnwrapped(t *testing.T) { + manager := NewManager(DefaultConfig(), managerTestStore{}, managerTestProviderManager{}, logrus.New()) + + if _, err := manager.GetQuota(context.Background(), "missing"); err != ErrUsageNotFound { + t.Fatalf("GetQuota() error = %v, want ErrUsageNotFound (identity, via ==)", err) + } + if _, err := manager.GetQuotaNoCache(context.Background(), "missing"); err != ErrUsageNotFound { + t.Fatalf("GetQuotaNoCache() error = %v, want ErrUsageNotFound (identity, via ==)", err) + } +} + func TestRefreshBoundsConcurrency(t *testing.T) { providers := make([]*typ.Provider, 20) for i := range providers { diff --git a/internal/server/config/provider.go b/internal/server/config/provider.go index 4d28c719e..a25c56cf3 100644 --- a/internal/server/config/provider.go +++ b/internal/server/config/provider.go @@ -110,7 +110,6 @@ func (c *Config) GetProviderByUUID(uuid string) (*typ.Provider, error) { if c.providerStore == nil { return nil, fmt.Errorf("provider store not initialized") } - provider, err := c.providerStore.GetByUUID(uuid) if err != nil { return nil, fmt.Errorf("provider '%s' not found: %w", uuid, err) diff --git a/internal/server/module/plugin/handler.go b/internal/server/module/plugin/handler.go new file mode 100644 index 000000000..36b99bd1c --- /dev/null +++ b/internal/server/module/plugin/handler.go @@ -0,0 +1,207 @@ +// Package plugin handles HTTP endpoints for registering external plugin code +// as a tingly-box upstream. A plugin is an ordinary OpenAI-compatible HTTP +// provider tagged "plugin" (see typ.Provider.IsPlugin) — routing is +// unchanged, and liveness is handled by the same per-service circuit breaker +// that already protects every other provider. There is deliberately no +// separate plugin lifecycle (lease/heartbeat/expiry): that would duplicate +// the breaker for a single-operator box. If a plugin is retired, delete its +// provider like any other, via the provider module's DELETE endpoint. +package plugin + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/tingly-dev/tingly-box/ai" + "github.com/tingly-dev/tingly-box/internal/constant" + "github.com/tingly-dev/tingly-box/internal/loadbalance" + "github.com/tingly-dev/tingly-box/internal/server/config" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// Handler handles plugin registration HTTP requests. Its only dependency is +// the shared config — plugin registration is just provider + rule creation, +// so it needs nothing else from the server. +type Handler struct { + config *config.Config +} + +// NewHandler creates a plugin Handler. +func NewHandler(cfg *config.Config) *Handler { + return &Handler{config: cfg} +} + +// RegisterPlugin creates or updates a plugin-tagged provider (and optionally +// binds a rule to it) so "configure this rule with a plugin" is one call. +func (h *Handler) RegisterPlugin(c *gin.Context) { + var req RegisterPluginRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + + modelID := req.ModelID + if modelID == "" { + modelID = "plugin/" + req.Name + } + + apiStyle, err := normalizeAPIStyle(req.APIStyle) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + + provider, err := h.upsertPluginProvider(req.Name, req.Endpoint, req.Token, apiStyle) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": "failed to register plugin provider: " + err.Error(), + }) + return + } + + resp := RegisterPluginResponse{ + ProviderUUID: provider.UUID, + ModelID: modelID, + Note: "Provider registered. Bind a rule (pass `scenario`) to make the model selectable.", + } + + // One-step bind: ensure the rule whose single service is this plugin. + if req.Scenario != "" { + ruleUUID, err := h.ensurePluginRule(req.Scenario, modelID, provider.UUID, req.Name, req.Tier) + if err != nil { + resp.Note = "Provider registered, but rule binding failed: " + err.Error() + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) + return + } + resp.Scenario = req.Scenario + resp.RuleUUID = ruleUUID + resp.Ready = true + resp.Note = "Plugin wired in. Select model " + modelID + " under scenario " + req.Scenario + "." + } + + logrus.WithFields(logrus.Fields{ + "plugin": req.Name, + "endpoint": req.Endpoint, + "model_id": modelID, + "scenario": req.Scenario, + "ready": resp.Ready, + }).Info("Registered plugin provider") + + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) +} + +// normalizeAPIStyle validates the caller-supplied wire protocol, defaulting +// the empty string to "openai" (the style tb assumed before plugins could +// declare one). Anthropic is the SDK's own default for new plugins, but that +// is a Python-side policy — the wire-level default stays put for anyone +// calling this endpoint directly without an api_style. +func normalizeAPIStyle(raw string) (ai.APIStyle, error) { + switch raw { + case "": + return ai.APIStyleOpenAI, nil + case string(ai.APIStyleOpenAI), string(ai.APIStyleAnthropic): + return ai.APIStyle(raw), nil + default: + return "", &bindError{"api_style must be \"openai\" or \"anthropic\", got " + raw} + } +} + +// upsertPluginProvider creates a plugin-tagged provider, or updates the +// endpoint/token/style of an existing one with the same name. Idempotent by +// name so a plugin can safely re-register (e.g. on every process start). +func (h *Handler) upsertPluginProvider(name, endpoint, token string, apiStyle ai.APIStyle) (*typ.Provider, error) { + if existing, err := h.config.GetProviderByName(name); err == nil && existing.IsPlugin() { + existing.APIBase = endpoint + existing.APIStyle = apiStyle + existing.Token = token + existing.NoKeyRequired = token == "" + existing.Enabled = true + if err := h.config.UpdateProvider(existing.UUID, existing); err != nil { + return nil, err + } + return existing, nil + } + + provider := &typ.Provider{ + UUID: config.GenerateUUID(), + Name: name, + APIBase: endpoint, + APIStyle: apiStyle, + Token: token, + NoKeyRequired: token == "", + Enabled: true, + AuthType: typ.AuthTypeAPIKey, + Timeout: constant.DefaultRequestTimeout, + Tags: []string{typ.PluginTag}, + } + if err := h.config.AddProvider(provider); err != nil { + return nil, err + } + return provider, nil +} + +// ListPlugins returns the plugin-tagged providers, with the model id(s) each +// currently routes (derived from the rules bound to it) for display. +func (h *Handler) ListPlugins(c *gin.Context) { + modelsByProvider := map[string]string{} + for _, rule := range h.config.GetRequestConfigs() { + for _, svc := range rule.Services { + if svc == nil { + continue + } + if _, ok := modelsByProvider[svc.Provider]; !ok { + modelsByProvider[svc.Provider] = rule.RequestModel + } + } + } + + plugins := []PluginInfo{} + for _, p := range h.config.ListProviders() { + if !p.IsPlugin() { + continue + } + plugins = append(plugins, PluginInfo{ + UUID: p.UUID, + Name: p.Name, + Endpoint: p.APIBase, + ModelID: modelsByProvider[p.UUID], + }) + } + c.JSON(http.StatusOK, PluginsResponse{Success: true, Data: plugins}) +} + +// ensurePluginRule idempotently ensures a rule exists under scenario whose single +// tier-service points at the given provider id for modelID. Returns the rule UUID. +func (h *Handler) ensurePluginRule(scenario, modelID, providerID, name string, tier int) (string, error) { + scn := typ.RuleScenario(scenario) + if !typ.CanBindRulesToScenario(scn) { + return "", &bindError{"scenario " + scenario + " is not bindable"} + } + for _, rule := range h.config.GetRequestConfigs() { + if rule.GetScenario() == scn && rule.RequestModel == modelID { + return rule.UUID, nil // already bound (idempotent) + } + } + rule := typ.Rule{ + UUID: config.GenerateUUID(), + Scenario: scn, + RequestModel: modelID, + Description: "Plugin: " + name, + Active: true, + LBTactic: typ.NewDefaultTactic(loadbalance.TacticTier), + Services: []*loadbalance.Service{ + {Provider: providerID, Model: modelID, Weight: 1, Active: true, Tier: tier}, + }, + } + if err := h.config.AddRule(rule); err != nil { + return "", err + } + return rule.UUID, nil +} + +type bindError struct{ msg string } + +func (e *bindError) Error() string { return e.msg } diff --git a/internal/server/module/plugin/handler_test.go b/internal/server/module/plugin/handler_test.go new file mode 100644 index 000000000..a70a9eadb --- /dev/null +++ b/internal/server/module/plugin/handler_test.go @@ -0,0 +1,254 @@ +package plugin + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/tingly-dev/tingly-box/internal/server/config" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// postJSON drives a gin handler with a JSON body and returns the recorder and +// the parsed response envelope. +func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + gin.SetMode(gin.TestMode) + raw, _ := json.Marshal(body) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(raw)) + c.Request.Header.Set("Content-Type", "application/json") + h(c) + var parsed map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &parsed) + return w, parsed +} + +func newTestHandler(t *testing.T) *Handler { + t.Helper() + cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + return NewHandler(cfg) +} + +func TestRegisterPlugin_BindsRule(t *testing.T) { + h := newTestHandler(t) + + w, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", + Endpoint: "http://127.0.0.1:8765/v1", + ModelID: "plugin/my-rag", + Scenario: string(typ.ScenarioExperiment), + }) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", w.Code, w.Body.String()) + } + data, _ := resp["data"].(map[string]any) + if data["ready"] != true { + t.Fatalf("expected ready=true, got %v (note=%v)", data["ready"], data["note"]) + } + if data["model_id"] != "plugin/my-rag" { + t.Fatalf("model_id = %v", data["model_id"]) + } + providerUUID, _ := data["provider_uuid"].(string) + if providerUUID == "" { + t.Fatalf("expected a provider_uuid") + } + + // The provider must be persisted and tagged as a plugin. + prov, err := h.config.GetProviderByUUID(providerUUID) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if !prov.IsPlugin() { + t.Fatalf("provider is not tagged as plugin: %+v", prov) + } + + // A rule must exist under the scenario whose single service is the plugin. + var found bool + for _, rule := range h.config.GetRequestConfigs() { + if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { + found = true + if len(rule.Services) != 1 || rule.Services[0].Provider != providerUUID { + t.Fatalf("rule service does not point at plugin provider: %+v", rule.Services) + } + } + } + if !found { + t.Fatalf("no rule bound for the plugin under experiment scenario") + } +} + +func TestRegisterPlugin_APIStyleDefaultsToOpenAI(t *testing.T) { + h := newTestHandler(t) + + _, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "no-style", Endpoint: "http://127.0.0.1:8765/v1", + }) + uuid := resp["data"].(map[string]any)["provider_uuid"].(string) + prov, err := h.config.GetProviderByUUID(uuid) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIStyle != "openai" { + t.Fatalf("expected default api_style openai, got %q", prov.APIStyle) + } +} + +func TestRegisterPlugin_APIStyleAnthropic(t *testing.T) { + h := newTestHandler(t) + + _, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "anthropic-plug", Endpoint: "http://127.0.0.1:8765", APIStyle: "anthropic", + }) + uuid := resp["data"].(map[string]any)["provider_uuid"].(string) + prov, err := h.config.GetProviderByUUID(uuid) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIStyle != "anthropic" { + t.Fatalf("expected api_style anthropic, got %q", prov.APIStyle) + } +} + +func TestRegisterPlugin_APIStyleRejectsUnknown(t *testing.T) { + h := newTestHandler(t) + + w, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "bad-style", Endpoint: "http://127.0.0.1:8765", APIStyle: "gemini", + }) + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400 for unknown api_style, got %d (%v)", w.Code, resp) + } +} + +func TestRegisterPlugin_ReregisterUpdatesAPIStyle(t *testing.T) { + h := newTestHandler(t) + + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "switches", Endpoint: "http://127.0.0.1:8765", + }) + _, second := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "switches", Endpoint: "http://127.0.0.1:8765", APIStyle: "anthropic", + }) + uuid := second["data"].(map[string]any)["provider_uuid"].(string) + prov, err := h.config.GetProviderByUUID(uuid) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIStyle != "anthropic" { + t.Fatalf("expected re-register to update api_style to anthropic, got %q", prov.APIStyle) + } +} + +func TestRegisterPlugin_ProviderOnly(t *testing.T) { + h := newTestHandler(t) + + _, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "solo", + Endpoint: "http://127.0.0.1:9000/v1", + }) + data, _ := resp["data"].(map[string]any) + if data["ready"] == true { + t.Fatalf("expected ready=false when no scenario given") + } + // model id defaults to plugin/ + if data["model_id"] != "plugin/solo" { + t.Fatalf("model_id default = %v", data["model_id"]) + } +} + +func TestRegisterPlugin_ReregisterUpdatesInPlace(t *testing.T) { + h := newTestHandler(t) + + _, first := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", Endpoint: "http://127.0.0.1:8765/v1", + }) + firstUUID := first["data"].(map[string]any)["provider_uuid"].(string) + + // Re-register (e.g. the plugin process restarted on a different port). + _, second := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", Endpoint: "http://127.0.0.1:9999/v1", + }) + secondUUID := second["data"].(map[string]any)["provider_uuid"].(string) + + if firstUUID != secondUUID { + t.Fatalf("re-register should update the same provider, got %s then %s", firstUUID, secondUUID) + } + + prov, err := h.config.GetProviderByUUID(firstUUID) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIBase != "http://127.0.0.1:9999/v1" { + t.Fatalf("expected endpoint to be updated in place, got %s", prov.APIBase) + } + + // Exactly one provider named my-rag — no duplicate created. + count := 0 + for _, p := range h.config.ListProviders() { + if p.Name == "my-rag" { + count++ + } + } + if count != 1 { + t.Fatalf("re-register must not duplicate the provider, got %d", count) + } +} + +func TestRegisterPlugin_ReregisterIsIdempotentForRule(t *testing.T) { + h := newTestHandler(t) + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "p", Endpoint: "http://a/v1", Scenario: string(typ.ScenarioExperiment), + }) + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "p", Endpoint: "http://b/v1", Scenario: string(typ.ScenarioExperiment), + }) + count := 0 + for _, rule := range h.config.GetRequestConfigs() { + if rule.RequestModel == "plugin/p" { + count++ + } + } + if count != 1 { + t.Fatalf("re-register must not duplicate the rule, got %d", count) + } +} + +func TestListPlugins_FiltersPluginTag(t *testing.T) { + h := newTestHandler(t) + // a normal provider + if err := h.config.AddProvider(&typ.Provider{ + Name: "real", APIBase: "https://api.example.com/v1", APIStyle: "openai", Enabled: true, + }); err != nil { + t.Fatalf("AddProvider: %v", err) + } + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "plug", Endpoint: "http://127.0.0.1:8765/v1", ModelID: "plugin/plug", + Scenario: string(typ.ScenarioExperiment), + }) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + h.ListPlugins(c) + + var resp PluginsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(resp.Data) != 1 || resp.Data[0].Name != "plug" { + t.Fatalf("expected exactly the plugin provider, got %+v", resp.Data) + } + if resp.Data[0].ModelID != "plugin/plug" { + t.Fatalf("expected model id derived from the bound rule, got %q", resp.Data[0].ModelID) + } +} diff --git a/internal/server/module/plugin/routes.go b/internal/server/module/plugin/routes.go new file mode 100644 index 000000000..c78f1b55a --- /dev/null +++ b/internal/server/module/plugin/routes.go @@ -0,0 +1,23 @@ +package plugin + +import ( + "github.com/tingly-dev/tingly-box/swagger" +) + +// RegisterRoutes wires the plugin endpoints onto the given route group. +func RegisterRoutes(api *swagger.RouteGroup, h *Handler) { + // Register (or update) external plugin code as an upstream, optionally + // binding a rule — an idempotent upsert-by-name. + api.POST("/plugins", h.RegisterPlugin, + swagger.WithDescription("Register (or update) external plugin code as an upstream, optionally binding a rule"), + swagger.WithTags("plugins"), + swagger.WithRequestModel(RegisterPluginRequest{}), + swagger.WithResponseModel(RegisterPluginResponse{}), + ) + + api.GET("/plugins", h.ListPlugins, + swagger.WithDescription("List registered plugin providers"), + swagger.WithTags("plugins"), + swagger.WithResponseModel(PluginsResponse{}), + ) +} diff --git a/internal/server/module/plugin/types.go b/internal/server/module/plugin/types.go new file mode 100644 index 000000000..e53c40bad --- /dev/null +++ b/internal/server/module/plugin/types.go @@ -0,0 +1,39 @@ +package plugin + +// RegisterPluginRequest registers external plugin code as a tingly-box upstream. +// It is idempotent by name: calling it again (e.g. every time the plugin +// process starts) updates the existing provider instead of duplicating it. +type RegisterPluginRequest struct { + Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` + Endpoint string `json:"endpoint" binding:"required" description:"Plugin base URL" example:"http://127.0.0.1:8765/v1"` + ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` + Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` + Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` + Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` + APIStyle string `json:"api_style,omitempty" description:"Wire protocol the plugin's endpoint speaks: \"openai\" or \"anthropic\"; empty defaults to \"openai\"" example:"anthropic"` +} + +// RegisterPluginResponse reports what was created or updated. +type RegisterPluginResponse struct { + ProviderUUID string `json:"provider_uuid"` + ModelID string `json:"model_id"` + Scenario string `json:"scenario,omitempty"` + RuleUUID string `json:"rule_uuid,omitempty"` + // Ready is true when a rule is bound, so clients can select the model now. + Ready bool `json:"ready"` + Note string `json:"note,omitempty"` +} + +// PluginInfo is a list view of a plugin provider. +type PluginInfo struct { + UUID string `json:"uuid"` + Name string `json:"name"` + Endpoint string `json:"endpoint"` + ModelID string `json:"model_id,omitempty"` +} + +// PluginsResponse wraps the plugin list. +type PluginsResponse struct { + Success bool `json:"success"` + Data []PluginInfo `json:"data"` +} diff --git a/internal/server/module/providerquota/handler_test.go b/internal/server/module/providerquota/handler_test.go new file mode 100644 index 000000000..ce7735c7c --- /dev/null +++ b/internal/server/module/providerquota/handler_test.go @@ -0,0 +1,124 @@ +package providerquota + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/tingly-dev/tingly-box/ai/quota" +) + +// fakeManager lets each test configure exactly what GetQuota returns per +// provider UUID, without a real store/fetcher. +type fakeManager struct { + quotas map[string]*quota.ProviderUsage + errs map[string]error +} + +func (f *fakeManager) GetQuota(_ context.Context, providerUUID string) (*quota.ProviderUsage, error) { + if err, ok := f.errs[providerUUID]; ok { + return nil, err + } + if u, ok := f.quotas[providerUUID]; ok { + return u, nil + } + return nil, quota.ErrUsageNotFound +} +func (f *fakeManager) GetQuotaNoCache(ctx context.Context, providerUUID string) (*quota.ProviderUsage, error) { + return f.GetQuota(ctx, providerUUID) +} +func (f *fakeManager) ListQuota(context.Context) ([]*quota.ProviderUsage, error) { return nil, nil } +func (f *fakeManager) Refresh(context.Context) ([]*quota.ProviderUsage, error) { return nil, nil } +func (f *fakeManager) RefreshProvider(context.Context, string) (*quota.ProviderUsage, error) { + return nil, nil +} +func (f *fakeManager) Summary(context.Context) (*quota.Summary, error) { return nil, nil } +func (f *fakeManager) IsProviderSupported(string) bool { return true } +func (f *fakeManager) StartAutoRefresh(context.Context) {} +func (f *fakeManager) StopAutoRefresh() {} + +func postJSON(t *testing.T, h gin.HandlerFunc, body any) *httptest.ResponseRecorder { + t.Helper() + gin.SetMode(gin.TestMode) + raw, _ := json.Marshal(body) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(raw)) + c.Request.Header.Set("Content-Type", "application/json") + h(c) + return w +} + +// TestBatchGetQuota_SkipsProvidersWithNoData is the regression test for the +// bug e2e testing surfaced: a provider with no quota data (e.g. a vmodel/ +// local provider with no registered fetcher) used to 500 the WHOLE batch +// request instead of just being omitted from the result — because +// Manager.GetQuota re-wrapped ErrUsageNotFound into a new error, breaking +// the handler's `err != quota.ErrUsageNotFound` identity check (fixed in +// ai/quota/manager.go). This test exercises the real (non-fake) comparison +// path in the handler with a manager that returns the sentinel directly. +func TestBatchGetQuota_SkipsProvidersWithNoData(t *testing.T) { + mgr := &fakeManager{ + quotas: map[string]*quota.ProviderUsage{ + "has-data": {ProviderUUID: "has-data", ProviderName: "Real"}, + }, + // "no-data" provider: absent from both maps -> ErrUsageNotFound. + } + h := NewHandler(mgr, logrus.StandardLogger()) + + w := postJSON(t, h.BatchGetQuota, BatchGetQuotaRequest{ProviderUUIDs: []string{"has-data", "no-data"}}) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String()) + } + var resp BatchGetQuotaResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if _, ok := resp.Data["has-data"]; !ok { + t.Fatalf("expected has-data in response, got %+v", resp.Data) + } + if _, ok := resp.Data["no-data"]; ok { + t.Fatalf("expected no-data to be omitted (not errored), got %+v", resp.Data) + } +} + +// TestBatchGetQuota_FailsOnlyWhenEveryProviderErrors confirms a genuine +// (non-not-found) error still surfaces when NO provider in the batch +// produced usable data — distinct from the not-found-is-a-skip case above. +func TestBatchGetQuota_FailsOnlyWhenEveryProviderErrors(t *testing.T) { + mgr := &fakeManager{ + errs: map[string]error{"broken": context.DeadlineExceeded}, + } + h := NewHandler(mgr, logrus.StandardLogger()) + + w := postJSON(t, h.BatchGetQuota, BatchGetQuotaRequest{ProviderUUIDs: []string{"broken"}}) + + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500; body=%s", w.Code, w.Body.String()) + } +} + +// TestGetQuota_NotFoundReturns404 pins the single-provider GetQuota's +// not-found response, the same sentinel path BatchGetQuota relies on. +func TestGetQuota_NotFoundReturns404(t *testing.T) { + mgr := &fakeManager{} + h := NewHandler(mgr, logrus.StandardLogger()) + + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/provider-quota/no-data", nil) + c.Params = gin.Params{{Key: "uuid", Value: "no-data"}} + h.GetQuota(c) + + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body=%s", w.Code, w.Body.String()) + } +} diff --git a/internal/server/sdk_session.go b/internal/server/sdk_session.go new file mode 100644 index 000000000..69849c1f7 --- /dev/null +++ b/internal/server/sdk_session.go @@ -0,0 +1,165 @@ +package server + +import ( + "net" + "net/http" + "net/url" + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// SDKSessionRequest is the body for POST /api/v1/sdk/session. +type SDKSessionRequest struct { + // Scenario is the rule scenario the SDK session should bind to. + // Defaults to "experiment" when empty. + Scenario string `json:"scenario"` + // Name is a human label that shows up in tingly-box logs as the caller, + // so experiments are distinguishable in the request history. + Name string `json:"name"` +} + +// SDKSessionResponse is returned by POST /api/v1/sdk/session. It hands the +// Python SDK everything it needs to construct a tingly-box-bound OpenAI / +// Anthropic client: a base URL, a bearer token, and the transports the +// scenario accepts. +type SDKSessionResponse struct { + // BaseURL is the scenario root, e.g. "http://127.0.0.1:12580/tingly/experiment". + // The OpenAI SDK should target BaseURL+"/v1"; the Anthropic SDK targets BaseURL. + BaseURL string `json:"base_url"` + // Token is the bearer token to authenticate against the gateway. In v0.1 + // this is the gateway model token (long-lived); scoped short-lived tokens + // are a follow-up. + Token string `json:"token"` + // Scenario is the resolved scenario id. + Scenario string `json:"scenario"` + // Transport is "openai", "anthropic", or "both", derived from the scenario + // descriptor. It tells the SDK which client styles are valid. + Transport string `json:"transport"` + // Ready is true when an active rule with at least one service is bound to + // the scenario. When false, requests will fail until the user binds a rule; + // the SDK's `tingly doctor` surfaces this as the next action. + Ready bool `json:"ready"` + // Services is the number of active services bound to the scenario's rule. + Services int `json:"services"` + // ExpiresAt is the token expiry. Empty in v0.1 (long-lived model token). + ExpiresAt string `json:"expires_at,omitempty"` +} + +// CreateSDKSession mints an SDK session for a scenario. It is the single +// gateway-side endpoint the `tingly` Python module relies on: given a scenario, +// it returns the base URL, bearer token, and accepted transports so a user can +// write an experiment or plugin in a handful of lines and reuse the gateway's +// routing, fallback, guard rails, quota, and logging. +func (s *Server) CreateSDKSession(c *gin.Context) { + var req SDKSessionRequest + if err := c.ShouldBindJSON(&req); err != nil { + // Tolerate an empty body — default everything. + req = SDKSessionRequest{} + } + + scenario := typ.RuleScenario(req.Scenario) + if scenario == "" { + scenario = typ.ScenarioExperiment + } + + descriptor, ok := typ.GetScenarioDescriptor(scenario) + if !ok || !descriptor.AllowRuleBinding { + c.JSON(http.StatusNotFound, gin.H{ + "success": false, + "error": "unknown or non-bindable scenario: " + string(scenario), + "valid_scenarios": bindableScenarioIDs(), + }) + return + } + + ready, services := s.scenarioRuleStatus(scenario) + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": SDKSessionResponse{ + BaseURL: s.scenarioBaseURL(scenario), + Token: s.config.GetModelToken(), + Scenario: string(scenario), + Transport: scenarioTransportLabel(descriptor), + Ready: ready, + Services: services, + }, + }) +} + +// scenarioBaseURL builds the externally reachable scenario root URL. A bind +// host of 0.0.0.0 / empty is rewritten to 127.0.0.1 so the returned URL is +// usable by a local SDK client. +func (s *Server) scenarioBaseURL(scenario typ.RuleScenario) string { + host := s.config.GetServerHost() + if host == "" || host == "0.0.0.0" || host == "::" { + host = "127.0.0.1" + } + port := s.config.GetServerPort() + if port == 0 { + port = 12580 + } + return (&url.URL{ + Scheme: "http", + Host: net.JoinHostPort(host, strconv.Itoa(port)), + Path: "/tingly/" + string(scenario.Base()), + }).String() +} + +// scenarioRuleStatus reports whether an active rule with at least one active +// service is bound to the scenario, and how many active services it has. +func (s *Server) scenarioRuleStatus(scenario typ.RuleScenario) (ready bool, services int) { + for i := range s.config.GetRequestConfigs() { + rule := s.config.GetRequestConfigs()[i] + if !rule.Active { + continue + } + if rule.GetScenario().Base() != scenario.Base() { + continue + } + n := len(rule.GetActiveServices()) + if n > services { + services = n + } + if n > 0 { + ready = true + } + } + return ready, services +} + +// scenarioTransportLabel collapses a descriptor's supported transports into the +// label the SDK understands: "openai", "anthropic", or "both". +func scenarioTransportLabel(descriptor typ.ScenarioDescriptor) string { + openai, anthropic := false, false + for _, t := range descriptor.SupportedTransport { + switch t { + case typ.TransportOpenAI: + openai = true + case typ.TransportAnthropic: + anthropic = true + } + } + switch { + case openai && anthropic: + return "both" + case anthropic: + return "anthropic" + default: + return "openai" + } +} + +// bindableScenarioIDs lists scenario ids a caller may bind an SDK session to. +func bindableScenarioIDs() []string { + var ids []string + for _, d := range typ.RegisteredScenarioDescriptors() { + if d.AllowRuleBinding { + ids = append(ids, string(d.ID)) + } + } + return ids +} diff --git a/internal/server/sdk_session_test.go b/internal/server/sdk_session_test.go new file mode 100644 index 000000000..a22f2a58f --- /dev/null +++ b/internal/server/sdk_session_test.go @@ -0,0 +1,72 @@ +package server + +import ( + "encoding/json" + "testing" + + "github.com/tingly-dev/tingly-box/internal/typ" +) + +func TestScenarioTransportLabel(t *testing.T) { + cases := []struct { + name string + transport []typ.ScenarioTransport + want string + }{ + {"both", []typ.ScenarioTransport{typ.TransportOpenAI, typ.TransportAnthropic}, "both"}, + {"anthropic-only", []typ.ScenarioTransport{typ.TransportAnthropic}, "anthropic"}, + {"openai-only", []typ.ScenarioTransport{typ.TransportOpenAI}, "openai"}, + {"embed-falls-to-openai", []typ.ScenarioTransport{typ.TransportEmbed}, "openai"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := scenarioTransportLabel(typ.ScenarioDescriptor{SupportedTransport: tc.transport}) + if got != tc.want { + t.Fatalf("scenarioTransportLabel(%v) = %q, want %q", tc.transport, got, tc.want) + } + }) + } +} + +func TestBindableScenarioIDsIncludesExperiment(t *testing.T) { + ids := bindableScenarioIDs() + found := false + for _, id := range ids { + if id == string(typ.ScenarioExperiment) { + found = true + } + } + if !found { + t.Fatalf("expected experiment scenario in bindable list, got %v", ids) + } +} + +// TestSDKSessionResponseShape freezes the JSON field names the Python SDK +// depends on. If a field is renamed here without updating the SDK, this fails. +func TestSDKSessionResponseShape(t *testing.T) { + resp := SDKSessionResponse{ + BaseURL: "http://127.0.0.1:12580/tingly/experiment", + Token: "tok", + Scenario: "experiment", + Transport: "both", + Ready: true, + Services: 2, + } + b, err := json.Marshal(resp) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, key := range []string{"base_url", "token", "scenario", "transport", "ready", "services"} { + if _, ok := m[key]; !ok { + t.Fatalf("response JSON missing %q field; have %v", key, m) + } + } + // expires_at is omitempty and absent here + if _, ok := m["expires_at"]; ok { + t.Fatalf("expires_at should be omitted when empty") + } +} diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index dbf858bb0..6433a897f 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -12,6 +12,7 @@ import ( "github.com/tingly-dev/tingly-box/internal/server/config" "github.com/tingly-dev/tingly-box/internal/server/module/info" "github.com/tingly-dev/tingly-box/internal/server/module/onboarding" + pluginmodule "github.com/tingly-dev/tingly-box/internal/server/module/plugin" probemodule "github.com/tingly-dev/tingly-box/internal/server/module/probe" providermodule "github.com/tingly-dev/tingly-box/internal/server/module/provider" "github.com/tingly-dev/tingly-box/internal/server/module/providertemplate" @@ -88,6 +89,15 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { swagger.WithResponseModel(gin.H{}), ) + // SDK session endpoint — mints a scenario-bound session (base URL, token, + // transports) for the `tingly` Python SDK / plugin experiments. + apiV1.POST("/sdk/session", s.CreateSDKSession, + swagger.WithTags("sdk"), + swagger.WithDescription("Mint a scoped SDK session for a scenario"), + swagger.WithRequestModel(SDKSessionRequest{}), + swagger.WithResponseModel(SDKSessionResponse{}), + ) + apiV2 := manager.NewGroup("api", "v2", "") apiV2.Router.Use(s.getUserAuthMiddleware()) @@ -391,6 +401,15 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { providerHandler := providermodule.NewHandler(s.config, s.quotaManager) providermodule.RegisterRoutes(apiV2, providerHandler) + // Plugin registration: a plugin is a provider tagged "plugin" (external + // OpenAI upstream). Independent module — its only dependency is config, so + // it owns its own handler rather than living on *Server. POST is an + // idempotent upsert-by-name that wires it in (provider + optional rule) in + // one step; liveness is handled by the same circuit breaker that covers + // every other provider — no separate lifecycle. + pluginHandler := pluginmodule.NewHandler(s.config) + pluginmodule.RegisterRoutes(apiV2, pluginHandler) + // Provider template endpoints providerTemplateHandler := providertemplate.NewHandler(s.templateManager) providertemplate.RegisterRoutes(apiV2, providerTemplateHandler) diff --git a/internal/typ/scenario_registry.go b/internal/typ/scenario_registry.go index 8e5f2f0c2..41083b88b 100644 --- a/internal/typ/scenario_registry.go +++ b/internal/typ/scenario_registry.go @@ -114,6 +114,17 @@ func builtinScenarioDescriptorFor(scenario RuleScenario) ScenarioDescriptor { AllowDirectPathUse: true, SupportsProfiles: true, } + case ScenarioExperiment: + // SDK / plugin experiment surface. Accepts both OpenAI and Anthropic + // transports so a Python experiment can use either SDK against the same + // /tingly/experiment endpoint. Profiles let users name parallel experiments. + return ScenarioDescriptor{ + ID: scenario, + SupportedTransport: []ScenarioTransport{TransportOpenAI, TransportAnthropic}, + AllowRuleBinding: true, + AllowDirectPathUse: true, + SupportsProfiles: true, + } case ScenarioGlobal: return ScenarioDescriptor{ ID: scenario, diff --git a/internal/typ/scenario_registry_test.go b/internal/typ/scenario_registry_test.go index 13a243118..806b68389 100644 --- a/internal/typ/scenario_registry_test.go +++ b/internal/typ/scenario_registry_test.go @@ -224,3 +224,22 @@ func TestValidateProfileName(t *testing.T) { } } } + +func TestExperimentScenarioDescriptor(t *testing.T) { + d, ok := GetScenarioDescriptor(ScenarioExperiment) + if !ok { + t.Fatalf("expected %q descriptor to be registered", ScenarioExperiment) + } + if !d.AllowRuleBinding || !d.AllowDirectPathUse { + t.Fatalf("expected experiment descriptor to allow rule binding and path use, got %+v", d) + } + if !d.SupportsProfiles { + t.Fatalf("expected experiment scenario to support profiles") + } + if !ScenarioSupportsTransport(ScenarioExperiment, TransportOpenAI) { + t.Fatalf("experiment scenario should support TransportOpenAI") + } + if !ScenarioSupportsTransport(ScenarioExperiment, TransportAnthropic) { + t.Fatalf("experiment scenario should support TransportAnthropic") + } +} diff --git a/internal/typ/type.go b/internal/typ/type.go index ba1f93cb4..55054ecdb 100644 --- a/internal/typ/type.go +++ b/internal/typ/type.go @@ -63,8 +63,9 @@ const ( ScenarioClaudeDesktop RuleScenario = "claude_desktop" ScenarioSmartGuide RuleScenario = "_smart_guide" ScenarioGlobal RuleScenario = "_global" // Global flags that apply to all scenarios - ScenarioEmbed RuleScenario = "embed" // Embedding application scenario; only serves /embeddings - ScenarioImageGen RuleScenario = "imagegen" // Image generation scenario; only serves /images/generations + ScenarioEmbed RuleScenario = "embed" // Embedding application scenario; only serves /embeddings + ScenarioImageGen RuleScenario = "imagegen" // Image generation scenario; only serves /images/generations + ScenarioExperiment RuleScenario = "experiment" // Python SDK / plugin experiment scenario; accepts OpenAI + Anthropic transports ) func BuiltinScenarios() []RuleScenario { @@ -83,6 +84,7 @@ func BuiltinScenarios() []RuleScenario { ScenarioGlobal, ScenarioEmbed, ScenarioImageGen, + ScenarioExperiment, } } @@ -343,6 +345,9 @@ type OAuthDetail = ai.OAuthDetail // Type alias for backward compatibility with common/provider type VModelDetail = ai.VModelDetail +// PluginTag marks a provider as backed by external plugin code (see Provider.IsPlugin). +const PluginTag = ai.PluginTag + // CredentialBundle holds multi-field credentials for non-bearer auth types // Type alias for backward compatibility with common/provider type CredentialBundle = ai.CredentialBundle diff --git a/openapi.json b/openapi.json index 9ae957298..317b59ccb 100644 --- a/openapi.json +++ b/openapi.json @@ -3833,6 +3833,39 @@ } } }, + "/api/v1/sdk/session": { + "post": { + "tags": [ + "sdk" + ], + "summary": "Mint a scoped SDK session for a scenario", + "description": "Mint a scoped SDK session for a scenario", + "operationId": "apiV1SdkSessionPost", + "requestBody": { + "description": "Request body", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SDKSessionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SDKSessionResponse" + } + } + } + } + } + } + }, "/api/v1/server/restart": { "post": { "tags": [ @@ -4789,6 +4822,59 @@ } } }, + "/api/v2/plugins": { + "get": { + "tags": [ + "plugins" + ], + "summary": "List registered plugin providers", + "description": "List registered plugin providers", + "operationId": "apiV2PluginsGet", + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PluginsResponse" + } + } + } + } + } + }, + "post": { + "tags": [ + "plugins" + ], + "summary": "Register (or update) external plugin code as an upstream, optionally binding a rule", + "description": "Register (or update) external plugin code as an upstream, optionally binding a rule", + "operationId": "apiV2PluginsPost", + "requestBody": { + "description": "Request body", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginResponse" + } + } + } + } + } + } + }, "/api/v2/probe": { "post": { "tags": [ @@ -9317,6 +9403,52 @@ "categories" ] }, + "PluginInfo": { + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "description": "Field endpoint" + }, + "model_id": { + "type": "string", + "description": "Field model_id" + }, + "name": { + "type": "string", + "description": "Field name" + }, + "uuid": { + "type": "string", + "description": "Field uuid" + } + }, + "required": [ + "uuid", + "name", + "endpoint" + ] + }, + "PluginsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "description": "Field data", + "items": { + "$ref": "#/components/schemas/PluginInfo" + } + }, + "success": { + "type": "boolean", + "description": "Field success" + } + }, + "required": [ + "success", + "data" + ] + }, "Policy": { "type": "object", "properties": { @@ -10251,6 +10383,83 @@ "message" ] }, + "RegisterPluginRequest": { + "type": "object", + "properties": { + "api_style": { + "type": "string", + "description": "Wire protocol the plugin's endpoint speaks: \"openai\" or \"anthropic\"; empty defaults to \"openai\"", + "example": "anthropic" + }, + "endpoint": { + "type": "string", + "description": "Plugin base URL", + "example": "http://127.0.0.1:8765/v1" + }, + "model_id": { + "type": "string", + "description": "Model id the plugin advertises", + "example": "plugin/my-rag" + }, + "name": { + "type": "string", + "description": "Plugin / provider name", + "example": "my-rag" + }, + "scenario": { + "type": "string", + "description": "Scenario to bind a rule under; omit to create only the provider", + "example": "experiment" + }, + "tier": { + "type": "integer", + "format": "int64", + "description": "Tier for the bound service (0 = highest priority)" + }, + "token": { + "type": "string", + "description": "Token tingly-box should send to the plugin (empty = no key)" + } + }, + "required": [ + "name", + "endpoint" + ] + }, + "RegisterPluginResponse": { + "type": "object", + "properties": { + "model_id": { + "type": "string", + "description": "Field model_id" + }, + "note": { + "type": "string", + "description": "Field note" + }, + "provider_uuid": { + "type": "string", + "description": "Field provider_uuid" + }, + "ready": { + "type": "boolean", + "description": "Field ready" + }, + "rule_uuid": { + "type": "string", + "description": "Field rule_uuid" + }, + "scenario": { + "type": "string", + "description": "Field scenario" + } + }, + "required": [ + "provider_uuid", + "model_id", + "ready" + ] + }, "RemoveSkillLocationResponse": { "type": "object", "properties": { @@ -10494,6 +10703,65 @@ "data" ] }, + "SDKSessionRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Field name" + }, + "scenario": { + "type": "string", + "description": "Field scenario" + } + }, + "required": [ + "scenario", + "name" + ] + }, + "SDKSessionResponse": { + "type": "object", + "properties": { + "base_url": { + "type": "string", + "description": "Field base_url" + }, + "expires_at": { + "type": "string", + "description": "Field expires_at" + }, + "ready": { + "type": "boolean", + "description": "Field ready" + }, + "scenario": { + "type": "string", + "description": "Field scenario" + }, + "services": { + "type": "integer", + "format": "int64", + "description": "Field services" + }, + "token": { + "type": "string", + "description": "Field token" + }, + "transport": { + "type": "string", + "description": "Field transport" + } + }, + "required": [ + "base_url", + "token", + "scenario", + "transport", + "ready", + "services" + ] + }, "ScanIdesResponse": { "type": "object", "properties": { @@ -13452,6 +13720,10 @@ "name": "onboarding", "description": "Operations related to onboarding" }, + { + "name": "plugins", + "description": "Operations related to plugins" + }, { "name": "providers", "description": "Operations related to providers" @@ -13468,6 +13740,10 @@ "name": "scenarios", "description": "Operations related to scenarios" }, + { + "name": "sdk", + "description": "Operations related to sdk" + }, { "name": "server", "description": "Operations related to server" diff --git a/sdk/python/README.md b/sdk/python/README.md new file mode 100644 index 000000000..4a559816c --- /dev/null +++ b/sdk/python/README.md @@ -0,0 +1,131 @@ +# tingly — Python SDK for tingly-box + +Write an LLM experiment or plugin in a handful of lines and reuse the +tingly-box gateway's power: provider routing, fallback, guard rails, quota and +logging. You write the idea; the box handles the plumbing. + +## Install + +```bash +pip install tingly +``` + +Ships with the `openai` and `anthropic` SDKs so `tb.openai` / `tb.anthropic` +give you full fine-grained control out of the box. + +## Experiment ASAP + +```python +import tingly + +tb = tingly.connect(scenario="experiment") # auto-discovers your local tb + +# One-shot, transport picked for you, model routed by tb: +print(tb.ask("Summarize tingly-box in one line", model="auto")) + +# Or use the SDK objects directly — already pointed at the gateway: +resp = tb.openai.chat.completions.create( + model="auto", + messages=[{"role": "user", "content": "hi"}], +) +resp = tb.anthropic.messages.create( + model="claude-sonnet-4-6", + max_tokens=256, + messages=[{"role": "user", "content": "hi"}], +) +``` + +Every call above flows through tingly-box, so guard rails, quota, logging and +fallback apply automatically — your experiment never has to know. + +## How `connect()` finds your box + +In order: explicit args → `TINGLY_BOX_URL` / `TINGLY_BOX_TOKEN` env → +`~/.tingly-box/sdk.json` → `~/.tingly-box/config.json` + localhost probe. + +The token is your **admin** token (tb's `UserToken`); the SDK uses it once to +mint a session, then uses the returned model token for the LLM calls. + +## Diagnose + +```bash +tingly doctor # traverses the real path and prints what works +tingly doctor --link # save gateway URL + token to ~/.tingly-box/sdk.json +``` + +A green `tingly doctor` is a guarantee your code will run. + +## Write a plugin (an AI server tb can route to) + +A plugin is an upstream tb can call two ways: **Anthropic Messages +(`/v1/messages`, primary)** and **OpenAI chat completions +(`/v1/chat/completions`, secondary)** — both real, both always served; which +one tb actually uses is a registration choice (`api_style`, `"anthropic"` by +default). Write one handler, serve it, register it — then any tb client can +select it as a model, regardless of which protocol *that* client speaks. + +```python +from tingly import Plugin + +plugin = Plugin(name="my-rag") # model id: plugin/my-rag + +@plugin.chat +def handle(req): + docs = retrieve(req.last_user_text()) + return plugin.llm.ask(f"Using {docs}, answer: {req.last_user_text()}") + +if __name__ == "__main__": + plugin.serve() # http://127.0.0.1:8765/v1 +``` + +```bash +tingly plugin init my-rag # scaffold module + tingly.toml +tingly plugin run my_rag_plugin.py # serve AND register with tb +``` + +`serve()` (and `tingly plugin run`) registers the plugin with tb once at +startup — an idempotent upsert-by-name, so restarting the plugin updates the +same provider instead of duplicating it. There is no heartbeat or lease: +liveness is handled by tb's existing per-service circuit breaker, the same +mechanism that protects every other provider. If the plugin goes down, the +next failed request trips the breaker and traffic tier-fails-over (when a +fallback tier is configured). Retiring a plugin is the same as retiring any +other provider — delete it in the tb UI. + +The server is stdlib-only (no FastAPI), supports streaming on both routes, and +`plugin.llm` calls back into tb (Anthropic-first) so the plugin reuses the +gateway for its own LLM work. + +### Example plugins + +`sdk/python/examples/` has three, each demonstrating a different real-world +pattern for the same idea — a plugin composing the box by calling back into +other tb rules: + +- **`rag_plugin.py`** — retrieval-augmented answers from a toy corpus, one + call back into tb for generation. +- **`critic_plugin.py`** — cross-model critique (`model="plugin/critic"`): + forwards the thing to review to a *different* rule/model and returns a + structured verdict. Self-critique is unreliable (a model can't reliably + catch its own mistakes); this is the pattern behind + [Zen MCP](https://github.com/jray2123/zen-mcp-server) and + [Consult7](https://github.com/szeider/consult7), and behind aider's + architect/editor split. +- **`fusion_plugin.py`** — multi-model consensus (`model="plugin/fusion"`): + polls a panel of rules/models concurrently, skips the judge call when they + already agree, otherwise a judge call synthesizes. Mirrors Consult7's 2026 + Fusion feature; the clearest illustration that a plugin can freely + originate more than one call, against more than one rule, per request. + +## Status + +- **Layer 1** (consume tb): `connect()` → `Client`. Done — Anthropic tried + first when a scenario supports both transports. +- **Layer 2** (be an AI server): `tingly.Plugin` + manifest + `register`. Done — + dual-protocol server (Anthropic primary, OpenAI secondary); tb-side + supervisor/lifecycle UI still pending (not required to use plugins today). +- **Layer 3** (tb routes to the plugin as a model): via provider-as-upstream. + Done, verified end-to-end including a plugin forwarding to another tb rule + and returning the result (`sdk/python/examples/e2e_run.sh`). + +See `.design/python-sdk.md` in the repo for the full design and diagrams. diff --git a/sdk/python/examples/critic_plugin.py b/sdk/python/examples/critic_plugin.py new file mode 100644 index 000000000..10c157719 --- /dev/null +++ b/sdk/python/examples/critic_plugin.py @@ -0,0 +1,89 @@ +"""A "critic" plugin: cross-model critique — the pattern behind Zen MCP and +Consult7 (an agent mid-task consults a *different* model for review) and +aider's architect/editor split (a separate model reviews before code lands). + +Self-critique — a model reviewing its own output — is unreliable: Huang et +al. (ICLR 2024) found LLMs cannot reliably self-correct without external +feedback. Cross-model critique (a genuinely different model reviews) is the +more robust variant, and it maps directly onto what a plugin is for: this +handler does zero LLM work itself, it only forwards the artifact-to-review to +a different tb rule/model via `plugin.use(...)` and shapes the structured +verdict that comes back. No hard-coded provider or key — same gateway, +different rule. + +Run it (serves on :8766 AND registers with tb on startup): + + pip install -e . # from sdk/python + python examples/critic_plugin.py + +Then from any tb client: model="plugin/critic", the message is the thing to +review (a diff, a draft answer, a decision); an optional system message adds +context the critic should weigh. +""" + +from __future__ import annotations + +import json + +from tingly import ChatRequest, Plugin + +# Where the critique itself is delegated — point CRITIC_SCENARIO / CRITIC_MODEL +# at a rule bound to a genuinely different (ideally stronger) model than +# whatever called this plugin; reviewing with the same model defeats the point. +CRITIC_SCENARIO = "experiment" +CRITIC_MODEL = "auto" + +CRITIQUE_PROMPT = """You are reviewing the following for correctness, risk and \ +missing considerations. Respond with ONLY JSON matching: +{{"verdict": "approve" | "revise", "issues": ["..."], "suggestion": "..."}} + +--- context --- +{context} + +--- to review --- +{content} +""" + +plugin = Plugin( + name="critic", + scenario="experiment", # bind a rule under this scenario on register + description="Cross-model critique — delegates review to a different rule/model", +) + + +@plugin.chat +def handle(req: ChatRequest) -> str: + content = req.last_user_text() + context = req.system_text() or "(none)" + prompt = CRITIQUE_PROMPT.format(context=context, content=content) + + # The one line that matters: hand the review to a DIFFERENT tb rule. + raw = plugin.use(CRITIC_SCENARIO).ask(prompt, model=CRITIC_MODEL, max_tokens=1024) + return _format_verdict(_parse_verdict(raw)) + + +def _parse_verdict(raw: str) -> dict: + text = raw.strip() + if text.startswith("```"): + text = text.strip("`") + text = text.split("\n", 1)[1] if "\n" in text else text + try: + return json.loads(text) + except ValueError: + # The critic model didn't follow the JSON contract — degrade to a + # plain "revise" verdict carrying its raw text as the suggestion, + # rather than crashing the request. + return {"verdict": "revise", "issues": ["critic model did not return JSON"], "suggestion": raw} + + +def _format_verdict(verdict: dict) -> str: + lines = [f"verdict: {verdict.get('verdict', 'unknown')}"] + for issue in verdict.get("issues") or []: + lines.append(f"- {issue}") + if verdict.get("suggestion"): + lines.append(f"suggestion: {verdict['suggestion']}") + return "\n".join(lines) + + +if __name__ == "__main__": + plugin.serve(port=8766) diff --git a/sdk/python/examples/e2e_plugin.py b/sdk/python/examples/e2e_plugin.py new file mode 100644 index 000000000..d894c6270 --- /dev/null +++ b/sdk/python/examples/e2e_plugin.py @@ -0,0 +1,35 @@ +"""E2E plugin: served as an upstream, and calls BACK into tb's echo-model. + +Demonstrates the full hub: + client → tb (rule plugin/rag-demo) → THIS plugin → plugin.use("experiment") + → tb (rule echo-model → vmodel) → echoed text → back to client +""" + +from tingly import Plugin + +plugin = Plugin(name="rag-demo", scenario="experiment") + +CORPUS = { + "tingly-box": "tingly-box is a personal intelligence orchestrator.", + "plugin": "A plugin is an Anthropic/OpenAI-compatible upstream tb can route to.", +} + + +def retrieve(q: str) -> str: + hits = [t for k, t in CORPUS.items() if k in q.lower()] + return " ".join(hits) or "(no docs)" + + +@plugin.chat +def handle(req): + q = req.last_user_text() + docs = retrieve(q) + # Call back into tb against the echo-model rule (no real network needed). + echoed = plugin.use("experiment").ask( + f"[plugin-rag] docs={docs!r} q={q!r}", model="echo-model" + ) + return f"RAG via plugin → tb echo returned: {echoed}" + + +if __name__ == "__main__": + plugin.serve(port=8765) diff --git a/sdk/python/examples/e2e_run.sh b/sdk/python/examples/e2e_run.sh new file mode 100755 index 000000000..bbfb05012 --- /dev/null +++ b/sdk/python/examples/e2e_run.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# End-to-end demo of the full hub, using NO network / API keys: +# client → tb (rule plugin/rag-demo) → plugin → plugin.use("experiment") +# → tb (rule echo-model → vmodel) → echoed text → back to client +# +# Prereqs: +# go build -o /tmp/tb_e2e ./cli/tingly-box # the tb binary +# pip install httpx openai # SDK transitive + a transport +# Run: bash sdk/python/examples/e2e_run.sh +set -uo pipefail + +TB=${TB_BIN:-/tmp/tb_e2e} +CFG=$(mktemp -d) +PORT=18901 +BASE="http://127.0.0.1:$PORT" +SDK=/home/user/tingly-box/sdk/python +export PYTHONPATH=$SDK + +cleanup() { + [[ -n "${PLUG_PID:-}" ]] && kill "$PLUG_PID" 2>/dev/null + [[ -n "${TB_PID:-}" ]] && kill "$TB_PID" 2>/dev/null +} +trap cleanup EXIT + +echo "== 1. start tb (config-dir=$CFG, port=$PORT) ==" +"$TB" --config-dir "$CFG" start --port "$PORT" --ui --browser=false >/tmp/tb_e2e.log 2>&1 & +TB_PID=$! +for i in $(seq 1 60); do + curl -sf "$BASE/api/v1/info/health" >/dev/null 2>&1 && break + sleep 0.5 +done +curl -sf "$BASE/api/v1/info/health" >/dev/null || { echo "tb did not start"; tail -25 /tmp/tb_e2e.log; exit 1; } +echo " tb healthy at $BASE" + +# Tokens are generated fresh per config-dir; read them from the config file. +CFGFILE=$(find "$CFG" -name 'config.json' | head -1) +echo " config file: $CFGFILE" +UTOK=$(python3 -c "import json,sys;d=json.load(open('$CFGFILE'));print(d.get('user_token') or d.get('UserToken',''))") +MTOK=$(python3 -c "import json,sys;d=json.load(open('$CFGFILE'));print(d.get('model_token') or d.get('ModelToken',''))") +echo " user token: ${UTOK:0:16}… model token: ${MTOK:0:16}…" + +UADMIN=(-H "Authorization: Bearer $UTOK" -H "Content-Type: application/json") +UMODEL=(-H "Authorization: Bearer $MTOK" -H "Content-Type: application/json") + +echo "== 2. create vmodel provider (echo backend, no network) ==" +VRESP=$(curl -s "${UADMIN[@]}" -X POST "$BASE/api/v2/providers" -d '{ + "name":"vmodel-echo","api_base":"vmodel://local","api_style":"openai", + "auth_type":"vmodel","no_key_required":true,"enabled":true}') +VUUID=$(echo "$VRESP" | python3 -c "import sys,json;d=json.load(sys.stdin);print(d.get('data',{}).get('uuid') or d.get('uuid',''))") +echo " vmodel provider uuid: $VUUID" + +echo "== 3. create echo-model rule under experiment scenario ==" +curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ + \"scenario\":\"experiment\",\"request_model\":\"echo-model\",\"active\":true, + \"lb_tactic\":{\"type\":\"random\",\"params\":{}}, + \"services\":[{\"provider\":\"$VUUID\",\"model\":\"echo-model\",\"weight\":1,\"active\":true}]}" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'), d.get('data',{}).get('uuid',''))" + +echo "== 4. start the plugin — it registers with tb once on startup ==" +echo " (serve() → POST /api/v2/plugins, an idempotent upsert-by-name; no heartbeat)" +TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ + python3 "$SDK/examples/e2e_plugin.py" >/tmp/plugin_e2e.log 2>&1 & +PLUG_PID=$! +for i in $(seq 1 40); do + curl -sf "http://127.0.0.1:8765/health" >/dev/null 2>&1 && break + sleep 0.3 +done +curl -sf "http://127.0.0.1:8765/health" >/dev/null || { echo "plugin did not start"; cat /tmp/plugin_e2e.log; exit 1; } + +echo "== 5. tb sees the plugin provider (GET /api/v2/plugins) ==" +for i in $(seq 1 20); do + LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") + echo "$LIST" | grep -q 'rag-demo' && break + sleep 0.3 +done +echo "$LIST" | python3 -m json.tool + +echo "== 6. CLIENT CALL: model=plugin/rag-demo through tb ==" +echo " (client → tb → plugin → tb echo-model → back)" +curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ + "model":"plugin/rag-demo", + "messages":[{"role":"user","content":"What is tingly-box?"}]}' | python3 -m json.tool + +echo "== plugin log tail ==" +tail -6 /tmp/plugin_e2e.log + +echo "== 7. NO SEPARATE LIFECYCLE: kill the plugin (simulated crash) ==" +echo " (hard SIGKILL, no graceful shutdown — the provider is a normal DB row," +echo " same as any other provider, so it does NOT disappear from the list)" +kill -KILL "$PLUG_PID" 2>/dev/null +PLUG_PID="" +LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") +echo " GET /api/v2/plugins still shows it: $LIST" + +echo "== 8. client call after the plugin is dead ==" +echo " (liveness is the SAME per-service circuit breaker every provider gets;" +echo " with no fallback tier configured on this rule the request just errors —" +echo " add a tier-1 real model to the rule and this would tier-failover instead)" +curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ + "model":"plugin/rag-demo", + "messages":[{"role":"user","content":"still there?"}]}' \ + | python3 -c "import sys,json; d=json.load(sys.stdin); e=d.get('error', d); print(' ->', (json.dumps(e) if isinstance(e,dict) else str(e))[:200])" + +echo "== 9. restart the plugin → re-register upserts the SAME provider (no duplicate) ==" +TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ + python3 "$SDK/examples/e2e_plugin.py" >/tmp/plugin_e2e_2.log 2>&1 & +PLUG_PID=$! +for i in $(seq 1 40); do + curl -sf "http://127.0.0.1:8765/health" >/dev/null 2>&1 && break + sleep 0.3 +done +COUNT=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(sum(1 for p in d['data'] if p['name']=='rag-demo'))") +echo " plugin providers named rag-demo after restart: $COUNT (expect 1)" + +echo "== done ==" diff --git a/sdk/python/examples/fusion_plugin.py b/sdk/python/examples/fusion_plugin.py new file mode 100644 index 000000000..823f5f4d3 --- /dev/null +++ b/sdk/python/examples/fusion_plugin.py @@ -0,0 +1,81 @@ +"""A "fusion" plugin: parallel multi-model consensus, then a judge model +synthesizes — the pattern behind Consult7's 2026 Fusion feature (a panel of +frontier models answers in parallel; a judge model merges the answers; a +panel that already agrees skips the judge call). + +This is the clearest illustration of "tb is a hub of rules; a plugin can +freely originate calls against any of them": the handler calls BACK into tb +more than once, against DIFFERENT rules/models, concurrently, before +answering once. + +Run it (serves on :8767 AND registers with tb on startup): + + pip install -e . # from sdk/python + python examples/fusion_plugin.py + +Then from any tb client: model="plugin/fusion", the message is the question. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor + +from tingly import ChatRequest, Plugin + +# The panel: each entry is (scenario, model) called independently and +# concurrently. Point these at genuinely different rules/models — a panel of +# clones of the same model adds latency without adding a second opinion. +PANEL = [ + ("experiment", "auto"), + ("experiment", "auto"), +] +JUDGE_SCENARIO = "experiment" +JUDGE_MODEL = "auto" + +JUDGE_PROMPT = """Multiple models answered the same question independently. \ +Synthesize the single best answer, resolving disagreements and noting when \ +the panel disagreed. + +--- question --- +{question} + +--- panel answers --- +{answers} +""" + +plugin = Plugin( + name="fusion", + scenario="experiment", # bind a rule under this scenario on register + description="Multi-model consensus — panel of rules/models + judge synthesis", +) + + +@plugin.chat +def handle(req: ChatRequest) -> str: + question = req.last_user_text() + answers = _poll_panel(question) + + if len(set(answers)) == 1: + # The panel already agreed — the judge call would just restate this, + # so skip it and save a hop (mirrors Consult7 skipping the panel + # entirely for trivial prompts). + return answers[0] + + answers_block = "\n\n".join(f"[{i + 1}] {a}" for i, a in enumerate(answers)) + return plugin.use(JUDGE_SCENARIO).ask( + JUDGE_PROMPT.format(question=question, answers=answers_block), + model=JUDGE_MODEL, + ) + + +def _poll_panel(question: str) -> list: + with ThreadPoolExecutor(max_workers=len(PANEL)) as pool: + futures = [ + pool.submit(plugin.use(scenario).ask, question, model=model) + for scenario, model in PANEL + ] + return [f.result() for f in futures] + + +if __name__ == "__main__": + plugin.serve(port=8767) diff --git a/sdk/python/examples/rag_experiment.py b/sdk/python/examples/rag_experiment.py new file mode 100644 index 000000000..24580c2e7 --- /dev/null +++ b/sdk/python/examples/rag_experiment.py @@ -0,0 +1,54 @@ +"""A minimal RAG-style experiment that reuses the tingly-box gateway. + +Run a local tingly-box, then: + + pip install -e . # from sdk/python + python examples/rag_experiment.py + +Everything below routes through tb: provider selection, fallback, guard rails, +quota and logging are all applied for free. The experiment only owns its own +logic (here, a toy retriever). +""" + +import tingly + +# A stand-in "corpus". A real experiment would hit a vector store here. +CORPUS = { + "tingly-box": "tingly-box is a personal intelligence orchestrator: an LLM " + "gateway with remote control and guard rails.", + "sdk": "The tingly Python SDK lets you write an experiment in a handful of " + "lines and reuse the gateway.", +} + + +def retrieve(question: str) -> str: + q = question.lower() + hits = [text for key, text in CORPUS.items() if key in q] + return "\n".join(hits) or "(no matching documents)" + + +def main() -> None: + # Auto-discovers the local gateway and binds to the "experiment" scenario. + with tingly.connect(scenario="experiment", name="rag-experiment") as tb: + if not tb.ready: + print( + "Scenario 'experiment' has no active rule yet — bind one in the " + "tingly-box UI, or run `tingly doctor`." + ) + return + + question = "What is tingly-box?" + docs = retrieve(question) + answer = tb.ask( + f"Using only these documents:\n{docs}\n\nAnswer: {question}", + model="auto", + ) + print("Q:", question) + print("A:", answer) + + usage = tb.usage.this_session() + print(f"\n[usage] {usage.requests} request(s), {usage.total_tokens} tokens") + + +if __name__ == "__main__": + main() diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py new file mode 100644 index 000000000..947512347 --- /dev/null +++ b/sdk/python/examples/rag_plugin.py @@ -0,0 +1,53 @@ +"""A RAG plugin served as an upstream for tingly-box (Anthropic primary, OpenAI secondary). + +Run it (serves on :8765 AND registers with tb on startup): + + pip install -e . # from sdk/python + python examples/rag_plugin.py + +Registration is a one-shot, idempotent upsert by name — tb creates or updates +this plugin's provider (and the rule, since `scenario` is set below). There is +no heartbeat or lease; liveness is handled by tb's existing per-service circuit +breaker like any other provider. + +Now `model="plugin/rag-demo"` from Claude Code, Cursor, the tb UI, or another +`tingly.connect()` experiment routes here — with tb's guard rails, quota, +logging and tier-failover applied. The handler itself calls *back* into tb via +`plugin.llm` for the generation step. +""" + +from tingly import Plugin + +plugin = Plugin( + name="rag-demo", + scenario="experiment", # bind a rule under this scenario on register + description="Answers from a toy in-memory corpus", +) + +CORPUS = { + "tingly-box": "tingly-box is a personal intelligence orchestrator: an LLM " + "gateway with remote control and guard rails.", + "plugin": "A tingly plugin is an Anthropic/OpenAI-compatible upstream that " + "tingly-box can route to as a model.", +} + + +def retrieve(question: str) -> str: + q = question.lower() + hits = [text for key, text in CORPUS.items() if key in q] + return "\n".join(hits) or "(no matching documents)" + + +@plugin.chat +def handle(req): + question = req.last_user_text() + docs = retrieve(question) + # Generation goes back through tingly-box — no provider/key hard-coded here. + return plugin.llm.ask( + f"Using only these documents:\n{docs}\n\nAnswer: {question}", + model="auto", + ) + + +if __name__ == "__main__": + plugin.serve() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml new file mode 100644 index 000000000..8ee2f469b --- /dev/null +++ b/sdk/python/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "tingly" +version = "0.1.0" +description = "Python SDK for tingly-box — write an LLM experiment or plugin in a handful of lines and reuse the gateway's routing, fallback, guard rails, quota and logging." +readme = "README.md" +requires-python = ">=3.9" +license = { text = "MPL-2.0" } +authors = [{ name = "tingly.dev", email = "ops@tingly.dev" }] +keywords = ["tingly-box", "llm", "gateway", "openai", "anthropic", "plugin"] +dependencies = [ + "httpx>=0.27", + "pydantic>=2", + # Core, not optional: the value of the SDK is fine-grained control via the + # real provider SDKs (tb.openai / tb.anthropic expose every param, tool, + # streaming and beta header). ask() and plugin.llm route through them too. + "openai>=1.0", + "anthropic>=0.40", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "respx>=0.21"] + +[project.scripts] +tingly = "tingly.cli:main" + +[project.urls] +Homepage = "https://github.com/tingly-dev/tingly-box" +Documentation = "https://github.com/tingly-dev/tingly-box/blob/main/.design/python-sdk.md" + +[tool.hatch.build.targets.wheel] +packages = ["tingly"] + +[tool.pytest.ini_options] +markers = [ + "needs_tb: integration tests that require a running tingly-box gateway (opt-in)", +] +addopts = "-m 'not needs_tb'" diff --git a/sdk/python/tests/test_client_offline.py b/sdk/python/tests/test_client_offline.py new file mode 100644 index 000000000..aa3255a04 --- /dev/null +++ b/sdk/python/tests/test_client_offline.py @@ -0,0 +1,100 @@ +"""Client transport-routing tests with a fake session (no real SDKs invoked).""" + +import pytest + +from tingly.client import Client +from tingly.discovery import Session +from tingly.errors import TinglyError + + +def _client(transport: str) -> Client: + session = Session( + base_url="http://tb.test:12580/tingly/experiment", + token="model-tok", + scenario="experiment", + transport=transport, + ready=True, + services=1, + ) + return Client(session, "http://tb.test:12580", "admin", "exp", 30.0) + + +def test_anthropic_only_rejects_openai(): + c = _client("anthropic") + with pytest.raises(TinglyError): + _ = c.openai + + +def test_openai_only_rejects_anthropic(): + c = _client("openai") + with pytest.raises(TinglyError): + _ = c.anthropic + + +def test_both_exposes_identity(): + c = _client("both") + assert c.scenario == "experiment" + assert c.transport == "both" + assert c.ready is True + assert c.base_url.endswith("/tingly/experiment") + + +def test_ask_prefers_anthropic_when_both_supported(monkeypatch): + """Anthropic is tb's native protocol, so ask() tries it first when a + scenario supports both — flipped from the old OpenAI-first default.""" + calls = [] + c = _client("both") + monkeypatch.setattr(c, "_ask_anthropic", lambda *a, **k: calls.append("anthropic") or "ok") + monkeypatch.setattr(c, "_ask_openai", lambda *a, **k: calls.append("openai") or "ok") + c.ask("hi") + assert calls == ["anthropic"] + + +def test_ask_anthropic_passes_model_through_unmodified(monkeypatch): + """model="auto" must reach tb as-is (tb's rule resolution handles it) — + not get silently rewritten to a hardcoded model name.""" + c = _client("anthropic") + captured = {} + + class _FakeMessages: + def create(self, **kwargs): + captured.update(kwargs) + + class R: + content = [] + + return R() + + class _FakeAnthropic: + messages = _FakeMessages() + + monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) + c.ask("hi", model="auto") + assert captured["model"] == "auto" + + +def test_client_passes_scenario_root_to_transports(monkeypatch): + """The client hands the scenario root + model token to each builder; + per-transport URL shaping (e.g. the OpenAI /v1 suffix) happens inside the + builder.""" + captured = {} + + def fake_openai(base_url, token, timeout): + captured["openai_base"] = base_url + captured["openai_token"] = token + return object() + + def fake_anthropic(base_url, token, timeout): + captured["anthropic_base"] = base_url + return object() + + monkeypatch.setattr("tingly.transports.openai_compat.build_openai", fake_openai) + monkeypatch.setattr( + "tingly.transports.anthropic_compat.build_anthropic", fake_anthropic + ) + c = _client("both") + _ = c.openai + _ = c.anthropic + assert captured["openai_base"] == "http://tb.test:12580/tingly/experiment" + assert captured["openai_token"] == "model-tok" + assert captured["anthropic_base"] == "http://tb.test:12580/tingly/experiment" diff --git a/sdk/python/tests/test_config.py b/sdk/python/tests/test_config.py new file mode 100644 index 000000000..468fb56cd --- /dev/null +++ b/sdk/python/tests/test_config.py @@ -0,0 +1,57 @@ +"""Config resolution precedence tests (no network).""" + +import json + +import tingly.config as cfg + + +def test_args_win(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, "http://env:1/") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + r = cfg.resolve(base_url="http://arg:2", token="arg-token") + assert r.base_url == "http://arg:2" + assert r.token == "arg-token" + assert r.source == "args" + + +def test_env_fallback(monkeypatch, tmp_path): + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + monkeypatch.setenv(cfg.ENV_URL, "http://env:1") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + r = cfg.resolve() + assert r.base_url == "http://env:1" + assert r.token == "env-token" + assert r.source == "env" + + +def test_sdk_link_file(monkeypatch, tmp_path): + monkeypatch.delenv(cfg.ENV_URL, raising=False) + monkeypatch.delenv(cfg.ENV_TOKEN, raising=False) + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + (tmp_path / "sdk.json").write_text( + json.dumps({"base_url": "http://link:3", "token": "link-token"}) + ) + r = cfg.resolve() + assert r.base_url == "http://link:3" + assert r.token == "link-token" + assert r.source == "sdk.json" + + +def test_config_json_admin_token(monkeypatch, tmp_path): + monkeypatch.delenv(cfg.ENV_URL, raising=False) + monkeypatch.delenv(cfg.ENV_TOKEN, raising=False) + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + (tmp_path / "config.json").write_text(json.dumps({"UserToken": "admin-xyz"})) + r = cfg.resolve() + assert r.token == "admin-xyz" + # default localhost base when nothing else set + assert r.base_url.startswith("http://127.0.0.1:") + + +def test_default_localhost(monkeypatch, tmp_path): + monkeypatch.delenv(cfg.ENV_URL, raising=False) + monkeypatch.delenv(cfg.ENV_TOKEN, raising=False) + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + r = cfg.resolve() + assert r.base_url == f"http://{cfg.DEFAULT_HOST}:{cfg.DEFAULT_PORT}" + assert r.token is None diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py new file mode 100644 index 000000000..45ce0c311 --- /dev/null +++ b/sdk/python/tests/test_discovery.py @@ -0,0 +1,84 @@ +"""Discovery + session minting tests (gateway mocked with respx).""" + +import httpx +import pytest +import respx + +import tingly.discovery as disco +from tingly.errors import AuthError, GatewayUnreachableError, ScenarioNotFoundError + +BASE = "http://tb.test:12580" + + +@respx.mock +def test_probe_version_ok(): + # Discovery probes the unauthenticated health endpoint, not /info/version. + respx.get(f"{BASE}/api/v1/info/health").mock( + return_value=httpx.Response(200, json={"health": True, "status": "healthy"}) + ) + assert disco.probe_version(BASE) == "ok" + + +@respx.mock +def test_probe_version_down(): + respx.get(f"{BASE}/api/v1/info/health").mock( + return_value=httpx.Response(503) + ) + assert disco.probe_version(BASE) is None + + +@respx.mock +def test_create_session_ok(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "data": { + "base_url": f"{BASE}/tingly/experiment", + "token": "model-tok", + "scenario": "experiment", + "transport": "both", + "ready": True, + "services": 2, + }, + }, + ) + ) + s = disco.create_session(BASE, "admin", "experiment", name="exp") + assert s.base_url == f"{BASE}/tingly/experiment" + assert s.token == "model-tok" + assert s.transport == "both" + assert s.ready is True + assert s.services == 2 + + +@respx.mock +def test_create_session_auth_error(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + return_value=httpx.Response(401, json={"error": "nope"}) + ) + with pytest.raises(AuthError): + disco.create_session(BASE, "bad", "experiment") + + +@respx.mock +def test_create_session_scenario_not_found(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + return_value=httpx.Response( + 404, + json={"success": False, "valid_scenarios": ["experiment", "openai"]}, + ) + ) + with pytest.raises(ScenarioNotFoundError) as ei: + disco.create_session(BASE, "admin", "bogus") + assert "experiment" in ei.value.valid_scenarios + + +@respx.mock +def test_create_session_unreachable(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + side_effect=httpx.ConnectError("refused") + ) + with pytest.raises(GatewayUnreachableError): + disco.create_session(BASE, "admin", "experiment") diff --git a/sdk/python/tests/test_example_plugins.py b/sdk/python/tests/test_example_plugins.py new file mode 100644 index 000000000..e18a0ecf9 --- /dev/null +++ b/sdk/python/tests/test_example_plugins.py @@ -0,0 +1,136 @@ +"""Tests for the critic/fusion showcase plugins (sdk/python/examples/). + +These exercise handler logic only — plugin.use() is monkeypatched to a fake +client, so no real tb and no real model calls. The examples aren't part of +the installed `tingly` package, so they're loaded by file path, the same way +`tingly plugin run` loads a user's plugin script (see tingly/cli.py). +""" + +import importlib.util +import sys +from pathlib import Path + +from tingly.plugin.types import ChatRequest + +EXAMPLES = Path(__file__).parent.parent / "examples" + + +def _load(name): + path = EXAMPLES / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +def _req(content, system=None): + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": content}) + return ChatRequest.from_openai_body({"model": "x", "messages": messages}) + + +class _FakeClient: + """Stands in for a tingly.Client — records calls, replies with a fixed + string or (if given a callable) the result of calling it with the prompt.""" + + def __init__(self, reply): + self._reply = reply + self.calls = [] + + def ask(self, prompt, **kwargs): + self.calls.append((prompt, kwargs)) + return self._reply(prompt) if callable(self._reply) else self._reply + + +# -- critic ----------------------------------------------------------------- + +def test_critic_formats_valid_json_verdict(monkeypatch): + critic = _load("critic_plugin") + fake = _FakeClient('{"verdict": "approve", "issues": [], "suggestion": "looks good"}') + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("def f(): return 1/0", system="a python snippet")) + + assert result == "verdict: approve\nsuggestion: looks good" + prompt, kwargs = fake.calls[0] + assert "a python snippet" in prompt + assert "def f(): return 1/0" in prompt + assert kwargs["model"] == critic.CRITIC_MODEL + + +def test_critic_lists_issues(monkeypatch): + critic = _load("critic_plugin") + fake = _FakeClient('{"verdict": "revise", "issues": ["divides by zero"], "suggestion": "guard the denominator"}') + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("def f(): return 1/0")) + + assert "verdict: revise" in result + assert "- divides by zero" in result + assert "suggestion: guard the denominator" in result + + +def test_critic_degrades_gracefully_on_non_json(monkeypatch): + """A critic model that ignores the JSON contract must not crash the + request — it should surface as a 'revise' verdict carrying the raw text.""" + critic = _load("critic_plugin") + fake = _FakeClient("looks fine to me") + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("some code")) + + assert "verdict: revise" in result + assert "looks fine to me" in result + + +def test_critic_strips_markdown_code_fence(monkeypatch): + critic = _load("critic_plugin") + fake = _FakeClient('```json\n{"verdict": "approve", "issues": [], "suggestion": ""}\n```') + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("some code")) + + assert result == "verdict: approve" + + +# -- fusion ------------------------------------------------------------- + +def test_poll_panel_gathers_one_result_per_panel_entry(monkeypatch): + fusion = _load("fusion_plugin") + fake = _FakeClient("same-answer") + monkeypatch.setattr(fusion.plugin, "use", lambda scenario: fake) + + results = fusion._poll_panel("q") + + assert results == ["same-answer"] * len(fusion.PANEL) + assert len(fake.calls) == len(fusion.PANEL) + + +def test_fusion_skips_judge_when_panel_agrees(monkeypatch): + fusion = _load("fusion_plugin") + monkeypatch.setattr(fusion, "_poll_panel", lambda question: ["42", "42"]) + + def judge_should_not_be_called(scenario): + raise AssertionError("judge must not be called when the panel agrees") + + monkeypatch.setattr(fusion.plugin, "use", judge_should_not_be_called) + + assert fusion.handle(_req("what is 6*7?")) == "42" + + +def test_fusion_calls_judge_when_panel_disagrees(monkeypatch): + fusion = _load("fusion_plugin") + monkeypatch.setattr(fusion, "_poll_panel", lambda question: ["A", "B"]) + judge = _FakeClient("SYNTHESIZED") + monkeypatch.setattr(fusion.plugin, "use", lambda scenario: judge) + + result = fusion.handle(_req("question")) + + assert result == "SYNTHESIZED" + assert len(judge.calls) == 1 + judge_prompt = judge.calls[0][0] + assert "A" in judge_prompt and "B" in judge_prompt + assert "question" in judge_prompt diff --git a/sdk/python/tests/test_plugin_manifest.py b/sdk/python/tests/test_plugin_manifest.py new file mode 100644 index 000000000..8b45824d6 --- /dev/null +++ b/sdk/python/tests/test_plugin_manifest.py @@ -0,0 +1,72 @@ +"""Manifest round-trip + discovery tests.""" + +from pathlib import Path + +from tingly import Plugin +from tingly.plugin import manifest as m + + +def test_manifest_roundtrip(tmp_path: Path): + man = m.Manifest( + name="my-rag", + model_id="plugin/my-rag", + entrypoint="rag_plugin:plugin", + description='has "quotes" and \\ slash', + port=9001, + ) + man.write(tmp_path) + loaded = m.load(tmp_path) + assert loaded.name == "my-rag" + assert loaded.model_id == "plugin/my-rag" + assert loaded.entrypoint == "rag_plugin:plugin" + assert loaded.port == 9001 + assert loaded.description == 'has "quotes" and \\ slash' + + +def test_manifest_find_walks_up(tmp_path: Path): + m.Manifest(name="p", model_id="plugin/p", entrypoint="p:plugin").write(tmp_path) + nested = tmp_path / "a" / "b" + nested.mkdir(parents=True) + found = m.find(nested) + assert found is not None + assert found.name == "p" + + +def test_plugin_builds_manifest(): + plugin = Plugin(name="my-rag", description="d") + man = plugin.manifest(entrypoint="rag_plugin:plugin", port=8080) + assert man.model_id == "plugin/my-rag" + assert man.entrypoint == "rag_plugin:plugin" + assert man.port == 8080 + assert man.transport == "anthropic" + + +def test_plugin_manifest_transport_follows_api_style_override(): + plugin = Plugin(name="my-rag", api_style="openai") + man = plugin.manifest(entrypoint="rag_plugin:plugin") + assert man.transport == "openai" + + # an explicit transport= still wins over api_style + man2 = plugin.manifest(entrypoint="rag_plugin:plugin", transport="anthropic") + assert man2.transport == "anthropic" + + +def test_plugin_use_caches_per_scenario(monkeypatch): + import tingly.client as client_mod + from tingly import Plugin + + calls = [] + + def fake_connect(scenario, name): + calls.append((scenario, name)) + return object() + + monkeypatch.setattr(client_mod, "connect", fake_connect) + + plugin = Plugin(name="p", scenario="experiment") + a1 = plugin.llm + a2 = plugin.llm # default scenario, cached + b1 = plugin.use("claude_code") # different rule-set + assert a1 is a2 + assert b1 is not a1 + assert calls == [("experiment", "plugin:p"), ("claude_code", "plugin:p")] diff --git a/sdk/python/tests/test_plugin_register.py b/sdk/python/tests/test_plugin_register.py new file mode 100644 index 000000000..d7229f9f7 --- /dev/null +++ b/sdk/python/tests/test_plugin_register.py @@ -0,0 +1,117 @@ +"""Active config + plugin registration tests.""" + +import json + +import httpx +import pytest +import respx + +import tingly +import tingly.config as cfg +from tingly.plugin import register as plugin_register + +BASE = "http://tb.test:12580" + + +@pytest.fixture(autouse=True) +def _reset_override(): + cfg._OVERRIDE = None + yield + cfg._OVERRIDE = None + + +def test_configure_takes_precedence(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, "http://env:1") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + tingly.configure(url="http://configured:9", admin_token="cfg-token") + r = cfg.resolve() + assert r.base_url == "http://configured:9" + assert r.token == "cfg-token" + assert r.source == "configure" + + +def test_connection_token_by_env_reference(monkeypatch): + monkeypatch.setenv("MY_TB_SECRET", "secret-123") + conn = tingly.Connection(url="http://x", admin_token_env="MY_TB_SECRET") + assert conn.token() == "secret-123" + + +@respx.mock +def test_register_binds_rule(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, BASE) + monkeypatch.setenv(cfg.ENV_TOKEN, "admin") + + route = respx.post(f"{BASE}/api/v2/plugins").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": { + "provider_uuid": "uuid-1", "model_id": "plugin/x", + "scenario": "experiment", "rule_uuid": "rule-1", + "ready": True, "note": "Plugin wired in.", + }, + }) + ) + + result = plugin_register.register( + "x", "http://127.0.0.1:8765/v1", "plugin/x", scenario="experiment" + ) + assert route.called + assert result.provider_uuid == "uuid-1" + assert result.rule_uuid == "rule-1" + assert result.ready is True + # anthropic is the SDK's default wire protocol for new plugins + assert json.loads(route.calls.last.request.content)["api_style"] == "anthropic" + + +@respx.mock +def test_serve_registers_once(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, BASE) + monkeypatch.setenv(cfg.ENV_TOKEN, "admin") + route = respx.post(f"{BASE}/api/v2/plugins").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": {"provider_uuid": "pid", "model_id": "plugin/srv", + "scenario": "experiment", "ready": True}, + }) + ) + + from tingly import Plugin + + plugin = Plugin(name="srv", scenario="experiment") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False) + try: + assert isinstance(port, int) and port > 0 + assert route.called # registered exactly once on serve() + + # real plugin socket still serves (use urllib so respx doesn't intercept it) + import urllib.request + + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) as r: + assert r.status == 200 + finally: + plugin.stop() + + # stop() only tears down the HTTP server — nothing to deregister, the + # provider stays configured in tb (same as any other provider). + assert route.call_count == 1 + + +def test_serve_register_false_skips(monkeypatch): + from tingly import Plugin + + plugin = Plugin(name="noreg") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False, register=False) + try: + assert isinstance(port, int) and port > 0 + finally: + plugin.stop() diff --git a/sdk/python/tests/test_plugin_server.py b/sdk/python/tests/test_plugin_server.py new file mode 100644 index 000000000..c3d3692f7 --- /dev/null +++ b/sdk/python/tests/test_plugin_server.py @@ -0,0 +1,209 @@ +"""Plugin server tests — drive a real (ephemeral-port) plugin over HTTP. + +These pin the two wire contracts tingly-box relies on when it routes to a +plugin as an upstream — Anthropic /v1/messages (primary) and OpenAI +/v1/chat/completions (secondary) — plus buffered/streaming shape, /v1/models, +and auth. Both protocols share one handler; only response shaping differs. +""" + +import json + +import httpx +import pytest + +from tingly import ChatRequest, Plugin + + +@pytest.fixture +def served(): + plugin = Plugin(name="t-plug", model_id="plugin/t-plug") + + @plugin.chat + def handle(req: ChatRequest): + if req.stream: + return iter(["he", "llo ", req.last_user_text()]) + return f"echo: {req.last_user_text()}" + + port = plugin.serve(port=0, verbose=False, block=False) + base = f"http://127.0.0.1:{port}" + yield base + plugin.stop() + + +def test_health(served): + r = httpx.get(f"{served}/health", timeout=5) + assert r.status_code == 200 + assert r.json()["status"] == "ok" + + +def test_models_lists_model_id(served): + r = httpx.get(f"{served}/v1/models", timeout=5) + assert r.status_code == 200 + ids = [m["id"] for m in r.json()["data"]] + assert "plugin/t-plug" in ids + + +def test_chat_completion_shape(served): + r = httpx.post( + f"{served}/v1/chat/completions", + json={"model": "plugin/t-plug", "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert r.status_code == 200 + body = r.json() + assert body["object"] == "chat.completion" + assert body["choices"][0]["message"]["content"] == "echo: hi" + assert body["choices"][0]["finish_reason"] == "stop" + + +def test_chat_completion_streaming(served): + with httpx.stream( + "POST", + f"{served}/v1/chat/completions", + json={ + "model": "plugin/t-plug", + "messages": [{"role": "user", "content": "world"}], + "stream": True, + }, + timeout=5, + ) as r: + assert r.status_code == 200 + deltas = [] + saw_done = False + for line in r.iter_lines(): + if not line.startswith("data: "): + continue + data = line[len("data: "):] + if data == "[DONE]": + saw_done = True + continue + chunk = json.loads(data) + assert chunk["object"] == "chat.completion.chunk" + delta = chunk["choices"][0]["delta"] + if delta.get("content"): + deltas.append(delta["content"]) + assert "".join(deltas) == "hello world" + assert saw_done + + +def test_anthropic_message_shape(served): + r = httpx.post( + f"{served}/v1/messages", + json={ + "model": "plugin/t-plug", + "max_tokens": 256, + "messages": [{"role": "user", "content": "hi"}], + }, + timeout=5, + ) + assert r.status_code == 200 + body = r.json() + assert body["type"] == "message" + assert body["role"] == "assistant" + assert body["content"] == [{"type": "text", "text": "echo: hi"}] + assert body["stop_reason"] == "end_turn" + + +def test_anthropic_message_streaming(served): + with httpx.stream( + "POST", + f"{served}/v1/messages", + json={ + "model": "plugin/t-plug", + "max_tokens": 256, + "messages": [{"role": "user", "content": "world"}], + "stream": True, + }, + timeout=5, + ) as r: + assert r.status_code == 200 + events = [] + deltas = [] + for line in r.iter_lines(): + if line.startswith("event: "): + events.append(line[len("event: "):]) + elif line.startswith("data: "): + payload = json.loads(line[len("data: "):]) + if payload.get("type") == "content_block_delta": + deltas.append(payload["delta"]["text"]) + assert events == [ + "message_start", "content_block_start", "content_block_delta", + "content_block_delta", "content_block_delta", "content_block_stop", + "message_delta", "message_stop", + ] + assert "".join(deltas) == "hello world" + + +def test_anthropic_messages_route_ignores_query_string(served): + """tb's real Anthropic client appends ?beta=true to /v1/messages; the + plugin server must route on the path, not the raw request-target.""" + r = httpx.post( + f"{served}/v1/messages?beta=true", + json={"model": "plugin/t-plug", "max_tokens": 32, "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert r.status_code == 200 + assert r.json()["content"][0]["text"] == "echo: hi" + + +def test_anthropic_system_field_reaches_handler(): + plugin = Plugin(name="sys-plug") + + @plugin.chat + def handle(req: ChatRequest): + return f"system={req.system_text()!r} user={req.last_user_text()!r}" + + port = plugin.serve(port=0, verbose=False, block=False) + base = f"http://127.0.0.1:{port}" + try: + r = httpx.post( + f"{base}/v1/messages", + json={ + "model": "x", + "max_tokens": 64, + "system": "be terse", + "messages": [{"role": "user", "content": "hi"}], + }, + timeout=5, + ) + assert r.status_code == 200 + text = r.json()["content"][0]["text"] + assert text == "system='be terse' user='hi'" + finally: + plugin.stop() + + +def test_auth_enforced_when_key_set(): + plugin = Plugin(name="auth-plug", api_key="secret") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False) + base = f"http://127.0.0.1:{port}" + try: + bad = httpx.post( + f"{base}/v1/chat/completions", + json={"model": "x", "messages": []}, + timeout=5, + ) + assert bad.status_code == 401 + + good = httpx.post( + f"{base}/v1/chat/completions", + headers={"Authorization": "Bearer secret"}, + json={"model": "x", "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert good.status_code == 200 + + bad_anthropic = httpx.post( + f"{base}/v1/messages", + json={"model": "x", "max_tokens": 8, "messages": []}, + timeout=5, + ) + assert bad_anthropic.status_code == 401 + assert bad_anthropic.json()["type"] == "error" + finally: + plugin.stop() diff --git a/sdk/python/tests/test_transports.py b/sdk/python/tests/test_transports.py new file mode 100644 index 000000000..155bd0677 --- /dev/null +++ b/sdk/python/tests/test_transports.py @@ -0,0 +1,47 @@ +"""Transport URL-shaping tests, using a stubbed LLM SDK module. + +These pin the contract with the gateway routes: + - OpenAI SDK base_url = scenario_root + "/v1" -> /tingly//v1/chat/completions + - Anthropic SDK base_url = scenario_root -> /tingly//v1/messages +""" + +import sys +import types + +from tingly.transports import anthropic_compat, openai_compat + +ROOT = "http://tb.test:12580/tingly/experiment" + + +def test_openai_appends_v1(monkeypatch): + captured = {} + + fake = types.ModuleType("openai") + + class FakeOpenAI: + def __init__(self, base_url, api_key, timeout): + captured["base_url"] = base_url + captured["api_key"] = api_key + + fake.OpenAI = FakeOpenAI + monkeypatch.setitem(sys.modules, "openai", fake) + + openai_compat.build_openai(ROOT, "tok", 10.0) + assert captured["base_url"] == ROOT + "/v1" + assert captured["api_key"] == "tok" + + +def test_anthropic_no_v1(monkeypatch): + captured = {} + + fake = types.ModuleType("anthropic") + + class FakeAnthropic: + def __init__(self, base_url, api_key, timeout): + captured["base_url"] = base_url + + fake.Anthropic = FakeAnthropic + monkeypatch.setitem(sys.modules, "anthropic", fake) + + anthropic_compat.build_anthropic(ROOT, "tok", 10.0) + assert captured["base_url"] == ROOT diff --git a/sdk/python/tingly/__init__.py b/sdk/python/tingly/__init__.py new file mode 100644 index 000000000..e9fb3ec34 --- /dev/null +++ b/sdk/python/tingly/__init__.py @@ -0,0 +1,40 @@ +"""tingly — Python SDK for tingly-box. + +Write an LLM experiment or plugin in a handful of lines and reuse the gateway's +routing, fallback, guard rails, quota and logging: + + >>> import tingly + >>> tb = tingly.connect(scenario="experiment") + >>> tb.ask("Say hello", model="auto") +""" + +from __future__ import annotations + +from ._version import __version__ +from .client import Client, connect +from .config import Connection, configure +from .errors import ( + AuthError, + GatewayUnreachableError, + GuardrailBlockedError, + ScenarioNotFoundError, + TinglyError, + UpstreamError, +) +from .plugin import ChatRequest, Plugin + +__all__ = [ + "__version__", + "connect", + "configure", + "Connection", + "Client", + "Plugin", + "ChatRequest", + "TinglyError", + "GatewayUnreachableError", + "AuthError", + "ScenarioNotFoundError", + "GuardrailBlockedError", + "UpstreamError", +] diff --git a/sdk/python/tingly/_http.py b/sdk/python/tingly/_http.py new file mode 100644 index 000000000..bec761d55 --- /dev/null +++ b/sdk/python/tingly/_http.py @@ -0,0 +1,15 @@ +"""Small shared HTTP helpers for the tingly SDK.""" + +from __future__ import annotations + +from typing import Optional + +import httpx + + +def safe_json(resp: httpx.Response) -> Optional[dict]: + """Parse a JSON body, returning None instead of raising on malformed input.""" + try: + return resp.json() + except ValueError: + return None diff --git a/sdk/python/tingly/_version.py b/sdk/python/tingly/_version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/sdk/python/tingly/_version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py new file mode 100644 index 000000000..3e1dea5a5 --- /dev/null +++ b/sdk/python/tingly/cli.py @@ -0,0 +1,219 @@ +"""``tingly`` CLI — a single diagnostic command. + +`tingly doctor` traverses the *real* code path a user's program takes +(discovery → session → a live LLM round-trip) and prints what worked and what +didn't, so a green doctor is a guarantee that user code will run. + +`tingly doctor --link` writes ``~/.tingly-box/sdk.json`` so future runs need no +env vars. +""" + +from __future__ import annotations + +import argparse +import getpass +import json +import sys +from typing import Optional + +from . import config as _config +from . import discovery as _discovery +from . import scenarios as _scenarios + +OK = "OK" +FAIL = "FAIL" +WARN = "WARN" + + +def _row(label: str, detail: str, status: str) -> None: + print(f"{label:<14}{detail:<40}{status}") + + +def doctor(scenario: str, link: bool) -> int: + if link: + _do_link() + + resolved = _config.resolve() + + # 1. gateway reachable + alive = _discovery.probe_version(resolved.base_url) + if alive is None: + _row("gateway", resolved.base_url, FAIL) + print( + f"\nNo tingly-box gateway responding at {resolved.base_url} " + f"(resolved via {resolved.source}).\n" + "Start tb, set TINGLY_BOX_URL, or run `tingly doctor --link`." + ) + return 1 + _row("gateway", f"{resolved.base_url} (reachable)", OK) + _row("token", f"{resolved.source}", OK if resolved.token else WARN) + + # 2. mint a session (real path) + try: + session = _discovery.create_session( + base_url=resolved.base_url, + admin_token=resolved.token or "", + scenario=scenario, + name="tingly-doctor", + ) + except Exception as exc: # noqa: BLE001 - report any failure verbatim + _row("session", scenario, FAIL) + print(f"\n{type(exc).__name__}: {exc}") + return 1 + + scen_detail = f"{session.scenario} ({session.transport}, {session.services} svc)" + _row("scenario", scen_detail, OK if session.ready else WARN) + if not session.ready: + print( + f"\nScenario {session.scenario!r} has no active rule with a service. " + "Bind a rule to it in the tingly-box UI before sending requests." + ) + + # 3. live round-trip (only if ready) + if session.ready: + _live_check(session) + + return 0 + + +def _live_check(session: "_discovery.Session") -> None: + from .client import Client + + client = Client( + session=session, + gateway_url="", + admin_token="", + name="tingly-doctor", + timeout=30.0, + ) + try: + text = client.ask("Reply with the single word: pong", model="auto") + ok = isinstance(text, str) and len(text) > 0 + transport = "messages" if _scenarios.supports_anthropic(session.transport) else "chat.completions" + _row("llm test", transport, OK if ok else FAIL) + except Exception as exc: # noqa: BLE001 + _row("llm test", "round-trip", FAIL) + print(f"\n{type(exc).__name__}: {exc}") + finally: + client.close() + + +def _do_link() -> None: + """Prompt for the admin token and persist a link file.""" + path = _config.sdk_link_path() + base_url = input(f"Gateway URL [{_config.resolve().base_url}]: ").strip() + if not base_url: + base_url = _config.resolve().base_url + token = getpass.getpass("Admin token (TINGLY_BOX_TOKEN): ").strip() + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + json.dump({"base_url": base_url, "token": token}, fh, indent=2) + try: + path.chmod(0o600) + except OSError: + pass + print(f"Wrote {path}") + + +def _plugin_init(name: str) -> int: + """Scaffold a minimal plugin: a starter module + tingly.toml.""" + from pathlib import Path + + from .plugin.manifest import Manifest + + safe = name.replace("-", "_") + module = f"{safe}_plugin.py" + Path(module).write_text(_PLUGIN_TEMPLATE.format(name=name), encoding="utf-8") + Manifest( + name=name, + model_id=f"plugin/{name}", + entrypoint=f"{safe}_plugin:plugin", + description=f"{name} plugin", + ).write(Path.cwd()) + print(f"Created {module} and tingly.toml.") + print(f"Run it with: python {module} (or: tingly plugin run {module})") + return 0 + + +def _plugin_run(target: str) -> int: + """Import a plugin (``module:attr`` or a .py path) and serve it.""" + import importlib + import importlib.util + from pathlib import Path + + if target.endswith(".py") or "/" in target: + path = Path(target) + spec = importlib.util.spec_from_file_location(path.stem, path) + mod = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + spec.loader.exec_module(mod) # type: ignore[union-attr] + plugin = getattr(mod, "plugin", None) + else: + mod_name, _, attr = target.partition(":") + mod = importlib.import_module(mod_name) + plugin = getattr(mod, attr or "plugin", None) + + if plugin is None: + print(f"No `plugin` found in {target!r}") + return 1 + plugin.serve() + return 0 + + +def main(argv: Optional[list] = None) -> int: + parser = argparse.ArgumentParser(prog="tingly", description="tingly-box Python SDK") + sub = parser.add_subparsers(dest="command") + + p_doctor = sub.add_parser("doctor", help="diagnose the SDK ↔ gateway connection") + p_doctor.add_argument( + "--scenario", default=_scenarios.EXPERIMENT, help="scenario to test" + ) + p_doctor.add_argument( + "--link", action="store_true", help="prompt for and save gateway URL + token" + ) + + p_plugin = sub.add_parser("plugin", help="author / run a plugin") + psub = p_plugin.add_subparsers(dest="plugin_command") + p_init = psub.add_parser("init", help="scaffold a starter plugin") + p_init.add_argument("name", help="plugin name, e.g. my-rag") + # `run` serves the plugin AND registers it with tb (idempotent upsert on + # start), so there is no separate one-shot register command. + p_run = psub.add_parser("run", help="serve a plugin and register it with tb") + p_run.add_argument("target", help="e.g. my_rag_plugin:plugin or my_rag_plugin.py") + + args = parser.parse_args(argv) + if args.command == "doctor": + return doctor(args.scenario, args.link) + if args.command == "plugin": + if args.plugin_command == "init": + return _plugin_init(args.name) + if args.plugin_command == "run": + return _plugin_run(args.target) + p_plugin.print_help() + return 0 + + parser.print_help() + return 0 + + +_PLUGIN_TEMPLATE = '''"""A tingly-box plugin: an OpenAI-compatible AI server backed by the gateway.""" + +from tingly import Plugin + +plugin = Plugin(name="{name}") + + +@plugin.chat +def handle(req): + question = req.last_user_text() + # Your logic here. Call back into tingly-box for LLM work via plugin.llm: + return plugin.llm.ask(f"Answer concisely: {{question}}", model="auto") + + +if __name__ == "__main__": + plugin.serve() +''' + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sdk/python/tingly/client.py b/sdk/python/tingly/client.py new file mode 100644 index 000000000..91719dd0a --- /dev/null +++ b/sdk/python/tingly/client.py @@ -0,0 +1,235 @@ +"""The tingly Client and the ``connect()`` entrypoint. + +``connect()`` is the whole surface area for "experiment ASAP": auto-discover the +local gateway, mint a scenario-bound session, and hand back a Client whose +``.openai`` / ``.anthropic`` SDK objects are already pointed at tingly-box — so +every request inherits the gateway's routing, fallback, guard rails, quota and +logging without the experiment knowing anything about them. +""" + +from __future__ import annotations + +from typing import Any, Iterator, Optional + +from . import config as _config +from . import discovery as _discovery +from . import scenarios as _scenarios +from .errors import TinglyError +from .helpers.guardrails import GuardrailsView +from .helpers.usage import UsageView +from .transports import anthropic_compat, openai_compat + + +class Client: + """A tingly-box-bound LLM client. + + Construct via :func:`connect`, not directly. Holds the minted session (for + LLM calls) plus the gateway root + admin token (for management views). + """ + + def __init__( + self, + session: "_discovery.Session", + gateway_url: str, + admin_token: str, + name: str, + timeout: float, + ): + self._session = session + self._gateway_url = gateway_url + self._admin_token = admin_token + self.name = name + self._timeout = timeout + + self._openai: Optional[Any] = None + self._anthropic: Optional[Any] = None + + # -- identity -------------------------------------------------------- + + @property + def base_url(self) -> str: + return self._session.base_url + + @property + def scenario(self) -> str: + return self._session.scenario + + @property + def transport(self) -> str: + return self._session.transport + + @property + def ready(self) -> bool: + """True when the scenario has an active rule with at least one service.""" + return self._session.ready + + # -- SDK pass-throughs ---------------------------------------------- + + @property + def openai(self) -> Any: + """A lazily-built ``openai.OpenAI`` bound to this scenario.""" + if not _scenarios.supports_openai(self._session.transport): + raise TinglyError( + f"scenario {self.scenario!r} does not accept the OpenAI transport " + f"(transport={self.transport!r})" + ) + if self._openai is None: + self._openai = openai_compat.build_openai( + self._session.base_url, self._session.token, self._timeout + ) + return self._openai + + @property + def anthropic(self) -> Any: + """A lazily-built ``anthropic.Anthropic`` bound to this scenario.""" + if not _scenarios.supports_anthropic(self._session.transport): + raise TinglyError( + f"scenario {self.scenario!r} does not accept the Anthropic transport " + f"(transport={self.transport!r})" + ) + if self._anthropic is None: + self._anthropic = anthropic_compat.build_anthropic( + self._session.base_url, self._session.token, self._timeout + ) + return self._anthropic + + # -- convenience ----------------------------------------------------- + + def ask( + self, + prompt: str, + *, + model: str = "auto", + system: Optional[str] = None, + max_tokens: int = 1024, + stream: bool = False, + **kwargs: Any, + ): + """One-shot prompt → text, routed through tingly-box. + + Picks the transport from the scenario: Anthropic messages is tried + first (tb's native protocol); OpenAI-only scenarios fall back to chat + completions. ``model="auto"`` lets the gateway route. + """ + if _scenarios.supports_anthropic(self._session.transport): + return self._ask_anthropic(prompt, model, system, max_tokens, stream, **kwargs) + return self._ask_openai(prompt, model, system, stream, **kwargs) + + def _ask_openai(self, prompt, model, system, stream, **kwargs): + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + resp = self.openai.chat.completions.create( + model=model, messages=messages, stream=stream, **kwargs + ) + if stream: + return self._stream_openai(resp) + return resp.choices[0].message.content + + @staticmethod + def _stream_openai(resp) -> Iterator[str]: + for chunk in resp: + delta = chunk.choices[0].delta.content + if delta: + yield delta + + def _ask_anthropic(self, prompt, model, system, max_tokens, stream, **kwargs): + params = dict( + model=model, + max_tokens=max_tokens, + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + if system: + params["system"] = system + if stream: + return self._stream_anthropic(params) + resp = self.anthropic.messages.create(**params) + return "".join(block.text for block in resp.content if hasattr(block, "text")) + + def _stream_anthropic(self, params) -> Iterator[str]: + with self.anthropic.messages.stream(**params) as stream: + for text in stream.text_stream: + yield text + + # -- management views ------------------------------------------------ + + @property + def usage(self) -> UsageView: + return UsageView(self._gateway_url, self._admin_token, self.name, self._timeout) + + @property + def guardrails(self) -> GuardrailsView: + return GuardrailsView(self._gateway_url, self._admin_token, self._timeout) + + # -- lifecycle ------------------------------------------------------- + + def close(self) -> None: + for c in (self._openai, self._anthropic): + closer = getattr(c, "close", None) + if callable(closer): + try: + closer() + except Exception: + pass + + def __enter__(self) -> "Client": + return self + + def __exit__(self, *exc) -> None: + self.close() + + +def connect( + scenario: str = _scenarios.EXPERIMENT, + *, + base_url: Optional[str] = None, + token: Optional[str] = None, + timeout: float = 60.0, + name: Optional[str] = None, +) -> Client: + """Connect to a local tingly-box gateway and return a bound Client. + + Args: + scenario: the rule scenario to bind to. Defaults to ``"experiment"``. + base_url: gateway root, e.g. ``http://127.0.0.1:12580``. Auto-discovered + if omitted (env → ``~/.tingly-box/sdk.json`` → ``config.json`` → + localhost probe). + token: admin token used to provision the session. Auto-discovered if + omitted. + timeout: per-request timeout in seconds for the LLM clients. + name: a label that identifies this experiment in tingly-box logs. + + Raises: + GatewayUnreachableError: no gateway responded. + AuthError: the admin token was rejected. + ScenarioNotFoundError: the scenario is unknown or not bindable. + """ + resolved = _config.resolve(base_url=base_url, token=token) + + if _discovery.probe_version(resolved.base_url) is None: + from .errors import GatewayUnreachableError + + raise GatewayUnreachableError( + f"no tingly-box gateway responding at {resolved.base_url} " + f"(resolved via {resolved.source}). Is `tb` running? " + f"Set TINGLY_BOX_URL or run `tingly doctor`." + ) + + caller = name or "tingly-sdk" + session = _discovery.create_session( + base_url=resolved.base_url, + admin_token=resolved.token or "", + scenario=scenario, + name=caller, + timeout=timeout, + ) + + return Client( + session=session, + gateway_url=resolved.base_url, + admin_token=resolved.token or "", + name=caller, + timeout=timeout, + ) diff --git a/sdk/python/tingly/config.py b/sdk/python/tingly/config.py new file mode 100644 index 000000000..a9d9c452e --- /dev/null +++ b/sdk/python/tingly/config.py @@ -0,0 +1,164 @@ +"""Configuration resolution for the tingly SDK. + +Resolves ``(base_url, token)`` for connecting to a local tingly-box gateway, +following a fixed precedence so behaviour is predictable across dev and hosted +contexts: + + 1. Explicit arguments to ``connect()``. + 2. Environment: ``TINGLY_BOX_URL`` / ``TINGLY_BOX_TOKEN``. + 3. The SDK link file: ``~/.tingly-box/sdk.json`` (written by + ``tingly doctor --link`` or the tb UI "Connect SDK" panel). + 4. The tb config file: ``~/.tingly-box/config.json`` (admin ``UserToken``) + combined with a localhost probe. + +The ``token`` resolved here is the *admin* token used to provision an SDK +session via ``POST /api/v1/sdk/session``; the session itself hands back the +*model* token used for the actual LLM calls. + +This module performs no network I/O; the probe lives in ``discovery``. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +DEFAULT_PORT = 12580 +DEFAULT_HOST = "127.0.0.1" + +ENV_URL = "TINGLY_BOX_URL" +ENV_TOKEN = "TINGLY_BOX_TOKEN" + + +def config_dir() -> Path: + """Return the tingly-box config directory (``~/.tingly-box`` by default).""" + override = os.environ.get("TINGLY_BOX_HOME") + if override: + return Path(override) + return Path.home() / ".tingly-box" + + +def sdk_link_path() -> Path: + return config_dir() / "sdk.json" + + +def tb_config_path() -> Path: + return config_dir() / "config.json" + + +@dataclass +class Connection: + """An explicit, runtime-injected tb connection — for containers / CI / + remote where there is no ``~/.tingly-box``. + + Secrets may be given by **reference** (an env var name) so they are not + hard-coded; ``token`` resolves the literal or the env var at use time. + """ + + url: Optional[str] = None + admin_token: Optional[str] = None + admin_token_env: Optional[str] = None + + def token(self) -> Optional[str]: + if self.admin_token: + return self.admin_token + if self.admin_token_env: + return os.environ.get(self.admin_token_env) + return None + + +# Process-wide override set by configure(); highest precedence in resolve(). +_OVERRIDE: Optional[Connection] = None + + +def configure( + url: Optional[str] = None, + admin_token: Optional[str] = None, + admin_token_env: Optional[str] = None, +) -> None: + """Actively configure the tb target + credentials for this process. + + Takes precedence over env / files. Useful when a plugin must point at a + specific tb and use injected credentials rather than local discovery. + """ + global _OVERRIDE + _OVERRIDE = Connection(url=url, admin_token=admin_token, admin_token_env=admin_token_env) + + +@dataclass +class Resolved: + """A resolved gateway target plus where it came from (for diagnostics).""" + + base_url: Optional[str] + token: Optional[str] + source: str # "configure" | "args" | "env" | "sdk.json" | "config.json" | "probe-default" + + +def _read_json(path: Path) -> Optional[dict]: + try: + with path.open("r", encoding="utf-8") as fh: + return json.load(fh) + except (FileNotFoundError, ValueError, OSError): + return None + + +def resolve( + base_url: Optional[str] = None, + token: Optional[str] = None, +) -> Resolved: + """Resolve ``(base_url, token)`` by precedence, without touching the network. + + A missing piece is left as ``None`` for the caller (``discovery``) to fill — + e.g. an explicit ``base_url`` with no token still falls through to pick up a + token from env / files. + """ + src = "args" + + # Highest precedence: an explicit configure()/Connection override. + if _OVERRIDE is not None: + if base_url is None and _OVERRIDE.url: + base_url, src = _OVERRIDE.url, "configure" + if token is None and _OVERRIDE.token(): + token = _OVERRIDE.token() + if src == "args": + src = "configure" + + if base_url is None: + env_url = os.environ.get(ENV_URL) + if env_url: + base_url, src = env_url, "env" + if token is None: + env_token = os.environ.get(ENV_TOKEN) + if env_token: + token = env_token + if src == "args": + src = "env" + + if base_url is None or token is None: + link = _read_json(sdk_link_path()) + if link: + if base_url is None and link.get("base_url"): + base_url, src = link["base_url"], "sdk.json" + if token is None and link.get("token"): + token = link["token"] + if src == "args": + src = "sdk.json" + + if token is None: + cfg = _read_json(tb_config_path()) + if cfg: + # tb stores UserToken (admin) and ModelToken (LLM API key). We need + # the admin token to provision an SDK session. + token = cfg.get("UserToken") or cfg.get("user_token") + if token and src == "args": + src = "config.json" + + if base_url is None: + base_url = f"http://{DEFAULT_HOST}:{DEFAULT_PORT}" + if src == "args": + src = "probe-default" + + return Resolved(base_url=base_url, token=token, source=src) diff --git a/sdk/python/tingly/discovery.py b/sdk/python/tingly/discovery.py new file mode 100644 index 000000000..48181dfc1 --- /dev/null +++ b/sdk/python/tingly/discovery.py @@ -0,0 +1,97 @@ +"""Gateway discovery + SDK session minting. + +``connect()`` calls into here to turn a resolved ``(base_url, admin_token)`` +into a live :class:`Session`: it probes the gateway for liveness, then mints a +scenario-bound session via ``POST /api/v1/sdk/session``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import httpx + +from ._http import safe_json +from .errors import ( + AuthError, + GatewayUnreachableError, + ScenarioNotFoundError, +) + + +@dataclass +class Session: + """The minted SDK session — everything the transports need to bind.""" + + base_url: str # scenario root, e.g. http://127.0.0.1:12580/tingly/experiment + token: str # model token used for LLM calls + scenario: str + transport: str # "openai" | "anthropic" | "both" + ready: bool + services: int + + +def probe_version(base_url: str, timeout: float = 5.0) -> Optional[str]: + """Return a liveness marker if the gateway is reachable, else ``None``. + + Uses the unauthenticated ``/api/v1/info/health`` endpoint (the version + endpoint requires the admin token, so it cannot be used for discovery). + Returns the string ``"ok"`` when healthy — callers only check for truthiness. + """ + url = base_url.rstrip("/") + "/api/v1/info/health" + try: + resp = httpx.get(url, timeout=timeout) + except httpx.HTTPError: + return None + if resp.status_code != 200: + return None + return "ok" + + +def create_session( + base_url: str, + admin_token: str, + scenario: str, + name: Optional[str] = None, + timeout: float = 30.0, +) -> Session: + """Mint an SDK session against ``POST /api/v1/sdk/session``.""" + url = base_url.rstrip("/") + "/api/v1/sdk/session" + headers = {"Authorization": f"Bearer {admin_token}"} if admin_token else {} + body = {"scenario": scenario, "name": name or ""} + + try: + resp = httpx.post(url, json=body, headers=headers, timeout=timeout) + except httpx.HTTPError as exc: + raise GatewayUnreachableError( + f"could not reach tingly-box at {base_url}: {exc}" + ) from exc + + if resp.status_code == 401: + raise AuthError( + "tingly-box rejected the admin token. Set TINGLY_BOX_TOKEN or run " + "`tingly doctor --link`." + ) + + payload = safe_json(resp) + if resp.status_code == 404: + raise ScenarioNotFoundError( + scenario, (payload or {}).get("valid_scenarios") + ) + if resp.status_code != 200 or not payload or not payload.get("success"): + raise GatewayUnreachableError( + f"unexpected response from {url}: HTTP {resp.status_code} {resp.text[:200]}" + ) + + data = payload.get("data") or {} + return Session( + base_url=data["base_url"], + token=data.get("token", ""), + scenario=data.get("scenario", scenario), + transport=data.get("transport", "both"), + ready=bool(data.get("ready", False)), + services=int(data.get("services", 0)), + ) + + diff --git a/sdk/python/tingly/errors.py b/sdk/python/tingly/errors.py new file mode 100644 index 000000000..739cd8024 --- /dev/null +++ b/sdk/python/tingly/errors.py @@ -0,0 +1,64 @@ +"""Error hierarchy for the tingly SDK. + +Every failure a user can hit maps to one of these so plugin authors can react +precisely — and, in the guardrail case, explain to *their* user why a request +was refused rather than surfacing a bare "it failed". +""" + +from __future__ import annotations + +from typing import Optional + + +class TinglyError(Exception): + """Base class for all tingly SDK errors.""" + + +class GatewayUnreachableError(TinglyError): + """The tingly-box gateway could not be discovered or reached. + + Raised during ``connect()`` discovery or when a request cannot establish a + connection to the gateway. + """ + + +class AuthError(TinglyError): + """The gateway rejected the supplied token (HTTP 401).""" + + +class ScenarioNotFoundError(TinglyError): + """The requested scenario is unknown or not bindable (HTTP 404). + + ``valid_scenarios`` carries the gateway's list of bindable scenarios so the + caller can suggest a correct value. + """ + + def __init__(self, scenario: str, valid_scenarios: Optional[list] = None): + self.scenario = scenario + self.valid_scenarios = valid_scenarios or [] + valid = ", ".join(self.valid_scenarios) if self.valid_scenarios else "(none reported)" + super().__init__( + f"scenario {scenario!r} is unknown or not bindable. " + f"Valid scenarios: {valid}" + ) + + +class GuardrailBlockedError(TinglyError): + """tingly-box refused the request due to a guard-rail policy. + + ``policy_id`` and ``reason`` are surfaced so the caller can show *why*. + """ + + def __init__(self, reason: str, policy_id: Optional[str] = None): + self.policy_id = policy_id + self.reason = reason + prefix = f"[{policy_id}] " if policy_id else "" + super().__init__(f"{prefix}request blocked by guard rail: {reason}") + + +class UpstreamError(TinglyError): + """An upstream LLM provider returned a server error (HTTP 5xx).""" + + def __init__(self, message: str, status_code: Optional[int] = None): + self.status_code = status_code + super().__init__(message) diff --git a/sdk/python/tingly/helpers/__init__.py b/sdk/python/tingly/helpers/__init__.py new file mode 100644 index 000000000..8bd76b821 --- /dev/null +++ b/sdk/python/tingly/helpers/__init__.py @@ -0,0 +1 @@ +"""Observability + guard-rail helper views attached to a Client.""" diff --git a/sdk/python/tingly/helpers/guardrails.py b/sdk/python/tingly/helpers/guardrails.py new file mode 100644 index 000000000..b9ba66a89 --- /dev/null +++ b/sdk/python/tingly/helpers/guardrails.py @@ -0,0 +1,53 @@ +"""Guard-rail view — inspect what guard rails are active on the gateway. + +Guard rails in tingly-box run *inline* on the request path, so they are applied +automatically to every call an experiment makes — there is nothing the plugin +author must wire up. This view lets a plugin introspect the active policies +(e.g. to explain to its user what is enforced); a blocked request surfaces as a +:class:`~tingly.errors.GuardrailBlockedError` from the LLM call itself. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List + +import httpx + + +@dataclass +class GuardrailStatus: + enabled: bool = False + active_policies: int = 0 + policy_names: List[str] = None # type: ignore[assignment] + + def __post_init__(self): + if self.policy_names is None: + self.policy_names = [] + + +class GuardrailsView: + def __init__(self, gateway_url: str, admin_token: str, timeout: float): + self._gateway_url = gateway_url.rstrip("/") + self._admin_token = admin_token + self._timeout = timeout + + def status(self) -> GuardrailStatus: + """Return whether guard rails are enabled and how many policies are active.""" + url = f"{self._gateway_url}/api/v1/guardrails/config" + headers = {"Authorization": f"Bearer {self._admin_token}"} + try: + resp = httpx.get(url, headers=headers, timeout=self._timeout) + resp.raise_for_status() + payload = resp.json() + except (httpx.HTTPError, ValueError): + return GuardrailStatus() + + data = payload.get("data") or payload + policies = data.get("policies") or [] + names = [p.get("name", p.get("id", "?")) for p in policies if isinstance(p, dict)] + return GuardrailStatus( + enabled=bool(data.get("enabled", bool(policies))), + active_policies=len(policies), + policy_names=names, + ) diff --git a/sdk/python/tingly/helpers/usage.py b/sdk/python/tingly/helpers/usage.py new file mode 100644 index 000000000..336ef1b7b --- /dev/null +++ b/sdk/python/tingly/helpers/usage.py @@ -0,0 +1,58 @@ +"""Usage view — concrete token / request numbers, never aliases. + +v0.1 reads the gateway's request history (``/api/v1/requests``) filtered to this +SDK session's name. A dedicated per-session usage endpoint is a backend +follow-up; until then this gives real, inspectable numbers. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict + +import httpx + + +@dataclass +class UsageSummary: + requests: int = 0 + input_tokens: int = 0 + output_tokens: int = 0 + by_model: Dict[str, int] = field(default_factory=dict) + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + +class UsageView: + def __init__(self, gateway_url: str, admin_token: str, name: str, timeout: float): + self._gateway_url = gateway_url.rstrip("/") + self._admin_token = admin_token + self._name = name + self._timeout = timeout + + def this_session(self) -> UsageSummary: + """Return token / request totals for this session's named caller.""" + url = f"{self._gateway_url}/api/v1/requests" + headers = {"Authorization": f"Bearer {self._admin_token}"} + try: + resp = httpx.get(url, headers=headers, timeout=self._timeout) + resp.raise_for_status() + payload = resp.json() + except (httpx.HTTPError, ValueError): + return UsageSummary() + + records = payload.get("data") or payload.get("records") or [] + summary = UsageSummary() + for rec in records: + # Best-effort filter by the SDK caller name when the field exists. + source = rec.get("source") or rec.get("name") or "" + if self._name and self._name not in str(source): + continue + summary.requests += 1 + summary.input_tokens += int(rec.get("input_tokens", 0) or 0) + summary.output_tokens += int(rec.get("output_tokens", 0) or 0) + model = rec.get("model", "unknown") + summary.by_model[model] = summary.by_model.get(model, 0) + 1 + return summary diff --git a/sdk/python/tingly/plugin/__init__.py b/sdk/python/tingly/plugin/__init__.py new file mode 100644 index 000000000..f35149cd9 --- /dev/null +++ b/sdk/python/tingly/plugin/__init__.py @@ -0,0 +1,15 @@ +"""tingly.plugin — write an AI server tingly-box can route to as a model. + +This is Layer 2 of the SDK. A :class:`Plugin` is an upstream tb can call as +Anthropic Messages (primary) or OpenAI chat completions (secondary); register +it as a provider in tingly-box and any client can select ``model_id``, +inheriting the gateway's routing / fallback / guard rails / quota / logging. +""" + +from __future__ import annotations + +from .core import Plugin +from .manifest import Manifest +from .types import ChatRequest, Message + +__all__ = ["Plugin", "Manifest", "ChatRequest", "Message"] diff --git a/sdk/python/tingly/plugin/core.py b/sdk/python/tingly/plugin/core.py new file mode 100644 index 000000000..948434408 --- /dev/null +++ b/sdk/python/tingly/plugin/core.py @@ -0,0 +1,223 @@ +"""The ``Plugin`` class — write an AI server that tingly-box can route to. + +A plugin is an upstream tb can call two ways — Anthropic Messages (primary) +and OpenAI chat completions (secondary), both always served regardless of +registration — the author registers a single chat handler, and ``serve()`` +runs the HTTP server. tingly-box is then pointed at it as a provider +(``api_base = http://host:port``, model ``model_id``, ``api_style`` picking +which route tb calls), so the plugin composes with routing, fallback, guard +rails, quota and logging like any other model. + +The handler may call **back into** tingly-box via ``plugin.llm`` (a Layer-1 +:class:`~tingly.client.Client`) for its own LLM needs — the recursion shown in +the Layer 3 design graph. + + from tingly import Plugin + + plugin = Plugin(name="my-rag") + + @plugin.chat + def handle(req): + docs = retrieve(req.last_user_text()) + return plugin.llm.ask(f"Using {docs}, answer: {req.last_user_text()}") + + if __name__ == "__main__": + plugin.serve() +""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING, Callable, Optional + +from .manifest import Manifest +from .server import Dispatch, HandlerResult, make_server +from .types import ChatRequest + +if TYPE_CHECKING: + from ..config import Connection + +ChatHandler = Callable[[ChatRequest], HandlerResult] + + +class Plugin: + """An OpenAI-compatible AI server backed by the tingly-box gateway.""" + + def __init__( + self, + name: str, + *, + model_id: Optional[str] = None, + version: str = "0.1.0", + description: str = "", + api_key: str = "", + scenario: str = "experiment", + api_style: str = "anthropic", + ): + self.name = name + self.model_id = model_id or f"plugin/{name}" + self.version = version + self.description = description + self.api_key = api_key + self.scenario = scenario + # Which wire protocol tb should use to *call* this plugin + # (/v1/messages vs /v1/chat/completions) — the server answers both + # regardless, this only picks what registration advertises. + self.api_style = api_style + + self._handler: Optional[ChatHandler] = None + self._clients: dict = {} # scenario -> lazily-connected client + self._httpd = None + + # -- authoring ------------------------------------------------------- + + def chat(self, fn: ChatHandler) -> ChatHandler: + """Register the plugin's chat handler. Decorator form. + + The handler receives a :class:`ChatRequest` and returns either a string + (buffered) or an iterator of strings (streamed). + """ + self._handler = fn + return fn + + @property + def llm(self): + """A lazily-connected client for calling back into tingly-box. + + This is the plugin's default calling context (``self.scenario``). The + plugin reuses the gateway for its own model calls, so it never hard-codes + a provider or key — and ``ask(model=...)`` can target *any* model tb + routes. To drive a *different* rule-set, use :meth:`use`. + """ + return self.use(self.scenario) + + def use(self, scenario: str): + """Return a client bound to a specific scenario (rule-set) in tb. + + A plugin composes the box: it can hold clients to several scenarios and + pick a model on each, so "the plugin can use any other rule / model + configured in tb" is one call: + + self.use("claude_code").ask("…", model="claude-sonnet-4-6") + self.use("experiment").ask("…", model="auto") + """ + client = self._clients.get(scenario) + if client is None: + from ..client import connect + + client = connect(scenario=scenario, name=f"plugin:{self.name}") + self._clients[scenario] = client + return client + + # -- dispatch -------------------------------------------------------- + + def _dispatch(self, req: ChatRequest) -> HandlerResult: + if self._handler is None: + raise RuntimeError( + f"plugin {self.name!r} has no chat handler; decorate one with @plugin.chat" + ) + return self._handler(req) + + # -- manifest -------------------------------------------------------- + + def manifest(self, entrypoint: str, port: int = 8765, transport: Optional[str] = None) -> Manifest: + """Build a :class:`Manifest` describing this plugin for tingly-box. + + ``transport`` defaults to :attr:`api_style`. + """ + return Manifest( + name=self.name, + model_id=self.model_id, + entrypoint=entrypoint, + version=self.version, + transport=transport or self.api_style, + port=port, + description=self.description, + ) + + # -- serving --------------------------------------------------------- + + def serve( + self, + host: str = "127.0.0.1", + port: int = 8765, + *, + verbose: bool = True, + block: bool = True, + register: bool = True, + advertise_host: Optional[str] = None, + tb: Optional["Connection"] = None, + ) -> int: + """Run the plugin's HTTP server and (by default) register it with tb. + + Registration is a one-shot, idempotent upsert by name: tb creates or + updates a normal provider for this plugin (and the rule, if + ``scenario`` was set on the constructor). There is no heartbeat or + lease — once the plugin is registered it stays configured in tb until + someone deletes it, the same as any other provider. Liveness is + handled by tb's existing per-service circuit breaker: if the plugin + goes down, the next failed request trips it and traffic tier-fails- + over (when a fallback tier is configured). + + ``tb`` may be a :class:`tingly.config.Connection` to point at a + specific gateway / inject credentials (containers / CI / remote). + + Returns the bound port (resolved even when ``port=0``). ``block=False`` + runs the server on a daemon thread and returns immediately. + """ + if tb is not None: + from ..config import configure + + configure(url=tb.url, admin_token=tb.admin_token, admin_token_env=tb.admin_token_env) + + httpd, bound = make_server( + self._dispatch, + self.model_id, + host=host, + port=port, + api_key=self.api_key, + verbose=verbose, + ) + self._httpd = httpd + if verbose: + print( + f"[tingly] plugin {self.name!r} serving model {self.model_id!r} " + f"on http://{host}:{bound}/v1" + ) + + if register: + self._register(advertise_host or host, bound, verbose) + + if not block: + t = threading.Thread(target=httpd.serve_forever, daemon=True) + t.start() + return bound + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + finally: + self.stop() + return bound + + def _register(self, host: str, port: int, verbose: bool) -> None: + from .register import register + + endpoint = f"http://{host}:{port}/v1" + try: + result = register( + self.name, endpoint, self.model_id, + scenario=self.scenario, token=self.api_key, + api_style=self.api_style, + ) + except Exception as exc: # noqa: BLE001 - registration is best-effort + if verbose: + print(f"[tingly] plugin registration skipped: {exc}") + return + if verbose: + print(f"[tingly] {result.note}") + + def stop(self) -> None: + if self._httpd is not None: + self._httpd.shutdown() + self._httpd = None diff --git a/sdk/python/tingly/plugin/manifest.py b/sdk/python/tingly/plugin/manifest.py new file mode 100644 index 000000000..e38e2e04f --- /dev/null +++ b/sdk/python/tingly/plugin/manifest.py @@ -0,0 +1,95 @@ +"""``tingly.toml`` plugin manifest — read/write. + +The manifest is what tingly-box (Layer 2 supervisor) reads to install and run a +plugin, and what `tingly plugin register` uses to wire the plugin in as an +upstream provider. It is deliberately tiny: + + [plugin] + name = "my-rag" + model_id = "plugin/my-rag" + version = "0.1.0" + entrypoint = "rag_plugin:plugin" # module:attr that yields a Plugin + transport = "anthropic" # anthropic (primary) | openai (secondary) — the + # wire protocol tb should use to call this plugin; + # the server always answers both regardless + port = 8765 + description = "Answers from my private corpus" +""" + +from __future__ import annotations + +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Optional + +if sys.version_info >= (3, 11): + import tomllib # type: ignore +else: # pragma: no cover - 3.9/3.10 fallback + try: + import tomli as tomllib # type: ignore + except ImportError: + tomllib = None # type: ignore + +MANIFEST_NAME = "tingly.toml" + + +@dataclass +class Manifest: + name: str + model_id: str + entrypoint: str + version: str = "0.1.0" + transport: str = "anthropic" + port: int = 8765 + description: str = "" + + def to_toml(self) -> str: + d = asdict(self) + lines = ["[plugin]"] + for key in ("name", "model_id", "version", "entrypoint", "transport", "description"): + lines.append(f'{key} = "{_escape(str(d[key]))}"') + lines.append(f"port = {int(d['port'])}") + return "\n".join(lines) + "\n" + + def write(self, directory: Path) -> Path: + path = Path(directory) / MANIFEST_NAME + path.write_text(self.to_toml(), encoding="utf-8") + return path + + +def load(path: Path) -> Manifest: + """Load a manifest from a ``tingly.toml`` file or its containing directory.""" + p = Path(path) + if p.is_dir(): + p = p / MANIFEST_NAME + if tomllib is None: # pragma: no cover + raise RuntimeError( + "reading tingly.toml needs Python 3.11+ or the `tomli` package" + ) + with p.open("rb") as fh: + data = tomllib.load(fh) + plugin = data.get("plugin") or {} + return Manifest( + name=plugin["name"], + model_id=plugin.get("model_id", f"plugin/{plugin['name']}"), + entrypoint=plugin["entrypoint"], + version=plugin.get("version", "0.1.0"), + transport=plugin.get("transport", "anthropic"), + port=int(plugin.get("port", 8765)), + description=plugin.get("description", ""), + ) + + +def find(start: Optional[Path] = None) -> Optional[Manifest]: + """Search upward from ``start`` (cwd by default) for a ``tingly.toml``.""" + cur = Path(start or Path.cwd()).resolve() + for directory in [cur, *cur.parents]: + candidate = directory / MANIFEST_NAME + if candidate.exists(): + return load(candidate) + return None + + +def _escape(s: str) -> str: + return s.replace("\\", "\\\\").replace('"', '\\"') diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py new file mode 100644 index 000000000..5faea143d --- /dev/null +++ b/sdk/python/tingly/plugin/register.py @@ -0,0 +1,81 @@ +"""Register a running plugin with tingly-box. + +A plugin registers once at startup (idempotent upsert by name — calling it +again, e.g. on every restart, updates the existing provider rather than +duplicating it). There is no heartbeat or lease: liveness is handled by the +same per-service circuit breaker that already protects every other tb +provider — if the plugin goes down, the next failed request trips the breaker +and traffic tier-fails-over (when a fallback tier is configured). If a plugin +is retired, delete its provider like any other, in the tb UI. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import httpx + +from .. import config as _config +from .._http import safe_json +from ..errors import AuthError, GatewayUnreachableError + + +@dataclass +class RegisterResult: + provider_uuid: str + model_id: str + scenario: Optional[str] + rule_uuid: Optional[str] + ready: bool + note: str + + +def register( + name: str, + endpoint: str, + model_id: str, + *, + scenario: Optional[str] = None, + token: str = "", + tier: int = 0, + api_style: str = "anthropic", + gateway_url: Optional[str] = None, + admin_token: Optional[str] = None, + timeout: float = 30.0, +) -> RegisterResult: + """Register (or update) this plugin as a tb provider, optionally binding a rule. + + ``api_style`` tells tb which wire protocol to use when it calls this + plugin's endpoint — ``"anthropic"`` (the plugin server's primary route, + ``/v1/messages``) or ``"openai"`` (``/v1/chat/completions``). The plugin + server answers both regardless; this only picks which one tb sends. + """ + resolved = _config.resolve(base_url=gateway_url, token=admin_token) + headers = {"Authorization": f"Bearer {resolved.token or ''}"} + url = resolved.base_url.rstrip("/") + "/api/v2/plugins" + body = { + "name": name, "endpoint": endpoint, "model_id": model_id, + "scenario": scenario or "", "token": token, "tier": tier, + "api_style": api_style, + } + try: + resp = httpx.post(url, json=body, headers=headers, timeout=timeout) + except httpx.HTTPError as exc: + raise GatewayUnreachableError(f"could not reach tingly-box: {exc}") from exc + if resp.status_code == 401: + raise AuthError("tingly-box rejected the admin token during plugin register") + payload = safe_json(resp) or {} + if resp.status_code != 200 or not payload.get("success"): + raise GatewayUnreachableError( + f"plugin register failed: HTTP {resp.status_code} {resp.text[:200]}" + ) + d = payload.get("data") or {} + return RegisterResult( + provider_uuid=d.get("provider_uuid", ""), + model_id=d.get("model_id", model_id), + scenario=d.get("scenario") or None, + rule_uuid=d.get("rule_uuid") or None, + ready=bool(d.get("ready", False)), + note=d.get("note", ""), + ) diff --git a/sdk/python/tingly/plugin/server.py b/sdk/python/tingly/plugin/server.py new file mode 100644 index 000000000..e5bd981d8 --- /dev/null +++ b/sdk/python/tingly/plugin/server.py @@ -0,0 +1,290 @@ +"""A tiny HTTP server for plugins (stdlib only), speaking two wire protocols. + +Exposes exactly what tingly-box needs to treat the plugin as an upstream: + + POST /v1/messages -> Anthropic message (+ SSE when stream=true) — primary + POST /v1/chat/completions -> OpenAI chat.completion (+ SSE) — secondary + GET /v1/models -> the plugin's model id + GET /health -> liveness + +Anthropic is primary because that is tingly-box's native protocol; OpenAI +chat completions is kept as a secondary, equally-real path (not a shim) since +many callers still speak it. Both routes share the same normalized +:class:`ChatRequest` and the same handler — the author's code never sees the +wire-format difference, only the response shaping differs per route. + +No framework dependency — uses ``http.server.ThreadingHTTPServer`` so a plugin +stays a single ``pip install tingly`` away. Streaming is real SSE: a handler +that returns an iterator is emitted as protocol-appropriate chunk/event frames. +""" + +from __future__ import annotations + +import json +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Callable, Dict, Iterable, Iterator, Tuple, Union +from urllib.parse import urlsplit + +from .types import ChatRequest + +# A handler returns buffered text or an iterator of text deltas. +HandlerResult = Union[str, Iterable[str]] +Dispatch = Callable[[ChatRequest], HandlerResult] + + +class _Handler(BaseHTTPRequestHandler): + # set by make_server via closure attributes on the server instance + server_version = "tingly-plugin/0.1" + + def log_message(self, fmt, *args): # quiet by default; plugin owns logging + if getattr(self.server, "verbose", False): + super().log_message(fmt, *args) + + # -- routing --------------------------------------------------------- + + def _route_path(self) -> str: + """The request path with any query string (e.g. tb's ``?beta=true`` + on Anthropic calls) and trailing slash stripped.""" + return urlsplit(self.path).path.rstrip("/") + + def do_GET(self): + path = self._route_path() + if path == "/health": + return self._json(200, {"status": "ok"}) + if path in ("/v1/models", "/models"): + return self._models() + return self._json(404, {"error": {"message": "not found", "type": "not_found"}}) + + def do_POST(self): + path = self._route_path() + if path in ("/v1/messages", "/messages"): + return self._handle_chat(anthropic=True) + if path in ("/v1/chat/completions", "/chat/completions"): + return self._handle_chat(anthropic=False) + return self._json(404, {"error": {"message": "not found", "type": "not_found"}}) + + def _handle_chat(self, anthropic: bool): + if not self._authorized(): + if anthropic: + return self._json( + 401, {"type": "error", "error": {"type": "authentication_error", "message": "invalid token"}} + ) + return self._json( + 401, {"error": {"message": "invalid token", "type": "auth_error"}} + ) + body = self._read_json() + if body is None: + if anthropic: + return self._json( + 400, {"type": "error", "error": {"type": "invalid_request_error", "message": "invalid JSON body"}} + ) + return self._json( + 400, {"error": {"message": "invalid JSON body", "type": "invalid_request_error"}} + ) + + req = ( + ChatRequest.from_anthropic_body(body) if anthropic else ChatRequest.from_openai_body(body) + ) + try: + result = self.server.dispatch(req) # type: ignore[attr-defined] + except Exception as exc: # noqa: BLE001 - surface as upstream 500 + if anthropic: + return self._json( + 500, {"type": "error", "error": {"type": "api_error", "message": f"plugin handler error: {exc}"}} + ) + return self._json( + 500, + {"error": {"message": f"plugin handler error: {exc}", "type": "api_error"}}, + ) + + model = req.model or self.server.model_id # type: ignore[attr-defined] + if anthropic: + return self._stream_anthropic(result, model) if req.stream else self._complete_anthropic(result, model) + return self._stream(result, model) if req.stream else self._complete(result, model) + + # -- responses ------------------------------------------------------- + + def _models(self): + model_id = self.server.model_id # type: ignore[attr-defined] + self._json( + 200, + { + "object": "list", + "data": [ + { + "id": model_id, + "object": "model", + "created": int(time.time()), + "owned_by": "tingly-plugin", + } + ], + }, + ) + + def _complete(self, result: HandlerResult, model: str): + text = result if isinstance(result, str) else "".join(result) + self._json( + 200, + { + "id": _chat_id(), + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + }, + ) + + def _stream(self, result: HandlerResult, model: str): + cid = _chat_id() + created = int(time.time()) + # Close-delimited SSE: no Content-Length, connection closes at end so the + # client's stream iterator terminates after the [DONE] frame. + self.close_connection = True + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + + def frame(delta: Dict[str, Any], finish: Any = None) -> bytes: + payload = { + "id": cid, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + return f"data: {json.dumps(payload)}\n\n".encode("utf-8") + + # role preamble, then content deltas, then terminal frame + [DONE] + self.wfile.write(frame({"role": "assistant"})) + chunks: Iterator[str] = iter([result]) if isinstance(result, str) else iter(result) + for piece in chunks: + if piece: + self.wfile.write(frame({"content": piece})) + self.wfile.flush() + self.wfile.write(frame({}, finish="stop")) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + def _complete_anthropic(self, result: HandlerResult, model: str): + text = result if isinstance(result, str) else "".join(result) + self._json( + 200, + { + "id": _msg_id(), + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + }, + ) + + def _stream_anthropic(self, result: HandlerResult, model: str): + mid = _msg_id() + self.close_connection = True + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + + def event(name: str, payload: Dict[str, Any]) -> bytes: + return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode("utf-8") + + self.wfile.write(event("message_start", { + "type": "message_start", + "message": { + "id": mid, "type": "message", "role": "assistant", "model": model, + "content": [], "stop_reason": None, "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + }, + })) + self.wfile.write(event("content_block_start", { + "type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": ""}, + })) + chunks: Iterator[str] = iter([result]) if isinstance(result, str) else iter(result) + for piece in chunks: + if piece: + self.wfile.write(event("content_block_delta", { + "type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": piece}, + })) + self.wfile.flush() + self.wfile.write(event("content_block_stop", {"type": "content_block_stop", "index": 0})) + self.wfile.write(event("message_delta", { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 0}, + })) + self.wfile.write(event("message_stop", {"type": "message_stop"})) + self.wfile.flush() + + # -- helpers --------------------------------------------------------- + + def _authorized(self) -> bool: + expected = self.server.api_key # type: ignore[attr-defined] + if not expected: + return True + header = self.headers.get("Authorization", "") + token = header[7:] if header.startswith("Bearer ") else header + return token.strip() == expected + + def _read_json(self): + try: + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length) if length else b"" + return json.loads(raw or b"{}") + except (ValueError, OSError): + return None + + def _json(self, status: int, payload: Dict[str, Any]): + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +def _chat_id() -> str: + return "chatcmpl-" + uuid.uuid4().hex[:24] + + +def _msg_id() -> str: + return "msg_" + uuid.uuid4().hex[:24] + + +def make_server( + dispatch: Dispatch, + model_id: str, + host: str, + port: int, + api_key: str = "", + verbose: bool = False, +) -> Tuple[ThreadingHTTPServer, int]: + """Build (but do not start) the plugin's HTTP server. + + Returns the server and the resolved port (useful when ``port=0`` asks the + OS for an ephemeral one — handy in tests). + """ + httpd = ThreadingHTTPServer((host, port), _Handler) + # stash config on the server instance for the handler to read + httpd.dispatch = dispatch # type: ignore[attr-defined] + httpd.model_id = model_id # type: ignore[attr-defined] + httpd.api_key = api_key # type: ignore[attr-defined] + httpd.verbose = verbose # type: ignore[attr-defined] + return httpd, httpd.server_address[1] diff --git a/sdk/python/tingly/plugin/types.py b/sdk/python/tingly/plugin/types.py new file mode 100644 index 000000000..c757646e8 --- /dev/null +++ b/sdk/python/tingly/plugin/types.py @@ -0,0 +1,103 @@ +"""Request/response value types passed to a plugin's chat handler. + +The handler sees a normalized :class:`ChatRequest` regardless of wire details, +and returns either a ``str`` (buffered) or an iterator of ``str`` (streamed) — +the server shapes those into OpenAI ``chat.completion`` / ``chat.completion.chunk`` +payloads so tingly-box (or any OpenAI client) can consume the plugin as an +upstream model. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class Message: + role: str + content: str + + +@dataclass +class ChatRequest: + """A normalized chat request handed to a plugin handler.""" + + model: str + messages: List[Message] + stream: bool = False + # Everything else from the wire body, untouched (temperature, tools, …). + extra: Dict[str, Any] = field(default_factory=dict) + + def last_user_text(self) -> str: + """The text of the most recent user message (empty string if none).""" + for msg in reversed(self.messages): + if msg.role == "user": + return msg.content + return "" + + def system_text(self) -> Optional[str]: + for msg in self.messages: + if msg.role == "system": + return msg.content + return None + + @classmethod + def from_openai_body(cls, body: Dict[str, Any]) -> "ChatRequest": + raw_messages = body.get("messages") or [] + messages = [ + Message(role=m.get("role", "user"), content=_content_to_text(m.get("content"))) + for m in raw_messages + ] + known = {"model", "messages", "stream"} + extra = {k: v for k, v in body.items() if k not in known} + return cls( + model=body.get("model", ""), + messages=messages, + stream=bool(body.get("stream", False)), + extra=extra, + ) + + @classmethod + def from_anthropic_body(cls, body: Dict[str, Any]) -> "ChatRequest": + """Normalize an Anthropic Messages API body (POST /v1/messages). + + Anthropic keeps the system prompt in a top-level ``system`` field + rather than a message with ``role: "system"``; it is folded into the + message list as a leading system message so handlers can use + :meth:`system_text` / :meth:`last_user_text` the same way regardless + of which wire protocol the caller used. + """ + raw_messages = body.get("messages") or [] + messages = [ + Message(role=m.get("role", "user"), content=_content_to_text(m.get("content"))) + for m in raw_messages + ] + system_text = _content_to_text(body.get("system")) + if system_text: + messages.insert(0, Message(role="system", content=system_text)) + known = {"model", "messages", "system", "stream"} + extra = {k: v for k, v in body.items() if k not in known} + return cls( + model=body.get("model", ""), + messages=messages, + stream=bool(body.get("stream", False)), + extra=extra, + ) + + +def _content_to_text(content: Any) -> str: + """Flatten OpenAI content (str or list of parts) to plain text.""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for part in content: + if isinstance(part, dict): + parts.append(part.get("text", "")) + else: + parts.append(str(part)) + return "".join(parts) + return str(content) diff --git a/sdk/python/tingly/scenarios.py b/sdk/python/tingly/scenarios.py new file mode 100644 index 000000000..553db91a7 --- /dev/null +++ b/sdk/python/tingly/scenarios.py @@ -0,0 +1,30 @@ +"""Scenario + transport constants mirrored from the tingly-box backend. + +These are the bindable scenarios most relevant to SDK users. The authoritative +list lives in ``internal/typ/type.go``; the gateway validates the scenario at +session time and returns the accepted transport, so this module is for +convenience and is not load-bearing. +""" + +from __future__ import annotations + +# Default scenario for experiments / plugins. +EXPERIMENT = "experiment" + +# Other commonly useful bindable scenarios. +OPENAI = "openai" +ANTHROPIC = "anthropic" +AGENT = "agent" + +# Transport labels returned by the gateway session response. +TRANSPORT_OPENAI = "openai" +TRANSPORT_ANTHROPIC = "anthropic" +TRANSPORT_BOTH = "both" + + +def supports_openai(transport: str) -> bool: + return transport in (TRANSPORT_OPENAI, TRANSPORT_BOTH) + + +def supports_anthropic(transport: str) -> bool: + return transport in (TRANSPORT_ANTHROPIC, TRANSPORT_BOTH) diff --git a/sdk/python/tingly/transports/__init__.py b/sdk/python/tingly/transports/__init__.py new file mode 100644 index 000000000..dc0834bdc --- /dev/null +++ b/sdk/python/tingly/transports/__init__.py @@ -0,0 +1 @@ +"""Transport builders — bind the user's preferred LLM SDK to tingly-box.""" diff --git a/sdk/python/tingly/transports/anthropic_compat.py b/sdk/python/tingly/transports/anthropic_compat.py new file mode 100644 index 000000000..222d500e3 --- /dev/null +++ b/sdk/python/tingly/transports/anthropic_compat.py @@ -0,0 +1,43 @@ +"""Build an ``anthropic.Anthropic`` client bound to a tingly-box scenario. + +The Anthropic Python SDK appends ``/v1/messages`` to its ``base_url``, so we +target the scenario root *without* a version segment: e.g. +``http://127.0.0.1:12580/tingly/experiment`` → ``…/tingly/experiment/v1/messages``, +which matches the gateway's ``/tingly/:scenario/v1`` route group. +""" + +from __future__ import annotations + +from typing import Any + + +def build_anthropic(base_url: str, token: str, timeout: float) -> Any: + try: + import anthropic + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The Anthropic transport requires the `anthropic` package. " + "Reinstall tingly (it ships with anthropic)." + ) from exc + + return anthropic.Anthropic( + base_url=base_url.rstrip("/"), + api_key=token or "tingly-box", + timeout=timeout, + ) + + +def build_async_anthropic(base_url: str, token: str, timeout: float) -> Any: + try: + import anthropic + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The Anthropic transport requires the `anthropic` package. " + "Reinstall tingly (it ships with anthropic)." + ) from exc + + return anthropic.AsyncAnthropic( + base_url=base_url.rstrip("/"), + api_key=token or "tingly-box", + timeout=timeout, + ) diff --git a/sdk/python/tingly/transports/openai_compat.py b/sdk/python/tingly/transports/openai_compat.py new file mode 100644 index 000000000..0407b17d8 --- /dev/null +++ b/sdk/python/tingly/transports/openai_compat.py @@ -0,0 +1,44 @@ +"""Build an ``openai.OpenAI`` client bound to a tingly-box scenario. + +The OpenAI Python SDK appends ``/chat/completions`` (etc.) to its ``base_url`` +and expects the version segment to be part of it, so we target the scenario +root + ``/v1``: e.g. ``http://127.0.0.1:12580/tingly/experiment/v1`` → +``…/tingly/experiment/v1/chat/completions``, which matches the gateway's +``/tingly/:scenario/v1`` route group. +""" + +from __future__ import annotations + +from typing import Any + + +def build_openai(base_url: str, token: str, timeout: float) -> Any: + try: + import openai + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The OpenAI transport requires the `openai` package. " + "Reinstall tingly (it ships with openai)." + ) from exc + + return openai.OpenAI( + base_url=base_url.rstrip("/") + "/v1", + api_key=token or "tingly-box", + timeout=timeout, + ) + + +def build_async_openai(base_url: str, token: str, timeout: float) -> Any: + try: + import openai + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The OpenAI transport requires the `openai` package. " + "Reinstall tingly (it ships with openai)." + ) from exc + + return openai.AsyncOpenAI( + base_url=base_url.rstrip("/") + "/v1", + api_key=token or "tingly-box", + timeout=timeout, + )