From c37532b316cefd4b679477e8757a7ec6a5530f16 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 07:21:14 +0000 Subject: [PATCH 01/28] feat(sdk): add tingly Python module + /sdk/session gateway seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce Layer 1 of the plugin SDK: a pip-installable `tingly` module that lets a user write an LLM experiment or plugin in ~10 lines while reusing the tingly-box gateway (routing, fallback, guard rails, quota, logging). Backend: - New `experiment` scenario (OpenAI + Anthropic transports, rule-bindable, profile-capable) so SDK traffic gets its own isolated rule. - New `POST /api/v1/sdk/session` endpoint that mints a scenario-bound session (base_url + model token + accepted transports + readiness) from the admin token. No new LLM routes needed — /tingly/:scenario is already dynamic. - Tests freezing the response JSON contract and the experiment descriptor. - Regenerated openapi.json. Python (sdk/python/): - connect() with discovery precedence (args → env → sdk.json → config.json → localhost probe), Client with .openai/.anthropic/.ask/.usage/.guardrails, transport builders, error hierarchy, and a `tingly doctor` CLI that traverses the real path. - Offline test suite (config, discovery via respx, transports, client). Design notes in .design/python-sdk.md. --- .design/python-sdk.md | 125 ++++++++++ internal/server/sdk_session.go | 165 ++++++++++++ internal/server/sdk_session_test.go | 72 ++++++ internal/server/server_webui_api.go | 9 + internal/typ/scenario_registry.go | 11 + internal/typ/scenario_registry_test.go | 20 ++ internal/typ/type.go | 6 +- sdk/python/README.md | 60 +++++ sdk/python/examples/rag_experiment.py | 54 ++++ sdk/python/pyproject.toml | 42 ++++ sdk/python/tests/test_client_offline.py | 66 +++++ sdk/python/tests/test_config.py | 57 +++++ sdk/python/tests/test_discovery.py | 87 +++++++ sdk/python/tests/test_transports.py | 47 ++++ sdk/python/tingly/__init__.py | 34 +++ sdk/python/tingly/_version.py | 1 + sdk/python/tingly/cli.py | 140 +++++++++++ sdk/python/tingly/client.py | 236 ++++++++++++++++++ sdk/python/tingly/config.py | 115 +++++++++ sdk/python/tingly/discovery.py | 132 ++++++++++ sdk/python/tingly/errors.py | 64 +++++ sdk/python/tingly/helpers/__init__.py | 1 + sdk/python/tingly/helpers/guardrails.py | 53 ++++ sdk/python/tingly/helpers/usage.py | 58 +++++ sdk/python/tingly/scenarios.py | 30 +++ sdk/python/tingly/transports/__init__.py | 1 + .../tingly/transports/anthropic_compat.py | 43 ++++ sdk/python/tingly/transports/openai_compat.py | 44 ++++ 28 files changed, 1771 insertions(+), 2 deletions(-) create mode 100644 .design/python-sdk.md create mode 100644 internal/server/sdk_session.go create mode 100644 internal/server/sdk_session_test.go create mode 100644 sdk/python/README.md create mode 100644 sdk/python/examples/rag_experiment.py create mode 100644 sdk/python/pyproject.toml create mode 100644 sdk/python/tests/test_client_offline.py create mode 100644 sdk/python/tests/test_config.py create mode 100644 sdk/python/tests/test_discovery.py create mode 100644 sdk/python/tests/test_transports.py create mode 100644 sdk/python/tingly/__init__.py create mode 100644 sdk/python/tingly/_version.py create mode 100644 sdk/python/tingly/cli.py create mode 100644 sdk/python/tingly/client.py create mode 100644 sdk/python/tingly/config.py create mode 100644 sdk/python/tingly/discovery.py create mode 100644 sdk/python/tingly/errors.py create mode 100644 sdk/python/tingly/helpers/__init__.py create mode 100644 sdk/python/tingly/helpers/guardrails.py create mode 100644 sdk/python/tingly/helpers/usage.py create mode 100644 sdk/python/tingly/scenarios.py create mode 100644 sdk/python/tingly/transports/__init__.py create mode 100644 sdk/python/tingly/transports/anthropic_compat.py create mode 100644 sdk/python/tingly/transports/openai_compat.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md new file mode 100644 index 000000000..a6646e614 --- /dev/null +++ b/.design/python-sdk.md @@ -0,0 +1,125 @@ +# Python SDK (`tingly`) — design + +> Audience: tingly-box contributors touching the SDK seam (`sdk/python/`), the +> `/api/v1/sdk/session` endpoint, or the `experiment` scenario. + +## Why + +tb is a capable personal-intelligence gateway, but extending or experimenting +on top of it meant either editing the Go backend or hand-rolling HTTP calls +with the right base URL, token and scenario path. There was no fast seam for +"I have an idea, let me try it against my box in ten lines". + +`tingly` is that seam: a pip module where the user writes only their own logic +(prompt, retrieval, agent loop) and **reuses the gateway's power** — provider +routing, tier/fallback, guard rails, quota, logging — for free. + +This is **Layer 1** (client-side library). It deliberately ships before Layer 2 +(tb-hosted plugins with a manifest + sub-process supervision) and Layer 3 +(plugin-as-virtual-model via `vmodel/virtualserver`), both of which build on +this same module and the same `/sdk/session` provisioning seam. + +## Shape + +``` +sdk/python/ + tingly/ + client.py # Client + connect() ← the whole user surface + discovery.py # probe gateway + POST /sdk/session + config.py # (base_url, admin_token) resolution precedence + scenarios.py # scenario + transport constants + transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb + helpers/ # usage + guardrails views + cli.py # `tingly doctor` + errors.py # TinglyError hierarchy +``` + +## Request flow + +``` +connect(scenario="experiment") + │ + ├─ config.resolve() args → env → ~/.tingly-box/sdk.json → config.json → localhost + ├─ discovery.probe_version() GET /api/v1/info/version (liveness) + ├─ discovery.create_session() POST /api/v1/sdk/session (admin token → model token) + └─ Client(session, gateway_url, admin_token) + .openai → openai.OpenAI(base_url = scenario_root + "/v1") + .anthropic → anthropic.Anthropic(base_url = scenario_root) + .ask() → picks transport from session.transport + .usage → GET /api/v1/requests (admin token) + .guardrails → GET /api/v1/guardrails/config (admin token) +``` + +## Two-token model + +- **Admin token** (tb's `UserToken`): authorizes `POST /sdk/session`. Resolved + from `TINGLY_BOX_TOKEN` / `sdk.json` / `config.json:UserToken`. Provisioning + requires admin rights. +- **Model token** (tb's `ModelToken`): returned *by* the session, used as the + bearer for the actual LLM calls. The OpenAI/Anthropic clients carry this, not + the admin token. + +In v0.1 the session returns the existing long-lived model token (same as +`tbclient.GetConnectionConfig` / `GetClaudeCodeEnv` already do). Short-lived +scoped tokens (`expires_at`) are the obvious follow-up — the response field is +already present and `omitempty`. + +## Gateway seam: `POST /api/v1/sdk/session` + +Handler: `internal/server/sdk_session.go` (`CreateSDKSession`), registered in +`webui_api.go` under the authenticated `apiV1` group (so it needs the admin +token). + +Request `{ scenario, name }` → response +`{ base_url, token, scenario, transport, ready, services, expires_at? }`. + +- `base_url` is the scenario root `http://host:port/tingly/`. Bind + host `0.0.0.0`/`::` is rewritten to `127.0.0.1` so it's client-usable. +- `transport` is `openai`|`anthropic`|`both`, collapsed from the scenario + descriptor's `SupportedTransport`. +- `ready`/`services` report whether an active rule with ≥1 service is bound, so + `tingly doctor` can tell the user the next action instead of failing opaquely. +- Unknown / non-bindable scenario → 404 with `valid_scenarios` in the body. + +No new routes were needed for the LLM calls themselves: `/tingly/:scenario` and +`/tingly/:scenario/v1` are already dynamic, so `experiment` flows through the +existing mixin endpoints (`chat/completions`, `messages`, `responses`, …). + +## The `experiment` scenario + +Added to `internal/typ/type.go` (`ScenarioExperiment = "experiment"`) and the +descriptor registry (`scenario_registry.go`): OpenAI + Anthropic transports, +rule-bindable, path-usable, profile-capable. It exists so SDK traffic has its +own isolated rule instead of polluting `claude_code` / `openai` rules — and so +users can name parallel experiments via profiles (`experiment:p1`). + +## UX-principles alignment + +- **No mode picker.** `connect()` is identical in dev and (future) hosted + contexts; the environment decides discovery, not the user. +- **Smart defaults.** `scenario="experiment"`, `model="auto"`. +- **Concrete values.** `usage.this_session()` returns token counts, not aliases. +- **Diagnostics traverse the real path.** `tingly doctor` runs the actual + discover → session → live round-trip; green = user code will run. +- **Surface the artifact for the next action.** `ready=false` and + `GuardrailBlockedError(policy_id, reason)` tell the user exactly what to fix. + +## Testing + +- Python: `sdk/python/tests/` — config precedence, discovery/session (respx + mocked gateway), transport URL shaping, client transport routing. Integration + tests that need a live tb are marked `@needs_tb` and skipped by default. +- Go: `internal/server/sdk_session_test.go` freezes the response JSON field + names (contract with the SDK) and the transport-label logic; + `internal/typ/scenario_registry_test.go` pins the experiment descriptor. + +## Open follow-ups + +1. Scoped short-lived session tokens (`expires_at` + refresh on 401). +2. Dedicated `GET /api/v1/sdk/usage?session=` so usage doesn't scan + `/api/v1/requests`. +3. Async client (`AsyncClient`, `aask`) — transports already have async builders. +4. Layer 2: `tingly.Plugin`, manifest, sub-process supervision (reuse + `agentboot/process`), `/plugins//*` reverse proxy, lifecycle UI. +5. Layer 3: auto-register a plugin tool as a virtual model via + `vmodel/virtualserver`. diff --git a/internal/server/sdk_session.go b/internal/server/sdk_session.go new file mode 100644 index 000000000..69849c1f7 --- /dev/null +++ b/internal/server/sdk_session.go @@ -0,0 +1,165 @@ +package server + +import ( + "net" + "net/http" + "net/url" + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// SDKSessionRequest is the body for POST /api/v1/sdk/session. +type SDKSessionRequest struct { + // Scenario is the rule scenario the SDK session should bind to. + // Defaults to "experiment" when empty. + Scenario string `json:"scenario"` + // Name is a human label that shows up in tingly-box logs as the caller, + // so experiments are distinguishable in the request history. + Name string `json:"name"` +} + +// SDKSessionResponse is returned by POST /api/v1/sdk/session. It hands the +// Python SDK everything it needs to construct a tingly-box-bound OpenAI / +// Anthropic client: a base URL, a bearer token, and the transports the +// scenario accepts. +type SDKSessionResponse struct { + // BaseURL is the scenario root, e.g. "http://127.0.0.1:12580/tingly/experiment". + // The OpenAI SDK should target BaseURL+"/v1"; the Anthropic SDK targets BaseURL. + BaseURL string `json:"base_url"` + // Token is the bearer token to authenticate against the gateway. In v0.1 + // this is the gateway model token (long-lived); scoped short-lived tokens + // are a follow-up. + Token string `json:"token"` + // Scenario is the resolved scenario id. + Scenario string `json:"scenario"` + // Transport is "openai", "anthropic", or "both", derived from the scenario + // descriptor. It tells the SDK which client styles are valid. + Transport string `json:"transport"` + // Ready is true when an active rule with at least one service is bound to + // the scenario. When false, requests will fail until the user binds a rule; + // the SDK's `tingly doctor` surfaces this as the next action. + Ready bool `json:"ready"` + // Services is the number of active services bound to the scenario's rule. + Services int `json:"services"` + // ExpiresAt is the token expiry. Empty in v0.1 (long-lived model token). + ExpiresAt string `json:"expires_at,omitempty"` +} + +// CreateSDKSession mints an SDK session for a scenario. It is the single +// gateway-side endpoint the `tingly` Python module relies on: given a scenario, +// it returns the base URL, bearer token, and accepted transports so a user can +// write an experiment or plugin in a handful of lines and reuse the gateway's +// routing, fallback, guard rails, quota, and logging. +func (s *Server) CreateSDKSession(c *gin.Context) { + var req SDKSessionRequest + if err := c.ShouldBindJSON(&req); err != nil { + // Tolerate an empty body — default everything. + req = SDKSessionRequest{} + } + + scenario := typ.RuleScenario(req.Scenario) + if scenario == "" { + scenario = typ.ScenarioExperiment + } + + descriptor, ok := typ.GetScenarioDescriptor(scenario) + if !ok || !descriptor.AllowRuleBinding { + c.JSON(http.StatusNotFound, gin.H{ + "success": false, + "error": "unknown or non-bindable scenario: " + string(scenario), + "valid_scenarios": bindableScenarioIDs(), + }) + return + } + + ready, services := s.scenarioRuleStatus(scenario) + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": SDKSessionResponse{ + BaseURL: s.scenarioBaseURL(scenario), + Token: s.config.GetModelToken(), + Scenario: string(scenario), + Transport: scenarioTransportLabel(descriptor), + Ready: ready, + Services: services, + }, + }) +} + +// scenarioBaseURL builds the externally reachable scenario root URL. A bind +// host of 0.0.0.0 / empty is rewritten to 127.0.0.1 so the returned URL is +// usable by a local SDK client. +func (s *Server) scenarioBaseURL(scenario typ.RuleScenario) string { + host := s.config.GetServerHost() + if host == "" || host == "0.0.0.0" || host == "::" { + host = "127.0.0.1" + } + port := s.config.GetServerPort() + if port == 0 { + port = 12580 + } + return (&url.URL{ + Scheme: "http", + Host: net.JoinHostPort(host, strconv.Itoa(port)), + Path: "/tingly/" + string(scenario.Base()), + }).String() +} + +// scenarioRuleStatus reports whether an active rule with at least one active +// service is bound to the scenario, and how many active services it has. +func (s *Server) scenarioRuleStatus(scenario typ.RuleScenario) (ready bool, services int) { + for i := range s.config.GetRequestConfigs() { + rule := s.config.GetRequestConfigs()[i] + if !rule.Active { + continue + } + if rule.GetScenario().Base() != scenario.Base() { + continue + } + n := len(rule.GetActiveServices()) + if n > services { + services = n + } + if n > 0 { + ready = true + } + } + return ready, services +} + +// scenarioTransportLabel collapses a descriptor's supported transports into the +// label the SDK understands: "openai", "anthropic", or "both". +func scenarioTransportLabel(descriptor typ.ScenarioDescriptor) string { + openai, anthropic := false, false + for _, t := range descriptor.SupportedTransport { + switch t { + case typ.TransportOpenAI: + openai = true + case typ.TransportAnthropic: + anthropic = true + } + } + switch { + case openai && anthropic: + return "both" + case anthropic: + return "anthropic" + default: + return "openai" + } +} + +// bindableScenarioIDs lists scenario ids a caller may bind an SDK session to. +func bindableScenarioIDs() []string { + var ids []string + for _, d := range typ.RegisteredScenarioDescriptors() { + if d.AllowRuleBinding { + ids = append(ids, string(d.ID)) + } + } + return ids +} diff --git a/internal/server/sdk_session_test.go b/internal/server/sdk_session_test.go new file mode 100644 index 000000000..a22f2a58f --- /dev/null +++ b/internal/server/sdk_session_test.go @@ -0,0 +1,72 @@ +package server + +import ( + "encoding/json" + "testing" + + "github.com/tingly-dev/tingly-box/internal/typ" +) + +func TestScenarioTransportLabel(t *testing.T) { + cases := []struct { + name string + transport []typ.ScenarioTransport + want string + }{ + {"both", []typ.ScenarioTransport{typ.TransportOpenAI, typ.TransportAnthropic}, "both"}, + {"anthropic-only", []typ.ScenarioTransport{typ.TransportAnthropic}, "anthropic"}, + {"openai-only", []typ.ScenarioTransport{typ.TransportOpenAI}, "openai"}, + {"embed-falls-to-openai", []typ.ScenarioTransport{typ.TransportEmbed}, "openai"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := scenarioTransportLabel(typ.ScenarioDescriptor{SupportedTransport: tc.transport}) + if got != tc.want { + t.Fatalf("scenarioTransportLabel(%v) = %q, want %q", tc.transport, got, tc.want) + } + }) + } +} + +func TestBindableScenarioIDsIncludesExperiment(t *testing.T) { + ids := bindableScenarioIDs() + found := false + for _, id := range ids { + if id == string(typ.ScenarioExperiment) { + found = true + } + } + if !found { + t.Fatalf("expected experiment scenario in bindable list, got %v", ids) + } +} + +// TestSDKSessionResponseShape freezes the JSON field names the Python SDK +// depends on. If a field is renamed here without updating the SDK, this fails. +func TestSDKSessionResponseShape(t *testing.T) { + resp := SDKSessionResponse{ + BaseURL: "http://127.0.0.1:12580/tingly/experiment", + Token: "tok", + Scenario: "experiment", + Transport: "both", + Ready: true, + Services: 2, + } + b, err := json.Marshal(resp) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, key := range []string{"base_url", "token", "scenario", "transport", "ready", "services"} { + if _, ok := m[key]; !ok { + t.Fatalf("response JSON missing %q field; have %v", key, m) + } + } + // expires_at is omitempty and absent here + if _, ok := m["expires_at"]; ok { + t.Fatalf("expires_at should be omitted when empty") + } +} diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index dbf858bb0..a57f6a6b4 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -88,6 +88,15 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { swagger.WithResponseModel(gin.H{}), ) + // SDK session endpoint — mints a scenario-bound session (base URL, token, + // transports) for the `tingly` Python SDK / plugin experiments. + apiV1.POST("/sdk/session", s.CreateSDKSession, + swagger.WithTags("sdk"), + swagger.WithDescription("Mint a scoped SDK session for a scenario"), + swagger.WithRequestModel(SDKSessionRequest{}), + swagger.WithResponseModel(SDKSessionResponse{}), + ) + apiV2 := manager.NewGroup("api", "v2", "") apiV2.Router.Use(s.getUserAuthMiddleware()) diff --git a/internal/typ/scenario_registry.go b/internal/typ/scenario_registry.go index 8e5f2f0c2..41083b88b 100644 --- a/internal/typ/scenario_registry.go +++ b/internal/typ/scenario_registry.go @@ -114,6 +114,17 @@ func builtinScenarioDescriptorFor(scenario RuleScenario) ScenarioDescriptor { AllowDirectPathUse: true, SupportsProfiles: true, } + case ScenarioExperiment: + // SDK / plugin experiment surface. Accepts both OpenAI and Anthropic + // transports so a Python experiment can use either SDK against the same + // /tingly/experiment endpoint. Profiles let users name parallel experiments. + return ScenarioDescriptor{ + ID: scenario, + SupportedTransport: []ScenarioTransport{TransportOpenAI, TransportAnthropic}, + AllowRuleBinding: true, + AllowDirectPathUse: true, + SupportsProfiles: true, + } case ScenarioGlobal: return ScenarioDescriptor{ ID: scenario, diff --git a/internal/typ/scenario_registry_test.go b/internal/typ/scenario_registry_test.go index 13a243118..d7286e878 100644 --- a/internal/typ/scenario_registry_test.go +++ b/internal/typ/scenario_registry_test.go @@ -174,6 +174,7 @@ func TestRegisterScenario_RejectsConflictingDescriptor(t *testing.T) { } } +<<<<<<< HEAD func TestIsSimpleProfileAlias(t *testing.T) { cases := []struct { in string @@ -224,3 +225,22 @@ func TestValidateProfileName(t *testing.T) { } } } + +func TestExperimentScenarioDescriptor(t *testing.T) { + d, ok := GetScenarioDescriptor(ScenarioExperiment) + if !ok { + t.Fatalf("expected %q descriptor to be registered", ScenarioExperiment) + } + if !d.AllowRuleBinding || !d.AllowDirectPathUse { + t.Fatalf("expected experiment descriptor to allow rule binding and path use, got %+v", d) + } + if !d.SupportsProfiles { + t.Fatalf("expected experiment scenario to support profiles") + } + if !ScenarioSupportsTransport(ScenarioExperiment, TransportOpenAI) { + t.Fatalf("experiment scenario should support TransportOpenAI") + } + if !ScenarioSupportsTransport(ScenarioExperiment, TransportAnthropic) { + t.Fatalf("experiment scenario should support TransportAnthropic") + } +} diff --git a/internal/typ/type.go b/internal/typ/type.go index ba1f93cb4..c9e4dc22b 100644 --- a/internal/typ/type.go +++ b/internal/typ/type.go @@ -63,8 +63,9 @@ const ( ScenarioClaudeDesktop RuleScenario = "claude_desktop" ScenarioSmartGuide RuleScenario = "_smart_guide" ScenarioGlobal RuleScenario = "_global" // Global flags that apply to all scenarios - ScenarioEmbed RuleScenario = "embed" // Embedding application scenario; only serves /embeddings - ScenarioImageGen RuleScenario = "imagegen" // Image generation scenario; only serves /images/generations + ScenarioEmbed RuleScenario = "embed" // Embedding application scenario; only serves /embeddings + ScenarioImageGen RuleScenario = "imagegen" // Image generation scenario; only serves /images/generations + ScenarioExperiment RuleScenario = "experiment" // Python SDK / plugin experiment scenario; accepts OpenAI + Anthropic transports ) func BuiltinScenarios() []RuleScenario { @@ -83,6 +84,7 @@ func BuiltinScenarios() []RuleScenario { ScenarioGlobal, ScenarioEmbed, ScenarioImageGen, + ScenarioExperiment, } } diff --git a/sdk/python/README.md b/sdk/python/README.md new file mode 100644 index 000000000..d86aab581 --- /dev/null +++ b/sdk/python/README.md @@ -0,0 +1,60 @@ +# tingly — Python SDK for tingly-box + +Write an LLM experiment or plugin in a handful of lines and reuse the +tingly-box gateway's power: provider routing, fallback, guard rails, quota and +logging. You write the idea; the box handles the plumbing. + +## Install + +```bash +pip install tingly # core +pip install "tingly[all]" # + openai and anthropic SDKs +``` + +## Experiment ASAP + +```python +import tingly + +tb = tingly.connect(scenario="experiment") # auto-discovers your local tb + +# One-shot, transport picked for you, model routed by tb: +print(tb.ask("Summarize tingly-box in one line", model="auto")) + +# Or use the SDK objects directly — already pointed at the gateway: +resp = tb.openai.chat.completions.create( + model="auto", + messages=[{"role": "user", "content": "hi"}], +) +resp = tb.anthropic.messages.create( + model="claude-sonnet-4-6", + max_tokens=256, + messages=[{"role": "user", "content": "hi"}], +) +``` + +Every call above flows through tingly-box, so guard rails, quota, logging and +fallback apply automatically — your experiment never has to know. + +## How `connect()` finds your box + +In order: explicit args → `TINGLY_BOX_URL` / `TINGLY_BOX_TOKEN` env → +`~/.tingly-box/sdk.json` → `~/.tingly-box/config.json` + localhost probe. + +The token is your **admin** token (tb's `UserToken`); the SDK uses it once to +mint a session, then uses the returned model token for the LLM calls. + +## Diagnose + +```bash +tingly doctor # traverses the real path and prints what works +tingly doctor --link # save gateway URL + token to ~/.tingly-box/sdk.json +``` + +A green `tingly doctor` is a guarantee your code will run. + +## Status + +This is **Layer 1** (client-side library). Layer 2 (tb-hosted plugins with a +manifest and lifecycle UI) and Layer 3 (plugin-as-virtual-model) build on the +same module. See `.design/python-sdk.md` in the repo for the full design. diff --git a/sdk/python/examples/rag_experiment.py b/sdk/python/examples/rag_experiment.py new file mode 100644 index 000000000..24ea0a15c --- /dev/null +++ b/sdk/python/examples/rag_experiment.py @@ -0,0 +1,54 @@ +"""A minimal RAG-style experiment that reuses the tingly-box gateway. + +Run a local tingly-box, then: + + pip install -e ".[all]" # from sdk/python + python examples/rag_experiment.py + +Everything below routes through tb: provider selection, fallback, guard rails, +quota and logging are all applied for free. The experiment only owns its own +logic (here, a toy retriever). +""" + +import tingly + +# A stand-in "corpus". A real experiment would hit a vector store here. +CORPUS = { + "tingly-box": "tingly-box is a personal intelligence orchestrator: an LLM " + "gateway with remote control and guard rails.", + "sdk": "The tingly Python SDK lets you write an experiment in a handful of " + "lines and reuse the gateway.", +} + + +def retrieve(question: str) -> str: + q = question.lower() + hits = [text for key, text in CORPUS.items() if key in q] + return "\n".join(hits) or "(no matching documents)" + + +def main() -> None: + # Auto-discovers the local gateway and binds to the "experiment" scenario. + with tingly.connect(scenario="experiment", name="rag-experiment") as tb: + if not tb.ready: + print( + "Scenario 'experiment' has no active rule yet — bind one in the " + "tingly-box UI, or run `tingly doctor`." + ) + return + + question = "What is tingly-box?" + docs = retrieve(question) + answer = tb.ask( + f"Using only these documents:\n{docs}\n\nAnswer: {question}", + model="auto", + ) + print("Q:", question) + print("A:", answer) + + usage = tb.usage.this_session() + print(f"\n[usage] {usage.requests} request(s), {usage.total_tokens} tokens") + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml new file mode 100644 index 000000000..94ec935fb --- /dev/null +++ b/sdk/python/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "tingly" +version = "0.1.0" +description = "Python SDK for tingly-box — write an LLM experiment or plugin in a handful of lines and reuse the gateway's routing, fallback, guard rails, quota and logging." +readme = "README.md" +requires-python = ">=3.9" +license = { text = "MPL-2.0" } +authors = [{ name = "tingly.dev", email = "ops@tingly.dev" }] +keywords = ["tingly-box", "llm", "gateway", "openai", "anthropic", "plugin"] +dependencies = [ + "httpx>=0.27", + "pydantic>=2", +] + +[project.optional-dependencies] +# The LLM SDKs are optional so a plugin that only uses `tb.ask()` (which goes +# through httpx) does not have to pull both. `connect().openai` /.anthropic +# raise a clear error if the matching extra is missing. +openai = ["openai>=1.0"] +anthropic = ["anthropic>=0.40"] +all = ["openai>=1.0", "anthropic>=0.40"] +dev = ["pytest>=8", "respx>=0.21", "openai>=1.0", "anthropic>=0.40"] + +[project.scripts] +tingly = "tingly.cli:main" + +[project.urls] +Homepage = "https://github.com/tingly-dev/tingly-box" +Documentation = "https://github.com/tingly-dev/tingly-box/blob/main/.design/python-sdk.md" + +[tool.hatch.build.targets.wheel] +packages = ["tingly"] + +[tool.pytest.ini_options] +markers = [ + "needs_tb: integration tests that require a running tingly-box gateway (opt-in)", +] +addopts = "-m 'not needs_tb'" diff --git a/sdk/python/tests/test_client_offline.py b/sdk/python/tests/test_client_offline.py new file mode 100644 index 000000000..00b254626 --- /dev/null +++ b/sdk/python/tests/test_client_offline.py @@ -0,0 +1,66 @@ +"""Client transport-routing tests with a fake session (no real SDKs invoked).""" + +import pytest + +from tingly.client import Client +from tingly.discovery import Session +from tingly.errors import TinglyError + + +def _client(transport: str) -> Client: + session = Session( + base_url="http://tb.test:12580/tingly/experiment", + token="model-tok", + scenario="experiment", + transport=transport, + ready=True, + services=1, + ) + return Client(session, "http://tb.test:12580", "admin", "exp", 30.0) + + +def test_anthropic_only_rejects_openai(): + c = _client("anthropic") + with pytest.raises(TinglyError): + _ = c.openai + + +def test_openai_only_rejects_anthropic(): + c = _client("openai") + with pytest.raises(TinglyError): + _ = c.anthropic + + +def test_both_exposes_identity(): + c = _client("both") + assert c.scenario == "experiment" + assert c.transport == "both" + assert c.ready is True + assert c.base_url.endswith("/tingly/experiment") + + +def test_client_passes_scenario_root_to_transports(monkeypatch): + """The client hands the scenario root + model token to each builder; + per-transport URL shaping (e.g. the OpenAI /v1 suffix) happens inside the + builder.""" + captured = {} + + def fake_openai(base_url, token, timeout): + captured["openai_base"] = base_url + captured["openai_token"] = token + return object() + + def fake_anthropic(base_url, token, timeout): + captured["anthropic_base"] = base_url + return object() + + monkeypatch.setattr("tingly.transports.openai_compat.build_openai", fake_openai) + monkeypatch.setattr( + "tingly.transports.anthropic_compat.build_anthropic", fake_anthropic + ) + c = _client("both") + _ = c.openai + _ = c.anthropic + assert captured["openai_base"] == "http://tb.test:12580/tingly/experiment" + assert captured["openai_token"] == "model-tok" + assert captured["anthropic_base"] == "http://tb.test:12580/tingly/experiment" diff --git a/sdk/python/tests/test_config.py b/sdk/python/tests/test_config.py new file mode 100644 index 000000000..468fb56cd --- /dev/null +++ b/sdk/python/tests/test_config.py @@ -0,0 +1,57 @@ +"""Config resolution precedence tests (no network).""" + +import json + +import tingly.config as cfg + + +def test_args_win(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, "http://env:1/") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + r = cfg.resolve(base_url="http://arg:2", token="arg-token") + assert r.base_url == "http://arg:2" + assert r.token == "arg-token" + assert r.source == "args" + + +def test_env_fallback(monkeypatch, tmp_path): + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + monkeypatch.setenv(cfg.ENV_URL, "http://env:1") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + r = cfg.resolve() + assert r.base_url == "http://env:1" + assert r.token == "env-token" + assert r.source == "env" + + +def test_sdk_link_file(monkeypatch, tmp_path): + monkeypatch.delenv(cfg.ENV_URL, raising=False) + monkeypatch.delenv(cfg.ENV_TOKEN, raising=False) + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + (tmp_path / "sdk.json").write_text( + json.dumps({"base_url": "http://link:3", "token": "link-token"}) + ) + r = cfg.resolve() + assert r.base_url == "http://link:3" + assert r.token == "link-token" + assert r.source == "sdk.json" + + +def test_config_json_admin_token(monkeypatch, tmp_path): + monkeypatch.delenv(cfg.ENV_URL, raising=False) + monkeypatch.delenv(cfg.ENV_TOKEN, raising=False) + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + (tmp_path / "config.json").write_text(json.dumps({"UserToken": "admin-xyz"})) + r = cfg.resolve() + assert r.token == "admin-xyz" + # default localhost base when nothing else set + assert r.base_url.startswith("http://127.0.0.1:") + + +def test_default_localhost(monkeypatch, tmp_path): + monkeypatch.delenv(cfg.ENV_URL, raising=False) + monkeypatch.delenv(cfg.ENV_TOKEN, raising=False) + monkeypatch.setenv("TINGLY_BOX_HOME", str(tmp_path)) + r = cfg.resolve() + assert r.base_url == f"http://{cfg.DEFAULT_HOST}:{cfg.DEFAULT_PORT}" + assert r.token is None diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py new file mode 100644 index 000000000..3188ed876 --- /dev/null +++ b/sdk/python/tests/test_discovery.py @@ -0,0 +1,87 @@ +"""Discovery + session minting tests (gateway mocked with respx).""" + +import httpx +import pytest +import respx + +import tingly.discovery as disco +from tingly.errors import AuthError, GatewayUnreachableError, ScenarioNotFoundError + +BASE = "http://tb.test:12580" + + +def _version_route(): + return respx.get(f"{BASE}/api/v1/info/version").mock( + return_value=httpx.Response(200, json={"version": "1.2.3"}) + ) + + +@respx.mock +def test_probe_version_ok(): + _version_route() + assert disco.probe_version(BASE) == "1.2.3" + + +@respx.mock +def test_probe_version_down(): + respx.get(f"{BASE}/api/v1/info/version").mock( + return_value=httpx.Response(503) + ) + assert disco.probe_version(BASE) is None + + +@respx.mock +def test_create_session_ok(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "data": { + "base_url": f"{BASE}/tingly/experiment", + "token": "model-tok", + "scenario": "experiment", + "transport": "both", + "ready": True, + "services": 2, + }, + }, + ) + ) + s = disco.create_session(BASE, "admin", "experiment", name="exp") + assert s.base_url == f"{BASE}/tingly/experiment" + assert s.token == "model-tok" + assert s.transport == "both" + assert s.ready is True + assert s.services == 2 + + +@respx.mock +def test_create_session_auth_error(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + return_value=httpx.Response(401, json={"error": "nope"}) + ) + with pytest.raises(AuthError): + disco.create_session(BASE, "bad", "experiment") + + +@respx.mock +def test_create_session_scenario_not_found(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + return_value=httpx.Response( + 404, + json={"success": False, "valid_scenarios": ["experiment", "openai"]}, + ) + ) + with pytest.raises(ScenarioNotFoundError) as ei: + disco.create_session(BASE, "admin", "bogus") + assert "experiment" in ei.value.valid_scenarios + + +@respx.mock +def test_create_session_unreachable(): + respx.post(f"{BASE}/api/v1/sdk/session").mock( + side_effect=httpx.ConnectError("refused") + ) + with pytest.raises(GatewayUnreachableError): + disco.create_session(BASE, "admin", "experiment") diff --git a/sdk/python/tests/test_transports.py b/sdk/python/tests/test_transports.py new file mode 100644 index 000000000..155bd0677 --- /dev/null +++ b/sdk/python/tests/test_transports.py @@ -0,0 +1,47 @@ +"""Transport URL-shaping tests, using a stubbed LLM SDK module. + +These pin the contract with the gateway routes: + - OpenAI SDK base_url = scenario_root + "/v1" -> /tingly//v1/chat/completions + - Anthropic SDK base_url = scenario_root -> /tingly//v1/messages +""" + +import sys +import types + +from tingly.transports import anthropic_compat, openai_compat + +ROOT = "http://tb.test:12580/tingly/experiment" + + +def test_openai_appends_v1(monkeypatch): + captured = {} + + fake = types.ModuleType("openai") + + class FakeOpenAI: + def __init__(self, base_url, api_key, timeout): + captured["base_url"] = base_url + captured["api_key"] = api_key + + fake.OpenAI = FakeOpenAI + monkeypatch.setitem(sys.modules, "openai", fake) + + openai_compat.build_openai(ROOT, "tok", 10.0) + assert captured["base_url"] == ROOT + "/v1" + assert captured["api_key"] == "tok" + + +def test_anthropic_no_v1(monkeypatch): + captured = {} + + fake = types.ModuleType("anthropic") + + class FakeAnthropic: + def __init__(self, base_url, api_key, timeout): + captured["base_url"] = base_url + + fake.Anthropic = FakeAnthropic + monkeypatch.setitem(sys.modules, "anthropic", fake) + + anthropic_compat.build_anthropic(ROOT, "tok", 10.0) + assert captured["base_url"] == ROOT diff --git a/sdk/python/tingly/__init__.py b/sdk/python/tingly/__init__.py new file mode 100644 index 000000000..e0cc619e6 --- /dev/null +++ b/sdk/python/tingly/__init__.py @@ -0,0 +1,34 @@ +"""tingly — Python SDK for tingly-box. + +Write an LLM experiment or plugin in a handful of lines and reuse the gateway's +routing, fallback, guard rails, quota and logging: + + >>> import tingly + >>> tb = tingly.connect(scenario="experiment") + >>> tb.ask("Say hello", model="auto") +""" + +from __future__ import annotations + +from ._version import __version__ +from .client import Client, connect +from .errors import ( + AuthError, + GatewayUnreachableError, + GuardrailBlockedError, + ScenarioNotFoundError, + TinglyError, + UpstreamError, +) + +__all__ = [ + "__version__", + "connect", + "Client", + "TinglyError", + "GatewayUnreachableError", + "AuthError", + "ScenarioNotFoundError", + "GuardrailBlockedError", + "UpstreamError", +] diff --git a/sdk/python/tingly/_version.py b/sdk/python/tingly/_version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/sdk/python/tingly/_version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py new file mode 100644 index 000000000..04909eb0d --- /dev/null +++ b/sdk/python/tingly/cli.py @@ -0,0 +1,140 @@ +"""``tingly`` CLI — a single diagnostic command. + +`tingly doctor` traverses the *real* code path a user's program takes +(discovery → session → a live LLM round-trip) and prints what worked and what +didn't, so a green doctor is a guarantee that user code will run. + +`tingly doctor --link` writes ``~/.tingly-box/sdk.json`` so future runs need no +env vars. +""" + +from __future__ import annotations + +import argparse +import getpass +import json +import sys +from typing import Optional + +from . import config as _config +from . import discovery as _discovery +from . import scenarios as _scenarios + +OK = "OK" +FAIL = "FAIL" +WARN = "WARN" + + +def _row(label: str, detail: str, status: str) -> None: + print(f"{label:<14}{detail:<40}{status}") + + +def doctor(scenario: str, link: bool) -> int: + if link: + _do_link() + + resolved = _config.resolve() + + # 1. gateway reachable + version = _discovery.probe_version(resolved.base_url) + if version is None: + _row("gateway", resolved.base_url, FAIL) + print( + f"\nNo tingly-box gateway responding at {resolved.base_url} " + f"(resolved via {resolved.source}).\n" + "Start tb, set TINGLY_BOX_URL, or run `tingly doctor --link`." + ) + return 1 + _row("gateway", f"{resolved.base_url} (v{version})", OK) + _row("token", f"{resolved.source}", OK if resolved.token else WARN) + + # 2. mint a session (real path) + try: + session = _discovery.create_session( + base_url=resolved.base_url, + admin_token=resolved.token or "", + scenario=scenario, + name="tingly-doctor", + ) + except Exception as exc: # noqa: BLE001 - report any failure verbatim + _row("session", scenario, FAIL) + print(f"\n{type(exc).__name__}: {exc}") + return 1 + + scen_detail = f"{session.scenario} ({session.transport}, {session.services} svc)" + _row("scenario", scen_detail, OK if session.ready else WARN) + if not session.ready: + print( + f"\nScenario {session.scenario!r} has no active rule with a service. " + "Bind a rule to it in the tingly-box UI before sending requests." + ) + + # 3. live round-trip (only if ready) + if session.ready: + _live_check(session) + + return 0 + + +def _live_check(session: "_discovery.Session") -> None: + from .client import Client + + client = Client( + session=session, + gateway_url="", + admin_token="", + name="tingly-doctor", + timeout=30.0, + ) + try: + text = client.ask("Reply with the single word: pong", model="auto") + ok = isinstance(text, str) and len(text) > 0 + transport = "chat.completions" if _scenarios.supports_openai(session.transport) else "messages" + _row("llm test", transport, OK if ok else FAIL) + except Exception as exc: # noqa: BLE001 + _row("llm test", "round-trip", FAIL) + print(f"\n{type(exc).__name__}: {exc}") + finally: + client.close() + + +def _do_link() -> None: + """Prompt for the admin token and persist a link file.""" + path = _config.sdk_link_path() + base_url = input(f"Gateway URL [{_config.resolve().base_url}]: ").strip() + if not base_url: + base_url = _config.resolve().base_url + token = getpass.getpass("Admin token (TINGLY_BOX_TOKEN): ").strip() + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + json.dump({"base_url": base_url, "token": token}, fh, indent=2) + try: + path.chmod(0o600) + except OSError: + pass + print(f"Wrote {path}") + + +def main(argv: Optional[list] = None) -> int: + parser = argparse.ArgumentParser(prog="tingly", description="tingly-box Python SDK") + sub = parser.add_subparsers(dest="command") + + p_doctor = sub.add_parser("doctor", help="diagnose the SDK ↔ gateway connection") + p_doctor.add_argument( + "--scenario", default=_scenarios.EXPERIMENT, help="scenario to test" + ) + p_doctor.add_argument( + "--link", action="store_true", help="prompt for and save gateway URL + token" + ) + + args = parser.parse_args(argv) + if args.command == "doctor": + return doctor(args.scenario, args.link) + + parser.print_help() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sdk/python/tingly/client.py b/sdk/python/tingly/client.py new file mode 100644 index 000000000..a8334726e --- /dev/null +++ b/sdk/python/tingly/client.py @@ -0,0 +1,236 @@ +"""The tingly Client and the ``connect()`` entrypoint. + +``connect()`` is the whole surface area for "experiment ASAP": auto-discover the +local gateway, mint a scenario-bound session, and hand back a Client whose +``.openai`` / ``.anthropic`` SDK objects are already pointed at tingly-box — so +every request inherits the gateway's routing, fallback, guard rails, quota and +logging without the experiment knowing anything about them. +""" + +from __future__ import annotations + +from typing import Any, Iterator, Optional + +from . import config as _config +from . import discovery as _discovery +from . import scenarios as _scenarios +from .errors import TinglyError +from .helpers.guardrails import GuardrailsView +from .helpers.usage import UsageView +from .transports import anthropic_compat, openai_compat + + +class Client: + """A tingly-box-bound LLM client. + + Construct via :func:`connect`, not directly. Holds the minted session (for + LLM calls) plus the gateway root + admin token (for management views). + """ + + def __init__( + self, + session: "_discovery.Session", + gateway_url: str, + admin_token: str, + name: str, + timeout: float, + ): + self._session = session + self._gateway_url = gateway_url + self._admin_token = admin_token + self.name = name + self._timeout = timeout + + self._openai: Optional[Any] = None + self._anthropic: Optional[Any] = None + + # -- identity -------------------------------------------------------- + + @property + def base_url(self) -> str: + return self._session.base_url + + @property + def scenario(self) -> str: + return self._session.scenario + + @property + def transport(self) -> str: + return self._session.transport + + @property + def ready(self) -> bool: + """True when the scenario has an active rule with at least one service.""" + return self._session.ready + + # -- SDK pass-throughs ---------------------------------------------- + + @property + def openai(self) -> Any: + """A lazily-built ``openai.OpenAI`` bound to this scenario.""" + if not _scenarios.supports_openai(self._session.transport): + raise TinglyError( + f"scenario {self.scenario!r} does not accept the OpenAI transport " + f"(transport={self.transport!r})" + ) + if self._openai is None: + self._openai = openai_compat.build_openai( + self._session.base_url, self._session.token, self._timeout + ) + return self._openai + + @property + def anthropic(self) -> Any: + """A lazily-built ``anthropic.Anthropic`` bound to this scenario.""" + if not _scenarios.supports_anthropic(self._session.transport): + raise TinglyError( + f"scenario {self.scenario!r} does not accept the Anthropic transport " + f"(transport={self.transport!r})" + ) + if self._anthropic is None: + self._anthropic = anthropic_compat.build_anthropic( + self._session.base_url, self._session.token, self._timeout + ) + return self._anthropic + + # -- convenience ----------------------------------------------------- + + def ask( + self, + prompt: str, + *, + model: str = "auto", + system: Optional[str] = None, + max_tokens: int = 1024, + stream: bool = False, + **kwargs: Any, + ): + """One-shot prompt → text, routed through tingly-box. + + Picks the transport from the scenario: OpenAI-capable scenarios use + chat completions; Anthropic-only scenarios use messages. ``model="auto"`` + lets the gateway route. + """ + if _scenarios.supports_openai(self._session.transport): + return self._ask_openai(prompt, model, system, stream, **kwargs) + return self._ask_anthropic(prompt, model, system, max_tokens, stream, **kwargs) + + def _ask_openai(self, prompt, model, system, stream, **kwargs): + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + resp = self.openai.chat.completions.create( + model=model, messages=messages, stream=stream, **kwargs + ) + if stream: + return self._stream_openai(resp) + return resp.choices[0].message.content + + @staticmethod + def _stream_openai(resp) -> Iterator[str]: + for chunk in resp: + delta = chunk.choices[0].delta.content + if delta: + yield delta + + def _ask_anthropic(self, prompt, model, system, max_tokens, stream, **kwargs): + anthropic_model = model if model != "auto" else "claude-sonnet-4-6" + params = dict( + model=anthropic_model, + max_tokens=max_tokens, + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + if system: + params["system"] = system + if stream: + return self._stream_anthropic(params) + resp = self.anthropic.messages.create(**params) + return "".join(block.text for block in resp.content if hasattr(block, "text")) + + def _stream_anthropic(self, params) -> Iterator[str]: + with self.anthropic.messages.stream(**params) as stream: + for text in stream.text_stream: + yield text + + # -- management views ------------------------------------------------ + + @property + def usage(self) -> UsageView: + return UsageView(self._gateway_url, self._admin_token, self.name, self._timeout) + + @property + def guardrails(self) -> GuardrailsView: + return GuardrailsView(self._gateway_url, self._admin_token, self._timeout) + + # -- lifecycle ------------------------------------------------------- + + def close(self) -> None: + for c in (self._openai, self._anthropic): + closer = getattr(c, "close", None) + if callable(closer): + try: + closer() + except Exception: + pass + + def __enter__(self) -> "Client": + return self + + def __exit__(self, *exc) -> None: + self.close() + + +def connect( + scenario: str = _scenarios.EXPERIMENT, + *, + base_url: Optional[str] = None, + token: Optional[str] = None, + timeout: float = 60.0, + name: Optional[str] = None, +) -> Client: + """Connect to a local tingly-box gateway and return a bound Client. + + Args: + scenario: the rule scenario to bind to. Defaults to ``"experiment"``. + base_url: gateway root, e.g. ``http://127.0.0.1:12580``. Auto-discovered + if omitted (env → ``~/.tingly-box/sdk.json`` → ``config.json`` → + localhost probe). + token: admin token used to provision the session. Auto-discovered if + omitted. + timeout: per-request timeout in seconds for the LLM clients. + name: a label that identifies this experiment in tingly-box logs. + + Raises: + GatewayUnreachableError: no gateway responded. + AuthError: the admin token was rejected. + ScenarioNotFoundError: the scenario is unknown or not bindable. + """ + resolved = _config.resolve(base_url=base_url, token=token) + + if _discovery.probe_version(resolved.base_url) is None: + from .errors import GatewayUnreachableError + + raise GatewayUnreachableError( + f"no tingly-box gateway responding at {resolved.base_url} " + f"(resolved via {resolved.source}). Is `tb` running? " + f"Set TINGLY_BOX_URL or run `tingly doctor`." + ) + + caller = name or "tingly-sdk" + session = _discovery.create_session( + base_url=resolved.base_url, + admin_token=resolved.token or "", + scenario=scenario, + name=caller, + timeout=timeout, + ) + + return Client( + session=session, + gateway_url=resolved.base_url, + admin_token=resolved.token or "", + name=caller, + timeout=timeout, + ) diff --git a/sdk/python/tingly/config.py b/sdk/python/tingly/config.py new file mode 100644 index 000000000..8a924e04b --- /dev/null +++ b/sdk/python/tingly/config.py @@ -0,0 +1,115 @@ +"""Configuration resolution for the tingly SDK. + +Resolves ``(base_url, token)`` for connecting to a local tingly-box gateway, +following a fixed precedence so behaviour is predictable across dev and hosted +contexts: + + 1. Explicit arguments to ``connect()``. + 2. Environment: ``TINGLY_BOX_URL`` / ``TINGLY_BOX_TOKEN``. + 3. The SDK link file: ``~/.tingly-box/sdk.json`` (written by + ``tingly doctor --link`` or the tb UI "Connect SDK" panel). + 4. The tb config file: ``~/.tingly-box/config.json`` (admin ``UserToken``) + combined with a localhost probe. + +The ``token`` resolved here is the *admin* token used to provision an SDK +session via ``POST /api/v1/sdk/session``; the session itself hands back the +*model* token used for the actual LLM calls. + +This module performs no network I/O; the probe lives in ``discovery``. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +DEFAULT_PORT = 12580 +DEFAULT_HOST = "127.0.0.1" + +ENV_URL = "TINGLY_BOX_URL" +ENV_TOKEN = "TINGLY_BOX_TOKEN" + + +def config_dir() -> Path: + """Return the tingly-box config directory (``~/.tingly-box`` by default).""" + override = os.environ.get("TINGLY_BOX_HOME") + if override: + return Path(override) + return Path.home() / ".tingly-box" + + +def sdk_link_path() -> Path: + return config_dir() / "sdk.json" + + +def tb_config_path() -> Path: + return config_dir() / "config.json" + + +@dataclass +class Resolved: + """A resolved gateway target plus where it came from (for diagnostics).""" + + base_url: Optional[str] + token: Optional[str] + source: str # "args" | "env" | "sdk.json" | "config.json" | "probe-default" + + +def _read_json(path: Path) -> Optional[dict]: + try: + with path.open("r", encoding="utf-8") as fh: + return json.load(fh) + except (FileNotFoundError, ValueError, OSError): + return None + + +def resolve( + base_url: Optional[str] = None, + token: Optional[str] = None, +) -> Resolved: + """Resolve ``(base_url, token)`` by precedence, without touching the network. + + A missing piece is left as ``None`` for the caller (``discovery``) to fill — + e.g. an explicit ``base_url`` with no token still falls through to pick up a + token from env / files. + """ + src = "args" + if base_url is None: + env_url = os.environ.get(ENV_URL) + if env_url: + base_url, src = env_url, "env" + if token is None: + env_token = os.environ.get(ENV_TOKEN) + if env_token: + token = env_token + if src == "args": + src = "env" + + if base_url is None or token is None: + link = _read_json(sdk_link_path()) + if link: + if base_url is None and link.get("base_url"): + base_url, src = link["base_url"], "sdk.json" + if token is None and link.get("token"): + token = link["token"] + if src == "args": + src = "sdk.json" + + if token is None: + cfg = _read_json(tb_config_path()) + if cfg: + # tb stores UserToken (admin) and ModelToken (LLM API key). We need + # the admin token to provision an SDK session. + token = cfg.get("UserToken") or cfg.get("user_token") + if token and src == "args": + src = "config.json" + + if base_url is None: + base_url = f"http://{DEFAULT_HOST}:{DEFAULT_PORT}" + if src == "args": + src = "probe-default" + + return Resolved(base_url=base_url, token=token, source=src) diff --git a/sdk/python/tingly/discovery.py b/sdk/python/tingly/discovery.py new file mode 100644 index 000000000..60de8083d --- /dev/null +++ b/sdk/python/tingly/discovery.py @@ -0,0 +1,132 @@ +"""Gateway discovery + SDK session minting. + +``connect()`` calls into here to turn a resolved ``(base_url, admin_token)`` +into a live :class:`Session`: it probes the gateway for liveness, then mints a +scenario-bound session via ``POST /api/v1/sdk/session``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import httpx + +from . import config as _config +from .errors import ( + AuthError, + GatewayUnreachableError, + ScenarioNotFoundError, +) + + +@dataclass +class Session: + """The minted SDK session — everything the transports need to bind.""" + + base_url: str # scenario root, e.g. http://127.0.0.1:12580/tingly/experiment + token: str # model token used for LLM calls + scenario: str + transport: str # "openai" | "anthropic" | "both" + ready: bool + services: int + + +def probe_version(base_url: str, timeout: float = 5.0) -> Optional[str]: + """Return the gateway version if reachable, else ``None``.""" + url = base_url.rstrip("/") + "/api/v1/info/version" + try: + resp = httpx.get(url, timeout=timeout) + except httpx.HTTPError: + return None + if resp.status_code != 200: + return None + try: + data = resp.json() + except ValueError: + return None + # tolerate both {version: ...} and {data: {version: ...}} + return ( + data.get("version") + or (data.get("data") or {}).get("version") + or "unknown" + ) + + +def create_session( + base_url: str, + admin_token: str, + scenario: str, + name: Optional[str] = None, + timeout: float = 30.0, +) -> Session: + """Mint an SDK session against ``POST /api/v1/sdk/session``.""" + url = base_url.rstrip("/") + "/api/v1/sdk/session" + headers = {"Authorization": f"Bearer {admin_token}"} if admin_token else {} + body = {"scenario": scenario, "name": name or ""} + + try: + resp = httpx.post(url, json=body, headers=headers, timeout=timeout) + except httpx.HTTPError as exc: + raise GatewayUnreachableError( + f"could not reach tingly-box at {base_url}: {exc}" + ) from exc + + if resp.status_code == 401: + raise AuthError( + "tingly-box rejected the admin token. Set TINGLY_BOX_TOKEN or run " + "`tingly doctor --link`." + ) + + payload = _safe_json(resp) + if resp.status_code == 404: + raise ScenarioNotFoundError( + scenario, (payload or {}).get("valid_scenarios") + ) + if resp.status_code != 200 or not payload or not payload.get("success"): + raise GatewayUnreachableError( + f"unexpected response from {url}: HTTP {resp.status_code} {resp.text[:200]}" + ) + + data = payload.get("data") or {} + return Session( + base_url=data["base_url"], + token=data.get("token", ""), + scenario=data.get("scenario", scenario), + transport=data.get("transport", "both"), + ready=bool(data.get("ready", False)), + services=int(data.get("services", 0)), + ) + + +def discover_and_connect( + scenario: str, + base_url: Optional[str] = None, + token: Optional[str] = None, + name: Optional[str] = None, + timeout: float = 30.0, +) -> Session: + """Resolve config, verify reachability, and mint a session.""" + resolved = _config.resolve(base_url=base_url, token=token) + + if probe_version(resolved.base_url) is None: + raise GatewayUnreachableError( + f"no tingly-box gateway responding at {resolved.base_url} " + f"(resolved via {resolved.source}). Is `tb` running? " + f"Set TINGLY_BOX_URL or run `tingly doctor`." + ) + + return create_session( + base_url=resolved.base_url, + admin_token=resolved.token or "", + scenario=scenario, + name=name, + timeout=timeout, + ) + + +def _safe_json(resp: httpx.Response) -> Optional[dict]: + try: + return resp.json() + except ValueError: + return None diff --git a/sdk/python/tingly/errors.py b/sdk/python/tingly/errors.py new file mode 100644 index 000000000..739cd8024 --- /dev/null +++ b/sdk/python/tingly/errors.py @@ -0,0 +1,64 @@ +"""Error hierarchy for the tingly SDK. + +Every failure a user can hit maps to one of these so plugin authors can react +precisely — and, in the guardrail case, explain to *their* user why a request +was refused rather than surfacing a bare "it failed". +""" + +from __future__ import annotations + +from typing import Optional + + +class TinglyError(Exception): + """Base class for all tingly SDK errors.""" + + +class GatewayUnreachableError(TinglyError): + """The tingly-box gateway could not be discovered or reached. + + Raised during ``connect()`` discovery or when a request cannot establish a + connection to the gateway. + """ + + +class AuthError(TinglyError): + """The gateway rejected the supplied token (HTTP 401).""" + + +class ScenarioNotFoundError(TinglyError): + """The requested scenario is unknown or not bindable (HTTP 404). + + ``valid_scenarios`` carries the gateway's list of bindable scenarios so the + caller can suggest a correct value. + """ + + def __init__(self, scenario: str, valid_scenarios: Optional[list] = None): + self.scenario = scenario + self.valid_scenarios = valid_scenarios or [] + valid = ", ".join(self.valid_scenarios) if self.valid_scenarios else "(none reported)" + super().__init__( + f"scenario {scenario!r} is unknown or not bindable. " + f"Valid scenarios: {valid}" + ) + + +class GuardrailBlockedError(TinglyError): + """tingly-box refused the request due to a guard-rail policy. + + ``policy_id`` and ``reason`` are surfaced so the caller can show *why*. + """ + + def __init__(self, reason: str, policy_id: Optional[str] = None): + self.policy_id = policy_id + self.reason = reason + prefix = f"[{policy_id}] " if policy_id else "" + super().__init__(f"{prefix}request blocked by guard rail: {reason}") + + +class UpstreamError(TinglyError): + """An upstream LLM provider returned a server error (HTTP 5xx).""" + + def __init__(self, message: str, status_code: Optional[int] = None): + self.status_code = status_code + super().__init__(message) diff --git a/sdk/python/tingly/helpers/__init__.py b/sdk/python/tingly/helpers/__init__.py new file mode 100644 index 000000000..8bd76b821 --- /dev/null +++ b/sdk/python/tingly/helpers/__init__.py @@ -0,0 +1 @@ +"""Observability + guard-rail helper views attached to a Client.""" diff --git a/sdk/python/tingly/helpers/guardrails.py b/sdk/python/tingly/helpers/guardrails.py new file mode 100644 index 000000000..b9ba66a89 --- /dev/null +++ b/sdk/python/tingly/helpers/guardrails.py @@ -0,0 +1,53 @@ +"""Guard-rail view — inspect what guard rails are active on the gateway. + +Guard rails in tingly-box run *inline* on the request path, so they are applied +automatically to every call an experiment makes — there is nothing the plugin +author must wire up. This view lets a plugin introspect the active policies +(e.g. to explain to its user what is enforced); a blocked request surfaces as a +:class:`~tingly.errors.GuardrailBlockedError` from the LLM call itself. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List + +import httpx + + +@dataclass +class GuardrailStatus: + enabled: bool = False + active_policies: int = 0 + policy_names: List[str] = None # type: ignore[assignment] + + def __post_init__(self): + if self.policy_names is None: + self.policy_names = [] + + +class GuardrailsView: + def __init__(self, gateway_url: str, admin_token: str, timeout: float): + self._gateway_url = gateway_url.rstrip("/") + self._admin_token = admin_token + self._timeout = timeout + + def status(self) -> GuardrailStatus: + """Return whether guard rails are enabled and how many policies are active.""" + url = f"{self._gateway_url}/api/v1/guardrails/config" + headers = {"Authorization": f"Bearer {self._admin_token}"} + try: + resp = httpx.get(url, headers=headers, timeout=self._timeout) + resp.raise_for_status() + payload = resp.json() + except (httpx.HTTPError, ValueError): + return GuardrailStatus() + + data = payload.get("data") or payload + policies = data.get("policies") or [] + names = [p.get("name", p.get("id", "?")) for p in policies if isinstance(p, dict)] + return GuardrailStatus( + enabled=bool(data.get("enabled", bool(policies))), + active_policies=len(policies), + policy_names=names, + ) diff --git a/sdk/python/tingly/helpers/usage.py b/sdk/python/tingly/helpers/usage.py new file mode 100644 index 000000000..336ef1b7b --- /dev/null +++ b/sdk/python/tingly/helpers/usage.py @@ -0,0 +1,58 @@ +"""Usage view — concrete token / request numbers, never aliases. + +v0.1 reads the gateway's request history (``/api/v1/requests``) filtered to this +SDK session's name. A dedicated per-session usage endpoint is a backend +follow-up; until then this gives real, inspectable numbers. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict + +import httpx + + +@dataclass +class UsageSummary: + requests: int = 0 + input_tokens: int = 0 + output_tokens: int = 0 + by_model: Dict[str, int] = field(default_factory=dict) + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + +class UsageView: + def __init__(self, gateway_url: str, admin_token: str, name: str, timeout: float): + self._gateway_url = gateway_url.rstrip("/") + self._admin_token = admin_token + self._name = name + self._timeout = timeout + + def this_session(self) -> UsageSummary: + """Return token / request totals for this session's named caller.""" + url = f"{self._gateway_url}/api/v1/requests" + headers = {"Authorization": f"Bearer {self._admin_token}"} + try: + resp = httpx.get(url, headers=headers, timeout=self._timeout) + resp.raise_for_status() + payload = resp.json() + except (httpx.HTTPError, ValueError): + return UsageSummary() + + records = payload.get("data") or payload.get("records") or [] + summary = UsageSummary() + for rec in records: + # Best-effort filter by the SDK caller name when the field exists. + source = rec.get("source") or rec.get("name") or "" + if self._name and self._name not in str(source): + continue + summary.requests += 1 + summary.input_tokens += int(rec.get("input_tokens", 0) or 0) + summary.output_tokens += int(rec.get("output_tokens", 0) or 0) + model = rec.get("model", "unknown") + summary.by_model[model] = summary.by_model.get(model, 0) + 1 + return summary diff --git a/sdk/python/tingly/scenarios.py b/sdk/python/tingly/scenarios.py new file mode 100644 index 000000000..553db91a7 --- /dev/null +++ b/sdk/python/tingly/scenarios.py @@ -0,0 +1,30 @@ +"""Scenario + transport constants mirrored from the tingly-box backend. + +These are the bindable scenarios most relevant to SDK users. The authoritative +list lives in ``internal/typ/type.go``; the gateway validates the scenario at +session time and returns the accepted transport, so this module is for +convenience and is not load-bearing. +""" + +from __future__ import annotations + +# Default scenario for experiments / plugins. +EXPERIMENT = "experiment" + +# Other commonly useful bindable scenarios. +OPENAI = "openai" +ANTHROPIC = "anthropic" +AGENT = "agent" + +# Transport labels returned by the gateway session response. +TRANSPORT_OPENAI = "openai" +TRANSPORT_ANTHROPIC = "anthropic" +TRANSPORT_BOTH = "both" + + +def supports_openai(transport: str) -> bool: + return transport in (TRANSPORT_OPENAI, TRANSPORT_BOTH) + + +def supports_anthropic(transport: str) -> bool: + return transport in (TRANSPORT_ANTHROPIC, TRANSPORT_BOTH) diff --git a/sdk/python/tingly/transports/__init__.py b/sdk/python/tingly/transports/__init__.py new file mode 100644 index 000000000..dc0834bdc --- /dev/null +++ b/sdk/python/tingly/transports/__init__.py @@ -0,0 +1 @@ +"""Transport builders — bind the user's preferred LLM SDK to tingly-box.""" diff --git a/sdk/python/tingly/transports/anthropic_compat.py b/sdk/python/tingly/transports/anthropic_compat.py new file mode 100644 index 000000000..fb2aac301 --- /dev/null +++ b/sdk/python/tingly/transports/anthropic_compat.py @@ -0,0 +1,43 @@ +"""Build an ``anthropic.Anthropic`` client bound to a tingly-box scenario. + +The Anthropic Python SDK appends ``/v1/messages`` to its ``base_url``, so we +target the scenario root *without* a version segment: e.g. +``http://127.0.0.1:12580/tingly/experiment`` → ``…/tingly/experiment/v1/messages``, +which matches the gateway's ``/tingly/:scenario/v1`` route group. +""" + +from __future__ import annotations + +from typing import Any + + +def build_anthropic(base_url: str, token: str, timeout: float) -> Any: + try: + import anthropic + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The Anthropic transport requires the `anthropic` package. " + "Install it with `pip install tingly[anthropic]`." + ) from exc + + return anthropic.Anthropic( + base_url=base_url.rstrip("/"), + api_key=token or "tingly-box", + timeout=timeout, + ) + + +def build_async_anthropic(base_url: str, token: str, timeout: float) -> Any: + try: + import anthropic + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The Anthropic transport requires the `anthropic` package. " + "Install it with `pip install tingly[anthropic]`." + ) from exc + + return anthropic.AsyncAnthropic( + base_url=base_url.rstrip("/"), + api_key=token or "tingly-box", + timeout=timeout, + ) diff --git a/sdk/python/tingly/transports/openai_compat.py b/sdk/python/tingly/transports/openai_compat.py new file mode 100644 index 000000000..adc250fa6 --- /dev/null +++ b/sdk/python/tingly/transports/openai_compat.py @@ -0,0 +1,44 @@ +"""Build an ``openai.OpenAI`` client bound to a tingly-box scenario. + +The OpenAI Python SDK appends ``/chat/completions`` (etc.) to its ``base_url`` +and expects the version segment to be part of it, so we target the scenario +root + ``/v1``: e.g. ``http://127.0.0.1:12580/tingly/experiment/v1`` → +``…/tingly/experiment/v1/chat/completions``, which matches the gateway's +``/tingly/:scenario/v1`` route group. +""" + +from __future__ import annotations + +from typing import Any + + +def build_openai(base_url: str, token: str, timeout: float) -> Any: + try: + import openai + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The OpenAI transport requires the `openai` package. " + "Install it with `pip install tingly[openai]`." + ) from exc + + return openai.OpenAI( + base_url=base_url.rstrip("/") + "/v1", + api_key=token or "tingly-box", + timeout=timeout, + ) + + +def build_async_openai(base_url: str, token: str, timeout: float) -> Any: + try: + import openai + except ImportError as exc: # pragma: no cover - import guard + raise ImportError( + "The OpenAI transport requires the `openai` package. " + "Install it with `pip install tingly[openai]`." + ) from exc + + return openai.AsyncOpenAI( + base_url=base_url.rstrip("/") + "/v1", + api_key=token or "tingly-box", + timeout=timeout, + ) From 8c402913a89f91b27ba09be15394119fed275ba0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 07:25:02 +0000 Subject: [PATCH 02/28] docs(sdk): add end-to-end pencil graph to python-sdk.md --- .design/python-sdk.md | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index a6646e614..f2c9b1e32 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -50,6 +50,69 @@ connect(scenario="experiment") .guardrails → GET /api/v1/guardrails/config (admin token) ``` +## How it works (pencil) + +Two phases. **Provisioning** happens once in `connect()` (admin token, dashed +lines). **Inference** happens on every call (model token, solid lines) and +reuses the exact same gateway pipeline as any other tb client — the SDK adds no +new path through the box. + +``` + YOUR PYTHON tingly-box GATEWAY UPSTREAMS + ┌───────────────────────┐ ┌──────────────────────────────────┐ ┌───────────────┐ + │ import tingly │ │ │ │ Anthropic │ + │ tb = tingly.connect() │ │ /api/v1/... (admin auth) │ │ OpenAI │ + │ │ │ /tingly/:scn (model auth) │ │ Deepseek │ + └───────────┬───────────┘ └──────────────────────────────────┘ │ vLLM / local │ + │ └───────▲───────┘ + ── PROVISION (once, admin token) ─────────────────────────────────────────────────┊──────── + │ ┊ + config.resolve() ┊ + args→env→sdk.json→config.json→localhost ┊ + │ ┊ + │ GET /api/v1/info/version (liveness) ┄┄┄┄┄┄┄┄►┐ ┊ + │ POST /api/v1/sdk/session {scenario,name} ┄┄┄►│ CreateSDKSession ┊ + │ Authorization: Bearer │ · validate scenario in registry + │ │ · transport = openai|anthropic|both + │ ◄┄┄┄ {base_url, token=, │ · ready/services from active rule + │ transport, ready, services} ┄┄┄┄┄┄┄┄┄┄┄┄┄┘ + ▼ + Client ── builds lazily ──► openai.OpenAI(base_url = root+"/v1", api_key=ModelToken) + anthropic.Anthropic(base_url = root, api_key=ModelToken) + + ══ INFERENCE (every call, model token) ═══════════════════════════════════════════════════ + │ + tb.ask("...", model="auto") + tb.openai.chat.completions.create(...) + tb.anthropic.messages.create(...) + │ POST /tingly/experiment/v1/chat/completions ┌─────────────────────────┐ + │ POST /tingly/experiment/v1/messages │ the SAME pipeline as │ + │ Authorization: Bearer ─────────►│ any other tb client │ + │ │ │ + │ │ scenario → rule resolve │ + │ │ guard rails (in/out) │ + │ │ smart routing / tiers │ + │ │ circuit-breaker failover│──► pick + │ │ quota + usage logging │ upstream + │ │ protocol transform │──────────► + │ ◄──────────── response (+ usage recorded) ──────└─────────────────────────┘ (solid) + ▼ + tb.usage.this_session() GET /api/v1/requests (admin token, read-back) + tb.guardrails.status() GET /api/v1/guardrails/config (admin token, read-back) +``` + +Key reading of the graph: + +- The SDK never talks to providers directly — the rightmost column is reachable + **only** through the gateway box in the middle. That is the whole point: the + experiment inherits routing/fallback/guard-rails/quota for free. +- Provisioning (dashed) uses the **admin** token and the `/api/v1/*` control + plane; inference (solid) uses the **model** token and the `/tingly/:scenario` + data plane. Different tokens, different surfaces. +- The inference box is *unchanged* tb internals — the SDK contributes the new + `experiment` scenario and the one provisioning endpoint, nothing in the hot + path. + ## Two-token model - **Admin token** (tb's `UserToken`): authorizes `POST /sdk/session`. Resolved From e6146845587058a26978d2d54135ae65aaed57b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 07:31:25 +0000 Subject: [PATCH 03/28] docs(sdk): document Layer 3 plugin-as-upstream with pencil graph --- .design/python-sdk.md | 53 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index f2c9b1e32..30ba6484e 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -184,5 +184,54 @@ users can name parallel experiments via profiles (`experiment:p1`). 3. Async client (`AsyncClient`, `aask`) — transports already have async builders. 4. Layer 2: `tingly.Plugin`, manifest, sub-process supervision (reuse `agentboot/process`), `/plugins//*` reverse proxy, lifecycle UI. -5. Layer 3: auto-register a plugin tool as a virtual model via - `vmodel/virtualserver`. +5. Layer 3: expose a plugin as a model tb can route to (see below). + +## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) + +Layer 1 points the **data-flow into** tb: the plugin is a *consumer*. For tb to +*select* a plugin as a model, the flow inverts — the plugin becomes a +*producer*, an HTTP upstream tb calls out to. That is Layer 2's `Plugin.serve()`. + +There are two distinct "virtual model" notions; only one fits an out-of-process +Python plugin: + +| | in-process `vmodel` (`AuthType=virtual`) | provider-as-upstream | +|---|---|---| +| what | Go code implementing `openai.VirtualModel` / `anthropic.VirtualModel`, compiled in (`ai/provider.go:IsVirtual` → `virtualModelService`) | a normal provider whose `api_base` is an external HTTP server speaking `/v1/chat/completions` or `/v1/messages` | +| lives | inside tb's process | out-of-process, any language | +| Python plugin fit | ✗ (needs a Go shim forwarding to Python) | ✓ natural route | + +So a Python plugin is selected by registering it as a **provider/upstream**, not +via the in-process `vmodel` package. + +``` + ANY tb client tingly-box GATEWAY UPSTREAMS + ┌──────────────┐ ┌──────────────────────────────┐ + │ Claude Code │ │ HandleOpenAIChatCompletions │ tier 1 ┌──────────────┐ + │ Cursor │ model │ scenario → rule resolve │ ┌───────► │ Anthropic / │ + │ tb UI ├───────►│ guard rails (in/out) │ │ fallback│ OpenAI (real)│ + │ tingly.ask() │ "plugin│ smart routing / TIERS ──────┼──┤ └──────────────┘ + └──────────────┘ /my-rag"│ circuit-breaker failover │ │ tier 0 ┌──────────────┐ + │ quota + usage logging │ └───────► │ my-rag PLUGIN│ ◄─ Layer 2 + │ provider.api_base = plugin │ POST │ POST /v1/chat│ Plugin.serve() + └────────────────────────────────┘ /v1/chat │ /completions │ + └──────┬───────┘ + ctx.llm.ask() ┄┄┄┘ (plugin may + back INTO tb for its own LLM calls) +``` + +Wiring (no new gateway hot-path code — it's just a provider): + +1. **Plugin serves** `POST /v1/chat/completions` (Layer 2 `Plugin.serve()`). +2. **Register a provider**: `{name:"my-rag", api_base:"http://127.0.0.1:", + api_style:"openai", models:["plugin/my-rag"]}` — a *normal* provider, not + `AuthType=virtual`. +3. **Bind a rule/service**: model `plugin/my-rag` → that provider. +4. Now `model:"plugin/my-rag"` from any client resolves through the same + dispatcher as every other model. Put the plugin in tier 0 and a real model in + tier 1 and tb fails over automatically when the plugin is down. + +The deeper option — a true in-process `AuthType=virtual` vmodel — means writing +a small Go adapter implementing `openai.VirtualModel` that forwards to the Python +process. Only worth it to bundle the plugin with no separate port; +provider-as-upstream is simpler and already fully supported. From b1a98d026084161ccb441f557c3e0440bc06f2fb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 07:39:57 +0000 Subject: [PATCH 04/28] =?UTF-8?q?feat(sdk):=20Layer=202=20=E2=80=94=20ting?= =?UTF-8?q?ly.Plugin=20AI=20server=20+=20manifest=20+=20tb=20registration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the plugin SDK: write an OpenAI-compatible AI server in one class and let tingly-box route to it as a model. - Plugin: @plugin.chat handler (returns str or iterator of str), .serve() runs a stdlib ThreadingHTTPServer (no FastAPI) exposing /v1/chat/completions (buffered + real SSE), /v1/models, /health. - plugin.llm: lazy Layer-1 client so the plugin calls back into tb for its own generation instead of hard-coding a provider/key. - tingly.toml manifest (read/write/discover) describing name/model_id/ entrypoint/transport/port for a future tb-side supervisor. - register_with_tb(): creates a tb provider (POST /api/v1/providers) pointing at the plugin — the Layer 3 wiring. Rule/service binding left to the UI. - CLI: `tingly plugin {init,run,register}`. - Optional bearer-token auth on the plugin server. - Example examples/rag_plugin.py; tests for server wire-contract (incl. SSE), auth, and manifest round-trip. Full suite: 25 passing. Docs: .design/python-sdk.md gains a Layer 2 section; README plugin quickstart. --- .design/python-sdk.md | 77 ++++++++- sdk/python/README.md | 38 ++++- sdk/python/examples/rag_plugin.py | 48 ++++++ sdk/python/tests/test_plugin_manifest.py | 41 +++++ sdk/python/tests/test_plugin_server.py | 112 +++++++++++++ sdk/python/tingly/__init__.py | 3 + sdk/python/tingly/cli.py | 94 +++++++++++ sdk/python/tingly/plugin/__init__.py | 14 ++ sdk/python/tingly/plugin/core.py | 154 ++++++++++++++++++ sdk/python/tingly/plugin/manifest.py | 93 +++++++++++ sdk/python/tingly/plugin/register.py | 106 ++++++++++++ sdk/python/tingly/plugin/server.py | 198 +++++++++++++++++++++++ sdk/python/tingly/plugin/types.py | 76 +++++++++ 13 files changed, 1046 insertions(+), 8 deletions(-) create mode 100644 sdk/python/examples/rag_plugin.py create mode 100644 sdk/python/tests/test_plugin_manifest.py create mode 100644 sdk/python/tests/test_plugin_server.py create mode 100644 sdk/python/tingly/plugin/__init__.py create mode 100644 sdk/python/tingly/plugin/core.py create mode 100644 sdk/python/tingly/plugin/manifest.py create mode 100644 sdk/python/tingly/plugin/register.py create mode 100644 sdk/python/tingly/plugin/server.py create mode 100644 sdk/python/tingly/plugin/types.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 30ba6484e..f25b57925 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -24,13 +24,19 @@ this same module and the same `/sdk/session` provisioning seam. ``` sdk/python/ tingly/ - client.py # Client + connect() ← the whole user surface + client.py # Layer 1: Client + connect() ← consume tb discovery.py # probe gateway + POST /sdk/session config.py # (base_url, admin_token) resolution precedence scenarios.py # scenario + transport constants transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb helpers/ # usage + guardrails views - cli.py # `tingly doctor` + plugin/ # Layer 2: be an AI server tb routes to + core.py # Plugin class (@plugin.chat, .llm, .serve) + server.py # stdlib OpenAI-compatible HTTP server (+ SSE) + types.py # ChatRequest / Message + manifest.py # tingly.toml read/write + register.py # register the plugin as a tb provider (Layer 3) + cli.py # `tingly doctor` + `tingly plugin {init,run,register}` errors.py # TinglyError hierarchy ``` @@ -182,9 +188,70 @@ users can name parallel experiments via profiles (`experiment:p1`). 2. Dedicated `GET /api/v1/sdk/usage?session=` so usage doesn't scan `/api/v1/requests`. 3. Async client (`AsyncClient`, `aask`) — transports already have async builders. -4. Layer 2: `tingly.Plugin`, manifest, sub-process supervision (reuse - `agentboot/process`), `/plugins//*` reverse proxy, lifecycle UI. -5. Layer 3: expose a plugin as a model tb can route to (see below). +4. Layer 2 — Python side **done** (`tingly.Plugin`, manifest, OpenAI server, + `register`); remaining tb-side: sub-process supervisor from the manifest + (reuse `agentboot/process`), `/plugins//*` reverse proxy, lifecycle UI. + See the "Layer 2" section below. +5. Layer 3: expose a plugin as a model tb can route to (see "Layer 3" below). + +## Layer 2: write an AI server (`tingly.Plugin`) + +A plugin is an **OpenAI-compatible upstream**: the author writes one chat +handler, and `serve()` runs the HTTP server. The whole surface is one class. + +```python +from tingly import Plugin + +plugin = Plugin(name="my-rag") # model_id defaults to "plugin/my-rag" + +@plugin.chat +def handle(req): # req: ChatRequest + docs = retrieve(req.last_user_text()) + return plugin.llm.ask( # ← calls BACK into tb (Layer 1) + f"Using {docs}, answer: {req.last_user_text()}", model="auto" + ) + +if __name__ == "__main__": + plugin.serve() # http://127.0.0.1:8765/v1 +``` + +Design choices: + +- **No framework dependency.** The server is `http.server.ThreadingHTTPServer` + (stdlib), so a plugin is one `pip install tingly` away. It serves + `POST /v1/chat/completions` (buffered **and** real SSE streaming), + `GET /v1/models`, `GET /health` — exactly what tb needs to treat it as an + OpenAI upstream. +- **Handler contract is minimal.** Return a `str` (buffered) or an iterator of + `str` (streamed); the server shapes both into `chat.completion` / + `chat.completion.chunk`. The author never touches wire format. +- **`plugin.llm` is a lazy Layer-1 client.** The plugin reuses the gateway for + its own generation instead of hard-coding a provider/key — the recursion in + the Layer 3 graph. +- **`tingly.toml` manifest** (`manifest.py`) declares name / model_id / + entrypoint / transport / port, so a future tb-side supervisor can install and + run the plugin. `tingly plugin init` scaffolds a module + manifest. +- **Optional token auth.** `Plugin(api_key=...)` enforces a bearer token so only + tb (carrying the matching provider token) can call it. + +CLI: + +``` +tingly plugin init my-rag # scaffold my_rag_plugin.py + tingly.toml +tingly plugin run my_rag_plugin.py # serve it +tingly plugin register my-rag \ # wire it into tb as a provider (Layer 3) + --url http://127.0.0.1:8765/v1 --model-id plugin/my-rag +``` + +`register` uses the existing `POST /api/v1/providers` endpoint (admin token, +resolved like `connect()`). Creating the *rule/service* that maps the model into +a scenario is still a user/UI step — the provider is the part the SDK does +idempotently. + +**Not yet built (tb-side):** a sub-process supervisor that boots plugins from +their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy +mount, and the install/enable/logs/disable lifecycle UI. The Python side and the +provider wiring are complete; those are the remaining backend pieces. ## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) diff --git a/sdk/python/README.md b/sdk/python/README.md index d86aab581..81dc09115 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -53,8 +53,40 @@ tingly doctor --link # save gateway URL + token to ~/.tingly-box/sdk.json A green `tingly doctor` is a guarantee your code will run. +## Write a plugin (an AI server tb can route to) + +A plugin is an OpenAI-compatible upstream. Write one handler, serve it, register +it — then any tb client can select it as a model. + +```python +from tingly import Plugin + +plugin = Plugin(name="my-rag") # model id: plugin/my-rag + +@plugin.chat +def handle(req): + docs = retrieve(req.last_user_text()) + return plugin.llm.ask(f"Using {docs}, answer: {req.last_user_text()}") + +if __name__ == "__main__": + plugin.serve() # http://127.0.0.1:8765/v1 +``` + +```bash +tingly plugin init my-rag # scaffold module + tingly.toml +tingly plugin run my_rag_plugin.py # serve +tingly plugin register my-rag \ # wire into tb as a provider + --url http://127.0.0.1:8765/v1 --model-id plugin/my-rag +``` + +The server is stdlib-only (no FastAPI), supports streaming, and `plugin.llm` +calls back into tb so the plugin reuses the gateway for its own LLM work. + ## Status -This is **Layer 1** (client-side library). Layer 2 (tb-hosted plugins with a -manifest and lifecycle UI) and Layer 3 (plugin-as-virtual-model) build on the -same module. See `.design/python-sdk.md` in the repo for the full design. +- **Layer 1** (consume tb): `connect()` → `Client`. Done. +- **Layer 2** (be an AI server): `tingly.Plugin` + manifest + `register`. Python + side done; tb-side supervisor/lifecycle UI pending. +- **Layer 3** (tb routes to the plugin as a model): via provider-as-upstream. + +See `.design/python-sdk.md` in the repo for the full design and diagrams. diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py new file mode 100644 index 000000000..dede0c610 --- /dev/null +++ b/sdk/python/examples/rag_plugin.py @@ -0,0 +1,48 @@ +"""A RAG plugin served as an OpenAI-compatible upstream for tingly-box. + +Run it: + + pip install -e ".[all]" # from sdk/python + python examples/rag_plugin.py # serves on http://127.0.0.1:8765/v1 + +Then register it with tb so any client can select model `plugin/rag-demo`: + + tingly plugin register rag-demo --url http://127.0.0.1:8765/v1 --model-id plugin/rag-demo + +Now `model="plugin/rag-demo"` from Claude Code, Cursor, the tb UI, or another +`tingly.connect()` experiment routes here — with tb's guard rails, quota, +logging and tier-failover applied. The handler itself calls *back* into tb via +`plugin.llm` for the generation step. +""" + +from tingly import Plugin + +plugin = Plugin(name="rag-demo", description="Answers from a toy in-memory corpus") + +CORPUS = { + "tingly-box": "tingly-box is a personal intelligence orchestrator: an LLM " + "gateway with remote control and guard rails.", + "plugin": "A tingly plugin is an OpenAI-compatible upstream that tingly-box " + "can route to as a model.", +} + + +def retrieve(question: str) -> str: + q = question.lower() + hits = [text for key, text in CORPUS.items() if key in q] + return "\n".join(hits) or "(no matching documents)" + + +@plugin.chat +def handle(req): + question = req.last_user_text() + docs = retrieve(question) + # Generation goes back through tingly-box — no provider/key hard-coded here. + return plugin.llm.ask( + f"Using only these documents:\n{docs}\n\nAnswer: {question}", + model="auto", + ) + + +if __name__ == "__main__": + plugin.serve() diff --git a/sdk/python/tests/test_plugin_manifest.py b/sdk/python/tests/test_plugin_manifest.py new file mode 100644 index 000000000..322f4b00e --- /dev/null +++ b/sdk/python/tests/test_plugin_manifest.py @@ -0,0 +1,41 @@ +"""Manifest round-trip + discovery tests.""" + +from pathlib import Path + +from tingly import Plugin +from tingly.plugin import manifest as m + + +def test_manifest_roundtrip(tmp_path: Path): + man = m.Manifest( + name="my-rag", + model_id="plugin/my-rag", + entrypoint="rag_plugin:plugin", + description='has "quotes" and \\ slash', + port=9001, + ) + man.write(tmp_path) + loaded = m.load(tmp_path) + assert loaded.name == "my-rag" + assert loaded.model_id == "plugin/my-rag" + assert loaded.entrypoint == "rag_plugin:plugin" + assert loaded.port == 9001 + assert loaded.description == 'has "quotes" and \\ slash' + + +def test_manifest_find_walks_up(tmp_path: Path): + m.Manifest(name="p", model_id="plugin/p", entrypoint="p:plugin").write(tmp_path) + nested = tmp_path / "a" / "b" + nested.mkdir(parents=True) + found = m.find(nested) + assert found is not None + assert found.name == "p" + + +def test_plugin_builds_manifest(): + plugin = Plugin(name="my-rag", description="d") + man = plugin.manifest(entrypoint="rag_plugin:plugin", port=8080) + assert man.model_id == "plugin/my-rag" + assert man.entrypoint == "rag_plugin:plugin" + assert man.port == 8080 + assert man.transport == "openai" diff --git a/sdk/python/tests/test_plugin_server.py b/sdk/python/tests/test_plugin_server.py new file mode 100644 index 000000000..d556d91f3 --- /dev/null +++ b/sdk/python/tests/test_plugin_server.py @@ -0,0 +1,112 @@ +"""Plugin server tests — drive a real (ephemeral-port) plugin over HTTP. + +These pin the OpenAI wire contract tingly-box relies on when it routes to a +plugin as an upstream: chat.completion shape, SSE streaming, /v1/models, auth. +""" + +import json + +import httpx +import pytest + +from tingly import ChatRequest, Plugin + + +@pytest.fixture +def served(): + plugin = Plugin(name="t-plug", model_id="plugin/t-plug") + + @plugin.chat + def handle(req: ChatRequest): + if req.stream: + return iter(["he", "llo ", req.last_user_text()]) + return f"echo: {req.last_user_text()}" + + port = plugin.serve(port=0, verbose=False, block=False) + base = f"http://127.0.0.1:{port}" + yield base + plugin.stop() + + +def test_health(served): + r = httpx.get(f"{served}/health", timeout=5) + assert r.status_code == 200 + assert r.json()["status"] == "ok" + + +def test_models_lists_model_id(served): + r = httpx.get(f"{served}/v1/models", timeout=5) + assert r.status_code == 200 + ids = [m["id"] for m in r.json()["data"]] + assert "plugin/t-plug" in ids + + +def test_chat_completion_shape(served): + r = httpx.post( + f"{served}/v1/chat/completions", + json={"model": "plugin/t-plug", "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert r.status_code == 200 + body = r.json() + assert body["object"] == "chat.completion" + assert body["choices"][0]["message"]["content"] == "echo: hi" + assert body["choices"][0]["finish_reason"] == "stop" + + +def test_chat_completion_streaming(served): + with httpx.stream( + "POST", + f"{served}/v1/chat/completions", + json={ + "model": "plugin/t-plug", + "messages": [{"role": "user", "content": "world"}], + "stream": True, + }, + timeout=5, + ) as r: + assert r.status_code == 200 + deltas = [] + saw_done = False + for line in r.iter_lines(): + if not line.startswith("data: "): + continue + data = line[len("data: "):] + if data == "[DONE]": + saw_done = True + continue + chunk = json.loads(data) + assert chunk["object"] == "chat.completion.chunk" + delta = chunk["choices"][0]["delta"] + if delta.get("content"): + deltas.append(delta["content"]) + assert "".join(deltas) == "hello world" + assert saw_done + + +def test_auth_enforced_when_key_set(): + plugin = Plugin(name="auth-plug", api_key="secret") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False) + base = f"http://127.0.0.1:{port}" + try: + bad = httpx.post( + f"{base}/v1/chat/completions", + json={"model": "x", "messages": []}, + timeout=5, + ) + assert bad.status_code == 401 + + good = httpx.post( + f"{base}/v1/chat/completions", + headers={"Authorization": "Bearer secret"}, + json={"model": "x", "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert good.status_code == 200 + finally: + plugin.stop() diff --git a/sdk/python/tingly/__init__.py b/sdk/python/tingly/__init__.py index e0cc619e6..5e78ba6b8 100644 --- a/sdk/python/tingly/__init__.py +++ b/sdk/python/tingly/__init__.py @@ -20,11 +20,14 @@ TinglyError, UpstreamError, ) +from .plugin import ChatRequest, Plugin __all__ = [ "__version__", "connect", "Client", + "Plugin", + "ChatRequest", "TinglyError", "GatewayUnreachableError", "AuthError", diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py index 04909eb0d..d8b383dcc 100644 --- a/sdk/python/tingly/cli.py +++ b/sdk/python/tingly/cli.py @@ -116,6 +116,60 @@ def _do_link() -> None: print(f"Wrote {path}") +def _plugin_init(name: str) -> int: + """Scaffold a minimal plugin: a starter module + tingly.toml.""" + from pathlib import Path + + from .plugin.manifest import Manifest + + safe = name.replace("-", "_") + module = f"{safe}_plugin.py" + Path(module).write_text(_PLUGIN_TEMPLATE.format(name=name), encoding="utf-8") + Manifest( + name=name, + model_id=f"plugin/{name}", + entrypoint=f"{safe}_plugin:plugin", + description=f"{name} plugin", + ).write(Path.cwd()) + print(f"Created {module} and tingly.toml.") + print(f"Run it with: python {module} (or: tingly plugin run {module})") + return 0 + + +def _plugin_run(target: str) -> int: + """Import a plugin (``module:attr`` or a .py path) and serve it.""" + import importlib + import importlib.util + from pathlib import Path + + if target.endswith(".py") or "/" in target: + path = Path(target) + spec = importlib.util.spec_from_file_location(path.stem, path) + mod = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + spec.loader.exec_module(mod) # type: ignore[union-attr] + plugin = getattr(mod, "plugin", None) + else: + mod_name, _, attr = target.partition(":") + mod = importlib.import_module(mod_name) + plugin = getattr(mod, attr or "plugin", None) + + if plugin is None: + print(f"No `plugin` found in {target!r}") + return 1 + plugin.serve() + return 0 + + +def _plugin_register(name: str, plugin_url: str, model_id: str, token: str) -> int: + from .plugin.register import register_with_tb + + result = register_with_tb(name, plugin_url, model_id, token=token) + status = OK if result.created else WARN + _row("provider", f"{result.name} → {result.api_base}", status) + print("\n" + result.note) + return 0 + + def main(argv: Optional[list] = None) -> int: parser = argparse.ArgumentParser(prog="tingly", description="tingly-box Python SDK") sub = parser.add_subparsers(dest="command") @@ -128,13 +182,53 @@ def main(argv: Optional[list] = None) -> int: "--link", action="store_true", help="prompt for and save gateway URL + token" ) + p_plugin = sub.add_parser("plugin", help="author / run / register a plugin") + psub = p_plugin.add_subparsers(dest="plugin_command") + p_init = psub.add_parser("init", help="scaffold a starter plugin") + p_init.add_argument("name", help="plugin name, e.g. my-rag") + p_run = psub.add_parser("run", help="serve a plugin (module:attr or path.py)") + p_run.add_argument("target", help="e.g. my_rag_plugin:plugin or my_rag_plugin.py") + p_reg = psub.add_parser("register", help="register a running plugin with tb") + p_reg.add_argument("name", help="provider name to create in tb") + p_reg.add_argument("--url", required=True, help="plugin OpenAI base, e.g. http://127.0.0.1:8765/v1") + p_reg.add_argument("--model-id", required=True, help="model id, e.g. plugin/my-rag") + p_reg.add_argument("--token", default="", help="token tb should send to the plugin") + args = parser.parse_args(argv) if args.command == "doctor": return doctor(args.scenario, args.link) + if args.command == "plugin": + if args.plugin_command == "init": + return _plugin_init(args.name) + if args.plugin_command == "run": + return _plugin_run(args.target) + if args.plugin_command == "register": + return _plugin_register(args.name, args.url, args.model_id, args.token) + p_plugin.print_help() + return 0 parser.print_help() return 0 +_PLUGIN_TEMPLATE = '''"""A tingly-box plugin: an OpenAI-compatible AI server backed by the gateway.""" + +from tingly import Plugin + +plugin = Plugin(name="{name}") + + +@plugin.chat +def handle(req): + question = req.last_user_text() + # Your logic here. Call back into tingly-box for LLM work via plugin.llm: + return plugin.llm.ask(f"Answer concisely: {{question}}", model="auto") + + +if __name__ == "__main__": + plugin.serve() +''' + + if __name__ == "__main__": sys.exit(main()) diff --git a/sdk/python/tingly/plugin/__init__.py b/sdk/python/tingly/plugin/__init__.py new file mode 100644 index 000000000..290a33c75 --- /dev/null +++ b/sdk/python/tingly/plugin/__init__.py @@ -0,0 +1,14 @@ +"""tingly.plugin — write an AI server tingly-box can route to as a model. + +This is Layer 2 of the SDK. A :class:`Plugin` is an OpenAI-compatible upstream; +register it as a provider in tingly-box and any client can select ``model_id``, +inheriting the gateway's routing / fallback / guard rails / quota / logging. +""" + +from __future__ import annotations + +from .core import Plugin +from .manifest import Manifest +from .types import ChatRequest, Message + +__all__ = ["Plugin", "Manifest", "ChatRequest", "Message"] diff --git a/sdk/python/tingly/plugin/core.py b/sdk/python/tingly/plugin/core.py new file mode 100644 index 000000000..74826284f --- /dev/null +++ b/sdk/python/tingly/plugin/core.py @@ -0,0 +1,154 @@ +"""The ``Plugin`` class — write an AI server that tingly-box can route to. + +A plugin is an OpenAI-compatible upstream: the author registers a single chat +handler, and ``serve()`` runs the HTTP server. tingly-box is then pointed at it +as a provider (``api_base = http://host:port``, model ``model_id``), so the +plugin composes with routing, fallback, guard rails, quota and logging like any +other model. + +The handler may call **back into** tingly-box via ``plugin.llm`` (a Layer-1 +:class:`~tingly.client.Client`) for its own LLM needs — the recursion shown in +the Layer 3 design graph. + + from tingly import Plugin + + plugin = Plugin(name="my-rag") + + @plugin.chat + def handle(req): + docs = retrieve(req.last_user_text()) + return plugin.llm.ask(f"Using {docs}, answer: {req.last_user_text()}") + + if __name__ == "__main__": + plugin.serve() +""" + +from __future__ import annotations + +import threading +from typing import Callable, Optional + +from .manifest import Manifest +from .server import Dispatch, HandlerResult, make_server +from .types import ChatRequest + +ChatHandler = Callable[[ChatRequest], HandlerResult] + + +class Plugin: + """An OpenAI-compatible AI server backed by the tingly-box gateway.""" + + def __init__( + self, + name: str, + *, + model_id: Optional[str] = None, + version: str = "0.1.0", + description: str = "", + api_key: str = "", + scenario: str = "experiment", + ): + self.name = name + self.model_id = model_id or f"plugin/{name}" + self.version = version + self.description = description + self.api_key = api_key + self.scenario = scenario + + self._handler: Optional[ChatHandler] = None + self._llm = None # lazy Layer-1 client + self._httpd = None + + # -- authoring ------------------------------------------------------- + + def chat(self, fn: ChatHandler) -> ChatHandler: + """Register the plugin's chat handler. Decorator form. + + The handler receives a :class:`ChatRequest` and returns either a string + (buffered) or an iterator of strings (streamed). + """ + self._handler = fn + return fn + + @property + def llm(self): + """A lazily-connected Layer-1 client for calling back into tingly-box. + + Lets a plugin reuse the gateway for its own model calls, so it never + hard-codes a provider or key. + """ + if self._llm is None: + from ..client import connect + + self._llm = connect(scenario=self.scenario, name=f"plugin:{self.name}") + return self._llm + + # -- dispatch -------------------------------------------------------- + + def _dispatch(self, req: ChatRequest) -> HandlerResult: + if self._handler is None: + raise RuntimeError( + f"plugin {self.name!r} has no chat handler; decorate one with @plugin.chat" + ) + return self._handler(req) + + # -- manifest -------------------------------------------------------- + + def manifest(self, entrypoint: str, port: int = 8765, transport: str = "openai") -> Manifest: + """Build a :class:`Manifest` describing this plugin for tingly-box.""" + return Manifest( + name=self.name, + model_id=self.model_id, + entrypoint=entrypoint, + version=self.version, + transport=transport, + port=port, + description=self.description, + ) + + # -- serving --------------------------------------------------------- + + def serve( + self, + host: str = "127.0.0.1", + port: int = 8765, + *, + verbose: bool = True, + block: bool = True, + ) -> int: + """Run the plugin's HTTP server. + + Returns the bound port (resolved even when ``port=0``). With + ``block=False`` the server runs on a daemon thread and the call returns + immediately — handy for tests and for embedding. + """ + httpd, bound = make_server( + self._dispatch, + self.model_id, + host=host, + port=port, + api_key=self.api_key, + verbose=verbose, + ) + self._httpd = httpd + if verbose: + print( + f"[tingly] plugin {self.name!r} serving model {self.model_id!r} " + f"on http://{host}:{bound}/v1 (register as an OpenAI provider in tb)" + ) + if not block: + t = threading.Thread(target=httpd.serve_forever, daemon=True) + t.start() + return bound + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + finally: + httpd.shutdown() + return bound + + def stop(self) -> None: + if self._httpd is not None: + self._httpd.shutdown() + self._httpd = None diff --git a/sdk/python/tingly/plugin/manifest.py b/sdk/python/tingly/plugin/manifest.py new file mode 100644 index 000000000..513dd76ec --- /dev/null +++ b/sdk/python/tingly/plugin/manifest.py @@ -0,0 +1,93 @@ +"""``tingly.toml`` plugin manifest — read/write. + +The manifest is what tingly-box (Layer 2 supervisor) reads to install and run a +plugin, and what `tingly plugin register` uses to wire the plugin in as an +upstream provider. It is deliberately tiny: + + [plugin] + name = "my-rag" + model_id = "plugin/my-rag" + version = "0.1.0" + entrypoint = "rag_plugin:plugin" # module:attr that yields a Plugin + transport = "openai" # openai | anthropic + port = 8765 + description = "Answers from my private corpus" +""" + +from __future__ import annotations + +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Optional + +if sys.version_info >= (3, 11): + import tomllib # type: ignore +else: # pragma: no cover - 3.9/3.10 fallback + try: + import tomli as tomllib # type: ignore + except ImportError: + tomllib = None # type: ignore + +MANIFEST_NAME = "tingly.toml" + + +@dataclass +class Manifest: + name: str + model_id: str + entrypoint: str + version: str = "0.1.0" + transport: str = "openai" + port: int = 8765 + description: str = "" + + def to_toml(self) -> str: + d = asdict(self) + lines = ["[plugin]"] + for key in ("name", "model_id", "version", "entrypoint", "transport", "description"): + lines.append(f'{key} = "{_escape(str(d[key]))}"') + lines.append(f"port = {int(d['port'])}") + return "\n".join(lines) + "\n" + + def write(self, directory: Path) -> Path: + path = Path(directory) / MANIFEST_NAME + path.write_text(self.to_toml(), encoding="utf-8") + return path + + +def load(path: Path) -> Manifest: + """Load a manifest from a ``tingly.toml`` file or its containing directory.""" + p = Path(path) + if p.is_dir(): + p = p / MANIFEST_NAME + if tomllib is None: # pragma: no cover + raise RuntimeError( + "reading tingly.toml needs Python 3.11+ or the `tomli` package" + ) + with p.open("rb") as fh: + data = tomllib.load(fh) + plugin = data.get("plugin") or {} + return Manifest( + name=plugin["name"], + model_id=plugin.get("model_id", f"plugin/{plugin['name']}"), + entrypoint=plugin["entrypoint"], + version=plugin.get("version", "0.1.0"), + transport=plugin.get("transport", "openai"), + port=int(plugin.get("port", 8765)), + description=plugin.get("description", ""), + ) + + +def find(start: Optional[Path] = None) -> Optional[Manifest]: + """Search upward from ``start`` (cwd by default) for a ``tingly.toml``.""" + cur = Path(start or Path.cwd()).resolve() + for directory in [cur, *cur.parents]: + candidate = directory / MANIFEST_NAME + if candidate.exists(): + return load(candidate) + return None + + +def _escape(s: str) -> str: + return s.replace("\\", "\\\\").replace('"', '\\"') diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py new file mode 100644 index 000000000..ffa91bf44 --- /dev/null +++ b/sdk/python/tingly/plugin/register.py @@ -0,0 +1,106 @@ +"""Register a running plugin with tingly-box as an upstream provider. + +This is the Layer 3 wiring: it creates a provider whose ``api_base`` points at +the plugin's HTTP server, so tingly-box can route a model id to it. Creating the +*rule/service* that maps the model into a scenario is left to the user (or the +tb UI) for now — the provider is the part the SDK can do safely and idempotently. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import httpx + +from .. import config as _config +from ..errors import AuthError, GatewayUnreachableError + + +@dataclass +class RegisterResult: + provider_uuid: Optional[str] + name: str + api_base: str + model_id: str + created: bool + note: str + + +def register_with_tb( + name: str, + plugin_url: str, + model_id: str, + *, + gateway_url: Optional[str] = None, + admin_token: Optional[str] = None, + token: str = "", + timeout: float = 30.0, +) -> RegisterResult: + """Create (or report) a tingly-box provider pointing at the plugin. + + Args: + name: provider name to create in tb (e.g. the plugin name). + plugin_url: the plugin's OpenAI base, e.g. ``http://127.0.0.1:8765/v1``. + model_id: the model id the plugin advertises (e.g. ``plugin/my-rag``). + gateway_url / admin_token: tb gateway + admin token; auto-discovered if + omitted (same precedence as ``connect()``). + token: optional API token tb should send to the plugin (matches the + plugin's ``api_key`` if it enforces one). + """ + resolved = _config.resolve(base_url=gateway_url, token=admin_token) + headers = {"Authorization": f"Bearer {resolved.token or ''}"} + url = resolved.base_url.rstrip("/") + "/api/v1/providers" + payload = { + "name": name, + "api_base": plugin_url, + "api_style": "openai", + "token": token, + "no_key_required": token == "", + "enabled": True, + "auth_type": "api_key", + } + + try: + resp = httpx.post(url, json=payload, headers=headers, timeout=timeout) + except httpx.HTTPError as exc: + raise GatewayUnreachableError( + f"could not reach tingly-box at {resolved.base_url}: {exc}" + ) from exc + + if resp.status_code == 401: + raise AuthError("tingly-box rejected the admin token while creating the provider") + + note = ( + f"Provider created. Bind a rule mapping model {model_id!r} to this " + f"provider (tb UI → Rules) so clients can select it." + ) + created = resp.status_code in (200, 201) + if not created: + # A name clash (already registered) is fine and idempotent enough. + note = ( + f"Provider not created (HTTP {resp.status_code}: {resp.text[:160]}). " + f"It may already exist — verify in the tb UI." + ) + + data = _safe_json(resp) or {} + provider_uuid = ( + (data.get("data") or {}).get("uuid") + or data.get("uuid") + or None + ) + return RegisterResult( + provider_uuid=provider_uuid, + name=name, + api_base=plugin_url, + model_id=model_id, + created=created, + note=note, + ) + + +def _safe_json(resp: httpx.Response): + try: + return resp.json() + except ValueError: + return None diff --git a/sdk/python/tingly/plugin/server.py b/sdk/python/tingly/plugin/server.py new file mode 100644 index 000000000..8b2a90ef5 --- /dev/null +++ b/sdk/python/tingly/plugin/server.py @@ -0,0 +1,198 @@ +"""A tiny OpenAI-compatible HTTP server for plugins (stdlib only). + +Exposes exactly what tingly-box needs to treat the plugin as an upstream +OpenAI provider: + + POST /v1/chat/completions -> chat.completion (+ SSE when stream=true) + GET /v1/models -> the plugin's model id + GET /health -> liveness + +No framework dependency — uses ``http.server.ThreadingHTTPServer`` so a plugin +stays a single ``pip install tingly`` away. Streaming is real SSE: a handler +that returns an iterator is emitted as ``chat.completion.chunk`` frames. +""" + +from __future__ import annotations + +import json +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Callable, Dict, Iterable, Iterator, Tuple, Union + +from .types import ChatRequest + +# A handler returns buffered text or an iterator of text deltas. +HandlerResult = Union[str, Iterable[str]] +Dispatch = Callable[[ChatRequest], HandlerResult] + + +class _Handler(BaseHTTPRequestHandler): + # set by make_server via closure attributes on the server instance + server_version = "tingly-plugin/0.1" + + def log_message(self, fmt, *args): # quiet by default; plugin owns logging + if getattr(self.server, "verbose", False): + super().log_message(fmt, *args) + + # -- routing --------------------------------------------------------- + + def do_GET(self): + if self.path.rstrip("/") == "/health": + return self._json(200, {"status": "ok"}) + if self.path.rstrip("/") in ("/v1/models", "/models"): + return self._models() + return self._json(404, {"error": {"message": "not found", "type": "not_found"}}) + + def do_POST(self): + if self.path.rstrip("/") not in ("/v1/chat/completions", "/chat/completions"): + return self._json( + 404, {"error": {"message": "not found", "type": "not_found"}} + ) + if not self._authorized(): + return self._json( + 401, {"error": {"message": "invalid token", "type": "auth_error"}} + ) + body = self._read_json() + if body is None: + return self._json( + 400, {"error": {"message": "invalid JSON body", "type": "invalid_request_error"}} + ) + + req = ChatRequest.from_openai_body(body) + try: + result = self.server.dispatch(req) # type: ignore[attr-defined] + except Exception as exc: # noqa: BLE001 - surface as upstream 500 + return self._json( + 500, + {"error": {"message": f"plugin handler error: {exc}", "type": "api_error"}}, + ) + + model = req.model or self.server.model_id # type: ignore[attr-defined] + if req.stream: + return self._stream(result, model) + return self._complete(result, model) + + # -- responses ------------------------------------------------------- + + def _models(self): + model_id = self.server.model_id # type: ignore[attr-defined] + self._json( + 200, + { + "object": "list", + "data": [ + { + "id": model_id, + "object": "model", + "created": int(time.time()), + "owned_by": "tingly-plugin", + } + ], + }, + ) + + def _complete(self, result: HandlerResult, model: str): + text = result if isinstance(result, str) else "".join(result) + self._json( + 200, + { + "id": _chat_id(), + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + }, + ) + + def _stream(self, result: HandlerResult, model: str): + cid = _chat_id() + created = int(time.time()) + # Close-delimited SSE: no Content-Length, connection closes at end so the + # client's stream iterator terminates after the [DONE] frame. + self.close_connection = True + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + + def frame(delta: Dict[str, Any], finish: Any = None) -> bytes: + payload = { + "id": cid, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + return f"data: {json.dumps(payload)}\n\n".encode("utf-8") + + # role preamble, then content deltas, then terminal frame + [DONE] + self.wfile.write(frame({"role": "assistant"})) + chunks: Iterator[str] = iter([result]) if isinstance(result, str) else iter(result) + for piece in chunks: + if piece: + self.wfile.write(frame({"content": piece})) + self.wfile.flush() + self.wfile.write(frame({}, finish="stop")) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + # -- helpers --------------------------------------------------------- + + def _authorized(self) -> bool: + expected = self.server.api_key # type: ignore[attr-defined] + if not expected: + return True + header = self.headers.get("Authorization", "") + token = header[7:] if header.startswith("Bearer ") else header + return token.strip() == expected + + def _read_json(self): + try: + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length) if length else b"" + return json.loads(raw or b"{}") + except (ValueError, OSError): + return None + + def _json(self, status: int, payload: Dict[str, Any]): + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +def _chat_id() -> str: + return "chatcmpl-" + uuid.uuid4().hex[:24] + + +def make_server( + dispatch: Dispatch, + model_id: str, + host: str, + port: int, + api_key: str = "", + verbose: bool = False, +) -> Tuple[ThreadingHTTPServer, int]: + """Build (but do not start) the plugin's HTTP server. + + Returns the server and the resolved port (useful when ``port=0`` asks the + OS for an ephemeral one — handy in tests). + """ + httpd = ThreadingHTTPServer((host, port), _Handler) + # stash config on the server instance for the handler to read + httpd.dispatch = dispatch # type: ignore[attr-defined] + httpd.model_id = model_id # type: ignore[attr-defined] + httpd.api_key = api_key # type: ignore[attr-defined] + httpd.verbose = verbose # type: ignore[attr-defined] + return httpd, httpd.server_address[1] diff --git a/sdk/python/tingly/plugin/types.py b/sdk/python/tingly/plugin/types.py new file mode 100644 index 000000000..8d054af8c --- /dev/null +++ b/sdk/python/tingly/plugin/types.py @@ -0,0 +1,76 @@ +"""Request/response value types passed to a plugin's chat handler. + +The handler sees a normalized :class:`ChatRequest` regardless of wire details, +and returns either a ``str`` (buffered) or an iterator of ``str`` (streamed) — +the server shapes those into OpenAI ``chat.completion`` / ``chat.completion.chunk`` +payloads so tingly-box (or any OpenAI client) can consume the plugin as an +upstream model. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class Message: + role: str + content: str + + +@dataclass +class ChatRequest: + """A normalized chat request handed to a plugin handler.""" + + model: str + messages: List[Message] + stream: bool = False + # Everything else from the wire body, untouched (temperature, tools, …). + extra: Dict[str, Any] = field(default_factory=dict) + + def last_user_text(self) -> str: + """The text of the most recent user message (empty string if none).""" + for msg in reversed(self.messages): + if msg.role == "user": + return msg.content + return "" + + def system_text(self) -> Optional[str]: + for msg in self.messages: + if msg.role == "system": + return msg.content + return None + + @classmethod + def from_openai_body(cls, body: Dict[str, Any]) -> "ChatRequest": + raw_messages = body.get("messages") or [] + messages = [ + Message(role=m.get("role", "user"), content=_content_to_text(m.get("content"))) + for m in raw_messages + ] + known = {"model", "messages", "stream"} + extra = {k: v for k, v in body.items() if k not in known} + return cls( + model=body.get("model", ""), + messages=messages, + stream=bool(body.get("stream", False)), + extra=extra, + ) + + +def _content_to_text(content: Any) -> str: + """Flatten OpenAI content (str or list of parts) to plain text.""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for part in content: + if isinstance(part, dict): + parts.append(part.get("text", "")) + else: + parts.append(str(part)) + return "".join(parts) + return str(content) From b7cdfab03c7b61c93b62acb96234eea5a2112ae8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 08:01:30 +0000 Subject: [PATCH 05/28] docs(sdk): add Layer 2 plugin lifecycle pencil graph --- .design/python-sdk.md | 59 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index f25b57925..84004914f 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -215,6 +215,65 @@ if __name__ == "__main__": plugin.serve() # http://127.0.0.1:8765/v1 ``` +### How it works (pencil) + +Two things to read here: the **anatomy** of a plugin (left), and the **request +lifecycle** when tb routes a model to it (the numbered loop). Note the loop is a +cycle — the plugin's handler calls back *into* tb (steps 4–6), so tb is both the +caller (step 3) and the upstream-for-the-plugin (step 5). + +``` + A PLUGIN (one Python process) tingly-box GATEWAY + ┌───────────────────────────────────┐ ┌──────────────────────────────┐ + │ Plugin(name="my-rag") │ │ │ + │ │ │ provider: │ + │ @plugin.chat │ │ name=my-rag │ + │ def handle(req): ... │ │ api_base=http://…:8765/v1 │ + │ │ │ │ model=plugin/my-rag │ + │ │ returns str | iter[str] │ │ │ + │ ▼ │ │ rule: plugin/my-rag → ↑ │ + │ serve() → stdlib HTTP server │ └──────────────┬───────────────┘ + │ POST /v1/chat/completions ◄────┼──── (3) POST /v1/chat ──┘ ▲ + │ GET /v1/models │ (model=plugin/my-rag)│ (6) answer + │ GET /health │ │ + │ · buffered → chat.completion │ │ + │ · stream → SSE chunks ────┼──── (7) response ────────────┘ + │ │ │ + │ plugin.llm (lazy Layer-1 client)│ + │ │ │ + └────────┼──────────────────────────┘ + │ (4) plugin.llm.ask("…", model="auto") + │ = tingly.connect(scenario="experiment") → POST /tingly/experiment/v1/chat + ▼ + ┌──────────────────────────────────────────────────────────────┐ + │ tingly-box pipeline (SAME as any client — see Layer 1 graph) │ + │ scenario→rule · guard rails · routing/tiers · failover · │ + │ quota · logging · transform ─────────────────────────► (5) real upstream + └──────────────────────────────────────────────────────────────┘ (Anthropic/ + OpenAI/…) + + request lifecycle: + (1) client sends model="plugin/my-rag" to tb ── see Layer 3 graph + (2) tb resolves rule → provider my-rag (api_base = plugin) + (3) tb POSTs OpenAI /v1/chat/completions to the PLUGIN + (4) handler runs; calls plugin.llm.ask(...) ── back INTO tb + (5) tb routes that call to a real upstream, applies guard rails/quota/… + (6) generated text returns to the handler + (7) handler's str/iterator → OpenAI response/SSE back to tb → back to client +``` + +Key reading: + +- **One process, two roles.** As a *server* the plugin answers tb on + `:8765/v1`; as a *client* (`plugin.llm`) it consumes tb via Layer 1. Same + gateway, both directions. +- **The author writes only step 4's body.** Everything else — wire parsing, + response/SSE shaping (steps 3 & 7), discovery/session (step 4's connect), + routing/guard-rails (step 5) — is the SDK and the gateway. +- **Guard rails apply twice, correctly:** once on the inbound call to the + plugin (step 3, via the provider/rule), and again on the plugin's own LLM call + (step 5). Neither is wired by the author. + Design choices: - **No framework dependency.** The server is `http.server.ThreadingHTTPServer` From 08509b607f63ccff39329b0837e8e34aabff9634 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 08:22:24 +0000 Subject: [PATCH 06/28] =?UTF-8?q?refactor(sdk):=20reframe=20arch=20as=20ru?= =?UTF-8?q?le=E2=87=84plugin=20hub;=20add=20Plugin.use(scenario)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3-layer framing oversold the complexity. Collapse it to one idea: tb is a hub of rules, a rule's upstream can be a plugin, and a plugin can originate calls against any other rule/model/provider configured in tb. - Plugin.use(scenario) returns a per-scenario client (cached), so a plugin can drive ANY rule-set in tb, not just a fixed "experiment" scenario. plugin.llm is now just the default-scenario shortcut. - python-sdk.md: new "Architecture (one idea, not three layers)" section with a hub graph; connect/serve/register presented as three verbs for the one rule⇄plugin relationship; note the tb-side UX direction (a rule whose service is a plugin, one step). Tests: 26 passing (added use()-caching test). --- .design/python-sdk.md | 55 ++++++++++++++++++++++-- sdk/python/tests/test_plugin_manifest.py | 21 +++++++++ sdk/python/tingly/plugin/core.py | 30 ++++++++++--- 3 files changed, 95 insertions(+), 11 deletions(-) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 84004914f..40a8d3d60 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -14,10 +14,57 @@ with the right base URL, token and scenario path. There was no fast seam for (prompt, retrieval, agent loop) and **reuses the gateway's power** — provider routing, tier/fallback, guard rails, quota, logging — for free. -This is **Layer 1** (client-side library). It deliberately ships before Layer 2 -(tb-hosted plugins with a manifest + sub-process supervision) and Layer 3 -(plugin-as-virtual-model via `vmodel/virtualserver`), both of which build on -this same module and the same `/sdk/session` provisioning seam. +## Architecture (one idea, not three layers) + +There is a single concept: + +> **tb is a hub of rules. A rule's upstream can be a plugin. A plugin can +> originate calls against any other rule.** + +A client request matches a **rule** (as today). That rule's upstream is **plugin +code** instead of a provider — the only new thing. The plugin does its custom +work and, for any LLM work, calls **back into tb against any other rule / model / +provider** you have configured. tb stays the single router; a plugin is just a +graph node that happens to be user code and can also originate edges. + +``` + ┌──────────────────── tingly-box (the hub) ───────────────────┐ + clients │ │ + ┌─────────────┐ req │ rule A ──upstream──► PLUGIN CODE (your logic) │ + │ Claude Code │──────►│ (model=plugin/x) │ │ + │ Cursor │ │ │ calls back into tb: │ + │ tb UI │ │ rule B ◄──────────────────┤ use("…").ask(model="…") │ + │ tingly.ask()│ │ (→ Anthropic real) │ │ + └─────────────┘ │ rule C ◄──────────────────┤ (another model / provider) │ + │ rule D ◄──────────────────┘ (even another plugin) │ + │ │ │ + │ ▼ every edge gets: guard rails · routing/tiers · │ + │ failover · quota · logging │ + └──────────────────────────────────────────────────────────────┘ + │ + ▼ real upstreams (Anthropic / OpenAI / local …) +``` + +Everything else in this document is *how* that relationship is implemented with +today's pieces — three verbs for the one rule⇄plugin relationship: + +| verb | what it is | SDK surface | +|------|------------|-------------| +| **connect** | a plugin (or experiment) *consumes* a rule | `tingly.connect()` / `plugin.use(scenario).ask(model=…)` | +| **serve** | a plugin *is* a rule's upstream | `tingly.Plugin` (OpenAI server) | +| **register**| point a rule's upstream at the plugin | `register_with_tb()` → tb provider + rule | + +The historical "Layer 1/2/3" headings below map exactly to connect / serve / +register. They are an implementation tour, not three separate products. + +### tb-side direction (UX) + +Today "register" is *provider + rule binding* (two steps; provider-with- +`api_base` is the mechanism). The intended tb-side UX is **one step: a rule whose +service is a plugin** — same as `vmodel`'s in-process models are selected, but +for external plugin code. That eliminates the mode-picker (`ux-principles.md`) +and makes "configure this rule with a plugin" literal. + ## Shape diff --git a/sdk/python/tests/test_plugin_manifest.py b/sdk/python/tests/test_plugin_manifest.py index 322f4b00e..86e11560e 100644 --- a/sdk/python/tests/test_plugin_manifest.py +++ b/sdk/python/tests/test_plugin_manifest.py @@ -39,3 +39,24 @@ def test_plugin_builds_manifest(): assert man.entrypoint == "rag_plugin:plugin" assert man.port == 8080 assert man.transport == "openai" + + +def test_plugin_use_caches_per_scenario(monkeypatch): + import tingly.client as client_mod + from tingly import Plugin + + calls = [] + + def fake_connect(scenario, name): + calls.append((scenario, name)) + return object() + + monkeypatch.setattr(client_mod, "connect", fake_connect) + + plugin = Plugin(name="p", scenario="experiment") + a1 = plugin.llm + a2 = plugin.llm # default scenario, cached + b1 = plugin.use("claude_code") # different rule-set + assert a1 is a2 + assert b1 is not a1 + assert calls == [("experiment", "plugin:p"), ("claude_code", "plugin:p")] diff --git a/sdk/python/tingly/plugin/core.py b/sdk/python/tingly/plugin/core.py index 74826284f..ef4dc2109 100644 --- a/sdk/python/tingly/plugin/core.py +++ b/sdk/python/tingly/plugin/core.py @@ -56,7 +56,7 @@ def __init__( self.scenario = scenario self._handler: Optional[ChatHandler] = None - self._llm = None # lazy Layer-1 client + self._clients: dict = {} # scenario -> lazily-connected client self._httpd = None # -- authoring ------------------------------------------------------- @@ -72,16 +72,32 @@ def chat(self, fn: ChatHandler) -> ChatHandler: @property def llm(self): - """A lazily-connected Layer-1 client for calling back into tingly-box. + """A lazily-connected client for calling back into tingly-box. - Lets a plugin reuse the gateway for its own model calls, so it never - hard-codes a provider or key. + This is the plugin's default calling context (``self.scenario``). The + plugin reuses the gateway for its own model calls, so it never hard-codes + a provider or key — and ``ask(model=...)`` can target *any* model tb + routes. To drive a *different* rule-set, use :meth:`use`. """ - if self._llm is None: + return self.use(self.scenario) + + def use(self, scenario: str): + """Return a client bound to a specific scenario (rule-set) in tb. + + A plugin composes the box: it can hold clients to several scenarios and + pick a model on each, so "the plugin can use any other rule / model + configured in tb" is one call: + + self.use("claude_code").ask("…", model="claude-sonnet-4-6") + self.use("experiment").ask("…", model="auto") + """ + client = self._clients.get(scenario) + if client is None: from ..client import connect - self._llm = connect(scenario=self.scenario, name=f"plugin:{self.name}") - return self._llm + client = connect(scenario=scenario, name=f"plugin:{self.name}") + self._clients[scenario] = client + return client # -- dispatch -------------------------------------------------------- From f7276e45e19e786c1915fd4dd44978ed743aa602 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 09:09:35 +0000 Subject: [PATCH 07/28] feat(plugins): first-class plugin provider kind + one-step registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make "configure this rule with a plugin" a single step. Backend: - ai.PluginDetail + Provider.PluginDetail + IsPlugin(): mark a provider as backed by external plugin code. Distinct from AuthTypeVirtual (in-process vmodel) — a plugin is an ordinary OpenAI HTTP upstream, so routing is unchanged; the marker is metadata for UI grouping + lifecycle. - Persist plugin_detail via a new column (AutoMigrate), reconstructed unconditionally since it's independent of auth type. - POST /api/v2/plugins: create the plugin provider and, when a scenario is given, the rule whose single tier-service is the plugin. GET /api/v2/plugins lists plugin-kind providers. - Handler + store tests; openapi regenerated. SDK: - register_with_tb() now calls /api/v2/plugins with scenario (one-step wire-in), fixing the prior wrong /api/v1/providers path. `tingly plugin register --scenario` binds the rule. Returns rule_uuid/ready. Docs: python-sdk.md documents the plugin provider kind. 28 py + go tests pass. --- .design/python-sdk.md | 32 ++++- ai/provider.go | 27 ++++ internal/data/db/provider_store.go | 28 ++++ internal/server/plugin_provider.go | 168 +++++++++++++++++++++++ internal/server/plugin_provider_test.go | 131 ++++++++++++++++++ internal/server/server_webui_api.go | 15 ++ internal/typ/type.go | 3 + sdk/python/README.md | 4 +- sdk/python/examples/rag_plugin.py | 6 +- sdk/python/tests/test_plugin_register.py | 71 ++++++++++ sdk/python/tingly/cli.py | 19 ++- sdk/python/tingly/plugin/register.py | 67 +++++---- 12 files changed, 520 insertions(+), 51 deletions(-) create mode 100644 internal/server/plugin_provider.go create mode 100644 internal/server/plugin_provider_test.go create mode 100644 sdk/python/tests/test_plugin_register.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 40a8d3d60..a12f5140a 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -57,13 +57,31 @@ today's pieces — three verbs for the one rule⇄plugin relationship: The historical "Layer 1/2/3" headings below map exactly to connect / serve / register. They are an implementation tour, not three separate products. -### tb-side direction (UX) - -Today "register" is *provider + rule binding* (two steps; provider-with- -`api_base` is the mechanism). The intended tb-side UX is **one step: a rule whose -service is a plugin** — same as `vmodel`'s in-process models are selected, but -for external plugin code. That eliminates the mode-picker (`ux-principles.md`) -and makes "configure this rule with a plugin" literal. +### tb-side: a first-class "plugin" provider kind (implemented) + +"Register" is now **one step**: `POST /api/v2/plugins` creates a plugin-kind +provider *and* (when a scenario is given) the rule whose upstream is that plugin. + +- **`ai.PluginDetail`** + `Provider.PluginDetail` + `Provider.IsPlugin()` mark a + provider as backed by plugin code. It is **distinct from `AuthTypeVirtual`** + (in-process `vmodel`): a plugin is an ordinary OpenAI HTTP upstream + (`APIStyle=openai`, `api_key`/`no_key`), so **routing is unchanged** — the + marker is metadata for UI grouping + lifecycle discovery only. Persisted via a + new `plugin_detail` column, reconstructed unconditionally (independent of auth + type), AutoMigrate-created. +- **`POST /api/v2/plugins`** `{name, endpoint, model_id?, token?, scenario?, tier?}` + → creates the provider; if `scenario` is bindable, also creates a rule + (`RequestModel=model_id`, single tier service → the plugin). Returns + `{provider_uuid, model_id, scenario, rule_uuid, ready}`. +- **`GET /api/v2/plugins`** lists plugin-kind providers for the UI's plugin + section. +- SDK: `register_with_tb(..., scenario=…)` and `tingly plugin register + --scenario experiment` do the full one-step wire-in. + +This makes "configure this rule with a plugin" literal and eliminates the +provider+rule two-step mode-picker (`ux-principles.md`). Remaining tb-side work: +the rule-editor UI surfacing "plugin" as a service kind (frontend, codegen), and +the process supervisor/lifecycle. ## Shape diff --git a/ai/provider.go b/ai/provider.go index 26a576940..c9ceaccf0 100644 --- a/ai/provider.go +++ b/ai/provider.go @@ -53,6 +53,25 @@ type VModelDetail struct { LatencyProfile string `json:"latency_profile,omitempty"` } +// PluginDetail marks a provider as backed by external plugin code. A plugin +// provider is otherwise an ordinary OpenAI HTTP upstream (APIStyle=openai, +// APIBase=/v1, api_key / no_key auth) — there is NO routing change, the +// dispatcher treats it exactly like any other provider. This marker exists so +// plugins are a first-class concept (grouped in the UI, discoverable by future +// lifecycle tooling) and so "configure this rule with a plugin" is one step. +// +// Note: this is distinct from VModelDetail / AuthTypeVirtual, which is the +// in-process synthetic-model path. A plugin runs out-of-process and is reached +// over HTTP. +type PluginDetail struct { + // ModelID is the model id the plugin advertises (e.g. "plugin/my-rag"). + ModelID string `json:"model_id,omitempty"` + // Managed reports whether tingly-box supervises the plugin process. + // Reserved for the tb-side supervisor; false means the plugin is run + // externally and tb only routes to it. + Managed bool `json:"managed,omitempty"` +} + // CredentialBundle holds the credential fields for multi-field auth types // (AWS SigV4, Azure, GCP Vertex). Fields is a generic, schema-validated // key/value map so new credential shapes can be added as data rather than new @@ -199,6 +218,7 @@ type Provider struct { AuthType AuthType `json:"auth_type"` // api_key, oauth, vmodel, aws_sigv4, azure_key, gcp_sa OAuthDetail *OAuthDetail `json:"oauth_detail,omitempty"` // OAuth credentials (only for oauth auth type) VModelDetail *VModelDetail `json:"vmodel_detail,omitempty"` // Virtual-model config (only for vmodel auth type) + PluginDetail *PluginDetail `json:"plugin_detail,omitempty"` // Plugin config (set when this provider is backed by plugin code) Credential *CredentialBundle `json:"credential,omitempty"` // Multi-field credentials (only for multi-field auth types) Source ProviderSource `json:"source,omitempty"` // "user" (default) or "builtin" @@ -253,6 +273,13 @@ func (p *Provider) IsVirtual() bool { return p != nil && p.AuthType == AuthTypeVirtual } +// IsPlugin reports whether this provider is backed by external plugin code. +// Plugin providers route as ordinary OpenAI HTTP upstreams; this is metadata +// for UI grouping and lifecycle discovery only. +func (p *Provider) IsPlugin() bool { + return p != nil && p.PluginDetail != nil +} + // IsBuiltin reports whether this provider was seeded by the system and is // therefore protected from deletion/mutation. func (p *Provider) IsBuiltin() bool { diff --git a/internal/data/db/provider_store.go b/internal/data/db/provider_store.go index 959796692..a601bbb73 100644 --- a/internal/data/db/provider_store.go +++ b/internal/data/db/provider_store.go @@ -59,6 +59,12 @@ type ProviderRecord struct { // VModel-specific fields (only populated when AuthType == "vmodel") VModelDetail string `gorm:"column:vmodel_detail;type:text"` // JSON-encoded typ.VModelDetail + // PluginDetail marks a provider as backed by external plugin code. Unlike + // VModelDetail it is independent of AuthType (a plugin uses api_key auth), so + // it is persisted/loaded unconditionally. JSON-encoded typ.PluginDetail; empty + // for non-plugin providers. Added additively; AutoMigrate creates the column. + PluginDetail string `gorm:"column:plugin_detail;type:text"` + // Credential holds multi-field credentials for non-bearer auth types // (aws_sigv4, azure_key, gcp_sa). JSON-encoded typ.CredentialBundle. // Empty for api_key/oauth/vmodel. Added additively; AutoMigrate creates @@ -130,6 +136,14 @@ func (r *ProviderRecord) toProvider() *typ.Provider { provider.AuthType = typ.AuthTypeAPIKey } + // PluginDetail is independent of auth type — reconstruct it for any provider. + if r.PluginDetail != "" { + var detail typ.PluginDetail + if err := json.Unmarshal([]byte(r.PluginDetail), &detail); err == nil { + provider.PluginDetail = &detail + } + } + return provider } @@ -169,6 +183,12 @@ func toRecord(p *typ.Provider) *ProviderRecord { record.Tags = string(tagsJSON) } + // PluginDetail is independent of auth type. + if p.PluginDetail != nil { + pdJSON, _ := json.Marshal(p.PluginDetail) + record.PluginDetail = string(pdJSON) + } + // Set credentials based on auth type switch p.AuthType { case typ.AuthTypeOAuth: @@ -222,6 +242,14 @@ func updateRecordFromProvider(record *ProviderRecord, p *typ.Provider) { record.Tags = "" } + // PluginDetail is independent of auth type; set or clear unconditionally. + if p.PluginDetail != nil { + pdJSON, _ := json.Marshal(p.PluginDetail) + record.PluginDetail = string(pdJSON) + } else { + record.PluginDetail = "" + } + // Set credentials based on auth type switch p.AuthType { case typ.AuthTypeOAuth: diff --git a/internal/server/plugin_provider.go b/internal/server/plugin_provider.go new file mode 100644 index 000000000..ffb68c25c --- /dev/null +++ b/internal/server/plugin_provider.go @@ -0,0 +1,168 @@ +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/tingly-dev/tingly-box/internal/constant" + "github.com/tingly-dev/tingly-box/internal/loadbalance" + "github.com/tingly-dev/tingly-box/internal/server/config" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// RegisterPluginRequest registers external plugin code as a tingly-box upstream +// in one step: it creates a plugin-kind provider and, when a scenario is given, +// the rule whose upstream is that plugin. +type RegisterPluginRequest struct { + Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` + Endpoint string `json:"endpoint" binding:"required" description:"Plugin OpenAI base URL" example:"http://127.0.0.1:8765/v1"` + ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` + Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` + Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` + Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` +} + +// RegisterPluginResponse reports what was created. +type RegisterPluginResponse struct { + ProviderUUID string `json:"provider_uuid"` + ModelID string `json:"model_id"` + Scenario string `json:"scenario,omitempty"` + RuleUUID string `json:"rule_uuid,omitempty"` + // Ready is true when a rule was bound, so clients can select the model now. + Ready bool `json:"ready"` + Note string `json:"note,omitempty"` +} + +// PluginInfo is a list view of a plugin-kind provider. +type PluginInfo struct { + UUID string `json:"uuid"` + Name string `json:"name"` + Endpoint string `json:"endpoint"` + ModelID string `json:"model_id"` + Managed bool `json:"managed"` + Enabled bool `json:"enabled"` +} + +// PluginsResponse wraps the plugin list. +type PluginsResponse struct { + Success bool `json:"success"` + Data []PluginInfo `json:"data"` +} + +// RegisterPlugin creates a plugin-kind provider (and optionally binds a rule to +// it) so "configure this rule with a plugin" is a single call. A plugin +// provider is an ordinary OpenAI HTTP upstream — routing is unchanged; the +// PluginDetail marker makes it a first-class concept for the UI and lifecycle. +func (s *Server) RegisterPlugin(c *gin.Context) { + var req RegisterPluginRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + + modelID := req.ModelID + if modelID == "" { + modelID = "plugin/" + req.Name + } + + provider := &typ.Provider{ + UUID: config.GenerateUUID(), + Name: req.Name, + APIBase: req.Endpoint, + APIStyle: "openai", + Token: req.Token, + NoKeyRequired: req.Token == "", + Enabled: true, + AuthType: typ.AuthTypeAPIKey, + Timeout: constant.DefaultRequestTimeout, + PluginDetail: &typ.PluginDetail{ModelID: modelID}, + } + if err := s.config.AddProvider(provider); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": "failed to create plugin provider: " + err.Error(), + }) + return + } + + resp := RegisterPluginResponse{ + ProviderUUID: provider.UUID, + ModelID: modelID, + Note: "Provider created. Bind a rule (pass `scenario`) to make the model selectable.", + } + + // One-step bind: create the rule whose single service is this plugin. + if req.Scenario != "" { + scenario := typ.RuleScenario(req.Scenario) + if !typ.CanBindRulesToScenario(scenario) { + // Provider is created; surface the bind failure without 500ing. + resp.Note = "Provider created, but scenario " + req.Scenario + + " is not bindable; bind a rule manually." + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) + return + } + + rule := typ.Rule{ + UUID: config.GenerateUUID(), + Scenario: scenario, + RequestModel: modelID, + Description: "Plugin: " + req.Name, + Active: true, + LBTactic: typ.ParseTacticFromMap(loadbalance.TacticTier, nil), + Services: []*loadbalance.Service{ + { + Provider: provider.UUID, + Model: modelID, + Weight: 1, + Active: true, + Tier: req.Tier, + }, + }, + } + if err := s.config.AddRule(rule); err != nil { + resp.Note = "Provider created, but rule binding failed: " + err.Error() + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) + return + } + resp.Scenario = req.Scenario + resp.RuleUUID = rule.UUID + resp.Ready = true + resp.Note = "Plugin wired in. Select model " + modelID + " under scenario " + req.Scenario + "." + } + + logrus.WithFields(logrus.Fields{ + "plugin": req.Name, + "endpoint": req.Endpoint, + "model_id": modelID, + "scenario": req.Scenario, + "ready": resp.Ready, + }).Info("Registered plugin provider") + + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) +} + +// ListPlugins returns the plugin-kind providers, for the UI's plugin section. +func (s *Server) ListPlugins(c *gin.Context) { + var plugins []PluginInfo + for _, p := range s.config.ListProviders() { + if !p.IsPlugin() { + continue + } + modelID := "" + if p.PluginDetail != nil { + modelID = p.PluginDetail.ModelID + } + managed := p.PluginDetail != nil && p.PluginDetail.Managed + plugins = append(plugins, PluginInfo{ + UUID: p.UUID, + Name: p.Name, + Endpoint: p.APIBase, + ModelID: modelID, + Managed: managed, + Enabled: p.Enabled, + }) + } + c.JSON(http.StatusOK, PluginsResponse{Success: true, Data: plugins}) +} diff --git a/internal/server/plugin_provider_test.go b/internal/server/plugin_provider_test.go new file mode 100644 index 000000000..212d2474f --- /dev/null +++ b/internal/server/plugin_provider_test.go @@ -0,0 +1,131 @@ +package server + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/tingly-dev/tingly-box/internal/server/config" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +func newPluginTestServer(t *testing.T) *Server { + t.Helper() + cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + return &Server{config: cfg} +} + +func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + gin.SetMode(gin.TestMode) + raw, _ := json.Marshal(body) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(raw)) + c.Request.Header.Set("Content-Type", "application/json") + h(c) + var parsed map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &parsed) + return w, parsed +} + +func TestRegisterPlugin_BindsRule(t *testing.T) { + s := newPluginTestServer(t) + + w, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", + Endpoint: "http://127.0.0.1:8765/v1", + ModelID: "plugin/my-rag", + Scenario: string(typ.ScenarioExperiment), + }) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", w.Code, w.Body.String()) + } + data, _ := resp["data"].(map[string]any) + if data["ready"] != true { + t.Fatalf("expected ready=true, got %v (note=%v)", data["ready"], data["note"]) + } + if data["model_id"] != "plugin/my-rag" { + t.Fatalf("model_id = %v", data["model_id"]) + } + providerUUID, _ := data["provider_uuid"].(string) + if providerUUID == "" { + t.Fatalf("expected a provider_uuid") + } + + // The provider must be a plugin-kind provider. + prov, err := s.config.GetProviderByUUID(providerUUID) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if !prov.IsPlugin() { + t.Fatalf("provider is not marked as plugin: %+v", prov) + } + if prov.PluginDetail.ModelID != "plugin/my-rag" { + t.Fatalf("plugin model id = %q", prov.PluginDetail.ModelID) + } + + // A rule must exist under the scenario whose single service is the plugin. + var found bool + for _, rule := range s.config.GetRequestConfigs() { + if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { + found = true + if len(rule.Services) != 1 || rule.Services[0].Provider != providerUUID { + t.Fatalf("rule service does not point at plugin provider: %+v", rule.Services) + } + } + } + if !found { + t.Fatalf("no rule bound for the plugin under experiment scenario") + } +} + +func TestRegisterPlugin_ProviderOnly(t *testing.T) { + s := newPluginTestServer(t) + + _, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "solo", + Endpoint: "http://127.0.0.1:9000/v1", + }) + data, _ := resp["data"].(map[string]any) + if data["ready"] == true { + t.Fatalf("expected ready=false when no scenario given") + } + // model id defaults to plugin/ + if data["model_id"] != "plugin/solo" { + t.Fatalf("model_id default = %v", data["model_id"]) + } +} + +func TestListPlugins_FiltersPluginKind(t *testing.T) { + s := newPluginTestServer(t) + // a normal provider + if err := s.config.AddProvider(&typ.Provider{ + Name: "real", APIBase: "https://api.example.com/v1", APIStyle: "openai", Enabled: true, + }); err != nil { + t.Fatalf("AddProvider: %v", err) + } + // a plugin provider via the handler + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{Name: "plug", Endpoint: "http://127.0.0.1:8765/v1"}) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + s.ListPlugins(c) + + var resp PluginsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(resp.Data) != 1 || resp.Data[0].Name != "plug" { + t.Fatalf("expected exactly the plugin provider, got %+v", resp.Data) + } +} diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index a57f6a6b4..6aae6a393 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -400,6 +400,21 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { providerHandler := providermodule.NewHandler(s.config, s.quotaManager) providermodule.RegisterRoutes(apiV2, providerHandler) + // Plugin registration: a plugin is a first-class provider kind (external + // OpenAI upstream). POST wires it in (provider + optional rule) in one step. + api.POST("/plugins", s.RegisterPlugin, + swagger.WithDescription("Register external plugin code as an upstream (and optionally bind a rule)"), + swagger.WithTags("plugins"), + swagger.WithRequestModel(RegisterPluginRequest{}), + swagger.WithResponseModel(RegisterPluginResponse{}), + ) + + api.GET("/plugins", s.ListPlugins, + swagger.WithDescription("List registered plugin-kind providers"), + swagger.WithTags("plugins"), + swagger.WithResponseModel(PluginsResponse{}), + ) + // Provider template endpoints providerTemplateHandler := providertemplate.NewHandler(s.templateManager) providertemplate.RegisterRoutes(apiV2, providerTemplateHandler) diff --git a/internal/typ/type.go b/internal/typ/type.go index c9e4dc22b..98c38fd2b 100644 --- a/internal/typ/type.go +++ b/internal/typ/type.go @@ -345,6 +345,9 @@ type OAuthDetail = ai.OAuthDetail // Type alias for backward compatibility with common/provider type VModelDetail = ai.VModelDetail +// PluginDetail marks a provider as backed by external plugin code. +type PluginDetail = ai.PluginDetail + // CredentialBundle holds multi-field credentials for non-bearer auth types // Type alias for backward compatibility with common/provider type CredentialBundle = ai.CredentialBundle diff --git a/sdk/python/README.md b/sdk/python/README.md index 81dc09115..436087cf6 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -75,8 +75,8 @@ if __name__ == "__main__": ```bash tingly plugin init my-rag # scaffold module + tingly.toml tingly plugin run my_rag_plugin.py # serve -tingly plugin register my-rag \ # wire into tb as a provider - --url http://127.0.0.1:8765/v1 --model-id plugin/my-rag +tingly plugin register my-rag \ # one step: provider + rule + --url http://127.0.0.1:8765/v1 --model-id plugin/my-rag --scenario experiment ``` The server is stdlib-only (no FastAPI), supports streaming, and `plugin.llm` diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py index dede0c610..c0fd49cc8 100644 --- a/sdk/python/examples/rag_plugin.py +++ b/sdk/python/examples/rag_plugin.py @@ -5,9 +5,11 @@ pip install -e ".[all]" # from sdk/python python examples/rag_plugin.py # serves on http://127.0.0.1:8765/v1 -Then register it with tb so any client can select model `plugin/rag-demo`: +Then wire it into tb in one step (creates the provider + a rule) so any client +can select model `plugin/rag-demo`: - tingly plugin register rag-demo --url http://127.0.0.1:8765/v1 --model-id plugin/rag-demo + tingly plugin register rag-demo --url http://127.0.0.1:8765/v1 \ + --model-id plugin/rag-demo --scenario experiment Now `model="plugin/rag-demo"` from Claude Code, Cursor, the tb UI, or another `tingly.connect()` experiment routes here — with tb's guard rails, quota, diff --git a/sdk/python/tests/test_plugin_register.py b/sdk/python/tests/test_plugin_register.py new file mode 100644 index 000000000..00e1dd9db --- /dev/null +++ b/sdk/python/tests/test_plugin_register.py @@ -0,0 +1,71 @@ +"""register_with_tb hits the one-step /api/v2/plugins endpoint (respx mocked).""" + +import httpx +import respx + +from tingly.plugin.register import register_with_tb + +BASE = "http://tb.test:12580" + + +@respx.mock +def test_register_binds_rule(monkeypatch): + monkeypatch.setenv("TINGLY_BOX_URL", BASE) + monkeypatch.setenv("TINGLY_BOX_TOKEN", "admin") + + route = respx.post(f"{BASE}/api/v2/plugins").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "data": { + "provider_uuid": "uuid-1", + "model_id": "plugin/my-rag", + "scenario": "experiment", + "rule_uuid": "rule-1", + "ready": True, + "note": "Plugin wired in.", + }, + }, + ) + ) + + result = register_with_tb( + "my-rag", + "http://127.0.0.1:8765/v1", + "plugin/my-rag", + scenario="experiment", + ) + + assert route.called + sent = route.calls.last.request + assert sent.headers["Authorization"] == "Bearer admin" + assert result.provider_uuid == "uuid-1" + assert result.rule_uuid == "rule-1" + assert result.ready is True + assert result.scenario == "experiment" + + +@respx.mock +def test_register_provider_only(monkeypatch): + monkeypatch.setenv("TINGLY_BOX_URL", BASE) + monkeypatch.setenv("TINGLY_BOX_TOKEN", "admin") + + respx.post(f"{BASE}/api/v2/plugins").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "data": { + "provider_uuid": "uuid-2", + "model_id": "plugin/solo", + "ready": False, + "note": "Provider created.", + }, + }, + ) + ) + + result = register_with_tb("solo", "http://127.0.0.1:9000/v1", "plugin/solo") + assert result.ready is False + assert result.rule_uuid is None diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py index d8b383dcc..07d8cb738 100644 --- a/sdk/python/tingly/cli.py +++ b/sdk/python/tingly/cli.py @@ -160,12 +160,16 @@ def _plugin_run(target: str) -> int: return 0 -def _plugin_register(name: str, plugin_url: str, model_id: str, token: str) -> int: +def _plugin_register(name, plugin_url, model_id, token, scenario) -> int: from .plugin.register import register_with_tb - result = register_with_tb(name, plugin_url, model_id, token=token) - status = OK if result.created else WARN - _row("provider", f"{result.name} → {result.api_base}", status) + result = register_with_tb( + name, plugin_url, model_id, scenario=scenario or None, token=token + ) + status = OK if result.ready else WARN + _row("plugin", f"{result.name} → {result.api_base}", status) + if result.rule_uuid: + _row("rule", f"{result.scenario}: {result.model_id}", OK) print("\n" + result.note) return 0 @@ -193,6 +197,9 @@ def main(argv: Optional[list] = None) -> int: p_reg.add_argument("--url", required=True, help="plugin OpenAI base, e.g. http://127.0.0.1:8765/v1") p_reg.add_argument("--model-id", required=True, help="model id, e.g. plugin/my-rag") p_reg.add_argument("--token", default="", help="token tb should send to the plugin") + p_reg.add_argument( + "--scenario", default="", help="bind a rule under this scenario (e.g. experiment)" + ) args = parser.parse_args(argv) if args.command == "doctor": @@ -203,7 +210,9 @@ def main(argv: Optional[list] = None) -> int: if args.plugin_command == "run": return _plugin_run(args.target) if args.plugin_command == "register": - return _plugin_register(args.name, args.url, args.model_id, args.token) + return _plugin_register( + args.name, args.url, args.model_id, args.token, args.scenario + ) p_plugin.print_help() return 0 diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py index ffa91bf44..a3a1d1dc8 100644 --- a/sdk/python/tingly/plugin/register.py +++ b/sdk/python/tingly/plugin/register.py @@ -1,9 +1,9 @@ -"""Register a running plugin with tingly-box as an upstream provider. +"""Register a running plugin with tingly-box in one step. -This is the Layer 3 wiring: it creates a provider whose ``api_base`` points at -the plugin's HTTP server, so tingly-box can route a model id to it. Creating the -*rule/service* that maps the model into a scenario is left to the user (or the -tb UI) for now — the provider is the part the SDK can do safely and idempotently. +Calls the first-class ``POST /api/v2/plugins`` endpoint, which creates a +plugin-kind provider pointing at the plugin's HTTP server and — when a scenario +is given — the rule whose upstream is that plugin. The plugin then composes with +tb's routing, fallback, guard rails, quota and logging like any other model. """ from __future__ import annotations @@ -23,7 +23,9 @@ class RegisterResult: name: str api_base: str model_id: str - created: bool + scenario: Optional[str] + rule_uuid: Optional[str] + ready: bool note: str @@ -32,33 +34,37 @@ def register_with_tb( plugin_url: str, model_id: str, *, + scenario: Optional[str] = None, gateway_url: Optional[str] = None, admin_token: Optional[str] = None, token: str = "", + tier: int = 0, timeout: float = 30.0, ) -> RegisterResult: - """Create (or report) a tingly-box provider pointing at the plugin. + """Wire a plugin into tingly-box in one call (provider + optional rule). Args: - name: provider name to create in tb (e.g. the plugin name). + name: plugin / provider name to create in tb. plugin_url: the plugin's OpenAI base, e.g. ``http://127.0.0.1:8765/v1``. model_id: the model id the plugin advertises (e.g. ``plugin/my-rag``). + scenario: bind a rule under this scenario so the model is selectable + immediately; omit to create only the provider. gateway_url / admin_token: tb gateway + admin token; auto-discovered if omitted (same precedence as ``connect()``). token: optional API token tb should send to the plugin (matches the plugin's ``api_key`` if it enforces one). + tier: tier for the bound service (0 = highest priority). """ resolved = _config.resolve(base_url=gateway_url, token=admin_token) headers = {"Authorization": f"Bearer {resolved.token or ''}"} - url = resolved.base_url.rstrip("/") + "/api/v1/providers" + url = resolved.base_url.rstrip("/") + "/api/v2/plugins" payload = { "name": name, - "api_base": plugin_url, - "api_style": "openai", + "endpoint": plugin_url, + "model_id": model_id, "token": token, - "no_key_required": token == "", - "enabled": True, - "auth_type": "api_key", + "scenario": scenario or "", + "tier": tier, } try: @@ -69,33 +75,24 @@ def register_with_tb( ) from exc if resp.status_code == 401: - raise AuthError("tingly-box rejected the admin token while creating the provider") + raise AuthError("tingly-box rejected the admin token while registering the plugin") - note = ( - f"Provider created. Bind a rule mapping model {model_id!r} to this " - f"provider (tb UI → Rules) so clients can select it." - ) - created = resp.status_code in (200, 201) - if not created: - # A name clash (already registered) is fine and idempotent enough. - note = ( - f"Provider not created (HTTP {resp.status_code}: {resp.text[:160]}). " - f"It may already exist — verify in the tb UI." + payload_data = _safe_json(resp) or {} + if resp.status_code not in (200, 201) or not payload_data.get("success"): + raise GatewayUnreachableError( + f"plugin registration failed: HTTP {resp.status_code} {resp.text[:200]}" ) - data = _safe_json(resp) or {} - provider_uuid = ( - (data.get("data") or {}).get("uuid") - or data.get("uuid") - or None - ) + data = payload_data.get("data") or {} return RegisterResult( - provider_uuid=provider_uuid, + provider_uuid=data.get("provider_uuid"), name=name, api_base=plugin_url, - model_id=model_id, - created=created, - note=note, + model_id=data.get("model_id", model_id), + scenario=data.get("scenario") or None, + rule_uuid=data.get("rule_uuid") or None, + ready=bool(data.get("ready", False)), + note=data.get("note", ""), ) From d244ebff9e99b610387f0f01fd681808630f035e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 09:25:38 +0000 Subject: [PATCH 08/28] fix(sdk): probe unauthenticated /info/health; add full-hub e2e example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running a real end-to-end (client → tb → plugin → plugin calls back into tb → echo-model vmodel) surfaced a discovery bug: probe_version hit the auth-gated /api/v1/info/version, so a plugin's callback to tb always failed discovery. Probe /api/v1/info/health (unauthenticated) instead; doctor shows "reachable". - examples/e2e_plugin.py: plugin whose handler calls back into tb's echo-model. - examples/e2e_run.sh: orchestrates tb + plugin + registration + client call with no network/API keys (vmodel echo stands in for a real provider). - Tests updated for the health probe. 28 py tests pass; verified the e2e returns the round-tripped string through the real tb binary. --- sdk/python/examples/e2e_plugin.py | 35 +++++++++++++ sdk/python/examples/e2e_run.sh | 83 ++++++++++++++++++++++++++++++ sdk/python/tests/test_discovery.py | 15 +++--- sdk/python/tingly/cli.py | 6 +-- sdk/python/tingly/discovery.py | 20 +++---- 5 files changed, 135 insertions(+), 24 deletions(-) create mode 100644 sdk/python/examples/e2e_plugin.py create mode 100755 sdk/python/examples/e2e_run.sh diff --git a/sdk/python/examples/e2e_plugin.py b/sdk/python/examples/e2e_plugin.py new file mode 100644 index 000000000..a83e8096d --- /dev/null +++ b/sdk/python/examples/e2e_plugin.py @@ -0,0 +1,35 @@ +"""E2E plugin: served as an upstream, and calls BACK into tb's echo-model. + +Demonstrates the full hub: + client → tb (rule plugin/rag-demo) → THIS plugin → plugin.use("experiment") + → tb (rule echo-model → vmodel) → echoed text → back to client +""" + +from tingly import Plugin + +plugin = Plugin(name="rag-demo", scenario="experiment") + +CORPUS = { + "tingly-box": "tingly-box is a personal intelligence orchestrator.", + "plugin": "A plugin is an OpenAI-compatible upstream tb can route to.", +} + + +def retrieve(q: str) -> str: + hits = [t for k, t in CORPUS.items() if k in q.lower()] + return " ".join(hits) or "(no docs)" + + +@plugin.chat +def handle(req): + q = req.last_user_text() + docs = retrieve(q) + # Call back into tb against the echo-model rule (no real network needed). + echoed = plugin.use("experiment").ask( + f"[plugin-rag] docs={docs!r} q={q!r}", model="echo-model" + ) + return f"RAG via plugin → tb echo returned: {echoed}" + + +if __name__ == "__main__": + plugin.serve(port=8765) diff --git a/sdk/python/examples/e2e_run.sh b/sdk/python/examples/e2e_run.sh new file mode 100755 index 000000000..4d2ad9c8a --- /dev/null +++ b/sdk/python/examples/e2e_run.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# End-to-end demo of the full hub, using NO network / API keys: +# client → tb (rule plugin/rag-demo) → plugin → plugin.use("experiment") +# → tb (rule echo-model → vmodel) → echoed text → back to client +# +# Prereqs: +# go build -o /tmp/tb_e2e ./cli/tingly-box # the tb binary +# pip install httpx openai # SDK transitive + a transport +# Run: bash sdk/python/examples/e2e_run.sh +set -uo pipefail + +TB=${TB_BIN:-/tmp/tb_e2e} +CFG=$(mktemp -d) +PORT=18901 +BASE="http://127.0.0.1:$PORT" +SDK=/home/user/tingly-box/sdk/python +export PYTHONPATH=$SDK + +cleanup() { + [[ -n "${PLUG_PID:-}" ]] && kill "$PLUG_PID" 2>/dev/null + [[ -n "${TB_PID:-}" ]] && kill "$TB_PID" 2>/dev/null +} +trap cleanup EXIT + +echo "== 1. start tb (config-dir=$CFG, port=$PORT) ==" +"$TB" --config-dir "$CFG" start --port "$PORT" --ui --browser=false >/tmp/tb_e2e.log 2>&1 & +TB_PID=$! +for i in $(seq 1 60); do + curl -sf "$BASE/api/v1/info/health" >/dev/null 2>&1 && break + sleep 0.5 +done +curl -sf "$BASE/api/v1/info/health" >/dev/null || { echo "tb did not start"; tail -25 /tmp/tb_e2e.log; exit 1; } +echo " tb healthy at $BASE" + +# Tokens are generated fresh per config-dir; read them from the config file. +CFGFILE=$(find "$CFG" -name 'config.json' | head -1) +echo " config file: $CFGFILE" +UTOK=$(python3 -c "import json,sys;d=json.load(open('$CFGFILE'));print(d.get('user_token') or d.get('UserToken',''))") +MTOK=$(python3 -c "import json,sys;d=json.load(open('$CFGFILE'));print(d.get('model_token') or d.get('ModelToken',''))") +echo " user token: ${UTOK:0:16}… model token: ${MTOK:0:16}…" + +UADMIN=(-H "Authorization: Bearer $UTOK" -H "Content-Type: application/json") +UMODEL=(-H "Authorization: Bearer $MTOK" -H "Content-Type: application/json") + +echo "== 2. create vmodel provider (echo backend, no network) ==" +VRESP=$(curl -s "${UADMIN[@]}" -X POST "$BASE/api/v2/providers" -d '{ + "name":"vmodel-echo","api_base":"vmodel://local","api_style":"openai", + "auth_type":"vmodel","no_key_required":true,"enabled":true}') +VUUID=$(echo "$VRESP" | python3 -c "import sys,json;d=json.load(sys.stdin);print(d.get('data',{}).get('uuid') or d.get('uuid',''))") +echo " vmodel provider uuid: $VUUID" + +echo "== 3. create echo-model rule under experiment scenario ==" +curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ + \"scenario\":\"experiment\",\"request_model\":\"echo-model\",\"active\":true, + \"lb_tactic\":{\"type\":\"random\",\"params\":{}}, + \"services\":[{\"provider\":\"$VUUID\",\"model\":\"echo-model\",\"weight\":1,\"active\":true}]}" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'), d.get('data',{}).get('uuid',''))" + +echo "== 4. start the plugin (serves OpenAI on :8765, calls back into tb) ==" +TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ + python3 "$SDK/examples/e2e_plugin.py" >/tmp/plugin_e2e.log 2>&1 & +PLUG_PID=$! +for i in $(seq 1 40); do + curl -sf "http://127.0.0.1:8765/health" >/dev/null 2>&1 && break + sleep 0.3 +done +curl -sf "http://127.0.0.1:8765/health" >/dev/null || { echo "plugin did not start"; cat /tmp/plugin_e2e.log; exit 1; } +echo " plugin up: $(curl -s http://127.0.0.1:8765/v1/models)" + +echo "== 5. register the plugin with tb (one step: provider + rule) ==" +curl -s "${UADMIN[@]}" -X POST "$BASE/api/v2/plugins" -d '{ + "name":"rag-demo","endpoint":"http://127.0.0.1:8765/v1", + "model_id":"plugin/rag-demo","scenario":"experiment"}' | python3 -m json.tool + +echo "== 6. CLIENT CALL: model=plugin/rag-demo through tb ==" +echo " (client → tb → plugin → tb echo-model → back)" +curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ + "model":"plugin/rag-demo", + "messages":[{"role":"user","content":"What is tingly-box?"}]}' | python3 -m json.tool + +echo "== plugin log tail ==" +tail -5 /tmp/plugin_e2e.log +echo "== done ==" diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py index 3188ed876..45ce0c311 100644 --- a/sdk/python/tests/test_discovery.py +++ b/sdk/python/tests/test_discovery.py @@ -10,21 +10,18 @@ BASE = "http://tb.test:12580" -def _version_route(): - return respx.get(f"{BASE}/api/v1/info/version").mock( - return_value=httpx.Response(200, json={"version": "1.2.3"}) - ) - - @respx.mock def test_probe_version_ok(): - _version_route() - assert disco.probe_version(BASE) == "1.2.3" + # Discovery probes the unauthenticated health endpoint, not /info/version. + respx.get(f"{BASE}/api/v1/info/health").mock( + return_value=httpx.Response(200, json={"health": True, "status": "healthy"}) + ) + assert disco.probe_version(BASE) == "ok" @respx.mock def test_probe_version_down(): - respx.get(f"{BASE}/api/v1/info/version").mock( + respx.get(f"{BASE}/api/v1/info/health").mock( return_value=httpx.Response(503) ) assert disco.probe_version(BASE) is None diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py index 07d8cb738..adf57a0a5 100644 --- a/sdk/python/tingly/cli.py +++ b/sdk/python/tingly/cli.py @@ -36,8 +36,8 @@ def doctor(scenario: str, link: bool) -> int: resolved = _config.resolve() # 1. gateway reachable - version = _discovery.probe_version(resolved.base_url) - if version is None: + alive = _discovery.probe_version(resolved.base_url) + if alive is None: _row("gateway", resolved.base_url, FAIL) print( f"\nNo tingly-box gateway responding at {resolved.base_url} " @@ -45,7 +45,7 @@ def doctor(scenario: str, link: bool) -> int: "Start tb, set TINGLY_BOX_URL, or run `tingly doctor --link`." ) return 1 - _row("gateway", f"{resolved.base_url} (v{version})", OK) + _row("gateway", f"{resolved.base_url} (reachable)", OK) _row("token", f"{resolved.source}", OK if resolved.token else WARN) # 2. mint a session (real path) diff --git a/sdk/python/tingly/discovery.py b/sdk/python/tingly/discovery.py index 60de8083d..ba4c8d824 100644 --- a/sdk/python/tingly/discovery.py +++ b/sdk/python/tingly/discovery.py @@ -33,24 +33,20 @@ class Session: def probe_version(base_url: str, timeout: float = 5.0) -> Optional[str]: - """Return the gateway version if reachable, else ``None``.""" - url = base_url.rstrip("/") + "/api/v1/info/version" + """Return a liveness marker if the gateway is reachable, else ``None``. + + Uses the unauthenticated ``/api/v1/info/health`` endpoint (the version + endpoint requires the admin token, so it cannot be used for discovery). + Returns the string ``"ok"`` when healthy — callers only check for truthiness. + """ + url = base_url.rstrip("/") + "/api/v1/info/health" try: resp = httpx.get(url, timeout=timeout) except httpx.HTTPError: return None if resp.status_code != 200: return None - try: - data = resp.json() - except ValueError: - return None - # tolerate both {version: ...} and {data: {version: ...}} - return ( - data.get("version") - or (data.get("data") or {}).get("version") - or "unknown" - ) + return "ok" def create_session( From 461c4721e2ba7383cfb32ce8f2271257f4bddb3c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 09:44:01 +0000 Subject: [PATCH 09/28] chore(sdk): make openai/anthropic core deps (fine-grained control is the point) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK's value is fine-grained control via the real provider SDKs (tb.openai / tb.anthropic expose every param, tool, streaming, beta header), and ask() / plugin.llm route through them — so they are core dependencies, not optional extras. Removes the install second-step that made the e2e callback fail until `pip install openai`. Updated README/examples/import-guard messages. --- sdk/python/README.md | 6 ++++-- sdk/python/examples/rag_experiment.py | 2 +- sdk/python/examples/rag_plugin.py | 2 +- sdk/python/pyproject.toml | 13 ++++++------- sdk/python/tingly/transports/anthropic_compat.py | 4 ++-- sdk/python/tingly/transports/openai_compat.py | 4 ++-- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/sdk/python/README.md b/sdk/python/README.md index 436087cf6..8805ad45d 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -7,10 +7,12 @@ logging. You write the idea; the box handles the plumbing. ## Install ```bash -pip install tingly # core -pip install "tingly[all]" # + openai and anthropic SDKs +pip install tingly ``` +Ships with the `openai` and `anthropic` SDKs so `tb.openai` / `tb.anthropic` +give you full fine-grained control out of the box. + ## Experiment ASAP ```python diff --git a/sdk/python/examples/rag_experiment.py b/sdk/python/examples/rag_experiment.py index 24ea0a15c..24580c2e7 100644 --- a/sdk/python/examples/rag_experiment.py +++ b/sdk/python/examples/rag_experiment.py @@ -2,7 +2,7 @@ Run a local tingly-box, then: - pip install -e ".[all]" # from sdk/python + pip install -e . # from sdk/python python examples/rag_experiment.py Everything below routes through tb: provider selection, fallback, guard rails, diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py index c0fd49cc8..6bc22d6f9 100644 --- a/sdk/python/examples/rag_plugin.py +++ b/sdk/python/examples/rag_plugin.py @@ -2,7 +2,7 @@ Run it: - pip install -e ".[all]" # from sdk/python + pip install -e . # from sdk/python python examples/rag_plugin.py # serves on http://127.0.0.1:8765/v1 Then wire it into tb in one step (creates the provider + a rule) so any client diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 94ec935fb..8ee2f469b 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -14,16 +14,15 @@ keywords = ["tingly-box", "llm", "gateway", "openai", "anthropic", "plugin"] dependencies = [ "httpx>=0.27", "pydantic>=2", + # Core, not optional: the value of the SDK is fine-grained control via the + # real provider SDKs (tb.openai / tb.anthropic expose every param, tool, + # streaming and beta header). ask() and plugin.llm route through them too. + "openai>=1.0", + "anthropic>=0.40", ] [project.optional-dependencies] -# The LLM SDKs are optional so a plugin that only uses `tb.ask()` (which goes -# through httpx) does not have to pull both. `connect().openai` /.anthropic -# raise a clear error if the matching extra is missing. -openai = ["openai>=1.0"] -anthropic = ["anthropic>=0.40"] -all = ["openai>=1.0", "anthropic>=0.40"] -dev = ["pytest>=8", "respx>=0.21", "openai>=1.0", "anthropic>=0.40"] +dev = ["pytest>=8", "respx>=0.21"] [project.scripts] tingly = "tingly.cli:main" diff --git a/sdk/python/tingly/transports/anthropic_compat.py b/sdk/python/tingly/transports/anthropic_compat.py index fb2aac301..222d500e3 100644 --- a/sdk/python/tingly/transports/anthropic_compat.py +++ b/sdk/python/tingly/transports/anthropic_compat.py @@ -17,7 +17,7 @@ def build_anthropic(base_url: str, token: str, timeout: float) -> Any: except ImportError as exc: # pragma: no cover - import guard raise ImportError( "The Anthropic transport requires the `anthropic` package. " - "Install it with `pip install tingly[anthropic]`." + "Reinstall tingly (it ships with anthropic)." ) from exc return anthropic.Anthropic( @@ -33,7 +33,7 @@ def build_async_anthropic(base_url: str, token: str, timeout: float) -> Any: except ImportError as exc: # pragma: no cover - import guard raise ImportError( "The Anthropic transport requires the `anthropic` package. " - "Install it with `pip install tingly[anthropic]`." + "Reinstall tingly (it ships with anthropic)." ) from exc return anthropic.AsyncAnthropic( diff --git a/sdk/python/tingly/transports/openai_compat.py b/sdk/python/tingly/transports/openai_compat.py index adc250fa6..0407b17d8 100644 --- a/sdk/python/tingly/transports/openai_compat.py +++ b/sdk/python/tingly/transports/openai_compat.py @@ -18,7 +18,7 @@ def build_openai(base_url: str, token: str, timeout: float) -> Any: except ImportError as exc: # pragma: no cover - import guard raise ImportError( "The OpenAI transport requires the `openai` package. " - "Install it with `pip install tingly[openai]`." + "Reinstall tingly (it ships with openai)." ) from exc return openai.OpenAI( @@ -34,7 +34,7 @@ def build_async_openai(base_url: str, token: str, timeout: float) -> Any: except ImportError as exc: # pragma: no cover - import guard raise ImportError( "The OpenAI transport requires the `openai` package. " - "Install it with `pip install tingly[openai]`." + "Reinstall tingly (it ships with openai)." ) from exc return openai.AsyncOpenAI( From c58d1fdcc91cf9e6c82fc60624cf437d575c93f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 11:50:37 +0000 Subject: [PATCH 10/28] feat(plugins): dynamic ephemeral registration + active SDK configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Treat a plugin as a runtime service instance, not a static config entry. tb-side: - In-memory PluginRegistry (process-local) holding leased, TTL'd plugin instances; stable id from name (UUIDv5), rotating lease per register, lazy expiry. Nothing persisted. - Config gains EphemeralProviderResolver: GetProviderByUUID / validateRuleServices fall back to the registry, so routing resolves live plugins transparently and an expired one isn't found → existing tier failover. db layer stays pure. - DNS-style layering: durable rule (the "name", idempotent) + ephemeral instance. - Endpoints: POST /api/v2/plugins/{register,heartbeat,deregister}, GET /plugins lists live + pinned. Persistent POST /api/v2/plugins stays as the "pin" path. SDK: - tingly.configure() / Connection: inject tb url + credentials at runtime (secrets by env reference), top-precedence in config.resolve(). - plugin/runtime.py: register/heartbeat/deregister + Heartbeater thread. - Plugin.serve(register=True, scenario=, ttl_seconds=, tb=Connection): self- register, background heartbeat, deregister on shutdown. Verified end-to-end (examples/e2e_run.sh): plugin self-registers as a live ephemeral instance, client call routes through it and back into tb — no network. Go + 33 py tests pass; openapi regenerated. --- .design/python-sdk.md | 35 ++++- internal/server/config/config.go | 11 ++ internal/server/config/provider.go | 23 +++ internal/server/plugin_dynamic_test.go | 101 ++++++++++++ internal/server/plugin_provider.go | 196 +++++++++++++++++++----- internal/server/plugin_registry.go | 158 +++++++++++++++++++ internal/server/plugin_registry_test.go | 81 ++++++++++ internal/server/server.go | 15 +- internal/server/server_webui_api.go | 23 ++- sdk/python/examples/e2e_run.sh | 15 +- sdk/python/tests/test_runtime.py | 124 +++++++++++++++ sdk/python/tingly/__init__.py | 3 + sdk/python/tingly/config.py | 51 +++++- sdk/python/tingly/plugin/core.py | 61 +++++++- sdk/python/tingly/plugin/runtime.py | 132 ++++++++++++++++ 15 files changed, 973 insertions(+), 56 deletions(-) create mode 100644 internal/server/plugin_dynamic_test.go create mode 100644 internal/server/plugin_registry.go create mode 100644 internal/server/plugin_registry_test.go create mode 100644 sdk/python/tests/test_runtime.py create mode 100644 sdk/python/tingly/plugin/runtime.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index a12f5140a..92cc74a41 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -79,9 +79,38 @@ provider *and* (when a scenario is given) the rule whose upstream is that plugin --scenario experiment` do the full one-step wire-in. This makes "configure this rule with a plugin" literal and eliminates the -provider+rule two-step mode-picker (`ux-principles.md`). Remaining tb-side work: -the rule-editor UI surfacing "plugin" as a service kind (frontend, codegen), and -the process supervisor/lifecycle. +provider+rule two-step mode-picker (`ux-principles.md`). + +### Plugin as runtime service (dynamic registration, implemented) + +A plugin is a **runtime instance**, not a static config entry. It registers at +startup, heartbeats to hold a lease, and is auto-removed when it stops/dies — +nothing persisted. Differs from a standard provider (durable, operator-managed). + +- **In-memory `PluginRegistry`** on the Server (process-local, matching tb's + circuit-breaker stance — no shared store). Stable id from name (UUIDv5) so a + restart re-registers under the same id; rotating `lease_id` per register. +- **Config hook** `EphemeralProviderResolver`: `GetProviderByUUID` / + `validateRuleServices` fall back to the registry, so **routing resolves live + plugins transparently** and an expired one simply isn't found → existing tier + failover routes to a tier-1 real model. The db layer stays pure persistence. +- **DNS-style layering**: the rule (the durable "name") is ensured idempotently; + the instance (endpoint + liveness) is ephemeral. No live instance ⇒ failover. +- Endpoints (apiV2): `POST /plugins/register` (leased; ensures rule), + `POST /plugins/heartbeat`, `POST /plugins/deregister`, `GET /plugins` (live + + pinned). The persistent `POST /api/v2/plugins` remains as the **pin** path. + +**Active configuration** (SDK): `tingly.configure(url=, admin_token_env=)` / +`Connection` inject the tb target + credentials at runtime (secrets by env +reference), top-precedence in `config.resolve()` — for containers / CI / remote +where there is no `~/.tingly-box`. `Plugin.serve(register=True, scenario=…, +ttl_seconds=…, tb=Connection(...))` self-registers, heartbeats on a background +thread, and deregisters on shutdown. + +Verified end-to-end (`examples/e2e_run.sh`): the plugin self-registers as a live +`ephemeral` instance, a client call routes through it, and it calls back into tb +— no network/keys. Remaining tb-side: rule-editor UI "plugin" kind (frontend), +process supervisor, scoped inference tokens, fully-ephemeral binding. ## Shape diff --git a/internal/server/config/config.go b/internal/server/config/config.go index 921d7356c..40637d6ce 100644 --- a/internal/server/config/config.go +++ b/internal/server/config/config.go @@ -119,6 +119,11 @@ type Config struct { imbotSettingsStore *db.ImBotSettingsStore templateManager *data.TemplateManager + // ephemeralResolver resolves non-persisted provider UUIDs (live plugin + // instances). Consulted as a fallback by GetProviderByUUID / + // validateRuleServices. Guarded by mu. + ephemeralResolver EphemeralProviderResolver + // Provider lifecycle hooks providerUpdateHooks []ProviderUpdateHook providerDeleteHooks []ProviderDeleteHook @@ -2536,6 +2541,12 @@ func (c *Config) validateRuleServices(rule typ.Rule) error { provider, err := c.providerStore.GetByUUID(svc.Provider) if err != nil { + // A live plugin instance is a valid (ephemeral) provider target. + if c.ephemeralResolver != nil { + if _, ok := c.ephemeralResolver.Resolve(svc.Provider); ok { + continue + } + } return fmt.Errorf("service references non-existent provider '%s': %w", svc.Provider, err) } if provider == nil { diff --git a/internal/server/config/provider.go b/internal/server/config/provider.go index 4d28c719e..48bd5537d 100644 --- a/internal/server/config/provider.go +++ b/internal/server/config/provider.go @@ -23,6 +23,22 @@ type ProviderDeleteHook interface { OnProviderDelete(uuid string) } +// EphemeralProviderResolver resolves provider UUIDs that are not persisted in +// the provider store — e.g. live plugin instances held in an in-memory registry. +// It is consulted as a fallback by provider lookups so dynamically-registered +// plugins route transparently, while keeping the db layer pure persistence. +type EphemeralProviderResolver interface { + Resolve(uuid string) (*typ.Provider, bool) +} + +// SetEphemeralProviderResolver installs the fallback resolver (e.g. the Server's +// plugin registry). Safe to call once during server construction. +func (c *Config) SetEphemeralProviderResolver(r EphemeralProviderResolver) { + c.mu.Lock() + defer c.mu.Unlock() + c.ephemeralResolver = r +} + // migrateProvidersToDB migrates providers from JSON config to database. // This is a one-time migration that runs on startup if the database is empty. // After migration (or if the database is already authoritative), the JSON @@ -113,6 +129,13 @@ func (c *Config) GetProviderByUUID(uuid string) (*typ.Provider, error) { provider, err := c.providerStore.GetByUUID(uuid) if err != nil { + // Fall back to the ephemeral resolver (live plugin instances that are not + // persisted). A miss here means the provider is truly unavailable. + if c.ephemeralResolver != nil { + if p, ok := c.ephemeralResolver.Resolve(uuid); ok { + return p, nil + } + } return nil, fmt.Errorf("provider '%s' not found: %w", uuid, err) } return provider, nil diff --git a/internal/server/plugin_dynamic_test.go b/internal/server/plugin_dynamic_test.go new file mode 100644 index 000000000..ea8aae741 --- /dev/null +++ b/internal/server/plugin_dynamic_test.go @@ -0,0 +1,101 @@ +package server + +import ( + "testing" + + "github.com/tingly-dev/tingly-box/internal/server/config" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +func newDynamicPluginServer(t *testing.T) *Server { + t.Helper() + cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + reg := NewPluginRegistry() + cfg.SetEphemeralProviderResolver(reg) + return &Server{config: cfg, pluginRegistry: reg} +} + +func TestRegisterPluginDynamic_RoutesAndExpires(t *testing.T) { + s := newDynamicPluginServer(t) + + w, resp := postJSON(t, s.RegisterPluginDynamic, RegisterPluginDynamicRequest{ + Name: "my-rag", + Endpoint: "http://127.0.0.1:8765/v1", + ModelID: "plugin/my-rag", + Scenario: string(typ.ScenarioExperiment), + }) + if w.Code != 200 { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + data, _ := resp["data"].(map[string]any) + pluginID, _ := data["plugin_id"].(string) + leaseID, _ := data["lease_id"].(string) + if pluginID == "" || leaseID == "" { + t.Fatalf("missing ids: %v", data) + } + + // No persistent provider was created. + for _, p := range s.config.ListProviders() { + if p.UUID == pluginID { + t.Fatalf("dynamic registration must NOT persist a provider") + } + } + + // But routing resolution (the real dispatch chokepoint) finds the live instance. + prov, err := s.config.GetProviderByUUID(pluginID) + if err != nil || !prov.IsPlugin() { + t.Fatalf("expected live ephemeral resolution, err=%v prov=%+v", err, prov) + } + + // The durable rule (the name) was bound to the plugin id. + var bound bool + for _, rule := range s.config.GetRequestConfigs() { + if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { + bound = true + if rule.Services[0].Provider != pluginID { + t.Fatalf("rule service should reference plugin id, got %s", rule.Services[0].Provider) + } + } + } + if !bound { + t.Fatalf("expected a durable rule bound to the plugin") + } + + // After deregister, the instance is gone → routing can no longer resolve it + // (→ tier failover in a real request). + postJSON(t, s.DeregisterPlugin, PluginLeaseRequest{LeaseID: leaseID}) + if _, err := s.config.GetProviderByUUID(pluginID); err == nil { + t.Fatalf("provider must be unresolved after deregister") + } +} + +func TestHeartbeatPlugin_UnknownLease(t *testing.T) { + s := newDynamicPluginServer(t) + w, _ := postJSON(t, s.HeartbeatPlugin, PluginLeaseRequest{LeaseID: "nope"}) + if w.Code != 404 { + t.Fatalf("expected 404 for unknown lease, got %d", w.Code) + } +} + +func TestReRegisterIsIdempotentForRule(t *testing.T) { + s := newDynamicPluginServer(t) + postJSON(t, s.RegisterPluginDynamic, RegisterPluginDynamicRequest{ + Name: "p", Endpoint: "http://a/v1", Scenario: string(typ.ScenarioExperiment), + }) + postJSON(t, s.RegisterPluginDynamic, RegisterPluginDynamicRequest{ + Name: "p", Endpoint: "http://b/v1", Scenario: string(typ.ScenarioExperiment), + }) + // exactly one rule for plugin/p + count := 0 + for _, rule := range s.config.GetRequestConfigs() { + if rule.RequestModel == "plugin/p" { + count++ + } + } + if count != 1 { + t.Fatalf("re-register must not duplicate the rule, got %d", count) + } +} diff --git a/internal/server/plugin_provider.go b/internal/server/plugin_provider.go index ffb68c25c..ace17d9e6 100644 --- a/internal/server/plugin_provider.go +++ b/internal/server/plugin_provider.go @@ -2,6 +2,7 @@ package server import ( "net/http" + "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" @@ -37,12 +38,13 @@ type RegisterPluginResponse struct { // PluginInfo is a list view of a plugin-kind provider. type PluginInfo struct { - UUID string `json:"uuid"` - Name string `json:"name"` - Endpoint string `json:"endpoint"` - ModelID string `json:"model_id"` - Managed bool `json:"managed"` - Enabled bool `json:"enabled"` + UUID string `json:"uuid"` + Name string `json:"name"` + Endpoint string `json:"endpoint"` + ModelID string `json:"model_id"` + Managed bool `json:"managed"` + Enabled bool `json:"enabled"` + Ephemeral bool `json:"ephemeral"` // true for live dynamic registrations } // PluginsResponse wraps the plugin list. @@ -95,39 +97,14 @@ func (s *Server) RegisterPlugin(c *gin.Context) { // One-step bind: create the rule whose single service is this plugin. if req.Scenario != "" { - scenario := typ.RuleScenario(req.Scenario) - if !typ.CanBindRulesToScenario(scenario) { - // Provider is created; surface the bind failure without 500ing. - resp.Note = "Provider created, but scenario " + req.Scenario + - " is not bindable; bind a rule manually." - c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) - return - } - - rule := typ.Rule{ - UUID: config.GenerateUUID(), - Scenario: scenario, - RequestModel: modelID, - Description: "Plugin: " + req.Name, - Active: true, - LBTactic: typ.ParseTacticFromMap(loadbalance.TacticTier, nil), - Services: []*loadbalance.Service{ - { - Provider: provider.UUID, - Model: modelID, - Weight: 1, - Active: true, - Tier: req.Tier, - }, - }, - } - if err := s.config.AddRule(rule); err != nil { + ruleUUID, err := s.ensurePluginRule(req.Scenario, modelID, provider.UUID, req.Name, req.Tier) + if err != nil { resp.Note = "Provider created, but rule binding failed: " + err.Error() c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) return } resp.Scenario = req.Scenario - resp.RuleUUID = rule.UUID + resp.RuleUUID = ruleUUID resp.Ready = true resp.Note = "Plugin wired in. Select model " + modelID + " under scenario " + req.Scenario + "." } @@ -143,11 +120,30 @@ func (s *Server) RegisterPlugin(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } -// ListPlugins returns the plugin-kind providers, for the UI's plugin section. +// ListPlugins returns plugin providers: live ephemeral instances from the +// registry plus any pinned/persistent plugin-kind providers. func (s *Server) ListPlugins(c *gin.Context) { var plugins []PluginInfo + seen := map[string]bool{} + + // Live, dynamically-registered instances first. + if s.pluginRegistry != nil { + for _, reg := range s.pluginRegistry.List() { + seen[reg.ID] = true + plugins = append(plugins, PluginInfo{ + UUID: reg.ID, + Name: reg.Name, + Endpoint: reg.Endpoint, + ModelID: reg.ModelID, + Enabled: true, + Ephemeral: true, + }) + } + } + + // Pinned / persistent plugin providers. for _, p := range s.config.ListProviders() { - if !p.IsPlugin() { + if !p.IsPlugin() || seen[p.UUID] { continue } modelID := "" @@ -166,3 +162,131 @@ func (s *Server) ListPlugins(c *gin.Context) { } c.JSON(http.StatusOK, PluginsResponse{Success: true, Data: plugins}) } + +// RegisterPluginDynamicRequest registers a live, ephemeral plugin instance. +type RegisterPluginDynamicRequest struct { + Name string `json:"name" binding:"required" example:"my-rag"` + Endpoint string `json:"endpoint" binding:"required" example:"http://127.0.0.1:8765/v1"` + ModelID string `json:"model_id,omitempty" example:"plugin/my-rag"` + Token string `json:"token,omitempty"` + Scenario string `json:"scenario,omitempty" example:"experiment"` + Tier int `json:"tier,omitempty"` + TTLSeconds int `json:"ttl_seconds,omitempty" example:"30"` +} + +// RegisterPluginDynamicResponse reports the lease for an ephemeral registration. +type RegisterPluginDynamicResponse struct { + PluginID string `json:"plugin_id"` + LeaseID string `json:"lease_id"` + ModelID string `json:"model_id"` + Scenario string `json:"scenario,omitempty"` + RuleUUID string `json:"rule_uuid,omitempty"` + TTLSeconds int `json:"ttl_seconds"` + Note string `json:"note,omitempty"` +} + +// RegisterPluginDynamic registers a live plugin instance in the in-memory +// registry (NOT persisted). The plugin keeps it alive by heartbeating; it is +// auto-removed when the lease expires or the plugin deregisters. When a scenario +// is given, the durable rule (the stable "name") is ensured idempotently. +func (s *Server) RegisterPluginDynamic(c *gin.Context) { + var req RegisterPluginDynamicRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + ttl := time.Duration(req.TTLSeconds) * time.Second + reg := s.pluginRegistry.Register(req.Name, req.Endpoint, req.ModelID, req.Scenario, req.Token, ttl) + + resp := RegisterPluginDynamicResponse{ + PluginID: reg.ID, + LeaseID: reg.LeaseID, + ModelID: reg.ModelID, + TTLSeconds: int(time.Until(reg.ExpiresAt).Seconds()), + Note: "Registered (ephemeral). Heartbeat to keep alive; deregister to remove.", + } + + if req.Scenario != "" { + // The durable binding references the stable plugin id; the live instance + // is resolved from the registry at request time. + ruleUUID, err := s.ensurePluginRule(req.Scenario, reg.ModelID, reg.ID, req.Name, req.Tier) + if err != nil { + resp.Note = "Registered, but rule binding failed: " + err.Error() + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) + return + } + resp.Scenario = req.Scenario + resp.RuleUUID = ruleUUID + } + + logrus.WithFields(logrus.Fields{ + "plugin": req.Name, "endpoint": req.Endpoint, "model_id": reg.ModelID, + "scenario": req.Scenario, "ttl_s": resp.TTLSeconds, + }).Info("Registered dynamic plugin instance") + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) +} + +// PluginLeaseRequest carries a lease id for heartbeat/deregister. +type PluginLeaseRequest struct { + LeaseID string `json:"lease_id" binding:"required"` + TTLSeconds int `json:"ttl_seconds,omitempty"` +} + +// HeartbeatPlugin extends a plugin instance's lease. +func (s *Server) HeartbeatPlugin(c *gin.Context) { + var req PluginLeaseRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + ttl := time.Duration(req.TTLSeconds) * time.Second + if !s.pluginRegistry.Heartbeat(req.LeaseID, ttl) { + c.JSON(http.StatusNotFound, gin.H{"success": false, "error": "unknown or expired lease"}) + return + } + c.JSON(http.StatusOK, gin.H{"success": true}) +} + +// DeregisterPlugin removes a live plugin instance immediately. +func (s *Server) DeregisterPlugin(c *gin.Context) { + var req PluginLeaseRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + removed := s.pluginRegistry.Deregister(req.LeaseID) + c.JSON(http.StatusOK, gin.H{"success": true, "data": gin.H{"removed": removed}}) +} + +// ensurePluginRule idempotently ensures a rule exists under scenario whose single +// tier-service points at the given provider id for modelID. Returns the rule UUID. +func (s *Server) ensurePluginRule(scenario, modelID, providerID, name string, tier int) (string, error) { + scn := typ.RuleScenario(scenario) + if !typ.CanBindRulesToScenario(scn) { + return "", &pluginBindError{"scenario " + scenario + " is not bindable"} + } + for _, rule := range s.config.GetRequestConfigs() { + if rule.GetScenario() == scn && rule.RequestModel == modelID { + return rule.UUID, nil // already bound (idempotent) + } + } + rule := typ.Rule{ + UUID: config.GenerateUUID(), + Scenario: scn, + RequestModel: modelID, + Description: "Plugin: " + name, + Active: true, + LBTactic: typ.ParseTacticFromMap(loadbalance.TacticTier, nil), + Services: []*loadbalance.Service{ + {Provider: providerID, Model: modelID, Weight: 1, Active: true, Tier: tier}, + }, + } + if err := s.config.AddRule(rule); err != nil { + return "", err + } + return rule.UUID, nil +} + +type pluginBindError struct{ msg string } + +func (e *pluginBindError) Error() string { return e.msg } diff --git a/internal/server/plugin_registry.go b/internal/server/plugin_registry.go new file mode 100644 index 000000000..c8e69caec --- /dev/null +++ b/internal/server/plugin_registry.go @@ -0,0 +1,158 @@ +package server + +import ( + "sync" + "time" + + "github.com/google/uuid" + + "github.com/tingly-dev/tingly-box/internal/constant" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// pluginNamespace is the UUIDv5 namespace used to derive a stable plugin id from +// its name, so a plugin that restarts re-registers under the same id and any +// durable rule that references it keeps pointing correctly. +var pluginNamespace = uuid.MustParse("3b1e0a2c-7c4e-5a9b-bf21-9f6d2c8e4a10") + +const defaultPluginTTL = 30 * time.Second + +// PluginRegistration is a live, ephemeral plugin instance. +type PluginRegistration struct { + ID string // deterministic from Name (UUIDv5) + Name string // plugin / provider name + Endpoint string // OpenAI base, e.g. http://127.0.0.1:8765/v1 + ModelID string // advertised model id, e.g. plugin/my-rag + Scenario string // scenario the durable rule was bound under (optional) + Token string // token tb sends to the plugin (optional) + LeaseID string // rotates each register; required to heartbeat/deregister + ExpiresAt time.Time + LastSeen time.Time +} + +// PluginRegistry holds live plugin instances in memory. It is intentionally +// process-local (no shared store), matching tb's existing routing state stance. +// Instances auto-expire when their lease is not renewed; nothing is persisted. +type PluginRegistry struct { + mu sync.RWMutex + byID map[string]*PluginRegistration + ttl time.Duration +} + +// NewPluginRegistry creates an empty registry with the default lease TTL. +func NewPluginRegistry() *PluginRegistry { + return &PluginRegistry{byID: map[string]*PluginRegistration{}, ttl: defaultPluginTTL} +} + +// PluginID derives the stable id for a plugin name. +func PluginID(name string) string { + return uuid.NewSHA1(pluginNamespace, []byte(name)).String() +} + +// Register adds or refreshes a plugin instance and returns the live record. +// ttl <= 0 uses the registry default. +func (r *PluginRegistry) Register(name, endpoint, modelID, scenario, token string, ttl time.Duration) *PluginRegistration { + if ttl <= 0 { + ttl = r.ttl + } + if modelID == "" { + modelID = "plugin/" + name + } + now := time.Now() + reg := &PluginRegistration{ + ID: PluginID(name), + Name: name, + Endpoint: endpoint, + ModelID: modelID, + Scenario: scenario, + Token: token, + LeaseID: uuid.NewString(), + ExpiresAt: now.Add(ttl), + LastSeen: now, + } + r.mu.Lock() + r.byID[reg.ID] = reg + r.mu.Unlock() + return reg +} + +// Heartbeat extends the lease identified by leaseID. Returns false if no live +// registration matches (unknown or already expired). +func (r *PluginRegistry) Heartbeat(leaseID string, ttl time.Duration) bool { + if ttl <= 0 { + ttl = r.ttl + } + now := time.Now() + r.mu.Lock() + defer r.mu.Unlock() + for _, reg := range r.byID { + if reg.LeaseID == leaseID { + if now.After(reg.ExpiresAt) { + delete(r.byID, reg.ID) + return false + } + reg.ExpiresAt = now.Add(ttl) + reg.LastSeen = now + return true + } + } + return false +} + +// Deregister removes the instance for leaseID. Returns true if one was removed. +func (r *PluginRegistry) Deregister(leaseID string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for id, reg := range r.byID { + if reg.LeaseID == leaseID { + delete(r.byID, id) + return true + } + } + return false +} + +// Resolve synthesizes a plugin-kind provider for a live instance by id. Expired +// instances are treated as absent (and reaped). Implements +// config.EphemeralProviderResolver. +func (r *PluginRegistry) Resolve(id string) (*typ.Provider, bool) { + r.mu.Lock() + defer r.mu.Unlock() + reg, ok := r.byID[id] + if !ok { + return nil, false + } + if time.Now().After(reg.ExpiresAt) { + delete(r.byID, id) + return nil, false + } + return &typ.Provider{ + UUID: reg.ID, + Name: reg.Name, + APIBase: reg.Endpoint, + APIStyle: "openai", + Token: reg.Token, + NoKeyRequired: reg.Token == "", + Enabled: true, + AuthType: typ.AuthTypeAPIKey, + Timeout: constant.DefaultRequestTimeout, + PluginDetail: &typ.PluginDetail{ModelID: reg.ModelID}, + }, true +} + +// List returns the currently-live registrations (expired ones are reaped). +func (r *PluginRegistry) List() []*PluginRegistration { + now := time.Now() + r.mu.Lock() + defer r.mu.Unlock() + out := make([]*PluginRegistration, 0, len(r.byID)) + for id, reg := range r.byID { + if now.After(reg.ExpiresAt) { + delete(r.byID, id) + continue + } + clone := *reg + out = append(out, &clone) + } + return out +} diff --git a/internal/server/plugin_registry_test.go b/internal/server/plugin_registry_test.go new file mode 100644 index 000000000..ad6fef744 --- /dev/null +++ b/internal/server/plugin_registry_test.go @@ -0,0 +1,81 @@ +package server + +import ( + "testing" + "time" +) + +func TestPluginRegistry_RegisterResolveExpire(t *testing.T) { + r := NewPluginRegistry() + reg := r.Register("my-rag", "http://127.0.0.1:8765/v1", "plugin/my-rag", "experiment", "", 50*time.Millisecond) + + // stable id from name + if reg.ID != PluginID("my-rag") { + t.Fatalf("id not derived from name: %s", reg.ID) + } + + // resolves to a live plugin-kind provider + p, ok := r.Resolve(reg.ID) + if !ok { + t.Fatalf("expected live resolution") + } + if !p.IsPlugin() || p.APIBase != "http://127.0.0.1:8765/v1" || p.PluginDetail.ModelID != "plugin/my-rag" { + t.Fatalf("synthesized provider wrong: %+v", p) + } + + // after TTL it is gone (auto-expire on resolve) + time.Sleep(70 * time.Millisecond) + if _, ok := r.Resolve(reg.ID); ok { + t.Fatalf("expected expiry after TTL") + } +} + +func TestPluginRegistry_HeartbeatKeepsAlive(t *testing.T) { + r := NewPluginRegistry() + reg := r.Register("p", "http://x/v1", "", "", "", 60*time.Millisecond) + + time.Sleep(40 * time.Millisecond) + if !r.Heartbeat(reg.LeaseID, 60*time.Millisecond) { + t.Fatalf("heartbeat should succeed before expiry") + } + time.Sleep(40 * time.Millisecond) // 80ms since register, but heartbeat reset it + if _, ok := r.Resolve(reg.ID); !ok { + t.Fatalf("heartbeat should have kept the instance alive") + } + + // unknown lease + if r.Heartbeat("nope", 0) { + t.Fatalf("unknown lease must not heartbeat") + } +} + +func TestPluginRegistry_Deregister(t *testing.T) { + r := NewPluginRegistry() + reg := r.Register("p", "http://x/v1", "", "", "", time.Minute) + if !r.Deregister(reg.LeaseID) { + t.Fatalf("deregister should remove the instance") + } + if _, ok := r.Resolve(reg.ID); ok { + t.Fatalf("resolve should miss after deregister") + } + if r.Deregister(reg.LeaseID) { + t.Fatalf("second deregister should be a no-op") + } +} + +func TestPluginRegistry_ReRegisterReusesID(t *testing.T) { + r := NewPluginRegistry() + a := r.Register("same", "http://a/v1", "", "", "", time.Minute) + b := r.Register("same", "http://b/v1", "", "", "", time.Minute) + if a.ID != b.ID { + t.Fatalf("re-register must reuse the stable id") + } + if a.LeaseID == b.LeaseID { + t.Fatalf("lease must rotate on re-register") + } + // latest endpoint wins + p, _ := r.Resolve(b.ID) + if p.APIBase != "http://b/v1" { + t.Fatalf("latest registration endpoint should win, got %s", p.APIBase) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index e7c755033..070a360e7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -66,6 +66,10 @@ type Server struct { loadBalancerAPI *LoadBalancerAPI healthMonitor *loadbalance.HealthMonitor + // pluginRegistry holds live, ephemeral plugin instances (dynamic + // registration). It is the EphemeralProviderResolver for the config. + pluginRegistry *PluginRegistry + // client pool for caching clientPool *client.ClientPool @@ -210,10 +214,15 @@ func NewServer(cfg *config.Config, opts ...ServerOption) *Server { // Default options server := &Server{ - config: cfg, - ctx: ctx, - cancel: cancel, + config: cfg, + ctx: ctx, + cancel: cancel, + pluginRegistry: NewPluginRegistry(), } + // Live plugin instances resolve through this in-memory registry as an + // ephemeral provider fallback, so dynamically-registered plugins route + // without a persisted provider row. + cfg.SetEphemeralProviderResolver(server.pluginRegistry) // Apply all options (defaults + provided) for _, opt := range allOpts { diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index 6aae6a393..7dacc92c1 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -410,11 +410,32 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { ) api.GET("/plugins", s.ListPlugins, - swagger.WithDescription("List registered plugin-kind providers"), + swagger.WithDescription("List registered plugin-kind providers (live + pinned)"), swagger.WithTags("plugins"), swagger.WithResponseModel(PluginsResponse{}), ) + // Dynamic (ephemeral) plugin lifecycle: register a live instance, keep it + // alive by heartbeat, and deregister on shutdown. Nothing is persisted. + api.POST("/plugins/register", s.RegisterPluginDynamic, + swagger.WithDescription("Register a live, ephemeral plugin instance (leased)"), + swagger.WithTags("plugins"), + swagger.WithRequestModel(RegisterPluginDynamicRequest{}), + swagger.WithResponseModel(RegisterPluginDynamicResponse{}), + ) + api.POST("/plugins/heartbeat", s.HeartbeatPlugin, + swagger.WithDescription("Extend a plugin instance's lease"), + swagger.WithTags("plugins"), + swagger.WithRequestModel(PluginLeaseRequest{}), + swagger.WithResponseModel(gin.H{}), + ) + api.POST("/plugins/deregister", s.DeregisterPlugin, + swagger.WithDescription("Remove a live plugin instance immediately"), + swagger.WithTags("plugins"), + swagger.WithRequestModel(PluginLeaseRequest{}), + swagger.WithResponseModel(gin.H{}), + ) + // Provider template endpoints providerTemplateHandler := providertemplate.NewHandler(s.templateManager) providertemplate.RegisterRoutes(apiV2, providerTemplateHandler) diff --git a/sdk/python/examples/e2e_run.sh b/sdk/python/examples/e2e_run.sh index 4d2ad9c8a..5d331caca 100755 --- a/sdk/python/examples/e2e_run.sh +++ b/sdk/python/examples/e2e_run.sh @@ -56,7 +56,8 @@ curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ \"services\":[{\"provider\":\"$VUUID\",\"model\":\"echo-model\",\"weight\":1,\"active\":true}]}" \ | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'), d.get('data',{}).get('uuid',''))" -echo "== 4. start the plugin (serves OpenAI on :8765, calls back into tb) ==" +echo "== 4. start the plugin — it DYNAMICALLY self-registers with tb ==" +echo " (serve(register=True) → POST /plugins/register + heartbeat; nothing persisted)" TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ python3 "$SDK/examples/e2e_plugin.py" >/tmp/plugin_e2e.log 2>&1 & PLUG_PID=$! @@ -65,12 +66,14 @@ for i in $(seq 1 40); do sleep 0.3 done curl -sf "http://127.0.0.1:8765/health" >/dev/null || { echo "plugin did not start"; cat /tmp/plugin_e2e.log; exit 1; } -echo " plugin up: $(curl -s http://127.0.0.1:8765/v1/models)" -echo "== 5. register the plugin with tb (one step: provider + rule) ==" -curl -s "${UADMIN[@]}" -X POST "$BASE/api/v2/plugins" -d '{ - "name":"rag-demo","endpoint":"http://127.0.0.1:8765/v1", - "model_id":"plugin/rag-demo","scenario":"experiment"}' | python3 -m json.tool +echo "== 5. tb sees the LIVE ephemeral instance (GET /api/v2/plugins) ==" +for i in $(seq 1 20); do + LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") + echo "$LIST" | grep -q 'rag-demo' && break + sleep 0.3 +done +echo "$LIST" | python3 -m json.tool echo "== 6. CLIENT CALL: model=plugin/rag-demo through tb ==" echo " (client → tb → plugin → tb echo-model → back)" diff --git a/sdk/python/tests/test_runtime.py b/sdk/python/tests/test_runtime.py new file mode 100644 index 000000000..b1bac8008 --- /dev/null +++ b/sdk/python/tests/test_runtime.py @@ -0,0 +1,124 @@ +"""Active config + dynamic registration tests.""" + +import threading + +import httpx +import pytest +import respx + +import tingly +import tingly.config as cfg +from tingly.plugin import runtime + +BASE = "http://tb.test:12580" + + +@pytest.fixture(autouse=True) +def _reset_override(): + cfg._OVERRIDE = None + yield + cfg._OVERRIDE = None + + +def test_configure_takes_precedence(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, "http://env:1") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + tingly.configure(url="http://configured:9", admin_token="cfg-token") + r = cfg.resolve() + assert r.base_url == "http://configured:9" + assert r.token == "cfg-token" + assert r.source == "configure" + + +def test_connection_token_by_env_reference(monkeypatch): + monkeypatch.setenv("MY_TB_SECRET", "secret-123") + conn = tingly.Connection(url="http://x", admin_token_env="MY_TB_SECRET") + assert conn.token() == "secret-123" + + +@respx.mock +def test_runtime_register_heartbeat_deregister(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, BASE) + monkeypatch.setenv(cfg.ENV_TOKEN, "admin") + + reg_route = respx.post(f"{BASE}/api/v2/plugins/register").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": { + "plugin_id": "pid-1", "lease_id": "lease-1", "model_id": "plugin/x", + "scenario": "experiment", "rule_uuid": "rule-1", "ttl_seconds": 30, + }, + }) + ) + hb_route = respx.post(f"{BASE}/api/v2/plugins/heartbeat").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + dr_route = respx.post(f"{BASE}/api/v2/plugins/deregister").mock( + return_value=httpx.Response(200, json={"success": True, "data": {"removed": True}}) + ) + + lease = runtime.register("x", "http://127.0.0.1:8765/v1", "plugin/x", scenario="experiment") + assert reg_route.called + assert lease.lease_id == "lease-1" + assert lease.rule_uuid == "rule-1" + + assert runtime.heartbeat(lease) is True + assert hb_route.called + + runtime.deregister(lease) + assert dr_route.called + + +@respx.mock +def test_serve_registers_and_deregisters(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, BASE) + monkeypatch.setenv(cfg.ENV_TOKEN, "admin") + respx.post(f"{BASE}/api/v2/plugins/register").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": {"plugin_id": "pid", "lease_id": "L", "model_id": "plugin/srv", + "scenario": "experiment", "ttl_seconds": 30}, + }) + ) + dr = respx.post(f"{BASE}/api/v2/plugins/deregister").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + + from tingly import Plugin + + plugin = Plugin(name="srv", scenario="experiment") + + @plugin.chat + def handle(req): + return "ok" + + # ttl high so the heartbeat thread doesn't fire during the test + port = plugin.serve(port=0, verbose=False, block=False, ttl_seconds=300) + assert isinstance(port, int) and port > 0 + assert plugin._lease is not None and plugin._lease.lease_id == "L" + + # real plugin socket still serves (use urllib so respx doesn't intercept it) + import urllib.request + + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) as r: + assert r.status == 200 + + plugin.stop() + assert dr.called # deregistered on shutdown + assert plugin._lease is None + + +def test_serve_register_false_skips(monkeypatch): + from tingly import Plugin + + plugin = Plugin(name="noreg") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False, register=False) + try: + assert plugin._lease is None + finally: + plugin.stop() diff --git a/sdk/python/tingly/__init__.py b/sdk/python/tingly/__init__.py index 5e78ba6b8..e9fb3ec34 100644 --- a/sdk/python/tingly/__init__.py +++ b/sdk/python/tingly/__init__.py @@ -12,6 +12,7 @@ from ._version import __version__ from .client import Client, connect +from .config import Connection, configure from .errors import ( AuthError, GatewayUnreachableError, @@ -25,6 +26,8 @@ __all__ = [ "__version__", "connect", + "configure", + "Connection", "Client", "Plugin", "ChatRequest", diff --git a/sdk/python/tingly/config.py b/sdk/python/tingly/config.py index 8a924e04b..a9d9c452e 100644 --- a/sdk/python/tingly/config.py +++ b/sdk/python/tingly/config.py @@ -49,13 +49,52 @@ def tb_config_path() -> Path: return config_dir() / "config.json" +@dataclass +class Connection: + """An explicit, runtime-injected tb connection — for containers / CI / + remote where there is no ``~/.tingly-box``. + + Secrets may be given by **reference** (an env var name) so they are not + hard-coded; ``token`` resolves the literal or the env var at use time. + """ + + url: Optional[str] = None + admin_token: Optional[str] = None + admin_token_env: Optional[str] = None + + def token(self) -> Optional[str]: + if self.admin_token: + return self.admin_token + if self.admin_token_env: + return os.environ.get(self.admin_token_env) + return None + + +# Process-wide override set by configure(); highest precedence in resolve(). +_OVERRIDE: Optional[Connection] = None + + +def configure( + url: Optional[str] = None, + admin_token: Optional[str] = None, + admin_token_env: Optional[str] = None, +) -> None: + """Actively configure the tb target + credentials for this process. + + Takes precedence over env / files. Useful when a plugin must point at a + specific tb and use injected credentials rather than local discovery. + """ + global _OVERRIDE + _OVERRIDE = Connection(url=url, admin_token=admin_token, admin_token_env=admin_token_env) + + @dataclass class Resolved: """A resolved gateway target plus where it came from (for diagnostics).""" base_url: Optional[str] token: Optional[str] - source: str # "args" | "env" | "sdk.json" | "config.json" | "probe-default" + source: str # "configure" | "args" | "env" | "sdk.json" | "config.json" | "probe-default" def _read_json(path: Path) -> Optional[dict]: @@ -77,6 +116,16 @@ def resolve( token from env / files. """ src = "args" + + # Highest precedence: an explicit configure()/Connection override. + if _OVERRIDE is not None: + if base_url is None and _OVERRIDE.url: + base_url, src = _OVERRIDE.url, "configure" + if token is None and _OVERRIDE.token(): + token = _OVERRIDE.token() + if src == "args": + src = "configure" + if base_url is None: env_url = os.environ.get(ENV_URL) if env_url: diff --git a/sdk/python/tingly/plugin/core.py b/sdk/python/tingly/plugin/core.py index ef4dc2109..6111287f1 100644 --- a/sdk/python/tingly/plugin/core.py +++ b/sdk/python/tingly/plugin/core.py @@ -58,6 +58,8 @@ def __init__( self._handler: Optional[ChatHandler] = None self._clients: dict = {} # scenario -> lazily-connected client self._httpd = None + self._lease = None # runtime.Lease when dynamically registered + self._heartbeater = None # -- authoring ------------------------------------------------------- @@ -131,13 +133,26 @@ def serve( *, verbose: bool = True, block: bool = True, + register: bool = True, + advertise_host: Optional[str] = None, + ttl_seconds: int = 30, + tb: Optional[Any] = None, ) -> int: - """Run the plugin's HTTP server. + """Run the plugin's HTTP server and (by default) register it with tb. - Returns the bound port (resolved even when ``port=0``). With - ``block=False`` the server runs on a daemon thread and the call returns - immediately — handy for tests and for embedding. + Dynamic registration is ephemeral: the plugin appears in tb only while it + runs — a background heartbeat keeps the lease, and it deregisters on + shutdown. ``tb`` may be a :class:`tingly.config.Connection` to point at a + specific gateway / inject credentials (containers / CI / remote). + + Returns the bound port (resolved even when ``port=0``). ``block=False`` + runs the server on a daemon thread and returns immediately. """ + if tb is not None: + from ..config import configure + + configure(url=tb.url, admin_token=tb.admin_token, admin_token_env=tb.admin_token_env) + httpd, bound = make_server( self._dispatch, self.model_id, @@ -150,8 +165,12 @@ def serve( if verbose: print( f"[tingly] plugin {self.name!r} serving model {self.model_id!r} " - f"on http://{host}:{bound}/v1 (register as an OpenAI provider in tb)" + f"on http://{host}:{bound}/v1" ) + + if register: + self._register(advertise_host or host, bound, ttl_seconds, verbose) + if not block: t = threading.Thread(target=httpd.serve_forever, daemon=True) t.start() @@ -161,10 +180,40 @@ def serve( except KeyboardInterrupt: pass finally: - httpd.shutdown() + self.stop() return bound + def _register(self, host: str, port: int, ttl_seconds: int, verbose: bool) -> None: + from . import runtime + + endpoint = f"http://{host}:{port}/v1" + try: + lease = runtime.register( + self.name, endpoint, self.model_id, + scenario=self.scenario, token=self.api_key, ttl_seconds=ttl_seconds, + ) + except Exception as exc: # noqa: BLE001 - registration is best-effort + if verbose: + print(f"[tingly] plugin registration skipped: {exc}") + return + self._lease = lease + self._heartbeater = runtime.Heartbeater(lease).start() + if verbose: + print( + f"[tingly] registered '{self.name}' as model {lease.model_id!r}" + + (f" under scenario {lease.scenario!r}" if lease.scenario else "") + + f" (lease ttl={lease.ttl_seconds}s)" + ) + def stop(self) -> None: + if self._heartbeater is not None: + self._heartbeater.stop() + self._heartbeater = None + if self._lease is not None: + from . import runtime + + runtime.deregister(self._lease) + self._lease = None if self._httpd is not None: self._httpd.shutdown() self._httpd = None diff --git a/sdk/python/tingly/plugin/runtime.py b/sdk/python/tingly/plugin/runtime.py new file mode 100644 index 000000000..5eca9b8fe --- /dev/null +++ b/sdk/python/tingly/plugin/runtime.py @@ -0,0 +1,132 @@ +"""Dynamic (ephemeral) plugin registration with tingly-box. + +A plugin registers a live instance, keeps it alive with a heartbeat, and +deregisters on shutdown — so it appears in tb only while it runs. Nothing is +persisted; if the plugin dies, tb's lease expires and routing falls back (tier +failover) to a real model. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass +from typing import Optional + +import httpx + +from .. import config as _config +from ..errors import AuthError, GatewayUnreachableError + + +@dataclass +class Lease: + gateway_url: str + admin_token: str + plugin_id: str + lease_id: str + model_id: str + scenario: Optional[str] + rule_uuid: Optional[str] + ttl_seconds: int + + +def register( + name: str, + endpoint: str, + model_id: str, + *, + scenario: Optional[str] = None, + token: str = "", + tier: int = 0, + ttl_seconds: int = 30, + gateway_url: Optional[str] = None, + admin_token: Optional[str] = None, + timeout: float = 30.0, +) -> Lease: + """Register a live ephemeral plugin instance; returns a renewable lease.""" + resolved = _config.resolve(base_url=gateway_url, token=admin_token) + headers = {"Authorization": f"Bearer {resolved.token or ''}"} + url = resolved.base_url.rstrip("/") + "/api/v2/plugins/register" + body = { + "name": name, "endpoint": endpoint, "model_id": model_id, + "scenario": scenario or "", "token": token, "tier": tier, + "ttl_seconds": ttl_seconds, + } + try: + resp = httpx.post(url, json=body, headers=headers, timeout=timeout) + except httpx.HTTPError as exc: + raise GatewayUnreachableError(f"could not reach tingly-box: {exc}") from exc + if resp.status_code == 401: + raise AuthError("tingly-box rejected the admin token during plugin register") + payload = _json(resp) or {} + if resp.status_code != 200 or not payload.get("success"): + raise GatewayUnreachableError( + f"plugin register failed: HTTP {resp.status_code} {resp.text[:200]}" + ) + d = payload.get("data") or {} + return Lease( + gateway_url=resolved.base_url.rstrip("/"), + admin_token=resolved.token or "", + plugin_id=d.get("plugin_id", ""), + lease_id=d.get("lease_id", ""), + model_id=d.get("model_id", model_id), + scenario=d.get("scenario") or None, + rule_uuid=d.get("rule_uuid") or None, + ttl_seconds=int(d.get("ttl_seconds", ttl_seconds)), + ) + + +def heartbeat(lease: Lease, timeout: float = 10.0) -> bool: + """Extend the lease. Returns False if tb no longer knows it (re-register).""" + url = lease.gateway_url + "/api/v2/plugins/heartbeat" + headers = {"Authorization": f"Bearer {lease.admin_token}"} + try: + resp = httpx.post( + url, json={"lease_id": lease.lease_id, "ttl_seconds": lease.ttl_seconds}, + headers=headers, timeout=timeout, + ) + except httpx.HTTPError: + return False + return resp.status_code == 200 + + +def deregister(lease: Lease, timeout: float = 10.0) -> None: + """Remove the live instance now (best-effort).""" + url = lease.gateway_url + "/api/v2/plugins/deregister" + headers = {"Authorization": f"Bearer {lease.admin_token}"} + try: + httpx.post(url, json={"lease_id": lease.lease_id}, headers=headers, timeout=timeout) + except httpx.HTTPError: + pass + + +class Heartbeater: + """Background thread that renews a lease until stopped.""" + + def __init__(self, lease: Lease, interval: Optional[float] = None): + self._lease = lease + # Renew well within the TTL (default: a third of it, min 1s). + self._interval = interval or max(1.0, lease.ttl_seconds / 3.0) + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + + def start(self) -> "Heartbeater": + self._thread = threading.Thread(target=self._loop, daemon=True) + self._thread.start() + return self + + def _loop(self) -> None: + while not self._stop.wait(self._interval): + heartbeat(self._lease) + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + + +def _json(resp: httpx.Response): + try: + return resp.json() + except ValueError: + return None From b9d573009b3df4ab4b91c41869731ae68e99a294 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 12:23:04 +0000 Subject: [PATCH 11/28] refactor: simplify plugin SDK + gateway changes (no behavior change) - Delete dead discover_and_connect() (client.connect() already does it). - Extract shared safe_json into tingly/_http.py (was duplicated 3x across discovery/register/runtime). - Go: extract buildPluginProvider() factory shared by the persistent handler and the registry's live resolution (was duplicated field-for-field). - config.GetProviderByUUID: call the ephemeral resolver outside the RLock (capture the pointer, release, then resolve) so the hot path no longer holds the config lock across the registry's own lock. - validateRuleServices: apply the ephemeral fallback to smart-routing services too, matching regular services (a live plugin is a valid target there as well). Go + 33 py tests pass. --- internal/server/config/config.go | 6 +++++ internal/server/config/provider.go | 14 ++++++----- internal/server/plugin_provider.go | 32 +++++++++++++++---------- internal/server/plugin_registry.go | 14 +---------- sdk/python/tingly/_http.py | 15 ++++++++++++ sdk/python/tingly/discovery.py | 35 ++-------------------------- sdk/python/tingly/plugin/register.py | 10 ++------ sdk/python/tingly/plugin/runtime.py | 10 ++------ 8 files changed, 56 insertions(+), 80 deletions(-) create mode 100644 sdk/python/tingly/_http.py diff --git a/internal/server/config/config.go b/internal/server/config/config.go index 40637d6ce..dd8905857 100644 --- a/internal/server/config/config.go +++ b/internal/server/config/config.go @@ -2566,6 +2566,12 @@ func (c *Config) validateRuleServices(rule typ.Rule) error { provider, err := c.providerStore.GetByUUID(svc.Provider) if err != nil { + // A live plugin instance is a valid (ephemeral) provider target. + if c.ephemeralResolver != nil { + if _, ok := c.ephemeralResolver.Resolve(svc.Provider); ok { + continue + } + } return fmt.Errorf("smart routing service references non-existent provider '%s': %w", svc.Provider, err) } if provider == nil { diff --git a/internal/server/config/provider.go b/internal/server/config/provider.go index 48bd5537d..c924ff4ed 100644 --- a/internal/server/config/provider.go +++ b/internal/server/config/provider.go @@ -121,18 +121,20 @@ func (c *Config) AddProviderByName(name, apiBase, token string) error { // GetProviderByUUID returns a provider from database func (c *Config) GetProviderByUUID(uuid string) (*typ.Provider, error) { c.mu.RLock() - defer c.mu.RUnlock() - if c.providerStore == nil { + c.mu.RUnlock() return nil, fmt.Errorf("provider store not initialized") } - provider, err := c.providerStore.GetByUUID(uuid) + resolver := c.ephemeralResolver + c.mu.RUnlock() + if err != nil { // Fall back to the ephemeral resolver (live plugin instances that are not - // persisted). A miss here means the provider is truly unavailable. - if c.ephemeralResolver != nil { - if p, ok := c.ephemeralResolver.Resolve(uuid); ok { + // persisted). Called outside the config lock to avoid holding it across + // the registry's own lock. A miss means the provider is truly unavailable. + if resolver != nil { + if p, ok := resolver.Resolve(uuid); ok { return p, nil } } diff --git a/internal/server/plugin_provider.go b/internal/server/plugin_provider.go index ace17d9e6..bd02b329f 100644 --- a/internal/server/plugin_provider.go +++ b/internal/server/plugin_provider.go @@ -69,18 +69,7 @@ func (s *Server) RegisterPlugin(c *gin.Context) { modelID = "plugin/" + req.Name } - provider := &typ.Provider{ - UUID: config.GenerateUUID(), - Name: req.Name, - APIBase: req.Endpoint, - APIStyle: "openai", - Token: req.Token, - NoKeyRequired: req.Token == "", - Enabled: true, - AuthType: typ.AuthTypeAPIKey, - Timeout: constant.DefaultRequestTimeout, - PluginDetail: &typ.PluginDetail{ModelID: modelID}, - } + provider := buildPluginProvider(config.GenerateUUID(), req.Name, req.Endpoint, modelID, req.Token) if err := s.config.AddProvider(provider); err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "success": false, @@ -290,3 +279,22 @@ func (s *Server) ensurePluginRule(scenario, modelID, providerID, name string, ti type pluginBindError struct{ msg string } func (e *pluginBindError) Error() string { return e.msg } + +// buildPluginProvider constructs the plugin-kind provider shared by persistent +// registration and live (ephemeral) resolution. A plugin is an ordinary OpenAI +// HTTP upstream (api_key / no_key) plus the PluginDetail marker — routing is +// unchanged. +func buildPluginProvider(uuid, name, endpoint, modelID, token string) *typ.Provider { + return &typ.Provider{ + UUID: uuid, + Name: name, + APIBase: endpoint, + APIStyle: "openai", + Token: token, + NoKeyRequired: token == "", + Enabled: true, + AuthType: typ.AuthTypeAPIKey, + Timeout: constant.DefaultRequestTimeout, + PluginDetail: &typ.PluginDetail{ModelID: modelID}, + } +} diff --git a/internal/server/plugin_registry.go b/internal/server/plugin_registry.go index c8e69caec..95a61c03a 100644 --- a/internal/server/plugin_registry.go +++ b/internal/server/plugin_registry.go @@ -6,7 +6,6 @@ import ( "github.com/google/uuid" - "github.com/tingly-dev/tingly-box/internal/constant" "github.com/tingly-dev/tingly-box/internal/typ" ) @@ -126,18 +125,7 @@ func (r *PluginRegistry) Resolve(id string) (*typ.Provider, bool) { delete(r.byID, id) return nil, false } - return &typ.Provider{ - UUID: reg.ID, - Name: reg.Name, - APIBase: reg.Endpoint, - APIStyle: "openai", - Token: reg.Token, - NoKeyRequired: reg.Token == "", - Enabled: true, - AuthType: typ.AuthTypeAPIKey, - Timeout: constant.DefaultRequestTimeout, - PluginDetail: &typ.PluginDetail{ModelID: reg.ModelID}, - }, true + return buildPluginProvider(reg.ID, reg.Name, reg.Endpoint, reg.ModelID, reg.Token), true } // List returns the currently-live registrations (expired ones are reaped). diff --git a/sdk/python/tingly/_http.py b/sdk/python/tingly/_http.py new file mode 100644 index 000000000..bec761d55 --- /dev/null +++ b/sdk/python/tingly/_http.py @@ -0,0 +1,15 @@ +"""Small shared HTTP helpers for the tingly SDK.""" + +from __future__ import annotations + +from typing import Optional + +import httpx + + +def safe_json(resp: httpx.Response) -> Optional[dict]: + """Parse a JSON body, returning None instead of raising on malformed input.""" + try: + return resp.json() + except ValueError: + return None diff --git a/sdk/python/tingly/discovery.py b/sdk/python/tingly/discovery.py index ba4c8d824..48181dfc1 100644 --- a/sdk/python/tingly/discovery.py +++ b/sdk/python/tingly/discovery.py @@ -12,7 +12,7 @@ import httpx -from . import config as _config +from ._http import safe_json from .errors import ( AuthError, GatewayUnreachableError, @@ -74,7 +74,7 @@ def create_session( "`tingly doctor --link`." ) - payload = _safe_json(resp) + payload = safe_json(resp) if resp.status_code == 404: raise ScenarioNotFoundError( scenario, (payload or {}).get("valid_scenarios") @@ -95,34 +95,3 @@ def create_session( ) -def discover_and_connect( - scenario: str, - base_url: Optional[str] = None, - token: Optional[str] = None, - name: Optional[str] = None, - timeout: float = 30.0, -) -> Session: - """Resolve config, verify reachability, and mint a session.""" - resolved = _config.resolve(base_url=base_url, token=token) - - if probe_version(resolved.base_url) is None: - raise GatewayUnreachableError( - f"no tingly-box gateway responding at {resolved.base_url} " - f"(resolved via {resolved.source}). Is `tb` running? " - f"Set TINGLY_BOX_URL or run `tingly doctor`." - ) - - return create_session( - base_url=resolved.base_url, - admin_token=resolved.token or "", - scenario=scenario, - name=name, - timeout=timeout, - ) - - -def _safe_json(resp: httpx.Response) -> Optional[dict]: - try: - return resp.json() - except ValueError: - return None diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py index a3a1d1dc8..43011e402 100644 --- a/sdk/python/tingly/plugin/register.py +++ b/sdk/python/tingly/plugin/register.py @@ -14,6 +14,7 @@ import httpx from .. import config as _config +from .._http import safe_json from ..errors import AuthError, GatewayUnreachableError @@ -77,7 +78,7 @@ def register_with_tb( if resp.status_code == 401: raise AuthError("tingly-box rejected the admin token while registering the plugin") - payload_data = _safe_json(resp) or {} + payload_data = safe_json(resp) or {} if resp.status_code not in (200, 201) or not payload_data.get("success"): raise GatewayUnreachableError( f"plugin registration failed: HTTP {resp.status_code} {resp.text[:200]}" @@ -94,10 +95,3 @@ def register_with_tb( ready=bool(data.get("ready", False)), note=data.get("note", ""), ) - - -def _safe_json(resp: httpx.Response): - try: - return resp.json() - except ValueError: - return None diff --git a/sdk/python/tingly/plugin/runtime.py b/sdk/python/tingly/plugin/runtime.py index 5eca9b8fe..7b7c90462 100644 --- a/sdk/python/tingly/plugin/runtime.py +++ b/sdk/python/tingly/plugin/runtime.py @@ -15,6 +15,7 @@ import httpx from .. import config as _config +from .._http import safe_json from ..errors import AuthError, GatewayUnreachableError @@ -58,7 +59,7 @@ def register( raise GatewayUnreachableError(f"could not reach tingly-box: {exc}") from exc if resp.status_code == 401: raise AuthError("tingly-box rejected the admin token during plugin register") - payload = _json(resp) or {} + payload = safe_json(resp) or {} if resp.status_code != 200 or not payload.get("success"): raise GatewayUnreachableError( f"plugin register failed: HTTP {resp.status_code} {resp.text[:200]}" @@ -123,10 +124,3 @@ def stop(self) -> None: self._stop.set() if self._thread is not None: self._thread.join(timeout=2.0) - - -def _json(resp: httpx.Response): - try: - return resp.json() - except ValueError: - return None From 86e52fb6855b14624e5ae0e4b53c22cf6f1c7f45 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 16:35:29 +0000 Subject: [PATCH 12/28] refactor(plugins): collapse to ephemeral-only; drop persistent path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go plugin code carried two parallel registration mechanisms (persistent provider+rule AND ephemeral registry). Since the design is "plugins are dynamic/ephemeral, not固化", remove the persistent half — the bigger, more complex one: - Delete the persistent RegisterPlugin handler + POST /api/v2/plugins route; the ephemeral register is now the only one (renamed to /plugins/register → RegisterPlugin). - Remove the plugin_detail DB column and its marshal/unmarshal across all three provider_store paths (toProvider/toRecord/updateRecordFromProvider). The store is plugin-agnostic again. PluginDetail/IsPlugin survive as an in-memory marker the registry sets on the synthesized provider; drop the unused Managed field. - ListPlugins lists live registry instances only (no persistent branch / seen map / Managed); PluginInfo trimmed to {uuid,name,endpoint,model_id}. - Move provider synthesis inline into registry.Resolve (its only caller). - SDK: remove plugin/register.py + the `tingly plugin register` command — a one-shot register has no heartbeat and would expire; `tingly plugin run` already serves + self-registers + heartbeats. Docs/examples/openapi updated. Go + 31 py tests pass. --- .design/python-sdk.md | 42 ++---- ai/provider.go | 20 +-- internal/data/db/provider_store.go | 28 ---- internal/server/plugin_dynamic_test.go | 31 +++- internal/server/plugin_provider.go | 180 ++++------------------- internal/server/plugin_provider_test.go | 108 +------------- internal/server/plugin_registry.go | 16 +- internal/server/server_webui_api.go | 22 +-- sdk/python/README.md | 8 +- sdk/python/examples/rag_plugin.py | 17 ++- sdk/python/tests/test_plugin_register.py | 71 --------- sdk/python/tingly/cli.py | 32 +--- sdk/python/tingly/plugin/register.py | 97 ------------ 13 files changed, 116 insertions(+), 556 deletions(-) delete mode 100644 sdk/python/tests/test_plugin_register.py delete mode 100644 sdk/python/tingly/plugin/register.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 92cc74a41..4d1b87b8c 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -57,29 +57,17 @@ today's pieces — three verbs for the one rule⇄plugin relationship: The historical "Layer 1/2/3" headings below map exactly to connect / serve / register. They are an implementation tour, not three separate products. -### tb-side: a first-class "plugin" provider kind (implemented) - -"Register" is now **one step**: `POST /api/v2/plugins` creates a plugin-kind -provider *and* (when a scenario is given) the rule whose upstream is that plugin. - -- **`ai.PluginDetail`** + `Provider.PluginDetail` + `Provider.IsPlugin()` mark a - provider as backed by plugin code. It is **distinct from `AuthTypeVirtual`** - (in-process `vmodel`): a plugin is an ordinary OpenAI HTTP upstream - (`APIStyle=openai`, `api_key`/`no_key`), so **routing is unchanged** — the - marker is metadata for UI grouping + lifecycle discovery only. Persisted via a - new `plugin_detail` column, reconstructed unconditionally (independent of auth - type), AutoMigrate-created. -- **`POST /api/v2/plugins`** `{name, endpoint, model_id?, token?, scenario?, tier?}` - → creates the provider; if `scenario` is bindable, also creates a rule - (`RequestModel=model_id`, single tier service → the plugin). Returns - `{provider_uuid, model_id, scenario, rule_uuid, ready}`. -- **`GET /api/v2/plugins`** lists plugin-kind providers for the UI's plugin - section. -- SDK: `register_with_tb(..., scenario=…)` and `tingly plugin register - --scenario experiment` do the full one-step wire-in. - -This makes "configure this rule with a plugin" literal and eliminates the -provider+rule two-step mode-picker (`ux-principles.md`). +### tb-side: plugins are ephemeral, not a persisted provider kind + +An earlier iteration persisted a plugin as a provider row (a `plugin_detail` +column + a `POST /api/v2/plugins` "pin" path). That was removed: plugins are +**dynamic/ephemeral** (see below), so there is no persisted plugin provider, no +DB column, and no separate persistent endpoint. `ai.PluginDetail` + +`Provider.IsPlugin()` survive only as an **in-memory marker** the registry sets +on the synthesized provider (distinct from `AuthTypeVirtual`, the in-process +`vmodel` path). A plugin is otherwise an ordinary OpenAI HTTP upstream, so +**routing is unchanged**. "Pin" (durable opt-in) can return later as a flag if a +real need appears. ### Plugin as runtime service (dynamic registration, implemented) @@ -97,8 +85,8 @@ nothing persisted. Differs from a standard provider (durable, operator-managed). - **DNS-style layering**: the rule (the durable "name") is ensured idempotently; the instance (endpoint + liveness) is ephemeral. No live instance ⇒ failover. - Endpoints (apiV2): `POST /plugins/register` (leased; ensures rule), - `POST /plugins/heartbeat`, `POST /plugins/deregister`, `GET /plugins` (live + - pinned). The persistent `POST /api/v2/plugins` remains as the **pin** path. + `POST /plugins/heartbeat`, `POST /plugins/deregister`, `GET /plugins` (live + instances). **Active configuration** (SDK): `tingly.configure(url=, admin_token_env=)` / `Connection` inject the tb target + credentials at runtime (secrets by env @@ -129,8 +117,8 @@ sdk/python/ server.py # stdlib OpenAI-compatible HTTP server (+ SSE) types.py # ChatRequest / Message manifest.py # tingly.toml read/write - register.py # register the plugin as a tb provider (Layer 3) - cli.py # `tingly doctor` + `tingly plugin {init,run,register}` + runtime.py # dynamic register / heartbeat / deregister + cli.py # `tingly doctor` + `tingly plugin {init,run}` errors.py # TinglyError hierarchy ``` diff --git a/ai/provider.go b/ai/provider.go index c9ceaccf0..161cd1aed 100644 --- a/ai/provider.go +++ b/ai/provider.go @@ -53,23 +53,17 @@ type VModelDetail struct { LatencyProfile string `json:"latency_profile,omitempty"` } -// PluginDetail marks a provider as backed by external plugin code. A plugin -// provider is otherwise an ordinary OpenAI HTTP upstream (APIStyle=openai, -// APIBase=/v1, api_key / no_key auth) — there is NO routing change, the -// dispatcher treats it exactly like any other provider. This marker exists so -// plugins are a first-class concept (grouped in the UI, discoverable by future -// lifecycle tooling) and so "configure this rule with a plugin" is one step. +// PluginDetail marks a provider as backed by external plugin code. It is an +// in-memory marker only — plugins register dynamically and are never persisted, +// so the synthesized provider carries this to identify plugin traffic. A plugin +// is otherwise an ordinary OpenAI HTTP upstream (APIStyle=openai, api_key / +// no_key), so there is NO routing change. // -// Note: this is distinct from VModelDetail / AuthTypeVirtual, which is the -// in-process synthetic-model path. A plugin runs out-of-process and is reached -// over HTTP. +// Distinct from VModelDetail / AuthTypeVirtual (the in-process synthetic-model +// path): a plugin runs out-of-process and is reached over HTTP. type PluginDetail struct { // ModelID is the model id the plugin advertises (e.g. "plugin/my-rag"). ModelID string `json:"model_id,omitempty"` - // Managed reports whether tingly-box supervises the plugin process. - // Reserved for the tb-side supervisor; false means the plugin is run - // externally and tb only routes to it. - Managed bool `json:"managed,omitempty"` } // CredentialBundle holds the credential fields for multi-field auth types diff --git a/internal/data/db/provider_store.go b/internal/data/db/provider_store.go index a601bbb73..959796692 100644 --- a/internal/data/db/provider_store.go +++ b/internal/data/db/provider_store.go @@ -59,12 +59,6 @@ type ProviderRecord struct { // VModel-specific fields (only populated when AuthType == "vmodel") VModelDetail string `gorm:"column:vmodel_detail;type:text"` // JSON-encoded typ.VModelDetail - // PluginDetail marks a provider as backed by external plugin code. Unlike - // VModelDetail it is independent of AuthType (a plugin uses api_key auth), so - // it is persisted/loaded unconditionally. JSON-encoded typ.PluginDetail; empty - // for non-plugin providers. Added additively; AutoMigrate creates the column. - PluginDetail string `gorm:"column:plugin_detail;type:text"` - // Credential holds multi-field credentials for non-bearer auth types // (aws_sigv4, azure_key, gcp_sa). JSON-encoded typ.CredentialBundle. // Empty for api_key/oauth/vmodel. Added additively; AutoMigrate creates @@ -136,14 +130,6 @@ func (r *ProviderRecord) toProvider() *typ.Provider { provider.AuthType = typ.AuthTypeAPIKey } - // PluginDetail is independent of auth type — reconstruct it for any provider. - if r.PluginDetail != "" { - var detail typ.PluginDetail - if err := json.Unmarshal([]byte(r.PluginDetail), &detail); err == nil { - provider.PluginDetail = &detail - } - } - return provider } @@ -183,12 +169,6 @@ func toRecord(p *typ.Provider) *ProviderRecord { record.Tags = string(tagsJSON) } - // PluginDetail is independent of auth type. - if p.PluginDetail != nil { - pdJSON, _ := json.Marshal(p.PluginDetail) - record.PluginDetail = string(pdJSON) - } - // Set credentials based on auth type switch p.AuthType { case typ.AuthTypeOAuth: @@ -242,14 +222,6 @@ func updateRecordFromProvider(record *ProviderRecord, p *typ.Provider) { record.Tags = "" } - // PluginDetail is independent of auth type; set or clear unconditionally. - if p.PluginDetail != nil { - pdJSON, _ := json.Marshal(p.PluginDetail) - record.PluginDetail = string(pdJSON) - } else { - record.PluginDetail = "" - } - // Set credentials based on auth type switch p.AuthType { case typ.AuthTypeOAuth: diff --git a/internal/server/plugin_dynamic_test.go b/internal/server/plugin_dynamic_test.go index ea8aae741..469941e4c 100644 --- a/internal/server/plugin_dynamic_test.go +++ b/internal/server/plugin_dynamic_test.go @@ -1,8 +1,13 @@ package server import ( + "encoding/json" + "net/http" + "net/http/httptest" "testing" + "github.com/gin-gonic/gin" + "github.com/tingly-dev/tingly-box/internal/server/config" "github.com/tingly-dev/tingly-box/internal/typ" ) @@ -18,10 +23,10 @@ func newDynamicPluginServer(t *testing.T) *Server { return &Server{config: cfg, pluginRegistry: reg} } -func TestRegisterPluginDynamic_RoutesAndExpires(t *testing.T) { +func TestRegisterPlugin_RoutesAndExpires(t *testing.T) { s := newDynamicPluginServer(t) - w, resp := postJSON(t, s.RegisterPluginDynamic, RegisterPluginDynamicRequest{ + w, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ Name: "my-rag", Endpoint: "http://127.0.0.1:8765/v1", ModelID: "plugin/my-rag", @@ -80,12 +85,30 @@ func TestHeartbeatPlugin_UnknownLease(t *testing.T) { } } +func TestListPlugins_ShowsLiveInstances(t *testing.T) { + s := newDynamicPluginServer(t) + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{Name: "plug", Endpoint: "http://127.0.0.1:8765/v1"}) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + s.ListPlugins(c) + + var resp PluginsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(resp.Data) != 1 || resp.Data[0].Name != "plug" { + t.Fatalf("expected the one live plugin, got %+v", resp.Data) + } +} + func TestReRegisterIsIdempotentForRule(t *testing.T) { s := newDynamicPluginServer(t) - postJSON(t, s.RegisterPluginDynamic, RegisterPluginDynamicRequest{ + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ Name: "p", Endpoint: "http://a/v1", Scenario: string(typ.ScenarioExperiment), }) - postJSON(t, s.RegisterPluginDynamic, RegisterPluginDynamicRequest{ + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ Name: "p", Endpoint: "http://b/v1", Scenario: string(typ.ScenarioExperiment), }) // exactly one rule for plugin/p diff --git a/internal/server/plugin_provider.go b/internal/server/plugin_provider.go index bd02b329f..9a061d8df 100644 --- a/internal/server/plugin_provider.go +++ b/internal/server/plugin_provider.go @@ -7,44 +7,17 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "github.com/tingly-dev/tingly-box/internal/constant" "github.com/tingly-dev/tingly-box/internal/loadbalance" "github.com/tingly-dev/tingly-box/internal/server/config" "github.com/tingly-dev/tingly-box/internal/typ" ) -// RegisterPluginRequest registers external plugin code as a tingly-box upstream -// in one step: it creates a plugin-kind provider and, when a scenario is given, -// the rule whose upstream is that plugin. -type RegisterPluginRequest struct { - Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` - Endpoint string `json:"endpoint" binding:"required" description:"Plugin OpenAI base URL" example:"http://127.0.0.1:8765/v1"` - ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` - Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` - Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` - Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` -} - -// RegisterPluginResponse reports what was created. -type RegisterPluginResponse struct { - ProviderUUID string `json:"provider_uuid"` - ModelID string `json:"model_id"` - Scenario string `json:"scenario,omitempty"` - RuleUUID string `json:"rule_uuid,omitempty"` - // Ready is true when a rule was bound, so clients can select the model now. - Ready bool `json:"ready"` - Note string `json:"note,omitempty"` -} - -// PluginInfo is a list view of a plugin-kind provider. +// PluginInfo is a list view of a live plugin instance. type PluginInfo struct { - UUID string `json:"uuid"` - Name string `json:"name"` - Endpoint string `json:"endpoint"` - ModelID string `json:"model_id"` - Managed bool `json:"managed"` - Enabled bool `json:"enabled"` - Ephemeral bool `json:"ephemeral"` // true for live dynamic registrations + UUID string `json:"uuid"` + Name string `json:"name"` + Endpoint string `json:"endpoint"` + ModelID string `json:"model_id"` } // PluginsResponse wraps the plugin list. @@ -53,107 +26,23 @@ type PluginsResponse struct { Data []PluginInfo `json:"data"` } -// RegisterPlugin creates a plugin-kind provider (and optionally binds a rule to -// it) so "configure this rule with a plugin" is a single call. A plugin -// provider is an ordinary OpenAI HTTP upstream — routing is unchanged; the -// PluginDetail marker makes it a first-class concept for the UI and lifecycle. -func (s *Server) RegisterPlugin(c *gin.Context) { - var req RegisterPluginRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) - return - } - - modelID := req.ModelID - if modelID == "" { - modelID = "plugin/" + req.Name - } - - provider := buildPluginProvider(config.GenerateUUID(), req.Name, req.Endpoint, modelID, req.Token) - if err := s.config.AddProvider(provider); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "success": false, - "error": "failed to create plugin provider: " + err.Error(), - }) - return - } - - resp := RegisterPluginResponse{ - ProviderUUID: provider.UUID, - ModelID: modelID, - Note: "Provider created. Bind a rule (pass `scenario`) to make the model selectable.", - } - - // One-step bind: create the rule whose single service is this plugin. - if req.Scenario != "" { - ruleUUID, err := s.ensurePluginRule(req.Scenario, modelID, provider.UUID, req.Name, req.Tier) - if err != nil { - resp.Note = "Provider created, but rule binding failed: " + err.Error() - c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) - return - } - resp.Scenario = req.Scenario - resp.RuleUUID = ruleUUID - resp.Ready = true - resp.Note = "Plugin wired in. Select model " + modelID + " under scenario " + req.Scenario + "." - } - - logrus.WithFields(logrus.Fields{ - "plugin": req.Name, - "endpoint": req.Endpoint, - "model_id": modelID, - "scenario": req.Scenario, - "ready": resp.Ready, - }).Info("Registered plugin provider") - - c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) -} - -// ListPlugins returns plugin providers: live ephemeral instances from the -// registry plus any pinned/persistent plugin-kind providers. +// ListPlugins returns the live (dynamically-registered) plugin instances. func (s *Server) ListPlugins(c *gin.Context) { - var plugins []PluginInfo - seen := map[string]bool{} - - // Live, dynamically-registered instances first. - if s.pluginRegistry != nil { - for _, reg := range s.pluginRegistry.List() { - seen[reg.ID] = true - plugins = append(plugins, PluginInfo{ - UUID: reg.ID, - Name: reg.Name, - Endpoint: reg.Endpoint, - ModelID: reg.ModelID, - Enabled: true, - Ephemeral: true, - }) - } - } - - // Pinned / persistent plugin providers. - for _, p := range s.config.ListProviders() { - if !p.IsPlugin() || seen[p.UUID] { - continue - } - modelID := "" - if p.PluginDetail != nil { - modelID = p.PluginDetail.ModelID - } - managed := p.PluginDetail != nil && p.PluginDetail.Managed + plugins := []PluginInfo{} + for _, reg := range s.pluginRegistry.List() { plugins = append(plugins, PluginInfo{ - UUID: p.UUID, - Name: p.Name, - Endpoint: p.APIBase, - ModelID: modelID, - Managed: managed, - Enabled: p.Enabled, + UUID: reg.ID, + Name: reg.Name, + Endpoint: reg.Endpoint, + ModelID: reg.ModelID, }) } c.JSON(http.StatusOK, PluginsResponse{Success: true, Data: plugins}) } -// RegisterPluginDynamicRequest registers a live, ephemeral plugin instance. -type RegisterPluginDynamicRequest struct { +// RegisterPluginRequest registers a live, ephemeral plugin instance and, when a +// scenario is given, ensures the durable rule whose upstream is that plugin. +type RegisterPluginRequest struct { Name string `json:"name" binding:"required" example:"my-rag"` Endpoint string `json:"endpoint" binding:"required" example:"http://127.0.0.1:8765/v1"` ModelID string `json:"model_id,omitempty" example:"plugin/my-rag"` @@ -163,8 +52,8 @@ type RegisterPluginDynamicRequest struct { TTLSeconds int `json:"ttl_seconds,omitempty" example:"30"` } -// RegisterPluginDynamicResponse reports the lease for an ephemeral registration. -type RegisterPluginDynamicResponse struct { +// RegisterPluginResponse reports the lease for an ephemeral registration. +type RegisterPluginResponse struct { PluginID string `json:"plugin_id"` LeaseID string `json:"lease_id"` ModelID string `json:"model_id"` @@ -174,12 +63,12 @@ type RegisterPluginDynamicResponse struct { Note string `json:"note,omitempty"` } -// RegisterPluginDynamic registers a live plugin instance in the in-memory -// registry (NOT persisted). The plugin keeps it alive by heartbeating; it is -// auto-removed when the lease expires or the plugin deregisters. When a scenario -// is given, the durable rule (the stable "name") is ensured idempotently. -func (s *Server) RegisterPluginDynamic(c *gin.Context) { - var req RegisterPluginDynamicRequest +// RegisterPlugin registers a live plugin instance in the in-memory registry (NOT +// persisted). The plugin keeps it alive by heartbeating; it is auto-removed when +// the lease expires or the plugin deregisters. When a scenario is given, the +// durable rule (the stable "name") is ensured idempotently. +func (s *Server) RegisterPlugin(c *gin.Context) { + var req RegisterPluginRequest if err := c.ShouldBindJSON(&req); err != nil { c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) return @@ -187,7 +76,7 @@ func (s *Server) RegisterPluginDynamic(c *gin.Context) { ttl := time.Duration(req.TTLSeconds) * time.Second reg := s.pluginRegistry.Register(req.Name, req.Endpoint, req.ModelID, req.Scenario, req.Token, ttl) - resp := RegisterPluginDynamicResponse{ + resp := RegisterPluginResponse{ PluginID: reg.ID, LeaseID: reg.LeaseID, ModelID: reg.ModelID, @@ -211,7 +100,7 @@ func (s *Server) RegisterPluginDynamic(c *gin.Context) { logrus.WithFields(logrus.Fields{ "plugin": req.Name, "endpoint": req.Endpoint, "model_id": reg.ModelID, "scenario": req.Scenario, "ttl_s": resp.TTLSeconds, - }).Info("Registered dynamic plugin instance") + }).Info("Registered plugin instance") c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } @@ -279,22 +168,3 @@ func (s *Server) ensurePluginRule(scenario, modelID, providerID, name string, ti type pluginBindError struct{ msg string } func (e *pluginBindError) Error() string { return e.msg } - -// buildPluginProvider constructs the plugin-kind provider shared by persistent -// registration and live (ephemeral) resolution. A plugin is an ordinary OpenAI -// HTTP upstream (api_key / no_key) plus the PluginDetail marker — routing is -// unchanged. -func buildPluginProvider(uuid, name, endpoint, modelID, token string) *typ.Provider { - return &typ.Provider{ - UUID: uuid, - Name: name, - APIBase: endpoint, - APIStyle: "openai", - Token: token, - NoKeyRequired: token == "", - Enabled: true, - AuthType: typ.AuthTypeAPIKey, - Timeout: constant.DefaultRequestTimeout, - PluginDetail: &typ.PluginDetail{ModelID: modelID}, - } -} diff --git a/internal/server/plugin_provider_test.go b/internal/server/plugin_provider_test.go index 212d2474f..070cbd017 100644 --- a/internal/server/plugin_provider_test.go +++ b/internal/server/plugin_provider_test.go @@ -8,20 +8,10 @@ import ( "testing" "github.com/gin-gonic/gin" - - "github.com/tingly-dev/tingly-box/internal/server/config" - "github.com/tingly-dev/tingly-box/internal/typ" ) -func newPluginTestServer(t *testing.T) *Server { - t.Helper() - cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) - if err != nil { - t.Fatalf("NewConfig: %v", err) - } - return &Server{config: cfg} -} - +// postJSON drives a gin handler with a JSON body and returns the recorder and +// the parsed response envelope. Shared by the plugin endpoint tests. func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseRecorder, map[string]any) { t.Helper() gin.SetMode(gin.TestMode) @@ -35,97 +25,3 @@ func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseReco _ = json.Unmarshal(w.Body.Bytes(), &parsed) return w, parsed } - -func TestRegisterPlugin_BindsRule(t *testing.T) { - s := newPluginTestServer(t) - - w, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ - Name: "my-rag", - Endpoint: "http://127.0.0.1:8765/v1", - ModelID: "plugin/my-rag", - Scenario: string(typ.ScenarioExperiment), - }) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, body=%s", w.Code, w.Body.String()) - } - data, _ := resp["data"].(map[string]any) - if data["ready"] != true { - t.Fatalf("expected ready=true, got %v (note=%v)", data["ready"], data["note"]) - } - if data["model_id"] != "plugin/my-rag" { - t.Fatalf("model_id = %v", data["model_id"]) - } - providerUUID, _ := data["provider_uuid"].(string) - if providerUUID == "" { - t.Fatalf("expected a provider_uuid") - } - - // The provider must be a plugin-kind provider. - prov, err := s.config.GetProviderByUUID(providerUUID) - if err != nil { - t.Fatalf("GetProviderByUUID: %v", err) - } - if !prov.IsPlugin() { - t.Fatalf("provider is not marked as plugin: %+v", prov) - } - if prov.PluginDetail.ModelID != "plugin/my-rag" { - t.Fatalf("plugin model id = %q", prov.PluginDetail.ModelID) - } - - // A rule must exist under the scenario whose single service is the plugin. - var found bool - for _, rule := range s.config.GetRequestConfigs() { - if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { - found = true - if len(rule.Services) != 1 || rule.Services[0].Provider != providerUUID { - t.Fatalf("rule service does not point at plugin provider: %+v", rule.Services) - } - } - } - if !found { - t.Fatalf("no rule bound for the plugin under experiment scenario") - } -} - -func TestRegisterPlugin_ProviderOnly(t *testing.T) { - s := newPluginTestServer(t) - - _, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ - Name: "solo", - Endpoint: "http://127.0.0.1:9000/v1", - }) - data, _ := resp["data"].(map[string]any) - if data["ready"] == true { - t.Fatalf("expected ready=false when no scenario given") - } - // model id defaults to plugin/ - if data["model_id"] != "plugin/solo" { - t.Fatalf("model_id default = %v", data["model_id"]) - } -} - -func TestListPlugins_FiltersPluginKind(t *testing.T) { - s := newPluginTestServer(t) - // a normal provider - if err := s.config.AddProvider(&typ.Provider{ - Name: "real", APIBase: "https://api.example.com/v1", APIStyle: "openai", Enabled: true, - }); err != nil { - t.Fatalf("AddProvider: %v", err) - } - // a plugin provider via the handler - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{Name: "plug", Endpoint: "http://127.0.0.1:8765/v1"}) - - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodGet, "/", nil) - s.ListPlugins(c) - - var resp PluginsResponse - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if len(resp.Data) != 1 || resp.Data[0].Name != "plug" { - t.Fatalf("expected exactly the plugin provider, got %+v", resp.Data) - } -} diff --git a/internal/server/plugin_registry.go b/internal/server/plugin_registry.go index 95a61c03a..698f51930 100644 --- a/internal/server/plugin_registry.go +++ b/internal/server/plugin_registry.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" + "github.com/tingly-dev/tingly-box/internal/constant" "github.com/tingly-dev/tingly-box/internal/typ" ) @@ -125,7 +126,20 @@ func (r *PluginRegistry) Resolve(id string) (*typ.Provider, bool) { delete(r.byID, id) return nil, false } - return buildPluginProvider(reg.ID, reg.Name, reg.Endpoint, reg.ModelID, reg.Token), true + // A plugin is an ordinary OpenAI HTTP upstream plus the PluginDetail marker; + // routing treats it like any other provider. + return &typ.Provider{ + UUID: reg.ID, + Name: reg.Name, + APIBase: reg.Endpoint, + APIStyle: "openai", + Token: reg.Token, + NoKeyRequired: reg.Token == "", + Enabled: true, + AuthType: typ.AuthTypeAPIKey, + Timeout: constant.DefaultRequestTimeout, + PluginDetail: &typ.PluginDetail{ModelID: reg.ModelID}, + }, true } // List returns the currently-live registrations (expired ones are reaped). diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index 7dacc92c1..0aa51679b 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -400,28 +400,20 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { providerHandler := providermodule.NewHandler(s.config, s.quotaManager) providermodule.RegisterRoutes(apiV2, providerHandler) - // Plugin registration: a plugin is a first-class provider kind (external - // OpenAI upstream). POST wires it in (provider + optional rule) in one step. - api.POST("/plugins", s.RegisterPlugin, - swagger.WithDescription("Register external plugin code as an upstream (and optionally bind a rule)"), - swagger.WithTags("plugins"), - swagger.WithRequestModel(RegisterPluginRequest{}), - swagger.WithResponseModel(RegisterPluginResponse{}), - ) - api.GET("/plugins", s.ListPlugins, - swagger.WithDescription("List registered plugin-kind providers (live + pinned)"), + swagger.WithDescription("List live (dynamically-registered) plugin instances"), swagger.WithTags("plugins"), swagger.WithResponseModel(PluginsResponse{}), ) - // Dynamic (ephemeral) plugin lifecycle: register a live instance, keep it - // alive by heartbeat, and deregister on shutdown. Nothing is persisted. - api.POST("/plugins/register", s.RegisterPluginDynamic, + // Plugin lifecycle: register a live instance, keep it alive by heartbeat, and + // deregister on shutdown. Nothing is persisted — an expired instance simply + // falls out of routing (tier failover). + api.POST("/plugins/register", s.RegisterPlugin, swagger.WithDescription("Register a live, ephemeral plugin instance (leased)"), swagger.WithTags("plugins"), - swagger.WithRequestModel(RegisterPluginDynamicRequest{}), - swagger.WithResponseModel(RegisterPluginDynamicResponse{}), + swagger.WithRequestModel(RegisterPluginRequest{}), + swagger.WithResponseModel(RegisterPluginResponse{}), ) api.POST("/plugins/heartbeat", s.HeartbeatPlugin, swagger.WithDescription("Extend a plugin instance's lease"), diff --git a/sdk/python/README.md b/sdk/python/README.md index 8805ad45d..d003ed0a2 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -76,11 +76,13 @@ if __name__ == "__main__": ```bash tingly plugin init my-rag # scaffold module + tingly.toml -tingly plugin run my_rag_plugin.py # serve -tingly plugin register my-rag \ # one step: provider + rule - --url http://127.0.0.1:8765/v1 --model-id plugin/my-rag --scenario experiment +tingly plugin run my_rag_plugin.py # serve AND register with tb (ephemeral) ``` +`serve()` (and `tingly plugin run`) **dynamically registers** the plugin with tb +while it runs — leased, heartbeated, and auto-removed on exit. Nothing is +persisted; if the plugin stops, tb's lease expires and routing falls back. + The server is stdlib-only (no FastAPI), supports streaming, and `plugin.llm` calls back into tb so the plugin reuses the gateway for its own LLM work. diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py index 6bc22d6f9..3256cfffa 100644 --- a/sdk/python/examples/rag_plugin.py +++ b/sdk/python/examples/rag_plugin.py @@ -1,15 +1,12 @@ """A RAG plugin served as an OpenAI-compatible upstream for tingly-box. -Run it: +Run it (serves on :8765 AND self-registers with tb while running): pip install -e . # from sdk/python - python examples/rag_plugin.py # serves on http://127.0.0.1:8765/v1 + python examples/rag_plugin.py -Then wire it into tb in one step (creates the provider + a rule) so any client -can select model `plugin/rag-demo`: - - tingly plugin register rag-demo --url http://127.0.0.1:8765/v1 \ - --model-id plugin/rag-demo --scenario experiment +Registration is dynamic/ephemeral: the plugin leases a spot in tb, heartbeats to +keep it, and deregisters on exit. Nothing is persisted. Now `model="plugin/rag-demo"` from Claude Code, Cursor, the tb UI, or another `tingly.connect()` experiment routes here — with tb's guard rails, quota, @@ -19,7 +16,11 @@ from tingly import Plugin -plugin = Plugin(name="rag-demo", description="Answers from a toy in-memory corpus") +plugin = Plugin( + name="rag-demo", + scenario="experiment", # bind a rule under this scenario on register + description="Answers from a toy in-memory corpus", +) CORPUS = { "tingly-box": "tingly-box is a personal intelligence orchestrator: an LLM " diff --git a/sdk/python/tests/test_plugin_register.py b/sdk/python/tests/test_plugin_register.py deleted file mode 100644 index 00e1dd9db..000000000 --- a/sdk/python/tests/test_plugin_register.py +++ /dev/null @@ -1,71 +0,0 @@ -"""register_with_tb hits the one-step /api/v2/plugins endpoint (respx mocked).""" - -import httpx -import respx - -from tingly.plugin.register import register_with_tb - -BASE = "http://tb.test:12580" - - -@respx.mock -def test_register_binds_rule(monkeypatch): - monkeypatch.setenv("TINGLY_BOX_URL", BASE) - monkeypatch.setenv("TINGLY_BOX_TOKEN", "admin") - - route = respx.post(f"{BASE}/api/v2/plugins").mock( - return_value=httpx.Response( - 200, - json={ - "success": True, - "data": { - "provider_uuid": "uuid-1", - "model_id": "plugin/my-rag", - "scenario": "experiment", - "rule_uuid": "rule-1", - "ready": True, - "note": "Plugin wired in.", - }, - }, - ) - ) - - result = register_with_tb( - "my-rag", - "http://127.0.0.1:8765/v1", - "plugin/my-rag", - scenario="experiment", - ) - - assert route.called - sent = route.calls.last.request - assert sent.headers["Authorization"] == "Bearer admin" - assert result.provider_uuid == "uuid-1" - assert result.rule_uuid == "rule-1" - assert result.ready is True - assert result.scenario == "experiment" - - -@respx.mock -def test_register_provider_only(monkeypatch): - monkeypatch.setenv("TINGLY_BOX_URL", BASE) - monkeypatch.setenv("TINGLY_BOX_TOKEN", "admin") - - respx.post(f"{BASE}/api/v2/plugins").mock( - return_value=httpx.Response( - 200, - json={ - "success": True, - "data": { - "provider_uuid": "uuid-2", - "model_id": "plugin/solo", - "ready": False, - "note": "Provider created.", - }, - }, - ) - ) - - result = register_with_tb("solo", "http://127.0.0.1:9000/v1", "plugin/solo") - assert result.ready is False - assert result.rule_uuid is None diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py index adf57a0a5..08538cbfa 100644 --- a/sdk/python/tingly/cli.py +++ b/sdk/python/tingly/cli.py @@ -160,20 +160,6 @@ def _plugin_run(target: str) -> int: return 0 -def _plugin_register(name, plugin_url, model_id, token, scenario) -> int: - from .plugin.register import register_with_tb - - result = register_with_tb( - name, plugin_url, model_id, scenario=scenario or None, token=token - ) - status = OK if result.ready else WARN - _row("plugin", f"{result.name} → {result.api_base}", status) - if result.rule_uuid: - _row("rule", f"{result.scenario}: {result.model_id}", OK) - print("\n" + result.note) - return 0 - - def main(argv: Optional[list] = None) -> int: parser = argparse.ArgumentParser(prog="tingly", description="tingly-box Python SDK") sub = parser.add_subparsers(dest="command") @@ -186,20 +172,14 @@ def main(argv: Optional[list] = None) -> int: "--link", action="store_true", help="prompt for and save gateway URL + token" ) - p_plugin = sub.add_parser("plugin", help="author / run / register a plugin") + p_plugin = sub.add_parser("plugin", help="author / run a plugin") psub = p_plugin.add_subparsers(dest="plugin_command") p_init = psub.add_parser("init", help="scaffold a starter plugin") p_init.add_argument("name", help="plugin name, e.g. my-rag") - p_run = psub.add_parser("run", help="serve a plugin (module:attr or path.py)") + # `run` serves the plugin AND self-registers with tb (heartbeat + deregister + # on exit), so there is no separate one-shot register command. + p_run = psub.add_parser("run", help="serve a plugin and register it with tb") p_run.add_argument("target", help="e.g. my_rag_plugin:plugin or my_rag_plugin.py") - p_reg = psub.add_parser("register", help="register a running plugin with tb") - p_reg.add_argument("name", help="provider name to create in tb") - p_reg.add_argument("--url", required=True, help="plugin OpenAI base, e.g. http://127.0.0.1:8765/v1") - p_reg.add_argument("--model-id", required=True, help="model id, e.g. plugin/my-rag") - p_reg.add_argument("--token", default="", help="token tb should send to the plugin") - p_reg.add_argument( - "--scenario", default="", help="bind a rule under this scenario (e.g. experiment)" - ) args = parser.parse_args(argv) if args.command == "doctor": @@ -209,10 +189,6 @@ def main(argv: Optional[list] = None) -> int: return _plugin_init(args.name) if args.plugin_command == "run": return _plugin_run(args.target) - if args.plugin_command == "register": - return _plugin_register( - args.name, args.url, args.model_id, args.token, args.scenario - ) p_plugin.print_help() return 0 diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py deleted file mode 100644 index 43011e402..000000000 --- a/sdk/python/tingly/plugin/register.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Register a running plugin with tingly-box in one step. - -Calls the first-class ``POST /api/v2/plugins`` endpoint, which creates a -plugin-kind provider pointing at the plugin's HTTP server and — when a scenario -is given — the rule whose upstream is that plugin. The plugin then composes with -tb's routing, fallback, guard rails, quota and logging like any other model. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Optional - -import httpx - -from .. import config as _config -from .._http import safe_json -from ..errors import AuthError, GatewayUnreachableError - - -@dataclass -class RegisterResult: - provider_uuid: Optional[str] - name: str - api_base: str - model_id: str - scenario: Optional[str] - rule_uuid: Optional[str] - ready: bool - note: str - - -def register_with_tb( - name: str, - plugin_url: str, - model_id: str, - *, - scenario: Optional[str] = None, - gateway_url: Optional[str] = None, - admin_token: Optional[str] = None, - token: str = "", - tier: int = 0, - timeout: float = 30.0, -) -> RegisterResult: - """Wire a plugin into tingly-box in one call (provider + optional rule). - - Args: - name: plugin / provider name to create in tb. - plugin_url: the plugin's OpenAI base, e.g. ``http://127.0.0.1:8765/v1``. - model_id: the model id the plugin advertises (e.g. ``plugin/my-rag``). - scenario: bind a rule under this scenario so the model is selectable - immediately; omit to create only the provider. - gateway_url / admin_token: tb gateway + admin token; auto-discovered if - omitted (same precedence as ``connect()``). - token: optional API token tb should send to the plugin (matches the - plugin's ``api_key`` if it enforces one). - tier: tier for the bound service (0 = highest priority). - """ - resolved = _config.resolve(base_url=gateway_url, token=admin_token) - headers = {"Authorization": f"Bearer {resolved.token or ''}"} - url = resolved.base_url.rstrip("/") + "/api/v2/plugins" - payload = { - "name": name, - "endpoint": plugin_url, - "model_id": model_id, - "token": token, - "scenario": scenario or "", - "tier": tier, - } - - try: - resp = httpx.post(url, json=payload, headers=headers, timeout=timeout) - except httpx.HTTPError as exc: - raise GatewayUnreachableError( - f"could not reach tingly-box at {resolved.base_url}: {exc}" - ) from exc - - if resp.status_code == 401: - raise AuthError("tingly-box rejected the admin token while registering the plugin") - - payload_data = safe_json(resp) or {} - if resp.status_code not in (200, 201) or not payload_data.get("success"): - raise GatewayUnreachableError( - f"plugin registration failed: HTTP {resp.status_code} {resp.text[:200]}" - ) - - data = payload_data.get("data") or {} - return RegisterResult( - provider_uuid=data.get("provider_uuid"), - name=name, - api_base=plugin_url, - model_id=data.get("model_id", model_id), - scenario=data.get("scenario") or None, - rule_uuid=data.get("rule_uuid") or None, - ready=bool(data.get("ready", False)), - note=data.get("note", ""), - ) From dc75813957d14bd725535bee2676721e6b387142 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 00:41:46 +0000 Subject: [PATCH 13/28] test(e2e): demonstrate ephemeral plugin lifecycle (lease expiry on crash) Add steps 7-8 to examples/e2e_run.sh: hard-kill the plugin and show tb auto-removes the live instance once the lease lapses (GET /api/v2/plugins empty), then a client call to the model is no longer routable. e2e_plugin.py uses a short ttl so the demo doesn't wait. Verified end-to-end against the real tb binary (full hub round-trip in step 6, auto-removal in step 7). --- sdk/python/examples/e2e_plugin.py | 3 ++- sdk/python/examples/e2e_run.sh | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/sdk/python/examples/e2e_plugin.py b/sdk/python/examples/e2e_plugin.py index a83e8096d..ced25f593 100644 --- a/sdk/python/examples/e2e_plugin.py +++ b/sdk/python/examples/e2e_plugin.py @@ -32,4 +32,5 @@ def handle(req): if __name__ == "__main__": - plugin.serve(port=8765) + # Short lease so the e2e can show auto-removal on death without a long wait. + plugin.serve(port=8765, ttl_seconds=4) diff --git a/sdk/python/examples/e2e_run.sh b/sdk/python/examples/e2e_run.sh index 5d331caca..83c194ddc 100755 --- a/sdk/python/examples/e2e_run.sh +++ b/sdk/python/examples/e2e_run.sh @@ -82,5 +82,26 @@ curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d "messages":[{"role":"user","content":"What is tingly-box?"}]}' | python3 -m json.tool echo "== plugin log tail ==" -tail -5 /tmp/plugin_e2e.log +tail -6 /tmp/plugin_e2e.log + +echo "== 7. EPHEMERAL: kill the plugin → tb auto-removes it when the lease lapses ==" +echo " (hard SIGKILL = simulate a crash; no graceful deregister, nothing persisted)" +kill -KILL "$PLUG_PID" 2>/dev/null +PLUG_PID="" +echo " waiting for the 4s lease to expire..." +for i in $(seq 1 20); do + LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") + echo "$LIST" | grep -q 'rag-demo' || break + sleep 0.5 +done +echo " GET /api/v2/plugins now: $LIST" + +echo "== 8. client call after the plugin is gone → instance no longer routable ==" +echo " (the durable rule's only service is the dead plugin; a real setup would" +echo " keep a tier-1 real model and tier-failover here)" +curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ + "model":"plugin/rag-demo", + "messages":[{"role":"user","content":"still there?"}]}' \ + | python3 -c "import sys,json; d=json.load(sys.stdin); e=d.get('error', d); print(' ->', (json.dumps(e) if isinstance(e,dict) else str(e))[:200])" + echo "== done ==" From 849ce5fc1b5197f9840a2447ab7f34a342850dba Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 04:07:34 +0000 Subject: [PATCH 14/28] fix: resolve rebase-onto-main fixups (tactic API, route group, markers) Rebasing claude/epic-hopper-rqgfci onto the updated main surfaced a few mechanical breaks from upstream changes: - typ.ParseTacticFromMap was replaced by typ.NewDefaultTactic upstream; update the plugin rule-binding call site. - internal/server/webui_api.go was renamed to server_webui_api.go and its provider CRUD routes moved into providermodule (registered under apiV2); fix the plugin route registrations to use the apiV2 group that now exists in scope instead of the removed `api` variable. - Remove a stray leftover conflict marker in scenario_registry_test.go (the surrounding content was already correctly merged). - Regenerate openapi.json against the fully rebased tree (the mid-rebase commit had a stale intermediate snapshot). go build ./... and go test on the affected packages pass; 31 Python SDK tests pass. --- internal/server/plugin_provider.go | 2 +- internal/server/server_webui_api.go | 8 ++++---- internal/typ/scenario_registry_test.go | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/server/plugin_provider.go b/internal/server/plugin_provider.go index 9a061d8df..dca4efc8b 100644 --- a/internal/server/plugin_provider.go +++ b/internal/server/plugin_provider.go @@ -154,7 +154,7 @@ func (s *Server) ensurePluginRule(scenario, modelID, providerID, name string, ti RequestModel: modelID, Description: "Plugin: " + name, Active: true, - LBTactic: typ.ParseTacticFromMap(loadbalance.TacticTier, nil), + LBTactic: typ.NewDefaultTactic(loadbalance.TacticTier), Services: []*loadbalance.Service{ {Provider: providerID, Model: modelID, Weight: 1, Active: true, Tier: tier}, }, diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index 0aa51679b..b19d7880e 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -400,7 +400,7 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { providerHandler := providermodule.NewHandler(s.config, s.quotaManager) providermodule.RegisterRoutes(apiV2, providerHandler) - api.GET("/plugins", s.ListPlugins, + apiV2.GET("/plugins", s.ListPlugins, swagger.WithDescription("List live (dynamically-registered) plugin instances"), swagger.WithTags("plugins"), swagger.WithResponseModel(PluginsResponse{}), @@ -409,19 +409,19 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { // Plugin lifecycle: register a live instance, keep it alive by heartbeat, and // deregister on shutdown. Nothing is persisted — an expired instance simply // falls out of routing (tier failover). - api.POST("/plugins/register", s.RegisterPlugin, + apiV2.POST("/plugins/register", s.RegisterPlugin, swagger.WithDescription("Register a live, ephemeral plugin instance (leased)"), swagger.WithTags("plugins"), swagger.WithRequestModel(RegisterPluginRequest{}), swagger.WithResponseModel(RegisterPluginResponse{}), ) - api.POST("/plugins/heartbeat", s.HeartbeatPlugin, + apiV2.POST("/plugins/heartbeat", s.HeartbeatPlugin, swagger.WithDescription("Extend a plugin instance's lease"), swagger.WithTags("plugins"), swagger.WithRequestModel(PluginLeaseRequest{}), swagger.WithResponseModel(gin.H{}), ) - api.POST("/plugins/deregister", s.DeregisterPlugin, + apiV2.POST("/plugins/deregister", s.DeregisterPlugin, swagger.WithDescription("Remove a live plugin instance immediately"), swagger.WithTags("plugins"), swagger.WithRequestModel(PluginLeaseRequest{}), diff --git a/internal/typ/scenario_registry_test.go b/internal/typ/scenario_registry_test.go index d7286e878..806b68389 100644 --- a/internal/typ/scenario_registry_test.go +++ b/internal/typ/scenario_registry_test.go @@ -174,7 +174,6 @@ func TestRegisterScenario_RejectsConflictingDescriptor(t *testing.T) { } } -<<<<<<< HEAD func TestIsSimpleProfileAlias(t *testing.T) { cases := []struct { in string From a55febe049e5ba2091daa8127f8f75c963fee1de Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 04:42:35 +0000 Subject: [PATCH 15/28] refactor(plugins): drop ephemeral registry, plugin = tagged provider + circuit breaker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tb already has liveness detection: every (rule, service) is covered by the existing per-service circuit breaker. The lease/heartbeat/TTL ephemeral registry (in-memory PluginRegistry, EphemeralProviderResolver hook consulted on every provider lookup, background heartbeat thread in the SDK) was built to avoid a stale DB row after a plugin process stops — a cosmetic problem, not a correctness one — by reinventing distributed-service-discovery machinery that a personal, single-operator box doesn't need. Removed in favor of the much smaller design the codebase already supports: Go: - ai.Provider.IsPlugin() now checks the existing, generic Tags field for "plugin" instead of a dedicated PluginDetail struct/column. No new DB schema. - POST /api/v2/plugins is an idempotent upsert-by-name: register once at startup (and again on every restart) and it updates the same provider instead of duplicating it; ensures the rule when `scenario` is given. - GET /api/v2/plugins lists plugin-tagged providers, deriving the display model id from the bound rule (no extra field needed). - Removed: PluginRegistry, EphemeralProviderResolver (Config hook + fallback branches in GetProviderByUUID/validateRuleServices), the heartbeat/ deregister endpoints, PluginDetail type. - Retiring a plugin = deleting its provider, same as any other provider. Python SDK: - plugin/runtime.py (register/heartbeat/deregister/Heartbeater thread) replaced by plugin/register.py: a single register() call. - Plugin.serve() registers once at startup, no background thread; stop() just shuts down the HTTP server. Verified end-to-end against the real tb binary (examples/e2e_run.sh, 9 steps): plugin registers once, client call round-trips through it and back into tb; killing the plugin leaves the provider listed (same as any provider) and the next request fails with a plain connection error (would tier-failover with a fallback tier configured); restarting upserts the same provider, no duplicate. go test ./... green except two pre-existing, unrelated failures already present on the pushed rebase commit (smart_guide test-mock interface drift, statusline cache-usage test) — verified via git stash. 31 Python tests pass. Docs (.design/python-sdk.md) updated with the design history as a record of what was tried and why it didn't stick. --- .design/python-sdk.md | 102 ++++++------ ai/provider.go | 34 ++-- internal/server/config/config.go | 17 -- internal/server/config/provider.go | 30 +--- internal/server/plugin_dynamic_test.go | 124 -------------- internal/server/plugin_provider.go | 203 ++++++++++++++--------- internal/server/plugin_provider_test.go | 165 ++++++++++++++++++ internal/server/plugin_registry.go | 160 ------------------ internal/server/plugin_registry_test.go | 81 --------- internal/server/server.go | 15 +- internal/server/server_webui_api.go | 30 +--- internal/typ/type.go | 4 +- sdk/python/README.md | 13 +- sdk/python/examples/e2e_plugin.py | 3 +- sdk/python/examples/e2e_run.sh | 39 +++-- sdk/python/examples/rag_plugin.py | 8 +- sdk/python/tests/test_plugin_register.py | 113 +++++++++++++ sdk/python/tests/test_runtime.py | 124 -------------- sdk/python/tingly/cli.py | 4 +- sdk/python/tingly/plugin/core.py | 49 +++--- sdk/python/tingly/plugin/register.py | 73 ++++++++ sdk/python/tingly/plugin/runtime.py | 126 -------------- 22 files changed, 621 insertions(+), 896 deletions(-) delete mode 100644 internal/server/plugin_dynamic_test.go delete mode 100644 internal/server/plugin_registry.go delete mode 100644 internal/server/plugin_registry_test.go create mode 100644 sdk/python/tests/test_plugin_register.py delete mode 100644 sdk/python/tests/test_runtime.py create mode 100644 sdk/python/tingly/plugin/register.py delete mode 100644 sdk/python/tingly/plugin/runtime.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 4d1b87b8c..f248ef9b0 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -57,48 +57,52 @@ today's pieces — three verbs for the one rule⇄plugin relationship: The historical "Layer 1/2/3" headings below map exactly to connect / serve / register. They are an implementation tour, not three separate products. -### tb-side: plugins are ephemeral, not a persisted provider kind - -An earlier iteration persisted a plugin as a provider row (a `plugin_detail` -column + a `POST /api/v2/plugins` "pin" path). That was removed: plugins are -**dynamic/ephemeral** (see below), so there is no persisted plugin provider, no -DB column, and no separate persistent endpoint. `ai.PluginDetail` + -`Provider.IsPlugin()` survive only as an **in-memory marker** the registry sets -on the synthesized provider (distinct from `AuthTypeVirtual`, the in-process -`vmodel` path). A plugin is otherwise an ordinary OpenAI HTTP upstream, so -**routing is unchanged**. "Pin" (durable opt-in) can return later as a flag if a -real need appears. - -### Plugin as runtime service (dynamic registration, implemented) - -A plugin is a **runtime instance**, not a static config entry. It registers at -startup, heartbeats to hold a lease, and is auto-removed when it stops/dies — -nothing persisted. Differs from a standard provider (durable, operator-managed). - -- **In-memory `PluginRegistry`** on the Server (process-local, matching tb's - circuit-breaker stance — no shared store). Stable id from name (UUIDv5) so a - restart re-registers under the same id; rotating `lease_id` per register. -- **Config hook** `EphemeralProviderResolver`: `GetProviderByUUID` / - `validateRuleServices` fall back to the registry, so **routing resolves live - plugins transparently** and an expired one simply isn't found → existing tier - failover routes to a tier-1 real model. The db layer stays pure persistence. -- **DNS-style layering**: the rule (the durable "name") is ensured idempotently; - the instance (endpoint + liveness) is ephemeral. No live instance ⇒ failover. -- Endpoints (apiV2): `POST /plugins/register` (leased; ensures rule), - `POST /plugins/heartbeat`, `POST /plugins/deregister`, `GET /plugins` (live - instances). +### tb-side: a plugin is a normal, tagged provider (implemented) + +**Design history, briefly, because it's instructive.** Two earlier iterations +over-built this: first a persisted "plugin provider kind" with its own DB +column and a distinct registration endpoint; then a full ephemeral service- +discovery layer (in-memory registry, per-instance lease, heartbeat thread, +TTL expiry, a `Config` hook consulted on every provider lookup) built to avoid +leaving a stale DB row behind when a plugin process stopped. Both were removed. +The reason: **tb already has liveness detection** — every `(rule, service)` +pair is covered by the existing per-service circuit breaker +(`internal/loadbalance/breaker.go`). A dead plugin's first failed request trips +it exactly like a dead real provider; traffic tier-fails-over automatically +when a fallback tier is configured. Lease/heartbeat/TTL was reinventing that +mechanism — distributed-service-discovery machinery for a problem tb doesn't +have (a personal, single-operator box, not a multi-tenant cluster). See +`git log` on this file's directory for the two removed designs if useful as a +cautionary reference. + +**What shipped instead — the minimal version:** + +- A plugin is an ordinary provider (`APIStyle=openai`, `api_key`/`no_key`) + carrying the tag `"plugin"` in the existing, generic `Provider.Tags` field. + `Provider.IsPlugin()` checks for that tag. No new struct, no new DB column — + Tags already round-trips through the provider store unconditionally. +- **`POST /api/v2/plugins`** is an **idempotent upsert by name**: register once + at startup (and again on every restart) and it updates the same provider + instead of duplicating it. When `scenario` is given it also idempotently + ensures the rule. Response: `{provider_uuid, model_id, scenario, rule_uuid, + ready, note}`. +- **`GET /api/v2/plugins`** lists plugin-tagged providers, deriving the display + model id from the rule(s) bound to each (no extra field needed). +- **Retiring a plugin** is the same as retiring any other provider: delete it + in the tb UI. There is no separate lifecycle to reason about. **Active configuration** (SDK): `tingly.configure(url=, admin_token_env=)` / `Connection` inject the tb target + credentials at runtime (secrets by env reference), top-precedence in `config.resolve()` — for containers / CI / remote -where there is no `~/.tingly-box`. `Plugin.serve(register=True, scenario=…, -ttl_seconds=…, tb=Connection(...))` self-registers, heartbeats on a background -thread, and deregisters on shutdown. +where there is no `~/.tingly-box`. This part was cheap and answers a real need, +so it stayed. `Plugin.serve(register=True, scenario=…, tb=Connection(...))` +registers once at startup — no background thread, no lease to manage. -Verified end-to-end (`examples/e2e_run.sh`): the plugin self-registers as a live -`ephemeral` instance, a client call routes through it, and it calls back into tb -— no network/keys. Remaining tb-side: rule-editor UI "plugin" kind (frontend), -process supervisor, scoped inference tokens, fully-ephemeral binding. +Verified end-to-end (`examples/e2e_run.sh`): the plugin registers once, a +client call routes through it and back into tb (no network/keys); killing the +plugin leaves its provider listed (same as any provider) and the next request +fails with a plain connection error (add a tier-1 fallback to see failover +instead); restarting the plugin upserts the same provider, no duplicate. ## Shape @@ -117,7 +121,7 @@ sdk/python/ server.py # stdlib OpenAI-compatible HTTP server (+ SSE) types.py # ChatRequest / Message manifest.py # tingly.toml read/write - runtime.py # dynamic register / heartbeat / deregister + register.py # one-shot, idempotent register with tb cli.py # `tingly doctor` + `tingly plugin {init,run}` errors.py # TinglyError hierarchy ``` @@ -379,15 +383,14 @@ CLI: ``` tingly plugin init my-rag # scaffold my_rag_plugin.py + tingly.toml -tingly plugin run my_rag_plugin.py # serve it -tingly plugin register my-rag \ # wire it into tb as a provider (Layer 3) - --url http://127.0.0.1:8765/v1 --model-id plugin/my-rag +tingly plugin run my_rag_plugin.py # serve it AND register with tb ``` -`register` uses the existing `POST /api/v1/providers` endpoint (admin token, -resolved like `connect()`). Creating the *rule/service* that maps the model into -a scenario is still a user/UI step — the provider is the part the SDK does -idempotently. +`run` (via `Plugin.serve()`) registers with `POST /api/v2/plugins` on startup — +an idempotent upsert by name that creates/updates the provider *and* the rule +(when `scenario` is set on the constructor) in one call. There is no separate +`register` command: a one-shot register with nothing keeping it alive would be +meaningless once ephemeral lifecycle was cut (see the "tb-side" section above). **Not yet built (tb-side):** a sub-process supervisor that boots plugins from their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy @@ -431,10 +434,11 @@ via the in-process `vmodel` package. Wiring (no new gateway hot-path code — it's just a provider): 1. **Plugin serves** `POST /v1/chat/completions` (Layer 2 `Plugin.serve()`). -2. **Register a provider**: `{name:"my-rag", api_base:"http://127.0.0.1:", - api_style:"openai", models:["plugin/my-rag"]}` — a *normal* provider, not - `AuthType=virtual`. -3. **Bind a rule/service**: model `plugin/my-rag` → that provider. +2. **Register**: `POST /api/v2/plugins {name:"my-rag", endpoint:"http://127.0.0.1:/v1", + model_id:"plugin/my-rag", scenario:"experiment"}` creates a *normal* provider + (not `AuthType=virtual`, tagged `"plugin"`) — this is exactly what + `Plugin.serve()` does on startup. +3. That same call **binds the rule/service**: model `plugin/my-rag` → that provider. 4. Now `model:"plugin/my-rag"` from any client resolves through the same dispatcher as every other model. Put the plugin in tier 0 and a real model in tier 1 and tb fails over automatically when the plugin is down. diff --git a/ai/provider.go b/ai/provider.go index 161cd1aed..e3ab70571 100644 --- a/ai/provider.go +++ b/ai/provider.go @@ -53,18 +53,13 @@ type VModelDetail struct { LatencyProfile string `json:"latency_profile,omitempty"` } -// PluginDetail marks a provider as backed by external plugin code. It is an -// in-memory marker only — plugins register dynamically and are never persisted, -// so the synthesized provider carries this to identify plugin traffic. A plugin -// is otherwise an ordinary OpenAI HTTP upstream (APIStyle=openai, api_key / -// no_key), so there is NO routing change. -// -// Distinct from VModelDetail / AuthTypeVirtual (the in-process synthetic-model +// PluginTag is the Provider.Tags value that marks a provider as backed by +// external plugin code. A plugin provider is otherwise an ordinary OpenAI HTTP +// upstream (APIStyle=openai, api_key / no_key) — there is NO routing change; +// reusing the existing generic Tags field means plugin identity needs no new +// persisted column. Distinct from AuthTypeVirtual (the in-process vmodel // path): a plugin runs out-of-process and is reached over HTTP. -type PluginDetail struct { - // ModelID is the model id the plugin advertises (e.g. "plugin/my-rag"). - ModelID string `json:"model_id,omitempty"` -} +const PluginTag = "plugin" // CredentialBundle holds the credential fields for multi-field auth types // (AWS SigV4, Azure, GCP Vertex). Fields is a generic, schema-validated @@ -212,7 +207,6 @@ type Provider struct { AuthType AuthType `json:"auth_type"` // api_key, oauth, vmodel, aws_sigv4, azure_key, gcp_sa OAuthDetail *OAuthDetail `json:"oauth_detail,omitempty"` // OAuth credentials (only for oauth auth type) VModelDetail *VModelDetail `json:"vmodel_detail,omitempty"` // Virtual-model config (only for vmodel auth type) - PluginDetail *PluginDetail `json:"plugin_detail,omitempty"` // Plugin config (set when this provider is backed by plugin code) Credential *CredentialBundle `json:"credential,omitempty"` // Multi-field credentials (only for multi-field auth types) Source ProviderSource `json:"source,omitempty"` // "user" (default) or "builtin" @@ -267,11 +261,19 @@ func (p *Provider) IsVirtual() bool { return p != nil && p.AuthType == AuthTypeVirtual } -// IsPlugin reports whether this provider is backed by external plugin code. -// Plugin providers route as ordinary OpenAI HTTP upstreams; this is metadata -// for UI grouping and lifecycle discovery only. +// IsPlugin reports whether this provider is backed by external plugin code +// (carries the PluginTag). Plugin providers route as ordinary OpenAI HTTP +// upstreams; this is metadata for UI grouping only. func (p *Provider) IsPlugin() bool { - return p != nil && p.PluginDetail != nil + if p == nil { + return false + } + for _, tag := range p.Tags { + if tag == PluginTag { + return true + } + } + return false } // IsBuiltin reports whether this provider was seeded by the system and is diff --git a/internal/server/config/config.go b/internal/server/config/config.go index dd8905857..921d7356c 100644 --- a/internal/server/config/config.go +++ b/internal/server/config/config.go @@ -119,11 +119,6 @@ type Config struct { imbotSettingsStore *db.ImBotSettingsStore templateManager *data.TemplateManager - // ephemeralResolver resolves non-persisted provider UUIDs (live plugin - // instances). Consulted as a fallback by GetProviderByUUID / - // validateRuleServices. Guarded by mu. - ephemeralResolver EphemeralProviderResolver - // Provider lifecycle hooks providerUpdateHooks []ProviderUpdateHook providerDeleteHooks []ProviderDeleteHook @@ -2541,12 +2536,6 @@ func (c *Config) validateRuleServices(rule typ.Rule) error { provider, err := c.providerStore.GetByUUID(svc.Provider) if err != nil { - // A live plugin instance is a valid (ephemeral) provider target. - if c.ephemeralResolver != nil { - if _, ok := c.ephemeralResolver.Resolve(svc.Provider); ok { - continue - } - } return fmt.Errorf("service references non-existent provider '%s': %w", svc.Provider, err) } if provider == nil { @@ -2566,12 +2555,6 @@ func (c *Config) validateRuleServices(rule typ.Rule) error { provider, err := c.providerStore.GetByUUID(svc.Provider) if err != nil { - // A live plugin instance is a valid (ephemeral) provider target. - if c.ephemeralResolver != nil { - if _, ok := c.ephemeralResolver.Resolve(svc.Provider); ok { - continue - } - } return fmt.Errorf("smart routing service references non-existent provider '%s': %w", svc.Provider, err) } if provider == nil { diff --git a/internal/server/config/provider.go b/internal/server/config/provider.go index c924ff4ed..a25c56cf3 100644 --- a/internal/server/config/provider.go +++ b/internal/server/config/provider.go @@ -23,22 +23,6 @@ type ProviderDeleteHook interface { OnProviderDelete(uuid string) } -// EphemeralProviderResolver resolves provider UUIDs that are not persisted in -// the provider store — e.g. live plugin instances held in an in-memory registry. -// It is consulted as a fallback by provider lookups so dynamically-registered -// plugins route transparently, while keeping the db layer pure persistence. -type EphemeralProviderResolver interface { - Resolve(uuid string) (*typ.Provider, bool) -} - -// SetEphemeralProviderResolver installs the fallback resolver (e.g. the Server's -// plugin registry). Safe to call once during server construction. -func (c *Config) SetEphemeralProviderResolver(r EphemeralProviderResolver) { - c.mu.Lock() - defer c.mu.Unlock() - c.ephemeralResolver = r -} - // migrateProvidersToDB migrates providers from JSON config to database. // This is a one-time migration that runs on startup if the database is empty. // After migration (or if the database is already authoritative), the JSON @@ -121,23 +105,13 @@ func (c *Config) AddProviderByName(name, apiBase, token string) error { // GetProviderByUUID returns a provider from database func (c *Config) GetProviderByUUID(uuid string) (*typ.Provider, error) { c.mu.RLock() + defer c.mu.RUnlock() + if c.providerStore == nil { - c.mu.RUnlock() return nil, fmt.Errorf("provider store not initialized") } provider, err := c.providerStore.GetByUUID(uuid) - resolver := c.ephemeralResolver - c.mu.RUnlock() - if err != nil { - // Fall back to the ephemeral resolver (live plugin instances that are not - // persisted). Called outside the config lock to avoid holding it across - // the registry's own lock. A miss means the provider is truly unavailable. - if resolver != nil { - if p, ok := resolver.Resolve(uuid); ok { - return p, nil - } - } return nil, fmt.Errorf("provider '%s' not found: %w", uuid, err) } return provider, nil diff --git a/internal/server/plugin_dynamic_test.go b/internal/server/plugin_dynamic_test.go deleted file mode 100644 index 469941e4c..000000000 --- a/internal/server/plugin_dynamic_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package server - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/gin-gonic/gin" - - "github.com/tingly-dev/tingly-box/internal/server/config" - "github.com/tingly-dev/tingly-box/internal/typ" -) - -func newDynamicPluginServer(t *testing.T) *Server { - t.Helper() - cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) - if err != nil { - t.Fatalf("NewConfig: %v", err) - } - reg := NewPluginRegistry() - cfg.SetEphemeralProviderResolver(reg) - return &Server{config: cfg, pluginRegistry: reg} -} - -func TestRegisterPlugin_RoutesAndExpires(t *testing.T) { - s := newDynamicPluginServer(t) - - w, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ - Name: "my-rag", - Endpoint: "http://127.0.0.1:8765/v1", - ModelID: "plugin/my-rag", - Scenario: string(typ.ScenarioExperiment), - }) - if w.Code != 200 { - t.Fatalf("status %d: %s", w.Code, w.Body.String()) - } - data, _ := resp["data"].(map[string]any) - pluginID, _ := data["plugin_id"].(string) - leaseID, _ := data["lease_id"].(string) - if pluginID == "" || leaseID == "" { - t.Fatalf("missing ids: %v", data) - } - - // No persistent provider was created. - for _, p := range s.config.ListProviders() { - if p.UUID == pluginID { - t.Fatalf("dynamic registration must NOT persist a provider") - } - } - - // But routing resolution (the real dispatch chokepoint) finds the live instance. - prov, err := s.config.GetProviderByUUID(pluginID) - if err != nil || !prov.IsPlugin() { - t.Fatalf("expected live ephemeral resolution, err=%v prov=%+v", err, prov) - } - - // The durable rule (the name) was bound to the plugin id. - var bound bool - for _, rule := range s.config.GetRequestConfigs() { - if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { - bound = true - if rule.Services[0].Provider != pluginID { - t.Fatalf("rule service should reference plugin id, got %s", rule.Services[0].Provider) - } - } - } - if !bound { - t.Fatalf("expected a durable rule bound to the plugin") - } - - // After deregister, the instance is gone → routing can no longer resolve it - // (→ tier failover in a real request). - postJSON(t, s.DeregisterPlugin, PluginLeaseRequest{LeaseID: leaseID}) - if _, err := s.config.GetProviderByUUID(pluginID); err == nil { - t.Fatalf("provider must be unresolved after deregister") - } -} - -func TestHeartbeatPlugin_UnknownLease(t *testing.T) { - s := newDynamicPluginServer(t) - w, _ := postJSON(t, s.HeartbeatPlugin, PluginLeaseRequest{LeaseID: "nope"}) - if w.Code != 404 { - t.Fatalf("expected 404 for unknown lease, got %d", w.Code) - } -} - -func TestListPlugins_ShowsLiveInstances(t *testing.T) { - s := newDynamicPluginServer(t) - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{Name: "plug", Endpoint: "http://127.0.0.1:8765/v1"}) - - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodGet, "/", nil) - s.ListPlugins(c) - - var resp PluginsResponse - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if len(resp.Data) != 1 || resp.Data[0].Name != "plug" { - t.Fatalf("expected the one live plugin, got %+v", resp.Data) - } -} - -func TestReRegisterIsIdempotentForRule(t *testing.T) { - s := newDynamicPluginServer(t) - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ - Name: "p", Endpoint: "http://a/v1", Scenario: string(typ.ScenarioExperiment), - }) - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ - Name: "p", Endpoint: "http://b/v1", Scenario: string(typ.ScenarioExperiment), - }) - // exactly one rule for plugin/p - count := 0 - for _, rule := range s.config.GetRequestConfigs() { - if rule.RequestModel == "plugin/p" { - count++ - } - } - if count != 1 { - t.Fatalf("re-register must not duplicate the rule, got %d", count) - } -} diff --git a/internal/server/plugin_provider.go b/internal/server/plugin_provider.go index dca4efc8b..6c5f8a5b0 100644 --- a/internal/server/plugin_provider.go +++ b/internal/server/plugin_provider.go @@ -2,22 +2,45 @@ package server import ( "net/http" - "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + "github.com/tingly-dev/tingly-box/internal/constant" "github.com/tingly-dev/tingly-box/internal/loadbalance" "github.com/tingly-dev/tingly-box/internal/server/config" "github.com/tingly-dev/tingly-box/internal/typ" ) -// PluginInfo is a list view of a live plugin instance. +// RegisterPluginRequest registers external plugin code as a tingly-box upstream. +// It is idempotent by name: calling it again (e.g. every time the plugin +// process starts) updates the existing provider instead of duplicating it. +type RegisterPluginRequest struct { + Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` + Endpoint string `json:"endpoint" binding:"required" description:"Plugin OpenAI base URL" example:"http://127.0.0.1:8765/v1"` + ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` + Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` + Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` + Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` +} + +// RegisterPluginResponse reports what was created or updated. +type RegisterPluginResponse struct { + ProviderUUID string `json:"provider_uuid"` + ModelID string `json:"model_id"` + Scenario string `json:"scenario,omitempty"` + RuleUUID string `json:"rule_uuid,omitempty"` + // Ready is true when a rule is bound, so clients can select the model now. + Ready bool `json:"ready"` + Note string `json:"note,omitempty"` +} + +// PluginInfo is a list view of a plugin provider. type PluginInfo struct { UUID string `json:"uuid"` Name string `json:"name"` Endpoint string `json:"endpoint"` - ModelID string `json:"model_id"` + ModelID string `json:"model_id,omitempty"` } // PluginsResponse wraps the plugin list. @@ -26,114 +49,130 @@ type PluginsResponse struct { Data []PluginInfo `json:"data"` } -// ListPlugins returns the live (dynamically-registered) plugin instances. -func (s *Server) ListPlugins(c *gin.Context) { - plugins := []PluginInfo{} - for _, reg := range s.pluginRegistry.List() { - plugins = append(plugins, PluginInfo{ - UUID: reg.ID, - Name: reg.Name, - Endpoint: reg.Endpoint, - ModelID: reg.ModelID, - }) - } - c.JSON(http.StatusOK, PluginsResponse{Success: true, Data: plugins}) -} - -// RegisterPluginRequest registers a live, ephemeral plugin instance and, when a -// scenario is given, ensures the durable rule whose upstream is that plugin. -type RegisterPluginRequest struct { - Name string `json:"name" binding:"required" example:"my-rag"` - Endpoint string `json:"endpoint" binding:"required" example:"http://127.0.0.1:8765/v1"` - ModelID string `json:"model_id,omitempty" example:"plugin/my-rag"` - Token string `json:"token,omitempty"` - Scenario string `json:"scenario,omitempty" example:"experiment"` - Tier int `json:"tier,omitempty"` - TTLSeconds int `json:"ttl_seconds,omitempty" example:"30"` -} - -// RegisterPluginResponse reports the lease for an ephemeral registration. -type RegisterPluginResponse struct { - PluginID string `json:"plugin_id"` - LeaseID string `json:"lease_id"` - ModelID string `json:"model_id"` - Scenario string `json:"scenario,omitempty"` - RuleUUID string `json:"rule_uuid,omitempty"` - TTLSeconds int `json:"ttl_seconds"` - Note string `json:"note,omitempty"` -} - -// RegisterPlugin registers a live plugin instance in the in-memory registry (NOT -// persisted). The plugin keeps it alive by heartbeating; it is auto-removed when -// the lease expires or the plugin deregisters. When a scenario is given, the -// durable rule (the stable "name") is ensured idempotently. +// RegisterPlugin creates or updates a plugin-tagged provider (and optionally +// binds a rule to it) so "configure this rule with a plugin" is one call. +// +// A plugin provider is an ordinary OpenAI HTTP upstream — routing is +// unchanged, and liveness is handled by the same per-service circuit breaker +// that already protects every other provider: if the plugin process is down, +// the first failed request trips the breaker and traffic tier-fails-over +// (when a fallback tier is configured). There is deliberately no separate +// registration lifecycle (lease/heartbeat/expiry) for plugins — that would +// duplicate the breaker for a single-operator box. If a plugin is retired, +// delete its provider like any other (DELETE /api/v2/providers/:uuid). func (s *Server) RegisterPlugin(c *gin.Context) { var req RegisterPluginRequest if err := c.ShouldBindJSON(&req); err != nil { c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) return } - ttl := time.Duration(req.TTLSeconds) * time.Second - reg := s.pluginRegistry.Register(req.Name, req.Endpoint, req.ModelID, req.Scenario, req.Token, ttl) + + modelID := req.ModelID + if modelID == "" { + modelID = "plugin/" + req.Name + } + + provider, err := s.upsertPluginProvider(req.Name, req.Endpoint, req.Token) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": "failed to register plugin provider: " + err.Error(), + }) + return + } resp := RegisterPluginResponse{ - PluginID: reg.ID, - LeaseID: reg.LeaseID, - ModelID: reg.ModelID, - TTLSeconds: int(time.Until(reg.ExpiresAt).Seconds()), - Note: "Registered (ephemeral). Heartbeat to keep alive; deregister to remove.", + ProviderUUID: provider.UUID, + ModelID: modelID, + Note: "Provider registered. Bind a rule (pass `scenario`) to make the model selectable.", } + // One-step bind: ensure the rule whose single service is this plugin. if req.Scenario != "" { - // The durable binding references the stable plugin id; the live instance - // is resolved from the registry at request time. - ruleUUID, err := s.ensurePluginRule(req.Scenario, reg.ModelID, reg.ID, req.Name, req.Tier) + ruleUUID, err := s.ensurePluginRule(req.Scenario, modelID, provider.UUID, req.Name, req.Tier) if err != nil { - resp.Note = "Registered, but rule binding failed: " + err.Error() + resp.Note = "Provider registered, but rule binding failed: " + err.Error() c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) return } resp.Scenario = req.Scenario resp.RuleUUID = ruleUUID + resp.Ready = true + resp.Note = "Plugin wired in. Select model " + modelID + " under scenario " + req.Scenario + "." } logrus.WithFields(logrus.Fields{ - "plugin": req.Name, "endpoint": req.Endpoint, "model_id": reg.ModelID, - "scenario": req.Scenario, "ttl_s": resp.TTLSeconds, - }).Info("Registered plugin instance") + "plugin": req.Name, + "endpoint": req.Endpoint, + "model_id": modelID, + "scenario": req.Scenario, + "ready": resp.Ready, + }).Info("Registered plugin provider") + c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } -// PluginLeaseRequest carries a lease id for heartbeat/deregister. -type PluginLeaseRequest struct { - LeaseID string `json:"lease_id" binding:"required"` - TTLSeconds int `json:"ttl_seconds,omitempty"` -} +// upsertPluginProvider creates a plugin-tagged provider, or updates the +// endpoint/token of an existing one with the same name. Idempotent by name so +// a plugin can safely re-register (e.g. on every process start). +func (s *Server) upsertPluginProvider(name, endpoint, token string) (*typ.Provider, error) { + if existing, err := s.config.GetProviderByName(name); err == nil && existing.IsPlugin() { + existing.APIBase = endpoint + existing.Token = token + existing.NoKeyRequired = token == "" + existing.Enabled = true + if err := s.config.UpdateProvider(existing.UUID, existing); err != nil { + return nil, err + } + return existing, nil + } -// HeartbeatPlugin extends a plugin instance's lease. -func (s *Server) HeartbeatPlugin(c *gin.Context) { - var req PluginLeaseRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) - return + provider := &typ.Provider{ + UUID: config.GenerateUUID(), + Name: name, + APIBase: endpoint, + APIStyle: "openai", + Token: token, + NoKeyRequired: token == "", + Enabled: true, + AuthType: typ.AuthTypeAPIKey, + Timeout: constant.DefaultRequestTimeout, + Tags: []string{typ.PluginTag}, } - ttl := time.Duration(req.TTLSeconds) * time.Second - if !s.pluginRegistry.Heartbeat(req.LeaseID, ttl) { - c.JSON(http.StatusNotFound, gin.H{"success": false, "error": "unknown or expired lease"}) - return + if err := s.config.AddProvider(provider); err != nil { + return nil, err } - c.JSON(http.StatusOK, gin.H{"success": true}) + return provider, nil } -// DeregisterPlugin removes a live plugin instance immediately. -func (s *Server) DeregisterPlugin(c *gin.Context) { - var req PluginLeaseRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) - return +// ListPlugins returns the plugin-tagged providers, with the model id(s) each +// currently routes (derived from the rules bound to it) for display. +func (s *Server) ListPlugins(c *gin.Context) { + modelsByProvider := map[string]string{} + for _, rule := range s.config.GetRequestConfigs() { + for _, svc := range rule.Services { + if svc == nil { + continue + } + if _, ok := modelsByProvider[svc.Provider]; !ok { + modelsByProvider[svc.Provider] = rule.RequestModel + } + } } - removed := s.pluginRegistry.Deregister(req.LeaseID) - c.JSON(http.StatusOK, gin.H{"success": true, "data": gin.H{"removed": removed}}) + + plugins := []PluginInfo{} + for _, p := range s.config.ListProviders() { + if !p.IsPlugin() { + continue + } + plugins = append(plugins, PluginInfo{ + UUID: p.UUID, + Name: p.Name, + Endpoint: p.APIBase, + ModelID: modelsByProvider[p.UUID], + }) + } + c.JSON(http.StatusOK, PluginsResponse{Success: true, Data: plugins}) } // ensurePluginRule idempotently ensures a rule exists under scenario whose single diff --git a/internal/server/plugin_provider_test.go b/internal/server/plugin_provider_test.go index 070cbd017..725dc6bff 100644 --- a/internal/server/plugin_provider_test.go +++ b/internal/server/plugin_provider_test.go @@ -8,6 +8,9 @@ import ( "testing" "github.com/gin-gonic/gin" + + "github.com/tingly-dev/tingly-box/internal/server/config" + "github.com/tingly-dev/tingly-box/internal/typ" ) // postJSON drives a gin handler with a JSON body and returns the recorder and @@ -25,3 +28,165 @@ func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseReco _ = json.Unmarshal(w.Body.Bytes(), &parsed) return w, parsed } + +func newPluginTestServer(t *testing.T) *Server { + t.Helper() + cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + return &Server{config: cfg} +} + +func TestRegisterPlugin_BindsRule(t *testing.T) { + s := newPluginTestServer(t) + + w, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", + Endpoint: "http://127.0.0.1:8765/v1", + ModelID: "plugin/my-rag", + Scenario: string(typ.ScenarioExperiment), + }) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", w.Code, w.Body.String()) + } + data, _ := resp["data"].(map[string]any) + if data["ready"] != true { + t.Fatalf("expected ready=true, got %v (note=%v)", data["ready"], data["note"]) + } + if data["model_id"] != "plugin/my-rag" { + t.Fatalf("model_id = %v", data["model_id"]) + } + providerUUID, _ := data["provider_uuid"].(string) + if providerUUID == "" { + t.Fatalf("expected a provider_uuid") + } + + // The provider must be persisted and tagged as a plugin. + prov, err := s.config.GetProviderByUUID(providerUUID) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if !prov.IsPlugin() { + t.Fatalf("provider is not tagged as plugin: %+v", prov) + } + + // A rule must exist under the scenario whose single service is the plugin. + var found bool + for _, rule := range s.config.GetRequestConfigs() { + if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { + found = true + if len(rule.Services) != 1 || rule.Services[0].Provider != providerUUID { + t.Fatalf("rule service does not point at plugin provider: %+v", rule.Services) + } + } + } + if !found { + t.Fatalf("no rule bound for the plugin under experiment scenario") + } +} + +func TestRegisterPlugin_ProviderOnly(t *testing.T) { + s := newPluginTestServer(t) + + _, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "solo", + Endpoint: "http://127.0.0.1:9000/v1", + }) + data, _ := resp["data"].(map[string]any) + if data["ready"] == true { + t.Fatalf("expected ready=false when no scenario given") + } + // model id defaults to plugin/ + if data["model_id"] != "plugin/solo" { + t.Fatalf("model_id default = %v", data["model_id"]) + } +} + +func TestRegisterPlugin_ReregisterUpdatesInPlace(t *testing.T) { + s := newPluginTestServer(t) + + _, first := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", Endpoint: "http://127.0.0.1:8765/v1", + }) + firstUUID := first["data"].(map[string]any)["provider_uuid"].(string) + + // Re-register (e.g. the plugin process restarted on a different port). + _, second := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "my-rag", Endpoint: "http://127.0.0.1:9999/v1", + }) + secondUUID := second["data"].(map[string]any)["provider_uuid"].(string) + + if firstUUID != secondUUID { + t.Fatalf("re-register should update the same provider, got %s then %s", firstUUID, secondUUID) + } + + prov, err := s.config.GetProviderByUUID(firstUUID) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIBase != "http://127.0.0.1:9999/v1" { + t.Fatalf("expected endpoint to be updated in place, got %s", prov.APIBase) + } + + // Exactly one provider named my-rag — no duplicate created. + count := 0 + for _, p := range s.config.ListProviders() { + if p.Name == "my-rag" { + count++ + } + } + if count != 1 { + t.Fatalf("re-register must not duplicate the provider, got %d", count) + } +} + +func TestRegisterPlugin_ReregisterIsIdempotentForRule(t *testing.T) { + s := newPluginTestServer(t) + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "p", Endpoint: "http://a/v1", Scenario: string(typ.ScenarioExperiment), + }) + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "p", Endpoint: "http://b/v1", Scenario: string(typ.ScenarioExperiment), + }) + count := 0 + for _, rule := range s.config.GetRequestConfigs() { + if rule.RequestModel == "plugin/p" { + count++ + } + } + if count != 1 { + t.Fatalf("re-register must not duplicate the rule, got %d", count) + } +} + +func TestListPlugins_FiltersPluginTag(t *testing.T) { + s := newPluginTestServer(t) + // a normal provider + if err := s.config.AddProvider(&typ.Provider{ + Name: "real", APIBase: "https://api.example.com/v1", APIStyle: "openai", Enabled: true, + }); err != nil { + t.Fatalf("AddProvider: %v", err) + } + postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + Name: "plug", Endpoint: "http://127.0.0.1:8765/v1", ModelID: "plugin/plug", + Scenario: string(typ.ScenarioExperiment), + }) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + s.ListPlugins(c) + + var resp PluginsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(resp.Data) != 1 || resp.Data[0].Name != "plug" { + t.Fatalf("expected exactly the plugin provider, got %+v", resp.Data) + } + if resp.Data[0].ModelID != "plugin/plug" { + t.Fatalf("expected model id derived from the bound rule, got %q", resp.Data[0].ModelID) + } +} diff --git a/internal/server/plugin_registry.go b/internal/server/plugin_registry.go deleted file mode 100644 index 698f51930..000000000 --- a/internal/server/plugin_registry.go +++ /dev/null @@ -1,160 +0,0 @@ -package server - -import ( - "sync" - "time" - - "github.com/google/uuid" - - "github.com/tingly-dev/tingly-box/internal/constant" - "github.com/tingly-dev/tingly-box/internal/typ" -) - -// pluginNamespace is the UUIDv5 namespace used to derive a stable plugin id from -// its name, so a plugin that restarts re-registers under the same id and any -// durable rule that references it keeps pointing correctly. -var pluginNamespace = uuid.MustParse("3b1e0a2c-7c4e-5a9b-bf21-9f6d2c8e4a10") - -const defaultPluginTTL = 30 * time.Second - -// PluginRegistration is a live, ephemeral plugin instance. -type PluginRegistration struct { - ID string // deterministic from Name (UUIDv5) - Name string // plugin / provider name - Endpoint string // OpenAI base, e.g. http://127.0.0.1:8765/v1 - ModelID string // advertised model id, e.g. plugin/my-rag - Scenario string // scenario the durable rule was bound under (optional) - Token string // token tb sends to the plugin (optional) - LeaseID string // rotates each register; required to heartbeat/deregister - ExpiresAt time.Time - LastSeen time.Time -} - -// PluginRegistry holds live plugin instances in memory. It is intentionally -// process-local (no shared store), matching tb's existing routing state stance. -// Instances auto-expire when their lease is not renewed; nothing is persisted. -type PluginRegistry struct { - mu sync.RWMutex - byID map[string]*PluginRegistration - ttl time.Duration -} - -// NewPluginRegistry creates an empty registry with the default lease TTL. -func NewPluginRegistry() *PluginRegistry { - return &PluginRegistry{byID: map[string]*PluginRegistration{}, ttl: defaultPluginTTL} -} - -// PluginID derives the stable id for a plugin name. -func PluginID(name string) string { - return uuid.NewSHA1(pluginNamespace, []byte(name)).String() -} - -// Register adds or refreshes a plugin instance and returns the live record. -// ttl <= 0 uses the registry default. -func (r *PluginRegistry) Register(name, endpoint, modelID, scenario, token string, ttl time.Duration) *PluginRegistration { - if ttl <= 0 { - ttl = r.ttl - } - if modelID == "" { - modelID = "plugin/" + name - } - now := time.Now() - reg := &PluginRegistration{ - ID: PluginID(name), - Name: name, - Endpoint: endpoint, - ModelID: modelID, - Scenario: scenario, - Token: token, - LeaseID: uuid.NewString(), - ExpiresAt: now.Add(ttl), - LastSeen: now, - } - r.mu.Lock() - r.byID[reg.ID] = reg - r.mu.Unlock() - return reg -} - -// Heartbeat extends the lease identified by leaseID. Returns false if no live -// registration matches (unknown or already expired). -func (r *PluginRegistry) Heartbeat(leaseID string, ttl time.Duration) bool { - if ttl <= 0 { - ttl = r.ttl - } - now := time.Now() - r.mu.Lock() - defer r.mu.Unlock() - for _, reg := range r.byID { - if reg.LeaseID == leaseID { - if now.After(reg.ExpiresAt) { - delete(r.byID, reg.ID) - return false - } - reg.ExpiresAt = now.Add(ttl) - reg.LastSeen = now - return true - } - } - return false -} - -// Deregister removes the instance for leaseID. Returns true if one was removed. -func (r *PluginRegistry) Deregister(leaseID string) bool { - r.mu.Lock() - defer r.mu.Unlock() - for id, reg := range r.byID { - if reg.LeaseID == leaseID { - delete(r.byID, id) - return true - } - } - return false -} - -// Resolve synthesizes a plugin-kind provider for a live instance by id. Expired -// instances are treated as absent (and reaped). Implements -// config.EphemeralProviderResolver. -func (r *PluginRegistry) Resolve(id string) (*typ.Provider, bool) { - r.mu.Lock() - defer r.mu.Unlock() - reg, ok := r.byID[id] - if !ok { - return nil, false - } - if time.Now().After(reg.ExpiresAt) { - delete(r.byID, id) - return nil, false - } - // A plugin is an ordinary OpenAI HTTP upstream plus the PluginDetail marker; - // routing treats it like any other provider. - return &typ.Provider{ - UUID: reg.ID, - Name: reg.Name, - APIBase: reg.Endpoint, - APIStyle: "openai", - Token: reg.Token, - NoKeyRequired: reg.Token == "", - Enabled: true, - AuthType: typ.AuthTypeAPIKey, - Timeout: constant.DefaultRequestTimeout, - PluginDetail: &typ.PluginDetail{ModelID: reg.ModelID}, - }, true -} - -// List returns the currently-live registrations (expired ones are reaped). -func (r *PluginRegistry) List() []*PluginRegistration { - now := time.Now() - r.mu.Lock() - defer r.mu.Unlock() - out := make([]*PluginRegistration, 0, len(r.byID)) - for id, reg := range r.byID { - if now.After(reg.ExpiresAt) { - delete(r.byID, id) - continue - } - clone := *reg - out = append(out, &clone) - } - return out -} diff --git a/internal/server/plugin_registry_test.go b/internal/server/plugin_registry_test.go deleted file mode 100644 index ad6fef744..000000000 --- a/internal/server/plugin_registry_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package server - -import ( - "testing" - "time" -) - -func TestPluginRegistry_RegisterResolveExpire(t *testing.T) { - r := NewPluginRegistry() - reg := r.Register("my-rag", "http://127.0.0.1:8765/v1", "plugin/my-rag", "experiment", "", 50*time.Millisecond) - - // stable id from name - if reg.ID != PluginID("my-rag") { - t.Fatalf("id not derived from name: %s", reg.ID) - } - - // resolves to a live plugin-kind provider - p, ok := r.Resolve(reg.ID) - if !ok { - t.Fatalf("expected live resolution") - } - if !p.IsPlugin() || p.APIBase != "http://127.0.0.1:8765/v1" || p.PluginDetail.ModelID != "plugin/my-rag" { - t.Fatalf("synthesized provider wrong: %+v", p) - } - - // after TTL it is gone (auto-expire on resolve) - time.Sleep(70 * time.Millisecond) - if _, ok := r.Resolve(reg.ID); ok { - t.Fatalf("expected expiry after TTL") - } -} - -func TestPluginRegistry_HeartbeatKeepsAlive(t *testing.T) { - r := NewPluginRegistry() - reg := r.Register("p", "http://x/v1", "", "", "", 60*time.Millisecond) - - time.Sleep(40 * time.Millisecond) - if !r.Heartbeat(reg.LeaseID, 60*time.Millisecond) { - t.Fatalf("heartbeat should succeed before expiry") - } - time.Sleep(40 * time.Millisecond) // 80ms since register, but heartbeat reset it - if _, ok := r.Resolve(reg.ID); !ok { - t.Fatalf("heartbeat should have kept the instance alive") - } - - // unknown lease - if r.Heartbeat("nope", 0) { - t.Fatalf("unknown lease must not heartbeat") - } -} - -func TestPluginRegistry_Deregister(t *testing.T) { - r := NewPluginRegistry() - reg := r.Register("p", "http://x/v1", "", "", "", time.Minute) - if !r.Deregister(reg.LeaseID) { - t.Fatalf("deregister should remove the instance") - } - if _, ok := r.Resolve(reg.ID); ok { - t.Fatalf("resolve should miss after deregister") - } - if r.Deregister(reg.LeaseID) { - t.Fatalf("second deregister should be a no-op") - } -} - -func TestPluginRegistry_ReRegisterReusesID(t *testing.T) { - r := NewPluginRegistry() - a := r.Register("same", "http://a/v1", "", "", "", time.Minute) - b := r.Register("same", "http://b/v1", "", "", "", time.Minute) - if a.ID != b.ID { - t.Fatalf("re-register must reuse the stable id") - } - if a.LeaseID == b.LeaseID { - t.Fatalf("lease must rotate on re-register") - } - // latest endpoint wins - p, _ := r.Resolve(b.ID) - if p.APIBase != "http://b/v1" { - t.Fatalf("latest registration endpoint should win, got %s", p.APIBase) - } -} diff --git a/internal/server/server.go b/internal/server/server.go index 070a360e7..e7c755033 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -66,10 +66,6 @@ type Server struct { loadBalancerAPI *LoadBalancerAPI healthMonitor *loadbalance.HealthMonitor - // pluginRegistry holds live, ephemeral plugin instances (dynamic - // registration). It is the EphemeralProviderResolver for the config. - pluginRegistry *PluginRegistry - // client pool for caching clientPool *client.ClientPool @@ -214,15 +210,10 @@ func NewServer(cfg *config.Config, opts ...ServerOption) *Server { // Default options server := &Server{ - config: cfg, - ctx: ctx, - cancel: cancel, - pluginRegistry: NewPluginRegistry(), + config: cfg, + ctx: ctx, + cancel: cancel, } - // Live plugin instances resolve through this in-memory registry as an - // ephemeral provider fallback, so dynamically-registered plugins route - // without a persisted provider row. - cfg.SetEphemeralProviderResolver(server.pluginRegistry) // Apply all options (defaults + provided) for _, opt := range allOpts { diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index b19d7880e..dba73512f 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -400,32 +400,20 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { providerHandler := providermodule.NewHandler(s.config, s.quotaManager) providermodule.RegisterRoutes(apiV2, providerHandler) - apiV2.GET("/plugins", s.ListPlugins, - swagger.WithDescription("List live (dynamically-registered) plugin instances"), - swagger.WithTags("plugins"), - swagger.WithResponseModel(PluginsResponse{}), - ) - - // Plugin lifecycle: register a live instance, keep it alive by heartbeat, and - // deregister on shutdown. Nothing is persisted — an expired instance simply - // falls out of routing (tier failover). - apiV2.POST("/plugins/register", s.RegisterPlugin, - swagger.WithDescription("Register a live, ephemeral plugin instance (leased)"), + // Plugin registration: a plugin is a provider tagged "plugin" (external + // OpenAI upstream). POST is an idempotent upsert-by-name that wires it in + // (provider + optional rule) in one step; liveness is handled by the same + // circuit breaker that covers every other provider — no separate lifecycle. + apiV2.POST("/plugins", s.RegisterPlugin, + swagger.WithDescription("Register (or update) external plugin code as an upstream, optionally binding a rule"), swagger.WithTags("plugins"), swagger.WithRequestModel(RegisterPluginRequest{}), swagger.WithResponseModel(RegisterPluginResponse{}), ) - apiV2.POST("/plugins/heartbeat", s.HeartbeatPlugin, - swagger.WithDescription("Extend a plugin instance's lease"), - swagger.WithTags("plugins"), - swagger.WithRequestModel(PluginLeaseRequest{}), - swagger.WithResponseModel(gin.H{}), - ) - apiV2.POST("/plugins/deregister", s.DeregisterPlugin, - swagger.WithDescription("Remove a live plugin instance immediately"), + apiV2.GET("/plugins", s.ListPlugins, + swagger.WithDescription("List registered plugin providers"), swagger.WithTags("plugins"), - swagger.WithRequestModel(PluginLeaseRequest{}), - swagger.WithResponseModel(gin.H{}), + swagger.WithResponseModel(PluginsResponse{}), ) // Provider template endpoints diff --git a/internal/typ/type.go b/internal/typ/type.go index 98c38fd2b..55054ecdb 100644 --- a/internal/typ/type.go +++ b/internal/typ/type.go @@ -345,8 +345,8 @@ type OAuthDetail = ai.OAuthDetail // Type alias for backward compatibility with common/provider type VModelDetail = ai.VModelDetail -// PluginDetail marks a provider as backed by external plugin code. -type PluginDetail = ai.PluginDetail +// PluginTag marks a provider as backed by external plugin code (see Provider.IsPlugin). +const PluginTag = ai.PluginTag // CredentialBundle holds multi-field credentials for non-bearer auth types // Type alias for backward compatibility with common/provider diff --git a/sdk/python/README.md b/sdk/python/README.md index d003ed0a2..5fa885a7f 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -76,12 +76,17 @@ if __name__ == "__main__": ```bash tingly plugin init my-rag # scaffold module + tingly.toml -tingly plugin run my_rag_plugin.py # serve AND register with tb (ephemeral) +tingly plugin run my_rag_plugin.py # serve AND register with tb ``` -`serve()` (and `tingly plugin run`) **dynamically registers** the plugin with tb -while it runs — leased, heartbeated, and auto-removed on exit. Nothing is -persisted; if the plugin stops, tb's lease expires and routing falls back. +`serve()` (and `tingly plugin run`) registers the plugin with tb once at +startup — an idempotent upsert-by-name, so restarting the plugin updates the +same provider instead of duplicating it. There is no heartbeat or lease: +liveness is handled by tb's existing per-service circuit breaker, the same +mechanism that protects every other provider. If the plugin goes down, the +next failed request trips the breaker and traffic tier-fails-over (when a +fallback tier is configured). Retiring a plugin is the same as retiring any +other provider — delete it in the tb UI. The server is stdlib-only (no FastAPI), supports streaming, and `plugin.llm` calls back into tb so the plugin reuses the gateway for its own LLM work. diff --git a/sdk/python/examples/e2e_plugin.py b/sdk/python/examples/e2e_plugin.py index ced25f593..a83e8096d 100644 --- a/sdk/python/examples/e2e_plugin.py +++ b/sdk/python/examples/e2e_plugin.py @@ -32,5 +32,4 @@ def handle(req): if __name__ == "__main__": - # Short lease so the e2e can show auto-removal on death without a long wait. - plugin.serve(port=8765, ttl_seconds=4) + plugin.serve(port=8765) diff --git a/sdk/python/examples/e2e_run.sh b/sdk/python/examples/e2e_run.sh index 83c194ddc..bbfb05012 100755 --- a/sdk/python/examples/e2e_run.sh +++ b/sdk/python/examples/e2e_run.sh @@ -56,8 +56,8 @@ curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ \"services\":[{\"provider\":\"$VUUID\",\"model\":\"echo-model\",\"weight\":1,\"active\":true}]}" \ | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'), d.get('data',{}).get('uuid',''))" -echo "== 4. start the plugin — it DYNAMICALLY self-registers with tb ==" -echo " (serve(register=True) → POST /plugins/register + heartbeat; nothing persisted)" +echo "== 4. start the plugin — it registers with tb once on startup ==" +echo " (serve() → POST /api/v2/plugins, an idempotent upsert-by-name; no heartbeat)" TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ python3 "$SDK/examples/e2e_plugin.py" >/tmp/plugin_e2e.log 2>&1 & PLUG_PID=$! @@ -67,7 +67,7 @@ for i in $(seq 1 40); do done curl -sf "http://127.0.0.1:8765/health" >/dev/null || { echo "plugin did not start"; cat /tmp/plugin_e2e.log; exit 1; } -echo "== 5. tb sees the LIVE ephemeral instance (GET /api/v2/plugins) ==" +echo "== 5. tb sees the plugin provider (GET /api/v2/plugins) ==" for i in $(seq 1 20); do LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") echo "$LIST" | grep -q 'rag-demo' && break @@ -84,24 +84,33 @@ curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d echo "== plugin log tail ==" tail -6 /tmp/plugin_e2e.log -echo "== 7. EPHEMERAL: kill the plugin → tb auto-removes it when the lease lapses ==" -echo " (hard SIGKILL = simulate a crash; no graceful deregister, nothing persisted)" +echo "== 7. NO SEPARATE LIFECYCLE: kill the plugin (simulated crash) ==" +echo " (hard SIGKILL, no graceful shutdown — the provider is a normal DB row," +echo " same as any other provider, so it does NOT disappear from the list)" kill -KILL "$PLUG_PID" 2>/dev/null PLUG_PID="" -echo " waiting for the 4s lease to expire..." -for i in $(seq 1 20); do - LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") - echo "$LIST" | grep -q 'rag-demo' || break - sleep 0.5 -done -echo " GET /api/v2/plugins now: $LIST" +LIST=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins") +echo " GET /api/v2/plugins still shows it: $LIST" -echo "== 8. client call after the plugin is gone → instance no longer routable ==" -echo " (the durable rule's only service is the dead plugin; a real setup would" -echo " keep a tier-1 real model and tier-failover here)" +echo "== 8. client call after the plugin is dead ==" +echo " (liveness is the SAME per-service circuit breaker every provider gets;" +echo " with no fallback tier configured on this rule the request just errors —" +echo " add a tier-1 real model to the rule and this would tier-failover instead)" curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ "model":"plugin/rag-demo", "messages":[{"role":"user","content":"still there?"}]}' \ | python3 -c "import sys,json; d=json.load(sys.stdin); e=d.get('error', d); print(' ->', (json.dumps(e) if isinstance(e,dict) else str(e))[:200])" +echo "== 9. restart the plugin → re-register upserts the SAME provider (no duplicate) ==" +TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ + python3 "$SDK/examples/e2e_plugin.py" >/tmp/plugin_e2e_2.log 2>&1 & +PLUG_PID=$! +for i in $(seq 1 40); do + curl -sf "http://127.0.0.1:8765/health" >/dev/null 2>&1 && break + sleep 0.3 +done +COUNT=$(curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(sum(1 for p in d['data'] if p['name']=='rag-demo'))") +echo " plugin providers named rag-demo after restart: $COUNT (expect 1)" + echo "== done ==" diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py index 3256cfffa..f61e350c9 100644 --- a/sdk/python/examples/rag_plugin.py +++ b/sdk/python/examples/rag_plugin.py @@ -1,12 +1,14 @@ """A RAG plugin served as an OpenAI-compatible upstream for tingly-box. -Run it (serves on :8765 AND self-registers with tb while running): +Run it (serves on :8765 AND registers with tb on startup): pip install -e . # from sdk/python python examples/rag_plugin.py -Registration is dynamic/ephemeral: the plugin leases a spot in tb, heartbeats to -keep it, and deregisters on exit. Nothing is persisted. +Registration is a one-shot, idempotent upsert by name — tb creates or updates +this plugin's provider (and the rule, since `scenario` is set below). There is +no heartbeat or lease; liveness is handled by tb's existing per-service circuit +breaker like any other provider. Now `model="plugin/rag-demo"` from Claude Code, Cursor, the tb UI, or another `tingly.connect()` experiment routes here — with tb's guard rails, quota, diff --git a/sdk/python/tests/test_plugin_register.py b/sdk/python/tests/test_plugin_register.py new file mode 100644 index 000000000..9505875a6 --- /dev/null +++ b/sdk/python/tests/test_plugin_register.py @@ -0,0 +1,113 @@ +"""Active config + plugin registration tests.""" + +import httpx +import pytest +import respx + +import tingly +import tingly.config as cfg +from tingly.plugin import register as plugin_register + +BASE = "http://tb.test:12580" + + +@pytest.fixture(autouse=True) +def _reset_override(): + cfg._OVERRIDE = None + yield + cfg._OVERRIDE = None + + +def test_configure_takes_precedence(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, "http://env:1") + monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") + tingly.configure(url="http://configured:9", admin_token="cfg-token") + r = cfg.resolve() + assert r.base_url == "http://configured:9" + assert r.token == "cfg-token" + assert r.source == "configure" + + +def test_connection_token_by_env_reference(monkeypatch): + monkeypatch.setenv("MY_TB_SECRET", "secret-123") + conn = tingly.Connection(url="http://x", admin_token_env="MY_TB_SECRET") + assert conn.token() == "secret-123" + + +@respx.mock +def test_register_binds_rule(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, BASE) + monkeypatch.setenv(cfg.ENV_TOKEN, "admin") + + route = respx.post(f"{BASE}/api/v2/plugins").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": { + "provider_uuid": "uuid-1", "model_id": "plugin/x", + "scenario": "experiment", "rule_uuid": "rule-1", + "ready": True, "note": "Plugin wired in.", + }, + }) + ) + + result = plugin_register.register( + "x", "http://127.0.0.1:8765/v1", "plugin/x", scenario="experiment" + ) + assert route.called + assert result.provider_uuid == "uuid-1" + assert result.rule_uuid == "rule-1" + assert result.ready is True + + +@respx.mock +def test_serve_registers_once(monkeypatch): + monkeypatch.setenv(cfg.ENV_URL, BASE) + monkeypatch.setenv(cfg.ENV_TOKEN, "admin") + route = respx.post(f"{BASE}/api/v2/plugins").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": {"provider_uuid": "pid", "model_id": "plugin/srv", + "scenario": "experiment", "ready": True}, + }) + ) + + from tingly import Plugin + + plugin = Plugin(name="srv", scenario="experiment") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False) + try: + assert isinstance(port, int) and port > 0 + assert route.called # registered exactly once on serve() + + # real plugin socket still serves (use urllib so respx doesn't intercept it) + import urllib.request + + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) as r: + assert r.status == 200 + finally: + plugin.stop() + + # stop() only tears down the HTTP server — nothing to deregister, the + # provider stays configured in tb (same as any other provider). + assert route.call_count == 1 + + +def test_serve_register_false_skips(monkeypatch): + from tingly import Plugin + + plugin = Plugin(name="noreg") + + @plugin.chat + def handle(req): + return "ok" + + port = plugin.serve(port=0, verbose=False, block=False, register=False) + try: + assert isinstance(port, int) and port > 0 + finally: + plugin.stop() diff --git a/sdk/python/tests/test_runtime.py b/sdk/python/tests/test_runtime.py deleted file mode 100644 index b1bac8008..000000000 --- a/sdk/python/tests/test_runtime.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Active config + dynamic registration tests.""" - -import threading - -import httpx -import pytest -import respx - -import tingly -import tingly.config as cfg -from tingly.plugin import runtime - -BASE = "http://tb.test:12580" - - -@pytest.fixture(autouse=True) -def _reset_override(): - cfg._OVERRIDE = None - yield - cfg._OVERRIDE = None - - -def test_configure_takes_precedence(monkeypatch): - monkeypatch.setenv(cfg.ENV_URL, "http://env:1") - monkeypatch.setenv(cfg.ENV_TOKEN, "env-token") - tingly.configure(url="http://configured:9", admin_token="cfg-token") - r = cfg.resolve() - assert r.base_url == "http://configured:9" - assert r.token == "cfg-token" - assert r.source == "configure" - - -def test_connection_token_by_env_reference(monkeypatch): - monkeypatch.setenv("MY_TB_SECRET", "secret-123") - conn = tingly.Connection(url="http://x", admin_token_env="MY_TB_SECRET") - assert conn.token() == "secret-123" - - -@respx.mock -def test_runtime_register_heartbeat_deregister(monkeypatch): - monkeypatch.setenv(cfg.ENV_URL, BASE) - monkeypatch.setenv(cfg.ENV_TOKEN, "admin") - - reg_route = respx.post(f"{BASE}/api/v2/plugins/register").mock( - return_value=httpx.Response(200, json={ - "success": True, - "data": { - "plugin_id": "pid-1", "lease_id": "lease-1", "model_id": "plugin/x", - "scenario": "experiment", "rule_uuid": "rule-1", "ttl_seconds": 30, - }, - }) - ) - hb_route = respx.post(f"{BASE}/api/v2/plugins/heartbeat").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - dr_route = respx.post(f"{BASE}/api/v2/plugins/deregister").mock( - return_value=httpx.Response(200, json={"success": True, "data": {"removed": True}}) - ) - - lease = runtime.register("x", "http://127.0.0.1:8765/v1", "plugin/x", scenario="experiment") - assert reg_route.called - assert lease.lease_id == "lease-1" - assert lease.rule_uuid == "rule-1" - - assert runtime.heartbeat(lease) is True - assert hb_route.called - - runtime.deregister(lease) - assert dr_route.called - - -@respx.mock -def test_serve_registers_and_deregisters(monkeypatch): - monkeypatch.setenv(cfg.ENV_URL, BASE) - monkeypatch.setenv(cfg.ENV_TOKEN, "admin") - respx.post(f"{BASE}/api/v2/plugins/register").mock( - return_value=httpx.Response(200, json={ - "success": True, - "data": {"plugin_id": "pid", "lease_id": "L", "model_id": "plugin/srv", - "scenario": "experiment", "ttl_seconds": 30}, - }) - ) - dr = respx.post(f"{BASE}/api/v2/plugins/deregister").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - from tingly import Plugin - - plugin = Plugin(name="srv", scenario="experiment") - - @plugin.chat - def handle(req): - return "ok" - - # ttl high so the heartbeat thread doesn't fire during the test - port = plugin.serve(port=0, verbose=False, block=False, ttl_seconds=300) - assert isinstance(port, int) and port > 0 - assert plugin._lease is not None and plugin._lease.lease_id == "L" - - # real plugin socket still serves (use urllib so respx doesn't intercept it) - import urllib.request - - with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) as r: - assert r.status == 200 - - plugin.stop() - assert dr.called # deregistered on shutdown - assert plugin._lease is None - - -def test_serve_register_false_skips(monkeypatch): - from tingly import Plugin - - plugin = Plugin(name="noreg") - - @plugin.chat - def handle(req): - return "ok" - - port = plugin.serve(port=0, verbose=False, block=False, register=False) - try: - assert plugin._lease is None - finally: - plugin.stop() diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py index 08538cbfa..d574c93ca 100644 --- a/sdk/python/tingly/cli.py +++ b/sdk/python/tingly/cli.py @@ -176,8 +176,8 @@ def main(argv: Optional[list] = None) -> int: psub = p_plugin.add_subparsers(dest="plugin_command") p_init = psub.add_parser("init", help="scaffold a starter plugin") p_init.add_argument("name", help="plugin name, e.g. my-rag") - # `run` serves the plugin AND self-registers with tb (heartbeat + deregister - # on exit), so there is no separate one-shot register command. + # `run` serves the plugin AND registers it with tb (idempotent upsert on + # start), so there is no separate one-shot register command. p_run = psub.add_parser("run", help="serve a plugin and register it with tb") p_run.add_argument("target", help="e.g. my_rag_plugin:plugin or my_rag_plugin.py") diff --git a/sdk/python/tingly/plugin/core.py b/sdk/python/tingly/plugin/core.py index 6111287f1..5847c8f83 100644 --- a/sdk/python/tingly/plugin/core.py +++ b/sdk/python/tingly/plugin/core.py @@ -26,12 +26,15 @@ def handle(req): from __future__ import annotations import threading -from typing import Callable, Optional +from typing import TYPE_CHECKING, Callable, Optional from .manifest import Manifest from .server import Dispatch, HandlerResult, make_server from .types import ChatRequest +if TYPE_CHECKING: + from ..config import Connection + ChatHandler = Callable[[ChatRequest], HandlerResult] @@ -58,8 +61,6 @@ def __init__( self._handler: Optional[ChatHandler] = None self._clients: dict = {} # scenario -> lazily-connected client self._httpd = None - self._lease = None # runtime.Lease when dynamically registered - self._heartbeater = None # -- authoring ------------------------------------------------------- @@ -135,14 +136,20 @@ def serve( block: bool = True, register: bool = True, advertise_host: Optional[str] = None, - ttl_seconds: int = 30, - tb: Optional[Any] = None, + tb: Optional["Connection"] = None, ) -> int: """Run the plugin's HTTP server and (by default) register it with tb. - Dynamic registration is ephemeral: the plugin appears in tb only while it - runs — a background heartbeat keeps the lease, and it deregisters on - shutdown. ``tb`` may be a :class:`tingly.config.Connection` to point at a + Registration is a one-shot, idempotent upsert by name: tb creates or + updates a normal provider for this plugin (and the rule, if + ``scenario`` was set on the constructor). There is no heartbeat or + lease — once the plugin is registered it stays configured in tb until + someone deletes it, the same as any other provider. Liveness is + handled by tb's existing per-service circuit breaker: if the plugin + goes down, the next failed request trips it and traffic tier-fails- + over (when a fallback tier is configured). + + ``tb`` may be a :class:`tingly.config.Connection` to point at a specific gateway / inject credentials (containers / CI / remote). Returns the bound port (resolved even when ``port=0``). ``block=False`` @@ -169,7 +176,7 @@ def serve( ) if register: - self._register(advertise_host or host, bound, ttl_seconds, verbose) + self._register(advertise_host or host, bound, verbose) if not block: t = threading.Thread(target=httpd.serve_forever, daemon=True) @@ -183,37 +190,23 @@ def serve( self.stop() return bound - def _register(self, host: str, port: int, ttl_seconds: int, verbose: bool) -> None: - from . import runtime + def _register(self, host: str, port: int, verbose: bool) -> None: + from .register import register endpoint = f"http://{host}:{port}/v1" try: - lease = runtime.register( + result = register( self.name, endpoint, self.model_id, - scenario=self.scenario, token=self.api_key, ttl_seconds=ttl_seconds, + scenario=self.scenario, token=self.api_key, ) except Exception as exc: # noqa: BLE001 - registration is best-effort if verbose: print(f"[tingly] plugin registration skipped: {exc}") return - self._lease = lease - self._heartbeater = runtime.Heartbeater(lease).start() if verbose: - print( - f"[tingly] registered '{self.name}' as model {lease.model_id!r}" - + (f" under scenario {lease.scenario!r}" if lease.scenario else "") - + f" (lease ttl={lease.ttl_seconds}s)" - ) + print(f"[tingly] {result.note}") def stop(self) -> None: - if self._heartbeater is not None: - self._heartbeater.stop() - self._heartbeater = None - if self._lease is not None: - from . import runtime - - runtime.deregister(self._lease) - self._lease = None if self._httpd is not None: self._httpd.shutdown() self._httpd = None diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py new file mode 100644 index 000000000..b78862548 --- /dev/null +++ b/sdk/python/tingly/plugin/register.py @@ -0,0 +1,73 @@ +"""Register a running plugin with tingly-box. + +A plugin registers once at startup (idempotent upsert by name — calling it +again, e.g. on every restart, updates the existing provider rather than +duplicating it). There is no heartbeat or lease: liveness is handled by the +same per-service circuit breaker that already protects every other tb +provider — if the plugin goes down, the next failed request trips the breaker +and traffic tier-fails-over (when a fallback tier is configured). If a plugin +is retired, delete its provider like any other, in the tb UI. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import httpx + +from .. import config as _config +from .._http import safe_json +from ..errors import AuthError, GatewayUnreachableError + + +@dataclass +class RegisterResult: + provider_uuid: str + model_id: str + scenario: Optional[str] + rule_uuid: Optional[str] + ready: bool + note: str + + +def register( + name: str, + endpoint: str, + model_id: str, + *, + scenario: Optional[str] = None, + token: str = "", + tier: int = 0, + gateway_url: Optional[str] = None, + admin_token: Optional[str] = None, + timeout: float = 30.0, +) -> RegisterResult: + """Register (or update) this plugin as a tb provider, optionally binding a rule.""" + resolved = _config.resolve(base_url=gateway_url, token=admin_token) + headers = {"Authorization": f"Bearer {resolved.token or ''}"} + url = resolved.base_url.rstrip("/") + "/api/v2/plugins" + body = { + "name": name, "endpoint": endpoint, "model_id": model_id, + "scenario": scenario or "", "token": token, "tier": tier, + } + try: + resp = httpx.post(url, json=body, headers=headers, timeout=timeout) + except httpx.HTTPError as exc: + raise GatewayUnreachableError(f"could not reach tingly-box: {exc}") from exc + if resp.status_code == 401: + raise AuthError("tingly-box rejected the admin token during plugin register") + payload = safe_json(resp) or {} + if resp.status_code != 200 or not payload.get("success"): + raise GatewayUnreachableError( + f"plugin register failed: HTTP {resp.status_code} {resp.text[:200]}" + ) + d = payload.get("data") or {} + return RegisterResult( + provider_uuid=d.get("provider_uuid", ""), + model_id=d.get("model_id", model_id), + scenario=d.get("scenario") or None, + rule_uuid=d.get("rule_uuid") or None, + ready=bool(d.get("ready", False)), + note=d.get("note", ""), + ) diff --git a/sdk/python/tingly/plugin/runtime.py b/sdk/python/tingly/plugin/runtime.py deleted file mode 100644 index 7b7c90462..000000000 --- a/sdk/python/tingly/plugin/runtime.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Dynamic (ephemeral) plugin registration with tingly-box. - -A plugin registers a live instance, keeps it alive with a heartbeat, and -deregisters on shutdown — so it appears in tb only while it runs. Nothing is -persisted; if the plugin dies, tb's lease expires and routing falls back (tier -failover) to a real model. -""" - -from __future__ import annotations - -import threading -from dataclasses import dataclass -from typing import Optional - -import httpx - -from .. import config as _config -from .._http import safe_json -from ..errors import AuthError, GatewayUnreachableError - - -@dataclass -class Lease: - gateway_url: str - admin_token: str - plugin_id: str - lease_id: str - model_id: str - scenario: Optional[str] - rule_uuid: Optional[str] - ttl_seconds: int - - -def register( - name: str, - endpoint: str, - model_id: str, - *, - scenario: Optional[str] = None, - token: str = "", - tier: int = 0, - ttl_seconds: int = 30, - gateway_url: Optional[str] = None, - admin_token: Optional[str] = None, - timeout: float = 30.0, -) -> Lease: - """Register a live ephemeral plugin instance; returns a renewable lease.""" - resolved = _config.resolve(base_url=gateway_url, token=admin_token) - headers = {"Authorization": f"Bearer {resolved.token or ''}"} - url = resolved.base_url.rstrip("/") + "/api/v2/plugins/register" - body = { - "name": name, "endpoint": endpoint, "model_id": model_id, - "scenario": scenario or "", "token": token, "tier": tier, - "ttl_seconds": ttl_seconds, - } - try: - resp = httpx.post(url, json=body, headers=headers, timeout=timeout) - except httpx.HTTPError as exc: - raise GatewayUnreachableError(f"could not reach tingly-box: {exc}") from exc - if resp.status_code == 401: - raise AuthError("tingly-box rejected the admin token during plugin register") - payload = safe_json(resp) or {} - if resp.status_code != 200 or not payload.get("success"): - raise GatewayUnreachableError( - f"plugin register failed: HTTP {resp.status_code} {resp.text[:200]}" - ) - d = payload.get("data") or {} - return Lease( - gateway_url=resolved.base_url.rstrip("/"), - admin_token=resolved.token or "", - plugin_id=d.get("plugin_id", ""), - lease_id=d.get("lease_id", ""), - model_id=d.get("model_id", model_id), - scenario=d.get("scenario") or None, - rule_uuid=d.get("rule_uuid") or None, - ttl_seconds=int(d.get("ttl_seconds", ttl_seconds)), - ) - - -def heartbeat(lease: Lease, timeout: float = 10.0) -> bool: - """Extend the lease. Returns False if tb no longer knows it (re-register).""" - url = lease.gateway_url + "/api/v2/plugins/heartbeat" - headers = {"Authorization": f"Bearer {lease.admin_token}"} - try: - resp = httpx.post( - url, json={"lease_id": lease.lease_id, "ttl_seconds": lease.ttl_seconds}, - headers=headers, timeout=timeout, - ) - except httpx.HTTPError: - return False - return resp.status_code == 200 - - -def deregister(lease: Lease, timeout: float = 10.0) -> None: - """Remove the live instance now (best-effort).""" - url = lease.gateway_url + "/api/v2/plugins/deregister" - headers = {"Authorization": f"Bearer {lease.admin_token}"} - try: - httpx.post(url, json={"lease_id": lease.lease_id}, headers=headers, timeout=timeout) - except httpx.HTTPError: - pass - - -class Heartbeater: - """Background thread that renews a lease until stopped.""" - - def __init__(self, lease: Lease, interval: Optional[float] = None): - self._lease = lease - # Renew well within the TTL (default: a third of it, min 1s). - self._interval = interval or max(1.0, lease.ttl_seconds / 3.0) - self._stop = threading.Event() - self._thread: Optional[threading.Thread] = None - - def start(self) -> "Heartbeater": - self._thread = threading.Thread(target=self._loop, daemon=True) - self._thread.start() - return self - - def _loop(self) -> None: - while not self._stop.wait(self._interval): - heartbeat(self._lease) - - def stop(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) From 5d272b424f56ed604192d2afdd8e1f29e3fb8b2f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 05:27:11 +0000 Subject: [PATCH 16/28] refactor(plugins): extract into internal/server/module/plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plugin registration was living as methods directly on *Server (internal/server/plugin_provider.go), coupling it into the same file/struct as every other core server concern — even though its only dependency is *config.Config (no other Server field is touched). That's a real footgun: as the Server struct grows, a "just an API surface" concern like this drifts further from being independently reasoned about, tested, or reused. Move it into its own module, matching the pattern every other server concern already uses (module/provider, module/rule, module/providertemplate, ...): - internal/server/module/plugin/handler.go — Handler{config}, NewHandler(cfg), RegisterPlugin/ListPlugins + the upsert/ensure-rule helpers (unchanged logic). - types.go — request/response types (unchanged). - routes.go — RegisterRoutes(group, handler), mounted from server_webui_api.go via pluginmodule.NewHandler(s.config) + pluginmodule.RegisterRoutes(apiV2, pluginHandler). Zero behavior change: openapi.json is byte-identical before/after regen (same routes, same shapes). Tests moved 1:1 into the module (still 5/5 passing) plus a fresh full go test sweep and a real end-to-end run against the rebuilt tb binary (examples/e2e_run.sh, all 9 steps) to confirm the relocation didn't break anything at runtime. --- .design/python-sdk.md | 32 +++++-- .../plugin/handler.go} | 92 +++++++------------ .../plugin/handler_test.go} | 46 +++++----- internal/server/module/plugin/routes.go | 23 +++++ internal/server/module/plugin/types.go | 38 ++++++++ internal/server/server_webui_api.go | 22 ++--- 6 files changed, 148 insertions(+), 105 deletions(-) rename internal/server/{plugin_provider.go => module/plugin/handler.go} (55%) rename internal/server/{plugin_provider_test.go => module/plugin/handler_test.go} (82%) create mode 100644 internal/server/module/plugin/routes.go create mode 100644 internal/server/module/plugin/types.go diff --git a/.design/python-sdk.md b/.design/python-sdk.md index f248ef9b0..ea24123f5 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -59,24 +59,38 @@ register. They are an implementation tour, not three separate products. ### tb-side: a plugin is a normal, tagged provider (implemented) -**Design history, briefly, because it's instructive.** Two earlier iterations -over-built this: first a persisted "plugin provider kind" with its own DB -column and a distinct registration endpoint; then a full ephemeral service- -discovery layer (in-memory registry, per-instance lease, heartbeat thread, -TTL expiry, a `Config` hook consulted on every provider lookup) built to avoid -leaving a stale DB row behind when a plugin process stopped. Both were removed. -The reason: **tb already has liveness detection** — every `(rule, service)` +**Design history, briefly, because it's instructive.** Three earlier +iterations over-built this: first a persisted "plugin provider kind" with its +own DB column and a distinct registration endpoint; then a full ephemeral +service-discovery layer (in-memory registry, per-instance lease, heartbeat +thread, TTL expiry, a `Config` hook consulted on every provider lookup) built +to avoid leaving a stale DB row behind when a plugin process stopped; then, +even after that was cut down to an idempotent upsert, the handler methods +still lived directly on `*Server` in `internal/server/*.go`, coupling plugin +registration — a self-contained concern whose only dependency is `*config.Config` +— into the same file/struct as every other server concern. All three were +fixed. The circuit-breaker point: + +**tb already has liveness detection** — every `(rule, service)` pair is covered by the existing per-service circuit breaker (`internal/loadbalance/breaker.go`). A dead plugin's first failed request trips it exactly like a dead real provider; traffic tier-fails-over automatically when a fallback tier is configured. Lease/heartbeat/TTL was reinventing that mechanism — distributed-service-discovery machinery for a problem tb doesn't have (a personal, single-operator box, not a multi-tenant cluster). See -`git log` on this file's directory for the two removed designs if useful as a +`git log` on this file's directory for the removed designs if useful as a cautionary reference. -**What shipped instead — the minimal version:** +**What shipped instead — the minimal version, in its own module:** +- Lives in `internal/server/module/plugin/` (`handler.go` / `types.go` / + `routes.go`), matching the same pattern every other server concern already + uses (`module/provider`, `module/rule`, `module/providertemplate`, …): + a `Handler` struct constructed with only the dependencies it needs + (`NewHandler(cfg *config.Config)` — nothing else, since registration is just + provider + rule creation), and a `RegisterRoutes(group, handler)` mounted + from `server_webui_api.go`. Plugin logic no longer lives as methods on the + giant `*Server` struct. - A plugin is an ordinary provider (`APIStyle=openai`, `api_key`/`no_key`) carrying the tag `"plugin"` in the existing, generic `Provider.Tags` field. `Provider.IsPlugin()` checks for that tag. No new struct, no new DB column — diff --git a/internal/server/plugin_provider.go b/internal/server/module/plugin/handler.go similarity index 55% rename from internal/server/plugin_provider.go rename to internal/server/module/plugin/handler.go index 6c5f8a5b0..7fde16b3f 100644 --- a/internal/server/plugin_provider.go +++ b/internal/server/module/plugin/handler.go @@ -1,4 +1,12 @@ -package server +// Package plugin handles HTTP endpoints for registering external plugin code +// as a tingly-box upstream. A plugin is an ordinary OpenAI-compatible HTTP +// provider tagged "plugin" (see typ.Provider.IsPlugin) — routing is +// unchanged, and liveness is handled by the same per-service circuit breaker +// that already protects every other provider. There is deliberately no +// separate plugin lifecycle (lease/heartbeat/expiry): that would duplicate +// the breaker for a single-operator box. If a plugin is retired, delete its +// provider like any other, via the provider module's DELETE endpoint. +package plugin import ( "net/http" @@ -12,55 +20,21 @@ import ( "github.com/tingly-dev/tingly-box/internal/typ" ) -// RegisterPluginRequest registers external plugin code as a tingly-box upstream. -// It is idempotent by name: calling it again (e.g. every time the plugin -// process starts) updates the existing provider instead of duplicating it. -type RegisterPluginRequest struct { - Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` - Endpoint string `json:"endpoint" binding:"required" description:"Plugin OpenAI base URL" example:"http://127.0.0.1:8765/v1"` - ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` - Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` - Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` - Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` -} - -// RegisterPluginResponse reports what was created or updated. -type RegisterPluginResponse struct { - ProviderUUID string `json:"provider_uuid"` - ModelID string `json:"model_id"` - Scenario string `json:"scenario,omitempty"` - RuleUUID string `json:"rule_uuid,omitempty"` - // Ready is true when a rule is bound, so clients can select the model now. - Ready bool `json:"ready"` - Note string `json:"note,omitempty"` +// Handler handles plugin registration HTTP requests. Its only dependency is +// the shared config — plugin registration is just provider + rule creation, +// so it needs nothing else from the server. +type Handler struct { + config *config.Config } -// PluginInfo is a list view of a plugin provider. -type PluginInfo struct { - UUID string `json:"uuid"` - Name string `json:"name"` - Endpoint string `json:"endpoint"` - ModelID string `json:"model_id,omitempty"` -} - -// PluginsResponse wraps the plugin list. -type PluginsResponse struct { - Success bool `json:"success"` - Data []PluginInfo `json:"data"` +// NewHandler creates a plugin Handler. +func NewHandler(cfg *config.Config) *Handler { + return &Handler{config: cfg} } // RegisterPlugin creates or updates a plugin-tagged provider (and optionally // binds a rule to it) so "configure this rule with a plugin" is one call. -// -// A plugin provider is an ordinary OpenAI HTTP upstream — routing is -// unchanged, and liveness is handled by the same per-service circuit breaker -// that already protects every other provider: if the plugin process is down, -// the first failed request trips the breaker and traffic tier-fails-over -// (when a fallback tier is configured). There is deliberately no separate -// registration lifecycle (lease/heartbeat/expiry) for plugins — that would -// duplicate the breaker for a single-operator box. If a plugin is retired, -// delete its provider like any other (DELETE /api/v2/providers/:uuid). -func (s *Server) RegisterPlugin(c *gin.Context) { +func (h *Handler) RegisterPlugin(c *gin.Context) { var req RegisterPluginRequest if err := c.ShouldBindJSON(&req); err != nil { c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) @@ -72,7 +46,7 @@ func (s *Server) RegisterPlugin(c *gin.Context) { modelID = "plugin/" + req.Name } - provider, err := s.upsertPluginProvider(req.Name, req.Endpoint, req.Token) + provider, err := h.upsertPluginProvider(req.Name, req.Endpoint, req.Token) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "success": false, @@ -89,7 +63,7 @@ func (s *Server) RegisterPlugin(c *gin.Context) { // One-step bind: ensure the rule whose single service is this plugin. if req.Scenario != "" { - ruleUUID, err := s.ensurePluginRule(req.Scenario, modelID, provider.UUID, req.Name, req.Tier) + ruleUUID, err := h.ensurePluginRule(req.Scenario, modelID, provider.UUID, req.Name, req.Tier) if err != nil { resp.Note = "Provider registered, but rule binding failed: " + err.Error() c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) @@ -115,13 +89,13 @@ func (s *Server) RegisterPlugin(c *gin.Context) { // upsertPluginProvider creates a plugin-tagged provider, or updates the // endpoint/token of an existing one with the same name. Idempotent by name so // a plugin can safely re-register (e.g. on every process start). -func (s *Server) upsertPluginProvider(name, endpoint, token string) (*typ.Provider, error) { - if existing, err := s.config.GetProviderByName(name); err == nil && existing.IsPlugin() { +func (h *Handler) upsertPluginProvider(name, endpoint, token string) (*typ.Provider, error) { + if existing, err := h.config.GetProviderByName(name); err == nil && existing.IsPlugin() { existing.APIBase = endpoint existing.Token = token existing.NoKeyRequired = token == "" existing.Enabled = true - if err := s.config.UpdateProvider(existing.UUID, existing); err != nil { + if err := h.config.UpdateProvider(existing.UUID, existing); err != nil { return nil, err } return existing, nil @@ -139,7 +113,7 @@ func (s *Server) upsertPluginProvider(name, endpoint, token string) (*typ.Provid Timeout: constant.DefaultRequestTimeout, Tags: []string{typ.PluginTag}, } - if err := s.config.AddProvider(provider); err != nil { + if err := h.config.AddProvider(provider); err != nil { return nil, err } return provider, nil @@ -147,9 +121,9 @@ func (s *Server) upsertPluginProvider(name, endpoint, token string) (*typ.Provid // ListPlugins returns the plugin-tagged providers, with the model id(s) each // currently routes (derived from the rules bound to it) for display. -func (s *Server) ListPlugins(c *gin.Context) { +func (h *Handler) ListPlugins(c *gin.Context) { modelsByProvider := map[string]string{} - for _, rule := range s.config.GetRequestConfigs() { + for _, rule := range h.config.GetRequestConfigs() { for _, svc := range rule.Services { if svc == nil { continue @@ -161,7 +135,7 @@ func (s *Server) ListPlugins(c *gin.Context) { } plugins := []PluginInfo{} - for _, p := range s.config.ListProviders() { + for _, p := range h.config.ListProviders() { if !p.IsPlugin() { continue } @@ -177,12 +151,12 @@ func (s *Server) ListPlugins(c *gin.Context) { // ensurePluginRule idempotently ensures a rule exists under scenario whose single // tier-service points at the given provider id for modelID. Returns the rule UUID. -func (s *Server) ensurePluginRule(scenario, modelID, providerID, name string, tier int) (string, error) { +func (h *Handler) ensurePluginRule(scenario, modelID, providerID, name string, tier int) (string, error) { scn := typ.RuleScenario(scenario) if !typ.CanBindRulesToScenario(scn) { - return "", &pluginBindError{"scenario " + scenario + " is not bindable"} + return "", &bindError{"scenario " + scenario + " is not bindable"} } - for _, rule := range s.config.GetRequestConfigs() { + for _, rule := range h.config.GetRequestConfigs() { if rule.GetScenario() == scn && rule.RequestModel == modelID { return rule.UUID, nil // already bound (idempotent) } @@ -198,12 +172,12 @@ func (s *Server) ensurePluginRule(scenario, modelID, providerID, name string, ti {Provider: providerID, Model: modelID, Weight: 1, Active: true, Tier: tier}, }, } - if err := s.config.AddRule(rule); err != nil { + if err := h.config.AddRule(rule); err != nil { return "", err } return rule.UUID, nil } -type pluginBindError struct{ msg string } +type bindError struct{ msg string } -func (e *pluginBindError) Error() string { return e.msg } +func (e *bindError) Error() string { return e.msg } diff --git a/internal/server/plugin_provider_test.go b/internal/server/module/plugin/handler_test.go similarity index 82% rename from internal/server/plugin_provider_test.go rename to internal/server/module/plugin/handler_test.go index 725dc6bff..c855cd2b5 100644 --- a/internal/server/plugin_provider_test.go +++ b/internal/server/module/plugin/handler_test.go @@ -1,4 +1,4 @@ -package server +package plugin import ( "bytes" @@ -14,7 +14,7 @@ import ( ) // postJSON drives a gin handler with a JSON body and returns the recorder and -// the parsed response envelope. Shared by the plugin endpoint tests. +// the parsed response envelope. func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseRecorder, map[string]any) { t.Helper() gin.SetMode(gin.TestMode) @@ -29,19 +29,19 @@ func postJSON(t *testing.T, h gin.HandlerFunc, body any) (*httptest.ResponseReco return w, parsed } -func newPluginTestServer(t *testing.T) *Server { +func newTestHandler(t *testing.T) *Handler { t.Helper() cfg, err := config.NewConfig(config.WithConfigDir(t.TempDir())) if err != nil { t.Fatalf("NewConfig: %v", err) } - return &Server{config: cfg} + return NewHandler(cfg) } func TestRegisterPlugin_BindsRule(t *testing.T) { - s := newPluginTestServer(t) + h := newTestHandler(t) - w, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + w, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "my-rag", Endpoint: "http://127.0.0.1:8765/v1", ModelID: "plugin/my-rag", @@ -64,7 +64,7 @@ func TestRegisterPlugin_BindsRule(t *testing.T) { } // The provider must be persisted and tagged as a plugin. - prov, err := s.config.GetProviderByUUID(providerUUID) + prov, err := h.config.GetProviderByUUID(providerUUID) if err != nil { t.Fatalf("GetProviderByUUID: %v", err) } @@ -74,7 +74,7 @@ func TestRegisterPlugin_BindsRule(t *testing.T) { // A rule must exist under the scenario whose single service is the plugin. var found bool - for _, rule := range s.config.GetRequestConfigs() { + for _, rule := range h.config.GetRequestConfigs() { if rule.GetScenario() == typ.ScenarioExperiment && rule.RequestModel == "plugin/my-rag" { found = true if len(rule.Services) != 1 || rule.Services[0].Provider != providerUUID { @@ -88,9 +88,9 @@ func TestRegisterPlugin_BindsRule(t *testing.T) { } func TestRegisterPlugin_ProviderOnly(t *testing.T) { - s := newPluginTestServer(t) + h := newTestHandler(t) - _, resp := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + _, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "solo", Endpoint: "http://127.0.0.1:9000/v1", }) @@ -105,15 +105,15 @@ func TestRegisterPlugin_ProviderOnly(t *testing.T) { } func TestRegisterPlugin_ReregisterUpdatesInPlace(t *testing.T) { - s := newPluginTestServer(t) + h := newTestHandler(t) - _, first := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + _, first := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "my-rag", Endpoint: "http://127.0.0.1:8765/v1", }) firstUUID := first["data"].(map[string]any)["provider_uuid"].(string) // Re-register (e.g. the plugin process restarted on a different port). - _, second := postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + _, second := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "my-rag", Endpoint: "http://127.0.0.1:9999/v1", }) secondUUID := second["data"].(map[string]any)["provider_uuid"].(string) @@ -122,7 +122,7 @@ func TestRegisterPlugin_ReregisterUpdatesInPlace(t *testing.T) { t.Fatalf("re-register should update the same provider, got %s then %s", firstUUID, secondUUID) } - prov, err := s.config.GetProviderByUUID(firstUUID) + prov, err := h.config.GetProviderByUUID(firstUUID) if err != nil { t.Fatalf("GetProviderByUUID: %v", err) } @@ -132,7 +132,7 @@ func TestRegisterPlugin_ReregisterUpdatesInPlace(t *testing.T) { // Exactly one provider named my-rag — no duplicate created. count := 0 - for _, p := range s.config.ListProviders() { + for _, p := range h.config.ListProviders() { if p.Name == "my-rag" { count++ } @@ -143,15 +143,15 @@ func TestRegisterPlugin_ReregisterUpdatesInPlace(t *testing.T) { } func TestRegisterPlugin_ReregisterIsIdempotentForRule(t *testing.T) { - s := newPluginTestServer(t) - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + h := newTestHandler(t) + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "p", Endpoint: "http://a/v1", Scenario: string(typ.ScenarioExperiment), }) - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "p", Endpoint: "http://b/v1", Scenario: string(typ.ScenarioExperiment), }) count := 0 - for _, rule := range s.config.GetRequestConfigs() { + for _, rule := range h.config.GetRequestConfigs() { if rule.RequestModel == "plugin/p" { count++ } @@ -162,14 +162,14 @@ func TestRegisterPlugin_ReregisterIsIdempotentForRule(t *testing.T) { } func TestListPlugins_FiltersPluginTag(t *testing.T) { - s := newPluginTestServer(t) + h := newTestHandler(t) // a normal provider - if err := s.config.AddProvider(&typ.Provider{ + if err := h.config.AddProvider(&typ.Provider{ Name: "real", APIBase: "https://api.example.com/v1", APIStyle: "openai", Enabled: true, }); err != nil { t.Fatalf("AddProvider: %v", err) } - postJSON(t, s.RegisterPlugin, RegisterPluginRequest{ + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ Name: "plug", Endpoint: "http://127.0.0.1:8765/v1", ModelID: "plugin/plug", Scenario: string(typ.ScenarioExperiment), }) @@ -177,7 +177,7 @@ func TestListPlugins_FiltersPluginTag(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodGet, "/", nil) - s.ListPlugins(c) + h.ListPlugins(c) var resp PluginsResponse if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { diff --git a/internal/server/module/plugin/routes.go b/internal/server/module/plugin/routes.go new file mode 100644 index 000000000..c78f1b55a --- /dev/null +++ b/internal/server/module/plugin/routes.go @@ -0,0 +1,23 @@ +package plugin + +import ( + "github.com/tingly-dev/tingly-box/swagger" +) + +// RegisterRoutes wires the plugin endpoints onto the given route group. +func RegisterRoutes(api *swagger.RouteGroup, h *Handler) { + // Register (or update) external plugin code as an upstream, optionally + // binding a rule — an idempotent upsert-by-name. + api.POST("/plugins", h.RegisterPlugin, + swagger.WithDescription("Register (or update) external plugin code as an upstream, optionally binding a rule"), + swagger.WithTags("plugins"), + swagger.WithRequestModel(RegisterPluginRequest{}), + swagger.WithResponseModel(RegisterPluginResponse{}), + ) + + api.GET("/plugins", h.ListPlugins, + swagger.WithDescription("List registered plugin providers"), + swagger.WithTags("plugins"), + swagger.WithResponseModel(PluginsResponse{}), + ) +} diff --git a/internal/server/module/plugin/types.go b/internal/server/module/plugin/types.go new file mode 100644 index 000000000..38a5a3888 --- /dev/null +++ b/internal/server/module/plugin/types.go @@ -0,0 +1,38 @@ +package plugin + +// RegisterPluginRequest registers external plugin code as a tingly-box upstream. +// It is idempotent by name: calling it again (e.g. every time the plugin +// process starts) updates the existing provider instead of duplicating it. +type RegisterPluginRequest struct { + Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` + Endpoint string `json:"endpoint" binding:"required" description:"Plugin OpenAI base URL" example:"http://127.0.0.1:8765/v1"` + ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` + Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` + Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` + Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` +} + +// RegisterPluginResponse reports what was created or updated. +type RegisterPluginResponse struct { + ProviderUUID string `json:"provider_uuid"` + ModelID string `json:"model_id"` + Scenario string `json:"scenario,omitempty"` + RuleUUID string `json:"rule_uuid,omitempty"` + // Ready is true when a rule is bound, so clients can select the model now. + Ready bool `json:"ready"` + Note string `json:"note,omitempty"` +} + +// PluginInfo is a list view of a plugin provider. +type PluginInfo struct { + UUID string `json:"uuid"` + Name string `json:"name"` + Endpoint string `json:"endpoint"` + ModelID string `json:"model_id,omitempty"` +} + +// PluginsResponse wraps the plugin list. +type PluginsResponse struct { + Success bool `json:"success"` + Data []PluginInfo `json:"data"` +} diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index dba73512f..6433a897f 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -12,6 +12,7 @@ import ( "github.com/tingly-dev/tingly-box/internal/server/config" "github.com/tingly-dev/tingly-box/internal/server/module/info" "github.com/tingly-dev/tingly-box/internal/server/module/onboarding" + pluginmodule "github.com/tingly-dev/tingly-box/internal/server/module/plugin" probemodule "github.com/tingly-dev/tingly-box/internal/server/module/probe" providermodule "github.com/tingly-dev/tingly-box/internal/server/module/provider" "github.com/tingly-dev/tingly-box/internal/server/module/providertemplate" @@ -401,20 +402,13 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { providermodule.RegisterRoutes(apiV2, providerHandler) // Plugin registration: a plugin is a provider tagged "plugin" (external - // OpenAI upstream). POST is an idempotent upsert-by-name that wires it in - // (provider + optional rule) in one step; liveness is handled by the same - // circuit breaker that covers every other provider — no separate lifecycle. - apiV2.POST("/plugins", s.RegisterPlugin, - swagger.WithDescription("Register (or update) external plugin code as an upstream, optionally binding a rule"), - swagger.WithTags("plugins"), - swagger.WithRequestModel(RegisterPluginRequest{}), - swagger.WithResponseModel(RegisterPluginResponse{}), - ) - apiV2.GET("/plugins", s.ListPlugins, - swagger.WithDescription("List registered plugin providers"), - swagger.WithTags("plugins"), - swagger.WithResponseModel(PluginsResponse{}), - ) + // OpenAI upstream). Independent module — its only dependency is config, so + // it owns its own handler rather than living on *Server. POST is an + // idempotent upsert-by-name that wires it in (provider + optional rule) in + // one step; liveness is handled by the same circuit breaker that covers + // every other provider — no separate lifecycle. + pluginHandler := pluginmodule.NewHandler(s.config) + pluginmodule.RegisterRoutes(apiV2, pluginHandler) // Provider template endpoints providerTemplateHandler := providertemplate.NewHandler(s.templateManager) From 2d398222932f8a118bc771aedd57a094fcd17eea Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 07:45:31 +0000 Subject: [PATCH 17/28] docs(sdk): flag Plugin naming collision, reprioritize follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frontend already uses "Plugins" as a deliberately-unified name for per-rule feature flags (RulePluginsCard/PluginFeatures, see rule-flags.md). The SDK's tingly.Plugin / /api/v2/plugins reuses the same word for an unrelated concept (external code as upstream) — a collision per ux-principles.md #3 that's currently silent only because the lifecycle UI hasn't shipped yet. Flag it now, before it becomes UI copy. Also reorders open follow-ups: the sub-process supervisor and reverse-proxy mount are ordinary backend work and can proceed independently; only the lifecycle UI needs the naming question resolved first. --- .design/python-sdk.md | 50 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index ea24123f5..c7c2a838a 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -282,17 +282,51 @@ users can name parallel experiments via profiles (`experiment:p1`). names (contract with the SDK) and the transport-label logic; `internal/typ/scenario_registry_test.go` pins the experiment descriptor. +## ⚠️ Naming collision to resolve before the lifecycle UI + +The frontend already has a deliberately-unified name **"Plugins"** for a +completely different concept — per-rule feature flags (`smart_compact`, +`vision_proxy_service`, `clean_header`, `session_affinity`, …), surfaced via +`RulePluginsCard` / `FlagCatalogDialog` / `PluginFeatures` (see +`.design/rule-flags.md` §"统一命名：Plugins"). That unification itself +resolved an earlier "Plugin" / "Rule Extensions" mixed-usage collision — this +exact kind of debt has already been paid down once in this codebase. + +The SDK's `tingly.Plugin` / `POST /api/v2/plugins` / the `"plugin"` provider +tag names an unrelated concept — *external code acting as an upstream* — but +reuses the same word. Per `.design/ux-principles.md` §3 ("一个词在产品中只能指 +一件事"), this has to be split before it becomes user-visible. It's silent +today only because there's no lifecycle UI yet (see follow-up 4 below) — the +moment that ships, both meanings of "Plugin" appear in the same product, on +adjacent surfaces of the same rule (its flags card vs. its upstream +binding). Rename one side **now**, while the surface is still an API + +Python class and not yet UI copy — cheaper than renaming after users have +`tingly.toml` manifests and muscle memory. This SDK side is the newer, +smaller-footprint concept, so it's the one that should move; candidates: +`upstream plugin`, `connector`, `extension provider`. Needs a product +decision, not a unilateral rename — flagged here rather than acted on. + ## Open follow-ups -1. Scoped short-lived session tokens (`expires_at` + refresh on 401). -2. Dedicated `GET /api/v1/sdk/usage?session=` so usage doesn't scan +Roughly in priority order — 1–3 are backend/SDK-only (no naming exposure, safe +to build regardless of the rename above); 4 is blocked on that decision for +its UI portion specifically (the supervisor + reverse-proxy mount are not). + +1. Layer 2 tb-side remainder — Python side is **done** (`tingly.Plugin`, + manifest, OpenAI server, `register`); still missing: a sub-process + supervisor that boots plugins from their manifest (reuse + `agentboot/process`), a `/plugins//*` reverse-proxy mount, and the + install/enable/logs/disable lifecycle UI. The first two are ordinary + backend work; the UI is the piece that should wait on the naming decision + above. See the "Layer 2" section below. +2. Scoped short-lived session tokens (`expires_at` + refresh on 401). +3. Dedicated `GET /api/v1/sdk/usage?session=` so usage doesn't scan `/api/v1/requests`. -3. Async client (`AsyncClient`, `aask`) — transports already have async builders. -4. Layer 2 — Python side **done** (`tingly.Plugin`, manifest, OpenAI server, - `register`); remaining tb-side: sub-process supervisor from the manifest - (reuse `agentboot/process`), `/plugins//*` reverse proxy, lifecycle UI. - See the "Layer 2" section below. -5. Layer 3: expose a plugin as a model tb can route to (see "Layer 3" below). +4. Async client (`AsyncClient`, `aask`) — transports already have async builders. + +Layer 3 (expose a plugin as a model tb can route to) needed no new work of its +own — it's already fully supported as provider-as-upstream (see "Layer 3" +below) — so it's not listed as a follow-up. ## Layer 2: write an AI server (`tingly.Plugin`) From 5bb80e670a08b6c3b640126ed598a820edbd9c0c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 08:12:23 +0000 Subject: [PATCH 18/28] feat(sdk): narrow MVP to connect+send+plugin round-trip, Anthropic primary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rescopes the plugin work to a concrete, verified milestone: connect to tb, send a message, and have a plugin work end-to-end including forwarding to another tb rule and back — with Anthropic as the primary wire protocol and OpenAI chat completions kept as a real secondary path, not removed. - Plugin server (sdk/python/tingly/plugin/server.py) now answers both POST /v1/messages (Anthropic, primary) and POST /v1/chat/completions (OpenAI, secondary) off one shared handler; ChatRequest gains from_anthropic_body() to fold the top-level `system` field into a message. - Registration carries api_style (openai|anthropic) end to end: Go RegisterPluginRequest -> Handler.upsertPluginProvider -> provider.APIStyle (previously hardcoded to "openai"). Wire-level default stays "openai" for back-compat; the Python SDK's own default is "anthropic". - Client.ask() now tries the Anthropic transport first when a scenario supports both (was OpenAI-first), and no longer silently rewrites model="auto" to a hardcoded model name on the Anthropic path. - Fixed a real bug this surfaced: Provider.GetAccessToken() returned "" for no-key providers, and anthropic-sdk-go treats an empty key as "look for ambient credentials", erroring instead of sending an empty header like the OpenAI client does. Added ai.NoKeySentinelToken for AuthTypeAPIKey + NoKeyRequired + empty token, general fix beyond just plugins. - Verified live end-to-end with the real tb binary (examples/e2e_run.sh): client -> tb -> plugin (Anthropic route) -> plugin calls back into another tb rule -> answer composed -> back through tb -> client. - Design doc and README updated to match: scope milestone, protocol decision, corrected pencil graphs, pruned/reprioritized follow-ups. --- .design/python-sdk.md | 191 +++++++++++++----- ai/provider.go | 13 ++ ai/provider_test.go | 18 ++ internal/server/module/plugin/handler.go | 34 +++- internal/server/module/plugin/handler_test.go | 62 ++++++ internal/server/module/plugin/types.go | 3 +- sdk/python/README.md | 23 ++- sdk/python/examples/e2e_plugin.py | 2 +- sdk/python/examples/rag_plugin.py | 6 +- sdk/python/tests/test_client_offline.py | 34 ++++ sdk/python/tests/test_plugin_manifest.py | 10 + sdk/python/tests/test_plugin_register.py | 4 + sdk/python/tests/test_plugin_server.py | 101 ++++++++- sdk/python/tingly/cli.py | 2 +- sdk/python/tingly/client.py | 15 +- sdk/python/tingly/plugin/__init__.py | 5 +- sdk/python/tingly/plugin/core.py | 27 ++- sdk/python/tingly/plugin/manifest.py | 8 +- sdk/python/tingly/plugin/register.py | 10 +- sdk/python/tingly/plugin/server.py | 126 ++++++++++-- sdk/python/tingly/plugin/types.py | 27 +++ 21 files changed, 607 insertions(+), 114 deletions(-) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index c7c2a838a..df37edf06 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -14,6 +14,42 @@ with the right base URL, token and scenario path. There was no fast seam for (prompt, retrieval, agent loop) and **reuses the gateway's power** — provider routing, tier/fallback, guard rails, quota, logging — for free. +## Scope (current milestone) + +The near-term target is deliberately narrow — connect, send a message, and +have a plugin work end-to-end including forwarding to another tb rule and +back. All three are done and verified live (`examples/e2e_run.sh`, real `tb` +binary, no mocks): + +1. `tingly.connect()` reaches tb and mints a session (Layer 1). +2. `tb.ask(...)` / `tb.anthropic.messages.create(...)` send a message through + tb's pipeline and get a real answer back (Layer 1). +3. A `tingly.Plugin` runs, tb routes a model to it, and the plugin's handler + calls `plugin.llm.ask(...)` / `plugin.use(scenario).ask(...)` to forward + into any other rule and get the result back before answering (Layer 2 + + Layer 3). + +**Protocol scope, deliberately narrowed:** Anthropic is primary everywhere in +the SDK; OpenAI chat completions is a real secondary path — kept, not +removed, but not what new work defaults to. Concretely: + +- `Client.ask()` tries the Anthropic transport first when a scenario supports + both (flipped from OpenAI-first — see "Two-token model" / Request flow). +- The plugin's own HTTP server answers `POST /v1/messages` (Anthropic, + primary) and `POST /v1/chat/completions` (OpenAI, secondary) — both real, + sharing one handler and one normalized `ChatRequest`; only the response + shaping differs per route. +- New plugins register with `api_style="anthropic"` by default + (`Plugin(api_style=...)` overrides it, per-plugin); the wire-level default + at `POST /api/v2/plugins` itself (a caller that omits the field entirely) + stays `"openai"`, for back-compat with anything hitting the endpoint + directly. + +Out of scope for now, unchanged from before: the tb-side plugin sub-process +supervisor, the `/plugins//*` reverse-proxy mount, and the lifecycle UI +(see Open follow-ups). None of the three milestone points above need them — +a plugin author starts their own process today, same as any local dev server. + ## Architecture (one idea, not three layers) There is a single concept: @@ -51,7 +87,7 @@ today's pieces — three verbs for the one rule⇄plugin relationship: | verb | what it is | SDK surface | |------|------------|-------------| | **connect** | a plugin (or experiment) *consumes* a rule | `tingly.connect()` / `plugin.use(scenario).ask(model=…)` | -| **serve** | a plugin *is* a rule's upstream | `tingly.Plugin` (OpenAI server) | +| **serve** | a plugin *is* a rule's upstream | `tingly.Plugin` (Anthropic-primary server) | | **register**| point a rule's upstream at the plugin | `register_with_tb()` → tb provider + rule | The historical "Layer 1/2/3" headings below map exactly to connect / serve / @@ -91,15 +127,32 @@ cautionary reference. provider + rule creation), and a `RegisterRoutes(group, handler)` mounted from `server_webui_api.go`. Plugin logic no longer lives as methods on the giant `*Server` struct. -- A plugin is an ordinary provider (`APIStyle=openai`, `api_key`/`no_key`) - carrying the tag `"plugin"` in the existing, generic `Provider.Tags` field. - `Provider.IsPlugin()` checks for that tag. No new struct, no new DB column — - Tags already round-trips through the provider store unconditionally. +- A plugin is an ordinary provider (`APIStyle=openai|anthropic`, `api_key`/ + `no_key`) carrying the tag `"plugin"` in the existing, generic + `Provider.Tags` field. `Provider.IsPlugin()` checks for that tag. No new + struct, no new DB column — Tags already round-trips through the provider + store unconditionally. - **`POST /api/v2/plugins`** is an **idempotent upsert by name**: register once at startup (and again on every restart) and it updates the same provider instead of duplicating it. When `scenario` is given it also idempotently - ensures the rule. Response: `{provider_uuid, model_id, scenario, rule_uuid, - ready, note}`. + ensures the rule. Request carries `api_style` (`openai`|`anthropic`, empty + → `openai` at the wire level; the SDK's own default is `anthropic`, see + Scope above) — this is what tells tb which of the plugin's two routes to + call. Response: `{provider_uuid, model_id, scenario, rule_uuid, ready, note}`. +- **A real, non-obvious fix underneath this:** `Provider.GetAccessToken()` + returned `""` for a no-key provider, and the vendored `anthropic-sdk-go` + treats an empty API key as "go look for ambient credentials" — it does its + own discovery (env vars, `anthropic auth login` profile, …) and errors + loudly when none exist, instead of just sending an empty/absent header the + way the OpenAI client does. This was invisible while every plugin was + `APIStyle=openai`; it surfaced immediately once Anthropic became the + default and broke the very first live end-to-end run. Fixed by + `ai.NoKeySentinelToken` (`ai/provider.go`): when `AuthType=api_key`, + `Token==""` and `NoKeyRequired=true`, `GetAccessToken()` returns that + sentinel instead of `""` — a real (if meaningless) value the SDK is happy + to send as the header, which the plugin's own auth check (`api_key=""` → + accept anything) ignores. General fix, not plugin-specific: any + no-key-required Anthropic-style provider benefits. - **`GET /api/v2/plugins`** lists plugin-tagged providers, deriving the display model id from the rule(s) bound to each (no extra field needed). - **Retiring a plugin** is the same as retiring any other provider: delete it @@ -112,11 +165,19 @@ where there is no `~/.tingly-box`. This part was cheap and answers a real need, so it stayed. `Plugin.serve(register=True, scenario=…, tb=Connection(...))` registers once at startup — no background thread, no lease to manage. -Verified end-to-end (`examples/e2e_run.sh`): the plugin registers once, a -client call routes through it and back into tb (no network/keys); killing the -plugin leaves its provider listed (same as any provider) and the next request -fails with a plain connection error (add a tier-1 fallback to see failover -instead); restarting the plugin upserts the same provider, no duplicate. +Verified end-to-end (`examples/e2e_run.sh`, real `tb` binary, no network/keys): +the plugin registers once as an `api_style=anthropic` provider; a client's +OpenAI-shaped `chat/completions` call to `model=plugin/rag-demo` routes +through tb, which forwards it to the plugin as `POST /v1/messages?beta=true` +(tb's real Anthropic client — the `?beta=true` query string is why the +server routes on path only, ignoring the query); the plugin's handler calls +`plugin.use("experiment").ask(..., model="echo-model")`, itself now an +Anthropic-transport call, which tb routes to a `vmodel` provider and back; +the composed answer returns to the plugin, which tb reshapes back to +OpenAI `chat.completion` for the original caller. Killing the plugin leaves +its provider listed (same as any provider) and the next request fails with a +plain connection error (add a tier-1 fallback to see failover instead); +restarting the plugin upserts the same provider, no duplicate. ## Shape @@ -131,9 +192,9 @@ sdk/python/ transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb helpers/ # usage + guardrails views plugin/ # Layer 2: be an AI server tb routes to - core.py # Plugin class (@plugin.chat, .llm, .serve) - server.py # stdlib OpenAI-compatible HTTP server (+ SSE) - types.py # ChatRequest / Message + core.py # Plugin class (@plugin.chat, .llm, .serve, api_style) + server.py # stdlib HTTP server: /v1/messages (primary) + /v1/chat/completions (secondary), + SSE + types.py # ChatRequest / Message (from_anthropic_body / from_openai_body) manifest.py # tingly.toml read/write register.py # one-shot, idempotent register with tb cli.py # `tingly doctor` + `tingly plugin {init,run}` @@ -151,7 +212,7 @@ connect(scenario="experiment") └─ Client(session, gateway_url, admin_token) .openai → openai.OpenAI(base_url = scenario_root + "/v1") .anthropic → anthropic.Anthropic(base_url = scenario_root) - .ask() → picks transport from session.transport + .ask() → Anthropic first when the scenario supports both, else OpenAI .usage → GET /api/v1/requests (admin token) .guardrails → GET /api/v1/guardrails/config (admin token) ``` @@ -308,12 +369,14 @@ decision, not a unilateral rename — flagged here rather than acted on. ## Open follow-ups -Roughly in priority order — 1–3 are backend/SDK-only (no naming exposure, safe -to build regardless of the rename above); 4 is blocked on that decision for -its UI portion specifically (the supervisor + reverse-proxy mount are not). +The Scope milestone above (connect + send + plugin round-trip, Anthropic +primary / OpenAI secondary) is **done**. What's left, roughly in priority +order — 1–3 are backend/SDK-only (no naming exposure, safe to build +regardless of the rename above); 4 is blocked on that decision for its UI +portion specifically (the supervisor + reverse-proxy mount are not): 1. Layer 2 tb-side remainder — Python side is **done** (`tingly.Plugin`, - manifest, OpenAI server, `register`); still missing: a sub-process + manifest, dual-protocol server, `register`); still missing: a sub-process supervisor that boots plugins from their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy mount, and the install/enable/logs/disable lifecycle UI. The first two are ordinary @@ -330,8 +393,11 @@ below) — so it's not listed as a follow-up. ## Layer 2: write an AI server (`tingly.Plugin`) -A plugin is an **OpenAI-compatible upstream**: the author writes one chat -handler, and `serve()` runs the HTTP server. The whole surface is one class. +A plugin is an upstream tb can call two ways — **Anthropic Messages +(primary)** and **OpenAI chat completions (secondary)**, both real, both +always served regardless of what registration advertises. The author writes +one chat handler, and `serve()` runs the HTTP server. The whole surface is +one class. ```python from tingly import Plugin @@ -363,21 +429,24 @@ caller (step 3) and the upstream-for-the-plugin (step 5). │ │ │ provider: │ │ @plugin.chat │ │ name=my-rag │ │ def handle(req): ... │ │ api_base=http://…:8765/v1 │ - │ │ │ │ model=plugin/my-rag │ - │ │ returns str | iter[str] │ │ │ - │ ▼ │ │ rule: plugin/my-rag → ↑ │ - │ serve() → stdlib HTTP server │ └──────────────┬───────────────┘ - │ POST /v1/chat/completions ◄────┼──── (3) POST /v1/chat ──┘ ▲ - │ GET /v1/models │ (model=plugin/my-rag)│ (6) answer - │ GET /health │ │ - │ · buffered → chat.completion │ │ - │ · stream → SSE chunks ────┼──── (7) response ────────────┘ + │ │ │ │ api_style=anthropic (dflt) │ + │ │ returns str | iter[str] │ │ model=plugin/my-rag │ + │ ▼ │ │ │ + │ serve() → stdlib HTTP server │ │ rule: plugin/my-rag → ↑ │ + │ POST /v1/messages ◄────┼─ (3) POST /v1/messages?beta=true ──┘ ▲ │ + │ (primary, api_style match) │ (model=plugin/my-rag) │ │ + │ POST /v1/chat/completions ◄────┼─ (3') POST /v1/chat/completions ─────┘ │ + │ (secondary, if api_style=openai) (6) answer │ + │ GET /v1/models │ │ + │ GET /health │ │ + │ · buffered → message / chat.completion │ + │ · stream → SSE (message_* events / chat.completion.chunk) ── (7) ───┘ │ │ │ │ plugin.llm (lazy Layer-1 client)│ │ │ │ └────────┼──────────────────────────┘ │ (4) plugin.llm.ask("…", model="auto") - │ = tingly.connect(scenario="experiment") → POST /tingly/experiment/v1/chat + │ = tingly.connect(scenario="experiment") → POST /tingly/experiment/v1/messages ▼ ┌──────────────────────────────────────────────────────────────┐ │ tingly-box pipeline (SAME as any client — see Layer 1 graph) │ @@ -387,13 +456,14 @@ caller (step 3) and the upstream-for-the-plugin (step 5). OpenAI/…) request lifecycle: - (1) client sends model="plugin/my-rag" to tb ── see Layer 3 graph - (2) tb resolves rule → provider my-rag (api_base = plugin) - (3) tb POSTs OpenAI /v1/chat/completions to the PLUGIN - (4) handler runs; calls plugin.llm.ask(...) ── back INTO tb + (1) client sends model="plugin/my-rag" to tb, any protocol ── see Layer 3 graph + (2) tb resolves rule → provider my-rag (api_base = plugin, api_style picks the route) + (3) tb calls the PLUGIN on whichever route matches provider.api_style + (Anthropic /v1/messages by default; /v1/chat/completions if registered openai) + (4) handler runs; calls plugin.llm.ask(...) ── back INTO tb, Anthropic-first (5) tb routes that call to a real upstream, applies guard rails/quota/… (6) generated text returns to the handler - (7) handler's str/iterator → OpenAI response/SSE back to tb → back to client + (7) handler's str/iterator → response/SSE shaped for whichever route was hit → back to tb → back to client ``` Key reading: @@ -411,21 +481,30 @@ Key reading: Design choices: - **No framework dependency.** The server is `http.server.ThreadingHTTPServer` - (stdlib), so a plugin is one `pip install tingly` away. It serves - `POST /v1/chat/completions` (buffered **and** real SSE streaming), - `GET /v1/models`, `GET /health` — exactly what tb needs to treat it as an - OpenAI upstream. -- **Handler contract is minimal.** Return a `str` (buffered) or an iterator of - `str` (streamed); the server shapes both into `chat.completion` / - `chat.completion.chunk`. The author never touches wire format. + (stdlib), so a plugin is one `pip install tingly` away. It always serves + both `POST /v1/messages` (Anthropic, buffered **and** real SSE) and + `POST /v1/chat/completions` (OpenAI, same), plus `GET /v1/models`, + `GET /health` — which route tb actually uses is a registration choice + (`api_style`), not a server capability limit. +- **Handler contract is minimal and protocol-agnostic.** Return a `str` + (buffered) or an iterator of `str` (streamed); the server shapes it into + `message`/SSE `message_*` events for the Anthropic route or + `chat.completion`/`chat.completion.chunk` for the OpenAI route, whichever + was hit. The author never touches wire format either way. + `ChatRequest.from_anthropic_body` folds Anthropic's top-level `system` + field into a leading `role="system"` message so `req.system_text()` / + `req.last_user_text()` work the same regardless of which route the caller + used. - **`plugin.llm` is a lazy Layer-1 client.** The plugin reuses the gateway for its own generation instead of hard-coding a provider/key — the recursion in - the Layer 3 graph. + the Layer 3 graph. Its own `ask()` calls try Anthropic first (see Scope). - **`tingly.toml` manifest** (`manifest.py`) declares name / model_id / - entrypoint / transport / port, so a future tb-side supervisor can install and - run the plugin. `tingly plugin init` scaffolds a module + manifest. + entrypoint / transport (`anthropic` by default, tracks `Plugin.api_style`) / + port, so a future tb-side supervisor can install and run the plugin. + `tingly plugin init` scaffolds a module + manifest. - **Optional token auth.** `Plugin(api_key=...)` enforces a bearer token so only - tb (carrying the matching provider token) can call it. + tb (carrying the matching provider token) can call it — checked once, + ahead of both routes. CLI: @@ -472,20 +551,22 @@ via the in-process `vmodel` package. │ tingly.ask() │ "plugin│ smart routing / TIERS ──────┼──┤ └──────────────┘ └──────────────┘ /my-rag"│ circuit-breaker failover │ │ tier 0 ┌──────────────┐ │ quota + usage logging │ └───────► │ my-rag PLUGIN│ ◄─ Layer 2 - │ provider.api_base = plugin │ POST │ POST /v1/chat│ Plugin.serve() - └────────────────────────────────┘ /v1/chat │ /completions │ - └──────┬───────┘ + │ provider.api_base = plugin │ POST │ POST /v1/ │ Plugin.serve() + │ provider.api_style picks route│ /v1/msgs │ messages │ + └────────────────────────────────┘ (dflt) └──────┬───────┘ ctx.llm.ask() ┄┄┄┘ (plugin may back INTO tb for its own LLM calls) ``` Wiring (no new gateway hot-path code — it's just a provider): -1. **Plugin serves** `POST /v1/chat/completions` (Layer 2 `Plugin.serve()`). +1. **Plugin serves** both `POST /v1/messages` and `POST /v1/chat/completions` + (Layer 2 `Plugin.serve()`). 2. **Register**: `POST /api/v2/plugins {name:"my-rag", endpoint:"http://127.0.0.1:/v1", - model_id:"plugin/my-rag", scenario:"experiment"}` creates a *normal* provider - (not `AuthType=virtual`, tagged `"plugin"`) — this is exactly what - `Plugin.serve()` does on startup. + model_id:"plugin/my-rag", scenario:"experiment", api_style:"anthropic"}` + creates a *normal* provider (not `AuthType=virtual`, tagged `"plugin"`, + `APIStyle` set from `api_style`) — this is exactly what `Plugin.serve()` + does on startup, with `api_style` defaulting to `"anthropic"`. 3. That same call **binds the rule/service**: model `plugin/my-rag` → that provider. 4. Now `model:"plugin/my-rag"` from any client resolves through the same dispatcher as every other model. Put the plugin in tier 0 and a real model in diff --git a/ai/provider.go b/ai/provider.go index e3ab70571..3fd7fac74 100644 --- a/ai/provider.go +++ b/ai/provider.go @@ -360,6 +360,16 @@ func (p *Provider) ResolveEndpoint(clientStyle APIStyle) (string, APIStyle) { // value is never transmitted. const VModelSentinelToken = "EMPTY" +// NoKeySentinelToken satisfies the same non-empty-APIKey check for real +// outbound HTTP providers that genuinely take no key (NoKeyRequired=true; +// e.g. a local plugin process). Unlike VModelSentinelToken this value IS +// transmitted, as an Authorization/x-api-key header the receiving side is +// expected to ignore. It exists because some client SDKs (anthropic-sdk-go) +// treat an empty API key as "look for ambient credentials" and fail loudly +// when none are found, instead of just sending an empty/absent header the +// way the OpenAI client does. +const NoKeySentinelToken = "tingly-no-key" + // GetAccessToken returns the access token based on auth type func (p *Provider) GetAccessToken() string { switch p.AuthType { @@ -371,6 +381,9 @@ func (p *Provider) GetAccessToken() string { return VModelSentinelToken case AuthTypeAPIKey, "": // Default to api_key for backward compatibility + if p.Token == "" && p.NoKeyRequired { + return NoKeySentinelToken + } return p.Token } return "" diff --git a/ai/provider_test.go b/ai/provider_test.go index 97f4e8128..6f9b9f34f 100644 --- a/ai/provider_test.go +++ b/ai/provider_test.go @@ -373,6 +373,24 @@ func TestProvider_GetAccessToken(t *testing.T) { }, want: "", }, + { + name: "API key auth, no key required, empty token -> sentinel", + provider: &Provider{ + AuthType: AuthTypeAPIKey, + NoKeyRequired: true, + Token: "", + }, + want: NoKeySentinelToken, + }, + { + name: "API key auth, no key required, but a real token is still preferred", + provider: &Provider{ + AuthType: AuthTypeAPIKey, + NoKeyRequired: true, + Token: "sk-real", + }, + want: "sk-real", + }, } for _, tt := range tests { diff --git a/internal/server/module/plugin/handler.go b/internal/server/module/plugin/handler.go index 7fde16b3f..36b99bd1c 100644 --- a/internal/server/module/plugin/handler.go +++ b/internal/server/module/plugin/handler.go @@ -14,6 +14,7 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + "github.com/tingly-dev/tingly-box/ai" "github.com/tingly-dev/tingly-box/internal/constant" "github.com/tingly-dev/tingly-box/internal/loadbalance" "github.com/tingly-dev/tingly-box/internal/server/config" @@ -46,7 +47,13 @@ func (h *Handler) RegisterPlugin(c *gin.Context) { modelID = "plugin/" + req.Name } - provider, err := h.upsertPluginProvider(req.Name, req.Endpoint, req.Token) + apiStyle, err := normalizeAPIStyle(req.APIStyle) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + + provider, err := h.upsertPluginProvider(req.Name, req.Endpoint, req.Token, apiStyle) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "success": false, @@ -86,12 +93,29 @@ func (h *Handler) RegisterPlugin(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"success": true, "data": resp}) } +// normalizeAPIStyle validates the caller-supplied wire protocol, defaulting +// the empty string to "openai" (the style tb assumed before plugins could +// declare one). Anthropic is the SDK's own default for new plugins, but that +// is a Python-side policy — the wire-level default stays put for anyone +// calling this endpoint directly without an api_style. +func normalizeAPIStyle(raw string) (ai.APIStyle, error) { + switch raw { + case "": + return ai.APIStyleOpenAI, nil + case string(ai.APIStyleOpenAI), string(ai.APIStyleAnthropic): + return ai.APIStyle(raw), nil + default: + return "", &bindError{"api_style must be \"openai\" or \"anthropic\", got " + raw} + } +} + // upsertPluginProvider creates a plugin-tagged provider, or updates the -// endpoint/token of an existing one with the same name. Idempotent by name so -// a plugin can safely re-register (e.g. on every process start). -func (h *Handler) upsertPluginProvider(name, endpoint, token string) (*typ.Provider, error) { +// endpoint/token/style of an existing one with the same name. Idempotent by +// name so a plugin can safely re-register (e.g. on every process start). +func (h *Handler) upsertPluginProvider(name, endpoint, token string, apiStyle ai.APIStyle) (*typ.Provider, error) { if existing, err := h.config.GetProviderByName(name); err == nil && existing.IsPlugin() { existing.APIBase = endpoint + existing.APIStyle = apiStyle existing.Token = token existing.NoKeyRequired = token == "" existing.Enabled = true @@ -105,7 +129,7 @@ func (h *Handler) upsertPluginProvider(name, endpoint, token string) (*typ.Provi UUID: config.GenerateUUID(), Name: name, APIBase: endpoint, - APIStyle: "openai", + APIStyle: apiStyle, Token: token, NoKeyRequired: token == "", Enabled: true, diff --git a/internal/server/module/plugin/handler_test.go b/internal/server/module/plugin/handler_test.go index c855cd2b5..a70a9eadb 100644 --- a/internal/server/module/plugin/handler_test.go +++ b/internal/server/module/plugin/handler_test.go @@ -87,6 +87,68 @@ func TestRegisterPlugin_BindsRule(t *testing.T) { } } +func TestRegisterPlugin_APIStyleDefaultsToOpenAI(t *testing.T) { + h := newTestHandler(t) + + _, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "no-style", Endpoint: "http://127.0.0.1:8765/v1", + }) + uuid := resp["data"].(map[string]any)["provider_uuid"].(string) + prov, err := h.config.GetProviderByUUID(uuid) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIStyle != "openai" { + t.Fatalf("expected default api_style openai, got %q", prov.APIStyle) + } +} + +func TestRegisterPlugin_APIStyleAnthropic(t *testing.T) { + h := newTestHandler(t) + + _, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "anthropic-plug", Endpoint: "http://127.0.0.1:8765", APIStyle: "anthropic", + }) + uuid := resp["data"].(map[string]any)["provider_uuid"].(string) + prov, err := h.config.GetProviderByUUID(uuid) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIStyle != "anthropic" { + t.Fatalf("expected api_style anthropic, got %q", prov.APIStyle) + } +} + +func TestRegisterPlugin_APIStyleRejectsUnknown(t *testing.T) { + h := newTestHandler(t) + + w, resp := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "bad-style", Endpoint: "http://127.0.0.1:8765", APIStyle: "gemini", + }) + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400 for unknown api_style, got %d (%v)", w.Code, resp) + } +} + +func TestRegisterPlugin_ReregisterUpdatesAPIStyle(t *testing.T) { + h := newTestHandler(t) + + postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "switches", Endpoint: "http://127.0.0.1:8765", + }) + _, second := postJSON(t, h.RegisterPlugin, RegisterPluginRequest{ + Name: "switches", Endpoint: "http://127.0.0.1:8765", APIStyle: "anthropic", + }) + uuid := second["data"].(map[string]any)["provider_uuid"].(string) + prov, err := h.config.GetProviderByUUID(uuid) + if err != nil { + t.Fatalf("GetProviderByUUID: %v", err) + } + if prov.APIStyle != "anthropic" { + t.Fatalf("expected re-register to update api_style to anthropic, got %q", prov.APIStyle) + } +} + func TestRegisterPlugin_ProviderOnly(t *testing.T) { h := newTestHandler(t) diff --git a/internal/server/module/plugin/types.go b/internal/server/module/plugin/types.go index 38a5a3888..e53c40bad 100644 --- a/internal/server/module/plugin/types.go +++ b/internal/server/module/plugin/types.go @@ -5,11 +5,12 @@ package plugin // process starts) updates the existing provider instead of duplicating it. type RegisterPluginRequest struct { Name string `json:"name" binding:"required" description:"Plugin / provider name" example:"my-rag"` - Endpoint string `json:"endpoint" binding:"required" description:"Plugin OpenAI base URL" example:"http://127.0.0.1:8765/v1"` + Endpoint string `json:"endpoint" binding:"required" description:"Plugin base URL" example:"http://127.0.0.1:8765/v1"` ModelID string `json:"model_id,omitempty" description:"Model id the plugin advertises" example:"plugin/my-rag"` Token string `json:"token,omitempty" description:"Token tingly-box should send to the plugin (empty = no key)"` Scenario string `json:"scenario,omitempty" description:"Scenario to bind a rule under; omit to create only the provider" example:"experiment"` Tier int `json:"tier,omitempty" description:"Tier for the bound service (0 = highest priority)"` + APIStyle string `json:"api_style,omitempty" description:"Wire protocol the plugin's endpoint speaks: \"openai\" or \"anthropic\"; empty defaults to \"openai\"" example:"anthropic"` } // RegisterPluginResponse reports what was created or updated. diff --git a/sdk/python/README.md b/sdk/python/README.md index 5fa885a7f..a2af76f6e 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -57,8 +57,12 @@ A green `tingly doctor` is a guarantee your code will run. ## Write a plugin (an AI server tb can route to) -A plugin is an OpenAI-compatible upstream. Write one handler, serve it, register -it — then any tb client can select it as a model. +A plugin is an upstream tb can call two ways: **Anthropic Messages +(`/v1/messages`, primary)** and **OpenAI chat completions +(`/v1/chat/completions`, secondary)** — both real, both always served; which +one tb actually uses is a registration choice (`api_style`, `"anthropic"` by +default). Write one handler, serve it, register it — then any tb client can +select it as a model, regardless of which protocol *that* client speaks. ```python from tingly import Plugin @@ -88,14 +92,19 @@ next failed request trips the breaker and traffic tier-fails-over (when a fallback tier is configured). Retiring a plugin is the same as retiring any other provider — delete it in the tb UI. -The server is stdlib-only (no FastAPI), supports streaming, and `plugin.llm` -calls back into tb so the plugin reuses the gateway for its own LLM work. +The server is stdlib-only (no FastAPI), supports streaming on both routes, and +`plugin.llm` calls back into tb (Anthropic-first) so the plugin reuses the +gateway for its own LLM work. ## Status -- **Layer 1** (consume tb): `connect()` → `Client`. Done. -- **Layer 2** (be an AI server): `tingly.Plugin` + manifest + `register`. Python - side done; tb-side supervisor/lifecycle UI pending. +- **Layer 1** (consume tb): `connect()` → `Client`. Done — Anthropic tried + first when a scenario supports both transports. +- **Layer 2** (be an AI server): `tingly.Plugin` + manifest + `register`. Done — + dual-protocol server (Anthropic primary, OpenAI secondary); tb-side + supervisor/lifecycle UI still pending (not required to use plugins today). - **Layer 3** (tb routes to the plugin as a model): via provider-as-upstream. + Done, verified end-to-end including a plugin forwarding to another tb rule + and returning the result (`sdk/python/examples/e2e_run.sh`). See `.design/python-sdk.md` in the repo for the full design and diagrams. diff --git a/sdk/python/examples/e2e_plugin.py b/sdk/python/examples/e2e_plugin.py index a83e8096d..d894c6270 100644 --- a/sdk/python/examples/e2e_plugin.py +++ b/sdk/python/examples/e2e_plugin.py @@ -11,7 +11,7 @@ CORPUS = { "tingly-box": "tingly-box is a personal intelligence orchestrator.", - "plugin": "A plugin is an OpenAI-compatible upstream tb can route to.", + "plugin": "A plugin is an Anthropic/OpenAI-compatible upstream tb can route to.", } diff --git a/sdk/python/examples/rag_plugin.py b/sdk/python/examples/rag_plugin.py index f61e350c9..947512347 100644 --- a/sdk/python/examples/rag_plugin.py +++ b/sdk/python/examples/rag_plugin.py @@ -1,4 +1,4 @@ -"""A RAG plugin served as an OpenAI-compatible upstream for tingly-box. +"""A RAG plugin served as an upstream for tingly-box (Anthropic primary, OpenAI secondary). Run it (serves on :8765 AND registers with tb on startup): @@ -27,8 +27,8 @@ CORPUS = { "tingly-box": "tingly-box is a personal intelligence orchestrator: an LLM " "gateway with remote control and guard rails.", - "plugin": "A tingly plugin is an OpenAI-compatible upstream that tingly-box " - "can route to as a model.", + "plugin": "A tingly plugin is an Anthropic/OpenAI-compatible upstream that " + "tingly-box can route to as a model.", } diff --git a/sdk/python/tests/test_client_offline.py b/sdk/python/tests/test_client_offline.py index 00b254626..aa3255a04 100644 --- a/sdk/python/tests/test_client_offline.py +++ b/sdk/python/tests/test_client_offline.py @@ -39,6 +39,40 @@ def test_both_exposes_identity(): assert c.base_url.endswith("/tingly/experiment") +def test_ask_prefers_anthropic_when_both_supported(monkeypatch): + """Anthropic is tb's native protocol, so ask() tries it first when a + scenario supports both — flipped from the old OpenAI-first default.""" + calls = [] + c = _client("both") + monkeypatch.setattr(c, "_ask_anthropic", lambda *a, **k: calls.append("anthropic") or "ok") + monkeypatch.setattr(c, "_ask_openai", lambda *a, **k: calls.append("openai") or "ok") + c.ask("hi") + assert calls == ["anthropic"] + + +def test_ask_anthropic_passes_model_through_unmodified(monkeypatch): + """model="auto" must reach tb as-is (tb's rule resolution handles it) — + not get silently rewritten to a hardcoded model name.""" + c = _client("anthropic") + captured = {} + + class _FakeMessages: + def create(self, **kwargs): + captured.update(kwargs) + + class R: + content = [] + + return R() + + class _FakeAnthropic: + messages = _FakeMessages() + + monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) + c.ask("hi", model="auto") + assert captured["model"] == "auto" + + def test_client_passes_scenario_root_to_transports(monkeypatch): """The client hands the scenario root + model token to each builder; per-transport URL shaping (e.g. the OpenAI /v1 suffix) happens inside the diff --git a/sdk/python/tests/test_plugin_manifest.py b/sdk/python/tests/test_plugin_manifest.py index 86e11560e..8b45824d6 100644 --- a/sdk/python/tests/test_plugin_manifest.py +++ b/sdk/python/tests/test_plugin_manifest.py @@ -38,8 +38,18 @@ def test_plugin_builds_manifest(): assert man.model_id == "plugin/my-rag" assert man.entrypoint == "rag_plugin:plugin" assert man.port == 8080 + assert man.transport == "anthropic" + + +def test_plugin_manifest_transport_follows_api_style_override(): + plugin = Plugin(name="my-rag", api_style="openai") + man = plugin.manifest(entrypoint="rag_plugin:plugin") assert man.transport == "openai" + # an explicit transport= still wins over api_style + man2 = plugin.manifest(entrypoint="rag_plugin:plugin", transport="anthropic") + assert man2.transport == "anthropic" + def test_plugin_use_caches_per_scenario(monkeypatch): import tingly.client as client_mod diff --git a/sdk/python/tests/test_plugin_register.py b/sdk/python/tests/test_plugin_register.py index 9505875a6..d7229f9f7 100644 --- a/sdk/python/tests/test_plugin_register.py +++ b/sdk/python/tests/test_plugin_register.py @@ -1,5 +1,7 @@ """Active config + plugin registration tests.""" +import json + import httpx import pytest import respx @@ -57,6 +59,8 @@ def test_register_binds_rule(monkeypatch): assert result.provider_uuid == "uuid-1" assert result.rule_uuid == "rule-1" assert result.ready is True + # anthropic is the SDK's default wire protocol for new plugins + assert json.loads(route.calls.last.request.content)["api_style"] == "anthropic" @respx.mock diff --git a/sdk/python/tests/test_plugin_server.py b/sdk/python/tests/test_plugin_server.py index d556d91f3..c3d3692f7 100644 --- a/sdk/python/tests/test_plugin_server.py +++ b/sdk/python/tests/test_plugin_server.py @@ -1,7 +1,9 @@ """Plugin server tests — drive a real (ephemeral-port) plugin over HTTP. -These pin the OpenAI wire contract tingly-box relies on when it routes to a -plugin as an upstream: chat.completion shape, SSE streaming, /v1/models, auth. +These pin the two wire contracts tingly-box relies on when it routes to a +plugin as an upstream — Anthropic /v1/messages (primary) and OpenAI +/v1/chat/completions (secondary) — plus buffered/streaming shape, /v1/models, +and auth. Both protocols share one handler; only response shaping differs. """ import json @@ -84,6 +86,93 @@ def test_chat_completion_streaming(served): assert saw_done +def test_anthropic_message_shape(served): + r = httpx.post( + f"{served}/v1/messages", + json={ + "model": "plugin/t-plug", + "max_tokens": 256, + "messages": [{"role": "user", "content": "hi"}], + }, + timeout=5, + ) + assert r.status_code == 200 + body = r.json() + assert body["type"] == "message" + assert body["role"] == "assistant" + assert body["content"] == [{"type": "text", "text": "echo: hi"}] + assert body["stop_reason"] == "end_turn" + + +def test_anthropic_message_streaming(served): + with httpx.stream( + "POST", + f"{served}/v1/messages", + json={ + "model": "plugin/t-plug", + "max_tokens": 256, + "messages": [{"role": "user", "content": "world"}], + "stream": True, + }, + timeout=5, + ) as r: + assert r.status_code == 200 + events = [] + deltas = [] + for line in r.iter_lines(): + if line.startswith("event: "): + events.append(line[len("event: "):]) + elif line.startswith("data: "): + payload = json.loads(line[len("data: "):]) + if payload.get("type") == "content_block_delta": + deltas.append(payload["delta"]["text"]) + assert events == [ + "message_start", "content_block_start", "content_block_delta", + "content_block_delta", "content_block_delta", "content_block_stop", + "message_delta", "message_stop", + ] + assert "".join(deltas) == "hello world" + + +def test_anthropic_messages_route_ignores_query_string(served): + """tb's real Anthropic client appends ?beta=true to /v1/messages; the + plugin server must route on the path, not the raw request-target.""" + r = httpx.post( + f"{served}/v1/messages?beta=true", + json={"model": "plugin/t-plug", "max_tokens": 32, "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert r.status_code == 200 + assert r.json()["content"][0]["text"] == "echo: hi" + + +def test_anthropic_system_field_reaches_handler(): + plugin = Plugin(name="sys-plug") + + @plugin.chat + def handle(req: ChatRequest): + return f"system={req.system_text()!r} user={req.last_user_text()!r}" + + port = plugin.serve(port=0, verbose=False, block=False) + base = f"http://127.0.0.1:{port}" + try: + r = httpx.post( + f"{base}/v1/messages", + json={ + "model": "x", + "max_tokens": 64, + "system": "be terse", + "messages": [{"role": "user", "content": "hi"}], + }, + timeout=5, + ) + assert r.status_code == 200 + text = r.json()["content"][0]["text"] + assert text == "system='be terse' user='hi'" + finally: + plugin.stop() + + def test_auth_enforced_when_key_set(): plugin = Plugin(name="auth-plug", api_key="secret") @@ -108,5 +197,13 @@ def handle(req): timeout=5, ) assert good.status_code == 200 + + bad_anthropic = httpx.post( + f"{base}/v1/messages", + json={"model": "x", "max_tokens": 8, "messages": []}, + timeout=5, + ) + assert bad_anthropic.status_code == 401 + assert bad_anthropic.json()["type"] == "error" finally: plugin.stop() diff --git a/sdk/python/tingly/cli.py b/sdk/python/tingly/cli.py index d574c93ca..3e1dea5a5 100644 --- a/sdk/python/tingly/cli.py +++ b/sdk/python/tingly/cli.py @@ -89,7 +89,7 @@ def _live_check(session: "_discovery.Session") -> None: try: text = client.ask("Reply with the single word: pong", model="auto") ok = isinstance(text, str) and len(text) > 0 - transport = "chat.completions" if _scenarios.supports_openai(session.transport) else "messages" + transport = "messages" if _scenarios.supports_anthropic(session.transport) else "chat.completions" _row("llm test", transport, OK if ok else FAIL) except Exception as exc: # noqa: BLE001 _row("llm test", "round-trip", FAIL) diff --git a/sdk/python/tingly/client.py b/sdk/python/tingly/client.py index a8334726e..91719dd0a 100644 --- a/sdk/python/tingly/client.py +++ b/sdk/python/tingly/client.py @@ -107,13 +107,13 @@ def ask( ): """One-shot prompt → text, routed through tingly-box. - Picks the transport from the scenario: OpenAI-capable scenarios use - chat completions; Anthropic-only scenarios use messages. ``model="auto"`` - lets the gateway route. + Picks the transport from the scenario: Anthropic messages is tried + first (tb's native protocol); OpenAI-only scenarios fall back to chat + completions. ``model="auto"`` lets the gateway route. """ - if _scenarios.supports_openai(self._session.transport): - return self._ask_openai(prompt, model, system, stream, **kwargs) - return self._ask_anthropic(prompt, model, system, max_tokens, stream, **kwargs) + if _scenarios.supports_anthropic(self._session.transport): + return self._ask_anthropic(prompt, model, system, max_tokens, stream, **kwargs) + return self._ask_openai(prompt, model, system, stream, **kwargs) def _ask_openai(self, prompt, model, system, stream, **kwargs): messages = [] @@ -135,9 +135,8 @@ def _stream_openai(resp) -> Iterator[str]: yield delta def _ask_anthropic(self, prompt, model, system, max_tokens, stream, **kwargs): - anthropic_model = model if model != "auto" else "claude-sonnet-4-6" params = dict( - model=anthropic_model, + model=model, max_tokens=max_tokens, messages=[{"role": "user", "content": prompt}], **kwargs, diff --git a/sdk/python/tingly/plugin/__init__.py b/sdk/python/tingly/plugin/__init__.py index 290a33c75..f35149cd9 100644 --- a/sdk/python/tingly/plugin/__init__.py +++ b/sdk/python/tingly/plugin/__init__.py @@ -1,7 +1,8 @@ """tingly.plugin — write an AI server tingly-box can route to as a model. -This is Layer 2 of the SDK. A :class:`Plugin` is an OpenAI-compatible upstream; -register it as a provider in tingly-box and any client can select ``model_id``, +This is Layer 2 of the SDK. A :class:`Plugin` is an upstream tb can call as +Anthropic Messages (primary) or OpenAI chat completions (secondary); register +it as a provider in tingly-box and any client can select ``model_id``, inheriting the gateway's routing / fallback / guard rails / quota / logging. """ diff --git a/sdk/python/tingly/plugin/core.py b/sdk/python/tingly/plugin/core.py index 5847c8f83..948434408 100644 --- a/sdk/python/tingly/plugin/core.py +++ b/sdk/python/tingly/plugin/core.py @@ -1,10 +1,12 @@ """The ``Plugin`` class — write an AI server that tingly-box can route to. -A plugin is an OpenAI-compatible upstream: the author registers a single chat -handler, and ``serve()`` runs the HTTP server. tingly-box is then pointed at it -as a provider (``api_base = http://host:port``, model ``model_id``), so the -plugin composes with routing, fallback, guard rails, quota and logging like any -other model. +A plugin is an upstream tb can call two ways — Anthropic Messages (primary) +and OpenAI chat completions (secondary), both always served regardless of +registration — the author registers a single chat handler, and ``serve()`` +runs the HTTP server. tingly-box is then pointed at it as a provider +(``api_base = http://host:port``, model ``model_id``, ``api_style`` picking +which route tb calls), so the plugin composes with routing, fallback, guard +rails, quota and logging like any other model. The handler may call **back into** tingly-box via ``plugin.llm`` (a Layer-1 :class:`~tingly.client.Client`) for its own LLM needs — the recursion shown in @@ -50,6 +52,7 @@ def __init__( description: str = "", api_key: str = "", scenario: str = "experiment", + api_style: str = "anthropic", ): self.name = name self.model_id = model_id or f"plugin/{name}" @@ -57,6 +60,10 @@ def __init__( self.description = description self.api_key = api_key self.scenario = scenario + # Which wire protocol tb should use to *call* this plugin + # (/v1/messages vs /v1/chat/completions) — the server answers both + # regardless, this only picks what registration advertises. + self.api_style = api_style self._handler: Optional[ChatHandler] = None self._clients: dict = {} # scenario -> lazily-connected client @@ -113,14 +120,17 @@ def _dispatch(self, req: ChatRequest) -> HandlerResult: # -- manifest -------------------------------------------------------- - def manifest(self, entrypoint: str, port: int = 8765, transport: str = "openai") -> Manifest: - """Build a :class:`Manifest` describing this plugin for tingly-box.""" + def manifest(self, entrypoint: str, port: int = 8765, transport: Optional[str] = None) -> Manifest: + """Build a :class:`Manifest` describing this plugin for tingly-box. + + ``transport`` defaults to :attr:`api_style`. + """ return Manifest( name=self.name, model_id=self.model_id, entrypoint=entrypoint, version=self.version, - transport=transport, + transport=transport or self.api_style, port=port, description=self.description, ) @@ -198,6 +208,7 @@ def _register(self, host: str, port: int, verbose: bool) -> None: result = register( self.name, endpoint, self.model_id, scenario=self.scenario, token=self.api_key, + api_style=self.api_style, ) except Exception as exc: # noqa: BLE001 - registration is best-effort if verbose: diff --git a/sdk/python/tingly/plugin/manifest.py b/sdk/python/tingly/plugin/manifest.py index 513dd76ec..e38e2e04f 100644 --- a/sdk/python/tingly/plugin/manifest.py +++ b/sdk/python/tingly/plugin/manifest.py @@ -9,7 +9,9 @@ model_id = "plugin/my-rag" version = "0.1.0" entrypoint = "rag_plugin:plugin" # module:attr that yields a Plugin - transport = "openai" # openai | anthropic + transport = "anthropic" # anthropic (primary) | openai (secondary) — the + # wire protocol tb should use to call this plugin; + # the server always answers both regardless port = 8765 description = "Answers from my private corpus" """ @@ -38,7 +40,7 @@ class Manifest: model_id: str entrypoint: str version: str = "0.1.0" - transport: str = "openai" + transport: str = "anthropic" port: int = 8765 description: str = "" @@ -73,7 +75,7 @@ def load(path: Path) -> Manifest: model_id=plugin.get("model_id", f"plugin/{plugin['name']}"), entrypoint=plugin["entrypoint"], version=plugin.get("version", "0.1.0"), - transport=plugin.get("transport", "openai"), + transport=plugin.get("transport", "anthropic"), port=int(plugin.get("port", 8765)), description=plugin.get("description", ""), ) diff --git a/sdk/python/tingly/plugin/register.py b/sdk/python/tingly/plugin/register.py index b78862548..5faea143d 100644 --- a/sdk/python/tingly/plugin/register.py +++ b/sdk/python/tingly/plugin/register.py @@ -39,17 +39,25 @@ def register( scenario: Optional[str] = None, token: str = "", tier: int = 0, + api_style: str = "anthropic", gateway_url: Optional[str] = None, admin_token: Optional[str] = None, timeout: float = 30.0, ) -> RegisterResult: - """Register (or update) this plugin as a tb provider, optionally binding a rule.""" + """Register (or update) this plugin as a tb provider, optionally binding a rule. + + ``api_style`` tells tb which wire protocol to use when it calls this + plugin's endpoint — ``"anthropic"`` (the plugin server's primary route, + ``/v1/messages``) or ``"openai"`` (``/v1/chat/completions``). The plugin + server answers both regardless; this only picks which one tb sends. + """ resolved = _config.resolve(base_url=gateway_url, token=admin_token) headers = {"Authorization": f"Bearer {resolved.token or ''}"} url = resolved.base_url.rstrip("/") + "/api/v2/plugins" body = { "name": name, "endpoint": endpoint, "model_id": model_id, "scenario": scenario or "", "token": token, "tier": tier, + "api_style": api_style, } try: resp = httpx.post(url, json=body, headers=headers, timeout=timeout) diff --git a/sdk/python/tingly/plugin/server.py b/sdk/python/tingly/plugin/server.py index 8b2a90ef5..e5bd981d8 100644 --- a/sdk/python/tingly/plugin/server.py +++ b/sdk/python/tingly/plugin/server.py @@ -1,15 +1,21 @@ -"""A tiny OpenAI-compatible HTTP server for plugins (stdlib only). +"""A tiny HTTP server for plugins (stdlib only), speaking two wire protocols. -Exposes exactly what tingly-box needs to treat the plugin as an upstream -OpenAI provider: +Exposes exactly what tingly-box needs to treat the plugin as an upstream: - POST /v1/chat/completions -> chat.completion (+ SSE when stream=true) - GET /v1/models -> the plugin's model id - GET /health -> liveness + POST /v1/messages -> Anthropic message (+ SSE when stream=true) — primary + POST /v1/chat/completions -> OpenAI chat.completion (+ SSE) — secondary + GET /v1/models -> the plugin's model id + GET /health -> liveness + +Anthropic is primary because that is tingly-box's native protocol; OpenAI +chat completions is kept as a secondary, equally-real path (not a shim) since +many callers still speak it. Both routes share the same normalized +:class:`ChatRequest` and the same handler — the author's code never sees the +wire-format difference, only the response shaping differs per route. No framework dependency — uses ``http.server.ThreadingHTTPServer`` so a plugin stays a single ``pip install tingly`` away. Streaming is real SSE: a handler -that returns an iterator is emitted as ``chat.completion.chunk`` frames. +that returns an iterator is emitted as protocol-appropriate chunk/event frames. """ from __future__ import annotations @@ -19,6 +25,7 @@ import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, Callable, Dict, Iterable, Iterator, Tuple, Union +from urllib.parse import urlsplit from .types import ChatRequest @@ -37,41 +44,65 @@ def log_message(self, fmt, *args): # quiet by default; plugin owns logging # -- routing --------------------------------------------------------- + def _route_path(self) -> str: + """The request path with any query string (e.g. tb's ``?beta=true`` + on Anthropic calls) and trailing slash stripped.""" + return urlsplit(self.path).path.rstrip("/") + def do_GET(self): - if self.path.rstrip("/") == "/health": + path = self._route_path() + if path == "/health": return self._json(200, {"status": "ok"}) - if self.path.rstrip("/") in ("/v1/models", "/models"): + if path in ("/v1/models", "/models"): return self._models() return self._json(404, {"error": {"message": "not found", "type": "not_found"}}) def do_POST(self): - if self.path.rstrip("/") not in ("/v1/chat/completions", "/chat/completions"): - return self._json( - 404, {"error": {"message": "not found", "type": "not_found"}} - ) + path = self._route_path() + if path in ("/v1/messages", "/messages"): + return self._handle_chat(anthropic=True) + if path in ("/v1/chat/completions", "/chat/completions"): + return self._handle_chat(anthropic=False) + return self._json(404, {"error": {"message": "not found", "type": "not_found"}}) + + def _handle_chat(self, anthropic: bool): if not self._authorized(): + if anthropic: + return self._json( + 401, {"type": "error", "error": {"type": "authentication_error", "message": "invalid token"}} + ) return self._json( 401, {"error": {"message": "invalid token", "type": "auth_error"}} ) body = self._read_json() if body is None: + if anthropic: + return self._json( + 400, {"type": "error", "error": {"type": "invalid_request_error", "message": "invalid JSON body"}} + ) return self._json( 400, {"error": {"message": "invalid JSON body", "type": "invalid_request_error"}} ) - req = ChatRequest.from_openai_body(body) + req = ( + ChatRequest.from_anthropic_body(body) if anthropic else ChatRequest.from_openai_body(body) + ) try: result = self.server.dispatch(req) # type: ignore[attr-defined] except Exception as exc: # noqa: BLE001 - surface as upstream 500 + if anthropic: + return self._json( + 500, {"type": "error", "error": {"type": "api_error", "message": f"plugin handler error: {exc}"}} + ) return self._json( 500, {"error": {"message": f"plugin handler error: {exc}", "type": "api_error"}}, ) model = req.model or self.server.model_id # type: ignore[attr-defined] - if req.stream: - return self._stream(result, model) - return self._complete(result, model) + if anthropic: + return self._stream_anthropic(result, model) if req.stream else self._complete_anthropic(result, model) + return self._stream(result, model) if req.stream else self._complete(result, model) # -- responses ------------------------------------------------------- @@ -145,6 +176,63 @@ def frame(delta: Dict[str, Any], finish: Any = None) -> bytes: self.wfile.write(b"data: [DONE]\n\n") self.wfile.flush() + def _complete_anthropic(self, result: HandlerResult, model: str): + text = result if isinstance(result, str) else "".join(result) + self._json( + 200, + { + "id": _msg_id(), + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + }, + ) + + def _stream_anthropic(self, result: HandlerResult, model: str): + mid = _msg_id() + self.close_connection = True + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + + def event(name: str, payload: Dict[str, Any]) -> bytes: + return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode("utf-8") + + self.wfile.write(event("message_start", { + "type": "message_start", + "message": { + "id": mid, "type": "message", "role": "assistant", "model": model, + "content": [], "stop_reason": None, "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + }, + })) + self.wfile.write(event("content_block_start", { + "type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": ""}, + })) + chunks: Iterator[str] = iter([result]) if isinstance(result, str) else iter(result) + for piece in chunks: + if piece: + self.wfile.write(event("content_block_delta", { + "type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": piece}, + })) + self.wfile.flush() + self.wfile.write(event("content_block_stop", {"type": "content_block_stop", "index": 0})) + self.wfile.write(event("message_delta", { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 0}, + })) + self.wfile.write(event("message_stop", {"type": "message_stop"})) + self.wfile.flush() + # -- helpers --------------------------------------------------------- def _authorized(self) -> bool: @@ -176,6 +264,10 @@ def _chat_id() -> str: return "chatcmpl-" + uuid.uuid4().hex[:24] +def _msg_id() -> str: + return "msg_" + uuid.uuid4().hex[:24] + + def make_server( dispatch: Dispatch, model_id: str, diff --git a/sdk/python/tingly/plugin/types.py b/sdk/python/tingly/plugin/types.py index 8d054af8c..c757646e8 100644 --- a/sdk/python/tingly/plugin/types.py +++ b/sdk/python/tingly/plugin/types.py @@ -58,6 +58,33 @@ def from_openai_body(cls, body: Dict[str, Any]) -> "ChatRequest": extra=extra, ) + @classmethod + def from_anthropic_body(cls, body: Dict[str, Any]) -> "ChatRequest": + """Normalize an Anthropic Messages API body (POST /v1/messages). + + Anthropic keeps the system prompt in a top-level ``system`` field + rather than a message with ``role: "system"``; it is folded into the + message list as a leading system message so handlers can use + :meth:`system_text` / :meth:`last_user_text` the same way regardless + of which wire protocol the caller used. + """ + raw_messages = body.get("messages") or [] + messages = [ + Message(role=m.get("role", "user"), content=_content_to_text(m.get("content"))) + for m in raw_messages + ] + system_text = _content_to_text(body.get("system")) + if system_text: + messages.insert(0, Message(role="system", content=system_text)) + known = {"model", "messages", "system", "stream"} + extra = {k: v for k, v in body.items() if k not in known} + return cls( + model=body.get("model", ""), + messages=messages, + stream=bool(body.get("stream", False)), + extra=extra, + ) + def _content_to_text(content: Any) -> str: """Flatten OpenAI content (str or list of parts) to plain text.""" From 750e76498c8f7979294e7f6c58ff66125be2fddc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 11:30:31 +0000 Subject: [PATCH 19/28] feat(sdk): add critic and fusion showcase plugins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new example plugins demonstrating a plugin composing tb by calling back into other rules more than once, grounded in patterns already in real-world use rather than invented for the demo: - critic_plugin.py (model="plugin/critic"): cross-model critique — forwards the thing to review to a different rule/model, returns a structured verdict. Self-critique is unreliable (Huang et al., ICLR 2024); this is the pattern behind Zen MCP, Consult7, and aider's architect/editor split. Named "critic" rather than "advisor" deliberately — tb already has an unrelated, in-process "advisor" MCP tool, and reusing the name would repeat the naming collision already flagged in .design/python-sdk.md for "Plugin"/"Plugins". - fusion_plugin.py (model="plugin/fusion"): multi-model consensus — polls a panel of rules/models concurrently, skips the judge call when the panel already agrees, otherwise a judge call synthesizes. Mirrors Consult7's Fusion feature; the clearest illustration that a plugin can originate calls against any number of other rules per request. Both have unit tests (tests/test_example_plugins.py) pinning the branching logic via a monkeypatched plugin.use(), and were smoke-tested live (serve + /health + /v1/models) before committing. Design doc and README updated to reference them. --- .design/python-sdk.md | 40 +++++++ sdk/python/README.md | 21 ++++ sdk/python/examples/critic_plugin.py | 89 +++++++++++++++ sdk/python/examples/fusion_plugin.py | 81 ++++++++++++++ sdk/python/tests/test_example_plugins.py | 136 +++++++++++++++++++++++ 5 files changed, 367 insertions(+) create mode 100644 sdk/python/examples/critic_plugin.py create mode 100644 sdk/python/examples/fusion_plugin.py create mode 100644 sdk/python/tests/test_example_plugins.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index df37edf06..4a5ecf599 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -524,6 +524,46 @@ their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy mount, and the install/enable/logs/disable lifecycle UI. The Python side and the provider wiring are complete; those are the remaining backend pieces. +### Example plugins (`sdk/python/examples/`) + +Three, each a different real-world shape of "plugin composes the box by +calling back into other rules" — not toys picked at random, each maps onto a +pattern already in wide use: + +- **`rag_plugin.py`** — one call back into tb for generation over retrieved + context. The baseline shape. +- **`critic_plugin.py`** (`model="plugin/critic"`) — cross-model critique: + forwards the artifact-to-review to a *different* rule/model, returns a + structured `{verdict, issues, suggestion}`. Chosen over self-critique + deliberately: Huang et al. (ICLR 2024) found LLMs can't reliably + self-correct without external feedback, so a plugin reviewing with a + different model is the robust variant, not a stylistic choice. This is the + pattern behind [Zen MCP](https://github.com/jray2123/zen-mcp-server) and + [Consult7](https://github.com/szeider/consult7) (both real MCP servers + coding agents use today to consult another model mid-task) and behind + aider's architect/editor split. Named "critic" deliberately, not "advisor" + — tb already has an unrelated, in-process `advisor` MCP tool + (`internal/mcp/runtime/advisor_virtual.go` + the response-hook machinery in + `internal/server/servertool/`); reusing that name for an architecturally + different thing (plugin-as-upstream calling back into the gateway, vs. a + direct in-process upstream call) would be the same collision already + flagged above for "Plugin" vs. rule-flag "Plugins" — same fix, applied + before it started rather than after. +- **`fusion_plugin.py`** (`model="plugin/fusion"`) — multi-model consensus: + polls a panel of rules/models concurrently (`ThreadPoolExecutor`), skips + the judge call when the panel already agrees, otherwise a judge call + synthesizes. Mirrors Consult7's 2026 Fusion feature (a panel of frontier + models answers in parallel, a judge model merges). The clearest + illustration of the architecture line at the top of this document — a + plugin can freely originate calls against *any* number of other rules, not + just one. + +Both `critic_plugin.py` and `fusion_plugin.py` have unit tests +(`tests/test_example_plugins.py`) that monkeypatch `plugin.use` to a fake +client and pin the branching logic (JSON-verdict formatting and graceful +degradation on non-JSON; judge-skipped-on-agreement vs. judge-called-on- +disagreement) without needing a live tb. + ## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) Layer 1 points the **data-flow into** tb: the plugin is a *consumer*. For tb to diff --git a/sdk/python/README.md b/sdk/python/README.md index a2af76f6e..4a559816c 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -96,6 +96,27 @@ The server is stdlib-only (no FastAPI), supports streaming on both routes, and `plugin.llm` calls back into tb (Anthropic-first) so the plugin reuses the gateway for its own LLM work. +### Example plugins + +`sdk/python/examples/` has three, each demonstrating a different real-world +pattern for the same idea — a plugin composing the box by calling back into +other tb rules: + +- **`rag_plugin.py`** — retrieval-augmented answers from a toy corpus, one + call back into tb for generation. +- **`critic_plugin.py`** — cross-model critique (`model="plugin/critic"`): + forwards the thing to review to a *different* rule/model and returns a + structured verdict. Self-critique is unreliable (a model can't reliably + catch its own mistakes); this is the pattern behind + [Zen MCP](https://github.com/jray2123/zen-mcp-server) and + [Consult7](https://github.com/szeider/consult7), and behind aider's + architect/editor split. +- **`fusion_plugin.py`** — multi-model consensus (`model="plugin/fusion"`): + polls a panel of rules/models concurrently, skips the judge call when they + already agree, otherwise a judge call synthesizes. Mirrors Consult7's 2026 + Fusion feature; the clearest illustration that a plugin can freely + originate more than one call, against more than one rule, per request. + ## Status - **Layer 1** (consume tb): `connect()` → `Client`. Done — Anthropic tried diff --git a/sdk/python/examples/critic_plugin.py b/sdk/python/examples/critic_plugin.py new file mode 100644 index 000000000..10c157719 --- /dev/null +++ b/sdk/python/examples/critic_plugin.py @@ -0,0 +1,89 @@ +"""A "critic" plugin: cross-model critique — the pattern behind Zen MCP and +Consult7 (an agent mid-task consults a *different* model for review) and +aider's architect/editor split (a separate model reviews before code lands). + +Self-critique — a model reviewing its own output — is unreliable: Huang et +al. (ICLR 2024) found LLMs cannot reliably self-correct without external +feedback. Cross-model critique (a genuinely different model reviews) is the +more robust variant, and it maps directly onto what a plugin is for: this +handler does zero LLM work itself, it only forwards the artifact-to-review to +a different tb rule/model via `plugin.use(...)` and shapes the structured +verdict that comes back. No hard-coded provider or key — same gateway, +different rule. + +Run it (serves on :8766 AND registers with tb on startup): + + pip install -e . # from sdk/python + python examples/critic_plugin.py + +Then from any tb client: model="plugin/critic", the message is the thing to +review (a diff, a draft answer, a decision); an optional system message adds +context the critic should weigh. +""" + +from __future__ import annotations + +import json + +from tingly import ChatRequest, Plugin + +# Where the critique itself is delegated — point CRITIC_SCENARIO / CRITIC_MODEL +# at a rule bound to a genuinely different (ideally stronger) model than +# whatever called this plugin; reviewing with the same model defeats the point. +CRITIC_SCENARIO = "experiment" +CRITIC_MODEL = "auto" + +CRITIQUE_PROMPT = """You are reviewing the following for correctness, risk and \ +missing considerations. Respond with ONLY JSON matching: +{{"verdict": "approve" | "revise", "issues": ["..."], "suggestion": "..."}} + +--- context --- +{context} + +--- to review --- +{content} +""" + +plugin = Plugin( + name="critic", + scenario="experiment", # bind a rule under this scenario on register + description="Cross-model critique — delegates review to a different rule/model", +) + + +@plugin.chat +def handle(req: ChatRequest) -> str: + content = req.last_user_text() + context = req.system_text() or "(none)" + prompt = CRITIQUE_PROMPT.format(context=context, content=content) + + # The one line that matters: hand the review to a DIFFERENT tb rule. + raw = plugin.use(CRITIC_SCENARIO).ask(prompt, model=CRITIC_MODEL, max_tokens=1024) + return _format_verdict(_parse_verdict(raw)) + + +def _parse_verdict(raw: str) -> dict: + text = raw.strip() + if text.startswith("```"): + text = text.strip("`") + text = text.split("\n", 1)[1] if "\n" in text else text + try: + return json.loads(text) + except ValueError: + # The critic model didn't follow the JSON contract — degrade to a + # plain "revise" verdict carrying its raw text as the suggestion, + # rather than crashing the request. + return {"verdict": "revise", "issues": ["critic model did not return JSON"], "suggestion": raw} + + +def _format_verdict(verdict: dict) -> str: + lines = [f"verdict: {verdict.get('verdict', 'unknown')}"] + for issue in verdict.get("issues") or []: + lines.append(f"- {issue}") + if verdict.get("suggestion"): + lines.append(f"suggestion: {verdict['suggestion']}") + return "\n".join(lines) + + +if __name__ == "__main__": + plugin.serve(port=8766) diff --git a/sdk/python/examples/fusion_plugin.py b/sdk/python/examples/fusion_plugin.py new file mode 100644 index 000000000..823f5f4d3 --- /dev/null +++ b/sdk/python/examples/fusion_plugin.py @@ -0,0 +1,81 @@ +"""A "fusion" plugin: parallel multi-model consensus, then a judge model +synthesizes — the pattern behind Consult7's 2026 Fusion feature (a panel of +frontier models answers in parallel; a judge model merges the answers; a +panel that already agrees skips the judge call). + +This is the clearest illustration of "tb is a hub of rules; a plugin can +freely originate calls against any of them": the handler calls BACK into tb +more than once, against DIFFERENT rules/models, concurrently, before +answering once. + +Run it (serves on :8767 AND registers with tb on startup): + + pip install -e . # from sdk/python + python examples/fusion_plugin.py + +Then from any tb client: model="plugin/fusion", the message is the question. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor + +from tingly import ChatRequest, Plugin + +# The panel: each entry is (scenario, model) called independently and +# concurrently. Point these at genuinely different rules/models — a panel of +# clones of the same model adds latency without adding a second opinion. +PANEL = [ + ("experiment", "auto"), + ("experiment", "auto"), +] +JUDGE_SCENARIO = "experiment" +JUDGE_MODEL = "auto" + +JUDGE_PROMPT = """Multiple models answered the same question independently. \ +Synthesize the single best answer, resolving disagreements and noting when \ +the panel disagreed. + +--- question --- +{question} + +--- panel answers --- +{answers} +""" + +plugin = Plugin( + name="fusion", + scenario="experiment", # bind a rule under this scenario on register + description="Multi-model consensus — panel of rules/models + judge synthesis", +) + + +@plugin.chat +def handle(req: ChatRequest) -> str: + question = req.last_user_text() + answers = _poll_panel(question) + + if len(set(answers)) == 1: + # The panel already agreed — the judge call would just restate this, + # so skip it and save a hop (mirrors Consult7 skipping the panel + # entirely for trivial prompts). + return answers[0] + + answers_block = "\n\n".join(f"[{i + 1}] {a}" for i, a in enumerate(answers)) + return plugin.use(JUDGE_SCENARIO).ask( + JUDGE_PROMPT.format(question=question, answers=answers_block), + model=JUDGE_MODEL, + ) + + +def _poll_panel(question: str) -> list: + with ThreadPoolExecutor(max_workers=len(PANEL)) as pool: + futures = [ + pool.submit(plugin.use(scenario).ask, question, model=model) + for scenario, model in PANEL + ] + return [f.result() for f in futures] + + +if __name__ == "__main__": + plugin.serve(port=8767) diff --git a/sdk/python/tests/test_example_plugins.py b/sdk/python/tests/test_example_plugins.py new file mode 100644 index 000000000..e18a0ecf9 --- /dev/null +++ b/sdk/python/tests/test_example_plugins.py @@ -0,0 +1,136 @@ +"""Tests for the critic/fusion showcase plugins (sdk/python/examples/). + +These exercise handler logic only — plugin.use() is monkeypatched to a fake +client, so no real tb and no real model calls. The examples aren't part of +the installed `tingly` package, so they're loaded by file path, the same way +`tingly plugin run` loads a user's plugin script (see tingly/cli.py). +""" + +import importlib.util +import sys +from pathlib import Path + +from tingly.plugin.types import ChatRequest + +EXAMPLES = Path(__file__).parent.parent / "examples" + + +def _load(name): + path = EXAMPLES / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +def _req(content, system=None): + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": content}) + return ChatRequest.from_openai_body({"model": "x", "messages": messages}) + + +class _FakeClient: + """Stands in for a tingly.Client — records calls, replies with a fixed + string or (if given a callable) the result of calling it with the prompt.""" + + def __init__(self, reply): + self._reply = reply + self.calls = [] + + def ask(self, prompt, **kwargs): + self.calls.append((prompt, kwargs)) + return self._reply(prompt) if callable(self._reply) else self._reply + + +# -- critic ----------------------------------------------------------------- + +def test_critic_formats_valid_json_verdict(monkeypatch): + critic = _load("critic_plugin") + fake = _FakeClient('{"verdict": "approve", "issues": [], "suggestion": "looks good"}') + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("def f(): return 1/0", system="a python snippet")) + + assert result == "verdict: approve\nsuggestion: looks good" + prompt, kwargs = fake.calls[0] + assert "a python snippet" in prompt + assert "def f(): return 1/0" in prompt + assert kwargs["model"] == critic.CRITIC_MODEL + + +def test_critic_lists_issues(monkeypatch): + critic = _load("critic_plugin") + fake = _FakeClient('{"verdict": "revise", "issues": ["divides by zero"], "suggestion": "guard the denominator"}') + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("def f(): return 1/0")) + + assert "verdict: revise" in result + assert "- divides by zero" in result + assert "suggestion: guard the denominator" in result + + +def test_critic_degrades_gracefully_on_non_json(monkeypatch): + """A critic model that ignores the JSON contract must not crash the + request — it should surface as a 'revise' verdict carrying the raw text.""" + critic = _load("critic_plugin") + fake = _FakeClient("looks fine to me") + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("some code")) + + assert "verdict: revise" in result + assert "looks fine to me" in result + + +def test_critic_strips_markdown_code_fence(monkeypatch): + critic = _load("critic_plugin") + fake = _FakeClient('```json\n{"verdict": "approve", "issues": [], "suggestion": ""}\n```') + monkeypatch.setattr(critic.plugin, "use", lambda scenario: fake) + + result = critic.handle(_req("some code")) + + assert result == "verdict: approve" + + +# -- fusion ------------------------------------------------------------- + +def test_poll_panel_gathers_one_result_per_panel_entry(monkeypatch): + fusion = _load("fusion_plugin") + fake = _FakeClient("same-answer") + monkeypatch.setattr(fusion.plugin, "use", lambda scenario: fake) + + results = fusion._poll_panel("q") + + assert results == ["same-answer"] * len(fusion.PANEL) + assert len(fake.calls) == len(fusion.PANEL) + + +def test_fusion_skips_judge_when_panel_agrees(monkeypatch): + fusion = _load("fusion_plugin") + monkeypatch.setattr(fusion, "_poll_panel", lambda question: ["42", "42"]) + + def judge_should_not_be_called(scenario): + raise AssertionError("judge must not be called when the panel agrees") + + monkeypatch.setattr(fusion.plugin, "use", judge_should_not_be_called) + + assert fusion.handle(_req("what is 6*7?")) == "42" + + +def test_fusion_calls_judge_when_panel_disagrees(monkeypatch): + fusion = _load("fusion_plugin") + monkeypatch.setattr(fusion, "_poll_panel", lambda question: ["A", "B"]) + judge = _FakeClient("SYNTHESIZED") + monkeypatch.setattr(fusion.plugin, "use", lambda scenario: judge) + + result = fusion.handle(_req("question")) + + assert result == "SYNTHESIZED" + assert len(judge.calls) == 1 + judge_prompt = judge.calls[0][0] + assert "A" in judge_prompt and "B" in judge_prompt + assert "question" in judge_prompt From f386ac165ff6007c4c6ace0ccde21161c3258c25 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 11:40:46 +0000 Subject: [PATCH 20/28] feat(sdk): add Client.quota view and a quota-aware router plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the "dispatch" plugin shape requested: a plugin that doesn't generate an answer itself, it picks which single candidate rule/model to forward to and sends the request there only. - tingly/helpers/quota.py: QuotaView (list/get/batch/refresh) + ProviderQuota/ UsageWindow, wrapping tb's internal/server/module/providerquota endpoints (admin token, same apiV1 auth group as usage/guardrails). The three response shapes (envelope / bare / uuid-keyed map) are pinned exactly as the Go handler returns them, verified by reading handler.go directly since this module isn't swagger-annotated. headroom_percent collapses a provider's multiple usage windows (session/daily/monthly/balance/...) to the single most-constrained one, for a routing pick. - Client.quota property, alongside the existing .usage / .guardrails views. - examples/router_plugin.py (model="plugin/router"): quota-aware dispatch — the same idea as LiteLLM Router's usage-based-routing strategy, picking the candidate with the most remaining headroom and forwarding to just that one. Reads cached quota by default (LiteLLM's own docs warn a live check on every request adds real latency); .quota.refresh() is opt-in. - Unit tests: tests/test_quota.py (respx-mocked, pins the three response shapes + the headroom heuristics), tests/test_router_plugin.py (pins highest-headroom selection, "unknown" quota treated as unconstrained not zero, and single-hop forwarding — only the chosen candidate's scenario is ever touched). - Confirmed no built-in quota-aware routing exists in tb's Go gateway (internal/smart_routing, internal/loadbalance) — this is genuinely new behavior, not a Python reimplementation of existing gateway logic; noted in the design doc. Design doc and README updated to match. --- .design/python-sdk.md | 65 ++++++++-- sdk/python/README.md | 6 + sdk/python/examples/router_plugin.py | 76 ++++++++++++ sdk/python/tests/test_quota.py | 116 ++++++++++++++++++ sdk/python/tests/test_router_plugin.py | 110 +++++++++++++++++ sdk/python/tingly/client.py | 5 + sdk/python/tingly/helpers/quota.py | 158 +++++++++++++++++++++++++ 7 files changed, 528 insertions(+), 8 deletions(-) create mode 100644 sdk/python/examples/router_plugin.py create mode 100644 sdk/python/tests/test_quota.py create mode 100644 sdk/python/tests/test_router_plugin.py create mode 100644 sdk/python/tingly/helpers/quota.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 4a5ecf599..16dbdd4d3 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -190,7 +190,7 @@ sdk/python/ config.py # (base_url, admin_token) resolution precedence scenarios.py # scenario + transport constants transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb - helpers/ # usage + guardrails views + helpers/ # usage + guardrails + quota views plugin/ # Layer 2: be an AI server tb routes to core.py # Plugin class (@plugin.chat, .llm, .serve, api_style) server.py # stdlib HTTP server: /v1/messages (primary) + /v1/chat/completions (secondary), + SSE @@ -215,6 +215,7 @@ connect(scenario="experiment") .ask() → Anthropic first when the scenario supports both, else OpenAI .usage → GET /api/v1/requests (admin token) .guardrails → GET /api/v1/guardrails/config (admin token) + .quota → GET/POST /api/v1/provider-quota[...] (admin token) ``` ## How it works (pencil) @@ -524,9 +525,44 @@ their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy mount, and the install/enable/logs/disable lifecycle UI. The Python side and the provider wiring are complete; those are the remaining backend pieces. +### `Client.quota` — provider usage/limit windows, and a live refresh + +Added for `router_plugin.py` below, but attached to `Client` like `.usage` / +`.guardrails` so any caller can use it. Wraps +`GET /api/v1/provider-quota[...]` (`internal/server/module/providerquota/`, +admin token, same `apiV1` auth-middleware group as usage/guardrails): + +| SDK call | endpoint | shape | +|---|---|---| +| `quota.list()` | `GET /provider-quota` | `{meta, data:[ProviderUsage]}` | +| `quota.get(uuid)` | `GET /provider-quota/:uuid` | bare `ProviderUsage` (no envelope) | +| `quota.batch(uuids)` | `POST /provider-quota/batch` | `{data: {uuid: ProviderUsage}}` | +| `quota.refresh(uuid?)` | `POST /provider-quota/:uuid?/refresh` | live re-fetch from the upstream account, bypassing tb's cache | + +These three response shapes are genuinely different (envelope vs. bare vs. +uuid-keyed map) — not a Python-side inconsistency, that's what the Go handler +(`internal/server/module/providerquota/handler.go:66-177`) actually returns +for each; `QuotaView._from_json`-style parsing per method is intentional, not +an oversight. `provider-quota` isn't in `openapi.json` (no swagger tags on +that module yet), so these shapes were pinned by reading the handler +directly, not generated — worth re-checking if that module ever gets +swagger-annotated. + +A provider's quota is **not one number** — `ProviderUsage.windows` is a list +(session/daily/weekly/monthly/balance/model/...), each with its own +`used`/`limit`/`used_percent` (`ai/quota/types.go`). `ProviderQuota.headroom_percent` +collapses that to the single most-constrained window's remaining percent — +a deliberately naive heuristic for "which candidate is worse off right now" +in a routing pick, not a replacement for reading `.windows` when the +distinction between e.g. a session limit and a monthly cost budget matters. +tb itself has **no built-in quota-aware routing** (`internal/smart_routing` +and `internal/loadbalance` have zero references to `ai/quota` as of this +writing) — a plugin picking by remaining quota is genuinely new behavior, +not a Python reimplementation of something the gateway already does. + ### Example plugins (`sdk/python/examples/`) -Three, each a different real-world shape of "plugin composes the box by +Four, each a different real-world shape of "plugin composes the box by calling back into other rules" — not toys picked at random, each maps onto a pattern already in wide use: @@ -557,12 +593,25 @@ pattern already in wide use: illustration of the architecture line at the top of this document — a plugin can freely originate calls against *any* number of other rules, not just one. - -Both `critic_plugin.py` and `fusion_plugin.py` have unit tests -(`tests/test_example_plugins.py`) that monkeypatch `plugin.use` to a fake -client and pin the branching logic (JSON-verdict formatting and graceful -degradation on non-JSON; judge-skipped-on-agreement vs. judge-called-on- -disagreement) without needing a live tb. +- **`router_plugin.py`** (`model="plugin/router"`) — quota-aware dispatch: a + different shape from the three above, which all *generate* an answer + themselves. A router generates nothing — it picks the ONE candidate + `(scenario, model, provider_uuid)` with the most quota headroom (via the + `Client.quota` view above) and forwards to just that one; one hop total, + by design, not N. Same idea as LiteLLM Router's `usage-based-routing` + strategy (route to whichever deployment has the most remaining rate-limit + capacity), implemented as a plugin instead of gateway config — deliberately + reads cached quota by default and only calls `.quota.refresh()` when a + caller opts in, since LiteLLM's own docs warn that a live usage check on + every request adds real per-request latency. + +Every example plugin has unit tests (`tests/test_example_plugins.py`, +`tests/test_router_plugin.py`, `tests/test_quota.py`) that monkeypatch +`plugin.use`/`Client.quota` to fakes and pin the decision logic — JSON-verdict +formatting and graceful degradation on non-JSON (critic); judge-skipped-on- +agreement vs. judge-called-on-disagreement (fusion); highest-headroom +candidate selection and single-hop forwarding (router) — without needing a +live tb. ## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) diff --git a/sdk/python/README.md b/sdk/python/README.md index 4a559816c..4254bb284 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -116,6 +116,12 @@ other tb rules: already agree, otherwise a judge call synthesizes. Mirrors Consult7's 2026 Fusion feature; the clearest illustration that a plugin can freely originate more than one call, against more than one rule, per request. +- **`router_plugin.py`** — quota-aware dispatch (`model="plugin/router"`): a + different shape from the three above — it generates nothing itself, it + only *decides* which one candidate rule/model to forward to, using live + quota headroom (`tb.quota`, below) to pick. Same idea as LiteLLM Router's + `usage-based-routing` strategy, implemented as a plugin instead of gateway + config. ## Status diff --git a/sdk/python/examples/router_plugin.py b/sdk/python/examples/router_plugin.py new file mode 100644 index 000000000..0ec444162 --- /dev/null +++ b/sdk/python/examples/router_plugin.py @@ -0,0 +1,76 @@ +"""A "router" plugin: quota-aware dispatch — a different shape from +rag/critic/fusion. Those all *generate* an answer themselves (one or more +calls back into tb feed a response the plugin composes). A router generates +nothing; its only job is to DECIDE which one candidate rule/model actually +serves the request, then forward to just that one. + +This is the same idea as LiteLLM Router's `usage-based-routing` strategy — +route to whichever deployment has the most remaining rate-limit/quota +headroom right now, instead of a fixed priority order — implemented here as +a plugin instead of gateway config, using the SDK's quota views +(`Client.quota`, `sdk/python/tingly/helpers/quota.py`) added for exactly +this. + +Run it (serves on :8768 AND registers with tb on startup): + + pip install -e . # from sdk/python + python examples/router_plugin.py + +Then from any tb client: model="plugin/router", the message is the question. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from tingly import ChatRequest, Plugin + + +@dataclass +class Candidate: + scenario: str + model: str + provider_uuid: str # the provider backing (scenario, model) — quota is per-provider + + +# Fill in real provider UUIDs from the tb UI (Providers page) or +# `GET /api/v2/providers` — quota is tracked per provider, not per rule, so +# there is no way to infer these from the model name alone. +CANDIDATES = [ + Candidate(scenario="experiment", model="auto", provider_uuid="REPLACE_WITH_PROVIDER_UUID_1"), + Candidate(scenario="experiment", model="auto", provider_uuid="REPLACE_WITH_PROVIDER_UUID_2"), +] + +plugin = Plugin( + name="router", + scenario="experiment", # bind a rule under this scenario on register + description="Quota-aware dispatch — forwards to whichever candidate has the most headroom", +) + + +@plugin.chat +def handle(req: ChatRequest) -> str: + question = req.last_user_text() + chosen = _pick_candidate() + # The only call that matters: forward to the ONE chosen candidate, not + # every candidate — a router spends one hop total, not N (contrast with + # fusion_plugin.py, which deliberately spends N to get a second opinion). + return plugin.use(chosen.scenario).ask(question, model=chosen.model) + + +def _pick_candidate() -> Candidate: + """Cached quota (tb refreshes lazily, ~20 min TTL) is enough for most + routing decisions and costs nothing extra per request. Call + `plugin.llm.quota.refresh(uuid)` first, for a specific candidate, if a + request genuinely needs a number fresher than that — LiteLLM's own + usage-based-routing docs warn that a live check on every single request + adds real latency, so that should be the exception, not the default.""" + quotas = plugin.llm.quota.batch([c.provider_uuid for c in CANDIDATES]) + return max( + CANDIDATES, + key=lambda c: quotas[c.provider_uuid].headroom_percent if c.provider_uuid in quotas else 100.0, + ) + + +if __name__ == "__main__": + plugin.serve(port=8768) diff --git a/sdk/python/tests/test_quota.py b/sdk/python/tests/test_quota.py new file mode 100644 index 000000000..696b04abb --- /dev/null +++ b/sdk/python/tests/test_quota.py @@ -0,0 +1,116 @@ +"""QuotaView tests (gateway mocked with respx) — pin the exact response +shapes tb's provider-quota endpoints return (list/refresh wrap {meta,data}; +get/refresh-one are bare ProviderUsage; batch wraps {data: {uuid: usage}}), +plus the headroom_percent / remaining_percent heuristics used for routing. +""" + +import httpx +import respx + +from tingly.helpers.quota import ProviderQuota, QuotaView, UsageWindow + +BASE = "http://tb.test:12580" + + +def _view() -> QuotaView: + return QuotaView(BASE, "admin", 5.0) + + +def _window(**overrides): + base = {"key": "session", "type": "session", "used": 10, "limit": 100, "used_percent": 10} + base.update(overrides) + return base + + +@respx.mock +def test_list_unwraps_data_array(): + respx.get(f"{BASE}/api/v1/provider-quota").mock( + return_value=httpx.Response(200, json={ + "meta": {"total": 1, "updated_at": "2026-01-01T00:00:00Z"}, + "data": [{ + "provider_uuid": "p1", "provider_name": "Anthropic", "provider_type": "anthropic", + "windows": [_window()], + }], + }) + ) + result = _view().list() + assert len(result) == 1 + assert result[0].provider_uuid == "p1" + assert result[0].windows[0].used_percent == 10 + + +@respx.mock +def test_get_is_bare_provider_usage_no_envelope(): + respx.get(f"{BASE}/api/v1/provider-quota/p1").mock( + return_value=httpx.Response(200, json={ + "provider_uuid": "p1", "provider_name": "Anthropic", "provider_type": "anthropic", + "windows": [_window(used_percent=42)], + }) + ) + result = _view().get("p1") + assert result.provider_uuid == "p1" + assert result.windows[0].used_percent == 42 + + +@respx.mock +def test_batch_unwraps_uuid_keyed_map(): + respx.post(f"{BASE}/api/v1/provider-quota/batch").mock( + return_value=httpx.Response(200, json={ + "data": { + "p1": {"provider_uuid": "p1", "provider_name": "A", "provider_type": "anthropic", "windows": []}, + "p2": {"provider_uuid": "p2", "provider_name": "B", "provider_type": "openai", "windows": []}, + } + }) + ) + result = _view().batch(["p1", "p2"]) + assert set(result) == {"p1", "p2"} + assert result["p2"].provider_name == "B" + + +@respx.mock +def test_refresh_one_returns_bare_provider_usage(): + route = respx.post(f"{BASE}/api/v1/provider-quota/p1/refresh").mock( + return_value=httpx.Response(200, json={ + "provider_uuid": "p1", "provider_name": "A", "provider_type": "anthropic", "windows": [], + }) + ) + result = _view().refresh("p1") + assert route.called + assert result.provider_uuid == "p1" + + +@respx.mock +def test_refresh_all_hits_refresh_endpoint_and_returns_none(): + route = respx.post(f"{BASE}/api/v1/provider-quota/refresh").mock( + return_value=httpx.Response(200, json={"meta": {"total": 0, "updated_at": "x"}, "data": []}) + ) + assert _view().refresh() is None + assert route.called + + +# -- headroom heuristics -------------------------------------------------- + +def test_window_remaining_percent(): + assert UsageWindow._from_json(_window(used_percent=30)).remaining_percent == 70.0 + + +def test_window_remaining_percent_none_when_unlimited(): + # tb's convention: limit<=0 means "unlimited" — there's no percent of an + # unbounded quantity, so this must not be treated as "0% remaining". + assert UsageWindow._from_json(_window(limit=0)).remaining_percent is None + + +def test_provider_headroom_is_the_most_constrained_window(): + quota = ProviderQuota( + provider_uuid="p1", provider_name="A", provider_type="anthropic", + windows=[ + UsageWindow._from_json(_window(key="session", used_percent=10)), # 90% remaining + UsageWindow._from_json(_window(key="daily", used_percent=80)), # 20% remaining + ], + ) + assert quota.headroom_percent == 20.0 + + +def test_provider_headroom_defaults_to_100_with_no_bounded_windows(): + quota = ProviderQuota(provider_uuid="p1", provider_name="A", provider_type="anthropic", windows=[]) + assert quota.headroom_percent == 100.0 diff --git a/sdk/python/tests/test_router_plugin.py b/sdk/python/tests/test_router_plugin.py new file mode 100644 index 000000000..9ee098908 --- /dev/null +++ b/sdk/python/tests/test_router_plugin.py @@ -0,0 +1,110 @@ +"""Tests for the router showcase plugin (sdk/python/examples/router_plugin.py). + +Unlike critic/fusion, a router doesn't generate anything — it picks ONE +candidate by quota headroom and forwards only to that one. These tests pin +that decision logic with plugin.use()/quota monkeypatched, no real tb. +""" + +import importlib.util +import sys +from pathlib import Path + +from tingly.helpers.quota import ProviderQuota, UsageWindow +from tingly.plugin.types import ChatRequest + +EXAMPLES = Path(__file__).parent.parent / "examples" + + +def _load(name): + path = EXAMPLES / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +def _req(content): + return ChatRequest.from_openai_body({"model": "x", "messages": [{"role": "user", "content": content}]}) + + +def _quota(uuid, used_percent): + return ProviderQuota( + provider_uuid=uuid, provider_name=uuid, provider_type="anthropic", + windows=[UsageWindow(key="session", type="session", used=used_percent, limit=100, used_percent=used_percent)], + ) + + +class _FakeQuota: + def __init__(self, quotas): + self._quotas = quotas + + def batch(self, uuids): + return {u: self._quotas[u] for u in uuids if u in self._quotas} + + +class _FakeClient: + def __init__(self, reply=None, quota=None): + self._reply = reply + self.quota = quota + self.calls = [] + + def ask(self, prompt, **kwargs): + self.calls.append((prompt, kwargs)) + return self._reply + + +def test_pick_candidate_chooses_highest_headroom(monkeypatch): + router = _load("router_plugin") + monkeypatch.setattr(router, "CANDIDATES", [ + router.Candidate(scenario="experiment", model="m1", provider_uuid="u1"), + router.Candidate(scenario="experiment", model="m2", provider_uuid="u2"), + ]) + quotas = _FakeQuota({"u1": _quota("u1", used_percent=90), "u2": _quota("u2", used_percent=10)}) + monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(quota=quotas)) + + chosen = router._pick_candidate() + + assert chosen.provider_uuid == "u2" # 90% headroom beats 10% + + +def test_pick_candidate_defaults_missing_quota_to_full_headroom(monkeypatch): + """A candidate tb has no quota data for yet must not be starved out by + one that does — treat "unknown" the same as "unconstrained", not zero.""" + router = _load("router_plugin") + monkeypatch.setattr(router, "CANDIDATES", [ + router.Candidate(scenario="experiment", model="m1", provider_uuid="u1"), + router.Candidate(scenario="experiment", model="m2", provider_uuid="unknown"), + ]) + quotas = _FakeQuota({"u1": _quota("u1", used_percent=90)}) # "unknown" absent + monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(quota=quotas)) + + chosen = router._pick_candidate() + + assert chosen.provider_uuid == "unknown" + + +def test_handle_forwards_only_to_the_chosen_candidate(monkeypatch): + router = _load("router_plugin") + monkeypatch.setattr(router, "CANDIDATES", [ + router.Candidate(scenario="s1", model="m1", provider_uuid="u1"), + router.Candidate(scenario="s2", model="m2", provider_uuid="u2"), + ]) + quotas = _FakeQuota({"u1": _quota("u1", used_percent=90), "u2": _quota("u2", used_percent=10)}) + llm_client = _FakeClient(quota=quotas) + s2_client = _FakeClient(reply="answer from s2") + + def fake_use(scenario): + if scenario == router.plugin.scenario: + return llm_client + if scenario == "s2": + return s2_client + raise AssertionError(f"must not forward to the un-chosen candidate's scenario: {scenario!r}") + + monkeypatch.setattr(router.plugin, "use", fake_use) + + result = router.handle(_req("what's 2+2?")) + + assert result == "answer from s2" + assert len(s2_client.calls) == 1 + assert s2_client.calls[0][1]["model"] == "m2" diff --git a/sdk/python/tingly/client.py b/sdk/python/tingly/client.py index 91719dd0a..fe0cef908 100644 --- a/sdk/python/tingly/client.py +++ b/sdk/python/tingly/client.py @@ -16,6 +16,7 @@ from . import scenarios as _scenarios from .errors import TinglyError from .helpers.guardrails import GuardrailsView +from .helpers.quota import QuotaView from .helpers.usage import UsageView from .transports import anthropic_compat, openai_compat @@ -163,6 +164,10 @@ def usage(self) -> UsageView: def guardrails(self) -> GuardrailsView: return GuardrailsView(self._gateway_url, self._admin_token, self._timeout) + @property + def quota(self) -> QuotaView: + return QuotaView(self._gateway_url, self._admin_token, self._timeout) + # -- lifecycle ------------------------------------------------------- def close(self) -> None: diff --git a/sdk/python/tingly/helpers/quota.py b/sdk/python/tingly/helpers/quota.py new file mode 100644 index 000000000..13784c7d4 --- /dev/null +++ b/sdk/python/tingly/helpers/quota.py @@ -0,0 +1,158 @@ +"""Quota view — per-provider usage/limit windows, and a live refresh. + +tb tracks quota per provider as one or more named **windows** (session / +daily / weekly / monthly / balance / model / ...), each with its own +``used`` / ``limit`` / ``used_percent`` — a provider is rarely a single +number. ``list()`` / ``get()`` / ``batch()`` read tb's cache (tb itself +lazily re-fetches from the upstream account when a provider's cached snapshot +has expired, ~20 min TTL by default); ``refresh()`` forces a **live** +re-fetch right now. Prefer the cache for routing decisions made on every +request — LiteLLM's own usage-based-routing docs warn that hitting a live +usage source on every single request adds real per-request latency; reserve +``refresh()`` for when you specifically need a number fresher than the cache. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import httpx + + +@dataclass +class UsageWindow: + """One quota window (e.g. "session", "daily", "monthly TPM").""" + + key: str + type: str + used: float + limit: float + used_percent: float = 0.0 + unit: str = "" + label: str = "" + resets_at: Optional[str] = None + allowed: Optional[bool] = None + limit_reached: Optional[bool] = None + + @property + def remaining_percent(self) -> Optional[float]: + """0-100 remaining, or ``None`` when ``limit<=0`` (tb's convention + for "unlimited" — there is no percentage to be remaining *of*).""" + if self.limit <= 0: + return None + return max(0.0, 100.0 - self.used_percent) + + @classmethod + def _from_json(cls, d: Dict[str, Any]) -> "UsageWindow": + return cls( + key=d.get("key", ""), + type=d.get("type", ""), + used=d.get("used", 0) or 0, + limit=d.get("limit", 0) or 0, + used_percent=d.get("used_percent", 0) or 0, + unit=d.get("unit", ""), + label=d.get("label", ""), + resets_at=d.get("resets_at"), + allowed=d.get("allowed"), + limit_reached=d.get("limit_reached"), + ) + + +@dataclass +class ProviderQuota: + """A provider's quota snapshot — as cached by tb, or freshly fetched.""" + + provider_uuid: str + provider_name: str + provider_type: str + windows: List[UsageWindow] = field(default_factory=list) + last_error: str = "" + raw: Dict[str, Any] = field(default_factory=dict) + + @property + def headroom_percent(self) -> float: + """The most CONSTRAINED window's remaining percent — i.e. whichever + limit this provider will hit first. ``100.0`` when no window carries + a real limit (nothing to be constrained by). + + This is deliberately a single naive number for making a routing pick + between candidates at a glance (see ``examples/router_plugin.py``); + session/daily/cost windows are not fungible, so anything more + precise than "which one is worse off right now" should read + ``.windows`` directly instead of trusting this alone. + """ + percents = [w.remaining_percent for w in self.windows if w.remaining_percent is not None] + return min(percents) if percents else 100.0 + + @classmethod + def _from_json(cls, d: Dict[str, Any]) -> "ProviderQuota": + return cls( + provider_uuid=d.get("provider_uuid", ""), + provider_name=d.get("provider_name", ""), + provider_type=d.get("provider_type", ""), + windows=[UsageWindow._from_json(w) for w in d.get("windows") or []], + last_error=d.get("last_error", ""), + raw=d, + ) + + +class QuotaView: + def __init__(self, gateway_url: str, admin_token: str, timeout: float): + self._gateway_url = gateway_url.rstrip("/") + self._admin_token = admin_token + self._timeout = timeout + + def _headers(self) -> Dict[str, str]: + return {"Authorization": f"Bearer {self._admin_token}"} + + def list(self) -> List[ProviderQuota]: + """Every provider's cached quota.""" + resp = httpx.get( + f"{self._gateway_url}/api/v1/provider-quota", + headers=self._headers(), timeout=self._timeout, + ) + resp.raise_for_status() + data = resp.json().get("data") or [] + return [ProviderQuota._from_json(d) for d in data] + + def get(self, provider_uuid: str) -> ProviderQuota: + """One provider's cached quota (tb transparently refetches if the + cached snapshot has expired).""" + resp = httpx.get( + f"{self._gateway_url}/api/v1/provider-quota/{provider_uuid}", + headers=self._headers(), timeout=self._timeout, + ) + resp.raise_for_status() + return ProviderQuota._from_json(resp.json()) + + def batch(self, provider_uuids: List[str]) -> Dict[str, ProviderQuota]: + """Cached quota for a specific set of providers in one round trip — + the shape a router picking between N candidates actually wants.""" + resp = httpx.post( + f"{self._gateway_url}/api/v1/provider-quota/batch", + headers=self._headers(), json={"provider_uuids": provider_uuids}, + timeout=self._timeout, + ) + resp.raise_for_status() + data = resp.json().get("data") or {} + return {uuid: ProviderQuota._from_json(d) for uuid, d in data.items()} + + def refresh(self, provider_uuid: Optional[str] = None) -> Optional[ProviderQuota]: + """Force a LIVE re-fetch from the upstream account, bypassing tb's + cache entirely. Omit ``provider_uuid`` to refresh every enabled + provider (returns ``None`` in that case — use :meth:`list` to read + the results back).""" + if provider_uuid: + resp = httpx.post( + f"{self._gateway_url}/api/v1/provider-quota/{provider_uuid}/refresh", + headers=self._headers(), timeout=self._timeout, + ) + resp.raise_for_status() + return ProviderQuota._from_json(resp.json()) + resp = httpx.post( + f"{self._gateway_url}/api/v1/provider-quota/refresh", + headers=self._headers(), timeout=self._timeout, + ) + resp.raise_for_status() + return None From 59ba5171091c9e5699eea6268764610fe53fa6b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 12:25:46 +0000 Subject: [PATCH 21/28] feat: add X-Tingly-Pin-Provider for deterministic dispatch, close router's quota/execution gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit router_plugin.py picked a provider by quota and then called .ask(model=X) — but (scenario, model) resolves to a rule that can have more than one active service, and tb's own load balancer decides which one actually runs. Nothing guaranteed the provider that was quota-checked was the one that served the request. This adds the missing piece: a scoped, authenticated way to pin a request to one specific service of an already-resolved rule. - internal/server/routing/simple.go: new X-Tingly-Pin-Provider header, handled in SimpleSelector.SelectService alongside the existing X-Tingly-Probe-Service bypass. Unlike that header — unauthenticated by convention, can pin to ANY provider on the box, admin/diagnostics-only by design (.design/probe.md) — this one is scoped: the pinned provider MUST already be one of the resolved rule's own active services, or tb rejects the request (400). It also rides the normal model-token auth already required to reach /tingly/:scenario/..., no new auth mechanism needed. Refactored the post-selection bookkeeping (session/affinity/observability) into applySelectionResult() so both the normal pipeline and the pin override produce identically-instrumented results. - internal/server/routing/result.go: SourceProviderPin routing-source constant, alongside SourceProbePin. - Go tests pin the scoping guarantee (pin rejected when the provider isn't on the rule, or is inactive, or is disabled) and that the unpinned path is unaffected. Also verified live against the real tb binary: a tier0/tier1 rule normally selects tier0, the same call with the pin header selects tier1 instead, and a pin to an unrelated provider is rejected. - sdk/python/tingly/helpers/rules.py: new Client.rules view (GET /api/v1/rules?scenario=) — Rule.active_services is what a caller needs to know before it can even consider pinning. - Client.ask(..., pin_provider=...) sets X-Tingly-Pin-Provider (merges with caller-supplied extra_headers). tb.openai/tb.anthropic already accept extra_headers= natively from their vendor SDKs, so pinning works there with zero SDK change — ask()'s kwarg is purely for convenience. - router_plugin.py rewritten: candidates are now plain model names, resolved via Client.rules to their rule's services; a candidate whose rule doesn't resolve to exactly one active service is skipped as not safely routable by an external quota check, rather than guessed at. The forwarded call now passes pin_provider=, closing the gap between what was checked and what runs. Design doc and README updated: new "Two connection modes" section explains scenario+rule (tb decides) vs. scenario+rule+pin (caller decides among the rule's own services), and why the scoping check is what makes this safe to expose where the older probe header isn't. --- .design/python-sdk.md | 85 +++++++++--- internal/server/routing/result.go | 4 + internal/server/routing/simple.go | 47 ++++++- .../routing/simple_pin_provider_test.go | 118 ++++++++++++++++ sdk/python/README.md | 28 +++- sdk/python/examples/router_plugin.py | 101 +++++++++----- sdk/python/tests/test_client_offline.py | 66 +++++++++ sdk/python/tests/test_router_plugin.py | 127 ++++++++++++------ sdk/python/tests/test_rules.py | 79 +++++++++++ sdk/python/tingly/client.py | 17 +++ sdk/python/tingly/helpers/rules.py | 97 +++++++++++++ 11 files changed, 672 insertions(+), 97 deletions(-) create mode 100644 internal/server/routing/simple_pin_provider_test.go create mode 100644 sdk/python/tests/test_rules.py create mode 100644 sdk/python/tingly/helpers/rules.py diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 16dbdd4d3..01315f4e3 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -190,7 +190,7 @@ sdk/python/ config.py # (base_url, admin_token) resolution precedence scenarios.py # scenario + transport constants transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb - helpers/ # usage + guardrails + quota views + helpers/ # usage + guardrails + quota + rules views plugin/ # Layer 2: be an AI server tb routes to core.py # Plugin class (@plugin.chat, .llm, .serve, api_style) server.py # stdlib HTTP server: /v1/messages (primary) + /v1/chat/completions (secondary), + SSE @@ -216,6 +216,7 @@ connect(scenario="experiment") .usage → GET /api/v1/requests (admin token) .guardrails → GET /api/v1/guardrails/config (admin token) .quota → GET/POST /api/v1/provider-quota[...] (admin token) + .rules → GET /api/v1/rules?scenario= (admin token) ``` ## How it works (pencil) @@ -560,6 +561,56 @@ and `internal/loadbalance` have zero references to `ai/quota` as of this writing) — a plugin picking by remaining quota is genuinely new behavior, not a Python reimplementation of something the gateway already does. +### Two connection modes: scenario+rule, and scenario+rule+pin + +Every call this SDK makes goes through `(scenario, model)` → tb resolves a +**rule** → the rule's `Services[]` (possibly several, tiered) → tb's own +affinity/smart-routing/load-balancer picks **which** service actually runs. +That's mode 1 — "let tb decide" — and it's what `.ask()` has always done. + +Building `router_plugin.py` (below) surfaced a real gap: a plugin that picks +a provider by quota and then calls `.ask(model=X)` has no guarantee that's +the provider tb's load balancer actually uses when the rule has more than +one active service — the "decision" and the execution are two unrelated +code paths that happen to usually agree. Mode 2 closes that: + +- **`X-Tingly-Pin-Provider: `** (`internal/server/routing/simple.go`, + `SimpleSelector.SelectService`) — forces the resolved rule to use that + exact service, skipping affinity/smart-routing/load-balancing. The check + that makes this safe to expose to ordinary clients: the provider **must** + already be one of the resolved rule's own active `Services[]`, or tb + rejects the request (400) — this cannot be used to reach an unrelated + provider elsewhere on the box. It also runs on the *same* authenticated + data-plane path as every other call (the model token already required to + reach `/tingly/:scenario/...`), unlike the older `X-Tingly-Probe-Service` + (`internal/server/routing/simple.go`, `.design/probe.md`), which bypasses + auth entirely by convention (*"any caller that can reach the TB HTTP port + can send it"*) and pins to **any** provider — that header is only ever + injected internally by tb's own probe/diagnostics tooling, deliberately + never exposed to SDK users. `X-Tingly-Pin-Provider` is the scoped, + authenticated version of the same underlying mechanic + (`SourceProviderPin` vs. `SourceProbePin` in `internal/server/routing/result.go`). +- SDK surface: `Client.ask(..., pin_provider=)` sets the header + (merges with any caller-supplied `extra_headers`); `tb.openai` / + `tb.anthropic` accept it directly too, since both vendor SDKs already + support `extra_headers=` on `.create()` — no SDK change was even required + for that path, `ask()`'s kwarg is purely for convenience. +- **`Client.rules`** (`tingly/helpers/rules.py`, wraps `GET /api/v1/rules?scenario=`, + admin token) is how a caller finds out what's *pinnable*: + `rules.for_model(scenario, model)` returns the resolved `Rule`, whose + `.active_services` are the only valid `pin_provider` values for that model. + A rule with more than one active service has more than one valid pin — use + quota (or whatever signal) to choose among them; a candidate whose rule + doesn't resolve to exactly one service isn't safely routable by an + external quota check at all (`router_plugin.py` skips those, rather than + guessing). + +Verified live against the real `tb` binary (not just mocked): a rule with +provider A at tier 0 and B at tier 1 — an unpinned call selects A (normal +tier order, confirmed via `X-Tingly-Debug-Routing`); the same call with +`X-Tingly-Pin-Provider: ` selects B despite the tier order; a pin to a +provider not on that rule is rejected with 400. + ### Example plugins (`sdk/python/examples/`) Four, each a different real-world shape of "plugin composes the box by @@ -595,22 +646,26 @@ pattern already in wide use: just one. - **`router_plugin.py`** (`model="plugin/router"`) — quota-aware dispatch: a different shape from the three above, which all *generate* an answer - themselves. A router generates nothing — it picks the ONE candidate - `(scenario, model, provider_uuid)` with the most quota headroom (via the - `Client.quota` view above) and forwards to just that one; one hop total, - by design, not N. Same idea as LiteLLM Router's `usage-based-routing` - strategy (route to whichever deployment has the most remaining rate-limit - capacity), implemented as a plugin instead of gateway config — deliberately - reads cached quota by default and only calls `.quota.refresh()` when a - caller opts in, since LiteLLM's own docs warn that a live usage check on - every request adds real per-request latency. + themselves. A router generates nothing — for each candidate model it + resolves the rule via `Client.rules` (skipping any candidate whose rule + isn't pinned to exactly one active service — see "Two connection modes" + above), checks quota for that one provider, picks the candidate with the + most headroom, and forwards with `pin_provider=` so the provider that was + quota-checked is *guaranteed* to be the one that serves the request — one + hop total, by design, not N. Same idea as LiteLLM Router's + `usage-based-routing` strategy (route to whichever deployment has the most + remaining rate-limit capacity), implemented as a plugin instead of gateway + config — deliberately reads cached quota by default and only calls + `.quota.refresh()` when a caller opts in, since LiteLLM's own docs warn + that a live usage check on every request adds real per-request latency. Every example plugin has unit tests (`tests/test_example_plugins.py`, -`tests/test_router_plugin.py`, `tests/test_quota.py`) that monkeypatch -`plugin.use`/`Client.quota` to fakes and pin the decision logic — JSON-verdict -formatting and graceful degradation on non-JSON (critic); judge-skipped-on- -agreement vs. judge-called-on-disagreement (fusion); highest-headroom -candidate selection and single-hop forwarding (router) — without needing a +`tests/test_router_plugin.py`, `tests/test_quota.py`, `tests/test_rules.py`) +that monkeypatch `plugin.use`/`Client.quota`/`Client.rules` to fakes and pin +the decision logic — JSON-verdict formatting and graceful degradation on +non-JSON (critic); judge-skipped-on-agreement vs. judge-called-on-disagreement +(fusion); multi-service rules skipped as non-routable, highest-headroom +candidate selection, and `pin_provider=` forwarding (router) — without needing a live tb. ## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) diff --git a/internal/server/routing/result.go b/internal/server/routing/result.go index b69cb29a5..46d26012a 100644 --- a/internal/server/routing/result.go +++ b/internal/server/routing/result.go @@ -15,6 +15,10 @@ const ( // SourceProbePin marks the X-Tingly-Probe-Service bypass, which pins a // specific service without running the pipeline. SourceProbePin = "probe_pin" + // SourceProviderPin marks the X-Tingly-Pin-Provider override — a caller + // choosing which of the rule's OWN configured services to use, still + // authenticated and still scoped to that rule (contrast SourceProbePin). + SourceProviderPin = "provider_pin" ) // SelectionResult represents the output of service selection pipeline. diff --git a/internal/server/routing/simple.go b/internal/server/routing/simple.go index 628e9fc3f..138233c38 100644 --- a/internal/server/routing/simple.go +++ b/internal/server/routing/simple.go @@ -58,6 +58,30 @@ func (s *SimpleSelector) SelectService( // Build context (session ID resolved internally) ctx := NewSelectionContext(rule, req, c, scenario) + // X-Tingly-Pin-Provider: — an authenticated caller (this + // endpoint already required a valid model token to reach here) asking to + // use one SPECIFIC service already configured on THIS rule, instead of + // whatever affinity/smart-routing/load-balancing would otherwise pick. + // Deliberately scoped to rule.Services — unlike X-Tingly-Probe-Service + // above, this is safe to expose to normal clients precisely because it + // cannot reach a provider the rule wasn't already configured to use. + if pinnedUUID := c.GetHeader("X-Tingly-Pin-Provider"); pinnedUUID != "" { + svc := findActiveServiceByProvider(rule, pinnedUUID) + if svc == nil { + return nil, nil, fmt.Errorf("X-Tingly-Pin-Provider %q is not an active service on this rule", pinnedUUID) + } + provider, err := s.selector.config.GetProviderByUUID(pinnedUUID) + if err != nil || provider == nil { + return nil, nil, fmt.Errorf("pinned provider not found: %s", pinnedUUID) + } + if !provider.Enabled { + return nil, nil, fmt.Errorf("pinned provider disabled: %s", pinnedUUID) + } + result := &SelectionResult{Provider: provider, Service: svc, Source: SourceProviderPin, MatchedSmartRuleIndex: -1} + s.applySelectionResult(c, ctx, rule, scenario, result) + return provider, svc, nil + } + // Execute pipeline result, err := s.selector.Select(ctx) if err != nil { @@ -68,6 +92,27 @@ func (s *SimpleSelector) SelectService( return nil, nil, fmt.Errorf("selection returned nil result") } + s.applySelectionResult(c, ctx, rule, scenario, result) + + return result.Provider, result.Service, nil +} + +// findActiveServiceByProvider returns the rule's own active service bound to +// the given provider UUID, or nil if the rule has no such service — the +// scoping check that makes X-Tingly-Pin-Provider safe to expose to clients. +func findActiveServiceByProvider(rule *typ.Rule, providerUUID string) *loadbalance.Service { + for _, svc := range rule.GetActiveServices() { + if svc.Provider == providerUUID { + return svc + } + } + return nil +} + +// applySelectionResult stores session/affinity/observability context and +// emits debug headers for a selection result, however it was produced +// (the normal pipeline, or the X-Tingly-Pin-Provider override above). +func (s *SimpleSelector) applySelectionResult(c *gin.Context, ctx *SelectionContext, rule *typ.Rule, scenario typ.RuleScenario, result *SelectionResult) { // Automatically store sessionID in gin context for downstream handlers c.Set(constant.CtxKeySessionID, ctx.SessionID.String()) // The scoped affinity key (session + matched smart partition) — consumers @@ -106,8 +151,6 @@ func (s *SimpleSelector) SelectService( }).Infof("[routing] selected %s/%s via %s", result.Provider.UUID, result.Service.Model, result.Source) setRoutingDebugHeaders(c, result.Provider.Name, result.Provider.UUID, result.Service.Model, result.Source, result.MatchedSmartRuleIndex, result.EvaluatedStages) - - return result.Provider, result.Service, nil } // setRoutingDebugHeaders emits X-Tingly-Selected-* response headers describing diff --git a/internal/server/routing/simple_pin_provider_test.go b/internal/server/routing/simple_pin_provider_test.go new file mode 100644 index 000000000..b65838830 --- /dev/null +++ b/internal/server/routing/simple_pin_provider_test.go @@ -0,0 +1,118 @@ +package routing + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tingly-dev/tingly-box/internal/loadbalance" + "github.com/tingly-dev/tingly-box/internal/typ" +) + +// TestSelectService_PinProvider verifies that X-Tingly-Pin-Provider picks the +// named service from the rule's OWN services, overriding what the load +// balancer would otherwise choose — same mechanics as the probe pin, but +// scoped to services the rule already has configured. +func TestSelectService_PinProvider(t *testing.T) { + svcA := testService("provider-a", "claude-sonnet", true) + svcB := testService("provider-b", "claude-sonnet", true) + cfg := &mockConfig{ + providers: map[string]*typ.Provider{ + "provider-a": testProvider("provider-a", "ProviderA", true), + "provider-b": testProvider("provider-b", "ProviderB", true), + }, + } + // Pipeline would normally return provider-b via load balancer. + lb := &mockLoadBalancer{service: svcB} + store := newMockAffinityStore() + sel := NewServiceSelector(cfg, store, lb) + simple := NewSimpleSelector(sel) + + rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA, svcB}) + c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "provider-a") + + provider, svc, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) + require.NoError(t, err) + + assert.Equal(t, "provider-a", provider.UUID) + assert.Equal(t, "provider-a", svc.Provider) +} + +// TestSelectService_PinProvider_RejectsProviderNotOnRule is the scoping +// guarantee that makes this header safe to expose to clients: it cannot +// reach a provider the rule wasn't already configured with. +func TestSelectService_PinProvider_RejectsProviderNotOnRule(t *testing.T) { + svcA := testService("provider-a", "claude-sonnet", true) + cfg := &mockConfig{ + providers: map[string]*typ.Provider{ + "provider-a": testProvider("provider-a", "ProviderA", true), + "unrelated-provider": testProvider("unrelated-provider", "Unrelated", true), + }, + } + simple := newSimpleSelector(cfg) + rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA}) + c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "unrelated-provider") + + _, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "not an active service") +} + +// TestSelectService_PinProvider_RejectsInactiveService confirms a service +// present on the rule but not active cannot be pinned to either. +func TestSelectService_PinProvider_RejectsInactiveService(t *testing.T) { + svcA := testService("provider-a", "claude-sonnet", false) // inactive + cfg := &mockConfig{ + providers: map[string]*typ.Provider{ + "provider-a": testProvider("provider-a", "ProviderA", true), + }, + } + simple := newSimpleSelector(cfg) + rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA}) + c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "provider-a") + + _, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "not an active service") +} + +// TestSelectService_PinProvider_DisabledProvider errors when the pinned +// provider is itself disabled, even though it's a configured service. +func TestSelectService_PinProvider_DisabledProvider(t *testing.T) { + svcA := testService("provider-a", "claude-sonnet", true) + cfg := &mockConfig{ + providers: map[string]*typ.Provider{ + "provider-a": testProvider("provider-a", "ProviderA", false), // disabled + }, + } + simple := newSimpleSelector(cfg) + rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA}) + c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "provider-a") + + _, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "disabled") +} + +// TestSelectService_NoPinHeader_FallsThrough confirms that without the pin +// header the normal pipeline still runs unaffected. +func TestSelectService_NoPinHeader_FallsThrough(t *testing.T) { + svc := testService("provider-a", "claude-sonnet", true) + cfg := &mockConfig{ + providers: map[string]*typ.Provider{ + "provider-a": testProvider("provider-a", "ProviderA", true), + }, + } + lb := &mockLoadBalancer{service: svc} + store := newMockAffinityStore() + sel := NewServiceSelector(cfg, store, lb) + simple := NewSimpleSelector(sel) + + rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svc}) + c := ginCtxWithHeader(t, "", "") + + provider, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) + require.NoError(t, err) + assert.Equal(t, "provider-a", provider.UUID) +} diff --git a/sdk/python/README.md b/sdk/python/README.md index 4254bb284..d1f23c6a4 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -98,7 +98,7 @@ gateway for its own LLM work. ### Example plugins -`sdk/python/examples/` has three, each demonstrating a different real-world +`sdk/python/examples/` has four, each demonstrating a different real-world pattern for the same idea — a plugin composing the box by calling back into other tb rules: @@ -118,10 +118,28 @@ other tb rules: originate more than one call, against more than one rule, per request. - **`router_plugin.py`** — quota-aware dispatch (`model="plugin/router"`): a different shape from the three above — it generates nothing itself, it - only *decides* which one candidate rule/model to forward to, using live - quota headroom (`tb.quota`, below) to pick. Same idea as LiteLLM Router's - `usage-based-routing` strategy, implemented as a plugin instead of gateway - config. + only *decides* which one candidate to forward to, using quota headroom + (`tb.quota`) to pick. Same idea as LiteLLM Router's `usage-based-routing` + strategy. Forwards with `tb.ask(..., pin_provider=)` so the provider + it checked quota for is *guaranteed* to be the one that serves the + request — see "Deterministic dispatch" below for why that matters. + +## Deterministic dispatch (`pin_provider`) + +Normally `tb.ask(model=X)` resolves `(scenario, model)` to a rule and lets tb +itself pick which of that rule's services actually runs (affinity / smart +routing / load balancing — unchanged, still the default). When code needs to +*guarantee* a specific provider — like `router_plugin.py` above, which +already checked that provider's quota — pass `pin_provider`: + +```python +tb.ask("...", model="sonnet1", pin_provider=provider_uuid) +``` + +tb only allows pinning to a provider that's already one of the resolved +rule's own configured services (`tb.rules.for_model(scenario, model)` lists +them) — it rejects a pin to anything else. See `.design/python-sdk.md` §"Two +connection modes" for the full mechanics. ## Status diff --git a/sdk/python/examples/router_plugin.py b/sdk/python/examples/router_plugin.py index 0ec444162..5a9b0f925 100644 --- a/sdk/python/examples/router_plugin.py +++ b/sdk/python/examples/router_plugin.py @@ -1,15 +1,25 @@ """A "router" plugin: quota-aware dispatch — a different shape from rag/critic/fusion. Those all *generate* an answer themselves (one or more calls back into tb feed a response the plugin composes). A router generates -nothing; its only job is to DECIDE which one candidate rule/model actually -serves the request, then forward to just that one. - -This is the same idea as LiteLLM Router's `usage-based-routing` strategy — -route to whichever deployment has the most remaining rate-limit/quota -headroom right now, instead of a fixed priority order — implemented here as -a plugin instead of gateway config, using the SDK's quota views -(`Client.quota`, `sdk/python/tingly/helpers/quota.py`) added for exactly -this. +nothing; its only job is to DECIDE which one candidate model actually serves +the request, then forward to just that one — and to GUARANTEE the provider +it checked quota for is the provider that actually serves it. + +That guarantee is why this isn't just "pick a model and call .ask(model=)": +a model name resolves to a *rule*, and a rule can have more than one active +service (tiers, load-balanced) — tb, not this plugin, decides which one of +those actually runs. Checking quota for one provider and then calling +`.ask(model=X)` would silently mean nothing if tb's own load balancer picks +a different service within that rule. So each candidate here must resolve +(via `Client.rules`) to a rule with exactly ONE active service — a model +name dedicated to one specific provider — and the forwarded call passes +`pin_provider=` (`X-Tingly-Pin-Provider`, see .design/python-sdk.md) to force +that exact provider, closing the loop between "what was checked" and "what +was used". + +Same idea as LiteLLM Router's `usage-based-routing` strategy — route to +whichever deployment has the most remaining rate-limit/quota headroom right +now — implemented as a plugin instead of gateway config. Run it (serves on :8768 AND registers with tb on startup): @@ -22,52 +32,79 @@ from __future__ import annotations from dataclasses import dataclass +from typing import List from tingly import ChatRequest, Plugin - -@dataclass -class Candidate: - scenario: str - model: str - provider_uuid: str # the provider backing (scenario, model) — quota is per-provider - - -# Fill in real provider UUIDs from the tb UI (Providers page) or -# `GET /api/v2/providers` — quota is tracked per provider, not per rule, so -# there is no way to infer these from the model name alone. -CANDIDATES = [ - Candidate(scenario="experiment", model="auto", provider_uuid="REPLACE_WITH_PROVIDER_UUID_1"), - Candidate(scenario="experiment", model="auto", provider_uuid="REPLACE_WITH_PROVIDER_UUID_2"), -] +# Each entry must be a model name, in ROUTER_SCENARIO, whose rule you've +# configured to point at exactly ONE provider — e.g. an "anthropic" scenario +# with a "sonnet1" rule bound only to provider A and a "sonnet2" rule bound +# only to provider B. That 1:1 binding is what makes a quota-based pick mean +# something (see the module docstring); a rule with more than one active +# service is skipped as a candidate, not guessed at. +ROUTER_SCENARIO = "experiment" +CANDIDATE_MODELS = ["sonnet1", "sonnet2"] plugin = Plugin( name="router", - scenario="experiment", # bind a rule under this scenario on register + scenario=ROUTER_SCENARIO, # bind a rule under this scenario on register description="Quota-aware dispatch — forwards to whichever candidate has the most headroom", ) +@dataclass +class ResolvedCandidate: + model: str + provider_uuid: str # the ONE provider this model's rule is pinned to + + @plugin.chat def handle(req: ChatRequest) -> str: question = req.last_user_text() chosen = _pick_candidate() - # The only call that matters: forward to the ONE chosen candidate, not - # every candidate — a router spends one hop total, not N (contrast with - # fusion_plugin.py, which deliberately spends N to get a second opinion). - return plugin.use(chosen.scenario).ask(question, model=chosen.model) + # pin_provider is what makes this a real decision rather than a guess: + # the provider that was quota-checked is GUARANTEED to be the one that + # serves this request. + return plugin.use(ROUTER_SCENARIO).ask( + question, model=chosen.model, pin_provider=chosen.provider_uuid + ) -def _pick_candidate() -> Candidate: +def _resolve_candidates() -> List[ResolvedCandidate]: + """Resolve each candidate model to its rule's single pinned provider. + Skips (rather than guesses at) any candidate whose rule has zero or more + than one active service — quota can't mean anything for a model tb + itself load-balances across multiple providers.""" + rules = plugin.llm.rules + resolved = [] + for model in CANDIDATE_MODELS: + rule = rules.for_model(ROUTER_SCENARIO, model) + if rule is None: + continue + services = rule.active_services + if len(services) != 1: + continue # not a pinned single-provider rule — not routable by quota + resolved.append(ResolvedCandidate(model=model, provider_uuid=services[0].provider)) + return resolved + + +def _pick_candidate() -> ResolvedCandidate: """Cached quota (tb refreshes lazily, ~20 min TTL) is enough for most routing decisions and costs nothing extra per request. Call `plugin.llm.quota.refresh(uuid)` first, for a specific candidate, if a request genuinely needs a number fresher than that — LiteLLM's own usage-based-routing docs warn that a live check on every single request adds real latency, so that should be the exception, not the default.""" - quotas = plugin.llm.quota.batch([c.provider_uuid for c in CANDIDATES]) + candidates = _resolve_candidates() + if not candidates: + raise RuntimeError( + "no router candidate resolved to a single-provider rule — each " + "entry in CANDIDATE_MODELS must name a model whose rule has " + "exactly one active service (see the module docstring)" + ) + quotas = plugin.llm.quota.batch([c.provider_uuid for c in candidates]) return max( - CANDIDATES, + candidates, key=lambda c: quotas[c.provider_uuid].headroom_percent if c.provider_uuid in quotas else 100.0, ) diff --git a/sdk/python/tests/test_client_offline.py b/sdk/python/tests/test_client_offline.py index aa3255a04..e3b79b324 100644 --- a/sdk/python/tests/test_client_offline.py +++ b/sdk/python/tests/test_client_offline.py @@ -73,6 +73,72 @@ class _FakeAnthropic: assert captured["model"] == "auto" +def test_ask_pin_provider_sets_header(monkeypatch): + c = _client("anthropic") + captured = {} + + class _FakeMessages: + def create(self, **kwargs): + captured.update(kwargs) + + class R: + content = [] + + return R() + + class _FakeAnthropic: + messages = _FakeMessages() + + monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) + c.ask("hi", pin_provider="p1") + + assert captured["extra_headers"] == {"X-Tingly-Pin-Provider": "p1"} + + +def test_ask_pin_provider_merges_with_caller_extra_headers(monkeypatch): + c = _client("anthropic") + captured = {} + + class _FakeMessages: + def create(self, **kwargs): + captured.update(kwargs) + + class R: + content = [] + + return R() + + class _FakeAnthropic: + messages = _FakeMessages() + + monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) + c.ask("hi", pin_provider="p1", extra_headers={"X-Custom": "1"}) + + assert captured["extra_headers"] == {"X-Custom": "1", "X-Tingly-Pin-Provider": "p1"} + + +def test_ask_without_pin_provider_sends_no_pin_header(monkeypatch): + c = _client("anthropic") + captured = {} + + class _FakeMessages: + def create(self, **kwargs): + captured.update(kwargs) + + class R: + content = [] + + return R() + + class _FakeAnthropic: + messages = _FakeMessages() + + monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) + c.ask("hi") + + assert "extra_headers" not in captured + + def test_client_passes_scenario_root_to_transports(monkeypatch): """The client hands the scenario root + model token to each builder; per-transport URL shaping (e.g. the OpenAI /v1 suffix) happens inside the diff --git a/sdk/python/tests/test_router_plugin.py b/sdk/python/tests/test_router_plugin.py index 9ee098908..16da16d83 100644 --- a/sdk/python/tests/test_router_plugin.py +++ b/sdk/python/tests/test_router_plugin.py @@ -1,15 +1,20 @@ """Tests for the router showcase plugin (sdk/python/examples/router_plugin.py). -Unlike critic/fusion, a router doesn't generate anything — it picks ONE -candidate by quota headroom and forwards only to that one. These tests pin -that decision logic with plugin.use()/quota monkeypatched, no real tb. +Unlike critic/fusion, a router doesn't generate anything — it resolves each +candidate model to its rule's single provider, picks the one with the most +quota headroom, and forwards with pin_provider= to guarantee that provider +is the one that actually serves the request. These tests pin that decision +logic with plugin.use()/rules/quota monkeypatched, no real tb. """ import importlib.util import sys from pathlib import Path +import pytest + from tingly.helpers.quota import ProviderQuota, UsageWindow +from tingly.helpers.rules import Rule from tingly.plugin.types import ChatRequest EXAMPLES = Path(__file__).parent.parent / "examples" @@ -28,6 +33,15 @@ def _req(content): return ChatRequest.from_openai_body({"model": "x", "messages": [{"role": "user", "content": content}]}) +def _rule(model, *providers_active): + """providers_active: e.g. [("p1", True)] for one active service, or + [("p1", True), ("p2", True)] for a multi-service (non-routable) rule.""" + return Rule._from_json({ + "uuid": f"rule-{model}", "scenario": "experiment", "request_model": model, + "services": [{"provider": p, "model": model, "active": active} for p, active in providers_active], + }) + + def _quota(uuid, used_percent): return ProviderQuota( provider_uuid=uuid, provider_name=uuid, provider_type="anthropic", @@ -35,6 +49,14 @@ def _quota(uuid, used_percent): ) +class _FakeRules: + def __init__(self, rules_by_model): + self._rules_by_model = rules_by_model + + def for_model(self, scenario, model): + return self._rules_by_model.get(model) + + class _FakeQuota: def __init__(self, quotas): self._quotas = quotas @@ -42,10 +64,14 @@ def __init__(self, quotas): def batch(self, uuids): return {u: self._quotas[u] for u in uuids if u in self._quotas} + def refresh(self, provider_uuid=None): + raise AssertionError("refresh() should not be called by default routing") + class _FakeClient: - def __init__(self, reply=None, quota=None): + def __init__(self, reply=None, rules=None, quota=None): self._reply = reply + self.rules = rules self.quota = quota self.calls = [] @@ -54,57 +80,72 @@ def ask(self, prompt, **kwargs): return self._reply -def test_pick_candidate_chooses_highest_headroom(monkeypatch): +def test_resolve_candidates_skips_multi_service_rules(monkeypatch): router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATES", [ - router.Candidate(scenario="experiment", model="m1", provider_uuid="u1"), - router.Candidate(scenario="experiment", model="m2", provider_uuid="u2"), - ]) - quotas = _FakeQuota({"u1": _quota("u1", used_percent=90), "u2": _quota("u2", used_percent=10)}) - monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(quota=quotas)) + monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1", "sonnet2", "sonnet3"]) + rules = _FakeRules({ + "sonnet1": _rule("sonnet1", ("p1", True)), # single active service — routable + "sonnet2": _rule("sonnet2", ("p2", True), ("p3", True)), # two active services — skip + # "sonnet3" absent entirely (rule doesn't exist) — skip + }) + monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules)) - chosen = router._pick_candidate() + resolved = router._resolve_candidates() - assert chosen.provider_uuid == "u2" # 90% headroom beats 10% + assert [c.model for c in resolved] == ["sonnet1"] + assert resolved[0].provider_uuid == "p1" -def test_pick_candidate_defaults_missing_quota_to_full_headroom(monkeypatch): - """A candidate tb has no quota data for yet must not be starved out by - one that does — treat "unknown" the same as "unconstrained", not zero.""" +def test_resolve_candidates_skips_rule_with_no_active_services(monkeypatch): router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATES", [ - router.Candidate(scenario="experiment", model="m1", provider_uuid="u1"), - router.Candidate(scenario="experiment", model="m2", provider_uuid="unknown"), - ]) - quotas = _FakeQuota({"u1": _quota("u1", used_percent=90)}) # "unknown" absent - monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(quota=quotas)) + monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1"]) + rules = _FakeRules({"sonnet1": _rule("sonnet1", ("p1", False))}) # only inactive service + monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules)) + + assert router._resolve_candidates() == [] + + +def test_pick_candidate_chooses_highest_headroom(monkeypatch): + router = _load("router_plugin") + monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1", "sonnet2"]) + rules = _FakeRules({ + "sonnet1": _rule("sonnet1", ("p1", True)), + "sonnet2": _rule("sonnet2", ("p2", True)), + }) + quotas = _FakeQuota({"p1": _quota("p1", used_percent=90), "p2": _quota("p2", used_percent=10)}) + monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules, quota=quotas)) chosen = router._pick_candidate() - assert chosen.provider_uuid == "unknown" + assert chosen.model == "sonnet2" # 90% headroom beats 10% + assert chosen.provider_uuid == "p2" + + +def test_pick_candidate_raises_when_no_routable_candidates(monkeypatch): + router = _load("router_plugin") + monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1"]) + rules = _FakeRules({}) # no rule resolves + monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules)) + + with pytest.raises(RuntimeError, match="no router candidate"): + router._pick_candidate() -def test_handle_forwards_only_to_the_chosen_candidate(monkeypatch): +def test_handle_forwards_with_pin_provider_for_the_chosen_candidate(monkeypatch): router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATES", [ - router.Candidate(scenario="s1", model="m1", provider_uuid="u1"), - router.Candidate(scenario="s2", model="m2", provider_uuid="u2"), - ]) - quotas = _FakeQuota({"u1": _quota("u1", used_percent=90), "u2": _quota("u2", used_percent=10)}) - llm_client = _FakeClient(quota=quotas) - s2_client = _FakeClient(reply="answer from s2") - - def fake_use(scenario): - if scenario == router.plugin.scenario: - return llm_client - if scenario == "s2": - return s2_client - raise AssertionError(f"must not forward to the un-chosen candidate's scenario: {scenario!r}") - - monkeypatch.setattr(router.plugin, "use", fake_use) + monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1", "sonnet2"]) + rules = _FakeRules({ + "sonnet1": _rule("sonnet1", ("p1", True)), + "sonnet2": _rule("sonnet2", ("p2", True)), + }) + quotas = _FakeQuota({"p1": _quota("p1", used_percent=90), "p2": _quota("p2", used_percent=10)}) + shared = _FakeClient(reply="the answer", rules=rules, quota=quotas) + monkeypatch.setattr(router.plugin, "use", lambda scenario: shared) result = router.handle(_req("what's 2+2?")) - assert result == "answer from s2" - assert len(s2_client.calls) == 1 - assert s2_client.calls[0][1]["model"] == "m2" + assert result == "the answer" + assert len(shared.calls) == 1 + prompt, kwargs = shared.calls[0] + assert kwargs["model"] == "sonnet2" + assert kwargs["pin_provider"] == "p2" diff --git a/sdk/python/tests/test_rules.py b/sdk/python/tests/test_rules.py new file mode 100644 index 000000000..4979a8191 --- /dev/null +++ b/sdk/python/tests/test_rules.py @@ -0,0 +1,79 @@ +"""RulesView tests (gateway mocked with respx).""" + +import httpx +import respx + +from tingly.helpers.rules import Rule, RulesView + +BASE = "http://tb.test:12580" + + +def _view() -> RulesView: + return RulesView(BASE, "admin", 5.0) + + +@respx.mock +def test_list_requires_scenario_query_param(): + route = respx.get(f"{BASE}/api/v1/rules", params={"scenario": "experiment"}).mock( + return_value=httpx.Response(200, json={"success": True, "data": []}) + ) + _view().list("experiment") + assert route.called + + +@respx.mock +def test_list_parses_services(): + respx.get(f"{BASE}/api/v1/rules").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": [{ + "uuid": "r1", "scenario": "experiment", "request_model": "sonnet1", "active": True, + "services": [ + {"provider": "p1", "model": "claude-sonnet-4-6", "active": True, "weight": 1, "tier": 0}, + {"provider": "p2", "model": "claude-sonnet-4-6", "active": False, "weight": 1, "tier": 1}, + ], + }], + }) + ) + rules = _view().list("experiment") + assert len(rules) == 1 + assert rules[0].request_model == "sonnet1" + assert len(rules[0].services) == 2 + assert [s.provider for s in rules[0].active_services] == ["p1"] + + +@respx.mock +def test_for_model_finds_matching_rule(): + respx.get(f"{BASE}/api/v1/rules").mock( + return_value=httpx.Response(200, json={ + "success": True, + "data": [ + {"uuid": "r1", "scenario": "experiment", "request_model": "sonnet1", "services": []}, + {"uuid": "r2", "scenario": "experiment", "request_model": "sonnet2", "services": []}, + ], + }) + ) + rule = _view().for_model("experiment", "sonnet2") + assert rule is not None + assert rule.uuid == "r2" + + +@respx.mock +def test_for_model_returns_none_when_no_match(): + respx.get(f"{BASE}/api/v1/rules").mock( + return_value=httpx.Response(200, json={"success": True, "data": []}) + ) + assert _view().for_model("experiment", "nope") is None + + +def test_service_for_provider_ignores_inactive(): + rule = Rule._from_json({ + "uuid": "r1", "scenario": "experiment", "request_model": "sonnet1", + "services": [ + {"provider": "p1", "model": "m", "active": False}, + {"provider": "p2", "model": "m", "active": True}, + ], + }) + assert rule.service_for_provider("p1") is None + assert rule.service_for_provider("p2") is not None + assert rule.service_for_provider("unknown") is None diff --git a/sdk/python/tingly/client.py b/sdk/python/tingly/client.py index fe0cef908..e660cd16d 100644 --- a/sdk/python/tingly/client.py +++ b/sdk/python/tingly/client.py @@ -17,6 +17,7 @@ from .errors import TinglyError from .helpers.guardrails import GuardrailsView from .helpers.quota import QuotaView +from .helpers.rules import RulesView from .helpers.usage import UsageView from .transports import anthropic_compat, openai_compat @@ -104,6 +105,7 @@ def ask( system: Optional[str] = None, max_tokens: int = 1024, stream: bool = False, + pin_provider: Optional[str] = None, **kwargs: Any, ): """One-shot prompt → text, routed through tingly-box. @@ -111,7 +113,18 @@ def ask( Picks the transport from the scenario: Anthropic messages is tried first (tb's native protocol); OpenAI-only scenarios fall back to chat completions. ``model="auto"`` lets the gateway route. + + ``pin_provider`` sends ``X-Tingly-Pin-Provider``, forcing tb to use + that exact provider instead of letting affinity/smart-routing/load- + balancing decide — but only if it's one of the resolved rule's own + configured services (see ``Client.rules``); tb rejects a pin to any + other provider. Omit it for tb's normal behavior. """ + if pin_provider: + kwargs["extra_headers"] = { + **kwargs.get("extra_headers", {}), + "X-Tingly-Pin-Provider": pin_provider, + } if _scenarios.supports_anthropic(self._session.transport): return self._ask_anthropic(prompt, model, system, max_tokens, stream, **kwargs) return self._ask_openai(prompt, model, system, stream, **kwargs) @@ -168,6 +181,10 @@ def guardrails(self) -> GuardrailsView: def quota(self) -> QuotaView: return QuotaView(self._gateway_url, self._admin_token, self._timeout) + @property + def rules(self) -> RulesView: + return RulesView(self._gateway_url, self._admin_token, self._timeout) + # -- lifecycle ------------------------------------------------------- def close(self) -> None: diff --git a/sdk/python/tingly/helpers/rules.py b/sdk/python/tingly/helpers/rules.py new file mode 100644 index 000000000..b2ecbac76 --- /dev/null +++ b/sdk/python/tingly/helpers/rules.py @@ -0,0 +1,97 @@ +"""Rules view — read a scenario's rules and the services each one has +configured, so a caller can discover which provider(s) back a given model. + +Existing views (`usage`, `guardrails`, `quota`) all read *what happened* or +*what's available*; this one reads *how a model resolves* — the missing +piece for anything that wants to act on a specific one of a rule's services +(e.g. `Client.ask(..., pin_provider=...)`), rather than letting tb pick. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import httpx + + +@dataclass +class RuleService: + """One of a rule's configured services — a (provider, model) binding tb + can route to, at the given tier.""" + + provider: str # provider UUID + model: str + active: bool = True + weight: int = 1 + tier: int = 0 + + +@dataclass +class Rule: + uuid: str + scenario: str + request_model: str + active: bool = True + services: List[RuleService] = field(default_factory=list) + + @property + def active_services(self) -> List[RuleService]: + return [s for s in self.services if s.active] + + def service_for_provider(self, provider_uuid: str) -> Optional[RuleService]: + """The rule's own active service bound to this provider, if any — + the set of providers valid to pass as `pin_provider=` for this rule's + model. `None` means this provider isn't one of this rule's services; + tb will reject a pin to it (`X-Tingly-Pin-Provider` is scoped to the + resolved rule's own services, not any provider on the box).""" + for svc in self.active_services: + if svc.provider == provider_uuid: + return svc + return None + + @classmethod + def _from_json(cls, d: Dict[str, Any]) -> "Rule": + return cls( + uuid=d.get("uuid", ""), + scenario=d.get("scenario", ""), + request_model=d.get("request_model", ""), + active=d.get("active", True), + services=[ + RuleService( + provider=s.get("provider", ""), + model=s.get("model", ""), + active=s.get("active", True), + weight=s.get("weight", 1), + tier=s.get("tier", 0), + ) + for s in d.get("services") or [] + ], + ) + + +class RulesView: + def __init__(self, gateway_url: str, admin_token: str, timeout: float): + self._gateway_url = gateway_url.rstrip("/") + self._admin_token = admin_token + self._timeout = timeout + + def list(self, scenario: str) -> List[Rule]: + """All rules configured under a scenario (required — tb 400s without it).""" + resp = httpx.get( + f"{self._gateway_url}/api/v1/rules", + params={"scenario": scenario}, + headers={"Authorization": f"Bearer {self._admin_token}"}, + timeout=self._timeout, + ) + resp.raise_for_status() + data = resp.json().get("data") or [] + return [Rule._from_json(d) for d in data] + + def for_model(self, scenario: str, model: str) -> Optional[Rule]: + """The rule whose `request_model` matches, if any — the common case + (a router deciding which provider backs *this* model).""" + for rule in self.list(scenario): + if rule.request_model == model: + return rule + return None From e579af3fc2bd852efabbada1114ec19750e34292 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 12:36:50 +0000 Subject: [PATCH 22/28] test: add fixed e2e regression script for the two connection modes; fix quota batch 500 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sdk/python/examples/e2e_run_pin.sh is a permanent, repeatable (set -uo pipefail, explicit pass/fail, non-zero exit on failure) end-to-end script validating both connection modes against a real tb binary, no network/keys: 1. scenario+rule (tb decides): a tier0/tier1 rule's unpinned call selects tier0. 2. scenario+rule+pin_provider: the same call pinned to the tier1 provider overrides tier order; a pin to a provider not on that rule is rejected. 3. The same via the SDK (Client.ask(pin_provider=)). 4. router_plugin.py run for real end-to-end: registers, resolves its sonnet1/sonnet2 candidates via Client.rules, and its forwarded call is confirmed provider_pin-sourced in tb's own routing log. Running it live immediately surfaced a real bug: POST /provider-quota/batch 500'd the whole batch the moment it included a provider with no quota data (e.g. a vmodel provider — exactly what this no-network test setup uses), instead of just omitting that provider from the result. Root cause: Manager.GetQuota / GetQuotaNoCache (ai/quota/manager.go) re-wrapped a not-found store lookup into a new fmt.Errorf(...) instead of returning quota.ErrUsageNotFound itself, silently breaking the `err == quota.ErrUsageNotFound` identity check both GetQuota and BatchGetQuota rely on to treat "no data yet" as a skip rather than a failure. Fixed by returning the sentinel unwrapped. - ai/quota/manager_test.go: TestGetQuota_NotFoundIsUnwrapped pins the identity contract directly. - internal/server/module/providerquota/handler_test.go: new (this module had no tests before) — pins that a not-found provider is skipped in a batch response, a genuine error still fails the batch when nothing usable came back, and single-provider GetQuota 404s cleanly. Design doc and README updated to point at the new script and document the bug/fix. --- .design/python-sdk.md | 31 +++- ai/quota/manager.go | 19 +-- ai/quota/manager_test.go | 19 +++ .../module/providerquota/handler_test.go | 124 ++++++++++++++ sdk/python/README.md | 3 + sdk/python/examples/e2e_run_pin.sh | 152 ++++++++++++++++++ 6 files changed, 332 insertions(+), 16 deletions(-) create mode 100644 internal/server/module/providerquota/handler_test.go create mode 100755 sdk/python/examples/e2e_run_pin.sh diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 01315f4e3..1c3342524 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -605,11 +605,32 @@ code paths that happen to usually agree. Mode 2 closes that: external quota check at all (`router_plugin.py` skips those, rather than guessing). -Verified live against the real `tb` binary (not just mocked): a rule with -provider A at tier 0 and B at tier 1 — an unpinned call selects A (normal -tier order, confirmed via `X-Tingly-Debug-Routing`); the same call with -`X-Tingly-Pin-Provider: ` selects B despite the tier order; a pin to a -provider not on that rule is rejected with 400. +Verified live against the real `tb` binary (not just mocked) — a fixed, +repeatable regression script, `sdk/python/examples/e2e_run_pin.sh` (three +vmodel providers, no network/keys, `set -uo pipefail` + explicit pass/fail +assertions, non-zero exit on any failure): a rule with provider A at tier 0 +and B at tier 1 — an unpinned call selects A (normal tier order, confirmed +via `X-Tingly-Debug-Routing`); the same call with `X-Tingly-Pin-Provider: ` +selects B despite the tier order; a pin to a provider not on that rule is +rejected with 400; the same round-trip through `Client.ask(pin_provider=)`; +and `router_plugin.py` run for real end-to-end, resolving `sonnet1`/`sonnet2` +via `Client.rules`, and forwarding with a confirmed `provider_pin`-sourced +selection in tb's own routing log. + +**A real bug this surfaced**, fixed alongside it: `Manager.GetQuota` / +`GetQuotaNoCache` (`ai/quota/manager.go`) re-wrapped a not-found store lookup +into a *new* `fmt.Errorf(...)` instead of returning `quota.ErrUsageNotFound` +itself — silently breaking the `err == quota.ErrUsageNotFound` identity +check every caller (`internal/server/module/providerquota/handler.go`, both +`GetQuota` and `BatchGetQuota`) relies on to treat "no data yet" as a skip. +The practical effect: `POST /provider-quota/batch` 500'd the *entire* batch +the moment it included any provider with no quota fetcher (a vmodel/local +provider, exactly what a no-network test setup uses) instead of just +omitting that one provider from the result — `router_plugin.py`'s very first +live run hit this immediately. Fixed by returning the sentinel unwrapped; +covered by `ai/quota/manager_test.go::TestGetQuota_NotFoundIsUnwrapped` and +`internal/server/module/providerquota/handler_test.go` (new — this module +had no tests before). ### Example plugins (`sdk/python/examples/`) diff --git a/ai/quota/manager.go b/ai/quota/manager.go index 16bca2cea..85adc4a24 100644 --- a/ai/quota/manager.go +++ b/ai/quota/manager.go @@ -127,12 +127,15 @@ func (m *Manager) RefreshProvider(ctx context.Context, providerUUID string) (*Pr } // GetQuota returns cached quota data and refreshes it when expired. +// +// A not-found store lookup returns ErrUsageNotFound UNWRAPPED — callers +// (e.g. the provider-quota HTTP handlers) compare against that sentinel +// with == to treat "no data yet" as a skip rather than an error; wrapping it +// here (as this used to do, via fmt.Errorf) silently broke that comparison +// and turned every "no quota data" provider into a hard error upstream. func (m *Manager) GetQuota(ctx context.Context, providerUUID string) (*ProviderUsage, error) { usage, err := m.store.Get(ctx, providerUUID) if err != nil { - if err == ErrUsageNotFound { - return nil, fmt.Errorf("quota not found for provider: %s", providerUUID) - } return nil, err } @@ -146,15 +149,9 @@ func (m *Manager) GetQuota(ctx context.Context, providerUUID string) (*ProviderU } // GetQuotaNoCache returns the latest quota data stored in the database. +// See GetQuota above for why ErrUsageNotFound must reach the caller unwrapped. func (m *Manager) GetQuotaNoCache(ctx context.Context, providerUUID string) (*ProviderUsage, error) { - usage, err := m.store.Get(ctx, providerUUID) - if err != nil { - if err == ErrUsageNotFound { - return nil, fmt.Errorf("quota not found for provider: %s", providerUUID) - } - return nil, err - } - return usage, nil + return m.store.Get(ctx, providerUUID) } // ListQuota returns quota data for all providers. diff --git a/ai/quota/manager_test.go b/ai/quota/manager_test.go index 4b72fa6ea..4b6e52f36 100644 --- a/ai/quota/manager_test.go +++ b/ai/quota/manager_test.go @@ -57,6 +57,25 @@ func (f *concurrencyTestFetcher) Fetch(_ context.Context, provider *typ.Provider return &ProviderUsage{ProviderUUID: provider.UUID}, nil } +// TestGetQuota_NotFoundIsUnwrapped locks in that GetQuota (and +// GetQuotaNoCache) return ErrUsageNotFound identically to what the store +// returned — not a re-wrapped error. Callers such as the provider-quota +// batch handler compare with == against this sentinel to skip providers +// with no quota data instead of failing the whole request; a wrapped error +// silently breaks that comparison (this was a real bug: BatchGetQuota 500'd +// for any provider with no quota data, e.g. a vmodel/local provider, instead +// of just omitting it from the result). +func TestGetQuota_NotFoundIsUnwrapped(t *testing.T) { + manager := NewManager(DefaultConfig(), managerTestStore{}, managerTestProviderManager{}, logrus.New()) + + if _, err := manager.GetQuota(context.Background(), "missing"); err != ErrUsageNotFound { + t.Fatalf("GetQuota() error = %v, want ErrUsageNotFound (identity, via ==)", err) + } + if _, err := manager.GetQuotaNoCache(context.Background(), "missing"); err != ErrUsageNotFound { + t.Fatalf("GetQuotaNoCache() error = %v, want ErrUsageNotFound (identity, via ==)", err) + } +} + func TestRefreshBoundsConcurrency(t *testing.T) { providers := make([]*typ.Provider, 20) for i := range providers { diff --git a/internal/server/module/providerquota/handler_test.go b/internal/server/module/providerquota/handler_test.go new file mode 100644 index 000000000..ce7735c7c --- /dev/null +++ b/internal/server/module/providerquota/handler_test.go @@ -0,0 +1,124 @@ +package providerquota + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/tingly-dev/tingly-box/ai/quota" +) + +// fakeManager lets each test configure exactly what GetQuota returns per +// provider UUID, without a real store/fetcher. +type fakeManager struct { + quotas map[string]*quota.ProviderUsage + errs map[string]error +} + +func (f *fakeManager) GetQuota(_ context.Context, providerUUID string) (*quota.ProviderUsage, error) { + if err, ok := f.errs[providerUUID]; ok { + return nil, err + } + if u, ok := f.quotas[providerUUID]; ok { + return u, nil + } + return nil, quota.ErrUsageNotFound +} +func (f *fakeManager) GetQuotaNoCache(ctx context.Context, providerUUID string) (*quota.ProviderUsage, error) { + return f.GetQuota(ctx, providerUUID) +} +func (f *fakeManager) ListQuota(context.Context) ([]*quota.ProviderUsage, error) { return nil, nil } +func (f *fakeManager) Refresh(context.Context) ([]*quota.ProviderUsage, error) { return nil, nil } +func (f *fakeManager) RefreshProvider(context.Context, string) (*quota.ProviderUsage, error) { + return nil, nil +} +func (f *fakeManager) Summary(context.Context) (*quota.Summary, error) { return nil, nil } +func (f *fakeManager) IsProviderSupported(string) bool { return true } +func (f *fakeManager) StartAutoRefresh(context.Context) {} +func (f *fakeManager) StopAutoRefresh() {} + +func postJSON(t *testing.T, h gin.HandlerFunc, body any) *httptest.ResponseRecorder { + t.Helper() + gin.SetMode(gin.TestMode) + raw, _ := json.Marshal(body) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(raw)) + c.Request.Header.Set("Content-Type", "application/json") + h(c) + return w +} + +// TestBatchGetQuota_SkipsProvidersWithNoData is the regression test for the +// bug e2e testing surfaced: a provider with no quota data (e.g. a vmodel/ +// local provider with no registered fetcher) used to 500 the WHOLE batch +// request instead of just being omitted from the result — because +// Manager.GetQuota re-wrapped ErrUsageNotFound into a new error, breaking +// the handler's `err != quota.ErrUsageNotFound` identity check (fixed in +// ai/quota/manager.go). This test exercises the real (non-fake) comparison +// path in the handler with a manager that returns the sentinel directly. +func TestBatchGetQuota_SkipsProvidersWithNoData(t *testing.T) { + mgr := &fakeManager{ + quotas: map[string]*quota.ProviderUsage{ + "has-data": {ProviderUUID: "has-data", ProviderName: "Real"}, + }, + // "no-data" provider: absent from both maps -> ErrUsageNotFound. + } + h := NewHandler(mgr, logrus.StandardLogger()) + + w := postJSON(t, h.BatchGetQuota, BatchGetQuotaRequest{ProviderUUIDs: []string{"has-data", "no-data"}}) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String()) + } + var resp BatchGetQuotaResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if _, ok := resp.Data["has-data"]; !ok { + t.Fatalf("expected has-data in response, got %+v", resp.Data) + } + if _, ok := resp.Data["no-data"]; ok { + t.Fatalf("expected no-data to be omitted (not errored), got %+v", resp.Data) + } +} + +// TestBatchGetQuota_FailsOnlyWhenEveryProviderErrors confirms a genuine +// (non-not-found) error still surfaces when NO provider in the batch +// produced usable data — distinct from the not-found-is-a-skip case above. +func TestBatchGetQuota_FailsOnlyWhenEveryProviderErrors(t *testing.T) { + mgr := &fakeManager{ + errs: map[string]error{"broken": context.DeadlineExceeded}, + } + h := NewHandler(mgr, logrus.StandardLogger()) + + w := postJSON(t, h.BatchGetQuota, BatchGetQuotaRequest{ProviderUUIDs: []string{"broken"}}) + + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500; body=%s", w.Code, w.Body.String()) + } +} + +// TestGetQuota_NotFoundReturns404 pins the single-provider GetQuota's +// not-found response, the same sentinel path BatchGetQuota relies on. +func TestGetQuota_NotFoundReturns404(t *testing.T) { + mgr := &fakeManager{} + h := NewHandler(mgr, logrus.StandardLogger()) + + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/provider-quota/no-data", nil) + c.Params = gin.Params{{Key: "uuid", Value: "no-data"}} + h.GetQuota(c) + + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body=%s", w.Code, w.Body.String()) + } +} diff --git a/sdk/python/README.md b/sdk/python/README.md index d1f23c6a4..413b9d0a5 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -151,5 +151,8 @@ connection modes" for the full mechanics. - **Layer 3** (tb routes to the plugin as a model): via provider-as-upstream. Done, verified end-to-end including a plugin forwarding to another tb rule and returning the result (`sdk/python/examples/e2e_run.sh`). +- **Deterministic dispatch** (`pin_provider`, `Client.rules`): done, verified + end-to-end including `router_plugin.py` run for real + (`sdk/python/examples/e2e_run_pin.sh`). See `.design/python-sdk.md` in the repo for the full design and diagrams. diff --git a/sdk/python/examples/e2e_run_pin.sh b/sdk/python/examples/e2e_run_pin.sh new file mode 100755 index 000000000..5924cddd2 --- /dev/null +++ b/sdk/python/examples/e2e_run_pin.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# End-to-end test for the two tb connection modes and router_plugin.py's use +# of both, using NO network / API keys (vmodel providers only): +# +# mode 1: scenario + rule(model) — tb picks the service +# (affinity/smart-routing/load-balancer; tier order here) +# mode 2: scenario + rule(model) + pin_provider — caller picks, but tb +# only allows a provider already on that rule's own services +# +# router_plugin.py is a real, unmodified consumer of both: it resolves +# candidates via Client.rules (mode 1's information), then forwards with +# pin_provider (mode 2) so the provider it quota-checked is the one that +# actually serves the request. vmodel providers have no fetchable quota, so +# the pick is a deterministic tie-break here, not a real quota decision — +# this script proves the WIRING (rule resolution -> pin -> tb enforcement), +# not "quota routing found a numerically better answer" (that needs a real +# provider account, out of scope for a no-network e2e test). +# +# Prereqs: +# go build -o /tmp/tb_e2e ./cli/tingly-box +# pip install -e . # from sdk/python (needs `tingly` importable) +# Run: bash sdk/python/examples/e2e_run_pin.sh +set -uo pipefail + +TB=${TB_BIN:-/tmp/tb_e2e} +CFG=$(mktemp -d) +PORT=18903 +BASE="http://127.0.0.1:$PORT" +SDK=/home/user/tingly-box/sdk/python +TB_LOG=/tmp/tb_pin_e2e.log +export PYTHONPATH=$SDK + +FAILED=0 +pass() { echo " PASS: $1"; } +fail() { echo " FAIL: $1"; FAILED=1; } + +cleanup() { + [[ -n "${PLUG_PID:-}" ]] && kill "$PLUG_PID" 2>/dev/null + [[ -n "${TB_PID:-}" ]] && kill "$TB_PID" 2>/dev/null +} +trap cleanup EXIT + +echo "== 1. start tb (config-dir=$CFG, port=$PORT) ==" +"$TB" --config-dir "$CFG" start --port "$PORT" --ui --browser=false >"$TB_LOG" 2>&1 & +TB_PID=$! +for i in $(seq 1 60); do + curl -sf "$BASE/api/v1/info/health" >/dev/null 2>&1 && break + sleep 0.5 +done +curl -sf "$BASE/api/v1/info/health" >/dev/null || { echo "tb did not start"; tail -30 "$TB_LOG"; exit 1; } +echo " tb healthy at $BASE" + +CFGFILE=$(find "$CFG" -name 'config.json' | head -1) +UTOK=$(python3 -c "import json;d=json.load(open('$CFGFILE'));print(d.get('user_token') or d.get('UserToken',''))") +MTOK=$(python3 -c "import json;d=json.load(open('$CFGFILE'));print(d.get('model_token') or d.get('ModelToken',''))") +UADMIN=(-H "Authorization: Bearer $UTOK" -H "Content-Type: application/json") +UMODEL=(-H "Authorization: Bearer $MTOK" -H "Content-Type: application/json") + +echo "== 2. create three vmodel providers (A, B, C — no network) ==" +mk_provider() { + curl -s "${UADMIN[@]}" -X POST "$BASE/api/v2/providers" -d "{ + \"name\":\"$1\",\"api_base\":\"vmodel://local\",\"api_style\":\"openai\", + \"auth_type\":\"vmodel\",\"no_key_required\":true,\"enabled\":true}" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print(d.get('data',{}).get('uuid') or d.get('uuid',''))" +} +PA=$(mk_provider vmodel-a) +PB=$(mk_provider vmodel-b) +PC=$(mk_provider vmodel-c) +echo " A=$PA B=$PB C=$PC" + +echo "== 3. rule 'tiered-model': A@tier0, B@tier1 (mode 1 target) ==" +echo " (request_model is the client-facing name; each service's own 'model'" +echo " must be 'echo-model' — the only mock ID the no-network vmodel backend knows)" +curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ + \"scenario\":\"experiment\",\"request_model\":\"tiered-model\",\"active\":true, + \"lb_tactic\":{\"type\":\"tier\",\"params\":{}}, + \"services\":[{\"provider\":\"$PA\",\"model\":\"echo-model\",\"weight\":1,\"active\":true,\"tier\":0}, + {\"provider\":\"$PB\",\"model\":\"echo-model\",\"weight\":1,\"active\":true,\"tier\":1}]}" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'))" + +echo "== 4. rules 'sonnet1'->A and 'sonnet2'->B, single service each ==" +echo " (router_plugin.py's default CANDIDATE_MODELS — unmodified file, real names)" +mk_pinned_rule() { + curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ + \"scenario\":\"experiment\",\"request_model\":\"$1\",\"active\":true, + \"lb_tactic\":{\"type\":\"random\",\"params\":{}}, + \"services\":[{\"provider\":\"$2\",\"model\":\"echo-model\",\"weight\":1,\"active\":true}]}" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'))" +} +mk_pinned_rule sonnet1 "$PA" +mk_pinned_rule sonnet2 "$PB" + +echo "== 5. MODE 1 (scenario+rule): unpinned call -> tb picks tier0 = A ==" +SEL=$(curl -s "${UMODEL[@]}" -H "X-Tingly-Debug-Routing: 1" -D - -o /dev/null \ + -X POST "$BASE/tingly/experiment/v1/chat/completions" \ + -d '{"model":"tiered-model","messages":[{"role":"user","content":"hi"}]}' \ + | grep -i "x-tingly-selected-provider-uuid" | tr -d '\r' | awk '{print $2}') +[[ "$SEL" == "$PA" ]] && pass "unpinned call selected tier0 (A=$PA)" || fail "unpinned call selected '$SEL', expected A=$PA" + +echo "== 6. MODE 2 (scenario+rule+pin): pin to B overrides tier order ==" +SEL=$(curl -s "${UMODEL[@]}" -H "X-Tingly-Debug-Routing: 1" -H "X-Tingly-Pin-Provider: $PB" -D - -o /dev/null \ + -X POST "$BASE/tingly/experiment/v1/chat/completions" \ + -d '{"model":"tiered-model","messages":[{"role":"user","content":"hi"}]}' \ + | grep -i "x-tingly-selected-provider-uuid" | tr -d '\r' | awk '{print $2}') +[[ "$SEL" == "$PB" ]] && pass "pinned call selected B ($PB) despite tier0=A" || fail "pinned call selected '$SEL', expected B=$PB" + +echo "== 7. MODE 2 scoping: pin to C (not on this rule) is rejected ==" +ERR=$(curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -H "X-Tingly-Pin-Provider: $PC" -d '{ + "model":"tiered-model","messages":[{"role":"user","content":"hi"}]}') +echo "$ERR" | grep -q "not an active service" && pass "pin to unrelated provider C rejected" || fail "expected rejection, got: $ERR" + +echo "== 8. SDK-level: Client.ask(pin_provider=) round-trips through the real gateway ==" +python3 - "$BASE" "$UTOK" "$PB" <<'PY' && pass "Client.ask(pin_provider=) completed" || { echo " FAIL: SDK pin_provider call raised"; FAILED=1; } +import sys +import tingly +base, admin_token, want_provider = sys.argv[1], sys.argv[2], sys.argv[3] +tb = tingly.connect(base_url=base, token=admin_token, scenario="experiment") +text = tb.ask("hi", model="tiered-model", pin_provider=want_provider) +assert isinstance(text, str) and text, f"expected non-empty text, got {text!r}" +PY + +echo "== 9. router_plugin.py: real run — resolves sonnet1/sonnet2 via Client.rules," +echo " picks by quota (tied here, no live quota source), forwards with pin_provider ==" +TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ + python3 "$SDK/examples/router_plugin.py" >/tmp/router_e2e.log 2>&1 & +PLUG_PID=$! +for i in $(seq 1 40); do + curl -sf "http://127.0.0.1:8768/health" >/dev/null 2>&1 && break + sleep 0.3 +done +curl -sf "http://127.0.0.1:8768/health" >/dev/null || { fail "router plugin did not start"; cat /tmp/router_e2e.log; } + +for i in $(seq 1 20); do + curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins" | grep -q '"router"' && break + sleep 0.3 +done + +RESP=$(curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ + "model":"plugin/router","messages":[{"role":"user","content":"what is 2+2?"}]}') +echo "$RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); print(' response:', d.get('choices',[{}])[0].get('message',{}).get('content', d))" \ + && pass "model=plugin/router answered" || fail "router call failed: $RESP" + +echo "== 10. confirm the router's FORWARDED call actually used a provider pin (tb log) ==" +grep -q "source=provider_pin" "$TB_LOG" \ + && pass "tb log shows a provider_pin-sourced selection (router's forwarded call)" \ + || fail "no provider_pin selection found in $TB_LOG" + +kill -KILL "$PLUG_PID" 2>/dev/null +PLUG_PID="" + +echo "== done: $([[ $FAILED -eq 0 ]] && echo "ALL PASSED" || echo "SOME FAILED — see above") ==" +exit $FAILED From 9c1310725bcdde47ab33baf138915e0944ea3e72 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 12:43:44 +0000 Subject: [PATCH 23/28] chore: regenerate openapi.json after rebase onto main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conflicts during the rebase onto main were resolved by keeping "ours" for openapi.json at each step (it's fully generated) rather than hand-merging a generated file commit by commit; this regenerates it once, cleanly, now that the rebase is done. Purely additive vs. the rebased state — no schema was lost by the --ours resolutions along the way. --- openapi.json | 276 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) diff --git a/openapi.json b/openapi.json index 9ae957298..317b59ccb 100644 --- a/openapi.json +++ b/openapi.json @@ -3833,6 +3833,39 @@ } } }, + "/api/v1/sdk/session": { + "post": { + "tags": [ + "sdk" + ], + "summary": "Mint a scoped SDK session for a scenario", + "description": "Mint a scoped SDK session for a scenario", + "operationId": "apiV1SdkSessionPost", + "requestBody": { + "description": "Request body", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SDKSessionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SDKSessionResponse" + } + } + } + } + } + } + }, "/api/v1/server/restart": { "post": { "tags": [ @@ -4789,6 +4822,59 @@ } } }, + "/api/v2/plugins": { + "get": { + "tags": [ + "plugins" + ], + "summary": "List registered plugin providers", + "description": "List registered plugin providers", + "operationId": "apiV2PluginsGet", + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PluginsResponse" + } + } + } + } + } + }, + "post": { + "tags": [ + "plugins" + ], + "summary": "Register (or update) external plugin code as an upstream, optionally binding a rule", + "description": "Register (or update) external plugin code as an upstream, optionally binding a rule", + "operationId": "apiV2PluginsPost", + "requestBody": { + "description": "Request body", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginResponse" + } + } + } + } + } + } + }, "/api/v2/probe": { "post": { "tags": [ @@ -9317,6 +9403,52 @@ "categories" ] }, + "PluginInfo": { + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "description": "Field endpoint" + }, + "model_id": { + "type": "string", + "description": "Field model_id" + }, + "name": { + "type": "string", + "description": "Field name" + }, + "uuid": { + "type": "string", + "description": "Field uuid" + } + }, + "required": [ + "uuid", + "name", + "endpoint" + ] + }, + "PluginsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "description": "Field data", + "items": { + "$ref": "#/components/schemas/PluginInfo" + } + }, + "success": { + "type": "boolean", + "description": "Field success" + } + }, + "required": [ + "success", + "data" + ] + }, "Policy": { "type": "object", "properties": { @@ -10251,6 +10383,83 @@ "message" ] }, + "RegisterPluginRequest": { + "type": "object", + "properties": { + "api_style": { + "type": "string", + "description": "Wire protocol the plugin's endpoint speaks: \"openai\" or \"anthropic\"; empty defaults to \"openai\"", + "example": "anthropic" + }, + "endpoint": { + "type": "string", + "description": "Plugin base URL", + "example": "http://127.0.0.1:8765/v1" + }, + "model_id": { + "type": "string", + "description": "Model id the plugin advertises", + "example": "plugin/my-rag" + }, + "name": { + "type": "string", + "description": "Plugin / provider name", + "example": "my-rag" + }, + "scenario": { + "type": "string", + "description": "Scenario to bind a rule under; omit to create only the provider", + "example": "experiment" + }, + "tier": { + "type": "integer", + "format": "int64", + "description": "Tier for the bound service (0 = highest priority)" + }, + "token": { + "type": "string", + "description": "Token tingly-box should send to the plugin (empty = no key)" + } + }, + "required": [ + "name", + "endpoint" + ] + }, + "RegisterPluginResponse": { + "type": "object", + "properties": { + "model_id": { + "type": "string", + "description": "Field model_id" + }, + "note": { + "type": "string", + "description": "Field note" + }, + "provider_uuid": { + "type": "string", + "description": "Field provider_uuid" + }, + "ready": { + "type": "boolean", + "description": "Field ready" + }, + "rule_uuid": { + "type": "string", + "description": "Field rule_uuid" + }, + "scenario": { + "type": "string", + "description": "Field scenario" + } + }, + "required": [ + "provider_uuid", + "model_id", + "ready" + ] + }, "RemoveSkillLocationResponse": { "type": "object", "properties": { @@ -10494,6 +10703,65 @@ "data" ] }, + "SDKSessionRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Field name" + }, + "scenario": { + "type": "string", + "description": "Field scenario" + } + }, + "required": [ + "scenario", + "name" + ] + }, + "SDKSessionResponse": { + "type": "object", + "properties": { + "base_url": { + "type": "string", + "description": "Field base_url" + }, + "expires_at": { + "type": "string", + "description": "Field expires_at" + }, + "ready": { + "type": "boolean", + "description": "Field ready" + }, + "scenario": { + "type": "string", + "description": "Field scenario" + }, + "services": { + "type": "integer", + "format": "int64", + "description": "Field services" + }, + "token": { + "type": "string", + "description": "Field token" + }, + "transport": { + "type": "string", + "description": "Field transport" + } + }, + "required": [ + "base_url", + "token", + "scenario", + "transport", + "ready", + "services" + ] + }, "ScanIdesResponse": { "type": "object", "properties": { @@ -13452,6 +13720,10 @@ "name": "onboarding", "description": "Operations related to onboarding" }, + { + "name": "plugins", + "description": "Operations related to plugins" + }, { "name": "providers", "description": "Operations related to providers" @@ -13468,6 +13740,10 @@ "name": "scenarios", "description": "Operations related to scenarios" }, + { + "name": "sdk", + "description": "Operations related to sdk" + }, { "name": "server", "description": "Operations related to server" From c68df68f7c6467bf035a75a83f6f3cb8c425ba39 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 13:56:33 +0000 Subject: [PATCH 24/28] =?UTF-8?q?docs(sdk):=20add=20python-sdk.pencil.md?= =?UTF-8?q?=20=E2=80=94=20the=20full=20request-flow=20pencil=20graphs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Visual companion to python-sdk.md, following the same convention as tier-routing.pencil.md / loadbalance.pencil.md / failover.pencil.md: architecture overview, Layer 1 provisioning-vs-inference, Layer 2 plugin anatomy + request lifecycle, Layer 3 provider-as-upstream wiring — plus new diagrams not previously drawn anywhere: the two connection modes (scenario+rule vs. scenario+rule+pin_provider), a scoping contrast between X-Tingly-Pin-Provider and X-Tingly-Probe-Service, router_plugin.py's decide-then-pin flow, a hop-count comparison across all four example plugins, the two-token model's surface split, and a map of what each e2e script actually exercises. python-sdk.md gets a one-line pointer at the top, matching how tier-routing.md points at failover.pencil.md. --- .design/python-sdk.md | 5 + .design/python-sdk.pencil.md | 382 +++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 .design/python-sdk.pencil.md diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 1c3342524..d16b78db7 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -3,6 +3,11 @@ > Audience: tingly-box contributors touching the SDK seam (`sdk/python/`), the > `/api/v1/sdk/session` endpoint, or the `experiment` scenario. +Diagram: `.design/python-sdk.pencil.md` — the full request-flow / architecture +pencil graphs (provisioning vs. inference, a plugin's request lifecycle, the +two connection modes, `router_plugin.py`'s decide-then-pin flow, and a +hop-count comparison across all four example plugins). + ## Why tb is a capable personal-intelligence gateway, but extending or experimenting diff --git a/.design/python-sdk.pencil.md b/.design/python-sdk.pencil.md new file mode 100644 index 000000000..d3fdbea9d --- /dev/null +++ b/.design/python-sdk.pencil.md @@ -0,0 +1,382 @@ +# Python SDK (`tingly`) — Pencil Graph + +Visual companion to `python-sdk.md`. Where that document argues the *why* +and pins the *facts* (exact endpoints, field names, file:line references), +this one is the flow, end to end — architecture, provisioning vs. inference, +a plugin's request lifecycle, the two connection modes, and how the four +example plugins differ in shape. Read `python-sdk.md` first if a term here +is unfamiliar; nothing here is authoritative on its own. + +Contents: + +- Architecture — one idea, not three layers +- Layer 1 — provisioning vs. inference (`connect()`) +- Layer 2 — a plugin's anatomy and request lifecycle +- Layer 3 — provider-as-upstream wiring +- Two connection modes — scenario+rule vs. scenario+rule+pin +- `X-Tingly-Pin-Provider` vs. `X-Tingly-Probe-Service` +- `router_plugin.py` — the decide-then-pin flow +- Example plugin shapes — hop-count comparison +- Two-token model — which token opens which surface +- Verified live — what each e2e script actually exercises + +## Architecture — one idea, not three layers + +> tb is a hub of rules. A rule's upstream can be a plugin. A plugin can +> originate calls against any other rule. + +``` + ┌──────────────────── tingly-box (the hub) ───────────────────┐ + clients │ │ + ┌─────────────┐ req │ rule A ──upstream──► PLUGIN CODE (your logic) │ + │ Claude Code │──────►│ (model=plugin/x) │ │ + │ Cursor │ │ │ calls back into tb: │ + │ tb UI │ │ rule B ◄──────────────────┤ use("…").ask(model="…") │ + │ tingly.ask()│ │ (→ Anthropic real) │ │ + └─────────────┘ │ rule C ◄──────────────────┤ (another model / provider) │ + │ rule D ◄──────────────────┘ (even another plugin) │ + │ │ │ + │ ▼ every edge gets: guard rails · routing/tiers · │ + │ failover · quota · logging │ + └──────────────────────────────────────────────────────────────┘ + │ + ▼ real upstreams (Anthropic / OpenAI / local …) +``` + +Three verbs, one relationship: + +``` + connect a plugin (or experiment) CONSUMES a rule tingly.connect() / plugin.use(scenario).ask() + serve a plugin IS a rule's upstream tingly.Plugin (Anthropic-primary server) + register point a rule's upstream at the plugin POST /api/v2/plugins (idempotent upsert) +``` + +"Layer 1/2/3" below = connect / serve / register. An implementation tour of +one idea, not three products. + +## Layer 1 — provisioning vs. inference (`connect()`) + +Two phases. **Provisioning** happens once (admin token, dashed). **Inference** +happens on every call (model token, solid) and reuses the exact same gateway +pipeline as any other tb client — the SDK adds no new path through the box. + +``` + YOUR PYTHON tingly-box GATEWAY UPSTREAMS + ┌───────────────────────┐ ┌──────────────────────────────────┐ ┌───────────────┐ + │ import tingly │ │ │ │ Anthropic │ + │ tb = tingly.connect() │ │ /api/v1/... (admin auth) │ │ OpenAI │ + │ │ │ /tingly/:scn (model auth) │ │ Deepseek │ + └───────────┬───────────┘ └──────────────────────────────────┘ │ vLLM / local │ + │ └───────▲───────┘ + ── PROVISION (once, admin token) ─────────────────────────────────────────────────┊──────── + │ ┊ + config.resolve() ┊ + args→env→sdk.json→config.json→localhost ┊ + │ ┊ + │ GET /api/v1/info/version (liveness) ┄┄┄┄┄┄┄┄►┐ ┊ + │ POST /api/v1/sdk/session {scenario,name} ┄┄┄►│ CreateSDKSession ┊ + │ Authorization: Bearer │ · validate scenario in registry + │ │ · transport = openai|anthropic|both + │ ◄┄┄┄ {base_url, token=, │ · ready/services from active rule + │ transport, ready, services} ┄┄┄┄┄┄┄┄┄┄┄┄┄┘ + ▼ + Client ── builds lazily ──► openai.OpenAI(base_url = root+"/v1", api_key=ModelToken) + anthropic.Anthropic(base_url = root, api_key=ModelToken) + + ══ INFERENCE (every call, model token) ═══════════════════════════════════════════════════ + │ + tb.ask("...", model="auto") ← tries Anthropic first, falls back to OpenAI + tb.openai.chat.completions.create(...) + tb.anthropic.messages.create(...) + │ POST /tingly/experiment/v1/chat/completions ┌─────────────────────────┐ + │ POST /tingly/experiment/v1/messages │ the SAME pipeline as │ + │ Authorization: Bearer ─────────►│ any other tb client │ + │ │ │ + │ │ scenario → rule resolve │ + │ │ guard rails (in/out) │ + │ │ smart routing / tiers │ + │ │ circuit-breaker failover│──► pick + │ │ quota + usage logging │ upstream + │ │ protocol transform │──────────► + │ ◄──────────── response (+ usage recorded) ──────└─────────────────────────┘ (solid) + ▼ + tb.usage.this_session() GET /api/v1/requests (admin token, read-back) + tb.guardrails.status() GET /api/v1/guardrails/config (admin token, read-back) + tb.quota.list() GET /api/v1/provider-quota (admin token, read-back) + tb.rules.for_model(...) GET /api/v1/rules?scenario= (admin token, read-back) +``` + +Key reading: the SDK never talks to providers directly — the rightmost column +is reachable **only** through the gateway box in the middle. Provisioning uses +the *admin* token and the `/api/v1/*` control plane; inference uses the +*model* token and the `/tingly/:scenario` data plane. Different tokens, +different surfaces (detailed further down). + +## Layer 2 — a plugin's anatomy and request lifecycle + +The loop is a cycle — the plugin's handler calls back *into* tb (steps 4–6), +so tb is both the caller (step 3) and the upstream-for-the-plugin (step 5). + +``` + A PLUGIN (one Python process) tingly-box GATEWAY + ┌───────────────────────────────────┐ ┌──────────────────────────────┐ + │ Plugin(name="my-rag") │ │ │ + │ │ │ provider: │ + │ @plugin.chat │ │ name=my-rag │ + │ def handle(req): ... │ │ api_base=http://…:8765/v1 │ + │ │ │ │ api_style=anthropic (dflt) │ + │ │ returns str | iter[str] │ │ model=plugin/my-rag │ + │ ▼ │ │ │ + │ serve() → stdlib HTTP server │ │ rule: plugin/my-rag → ↑ │ + │ POST /v1/messages ◄────┼─ (3) POST /v1/messages?beta=true ──┘ ▲ │ + │ (primary, api_style match) │ (model=plugin/my-rag) │ │ + │ POST /v1/chat/completions ◄────┼─ (3') POST /v1/chat/completions ─────┘ │ + │ (secondary, if api_style=openai) (6) answer │ + │ GET /v1/models │ │ + │ GET /health │ │ + │ · buffered → message / chat.completion │ + │ · stream → SSE (message_* events / chat.completion.chunk) ── (7) ───┘ + │ │ │ + │ plugin.llm (lazy Layer-1 client)│ + │ │ │ + └────────┼──────────────────────────┘ + │ (4) plugin.llm.ask("…", model="auto") + │ = tingly.connect(scenario="experiment") → POST /tingly/experiment/v1/messages + ▼ + ┌──────────────────────────────────────────────────────────────┐ + │ tingly-box pipeline (SAME as any client — see Layer 1 graph) │ + │ scenario→rule · guard rails · routing/tiers · failover · │ + │ quota · logging · transform ─────────────────────────► (5) real upstream + └──────────────────────────────────────────────────────────────┘ (Anthropic/ + OpenAI/…) + + request lifecycle: + (1) client sends model="plugin/my-rag" to tb, any protocol ── see Layer 3 graph + (2) tb resolves rule → provider my-rag (api_base = plugin, api_style picks the route) + (3) tb calls the PLUGIN on whichever route matches provider.api_style + (4) handler runs; calls plugin.llm.ask(...) ── back INTO tb, Anthropic-first + (5) tb routes that call to a real upstream, applies guard rails/quota/… + (6) generated text returns to the handler + (7) handler's str/iterator → response/SSE shaped for whichever route was hit → back to client +``` + +One process, two roles: as a *server* the plugin answers tb on `:8765/v1`; as +a *client* (`plugin.llm`) it consumes tb via Layer 1. The author writes only +step 4's body — wire parsing, response/SSE shaping, discovery/session, +routing/guard-rails are the SDK and the gateway. Guard rails apply **twice**, +correctly: once on the inbound call to the plugin (step 3), again on the +plugin's own LLM call (step 5) — neither is wired by the author. + +## Layer 3 — provider-as-upstream wiring + +A Python plugin is out-of-process, so it's selected as a normal +**provider/upstream**, not the in-process `AuthType=virtual` `vmodel` path +(that needs a Go shim; not worth it here — see `python-sdk.md`). + +``` + ANY tb client tingly-box GATEWAY UPSTREAMS + ┌──────────────┐ ┌──────────────────────────────┐ + │ Claude Code │ │ HandleOpenAIChatCompletions │ tier 1 ┌──────────────┐ + │ Cursor │ model │ scenario → rule resolve │ ┌───────► │ Anthropic / │ + │ tb UI ├───────►│ guard rails (in/out) │ │ fallback│ OpenAI (real)│ + │ tingly.ask() │ "plugin│ smart routing / TIERS ──────┼──┤ └──────────────┘ + └──────────────┘ /my-rag"│ circuit-breaker failover │ │ tier 0 ┌──────────────┐ + │ quota + usage logging │ └───────► │ my-rag PLUGIN│ ◄─ Layer 2 + │ provider.api_base = plugin │ POST │ POST /v1/ │ Plugin.serve() + │ provider.api_style picks route│ /v1/msgs │ messages │ + └────────────────────────────────┘ (dflt) └──────┬───────┘ + ctx.llm.ask() ┄┄┄┘ (plugin may + back INTO tb for its own LLM calls) +``` + +Wiring is four steps, no new gateway hot-path code — it's just a provider: +plugin serves both routes → `POST /api/v2/plugins` creates the provider + +binds the rule/service → `model:"plugin/my-rag"` now resolves through the +same dispatcher as every other model → tier the plugin under a real model and +tb fails over automatically when the plugin is down. + +## Two connection modes — scenario+rule vs. scenario+rule+pin + +Every call still starts the same way: `(scenario, model)` resolves to a +**rule**. What differs is who picks *which* of the rule's services actually +runs. + +``` +MODE 1 — scenario + rule MODE 2 — scenario + rule + pin +tb DECIDES (Client.ask()'s default) CALLER decides, but SCOPED + +tb.ask(model="X") tb.ask(model="X", pin_provider=B_uuid) + │ │ + ▼ ▼ +(scenario, model) ──resolve──► rule (scenario, model) ──resolve──► rule (SAME step) + │ │ + ▼ ▼ + rule.Services[] rule.Services[] + ┌───────┬───────┬───────┐ ┌───────┬───────┬───────┐ + │ tier0 │ tier0 │ tier1 │ │ tier0 │ tier0 │ tier1 │ + │ Aa │ Ab │ B │ │ Aa │ Ab │ B ◄──┼── pin_provider=B_uuid + └───┬───┴───┬───┴───┬────┘ └───────┴───────┴───┬────┘ + │ │ │ │ + affinity → smart-routing → load-balancer scoping check: B_uuid ∈ rule.Services ? + │ │ + ▼ yes ──────┴────── no + ONE service picked │ │ + (tb's choice — may vary ▼ ▼ + run to run: tier order, SKIP affinity/routing/LB 400 "not an + session pin, load) entirely — USE B active service + on this rule" +``` + +`router_plugin.py` is what surfaced the gap mode 2 closes: picking a provider +by quota and then calling `.ask(model=X)` is a **guess**, not a decision, the +moment a rule has more than one active service — nothing stops tb's own +load-balancer from choosing differently. Mode 2 makes the pick binding. + +## `X-Tingly-Pin-Provider` vs. `X-Tingly-Probe-Service` + +Two headers do structurally the same bypass (`internal/server/routing/simple.go`) +but are not interchangeable — the scoping check is the entire difference: + +``` + X-Tingly-Probe-Service X-Tingly-Pin-Provider + (pre-existing, internal-only) (this branch, SDK-facing) + ────────────────────────────────────────────────────────────────────────────────── + header value ":" "" + rule resolution SKIPPED — a synthetic rule is NORMAL — the real rule is + built on the fly resolved first, same as mode 1 + valid pin targets ANY provider on the box only providers already in + THIS resolved rule's Services[] + auth none at the header level — any rides the SAME model-token + caller reaching tb's HTTP port auth already required for + can send it (.design/probe.md) /tingly/:scenario/... — nothing new + who sends it today tb's own probe/diagnostics UI any SDK caller — + (internal/probe/e2e.go) Client.ask(pin_provider=...) + routing source label SourceProbePin SourceProviderPin + safe for SDK exposure? NO — deliberately never exposed YES — that scoping check is + to plugin authors exactly what makes it safe +``` + +## `router_plugin.py` — the decide-then-pin flow + +A router *generates nothing* — its entire job is picking the ONE candidate +that gets the real call, then guaranteeing it lands there. + +``` +handle(req) + │ + question = req.last_user_text() + ▼ +_pick_candidate() + │ + ├─ for model in CANDIDATE_MODELS: e.g. ["sonnet1", "sonnet2"] + │ rule = Client.rules.for_model(scenario, model) + │ │ + │ ├─ rule is None ───────────────────────────► SKIP (model not configured) + │ │ + │ └─ len(rule.active_services) != 1 ──────────► SKIP (0 or >1 services — tb's own + │ LB would decide; a quota check + │ on ONE of several means nothing) + │ │ + │ └─ exactly 1 service ──► ResolvedCandidate(model, provider_uuid=services[0].provider) + │ + ▼ +resolved = [ (sonnet1 → A), (sonnet2 → B), … ] only single-provider rules survive + │ + ▼ +quotas = Client.quota.batch([c.provider_uuid for c in resolved]) ← ONE control-plane round trip + │ missing quota data for a candidate → headroom defaults to 100.0 + │ ("unknown" is NOT "starved" — see ProviderQuota.headroom_percent) + ▼ +chosen = max(resolved, key=lambda c: quotas[c.provider_uuid].headroom_percent) + ▼ +plugin.use(scenario).ask(question, model=chosen.model, + pin_provider=chosen.provider_uuid) + │ └── MODE 2 (above): the provider that was + │ quota-checked is GUARANTEED to serve this + ▼ +answer ── back to the original caller +``` + +No candidate resolves → `_pick_candidate()` raises loudly (`RuntimeError`), +rather than silently guessing at an unroutable model. + +## Example plugin shapes — hop-count comparison + +Four plugins, four different relationships to "how many times does this +handler call back into tb, and how is the final one chosen": + +``` +rag_plugin.py client ──► plugin ──► tb ──► real model (1 hop, fixed rule) + (generation over retrieved context) + +critic_plugin.py client ──► plugin ──► tb ──► DIFFERENT model (1 hop, fixed rule) + (cross-model critique) + +fusion_plugin.py client ──► plugin ──┬─► tb ──► model A ┐ + ├─► tb ──► model B ├── N hops (panel, concurrent) + └─► tb ──► model C ┘ + panel disagrees? ──► tb ──► judge (+1 hop) + panel agrees? ──► skip the judge hop + +router_plugin.py client ──► plugin ──► tb.rules (control plane — no model call) + ──► tb.quota (control plane — no model call) + ──► tb ──► ONE chosen model, PINNED (1 hop) +``` + +rag/critic/router all cost exactly one *generating* hop; router just spends +two extra *control-plane* round trips (rules, quota) deciding which one. +fusion is the only shape that deliberately spends more than one generating +hop per request — that's the point of asking a panel. + +## Two-token model — which token opens which surface + +``` + admin token (tb's UserToken) model token (tb's ModelToken) + ────────────────────────────── ────────────────────────────── + authorizes POST /api/v1/sdk/session /tingly/:scenario/... + GET/POST /api/v1/... : (chat/completions, messages — + requests (.usage) the actual LLM calls) + guardrails/config (.guardrails) + provider-quota[...] (.quota) + rules?scenario= (.rules) + resolved via args → env → sdk.json → returned BY the session response + config.json:UserToken (Client holds it, never re-resolved) + who calls it connect()'s provisioning step; Client.ask() / .openai / .anthropic + Client.usage/.guardrails/ + .quota/.rules (read-back views) + scope full admin — can inspect any scoped to inference on the scenario + rule/provider/quota on the box the session was minted for +``` + +Provisioning (admin token) happens once per `connect()`; inference (model +token) happens on every `.ask()` call. A plugin process typically holds both: +its own registration used the admin token once at startup, and every +`plugin.llm.ask(...)` afterward uses a model token from its own `connect()`. + +## Verified live — what each e2e script actually exercises + +Both are fixed, repeatable, real-`tb`-binary scripts — no mocks, no network, +no API keys (vmodel providers only) — with hard pass/fail assertions. + +``` +sdk/python/examples/e2e_run.sh sdk/python/examples/e2e_run_pin.sh +──────────────────────────────────────── ──────────────────────────────────────── +plugin registration MODE 1: unpinned call → tier0 selected + (idempotent upsert-by-name) MODE 2: pinned call → tier1 selected +round-trip: client → tb → plugin → (overrides tier order) + plugin.use(...) → tb → another rule MODE 2 scoping: pin to an unrelated + → back through tb → client provider → rejected (400) +crash (SIGKILL) → circuit breaker SDK-level: Client.ask(pin_provider=) + (no fallback tier ⇒ plain error; round-trips the same way + add tier-1 to see failover instead) router_plugin.py run for real: +re-register → same provider, no duplicate resolves sonnet1/sonnet2 via + Client.rules, checks quota, forwards + with pin_provider — tb's own routing + log confirms a provider_pin-sourced + selection for that forwarded call +``` + +`e2e_run_pin.sh`'s first live run also caught a real bug (`Manager.GetQuota` +re-wrapping `ErrUsageNotFound`, 500ing `POST /provider-quota/batch` for any +provider with no quota data) — fixed alongside it; see `python-sdk.md` for +the full writeup. From 58a5eea65ff83328a9b542d7877cbfb2a7585ce1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 14:03:23 +0000 Subject: [PATCH 25/28] =?UTF-8?q?docs(sdk):=20simplify=20python-sdk.pencil?= =?UTF-8?q?.md=20=E2=80=94=204=20diagrams=20instead=20of=2010?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first version was too dense to actually read as a quick reference — 10 sections, multi-column tables, box-drawing crammed with detail already covered in prose in python-sdk.md. Cut down to four small diagrams, each answering exactly one question: the one idea (plugin = rule whose upstream calls back), a request start to finish (connect -> ask -> plugin hop), the two ways to pick a provider (default vs pin_provider=, side by side, minimal), and router_plugin.py's decision flow as a plain linear list. Dropped as pencil content (better as prose, not diagrams): the pin_provider vs probe_service comparison table, the two-token model table, the hop-count comparison, and the e2e-script coverage map — none of those are actually flows, they were tables wearing a diagram's clothes. --- .design/python-sdk.md | 7 +- .design/python-sdk.pencil.md | 427 ++++++----------------------------- 2 files changed, 76 insertions(+), 358 deletions(-) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index d16b78db7..0815f4c92 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -3,10 +3,9 @@ > Audience: tingly-box contributors touching the SDK seam (`sdk/python/`), the > `/api/v1/sdk/session` endpoint, or the `experiment` scenario. -Diagram: `.design/python-sdk.pencil.md` — the full request-flow / architecture -pencil graphs (provisioning vs. inference, a plugin's request lifecycle, the -two connection modes, `router_plugin.py`'s decide-then-pin flow, and a -hop-count comparison across all four example plugins). +Diagram: `.design/python-sdk.pencil.md` — four simple pencil graphs: the one +idea, a request start to finish, the two ways to pick a provider, and +`router_plugin.py`'s decision flow. ## Why diff --git a/.design/python-sdk.pencil.md b/.design/python-sdk.pencil.md index d3fdbea9d..cd206a631 100644 --- a/.design/python-sdk.pencil.md +++ b/.design/python-sdk.pencil.md @@ -1,382 +1,101 @@ # Python SDK (`tingly`) — Pencil Graph -Visual companion to `python-sdk.md`. Where that document argues the *why* -and pins the *facts* (exact endpoints, field names, file:line references), -this one is the flow, end to end — architecture, provisioning vs. inference, -a plugin's request lifecycle, the two connection modes, and how the four -example plugins differ in shape. Read `python-sdk.md` first if a term here -is unfamiliar; nothing here is authoritative on its own. +Visual companion to `python-sdk.md`. Four pictures, each answering one +question. For exact endpoints / field names / file:line references, that +doc is the source of truth — this page is just the shape of things. Contents: -- Architecture — one idea, not three layers -- Layer 1 — provisioning vs. inference (`connect()`) -- Layer 2 — a plugin's anatomy and request lifecycle -- Layer 3 — provider-as-upstream wiring -- Two connection modes — scenario+rule vs. scenario+rule+pin -- `X-Tingly-Pin-Provider` vs. `X-Tingly-Probe-Service` -- `router_plugin.py` — the decide-then-pin flow -- Example plugin shapes — hop-count comparison -- Two-token model — which token opens which surface -- Verified live — what each e2e script actually exercises +- The one idea +- A request, start to finish +- Two ways to pick a provider +- `router_plugin.py` in one picture -## Architecture — one idea, not three layers - -> tb is a hub of rules. A rule's upstream can be a plugin. A plugin can -> originate calls against any other rule. - -``` - ┌──────────────────── tingly-box (the hub) ───────────────────┐ - clients │ │ - ┌─────────────┐ req │ rule A ──upstream──► PLUGIN CODE (your logic) │ - │ Claude Code │──────►│ (model=plugin/x) │ │ - │ Cursor │ │ │ calls back into tb: │ - │ tb UI │ │ rule B ◄──────────────────┤ use("…").ask(model="…") │ - │ tingly.ask()│ │ (→ Anthropic real) │ │ - └─────────────┘ │ rule C ◄──────────────────┤ (another model / provider) │ - │ rule D ◄──────────────────┘ (even another plugin) │ - │ │ │ - │ ▼ every edge gets: guard rails · routing/tiers · │ - │ failover · quota · logging │ - └──────────────────────────────────────────────────────────────┘ - │ - ▼ real upstreams (Anthropic / OpenAI / local …) -``` - -Three verbs, one relationship: +## The one idea ``` - connect a plugin (or experiment) CONSUMES a rule tingly.connect() / plugin.use(scenario).ask() - serve a plugin IS a rule's upstream tingly.Plugin (Anthropic-primary server) - register point a rule's upstream at the plugin POST /api/v2/plugins (idempotent upsert) + client tingly-box real upstream + ┌────────┐ model=x ┌──────────────────────┐ + │ any app │────────────►│ rule x → PLUGIN CODE │ + └────────┘ │ │ │ + │ │ calls back:│ + │ rule y ◄──┘ use(y) │ + └─────┬──────────────────┘ + ▼ + Anthropic / OpenAI / local … ``` -"Layer 1/2/3" below = connect / serve / register. An implementation tour of -one idea, not three products. - -## Layer 1 — provisioning vs. inference (`connect()`) +A plugin is just a rule whose upstream happens to be your code. It can call +*back* into any other rule to get its own answer — same gateway, same guard +rails / quota / logging, both directions. -Two phases. **Provisioning** happens once (admin token, dashed). **Inference** -happens on every call (model token, solid) and reuses the exact same gateway -pipeline as any other tb client — the SDK adds no new path through the box. +## A request, start to finish ``` - YOUR PYTHON tingly-box GATEWAY UPSTREAMS - ┌───────────────────────┐ ┌──────────────────────────────────┐ ┌───────────────┐ - │ import tingly │ │ │ │ Anthropic │ - │ tb = tingly.connect() │ │ /api/v1/... (admin auth) │ │ OpenAI │ - │ │ │ /tingly/:scn (model auth) │ │ Deepseek │ - └───────────┬───────────┘ └──────────────────────────────────┘ │ vLLM / local │ - │ └───────▲───────┘ - ── PROVISION (once, admin token) ─────────────────────────────────────────────────┊──────── - │ ┊ - config.resolve() ┊ - args→env→sdk.json→config.json→localhost ┊ - │ ┊ - │ GET /api/v1/info/version (liveness) ┄┄┄┄┄┄┄┄►┐ ┊ - │ POST /api/v1/sdk/session {scenario,name} ┄┄┄►│ CreateSDKSession ┊ - │ Authorization: Bearer │ · validate scenario in registry - │ │ · transport = openai|anthropic|both - │ ◄┄┄┄ {base_url, token=, │ · ready/services from active rule - │ transport, ready, services} ┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ▼ - Client ── builds lazily ──► openai.OpenAI(base_url = root+"/v1", api_key=ModelToken) - anthropic.Anthropic(base_url = root, api_key=ModelToken) + 1. connect() admin token ──► mint a session ──► model token - ══ INFERENCE (every call, model token) ═══════════════════════════════════════════════════ - │ - tb.ask("...", model="auto") ← tries Anthropic first, falls back to OpenAI - tb.openai.chat.completions.create(...) - tb.anthropic.messages.create(...) - │ POST /tingly/experiment/v1/chat/completions ┌─────────────────────────┐ - │ POST /tingly/experiment/v1/messages │ the SAME pipeline as │ - │ Authorization: Bearer ─────────►│ any other tb client │ - │ │ │ - │ │ scenario → rule resolve │ - │ │ guard rails (in/out) │ - │ │ smart routing / tiers │ - │ │ circuit-breaker failover│──► pick - │ │ quota + usage logging │ upstream - │ │ protocol transform │──────────► - │ ◄──────────── response (+ usage recorded) ──────└─────────────────────────┘ (solid) - ▼ - tb.usage.this_session() GET /api/v1/requests (admin token, read-back) - tb.guardrails.status() GET /api/v1/guardrails/config (admin token, read-back) - tb.quota.list() GET /api/v1/provider-quota (admin token, read-back) - tb.rules.for_model(...) GET /api/v1/rules?scenario= (admin token, read-back) -``` - -Key reading: the SDK never talks to providers directly — the rightmost column -is reachable **only** through the gateway box in the middle. Provisioning uses -the *admin* token and the `/api/v1/*` control plane; inference uses the -*model* token and the `/tingly/:scenario` data plane. Different tokens, -different surfaces (detailed further down). - -## Layer 2 — a plugin's anatomy and request lifecycle - -The loop is a cycle — the plugin's handler calls back *into* tb (steps 4–6), -so tb is both the caller (step 3) and the upstream-for-the-plugin (step 5). - -``` - A PLUGIN (one Python process) tingly-box GATEWAY - ┌───────────────────────────────────┐ ┌──────────────────────────────┐ - │ Plugin(name="my-rag") │ │ │ - │ │ │ provider: │ - │ @plugin.chat │ │ name=my-rag │ - │ def handle(req): ... │ │ api_base=http://…:8765/v1 │ - │ │ │ │ api_style=anthropic (dflt) │ - │ │ returns str | iter[str] │ │ model=plugin/my-rag │ - │ ▼ │ │ │ - │ serve() → stdlib HTTP server │ │ rule: plugin/my-rag → ↑ │ - │ POST /v1/messages ◄────┼─ (3) POST /v1/messages?beta=true ──┘ ▲ │ - │ (primary, api_style match) │ (model=plugin/my-rag) │ │ - │ POST /v1/chat/completions ◄────┼─ (3') POST /v1/chat/completions ─────┘ │ - │ (secondary, if api_style=openai) (6) answer │ - │ GET /v1/models │ │ - │ GET /health │ │ - │ · buffered → message / chat.completion │ - │ · stream → SSE (message_* events / chat.completion.chunk) ── (7) ───┘ - │ │ │ - │ plugin.llm (lazy Layer-1 client)│ - │ │ │ - └────────┼──────────────────────────┘ - │ (4) plugin.llm.ask("…", model="auto") - │ = tingly.connect(scenario="experiment") → POST /tingly/experiment/v1/messages - ▼ - ┌──────────────────────────────────────────────────────────────┐ - │ tingly-box pipeline (SAME as any client — see Layer 1 graph) │ - │ scenario→rule · guard rails · routing/tiers · failover · │ - │ quota · logging · transform ─────────────────────────► (5) real upstream - └──────────────────────────────────────────────────────────────┘ (Anthropic/ - OpenAI/…) + 2. tb.ask("...") model token ──► tb picks a rule ──► picks a service + │ + ▼ + real model answers - request lifecycle: - (1) client sends model="plugin/my-rag" to tb, any protocol ── see Layer 3 graph - (2) tb resolves rule → provider my-rag (api_base = plugin, api_style picks the route) - (3) tb calls the PLUGIN on whichever route matches provider.api_style - (4) handler runs; calls plugin.llm.ask(...) ── back INTO tb, Anthropic-first - (5) tb routes that call to a real upstream, applies guard rails/quota/… - (6) generated text returns to the handler - (7) handler's str/iterator → response/SSE shaped for whichever route was hit → back to client + 3. if that model IS a plugin: + tb calls the plugin instead of a real model (step 2, inbound) + the plugin's handler does step 2 AGAIN, on its own, to get ITS answer + the plugin's answer becomes tb's answer to the original caller ``` -One process, two roles: as a *server* the plugin answers tb on `:8765/v1`; as -a *client* (`plugin.llm`) it consumes tb via Layer 1. The author writes only -step 4's body — wire parsing, response/SSE shaping, discovery/session, -routing/guard-rails are the SDK and the gateway. Guard rails apply **twice**, -correctly: once on the inbound call to the plugin (step 3), again on the -plugin's own LLM call (step 5) — neither is wired by the author. +Steps 1 and 2 are all of Layer 1 (`Client`). Step 3 is Layer 2 (`Plugin`) — +same request, plugin just sits in the middle and calls back once. -## Layer 3 — provider-as-upstream wiring +## Two ways to pick a provider -A Python plugin is out-of-process, so it's selected as a normal -**provider/upstream**, not the in-process `AuthType=virtual` `vmodel` path -(that needs a Go shim; not worth it here — see `python-sdk.md`). +A model can have more than one provider behind it (tiers, fallback). Normally +tb picks. `pin_provider=` lets the caller pick instead — but only from what +that rule already offers. ``` - ANY tb client tingly-box GATEWAY UPSTREAMS - ┌──────────────┐ ┌──────────────────────────────┐ - │ Claude Code │ │ HandleOpenAIChatCompletions │ tier 1 ┌──────────────┐ - │ Cursor │ model │ scenario → rule resolve │ ┌───────► │ Anthropic / │ - │ tb UI ├───────►│ guard rails (in/out) │ │ fallback│ OpenAI (real)│ - │ tingly.ask() │ "plugin│ smart routing / TIERS ──────┼──┤ └──────────────┘ - └──────────────┘ /my-rag"│ circuit-breaker failover │ │ tier 0 ┌──────────────┐ - │ quota + usage logging │ └───────► │ my-rag PLUGIN│ ◄─ Layer 2 - │ provider.api_base = plugin │ POST │ POST /v1/ │ Plugin.serve() - │ provider.api_style picks route│ /v1/msgs │ messages │ - └────────────────────────────────┘ (dflt) └──────┬───────┘ - ctx.llm.ask() ┄┄┄┘ (plugin may - back INTO tb for its own LLM calls) + default pin_provider=B + ─────── ────────────── + tb.ask(model="x") tb.ask(model="x", pin_provider=B) + │ │ + ▼ ▼ + rule "x" → services [A, B] rule "x" → services [A, B] + │ │ + ▼ ▼ + tb picks ONE must B be in [A, B]? + (tiers / affinity / load) yes → use B, guaranteed + no → 400, rejected ``` -Wiring is four steps, no new gateway hot-path code — it's just a provider: -plugin serves both routes → `POST /api/v2/plugins` creates the provider + -binds the rule/service → `model:"plugin/my-rag"` now resolves through the -same dispatcher as every other model → tier the plugin under a real model and -tb fails over automatically when the plugin is down. - -## Two connection modes — scenario+rule vs. scenario+rule+pin - -Every call still starts the same way: `(scenario, model)` resolves to a -**rule**. What differs is who picks *which* of the rule's services actually -runs. - -``` -MODE 1 — scenario + rule MODE 2 — scenario + rule + pin -tb DECIDES (Client.ask()'s default) CALLER decides, but SCOPED - -tb.ask(model="X") tb.ask(model="X", pin_provider=B_uuid) - │ │ - ▼ ▼ -(scenario, model) ──resolve──► rule (scenario, model) ──resolve──► rule (SAME step) - │ │ - ▼ ▼ - rule.Services[] rule.Services[] - ┌───────┬───────┬───────┐ ┌───────┬───────┬───────┐ - │ tier0 │ tier0 │ tier1 │ │ tier0 │ tier0 │ tier1 │ - │ Aa │ Ab │ B │ │ Aa │ Ab │ B ◄──┼── pin_provider=B_uuid - └───┬───┴───┬───┴───┬────┘ └───────┴───────┴───┬────┘ - │ │ │ │ - affinity → smart-routing → load-balancer scoping check: B_uuid ∈ rule.Services ? - │ │ - ▼ yes ──────┴────── no - ONE service picked │ │ - (tb's choice — may vary ▼ ▼ - run to run: tier order, SKIP affinity/routing/LB 400 "not an - session pin, load) entirely — USE B active service - on this rule" -``` - -`router_plugin.py` is what surfaced the gap mode 2 closes: picking a provider -by quota and then calling `.ask(model=X)` is a **guess**, not a decision, the -moment a rule has more than one active service — nothing stops tb's own -load-balancer from choosing differently. Mode 2 makes the pick binding. - -## `X-Tingly-Pin-Provider` vs. `X-Tingly-Probe-Service` - -Two headers do structurally the same bypass (`internal/server/routing/simple.go`) -but are not interchangeable — the scoping check is the entire difference: - -``` - X-Tingly-Probe-Service X-Tingly-Pin-Provider - (pre-existing, internal-only) (this branch, SDK-facing) - ────────────────────────────────────────────────────────────────────────────────── - header value ":" "" - rule resolution SKIPPED — a synthetic rule is NORMAL — the real rule is - built on the fly resolved first, same as mode 1 - valid pin targets ANY provider on the box only providers already in - THIS resolved rule's Services[] - auth none at the header level — any rides the SAME model-token - caller reaching tb's HTTP port auth already required for - can send it (.design/probe.md) /tingly/:scenario/... — nothing new - who sends it today tb's own probe/diagnostics UI any SDK caller — - (internal/probe/e2e.go) Client.ask(pin_provider=...) - routing source label SourceProbePin SourceProviderPin - safe for SDK exposure? NO — deliberately never exposed YES — that scoping check is - to plugin authors exactly what makes it safe -``` - -## `router_plugin.py` — the decide-then-pin flow - -A router *generates nothing* — its entire job is picking the ONE candidate -that gets the real call, then guaranteeing it lands there. - -``` -handle(req) - │ - question = req.last_user_text() - ▼ -_pick_candidate() - │ - ├─ for model in CANDIDATE_MODELS: e.g. ["sonnet1", "sonnet2"] - │ rule = Client.rules.for_model(scenario, model) - │ │ - │ ├─ rule is None ───────────────────────────► SKIP (model not configured) - │ │ - │ └─ len(rule.active_services) != 1 ──────────► SKIP (0 or >1 services — tb's own - │ LB would decide; a quota check - │ on ONE of several means nothing) - │ │ - │ └─ exactly 1 service ──► ResolvedCandidate(model, provider_uuid=services[0].provider) - │ - ▼ -resolved = [ (sonnet1 → A), (sonnet2 → B), … ] only single-provider rules survive - │ - ▼ -quotas = Client.quota.batch([c.provider_uuid for c in resolved]) ← ONE control-plane round trip - │ missing quota data for a candidate → headroom defaults to 100.0 - │ ("unknown" is NOT "starved" — see ProviderQuota.headroom_percent) - ▼ -chosen = max(resolved, key=lambda c: quotas[c.provider_uuid].headroom_percent) - ▼ -plugin.use(scenario).ask(question, model=chosen.model, - pin_provider=chosen.provider_uuid) - │ └── MODE 2 (above): the provider that was - │ quota-checked is GUARANTEED to serve this - ▼ -answer ── back to the original caller -``` - -No candidate resolves → `_pick_candidate()` raises loudly (`RuntimeError`), -rather than silently guessing at an unroutable model. - -## Example plugin shapes — hop-count comparison - -Four plugins, four different relationships to "how many times does this -handler call back into tb, and how is the final one chosen": - -``` -rag_plugin.py client ──► plugin ──► tb ──► real model (1 hop, fixed rule) - (generation over retrieved context) - -critic_plugin.py client ──► plugin ──► tb ──► DIFFERENT model (1 hop, fixed rule) - (cross-model critique) - -fusion_plugin.py client ──► plugin ──┬─► tb ──► model A ┐ - ├─► tb ──► model B ├── N hops (panel, concurrent) - └─► tb ──► model C ┘ - panel disagrees? ──► tb ──► judge (+1 hop) - panel agrees? ──► skip the judge hop - -router_plugin.py client ──► plugin ──► tb.rules (control plane — no model call) - ──► tb.quota (control plane — no model call) - ──► tb ──► ONE chosen model, PINNED (1 hop) -``` - -rag/critic/router all cost exactly one *generating* hop; router just spends -two extra *control-plane* round trips (rules, quota) deciding which one. -fusion is the only shape that deliberately spends more than one generating -hop per request — that's the point of asking a panel. - -## Two-token model — which token opens which surface - -``` - admin token (tb's UserToken) model token (tb's ModelToken) - ────────────────────────────── ────────────────────────────── - authorizes POST /api/v1/sdk/session /tingly/:scenario/... - GET/POST /api/v1/... : (chat/completions, messages — - requests (.usage) the actual LLM calls) - guardrails/config (.guardrails) - provider-quota[...] (.quota) - rules?scenario= (.rules) - resolved via args → env → sdk.json → returned BY the session response - config.json:UserToken (Client holds it, never re-resolved) - who calls it connect()'s provisioning step; Client.ask() / .openai / .anthropic - Client.usage/.guardrails/ - .quota/.rules (read-back views) - scope full admin — can inspect any scoped to inference on the scenario - rule/provider/quota on the box the session was minted for -``` - -Provisioning (admin token) happens once per `connect()`; inference (model -token) happens on every `.ask()` call. A plugin process typically holds both: -its own registration used the admin token once at startup, and every -`plugin.llm.ask(...)` afterward uses a model token from its own `connect()`. - -## Verified live — what each e2e script actually exercises +Why this exists: a plugin that checks quota for provider A and then calls +`.ask(model="x")` is only *guessing* A will be used — tb might pick B +instead. `pin_provider=` turns the guess into a guarantee. -Both are fixed, repeatable, real-`tb`-binary scripts — no mocks, no network, -no API keys (vmodel providers only) — with hard pass/fail assertions. +## `router_plugin.py` in one picture ``` -sdk/python/examples/e2e_run.sh sdk/python/examples/e2e_run_pin.sh -──────────────────────────────────────── ──────────────────────────────────────── -plugin registration MODE 1: unpinned call → tier0 selected - (idempotent upsert-by-name) MODE 2: pinned call → tier1 selected -round-trip: client → tb → plugin → (overrides tier order) - plugin.use(...) → tb → another rule MODE 2 scoping: pin to an unrelated - → back through tb → client provider → rejected (400) -crash (SIGKILL) → circuit breaker SDK-level: Client.ask(pin_provider=) - (no fallback tier ⇒ plain error; round-trips the same way - add tier-1 to see failover instead) router_plugin.py run for real: -re-register → same provider, no duplicate resolves sonnet1/sonnet2 via - Client.rules, checks quota, forwards - with pin_provider — tb's own routing - log confirms a provider_pin-sourced - selection for that forwarded call + question in + │ + ▼ + for each candidate model: + keep it only if it maps to exactly ONE provider + (a model with several providers can't be quota-picked — tb decides that one) + │ + ▼ + check quota for each provider kept + │ + ▼ + pick the one with the most headroom + │ + ▼ + tb.ask(model=picked, pin_provider=picked's provider) ← the guarantee, above + │ + ▼ + answer out ``` -`e2e_run_pin.sh`'s first live run also caught a real bug (`Manager.GetQuota` -re-wrapping `ErrUsageNotFound`, 500ing `POST /provider-quota/batch` for any -provider with no quota data) — fixed alongside it; see `python-sdk.md` for -the full writeup. +Everything else — `critic_plugin.py` (ask a different model to review), +`fusion_plugin.py` (ask several, then a judge), `rag_plugin.py` (ask one, +with retrieved context) — is the same "call back into tb" from *A request, +start to finish* above, just with different logic in the handler. From f85e361c38a3a1889eb9f3a5b81cb7109d6148c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 02:51:34 +0000 Subject: [PATCH 26/28] docs(sdk): record why pin_provider stays two modes, not three MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses a real gap the pencil graph exposed: pin_provider only lets a caller choose among a rule's ALREADY-CONFIGURED services — there was no answer for "I haven't configured a rule for this model at all, I just want to hit provider+model directly." tb already has the mechanics for that (X-Tingly-Probe-Service builds a synthetic rule and skips persisted-rule resolution), but exposing an authenticated version of it to the SDK was considered and rejected: it would create requests invisible to the tb UI's rule list, with nowhere to hang guard rails/quota config — the same reasoning already written down for why X-Tingly-Probe-Service itself stays internal-only. Resolution: "no rule yet" isn't a routing problem, it's a one-time setup step (POST /api/v1/rule with a single service — exactly what router_plugin.py's own sonnet1/sonnet2 candidates already are). Recorded in both python-sdk.md (full reasoning) and python-sdk.pencil.md (two-line pointer, keeping the simplified pencil graph simple). --- .design/python-sdk.md | 16 ++++++++++++++++ .design/python-sdk.pencil.md | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index 0815f4c92..a2f361b97 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -609,6 +609,22 @@ code paths that happen to usually agree. Mode 2 closes that: external quota check at all (`router_plugin.py` skips those, rather than guessing). +**Deliberately still two modes, not three.** The obvious next question: what +if there's no rule at all yet for the model you want — do you need a *third*, +rule-free "just connect me straight to provider+model" mode? tb already has +the mechanics for exactly that: `X-Tingly-Probe-Service` builds a synthetic +rule on the fly and skips persisted-rule resolution entirely +(`internal/server/protocol_handler.go`, `determineRuleWithScenario`). We +considered exposing an authenticated version of that to the SDK and rejected +it: it would mean requests that don't show up as a rule in the tb UI, with +nowhere to hang guard rails/quota config, re-litigating the exact reasoning +`python-sdk.md` already gives for why `X-Tingly-Probe-Service` stays +internal-only. "No rule yet" isn't a routing problem, it's a one-time setup +step — `POST /api/v1/rule` with a single service, same as `router_plugin.py`'s +own `sonnet1`/`sonnet2` candidates already do. Cheap, idempotent, and it keeps +every reachable provider visible as a rule, which is the whole point of tb +being "a hub of rules" rather than a raw provider proxy. + Verified live against the real `tb` binary (not just mocked) — a fixed, repeatable regression script, `sdk/python/examples/e2e_run_pin.sh` (three vmodel providers, no network/keys, `set -uo pipefail` + explicit pass/fail diff --git a/.design/python-sdk.pencil.md b/.design/python-sdk.pencil.md index cd206a631..c7352f22a 100644 --- a/.design/python-sdk.pencil.md +++ b/.design/python-sdk.pencil.md @@ -72,6 +72,11 @@ Why this exists: a plugin that checks quota for provider A and then calls `.ask(model="x")` is only *guessing* A will be used — tb might pick B instead. `pin_provider=` turns the guess into a guarantee. +Only two modes — no "skip the rule entirely" third one. No rule for a +provider yet? Create a one-service rule for it (cheap, one-time), don't +bypass rule resolution to reach it — same reason `X-Tingly-Probe-Service` +(tb's internal, unauthenticated bypass) never got exposed to the SDK. + ## `router_plugin.py` in one picture ``` From e681a0b3dd7c1ef19f22f098b6e58f3aa7fc96c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 13:02:27 +0000 Subject: [PATCH 27/28] revert: drop router_plugin.py / quota+rules views / pin_provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cut the deepest, most speculative branch of this session's work: it started as "an example to showcase the plugin capability" and grew into a new authenticated gateway header, a routing-pipeline refactor, two new SDK helper views, and a dedicated e2e script — all serving one demo plugin, not a stated product requirement. Keeping it around would have been exactly the kind of premature infrastructure that becomes a maintenance burden before it has a real consumer. Removed: - sdk/python/examples/router_plugin.py + its tests - sdk/python/tingly/helpers/quota.py (QuotaView) + tests — no remaining consumer once router_plugin.py is gone - sdk/python/tingly/helpers/rules.py (RulesView) + tests — same - Client.ask(pin_provider=...) and the X-Tingly-Pin-Provider header/ SourceProviderPin routing source (internal/server/routing/simple.go, result.go) — reverted to the pre-pin_provider SelectService, folding the applySelectionResult extraction back inline since it only existed to share code with the now-removed pin branch - sdk/python/examples/e2e_run_pin.sh - The "Client.quota" / "Two connection modes" sections in the design doc, the router_plugin.py bullet and its pencil-graph diagrams Kept: - critic_plugin.py / fusion_plugin.py — explicitly requested earlier, no backend changes, use only the already-core plugin.use(scenario).ask() - ai/quota/manager.go's ErrUsageNotFound-unwrapping fix + its test — a real, independent correctness bug (POST /provider-quota/batch 500ing for any provider with no quota data), unrelated to whether the SDK exposes quota - internal/server/module/providerquota/handler_test.go — that module had no tests before; the regression coverage stands on its own Verified: go build/test and the full Python suite pass (45 tests, down from 67), and examples/e2e_run.sh still passes end-to-end against a real tb binary. --- .../routing/simple_pin_provider_test.go | 118 ------------- sdk/python/examples/e2e_run_pin.sh | 152 ----------------- sdk/python/examples/router_plugin.py | 113 ------------- sdk/python/tests/test_quota.py | 116 ------------- sdk/python/tests/test_router_plugin.py | 151 ----------------- sdk/python/tests/test_rules.py | 79 --------- sdk/python/tingly/helpers/quota.py | 158 ------------------ sdk/python/tingly/helpers/rules.py | 97 ----------- 8 files changed, 984 deletions(-) delete mode 100644 internal/server/routing/simple_pin_provider_test.go delete mode 100755 sdk/python/examples/e2e_run_pin.sh delete mode 100644 sdk/python/examples/router_plugin.py delete mode 100644 sdk/python/tests/test_quota.py delete mode 100644 sdk/python/tests/test_router_plugin.py delete mode 100644 sdk/python/tests/test_rules.py delete mode 100644 sdk/python/tingly/helpers/quota.py delete mode 100644 sdk/python/tingly/helpers/rules.py diff --git a/internal/server/routing/simple_pin_provider_test.go b/internal/server/routing/simple_pin_provider_test.go deleted file mode 100644 index b65838830..000000000 --- a/internal/server/routing/simple_pin_provider_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package routing - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/tingly-dev/tingly-box/internal/loadbalance" - "github.com/tingly-dev/tingly-box/internal/typ" -) - -// TestSelectService_PinProvider verifies that X-Tingly-Pin-Provider picks the -// named service from the rule's OWN services, overriding what the load -// balancer would otherwise choose — same mechanics as the probe pin, but -// scoped to services the rule already has configured. -func TestSelectService_PinProvider(t *testing.T) { - svcA := testService("provider-a", "claude-sonnet", true) - svcB := testService("provider-b", "claude-sonnet", true) - cfg := &mockConfig{ - providers: map[string]*typ.Provider{ - "provider-a": testProvider("provider-a", "ProviderA", true), - "provider-b": testProvider("provider-b", "ProviderB", true), - }, - } - // Pipeline would normally return provider-b via load balancer. - lb := &mockLoadBalancer{service: svcB} - store := newMockAffinityStore() - sel := NewServiceSelector(cfg, store, lb) - simple := NewSimpleSelector(sel) - - rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA, svcB}) - c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "provider-a") - - provider, svc, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) - require.NoError(t, err) - - assert.Equal(t, "provider-a", provider.UUID) - assert.Equal(t, "provider-a", svc.Provider) -} - -// TestSelectService_PinProvider_RejectsProviderNotOnRule is the scoping -// guarantee that makes this header safe to expose to clients: it cannot -// reach a provider the rule wasn't already configured with. -func TestSelectService_PinProvider_RejectsProviderNotOnRule(t *testing.T) { - svcA := testService("provider-a", "claude-sonnet", true) - cfg := &mockConfig{ - providers: map[string]*typ.Provider{ - "provider-a": testProvider("provider-a", "ProviderA", true), - "unrelated-provider": testProvider("unrelated-provider", "Unrelated", true), - }, - } - simple := newSimpleSelector(cfg) - rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA}) - c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "unrelated-provider") - - _, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "not an active service") -} - -// TestSelectService_PinProvider_RejectsInactiveService confirms a service -// present on the rule but not active cannot be pinned to either. -func TestSelectService_PinProvider_RejectsInactiveService(t *testing.T) { - svcA := testService("provider-a", "claude-sonnet", false) // inactive - cfg := &mockConfig{ - providers: map[string]*typ.Provider{ - "provider-a": testProvider("provider-a", "ProviderA", true), - }, - } - simple := newSimpleSelector(cfg) - rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA}) - c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "provider-a") - - _, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "not an active service") -} - -// TestSelectService_PinProvider_DisabledProvider errors when the pinned -// provider is itself disabled, even though it's a configured service. -func TestSelectService_PinProvider_DisabledProvider(t *testing.T) { - svcA := testService("provider-a", "claude-sonnet", true) - cfg := &mockConfig{ - providers: map[string]*typ.Provider{ - "provider-a": testProvider("provider-a", "ProviderA", false), // disabled - }, - } - simple := newSimpleSelector(cfg) - rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svcA}) - c := ginCtxWithHeader(t, "X-Tingly-Pin-Provider", "provider-a") - - _, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "disabled") -} - -// TestSelectService_NoPinHeader_FallsThrough confirms that without the pin -// header the normal pipeline still runs unaffected. -func TestSelectService_NoPinHeader_FallsThrough(t *testing.T) { - svc := testService("provider-a", "claude-sonnet", true) - cfg := &mockConfig{ - providers: map[string]*typ.Provider{ - "provider-a": testProvider("provider-a", "ProviderA", true), - }, - } - lb := &mockLoadBalancer{service: svc} - store := newMockAffinityStore() - sel := NewServiceSelector(cfg, store, lb) - simple := NewSimpleSelector(sel) - - rule := testRule("rule-1", "claude-sonnet", []*loadbalance.Service{svc}) - c := ginCtxWithHeader(t, "", "") - - provider, _, err := simple.SelectService(c, typ.ScenarioAnthropic, rule, nil) - require.NoError(t, err) - assert.Equal(t, "provider-a", provider.UUID) -} diff --git a/sdk/python/examples/e2e_run_pin.sh b/sdk/python/examples/e2e_run_pin.sh deleted file mode 100755 index 5924cddd2..000000000 --- a/sdk/python/examples/e2e_run_pin.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# End-to-end test for the two tb connection modes and router_plugin.py's use -# of both, using NO network / API keys (vmodel providers only): -# -# mode 1: scenario + rule(model) — tb picks the service -# (affinity/smart-routing/load-balancer; tier order here) -# mode 2: scenario + rule(model) + pin_provider — caller picks, but tb -# only allows a provider already on that rule's own services -# -# router_plugin.py is a real, unmodified consumer of both: it resolves -# candidates via Client.rules (mode 1's information), then forwards with -# pin_provider (mode 2) so the provider it quota-checked is the one that -# actually serves the request. vmodel providers have no fetchable quota, so -# the pick is a deterministic tie-break here, not a real quota decision — -# this script proves the WIRING (rule resolution -> pin -> tb enforcement), -# not "quota routing found a numerically better answer" (that needs a real -# provider account, out of scope for a no-network e2e test). -# -# Prereqs: -# go build -o /tmp/tb_e2e ./cli/tingly-box -# pip install -e . # from sdk/python (needs `tingly` importable) -# Run: bash sdk/python/examples/e2e_run_pin.sh -set -uo pipefail - -TB=${TB_BIN:-/tmp/tb_e2e} -CFG=$(mktemp -d) -PORT=18903 -BASE="http://127.0.0.1:$PORT" -SDK=/home/user/tingly-box/sdk/python -TB_LOG=/tmp/tb_pin_e2e.log -export PYTHONPATH=$SDK - -FAILED=0 -pass() { echo " PASS: $1"; } -fail() { echo " FAIL: $1"; FAILED=1; } - -cleanup() { - [[ -n "${PLUG_PID:-}" ]] && kill "$PLUG_PID" 2>/dev/null - [[ -n "${TB_PID:-}" ]] && kill "$TB_PID" 2>/dev/null -} -trap cleanup EXIT - -echo "== 1. start tb (config-dir=$CFG, port=$PORT) ==" -"$TB" --config-dir "$CFG" start --port "$PORT" --ui --browser=false >"$TB_LOG" 2>&1 & -TB_PID=$! -for i in $(seq 1 60); do - curl -sf "$BASE/api/v1/info/health" >/dev/null 2>&1 && break - sleep 0.5 -done -curl -sf "$BASE/api/v1/info/health" >/dev/null || { echo "tb did not start"; tail -30 "$TB_LOG"; exit 1; } -echo " tb healthy at $BASE" - -CFGFILE=$(find "$CFG" -name 'config.json' | head -1) -UTOK=$(python3 -c "import json;d=json.load(open('$CFGFILE'));print(d.get('user_token') or d.get('UserToken',''))") -MTOK=$(python3 -c "import json;d=json.load(open('$CFGFILE'));print(d.get('model_token') or d.get('ModelToken',''))") -UADMIN=(-H "Authorization: Bearer $UTOK" -H "Content-Type: application/json") -UMODEL=(-H "Authorization: Bearer $MTOK" -H "Content-Type: application/json") - -echo "== 2. create three vmodel providers (A, B, C — no network) ==" -mk_provider() { - curl -s "${UADMIN[@]}" -X POST "$BASE/api/v2/providers" -d "{ - \"name\":\"$1\",\"api_base\":\"vmodel://local\",\"api_style\":\"openai\", - \"auth_type\":\"vmodel\",\"no_key_required\":true,\"enabled\":true}" \ - | python3 -c "import sys,json;d=json.load(sys.stdin);print(d.get('data',{}).get('uuid') or d.get('uuid',''))" -} -PA=$(mk_provider vmodel-a) -PB=$(mk_provider vmodel-b) -PC=$(mk_provider vmodel-c) -echo " A=$PA B=$PB C=$PC" - -echo "== 3. rule 'tiered-model': A@tier0, B@tier1 (mode 1 target) ==" -echo " (request_model is the client-facing name; each service's own 'model'" -echo " must be 'echo-model' — the only mock ID the no-network vmodel backend knows)" -curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ - \"scenario\":\"experiment\",\"request_model\":\"tiered-model\",\"active\":true, - \"lb_tactic\":{\"type\":\"tier\",\"params\":{}}, - \"services\":[{\"provider\":\"$PA\",\"model\":\"echo-model\",\"weight\":1,\"active\":true,\"tier\":0}, - {\"provider\":\"$PB\",\"model\":\"echo-model\",\"weight\":1,\"active\":true,\"tier\":1}]}" \ - | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'))" - -echo "== 4. rules 'sonnet1'->A and 'sonnet2'->B, single service each ==" -echo " (router_plugin.py's default CANDIDATE_MODELS — unmodified file, real names)" -mk_pinned_rule() { - curl -s "${UADMIN[@]}" -X POST "$BASE/api/v1/rule" -d "{ - \"scenario\":\"experiment\",\"request_model\":\"$1\",\"active\":true, - \"lb_tactic\":{\"type\":\"random\",\"params\":{}}, - \"services\":[{\"provider\":\"$2\",\"model\":\"echo-model\",\"weight\":1,\"active\":true}]}" \ - | python3 -c "import sys,json;d=json.load(sys.stdin);print(' rule created:', d.get('success'))" -} -mk_pinned_rule sonnet1 "$PA" -mk_pinned_rule sonnet2 "$PB" - -echo "== 5. MODE 1 (scenario+rule): unpinned call -> tb picks tier0 = A ==" -SEL=$(curl -s "${UMODEL[@]}" -H "X-Tingly-Debug-Routing: 1" -D - -o /dev/null \ - -X POST "$BASE/tingly/experiment/v1/chat/completions" \ - -d '{"model":"tiered-model","messages":[{"role":"user","content":"hi"}]}' \ - | grep -i "x-tingly-selected-provider-uuid" | tr -d '\r' | awk '{print $2}') -[[ "$SEL" == "$PA" ]] && pass "unpinned call selected tier0 (A=$PA)" || fail "unpinned call selected '$SEL', expected A=$PA" - -echo "== 6. MODE 2 (scenario+rule+pin): pin to B overrides tier order ==" -SEL=$(curl -s "${UMODEL[@]}" -H "X-Tingly-Debug-Routing: 1" -H "X-Tingly-Pin-Provider: $PB" -D - -o /dev/null \ - -X POST "$BASE/tingly/experiment/v1/chat/completions" \ - -d '{"model":"tiered-model","messages":[{"role":"user","content":"hi"}]}' \ - | grep -i "x-tingly-selected-provider-uuid" | tr -d '\r' | awk '{print $2}') -[[ "$SEL" == "$PB" ]] && pass "pinned call selected B ($PB) despite tier0=A" || fail "pinned call selected '$SEL', expected B=$PB" - -echo "== 7. MODE 2 scoping: pin to C (not on this rule) is rejected ==" -ERR=$(curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -H "X-Tingly-Pin-Provider: $PC" -d '{ - "model":"tiered-model","messages":[{"role":"user","content":"hi"}]}') -echo "$ERR" | grep -q "not an active service" && pass "pin to unrelated provider C rejected" || fail "expected rejection, got: $ERR" - -echo "== 8. SDK-level: Client.ask(pin_provider=) round-trips through the real gateway ==" -python3 - "$BASE" "$UTOK" "$PB" <<'PY' && pass "Client.ask(pin_provider=) completed" || { echo " FAIL: SDK pin_provider call raised"; FAILED=1; } -import sys -import tingly -base, admin_token, want_provider = sys.argv[1], sys.argv[2], sys.argv[3] -tb = tingly.connect(base_url=base, token=admin_token, scenario="experiment") -text = tb.ask("hi", model="tiered-model", pin_provider=want_provider) -assert isinstance(text, str) and text, f"expected non-empty text, got {text!r}" -PY - -echo "== 9. router_plugin.py: real run — resolves sonnet1/sonnet2 via Client.rules," -echo " picks by quota (tied here, no live quota source), forwards with pin_provider ==" -TINGLY_BOX_URL="$BASE" TINGLY_BOX_TOKEN="$UTOK" \ - python3 "$SDK/examples/router_plugin.py" >/tmp/router_e2e.log 2>&1 & -PLUG_PID=$! -for i in $(seq 1 40); do - curl -sf "http://127.0.0.1:8768/health" >/dev/null 2>&1 && break - sleep 0.3 -done -curl -sf "http://127.0.0.1:8768/health" >/dev/null || { fail "router plugin did not start"; cat /tmp/router_e2e.log; } - -for i in $(seq 1 20); do - curl -s "${UADMIN[@]}" "$BASE/api/v2/plugins" | grep -q '"router"' && break - sleep 0.3 -done - -RESP=$(curl -s "${UMODEL[@]}" -X POST "$BASE/tingly/experiment/v1/chat/completions" -d '{ - "model":"plugin/router","messages":[{"role":"user","content":"what is 2+2?"}]}') -echo "$RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); print(' response:', d.get('choices',[{}])[0].get('message',{}).get('content', d))" \ - && pass "model=plugin/router answered" || fail "router call failed: $RESP" - -echo "== 10. confirm the router's FORWARDED call actually used a provider pin (tb log) ==" -grep -q "source=provider_pin" "$TB_LOG" \ - && pass "tb log shows a provider_pin-sourced selection (router's forwarded call)" \ - || fail "no provider_pin selection found in $TB_LOG" - -kill -KILL "$PLUG_PID" 2>/dev/null -PLUG_PID="" - -echo "== done: $([[ $FAILED -eq 0 ]] && echo "ALL PASSED" || echo "SOME FAILED — see above") ==" -exit $FAILED diff --git a/sdk/python/examples/router_plugin.py b/sdk/python/examples/router_plugin.py deleted file mode 100644 index 5a9b0f925..000000000 --- a/sdk/python/examples/router_plugin.py +++ /dev/null @@ -1,113 +0,0 @@ -"""A "router" plugin: quota-aware dispatch — a different shape from -rag/critic/fusion. Those all *generate* an answer themselves (one or more -calls back into tb feed a response the plugin composes). A router generates -nothing; its only job is to DECIDE which one candidate model actually serves -the request, then forward to just that one — and to GUARANTEE the provider -it checked quota for is the provider that actually serves it. - -That guarantee is why this isn't just "pick a model and call .ask(model=)": -a model name resolves to a *rule*, and a rule can have more than one active -service (tiers, load-balanced) — tb, not this plugin, decides which one of -those actually runs. Checking quota for one provider and then calling -`.ask(model=X)` would silently mean nothing if tb's own load balancer picks -a different service within that rule. So each candidate here must resolve -(via `Client.rules`) to a rule with exactly ONE active service — a model -name dedicated to one specific provider — and the forwarded call passes -`pin_provider=` (`X-Tingly-Pin-Provider`, see .design/python-sdk.md) to force -that exact provider, closing the loop between "what was checked" and "what -was used". - -Same idea as LiteLLM Router's `usage-based-routing` strategy — route to -whichever deployment has the most remaining rate-limit/quota headroom right -now — implemented as a plugin instead of gateway config. - -Run it (serves on :8768 AND registers with tb on startup): - - pip install -e . # from sdk/python - python examples/router_plugin.py - -Then from any tb client: model="plugin/router", the message is the question. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import List - -from tingly import ChatRequest, Plugin - -# Each entry must be a model name, in ROUTER_SCENARIO, whose rule you've -# configured to point at exactly ONE provider — e.g. an "anthropic" scenario -# with a "sonnet1" rule bound only to provider A and a "sonnet2" rule bound -# only to provider B. That 1:1 binding is what makes a quota-based pick mean -# something (see the module docstring); a rule with more than one active -# service is skipped as a candidate, not guessed at. -ROUTER_SCENARIO = "experiment" -CANDIDATE_MODELS = ["sonnet1", "sonnet2"] - -plugin = Plugin( - name="router", - scenario=ROUTER_SCENARIO, # bind a rule under this scenario on register - description="Quota-aware dispatch — forwards to whichever candidate has the most headroom", -) - - -@dataclass -class ResolvedCandidate: - model: str - provider_uuid: str # the ONE provider this model's rule is pinned to - - -@plugin.chat -def handle(req: ChatRequest) -> str: - question = req.last_user_text() - chosen = _pick_candidate() - # pin_provider is what makes this a real decision rather than a guess: - # the provider that was quota-checked is GUARANTEED to be the one that - # serves this request. - return plugin.use(ROUTER_SCENARIO).ask( - question, model=chosen.model, pin_provider=chosen.provider_uuid - ) - - -def _resolve_candidates() -> List[ResolvedCandidate]: - """Resolve each candidate model to its rule's single pinned provider. - Skips (rather than guesses at) any candidate whose rule has zero or more - than one active service — quota can't mean anything for a model tb - itself load-balances across multiple providers.""" - rules = plugin.llm.rules - resolved = [] - for model in CANDIDATE_MODELS: - rule = rules.for_model(ROUTER_SCENARIO, model) - if rule is None: - continue - services = rule.active_services - if len(services) != 1: - continue # not a pinned single-provider rule — not routable by quota - resolved.append(ResolvedCandidate(model=model, provider_uuid=services[0].provider)) - return resolved - - -def _pick_candidate() -> ResolvedCandidate: - """Cached quota (tb refreshes lazily, ~20 min TTL) is enough for most - routing decisions and costs nothing extra per request. Call - `plugin.llm.quota.refresh(uuid)` first, for a specific candidate, if a - request genuinely needs a number fresher than that — LiteLLM's own - usage-based-routing docs warn that a live check on every single request - adds real latency, so that should be the exception, not the default.""" - candidates = _resolve_candidates() - if not candidates: - raise RuntimeError( - "no router candidate resolved to a single-provider rule — each " - "entry in CANDIDATE_MODELS must name a model whose rule has " - "exactly one active service (see the module docstring)" - ) - quotas = plugin.llm.quota.batch([c.provider_uuid for c in candidates]) - return max( - candidates, - key=lambda c: quotas[c.provider_uuid].headroom_percent if c.provider_uuid in quotas else 100.0, - ) - - -if __name__ == "__main__": - plugin.serve(port=8768) diff --git a/sdk/python/tests/test_quota.py b/sdk/python/tests/test_quota.py deleted file mode 100644 index 696b04abb..000000000 --- a/sdk/python/tests/test_quota.py +++ /dev/null @@ -1,116 +0,0 @@ -"""QuotaView tests (gateway mocked with respx) — pin the exact response -shapes tb's provider-quota endpoints return (list/refresh wrap {meta,data}; -get/refresh-one are bare ProviderUsage; batch wraps {data: {uuid: usage}}), -plus the headroom_percent / remaining_percent heuristics used for routing. -""" - -import httpx -import respx - -from tingly.helpers.quota import ProviderQuota, QuotaView, UsageWindow - -BASE = "http://tb.test:12580" - - -def _view() -> QuotaView: - return QuotaView(BASE, "admin", 5.0) - - -def _window(**overrides): - base = {"key": "session", "type": "session", "used": 10, "limit": 100, "used_percent": 10} - base.update(overrides) - return base - - -@respx.mock -def test_list_unwraps_data_array(): - respx.get(f"{BASE}/api/v1/provider-quota").mock( - return_value=httpx.Response(200, json={ - "meta": {"total": 1, "updated_at": "2026-01-01T00:00:00Z"}, - "data": [{ - "provider_uuid": "p1", "provider_name": "Anthropic", "provider_type": "anthropic", - "windows": [_window()], - }], - }) - ) - result = _view().list() - assert len(result) == 1 - assert result[0].provider_uuid == "p1" - assert result[0].windows[0].used_percent == 10 - - -@respx.mock -def test_get_is_bare_provider_usage_no_envelope(): - respx.get(f"{BASE}/api/v1/provider-quota/p1").mock( - return_value=httpx.Response(200, json={ - "provider_uuid": "p1", "provider_name": "Anthropic", "provider_type": "anthropic", - "windows": [_window(used_percent=42)], - }) - ) - result = _view().get("p1") - assert result.provider_uuid == "p1" - assert result.windows[0].used_percent == 42 - - -@respx.mock -def test_batch_unwraps_uuid_keyed_map(): - respx.post(f"{BASE}/api/v1/provider-quota/batch").mock( - return_value=httpx.Response(200, json={ - "data": { - "p1": {"provider_uuid": "p1", "provider_name": "A", "provider_type": "anthropic", "windows": []}, - "p2": {"provider_uuid": "p2", "provider_name": "B", "provider_type": "openai", "windows": []}, - } - }) - ) - result = _view().batch(["p1", "p2"]) - assert set(result) == {"p1", "p2"} - assert result["p2"].provider_name == "B" - - -@respx.mock -def test_refresh_one_returns_bare_provider_usage(): - route = respx.post(f"{BASE}/api/v1/provider-quota/p1/refresh").mock( - return_value=httpx.Response(200, json={ - "provider_uuid": "p1", "provider_name": "A", "provider_type": "anthropic", "windows": [], - }) - ) - result = _view().refresh("p1") - assert route.called - assert result.provider_uuid == "p1" - - -@respx.mock -def test_refresh_all_hits_refresh_endpoint_and_returns_none(): - route = respx.post(f"{BASE}/api/v1/provider-quota/refresh").mock( - return_value=httpx.Response(200, json={"meta": {"total": 0, "updated_at": "x"}, "data": []}) - ) - assert _view().refresh() is None - assert route.called - - -# -- headroom heuristics -------------------------------------------------- - -def test_window_remaining_percent(): - assert UsageWindow._from_json(_window(used_percent=30)).remaining_percent == 70.0 - - -def test_window_remaining_percent_none_when_unlimited(): - # tb's convention: limit<=0 means "unlimited" — there's no percent of an - # unbounded quantity, so this must not be treated as "0% remaining". - assert UsageWindow._from_json(_window(limit=0)).remaining_percent is None - - -def test_provider_headroom_is_the_most_constrained_window(): - quota = ProviderQuota( - provider_uuid="p1", provider_name="A", provider_type="anthropic", - windows=[ - UsageWindow._from_json(_window(key="session", used_percent=10)), # 90% remaining - UsageWindow._from_json(_window(key="daily", used_percent=80)), # 20% remaining - ], - ) - assert quota.headroom_percent == 20.0 - - -def test_provider_headroom_defaults_to_100_with_no_bounded_windows(): - quota = ProviderQuota(provider_uuid="p1", provider_name="A", provider_type="anthropic", windows=[]) - assert quota.headroom_percent == 100.0 diff --git a/sdk/python/tests/test_router_plugin.py b/sdk/python/tests/test_router_plugin.py deleted file mode 100644 index 16da16d83..000000000 --- a/sdk/python/tests/test_router_plugin.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Tests for the router showcase plugin (sdk/python/examples/router_plugin.py). - -Unlike critic/fusion, a router doesn't generate anything — it resolves each -candidate model to its rule's single provider, picks the one with the most -quota headroom, and forwards with pin_provider= to guarantee that provider -is the one that actually serves the request. These tests pin that decision -logic with plugin.use()/rules/quota monkeypatched, no real tb. -""" - -import importlib.util -import sys -from pathlib import Path - -import pytest - -from tingly.helpers.quota import ProviderQuota, UsageWindow -from tingly.helpers.rules import Rule -from tingly.plugin.types import ChatRequest - -EXAMPLES = Path(__file__).parent.parent / "examples" - - -def _load(name): - path = EXAMPLES / f"{name}.py" - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - sys.modules[name] = mod - spec.loader.exec_module(mod) - return mod - - -def _req(content): - return ChatRequest.from_openai_body({"model": "x", "messages": [{"role": "user", "content": content}]}) - - -def _rule(model, *providers_active): - """providers_active: e.g. [("p1", True)] for one active service, or - [("p1", True), ("p2", True)] for a multi-service (non-routable) rule.""" - return Rule._from_json({ - "uuid": f"rule-{model}", "scenario": "experiment", "request_model": model, - "services": [{"provider": p, "model": model, "active": active} for p, active in providers_active], - }) - - -def _quota(uuid, used_percent): - return ProviderQuota( - provider_uuid=uuid, provider_name=uuid, provider_type="anthropic", - windows=[UsageWindow(key="session", type="session", used=used_percent, limit=100, used_percent=used_percent)], - ) - - -class _FakeRules: - def __init__(self, rules_by_model): - self._rules_by_model = rules_by_model - - def for_model(self, scenario, model): - return self._rules_by_model.get(model) - - -class _FakeQuota: - def __init__(self, quotas): - self._quotas = quotas - - def batch(self, uuids): - return {u: self._quotas[u] for u in uuids if u in self._quotas} - - def refresh(self, provider_uuid=None): - raise AssertionError("refresh() should not be called by default routing") - - -class _FakeClient: - def __init__(self, reply=None, rules=None, quota=None): - self._reply = reply - self.rules = rules - self.quota = quota - self.calls = [] - - def ask(self, prompt, **kwargs): - self.calls.append((prompt, kwargs)) - return self._reply - - -def test_resolve_candidates_skips_multi_service_rules(monkeypatch): - router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1", "sonnet2", "sonnet3"]) - rules = _FakeRules({ - "sonnet1": _rule("sonnet1", ("p1", True)), # single active service — routable - "sonnet2": _rule("sonnet2", ("p2", True), ("p3", True)), # two active services — skip - # "sonnet3" absent entirely (rule doesn't exist) — skip - }) - monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules)) - - resolved = router._resolve_candidates() - - assert [c.model for c in resolved] == ["sonnet1"] - assert resolved[0].provider_uuid == "p1" - - -def test_resolve_candidates_skips_rule_with_no_active_services(monkeypatch): - router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1"]) - rules = _FakeRules({"sonnet1": _rule("sonnet1", ("p1", False))}) # only inactive service - monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules)) - - assert router._resolve_candidates() == [] - - -def test_pick_candidate_chooses_highest_headroom(monkeypatch): - router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1", "sonnet2"]) - rules = _FakeRules({ - "sonnet1": _rule("sonnet1", ("p1", True)), - "sonnet2": _rule("sonnet2", ("p2", True)), - }) - quotas = _FakeQuota({"p1": _quota("p1", used_percent=90), "p2": _quota("p2", used_percent=10)}) - monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules, quota=quotas)) - - chosen = router._pick_candidate() - - assert chosen.model == "sonnet2" # 90% headroom beats 10% - assert chosen.provider_uuid == "p2" - - -def test_pick_candidate_raises_when_no_routable_candidates(monkeypatch): - router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1"]) - rules = _FakeRules({}) # no rule resolves - monkeypatch.setattr(router.plugin, "use", lambda scenario: _FakeClient(rules=rules)) - - with pytest.raises(RuntimeError, match="no router candidate"): - router._pick_candidate() - - -def test_handle_forwards_with_pin_provider_for_the_chosen_candidate(monkeypatch): - router = _load("router_plugin") - monkeypatch.setattr(router, "CANDIDATE_MODELS", ["sonnet1", "sonnet2"]) - rules = _FakeRules({ - "sonnet1": _rule("sonnet1", ("p1", True)), - "sonnet2": _rule("sonnet2", ("p2", True)), - }) - quotas = _FakeQuota({"p1": _quota("p1", used_percent=90), "p2": _quota("p2", used_percent=10)}) - shared = _FakeClient(reply="the answer", rules=rules, quota=quotas) - monkeypatch.setattr(router.plugin, "use", lambda scenario: shared) - - result = router.handle(_req("what's 2+2?")) - - assert result == "the answer" - assert len(shared.calls) == 1 - prompt, kwargs = shared.calls[0] - assert kwargs["model"] == "sonnet2" - assert kwargs["pin_provider"] == "p2" diff --git a/sdk/python/tests/test_rules.py b/sdk/python/tests/test_rules.py deleted file mode 100644 index 4979a8191..000000000 --- a/sdk/python/tests/test_rules.py +++ /dev/null @@ -1,79 +0,0 @@ -"""RulesView tests (gateway mocked with respx).""" - -import httpx -import respx - -from tingly.helpers.rules import Rule, RulesView - -BASE = "http://tb.test:12580" - - -def _view() -> RulesView: - return RulesView(BASE, "admin", 5.0) - - -@respx.mock -def test_list_requires_scenario_query_param(): - route = respx.get(f"{BASE}/api/v1/rules", params={"scenario": "experiment"}).mock( - return_value=httpx.Response(200, json={"success": True, "data": []}) - ) - _view().list("experiment") - assert route.called - - -@respx.mock -def test_list_parses_services(): - respx.get(f"{BASE}/api/v1/rules").mock( - return_value=httpx.Response(200, json={ - "success": True, - "data": [{ - "uuid": "r1", "scenario": "experiment", "request_model": "sonnet1", "active": True, - "services": [ - {"provider": "p1", "model": "claude-sonnet-4-6", "active": True, "weight": 1, "tier": 0}, - {"provider": "p2", "model": "claude-sonnet-4-6", "active": False, "weight": 1, "tier": 1}, - ], - }], - }) - ) - rules = _view().list("experiment") - assert len(rules) == 1 - assert rules[0].request_model == "sonnet1" - assert len(rules[0].services) == 2 - assert [s.provider for s in rules[0].active_services] == ["p1"] - - -@respx.mock -def test_for_model_finds_matching_rule(): - respx.get(f"{BASE}/api/v1/rules").mock( - return_value=httpx.Response(200, json={ - "success": True, - "data": [ - {"uuid": "r1", "scenario": "experiment", "request_model": "sonnet1", "services": []}, - {"uuid": "r2", "scenario": "experiment", "request_model": "sonnet2", "services": []}, - ], - }) - ) - rule = _view().for_model("experiment", "sonnet2") - assert rule is not None - assert rule.uuid == "r2" - - -@respx.mock -def test_for_model_returns_none_when_no_match(): - respx.get(f"{BASE}/api/v1/rules").mock( - return_value=httpx.Response(200, json={"success": True, "data": []}) - ) - assert _view().for_model("experiment", "nope") is None - - -def test_service_for_provider_ignores_inactive(): - rule = Rule._from_json({ - "uuid": "r1", "scenario": "experiment", "request_model": "sonnet1", - "services": [ - {"provider": "p1", "model": "m", "active": False}, - {"provider": "p2", "model": "m", "active": True}, - ], - }) - assert rule.service_for_provider("p1") is None - assert rule.service_for_provider("p2") is not None - assert rule.service_for_provider("unknown") is None diff --git a/sdk/python/tingly/helpers/quota.py b/sdk/python/tingly/helpers/quota.py deleted file mode 100644 index 13784c7d4..000000000 --- a/sdk/python/tingly/helpers/quota.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Quota view — per-provider usage/limit windows, and a live refresh. - -tb tracks quota per provider as one or more named **windows** (session / -daily / weekly / monthly / balance / model / ...), each with its own -``used`` / ``limit`` / ``used_percent`` — a provider is rarely a single -number. ``list()`` / ``get()`` / ``batch()`` read tb's cache (tb itself -lazily re-fetches from the upstream account when a provider's cached snapshot -has expired, ~20 min TTL by default); ``refresh()`` forces a **live** -re-fetch right now. Prefer the cache for routing decisions made on every -request — LiteLLM's own usage-based-routing docs warn that hitting a live -usage source on every single request adds real per-request latency; reserve -``refresh()`` for when you specifically need a number fresher than the cache. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -import httpx - - -@dataclass -class UsageWindow: - """One quota window (e.g. "session", "daily", "monthly TPM").""" - - key: str - type: str - used: float - limit: float - used_percent: float = 0.0 - unit: str = "" - label: str = "" - resets_at: Optional[str] = None - allowed: Optional[bool] = None - limit_reached: Optional[bool] = None - - @property - def remaining_percent(self) -> Optional[float]: - """0-100 remaining, or ``None`` when ``limit<=0`` (tb's convention - for "unlimited" — there is no percentage to be remaining *of*).""" - if self.limit <= 0: - return None - return max(0.0, 100.0 - self.used_percent) - - @classmethod - def _from_json(cls, d: Dict[str, Any]) -> "UsageWindow": - return cls( - key=d.get("key", ""), - type=d.get("type", ""), - used=d.get("used", 0) or 0, - limit=d.get("limit", 0) or 0, - used_percent=d.get("used_percent", 0) or 0, - unit=d.get("unit", ""), - label=d.get("label", ""), - resets_at=d.get("resets_at"), - allowed=d.get("allowed"), - limit_reached=d.get("limit_reached"), - ) - - -@dataclass -class ProviderQuota: - """A provider's quota snapshot — as cached by tb, or freshly fetched.""" - - provider_uuid: str - provider_name: str - provider_type: str - windows: List[UsageWindow] = field(default_factory=list) - last_error: str = "" - raw: Dict[str, Any] = field(default_factory=dict) - - @property - def headroom_percent(self) -> float: - """The most CONSTRAINED window's remaining percent — i.e. whichever - limit this provider will hit first. ``100.0`` when no window carries - a real limit (nothing to be constrained by). - - This is deliberately a single naive number for making a routing pick - between candidates at a glance (see ``examples/router_plugin.py``); - session/daily/cost windows are not fungible, so anything more - precise than "which one is worse off right now" should read - ``.windows`` directly instead of trusting this alone. - """ - percents = [w.remaining_percent for w in self.windows if w.remaining_percent is not None] - return min(percents) if percents else 100.0 - - @classmethod - def _from_json(cls, d: Dict[str, Any]) -> "ProviderQuota": - return cls( - provider_uuid=d.get("provider_uuid", ""), - provider_name=d.get("provider_name", ""), - provider_type=d.get("provider_type", ""), - windows=[UsageWindow._from_json(w) for w in d.get("windows") or []], - last_error=d.get("last_error", ""), - raw=d, - ) - - -class QuotaView: - def __init__(self, gateway_url: str, admin_token: str, timeout: float): - self._gateway_url = gateway_url.rstrip("/") - self._admin_token = admin_token - self._timeout = timeout - - def _headers(self) -> Dict[str, str]: - return {"Authorization": f"Bearer {self._admin_token}"} - - def list(self) -> List[ProviderQuota]: - """Every provider's cached quota.""" - resp = httpx.get( - f"{self._gateway_url}/api/v1/provider-quota", - headers=self._headers(), timeout=self._timeout, - ) - resp.raise_for_status() - data = resp.json().get("data") or [] - return [ProviderQuota._from_json(d) for d in data] - - def get(self, provider_uuid: str) -> ProviderQuota: - """One provider's cached quota (tb transparently refetches if the - cached snapshot has expired).""" - resp = httpx.get( - f"{self._gateway_url}/api/v1/provider-quota/{provider_uuid}", - headers=self._headers(), timeout=self._timeout, - ) - resp.raise_for_status() - return ProviderQuota._from_json(resp.json()) - - def batch(self, provider_uuids: List[str]) -> Dict[str, ProviderQuota]: - """Cached quota for a specific set of providers in one round trip — - the shape a router picking between N candidates actually wants.""" - resp = httpx.post( - f"{self._gateway_url}/api/v1/provider-quota/batch", - headers=self._headers(), json={"provider_uuids": provider_uuids}, - timeout=self._timeout, - ) - resp.raise_for_status() - data = resp.json().get("data") or {} - return {uuid: ProviderQuota._from_json(d) for uuid, d in data.items()} - - def refresh(self, provider_uuid: Optional[str] = None) -> Optional[ProviderQuota]: - """Force a LIVE re-fetch from the upstream account, bypassing tb's - cache entirely. Omit ``provider_uuid`` to refresh every enabled - provider (returns ``None`` in that case — use :meth:`list` to read - the results back).""" - if provider_uuid: - resp = httpx.post( - f"{self._gateway_url}/api/v1/provider-quota/{provider_uuid}/refresh", - headers=self._headers(), timeout=self._timeout, - ) - resp.raise_for_status() - return ProviderQuota._from_json(resp.json()) - resp = httpx.post( - f"{self._gateway_url}/api/v1/provider-quota/refresh", - headers=self._headers(), timeout=self._timeout, - ) - resp.raise_for_status() - return None diff --git a/sdk/python/tingly/helpers/rules.py b/sdk/python/tingly/helpers/rules.py deleted file mode 100644 index b2ecbac76..000000000 --- a/sdk/python/tingly/helpers/rules.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Rules view — read a scenario's rules and the services each one has -configured, so a caller can discover which provider(s) back a given model. - -Existing views (`usage`, `guardrails`, `quota`) all read *what happened* or -*what's available*; this one reads *how a model resolves* — the missing -piece for anything that wants to act on a specific one of a rule's services -(e.g. `Client.ask(..., pin_provider=...)`), rather than letting tb pick. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -import httpx - - -@dataclass -class RuleService: - """One of a rule's configured services — a (provider, model) binding tb - can route to, at the given tier.""" - - provider: str # provider UUID - model: str - active: bool = True - weight: int = 1 - tier: int = 0 - - -@dataclass -class Rule: - uuid: str - scenario: str - request_model: str - active: bool = True - services: List[RuleService] = field(default_factory=list) - - @property - def active_services(self) -> List[RuleService]: - return [s for s in self.services if s.active] - - def service_for_provider(self, provider_uuid: str) -> Optional[RuleService]: - """The rule's own active service bound to this provider, if any — - the set of providers valid to pass as `pin_provider=` for this rule's - model. `None` means this provider isn't one of this rule's services; - tb will reject a pin to it (`X-Tingly-Pin-Provider` is scoped to the - resolved rule's own services, not any provider on the box).""" - for svc in self.active_services: - if svc.provider == provider_uuid: - return svc - return None - - @classmethod - def _from_json(cls, d: Dict[str, Any]) -> "Rule": - return cls( - uuid=d.get("uuid", ""), - scenario=d.get("scenario", ""), - request_model=d.get("request_model", ""), - active=d.get("active", True), - services=[ - RuleService( - provider=s.get("provider", ""), - model=s.get("model", ""), - active=s.get("active", True), - weight=s.get("weight", 1), - tier=s.get("tier", 0), - ) - for s in d.get("services") or [] - ], - ) - - -class RulesView: - def __init__(self, gateway_url: str, admin_token: str, timeout: float): - self._gateway_url = gateway_url.rstrip("/") - self._admin_token = admin_token - self._timeout = timeout - - def list(self, scenario: str) -> List[Rule]: - """All rules configured under a scenario (required — tb 400s without it).""" - resp = httpx.get( - f"{self._gateway_url}/api/v1/rules", - params={"scenario": scenario}, - headers={"Authorization": f"Bearer {self._admin_token}"}, - timeout=self._timeout, - ) - resp.raise_for_status() - data = resp.json().get("data") or [] - return [Rule._from_json(d) for d in data] - - def for_model(self, scenario: str, model: str) -> Optional[Rule]: - """The rule whose `request_model` matches, if any — the common case - (a router deciding which provider backs *this* model).""" - for rule in self.list(scenario): - if rule.request_model == model: - return rule - return None From c05a4b44dcc5c8c514728b7db97c96fe8b514412 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 13:03:57 +0000 Subject: [PATCH 28/28] revert: finish dropping router_plugin.py / quota+rules / pin_provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous "revert" commit only picked up the deletions (git rm'd files) — a bad pathspec in the accompanying `git add` silently aborted before it staged the modified files, so client.py, simple.go, result.go, and the docs never actually lost their pin_provider/quota/rules content. This is that missing half: reverts Client.ask()'s pin_provider param, Client.quota/ Client.rules properties, the X-Tingly-Pin-Provider header handling in SimpleSelector.SelectService and its SourceProviderPin constant, the now-stale pin_provider tests in test_client_offline.py, and the design doc / README / pencil-graph sections describing all of it. Verified: go build + internal/server/routing tests, and the full Python suite (45 tests) all still pass with this actually applied. --- .design/python-sdk.md | 170 +++--------------------- .design/python-sdk.pencil.md | 69 +--------- internal/server/routing/result.go | 4 - internal/server/routing/simple.go | 47 +------ sdk/python/README.md | 29 +--- sdk/python/tests/test_client_offline.py | 66 --------- sdk/python/tingly/client.py | 22 --- 7 files changed, 28 insertions(+), 379 deletions(-) diff --git a/.design/python-sdk.md b/.design/python-sdk.md index a2f361b97..bb5e12f34 100644 --- a/.design/python-sdk.md +++ b/.design/python-sdk.md @@ -3,9 +3,8 @@ > Audience: tingly-box contributors touching the SDK seam (`sdk/python/`), the > `/api/v1/sdk/session` endpoint, or the `experiment` scenario. -Diagram: `.design/python-sdk.pencil.md` — four simple pencil graphs: the one -idea, a request start to finish, the two ways to pick a provider, and -`router_plugin.py`'s decision flow. +Diagram: `.design/python-sdk.pencil.md` — two simple pencil graphs: the one +idea, and a request start to finish. ## Why @@ -194,7 +193,7 @@ sdk/python/ config.py # (base_url, admin_token) resolution precedence scenarios.py # scenario + transport constants transports/ # build openai.OpenAI / anthropic.Anthropic bound to tb - helpers/ # usage + guardrails + quota + rules views + helpers/ # usage + guardrails views plugin/ # Layer 2: be an AI server tb routes to core.py # Plugin class (@plugin.chat, .llm, .serve, api_style) server.py # stdlib HTTP server: /v1/messages (primary) + /v1/chat/completions (secondary), + SSE @@ -219,8 +218,6 @@ connect(scenario="experiment") .ask() → Anthropic first when the scenario supports both, else OpenAI .usage → GET /api/v1/requests (admin token) .guardrails → GET /api/v1/guardrails/config (admin token) - .quota → GET/POST /api/v1/provider-quota[...] (admin token) - .rules → GET /api/v1/rules?scenario= (admin token) ``` ## How it works (pencil) @@ -530,131 +527,9 @@ their manifest (reuse `agentboot/process`), a `/plugins//*` reverse-proxy mount, and the install/enable/logs/disable lifecycle UI. The Python side and the provider wiring are complete; those are the remaining backend pieces. -### `Client.quota` — provider usage/limit windows, and a live refresh - -Added for `router_plugin.py` below, but attached to `Client` like `.usage` / -`.guardrails` so any caller can use it. Wraps -`GET /api/v1/provider-quota[...]` (`internal/server/module/providerquota/`, -admin token, same `apiV1` auth-middleware group as usage/guardrails): - -| SDK call | endpoint | shape | -|---|---|---| -| `quota.list()` | `GET /provider-quota` | `{meta, data:[ProviderUsage]}` | -| `quota.get(uuid)` | `GET /provider-quota/:uuid` | bare `ProviderUsage` (no envelope) | -| `quota.batch(uuids)` | `POST /provider-quota/batch` | `{data: {uuid: ProviderUsage}}` | -| `quota.refresh(uuid?)` | `POST /provider-quota/:uuid?/refresh` | live re-fetch from the upstream account, bypassing tb's cache | - -These three response shapes are genuinely different (envelope vs. bare vs. -uuid-keyed map) — not a Python-side inconsistency, that's what the Go handler -(`internal/server/module/providerquota/handler.go:66-177`) actually returns -for each; `QuotaView._from_json`-style parsing per method is intentional, not -an oversight. `provider-quota` isn't in `openapi.json` (no swagger tags on -that module yet), so these shapes were pinned by reading the handler -directly, not generated — worth re-checking if that module ever gets -swagger-annotated. - -A provider's quota is **not one number** — `ProviderUsage.windows` is a list -(session/daily/weekly/monthly/balance/model/...), each with its own -`used`/`limit`/`used_percent` (`ai/quota/types.go`). `ProviderQuota.headroom_percent` -collapses that to the single most-constrained window's remaining percent — -a deliberately naive heuristic for "which candidate is worse off right now" -in a routing pick, not a replacement for reading `.windows` when the -distinction between e.g. a session limit and a monthly cost budget matters. -tb itself has **no built-in quota-aware routing** (`internal/smart_routing` -and `internal/loadbalance` have zero references to `ai/quota` as of this -writing) — a plugin picking by remaining quota is genuinely new behavior, -not a Python reimplementation of something the gateway already does. - -### Two connection modes: scenario+rule, and scenario+rule+pin - -Every call this SDK makes goes through `(scenario, model)` → tb resolves a -**rule** → the rule's `Services[]` (possibly several, tiered) → tb's own -affinity/smart-routing/load-balancer picks **which** service actually runs. -That's mode 1 — "let tb decide" — and it's what `.ask()` has always done. - -Building `router_plugin.py` (below) surfaced a real gap: a plugin that picks -a provider by quota and then calls `.ask(model=X)` has no guarantee that's -the provider tb's load balancer actually uses when the rule has more than -one active service — the "decision" and the execution are two unrelated -code paths that happen to usually agree. Mode 2 closes that: - -- **`X-Tingly-Pin-Provider: `** (`internal/server/routing/simple.go`, - `SimpleSelector.SelectService`) — forces the resolved rule to use that - exact service, skipping affinity/smart-routing/load-balancing. The check - that makes this safe to expose to ordinary clients: the provider **must** - already be one of the resolved rule's own active `Services[]`, or tb - rejects the request (400) — this cannot be used to reach an unrelated - provider elsewhere on the box. It also runs on the *same* authenticated - data-plane path as every other call (the model token already required to - reach `/tingly/:scenario/...`), unlike the older `X-Tingly-Probe-Service` - (`internal/server/routing/simple.go`, `.design/probe.md`), which bypasses - auth entirely by convention (*"any caller that can reach the TB HTTP port - can send it"*) and pins to **any** provider — that header is only ever - injected internally by tb's own probe/diagnostics tooling, deliberately - never exposed to SDK users. `X-Tingly-Pin-Provider` is the scoped, - authenticated version of the same underlying mechanic - (`SourceProviderPin` vs. `SourceProbePin` in `internal/server/routing/result.go`). -- SDK surface: `Client.ask(..., pin_provider=)` sets the header - (merges with any caller-supplied `extra_headers`); `tb.openai` / - `tb.anthropic` accept it directly too, since both vendor SDKs already - support `extra_headers=` on `.create()` — no SDK change was even required - for that path, `ask()`'s kwarg is purely for convenience. -- **`Client.rules`** (`tingly/helpers/rules.py`, wraps `GET /api/v1/rules?scenario=`, - admin token) is how a caller finds out what's *pinnable*: - `rules.for_model(scenario, model)` returns the resolved `Rule`, whose - `.active_services` are the only valid `pin_provider` values for that model. - A rule with more than one active service has more than one valid pin — use - quota (or whatever signal) to choose among them; a candidate whose rule - doesn't resolve to exactly one service isn't safely routable by an - external quota check at all (`router_plugin.py` skips those, rather than - guessing). - -**Deliberately still two modes, not three.** The obvious next question: what -if there's no rule at all yet for the model you want — do you need a *third*, -rule-free "just connect me straight to provider+model" mode? tb already has -the mechanics for exactly that: `X-Tingly-Probe-Service` builds a synthetic -rule on the fly and skips persisted-rule resolution entirely -(`internal/server/protocol_handler.go`, `determineRuleWithScenario`). We -considered exposing an authenticated version of that to the SDK and rejected -it: it would mean requests that don't show up as a rule in the tb UI, with -nowhere to hang guard rails/quota config, re-litigating the exact reasoning -`python-sdk.md` already gives for why `X-Tingly-Probe-Service` stays -internal-only. "No rule yet" isn't a routing problem, it's a one-time setup -step — `POST /api/v1/rule` with a single service, same as `router_plugin.py`'s -own `sonnet1`/`sonnet2` candidates already do. Cheap, idempotent, and it keeps -every reachable provider visible as a rule, which is the whole point of tb -being "a hub of rules" rather than a raw provider proxy. - -Verified live against the real `tb` binary (not just mocked) — a fixed, -repeatable regression script, `sdk/python/examples/e2e_run_pin.sh` (three -vmodel providers, no network/keys, `set -uo pipefail` + explicit pass/fail -assertions, non-zero exit on any failure): a rule with provider A at tier 0 -and B at tier 1 — an unpinned call selects A (normal tier order, confirmed -via `X-Tingly-Debug-Routing`); the same call with `X-Tingly-Pin-Provider: ` -selects B despite the tier order; a pin to a provider not on that rule is -rejected with 400; the same round-trip through `Client.ask(pin_provider=)`; -and `router_plugin.py` run for real end-to-end, resolving `sonnet1`/`sonnet2` -via `Client.rules`, and forwarding with a confirmed `provider_pin`-sourced -selection in tb's own routing log. - -**A real bug this surfaced**, fixed alongside it: `Manager.GetQuota` / -`GetQuotaNoCache` (`ai/quota/manager.go`) re-wrapped a not-found store lookup -into a *new* `fmt.Errorf(...)` instead of returning `quota.ErrUsageNotFound` -itself — silently breaking the `err == quota.ErrUsageNotFound` identity -check every caller (`internal/server/module/providerquota/handler.go`, both -`GetQuota` and `BatchGetQuota`) relies on to treat "no data yet" as a skip. -The practical effect: `POST /provider-quota/batch` 500'd the *entire* batch -the moment it included any provider with no quota fetcher (a vmodel/local -provider, exactly what a no-network test setup uses) instead of just -omitting that one provider from the result — `router_plugin.py`'s very first -live run hit this immediately. Fixed by returning the sentinel unwrapped; -covered by `ai/quota/manager_test.go::TestGetQuota_NotFoundIsUnwrapped` and -`internal/server/module/providerquota/handler_test.go` (new — this module -had no tests before). - ### Example plugins (`sdk/python/examples/`) -Four, each a different real-world shape of "plugin composes the box by +Three, each a different real-world shape of "plugin composes the box by calling back into other rules" — not toys picked at random, each maps onto a pattern already in wide use: @@ -685,29 +560,20 @@ pattern already in wide use: illustration of the architecture line at the top of this document — a plugin can freely originate calls against *any* number of other rules, not just one. -- **`router_plugin.py`** (`model="plugin/router"`) — quota-aware dispatch: a - different shape from the three above, which all *generate* an answer - themselves. A router generates nothing — for each candidate model it - resolves the rule via `Client.rules` (skipping any candidate whose rule - isn't pinned to exactly one active service — see "Two connection modes" - above), checks quota for that one provider, picks the candidate with the - most headroom, and forwards with `pin_provider=` so the provider that was - quota-checked is *guaranteed* to be the one that serves the request — one - hop total, by design, not N. Same idea as LiteLLM Router's - `usage-based-routing` strategy (route to whichever deployment has the most - remaining rate-limit capacity), implemented as a plugin instead of gateway - config — deliberately reads cached quota by default and only calls - `.quota.refresh()` when a caller opts in, since LiteLLM's own docs warn - that a live usage check on every request adds real per-request latency. - -Every example plugin has unit tests (`tests/test_example_plugins.py`, -`tests/test_router_plugin.py`, `tests/test_quota.py`, `tests/test_rules.py`) -that monkeypatch `plugin.use`/`Client.quota`/`Client.rules` to fakes and pin -the decision logic — JSON-verdict formatting and graceful degradation on -non-JSON (critic); judge-skipped-on-agreement vs. judge-called-on-disagreement -(fusion); multi-service rules skipped as non-routable, highest-headroom -candidate selection, and `pin_provider=` forwarding (router) — without needing a -live tb. + +Every example plugin has unit tests (`tests/test_example_plugins.py`) that +monkeypatch `plugin.use` to a fake client and pin the decision logic — +JSON-verdict formatting and graceful degradation on non-JSON (critic); +judge-skipped-on-agreement vs. judge-called-on-disagreement (fusion) — +without needing a live tb. + +Deliberately not built (yet): quota-aware / deterministic-dispatch routing +(picking a candidate rule by remaining quota, forcing a specific service +within a rule). It's a real, well-scoped follow-up — not out of scope +forever, just not needed for the current milestone above, and speculative +infrastructure built ahead of a real consumer tends to become exactly the +kind of thing that needs redesigning once an actual use shows up. Revisit +if/when a concrete need for it appears. ## Layer 3: can tb *use* a plugin as a model? (yes — as an upstream) diff --git a/.design/python-sdk.pencil.md b/.design/python-sdk.pencil.md index c7352f22a..19f856235 100644 --- a/.design/python-sdk.pencil.md +++ b/.design/python-sdk.pencil.md @@ -1,15 +1,13 @@ # Python SDK (`tingly`) — Pencil Graph -Visual companion to `python-sdk.md`. Four pictures, each answering one -question. For exact endpoints / field names / file:line references, that -doc is the source of truth — this page is just the shape of things. +Visual companion to `python-sdk.md`. Two pictures. For exact endpoints / +field names / file:line references, that doc is the source of truth — this +page is just the shape of things. Contents: - The one idea - A request, start to finish -- Two ways to pick a provider -- `router_plugin.py` in one picture ## The one idea @@ -47,60 +45,7 @@ rails / quota / logging, both directions. Steps 1 and 2 are all of Layer 1 (`Client`). Step 3 is Layer 2 (`Plugin`) — same request, plugin just sits in the middle and calls back once. - -## Two ways to pick a provider - -A model can have more than one provider behind it (tiers, fallback). Normally -tb picks. `pin_provider=` lets the caller pick instead — but only from what -that rule already offers. - -``` - default pin_provider=B - ─────── ────────────── - tb.ask(model="x") tb.ask(model="x", pin_provider=B) - │ │ - ▼ ▼ - rule "x" → services [A, B] rule "x" → services [A, B] - │ │ - ▼ ▼ - tb picks ONE must B be in [A, B]? - (tiers / affinity / load) yes → use B, guaranteed - no → 400, rejected -``` - -Why this exists: a plugin that checks quota for provider A and then calls -`.ask(model="x")` is only *guessing* A will be used — tb might pick B -instead. `pin_provider=` turns the guess into a guarantee. - -Only two modes — no "skip the rule entirely" third one. No rule for a -provider yet? Create a one-service rule for it (cheap, one-time), don't -bypass rule resolution to reach it — same reason `X-Tingly-Probe-Service` -(tb's internal, unauthenticated bypass) never got exposed to the SDK. - -## `router_plugin.py` in one picture - -``` - question in - │ - ▼ - for each candidate model: - keep it only if it maps to exactly ONE provider - (a model with several providers can't be quota-picked — tb decides that one) - │ - ▼ - check quota for each provider kept - │ - ▼ - pick the one with the most headroom - │ - ▼ - tb.ask(model=picked, pin_provider=picked's provider) ← the guarantee, above - │ - ▼ - answer out -``` - -Everything else — `critic_plugin.py` (ask a different model to review), -`fusion_plugin.py` (ask several, then a judge), `rag_plugin.py` (ask one, -with retrieved context) — is the same "call back into tb" from *A request, -start to finish* above, just with different logic in the handler. +`critic_plugin.py` (ask a different model to review), `fusion_plugin.py` +(ask several, then a judge), and `rag_plugin.py` (ask one, with retrieved +context) are all step 3 with different logic in the handler — no new +mechanism. diff --git a/internal/server/routing/result.go b/internal/server/routing/result.go index 46d26012a..b69cb29a5 100644 --- a/internal/server/routing/result.go +++ b/internal/server/routing/result.go @@ -15,10 +15,6 @@ const ( // SourceProbePin marks the X-Tingly-Probe-Service bypass, which pins a // specific service without running the pipeline. SourceProbePin = "probe_pin" - // SourceProviderPin marks the X-Tingly-Pin-Provider override — a caller - // choosing which of the rule's OWN configured services to use, still - // authenticated and still scoped to that rule (contrast SourceProbePin). - SourceProviderPin = "provider_pin" ) // SelectionResult represents the output of service selection pipeline. diff --git a/internal/server/routing/simple.go b/internal/server/routing/simple.go index 138233c38..628e9fc3f 100644 --- a/internal/server/routing/simple.go +++ b/internal/server/routing/simple.go @@ -58,30 +58,6 @@ func (s *SimpleSelector) SelectService( // Build context (session ID resolved internally) ctx := NewSelectionContext(rule, req, c, scenario) - // X-Tingly-Pin-Provider: — an authenticated caller (this - // endpoint already required a valid model token to reach here) asking to - // use one SPECIFIC service already configured on THIS rule, instead of - // whatever affinity/smart-routing/load-balancing would otherwise pick. - // Deliberately scoped to rule.Services — unlike X-Tingly-Probe-Service - // above, this is safe to expose to normal clients precisely because it - // cannot reach a provider the rule wasn't already configured to use. - if pinnedUUID := c.GetHeader("X-Tingly-Pin-Provider"); pinnedUUID != "" { - svc := findActiveServiceByProvider(rule, pinnedUUID) - if svc == nil { - return nil, nil, fmt.Errorf("X-Tingly-Pin-Provider %q is not an active service on this rule", pinnedUUID) - } - provider, err := s.selector.config.GetProviderByUUID(pinnedUUID) - if err != nil || provider == nil { - return nil, nil, fmt.Errorf("pinned provider not found: %s", pinnedUUID) - } - if !provider.Enabled { - return nil, nil, fmt.Errorf("pinned provider disabled: %s", pinnedUUID) - } - result := &SelectionResult{Provider: provider, Service: svc, Source: SourceProviderPin, MatchedSmartRuleIndex: -1} - s.applySelectionResult(c, ctx, rule, scenario, result) - return provider, svc, nil - } - // Execute pipeline result, err := s.selector.Select(ctx) if err != nil { @@ -92,27 +68,6 @@ func (s *SimpleSelector) SelectService( return nil, nil, fmt.Errorf("selection returned nil result") } - s.applySelectionResult(c, ctx, rule, scenario, result) - - return result.Provider, result.Service, nil -} - -// findActiveServiceByProvider returns the rule's own active service bound to -// the given provider UUID, or nil if the rule has no such service — the -// scoping check that makes X-Tingly-Pin-Provider safe to expose to clients. -func findActiveServiceByProvider(rule *typ.Rule, providerUUID string) *loadbalance.Service { - for _, svc := range rule.GetActiveServices() { - if svc.Provider == providerUUID { - return svc - } - } - return nil -} - -// applySelectionResult stores session/affinity/observability context and -// emits debug headers for a selection result, however it was produced -// (the normal pipeline, or the X-Tingly-Pin-Provider override above). -func (s *SimpleSelector) applySelectionResult(c *gin.Context, ctx *SelectionContext, rule *typ.Rule, scenario typ.RuleScenario, result *SelectionResult) { // Automatically store sessionID in gin context for downstream handlers c.Set(constant.CtxKeySessionID, ctx.SessionID.String()) // The scoped affinity key (session + matched smart partition) — consumers @@ -151,6 +106,8 @@ func (s *SimpleSelector) applySelectionResult(c *gin.Context, ctx *SelectionCont }).Infof("[routing] selected %s/%s via %s", result.Provider.UUID, result.Service.Model, result.Source) setRoutingDebugHeaders(c, result.Provider.Name, result.Provider.UUID, result.Service.Model, result.Source, result.MatchedSmartRuleIndex, result.EvaluatedStages) + + return result.Provider, result.Service, nil } // setRoutingDebugHeaders emits X-Tingly-Selected-* response headers describing diff --git a/sdk/python/README.md b/sdk/python/README.md index 413b9d0a5..4a559816c 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -98,7 +98,7 @@ gateway for its own LLM work. ### Example plugins -`sdk/python/examples/` has four, each demonstrating a different real-world +`sdk/python/examples/` has three, each demonstrating a different real-world pattern for the same idea — a plugin composing the box by calling back into other tb rules: @@ -116,30 +116,6 @@ other tb rules: already agree, otherwise a judge call synthesizes. Mirrors Consult7's 2026 Fusion feature; the clearest illustration that a plugin can freely originate more than one call, against more than one rule, per request. -- **`router_plugin.py`** — quota-aware dispatch (`model="plugin/router"`): a - different shape from the three above — it generates nothing itself, it - only *decides* which one candidate to forward to, using quota headroom - (`tb.quota`) to pick. Same idea as LiteLLM Router's `usage-based-routing` - strategy. Forwards with `tb.ask(..., pin_provider=)` so the provider - it checked quota for is *guaranteed* to be the one that serves the - request — see "Deterministic dispatch" below for why that matters. - -## Deterministic dispatch (`pin_provider`) - -Normally `tb.ask(model=X)` resolves `(scenario, model)` to a rule and lets tb -itself pick which of that rule's services actually runs (affinity / smart -routing / load balancing — unchanged, still the default). When code needs to -*guarantee* a specific provider — like `router_plugin.py` above, which -already checked that provider's quota — pass `pin_provider`: - -```python -tb.ask("...", model="sonnet1", pin_provider=provider_uuid) -``` - -tb only allows pinning to a provider that's already one of the resolved -rule's own configured services (`tb.rules.for_model(scenario, model)` lists -them) — it rejects a pin to anything else. See `.design/python-sdk.md` §"Two -connection modes" for the full mechanics. ## Status @@ -151,8 +127,5 @@ connection modes" for the full mechanics. - **Layer 3** (tb routes to the plugin as a model): via provider-as-upstream. Done, verified end-to-end including a plugin forwarding to another tb rule and returning the result (`sdk/python/examples/e2e_run.sh`). -- **Deterministic dispatch** (`pin_provider`, `Client.rules`): done, verified - end-to-end including `router_plugin.py` run for real - (`sdk/python/examples/e2e_run_pin.sh`). See `.design/python-sdk.md` in the repo for the full design and diagrams. diff --git a/sdk/python/tests/test_client_offline.py b/sdk/python/tests/test_client_offline.py index e3b79b324..aa3255a04 100644 --- a/sdk/python/tests/test_client_offline.py +++ b/sdk/python/tests/test_client_offline.py @@ -73,72 +73,6 @@ class _FakeAnthropic: assert captured["model"] == "auto" -def test_ask_pin_provider_sets_header(monkeypatch): - c = _client("anthropic") - captured = {} - - class _FakeMessages: - def create(self, **kwargs): - captured.update(kwargs) - - class R: - content = [] - - return R() - - class _FakeAnthropic: - messages = _FakeMessages() - - monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) - c.ask("hi", pin_provider="p1") - - assert captured["extra_headers"] == {"X-Tingly-Pin-Provider": "p1"} - - -def test_ask_pin_provider_merges_with_caller_extra_headers(monkeypatch): - c = _client("anthropic") - captured = {} - - class _FakeMessages: - def create(self, **kwargs): - captured.update(kwargs) - - class R: - content = [] - - return R() - - class _FakeAnthropic: - messages = _FakeMessages() - - monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) - c.ask("hi", pin_provider="p1", extra_headers={"X-Custom": "1"}) - - assert captured["extra_headers"] == {"X-Custom": "1", "X-Tingly-Pin-Provider": "p1"} - - -def test_ask_without_pin_provider_sends_no_pin_header(monkeypatch): - c = _client("anthropic") - captured = {} - - class _FakeMessages: - def create(self, **kwargs): - captured.update(kwargs) - - class R: - content = [] - - return R() - - class _FakeAnthropic: - messages = _FakeMessages() - - monkeypatch.setattr(Client, "anthropic", property(lambda self: _FakeAnthropic())) - c.ask("hi") - - assert "extra_headers" not in captured - - def test_client_passes_scenario_root_to_transports(monkeypatch): """The client hands the scenario root + model token to each builder; per-transport URL shaping (e.g. the OpenAI /v1 suffix) happens inside the diff --git a/sdk/python/tingly/client.py b/sdk/python/tingly/client.py index e660cd16d..91719dd0a 100644 --- a/sdk/python/tingly/client.py +++ b/sdk/python/tingly/client.py @@ -16,8 +16,6 @@ from . import scenarios as _scenarios from .errors import TinglyError from .helpers.guardrails import GuardrailsView -from .helpers.quota import QuotaView -from .helpers.rules import RulesView from .helpers.usage import UsageView from .transports import anthropic_compat, openai_compat @@ -105,7 +103,6 @@ def ask( system: Optional[str] = None, max_tokens: int = 1024, stream: bool = False, - pin_provider: Optional[str] = None, **kwargs: Any, ): """One-shot prompt → text, routed through tingly-box. @@ -113,18 +110,7 @@ def ask( Picks the transport from the scenario: Anthropic messages is tried first (tb's native protocol); OpenAI-only scenarios fall back to chat completions. ``model="auto"`` lets the gateway route. - - ``pin_provider`` sends ``X-Tingly-Pin-Provider``, forcing tb to use - that exact provider instead of letting affinity/smart-routing/load- - balancing decide — but only if it's one of the resolved rule's own - configured services (see ``Client.rules``); tb rejects a pin to any - other provider. Omit it for tb's normal behavior. """ - if pin_provider: - kwargs["extra_headers"] = { - **kwargs.get("extra_headers", {}), - "X-Tingly-Pin-Provider": pin_provider, - } if _scenarios.supports_anthropic(self._session.transport): return self._ask_anthropic(prompt, model, system, max_tokens, stream, **kwargs) return self._ask_openai(prompt, model, system, stream, **kwargs) @@ -177,14 +163,6 @@ def usage(self) -> UsageView: def guardrails(self) -> GuardrailsView: return GuardrailsView(self._gateway_url, self._admin_token, self._timeout) - @property - def quota(self) -> QuotaView: - return QuotaView(self._gateway_url, self._admin_token, self._timeout) - - @property - def rules(self) -> RulesView: - return RulesView(self._gateway_url, self._admin_token, self._timeout) - # -- lifecycle ------------------------------------------------------- def close(self) -> None: