From 70cd31ea0faf02752ad15f0bdc67c41960d1474c Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 17:12:04 +0800 Subject: [PATCH 1/9] refactor: better naming for probe related and lint fix. --- .design/probe.md | 4 +- .design/third-party-credentials.md | 2 +- internal/probe/{e2e.go => e2e_probe.go} | 32 ++++---- internal/probe/e2e_probe_test.go | 12 +-- .../probe/{lightweight.go => light_probe.go} | 20 ++--- internal/probe/probetools.go | 24 +++--- internal/probe/result.go | 62 ---------------- internal/probe/{sdkprobe.go => sdk.go} | 33 ++++----- internal/probe/types.go | 73 +++++++++++++++++-- internal/probe/types_test.go | 4 +- internal/server/module/probe/handler.go | 10 +-- internal/server/server.go | 14 ++-- internal/server/server_webui_api.go | 2 +- 13 files changed, 143 insertions(+), 149 deletions(-) rename internal/probe/{e2e.go => e2e_probe.go} (90%) rename internal/probe/{lightweight.go => light_probe.go} (87%) delete mode 100644 internal/probe/result.go rename internal/probe/{sdkprobe.go => sdk.go} (86%) diff --git a/.design/probe.md b/.design/probe.md index eb153b0c1..e15d15398 100644 --- a/.design/probe.md +++ b/.design/probe.md @@ -4,8 +4,8 @@ The probe subsystem performs SDK-level end-to-end connectivity tests for providers and rules. There are two probe strategies: -- **Lightweight** (`internal/probe/lightweight.go`): HTTP-level checks (OPTIONS, `/models`, `/chat/completions`) with no SDK. Used during provider onboarding to validate credentials quickly. -- **E2E** (`internal/probe/e2e.go`): Full SDK round-trip using the same client methods as production traffic (ChatCompletionsNew, ResponsesNew, MessagesNew, GenerateContent). This catches provider quirks that only show up under the real code path. +- **Lightweight** (`../internal/probe/light_probe.go`): HTTP-level checks (OPTIONS, `/models`, `/chat/completions`) with no SDK. Used during provider onboarding to validate credentials quickly. +- **E2E** (`../internal/probe/e2e_probe.go`): Full SDK round-trip using the same client methods as production traffic (ChatCompletionsNew, ResponsesNew, MessagesNew, GenerateContent). This catches provider quirks that only show up under the real code path. ## E2E Target Types diff --git a/.design/third-party-credentials.md b/.design/third-party-credentials.md index 74b77c5fe..f4129c212 100644 --- a/.design/third-party-credentials.md +++ b/.design/third-party-credentials.md @@ -209,7 +209,7 @@ verified against AWS/Google/Anthropic/Microsoft docs (Jul 2026): Bedrock generic form; responses return credentials in full). - **Test Connection** through the signed client path. - `GetAccessToken()` returns `""` for cloud types; the manual-header call sites - outside the client constructors (`internal/probe/sdkprobe.go` lightweight + outside the client constructors (`../internal/probe/sdk.go` lightweight probe, `internal/tbclient`) send unauthenticated requests for cloud providers. A central "apply credential" seam would fix all of them at once. - Backend derivation of `api_base` from the bundle (today the frontend computes diff --git a/internal/probe/e2e.go b/internal/probe/e2e_probe.go similarity index 90% rename from internal/probe/e2e.go rename to internal/probe/e2e_probe.go index c810f20a4..7c2194b72 100644 --- a/internal/probe/e2e.go +++ b/internal/probe/e2e_probe.go @@ -14,18 +14,18 @@ import ( "github.com/tingly-dev/tingly-box/internal/typ" ) -// E2EService runs SDK-level end-to-end probes against a rule, a saved +// E2EProber runs SDK-level end-to-end probes against a rule, a saved // provider, or an inline provider config. It is independent of *Server and // is wired in NewServer. -type E2EService struct { +type E2EProber struct { config *config.Config clientPool *client.ClientPool endpointCache *endpointProbeCache } -// NewE2EService constructs a E2EService. -func NewE2EService(cfg *config.Config, pool *client.ClientPool) *E2EService { - return &E2EService{ +// NewE2EProber constructs a E2EProber. +func NewE2EProber(cfg *config.Config, pool *client.ClientPool) *E2EProber { + return &E2EProber{ config: cfg, clientPool: pool, endpointCache: newEndpointProbeCache(), @@ -33,7 +33,7 @@ func NewE2EService(cfg *config.Config, pool *client.ClientPool) *E2EService { } // Probe performs a non-streaming probe against the target described by req. -func (e *E2EService) Probe(ctx context.Context, req *E2ERequest) (*E2EData, error) { +func (e *E2EProber) Probe(ctx context.Context, req *E2ERequest) (*E2EData, error) { provider, model, probeHeaders, err := e.resolveTargetToProviderModel(ctx, req) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (e *E2EService) Probe(ctx context.Context, req *E2ERequest) (*E2EData, erro cacheable := req.TargetType == E2ETargetProvider && req.Direct && (req.Endpoint == "chat" || req.Endpoint == "responses") if cacheable && e.endpointCache.hit(provider.UUID, model, req.Endpoint) { - return &ProbeResult{Success: true, Message: "Verified recently (cached)"}, nil + return &Result{Success: true, Message: "Verified recently (cached)"}, nil } if len(probeHeaders) > 0 { @@ -62,7 +62,7 @@ func (e *E2EService) Probe(ctx context.Context, req *E2ERequest) (*E2EData, erro } // ProbeStream performs a streaming probe against the target described by req. -func (e *E2EService) ProbeStream(ctx context.Context, req *E2ERequest) (*E2EData, error) { +func (e *E2EProber) ProbeStream(ctx context.Context, req *E2ERequest) (*E2EData, error) { provider, model, probeHeaders, err := e.resolveTargetToProviderModel(ctx, req) if err != nil { return nil, err @@ -77,7 +77,7 @@ func (e *E2EService) ProbeStream(ctx context.Context, req *E2ERequest) (*E2EData // resolveTargetToProviderModel resolves an E2ERequest to a provider, model, // and optional probe headers. Probe headers are injected into SDK HTTP calls // via probeHeaderRoundTripper so that TB's own loopback endpoint can read them. -func (e *E2EService) resolveTargetToProviderModel(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { +func (e *E2EProber) resolveTargetToProviderModel(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { var ( provider *typ.Provider model string @@ -107,7 +107,7 @@ func (e *E2EService) resolveTargetToProviderModel(ctx context.Context, req *E2ER return provider, model, probeHeaders, nil } -func (e *E2EService) resolveVModelLoopbackTarget(ctx context.Context, provider *typ.Provider, model string) (*typ.Provider, string, error) { +func (e *E2EProber) resolveVModelLoopbackTarget(ctx context.Context, provider *typ.Provider, model string) (*typ.Provider, string, error) { port := e.config.GetServerPort() if port == 0 { return nil, "", fmt.Errorf("server port unknown; cannot probe vmodel provider %q", provider.Name) @@ -127,7 +127,7 @@ func (e *E2EService) resolveVModelLoopbackTarget(ctx context.Context, provider * }) } -func (e *E2EService) resolveProviderTarget(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { +func (e *E2EProber) resolveProviderTarget(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { provider, err := e.config.GetProviderByUUID(req.ProviderUUID) if err != nil || provider == nil { return nil, "", nil, fmt.Errorf("provider not found: %s", req.ProviderUUID) @@ -211,7 +211,7 @@ func resolveOpenAIProbeEndpoint(override string, provider *typ.Provider) string case "chat", "responses": return override default: - if isCodexOAuth(provider) { + if provider.IsCodexProvider() { return "responses" } return "chat" @@ -241,7 +241,7 @@ func defaultScenarioForAPIStyle(style protocol.APIStyle) (typ.RuleScenario, bool } } -func (e *E2EService) resolveProviderConfigTarget(_ context.Context, req *E2ERequest) (*typ.Provider, string, error) { +func (e *E2EProber) resolveProviderConfigTarget(_ context.Context, req *E2ERequest) (*typ.Provider, string, error) { if req.APIBase == "" || req.APIStyle == "" || req.Token == "" { return nil, "", fmt.Errorf("provider_config target requires api_base, api_style, and token") } @@ -269,7 +269,7 @@ func (e *E2EService) resolveProviderConfigTarget(_ context.Context, req *E2ERequ return provider, model, nil } -func (e *E2EService) resolveRuleTarget(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { +func (e *E2EProber) resolveRuleTarget(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { rule := e.config.GetRuleByUUID(req.RuleUUID) if rule == nil { return nil, "", nil, fmt.Errorf("rule not found: %s", req.RuleUUID) @@ -319,7 +319,7 @@ func (e *E2EService) resolveRuleTarget(ctx context.Context, req *E2ERequest) (*t // endpointOverride forces which OpenAI endpoint to hit ("chat"/"responses"); // pass "" for resolveOpenAIProbeEndpoint's default (Codex OAuth -> responses, // everything else -> chat). -func (e *E2EService) ProbeProviderWithSDK(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { +func (e *E2EProber) ProbeProviderWithSDK(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { mode := testMode _, wrapProbeHeaders := client.GetProbeHeaders(ctx) @@ -405,6 +405,6 @@ func applyRoutingCapture(result *E2EData, cap *client.RoutingCapture) { } } -func (e *E2EService) probeProviderStream(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { +func (e *E2EProber) probeProviderStream(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { return e.ProbeProviderWithSDK(ctx, provider, model, message, testMode, endpointOverride) } diff --git a/internal/probe/e2e_probe_test.go b/internal/probe/e2e_probe_test.go index b7b534e9c..cb66ebf34 100644 --- a/internal/probe/e2e_probe_test.go +++ b/internal/probe/e2e_probe_test.go @@ -51,7 +51,7 @@ func TestResolveProviderTarget_OpenAI_RoutesLoopback(t *testing.T) { } addProvider(t, cfg, p) - svc := &E2EService{config: cfg} + svc := &E2EProber{config: cfg} req := &E2ERequest{ TargetType: E2ETargetProvider, ProviderUUID: "p-openai", @@ -85,7 +85,7 @@ func TestResolveProviderTarget_Anthropic_RoutesLoopback(t *testing.T) { } addProvider(t, cfg, p) - svc := &E2EService{config: cfg} + svc := &E2EProber{config: cfg} req := &E2ERequest{ TargetType: E2ETargetProvider, ProviderUUID: "p-anthropic", @@ -117,7 +117,7 @@ func TestResolveProviderTarget_Google_DirectSDK(t *testing.T) { } addProvider(t, cfg, p) - svc := &E2EService{config: cfg} + svc := &E2EProber{config: cfg} req := &E2ERequest{ TargetType: E2ETargetProvider, ProviderUUID: "p-google", @@ -148,7 +148,7 @@ func TestResolveProviderTarget_NoPort_FallsBackDirect(t *testing.T) { } addProvider(t, cfg, p) - svc := &E2EService{config: cfg} + svc := &E2EProber{config: cfg} req := &E2ERequest{ TargetType: E2ETargetProvider, ProviderUUID: "p-openai", @@ -178,7 +178,7 @@ func TestResolveProviderTarget_DisabledProvider_Errors(t *testing.T) { p.Enabled = false require.NoError(t, cfg.UpdateProvider("p-disabled", p)) - svc := &E2EService{config: cfg} + svc := &E2EProber{config: cfg} req := &E2ERequest{ TargetType: E2ETargetProvider, ProviderUUID: "p-disabled", @@ -283,7 +283,7 @@ func TestProbe_CachedEndpointCheck_SkipsDispatch(t *testing.T) { } addProvider(t, cfg, p) - svc := NewE2EService(cfg, nil) // nil clientPool: dispatch would panic + svc := NewE2EProber(cfg, nil) // nil clientPool: dispatch would panic svc.endpointCache.remember("p-cache", "gpt-4o", "responses") result, err := svc.Probe(context.Background(), &E2ERequest{ diff --git a/internal/probe/lightweight.go b/internal/probe/light_probe.go similarity index 87% rename from internal/probe/lightweight.go rename to internal/probe/light_probe.go index 3a40e38f4..b7781fd48 100644 --- a/internal/probe/lightweight.go +++ b/internal/probe/light_probe.go @@ -10,23 +10,23 @@ import ( "github.com/tingly-dev/tingly-box/internal/typ" ) -// LightweightService runs the optional "Test Connection" probe used when a +// LightProber runs the optional "Test Connection" probe used when a // user adds an API key. It pokes OPTIONS, /models, /chat/completions, and // /responses and returns a per-endpoint report; results are advisory only // and do not block onboarding. Independent of *Server. -type LightweightService struct { +type LightProber struct { pool *client.ClientPool } -// NewLightweightService constructs a LightweightService backed by the given client pool. -func NewLightweightService(pool *client.ClientPool) *LightweightService { - return &LightweightService{pool: pool} +// NewLightProber constructs a LightProber backed by the given client pool. +func NewLightProber(pool *client.ClientPool) *LightProber { + return &LightProber{pool: pool} } // Probe runs every applicable sub-probe for the provider and returns a // populated LightweightProbeResponseData. Never returns an error — partial // failure is encoded in the per-endpoint fields and the Valid summary. -func (l *LightweightService) Probe(ctx context.Context, provider *typ.Provider) *LightweightProbeResponseData { +func (l *LightProber) Probe(ctx context.Context, provider *typ.Provider) *LightweightProbeResponseData { data := &LightweightProbeResponseData{ Provider: provider.Name, APIBase: provider.APIBase, @@ -95,7 +95,7 @@ type modelsReport struct { Warning string } -func (l *LightweightService) probeOptionsEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { +func (l *LightProber) probeOptionsEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { startTime := time.Now() switch provider.APIStyle { @@ -113,7 +113,7 @@ func (l *LightweightService) probeOptionsEndpoint(ctx context.Context, provider return endpointReport{false, fmt.Sprintf("OPTIONS failed: %s", result.ErrorMessage), responseTime} } -func (l *LightweightService) probeModelsEndpoint(ctx context.Context, provider *typ.Provider) modelsReport { +func (l *LightProber) probeModelsEndpoint(ctx context.Context, provider *typ.Provider) modelsReport { startTime := time.Now() var lister client.ModelLister @@ -174,7 +174,7 @@ func (l *LightweightService) probeModelsEndpoint(ctx context.Context, provider * } } -func (l *LightweightService) probeChatEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { +func (l *LightProber) probeChatEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { startTime := time.Now() c := l.pool.GetOpenAIClient(context.Background(), provider, "") @@ -197,7 +197,7 @@ func (l *LightweightService) probeChatEndpoint(ctx context.Context, provider *ty return endpointReport{false, "Chat endpoint returned no content", responseTime} } -func (l *LightweightService) probeResponsesEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { +func (l *LightProber) probeResponsesEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { startTime := time.Now() c := l.pool.GetOpenAIClient(context.Background(), provider, "") diff --git a/internal/probe/probetools.go b/internal/probe/probetools.go index cb443860d..b91b46578 100644 --- a/internal/probe/probetools.go +++ b/internal/probe/probetools.go @@ -17,8 +17,8 @@ func getProbeToolsAnthropic() []anthropic.ToolUnionParam { Name: "bash", InputSchema: anthropic.ToolInputSchemaParam{ Type: "object", - Properties: map[string]interface{}{ - "command": map[string]interface{}{ + Properties: map[string]any{ + "command": map[string]any{ "type": "string", "description": "The bash command to execute (e.g., 'ls -la', 'pwd', 'cat file.txt')", }, @@ -32,8 +32,8 @@ func getProbeToolsAnthropic() []anthropic.ToolUnionParam { Name: "get_status", InputSchema: anthropic.ToolInputSchemaParam{ Type: "object", - Properties: map[string]interface{}{ - "verbose": map[string]interface{}{ + Properties: map[string]any{ + "verbose": map[string]any{ "type": "boolean", "description": "Whether to include verbose status information", }, @@ -54,8 +54,8 @@ func getProbeToolsOpenAI() []openai.ChatCompletionToolUnionParam { Parameters: shared.FunctionParameters{ "type:": "object", "additionalProperties": false, - "properties": map[string]interface{}{ - "command": map[string]interface{}{ + "properties": map[string]any{ + "command": map[string]any{ "type": "string", "description": "The bash command to execute", }, @@ -69,8 +69,8 @@ func getProbeToolsOpenAI() []openai.ChatCompletionToolUnionParam { Parameters: shared.FunctionParameters{ "type": "object", "additionalProperties": false, - "properties": map[string]interface{}{ - "verbose": map[string]interface{}{ + "properties": map[string]any{ + "verbose": map[string]any{ "type": "boolean", "description": "Whether to include verbose information", }, @@ -89,8 +89,8 @@ func getProbeToolsResponses() []responses.ToolUnionParam { map[string]any{ "type": "object", "additionalProperties": false, - "properties": map[string]interface{}{ - "command": map[string]interface{}{ + "properties": map[string]any{ + "command": map[string]any{ "type": "string", "description": "The bash command to execute", }, @@ -104,8 +104,8 @@ func getProbeToolsResponses() []responses.ToolUnionParam { map[string]any{ "type": "object", "additionalProperties": false, - "properties": map[string]interface{}{ - "verbose": map[string]interface{}{ + "properties": map[string]any{ + "verbose": map[string]any{ "type": "boolean", "description": "Whether to include verbose information", }, diff --git a/internal/probe/result.go b/internal/probe/result.go deleted file mode 100644 index 8754599d3..000000000 --- a/internal/probe/result.go +++ /dev/null @@ -1,62 +0,0 @@ -package probe - -// ProbeResult is the canonical SDK-level probe result, shared by the E2E and -// lightweight probe strategies. It doubles as the JSON payload returned by the -// probe HTTP endpoints (exposed under the E2EData alias). -type ProbeResult struct { - // Basic fields - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Content string `json:"content,omitempty"` - LatencyMs int64 `json:"latency_ms"` - ModelsCount int `json:"models_count,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - - // Streaming mode indicator - Stream bool `json:"stream,omitempty"` - - // Token usage - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - - // Tool calls (for tool mode) - ToolCalls []ProbeToolCall `json:"tool_calls,omitempty"` - - // Request URL (for debugging) - RequestURL string `json:"request_url,omitempty"` - - // Routing trace — populated for TB-loopback probes (provider and rule targets). - // Empty for direct probes and provider_config probes. - SelectedProvider string `json:"selected_provider,omitempty"` - SelectedProviderUUID string `json:"selected_provider_uuid,omitempty"` - SelectedModel string `json:"selected_model,omitempty"` - RoutingSource string `json:"routing_source,omitempty"` - MatchedSmartRule *int `json:"matched_smart_rule,omitempty"` // nil = none, ≥0 = index - - // Execution-level facts — the real upstream endpoint TB used, the matched - // rule, and the flags it applied. Populated for TB-loopback probes. - UpstreamAPI string `json:"upstream_api,omitempty"` - UpstreamURL string `json:"upstream_url,omitempty"` - MatchedRule string `json:"matched_rule,omitempty"` - MatchedRuleDesc string `json:"matched_rule_desc,omitempty"` - AppliedFlags string `json:"applied_flags,omitempty"` -} - -// ProbeToolCall represents a tool call in a probe response. -type ProbeToolCall struct { - ID string `json:"id"` - Name string `json:"name"` - Input map[string]interface{} `json:"input"` -} - -// toProbeResult builds a ProbeResult carrying the raw (JSON-marshaled) -// upstream response for a successful probe. -func toProbeResult(content string, latencyMs int64, requestURL string, isStreaming bool) *ProbeResult { - return &ProbeResult{ - Content: content, - LatencyMs: latencyMs, - RequestURL: requestURL, - Stream: isStreaming, - } -} diff --git a/internal/probe/sdkprobe.go b/internal/probe/sdk.go similarity index 86% rename from internal/probe/sdkprobe.go rename to internal/probe/sdk.go index 01469b057..1ec208b44 100644 --- a/internal/probe/sdkprobe.go +++ b/internal/probe/sdk.go @@ -14,7 +14,6 @@ import ( "github.com/openai/openai-go/v3/responses" "google.golang.org/genai" - "github.com/tingly-dev/tingly-box/ai" "github.com/tingly-dev/tingly-box/internal/client" "github.com/tingly-dev/tingly-box/internal/protocol" "github.com/tingly-dev/tingly-box/internal/typ" @@ -32,7 +31,7 @@ const probeEchoInstruction = "work as `echo` if possible" // path. The client package therefore no longer owns any probe-specific code. // probeOpenAIChat builds and dispatches a minimal Chat Completions probe. -func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model, message string, mode E2EMode) (*ProbeResult, error) { +func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model, message string, mode E2EMode) (*Result, error) { start := time.Now() params := openai.ChatCompletionNewParams{ Model: model, @@ -61,7 +60,7 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model return nil, fmt.Errorf("chat streaming not supported by provider") } defer stream.Close() - var chunks []interface{} + var chunks []any for stream.Next() { chunks = append(chunks, stream.Current()) } @@ -73,7 +72,7 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model } // probeOpenAIResponses builds and dispatches a minimal Responses API probe. -func probeOpenAIResponses(ctx context.Context, oc client.OpenAIClientInterface, model, message string, mode E2EMode) (*ProbeResult, error) { +func probeOpenAIResponses(ctx context.Context, oc client.OpenAIClientInterface, model, message string, mode E2EMode) (*Result, error) { start := time.Now() params := responses.ResponseNewParams{ Model: model, @@ -111,7 +110,7 @@ func probeOpenAIResponses(ctx context.Context, oc client.OpenAIClientInterface, return nil, fmt.Errorf("responses streaming not supported by provider") } defer stream.Close() - var chunks []interface{} + var chunks []any for stream.Next() { chunks = append(chunks, stream.Current()) } @@ -123,7 +122,7 @@ func probeOpenAIResponses(ctx context.Context, oc client.OpenAIClientInterface, } // probeAnthropicMessages builds and dispatches a minimal Messages probe. -func probeAnthropicMessages(ctx context.Context, ac client.AnthropicClientInterface, model, message string, mode E2EMode) (*ProbeResult, error) { +func probeAnthropicMessages(ctx context.Context, ac client.AnthropicClientInterface, model, message string, mode E2EMode) (*Result, error) { start := time.Now() provider := ac.GetProvider() @@ -160,7 +159,7 @@ func probeAnthropicMessages(ctx context.Context, ac client.AnthropicClientInterf return nil, fmt.Errorf("messages streaming not supported by provider") } defer stream.Close() - var chunks []interface{} + var chunks []any for stream.Next() { chunks = append(chunks, stream.Current()) } @@ -172,7 +171,7 @@ func probeAnthropicMessages(ctx context.Context, ac client.AnthropicClientInterf } // probeGoogleGenerate builds and dispatches a minimal GenerateContent probe. -func probeGoogleGenerate(ctx context.Context, gc *client.GoogleClient, model, message string, mode E2EMode) (*ProbeResult, error) { +func probeGoogleGenerate(ctx context.Context, gc *client.GoogleClient, model, message string, mode E2EMode) (*Result, error) { start := time.Now() contents := []*genai.Content{ {Role: "user", Parts: []*genai.Part{{Text: message}}}, @@ -189,7 +188,7 @@ func probeGoogleGenerate(ctx context.Context, gc *client.GoogleClient, model, me return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false), nil } - var chunks []interface{} + var chunks []any for resp, err := range gc.GenerateContentStream(ctx, model, contents, config) { if err != nil { return nil, err @@ -203,7 +202,7 @@ func probeGoogleGenerate(ctx context.Context, gc *client.GoogleClient, model, me // probeOptions issues a bare OPTIONS request to the provider base URL with the // auth headers appropriate for its API style. Used by the lightweight probe; // results are advisory. -func probeOptions(ctx context.Context, provider *typ.Provider) ProbeResult { +func probeOptions(ctx context.Context, provider *typ.Provider) Result { start := time.Now() url := provider.APIBase @@ -228,7 +227,7 @@ func probeOptions(ctx context.Context, provider *typ.Provider) ProbeResult { req, err := http.NewRequestWithContext(ctx, http.MethodOptions, url, nil) if err != nil { - return ProbeResult{Success: false, ErrorMessage: fmt.Sprintf("Failed to create OPTIONS request: %v", err)} + return Result{Success: false, ErrorMessage: fmt.Sprintf("Failed to create OPTIONS request: %v", err)} } req.Header = header @@ -236,18 +235,12 @@ func probeOptions(ctx context.Context, provider *typ.Provider) ProbeResult { resp, err := httpClient.Do(req) latencyMs := time.Since(start).Milliseconds() if err != nil { - return ProbeResult{Success: false, ErrorMessage: fmt.Sprintf("OPTIONS request failed: %v", err), LatencyMs: latencyMs} + return Result{Success: false, ErrorMessage: fmt.Sprintf("OPTIONS request failed: %v", err), LatencyMs: latencyMs} } defer resp.Body.Close() if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return ProbeResult{Success: true, Message: "OPTIONS request successful", LatencyMs: latencyMs} + return Result{Success: true, Message: "OPTIONS request successful", LatencyMs: latencyMs} } - return ProbeResult{Success: false, ErrorMessage: fmt.Sprintf("OPTIONS request failed with status: %d", resp.StatusCode), LatencyMs: latencyMs} -} - -// isCodexOAuth reports whether the provider is a Codex OAuth provider, which -// only speaks the Responses API. -func isCodexOAuth(provider *typ.Provider) bool { - return provider.OAuthIssuer() == ai.IssuerCodex + return Result{Success: false, ErrorMessage: fmt.Sprintf("OPTIONS request failed with status: %d", resp.StatusCode), LatencyMs: latencyMs} } diff --git a/internal/probe/types.go b/internal/probe/types.go index f48af4f7c..2c6c8eebe 100644 --- a/internal/probe/types.go +++ b/internal/probe/types.go @@ -12,14 +12,75 @@ import ( "github.com/tingly-dev/tingly-box/internal/typ" ) -// ProbeRequest represents the request to probe/test a provider and model. -type ProbeRequest struct { +// Result is the canonical SDK-level probe result, shared by the E2E and +// lightweight probe strategies. It doubles as the JSON payload returned by the +// probe HTTP endpoints (exposed under the E2EData alias). +type Result struct { + // Basic fields + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Content string `json:"content,omitempty"` + LatencyMs int64 `json:"latency_ms"` + ModelsCount int `json:"models_count,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + + // Streaming mode indicator + Stream bool `json:"stream,omitempty"` + + // Token usage + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` + + // Tool calls (for tool mode) + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + + // Request URL (for debugging) + RequestURL string `json:"request_url,omitempty"` + + // Routing trace — populated for TB-loopback probes (provider and rule targets). + // Empty for direct probes and provider_config probes. + SelectedProvider string `json:"selected_provider,omitempty"` + SelectedProviderUUID string `json:"selected_provider_uuid,omitempty"` + SelectedModel string `json:"selected_model,omitempty"` + RoutingSource string `json:"routing_source,omitempty"` + MatchedSmartRule *int `json:"matched_smart_rule,omitempty"` // nil = none, ≥0 = index + + // Execution-level facts — the real upstream endpoint TB used, the matched + // rule, and the flags it applied. Populated for TB-loopback probes. + UpstreamAPI string `json:"upstream_api,omitempty"` + UpstreamURL string `json:"upstream_url,omitempty"` + MatchedRule string `json:"matched_rule,omitempty"` + MatchedRuleDesc string `json:"matched_rule_desc,omitempty"` + AppliedFlags string `json:"applied_flags,omitempty"` +} + +// ToolCall represents a tool call in a probe response. +type ToolCall struct { + ID string `json:"id"` + Name string `json:"name"` + Input map[string]any `json:"input"` +} + +// toProbeResult builds a Result carrying the raw (JSON-marshaled) +// upstream response for a successful probe. +func toProbeResult(content string, latencyMs int64, requestURL string, isStreaming bool) *Result { + return &Result{ + Content: content, + LatencyMs: latencyMs, + RequestURL: requestURL, + Stream: isStreaming, + } +} + +// Request represents the request to probe/test a provider and model. +type Request struct { Provider string `json:"provider" binding:"required" description:"Provider UUID to test against" example:"550e8400-e29b-41d4-a716-446655440000"` Model string `json:"model" binding:"required" description:"Model name to test against" example:"gpt-4-latest"` } -// ProbeProviderRequest represents the request to probe/test a provider's API key and connectivity. -type ProbeProviderRequest struct { +// ProviderRequest represents the request to probe/test a provider's API key and connectivity. +type ProviderRequest struct { Name string `json:"name" binding:"required" description:"Provider name" example:"openai"` APIBase string `json:"api_base" binding:"required" description:"API base URL" example:"https://api.openai.com/v1"` APIStyle string `json:"api_style" binding:"required,oneof=openai anthropic" description:"API style" example:"openai"` @@ -127,10 +188,10 @@ type E2ERequest struct { Endpoint string `json:"endpoint,omitempty" example:"responses"` } -// E2EData is an alias to ProbeResult — the canonical SDK-level probe result. +// E2EData is an alias to Result — the canonical SDK-level probe result. // Aliased so service-layer Response wrappers and swagger registrations can // keep referring to the historical E2EData name. -type E2EData = ProbeResult +type E2EData = Result // E2EResponseChunk represents a streaming response chunk. type E2EResponseChunk struct { diff --git a/internal/probe/types_test.go b/internal/probe/types_test.go index 66074e287..27778bc56 100644 --- a/internal/probe/types_test.go +++ b/internal/probe/types_test.go @@ -1,6 +1,7 @@ package probe import ( + "errors" "strings" "testing" @@ -119,7 +120,8 @@ func TestValidateE2ERequest(t *testing.T) { if err == nil { t.Fatalf("ValidateE2ERequest expected error for field %q, got nil", tt.wantErr) } - ve, ok := err.(*ValidationError) + var ve *ValidationError + ok := errors.As(err, &ve) if !ok { t.Fatalf("ValidateE2ERequest returned %T, want *ValidationError", err) } diff --git a/internal/server/module/probe/handler.go b/internal/server/module/probe/handler.go index 59ea79bae..c364a8d9f 100644 --- a/internal/server/module/probe/handler.go +++ b/internal/server/module/probe/handler.go @@ -15,13 +15,13 @@ import ( // lightweight services; adaptive can be hung off the same struct when that // strategy is decoupled from *Server. type Handler struct { - e2e *probe.E2EService - lightweight *probe.LightweightService + e2e *probe.E2EProber + light *probe.LightProber } // NewHandler builds a Handler around the given probe services. -func NewHandler(e2e *probe.E2EService, lightweight *probe.LightweightService) *Handler { - return &Handler{e2e: e2e, lightweight: lightweight} +func NewHandler(e2e *probe.E2EProber, light *probe.LightProber) *Handler { + return &Handler{e2e: e2e, light: light} } // errorDetail mirrors the JSON shape of the server's global ErrorDetail so @@ -141,6 +141,6 @@ func (h *Handler) HandleLightweightProbe(c *gin.Context) { provider.AuthType = typ.AuthType(req.AuthType) } - data := h.lightweight.Probe(c.Request.Context(), provider) + data := h.light.Probe(c.Request.Context(), provider) c.JSON(http.StatusOK, LightweightResponse{Success: true, Data: data}) } diff --git a/internal/server/server.go b/internal/server/server.go index a74b43f40..d26d7b1fe 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -104,11 +104,11 @@ type Server struct { // template manager for provider templates templateManager *data.TemplateManager - // probeE2EService runs SDK-level end-to-end probes for the /api/v2/probe endpoint. - probeE2EService *probe.E2EService + // probeE2e runs SDK-level end-to-end probes for the /api/v2/probe endpoint. + probeE2e *probe.E2EProber - // probeLightweight powers /api/v2/probe/lightweight — optional key validation. - probeLightweight *probe.LightweightService + // probeLight powers /api/v2/probe/lightweight — optional key validation. + probeLight *probe.LightProber // mcp runtime for external MCP tools mcpRuntime *mcpruntime.Runtime @@ -402,8 +402,8 @@ func NewServer(cfg *config.Config, opts ...ServerOption) *Server { server.registerAdviserFromConfig() // E2E probe service handles /api/v2/probe end-to-end without touching *Server. - server.probeE2EService = probe.NewE2EService(cfg, server.clientPool) - server.probeLightweight = probe.NewLightweightService(server.clientPool) + server.probeE2e = probe.NewE2EProber(cfg, server.clientPool) + server.probeLight = probe.NewLightProber(server.clientPool) // Initialize OTel for token metrics and tracing. Telemetry is export-only // (optional OTLP); persistent usage records are written directly by the @@ -536,7 +536,7 @@ func NewServer(cfg *config.Config, opts ...ServerOption) *Server { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - return server.probeLightweight.Probe(ctx, provider).Valid + return server.probeLight.Probe(ctx, provider).Valid }) } diff --git a/internal/server/server_webui_api.go b/internal/server/server_webui_api.go index dbf858bb0..6f836645e 100644 --- a/internal/server/server_webui_api.go +++ b/internal/server/server_webui_api.go @@ -366,7 +366,7 @@ func (s *Server) UseWebAPIEndpoints(manager *swagger.RouteManager) { onboarding.RegisterRoutes(apiV1, onboardingHandler) // E2E + lightweight probe endpoints - probemodule.RegisterRoutes(apiV2, probemodule.NewHandler(s.probeE2EService, s.probeLightweight)) + probemodule.RegisterRoutes(apiV2, probemodule.NewHandler(s.probeE2e, s.probeLight)) // Token Management apiV1.POST("/token", s.webHandler.GenerateToken, From a73603d83e2042160ff983e20c77ecca21d07d79 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 18:01:55 +0800 Subject: [PATCH 2/9] feat(probe): surface canonical token usage + tool calls; drop dead code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Token usage was declared on Result but never populated — every SDK probe JSON-marshaled the whole response into Content and returned, leaving the usage fields at zero. Tool calls were never lifted either. Resolve by reusing the existing internal/protocol/usage extractors rather than re-implementing parsing: - Result now carries a single Usage *protocol.TokenUsage in the canonical shape (input_tokens/output_tokens/cache_read_tokens/cache_write_tokens/ reasoning_tokens) — the same vocabulary the rest of TB emits and the frontend renders. No parallel flat token fields. - Non-stream probes call FromOpenAIChatCompletion/FromOpenAIResponses/ FromAnthropicMessage on resp.Usage. Stream probes take usage from the final chunk (OpenAI, with stream_options.include_usage now requested) / response.completed event (Responses) / AnthropicAccumulator (Anthropic). - Tool mode lifts calls into ToolCalls via per-provider extractors. - Success is now set in toProbeResult (was a never-true bug); LatencyMs is owned by the SDK probe (pure upstream time) — the handler no longer overwrites it. Also remove meaningless/dead implementation surfaced during review: - defaultModelForAPIStyle + model guessing: a probe tests a specific model the user picked; an empty model now errors explicitly instead of silently inventing claude-3-haiku/gpt-3.5-turbo. - ProbeStream + probeProviderStream: 1-line passthroughs; Probe handles all modes (stream decision lives in the SDK helpers via testMode). - ProbeProviderWithSDK -> unexported probeProviderWithSDK (the testProviderConnectivity reuse it was exported for no longer exists); routing-capture wiring deduped via a maybeCapture closure. - Dead types Request, ProviderRequest, ProbeProviderResponseData, and the orphaned ProbeProviderResponse envelope (zero refs/swagger/frontend). Google is out of scope: no usage/tool-call extraction, probes unchanged. --- internal/probe/e2e_probe.go | 102 ++++++++---------- internal/probe/light_probe.go | 4 +- internal/probe/sdk.go | 132 +++++++++++++++++++++--- internal/probe/types.go | 88 +++++++--------- internal/probe/types_test.go | 84 +++++++++++++++ internal/server/module/probe/handler.go | 20 ++-- internal/server/server_types.go | 11 -- 7 files changed, 293 insertions(+), 148 deletions(-) diff --git a/internal/probe/e2e_probe.go b/internal/probe/e2e_probe.go index 7c2194b72..a494779b1 100644 --- a/internal/probe/e2e_probe.go +++ b/internal/probe/e2e_probe.go @@ -32,7 +32,10 @@ func NewE2EProber(cfg *config.Config, pool *client.ClientPool) *E2EProber { } } -// Probe performs a non-streaming probe against the target described by req. +// Probe performs an SDK probe against the target described by req. It serves +// all test modes — simple/streaming/tool — the stream decision is made inside +// the SDK helpers from req.TestMode. Only the narrow direct-endpoint +// capability-check shape is cached; everything else dispatches for real. func (e *E2EProber) Probe(ctx context.Context, req *E2ERequest) (*E2EData, error) { provider, model, probeHeaders, err := e.resolveTargetToProviderModel(ctx, req) if err != nil { @@ -54,26 +57,13 @@ func (e *E2EProber) Probe(ctx context.Context, req *E2ERequest) (*E2EData, error ctx = client.WithProbeHeaders(ctx, probeHeaders) } message := E2EMessage(req.TestMode, req.Message) - result, err := e.ProbeProviderWithSDK(ctx, provider, model, message, req.TestMode, req.Endpoint) + result, err := e.probeProviderWithSDK(ctx, provider, model, message, req.TestMode, req.Endpoint) if cacheable && err == nil && result != nil && result.Success { e.endpointCache.remember(provider.UUID, model, req.Endpoint) } return result, err } -// ProbeStream performs a streaming probe against the target described by req. -func (e *E2EProber) ProbeStream(ctx context.Context, req *E2ERequest) (*E2EData, error) { - provider, model, probeHeaders, err := e.resolveTargetToProviderModel(ctx, req) - if err != nil { - return nil, err - } - if len(probeHeaders) > 0 { - ctx = client.WithProbeHeaders(ctx, probeHeaders) - } - message := E2EMessage(req.TestMode, req.Message) - return e.probeProviderStream(ctx, provider, model, message, req.TestMode, req.Endpoint) -} - // resolveTargetToProviderModel resolves an E2ERequest to a provider, model, // and optional probe headers. Probe headers are injected into SDK HTTP calls // via probeHeaderRoundTripper so that TB's own loopback endpoint can read them. @@ -141,10 +131,8 @@ func (e *E2EProber) resolveProviderTarget(ctx context.Context, req *E2ERequest) if model == "" { if len(provider.Models) > 0 { model = provider.Models[0] - } else if provider.APIStyle == protocol.APIStyleAnthropic { - model = "claude-3-haiku-20240307" } else { - model = "gpt-3.5-turbo" + return nil, "", nil, fmt.Errorf("no model specified and provider %q has no models to default to", provider.Name) } } @@ -201,11 +189,9 @@ func (e *E2EProber) resolveProviderTarget(ctx context.Context, req *E2ERequest) return loopbackProvider, loopbackModel, probeHeaders, nil } -// resolveOpenAIProbeEndpoint decides which OpenAI endpoint a probe should hit, -// folding both special cases that previously lived as separate branches in -// ProbeProviderWithSDK into one place: an explicit override always wins; -// absent that, Codex OAuth providers only speak Responses, everything else -// defaults to Chat. +// resolveOpenAIProbeEndpoint decides which OpenAI endpoint a probe should hit: +// an explicit override always wins; absent that, Codex OAuth providers only +// speak Responses, everything else defaults to Chat. func resolveOpenAIProbeEndpoint(override string, provider *typ.Provider) string { switch override { case "chat", "responses": @@ -256,14 +242,7 @@ func (e *E2EProber) resolveProviderConfigTarget(_ context.Context, req *E2EReque model := req.Model if model == "" { - switch provider.APIStyle { - case protocol.APIStyleAnthropic: - model = "claude-3-haiku-20240307" - case protocol.APIStyleGoogle: - model = "gemini-2.0-flash-exp" - default: - model = "gpt-3.5-turbo" - } + return nil, "", fmt.Errorf("no model specified for provider_config probe") } return provider, model, nil @@ -313,19 +292,35 @@ func (e *E2EProber) resolveRuleTarget(ctx context.Context, req *E2ERequest) (*ty return provider, model, probeHeaders, nil } -// ProbeProviderWithSDK runs an SDK probe by dispatching a minimal request -// through the provider's real-traffic client methods. Public because the -// server's provider onboarding path (testProviderConnectivity) reuses it. +// probeProviderWithSDK dispatches a minimal request through the provider's +// real-traffic client methods (the same methods production uses, so provider +// quirks cannot drift from the real path). The stream-vs-non-stream decision +// is made inside each per-provider helper from testMode. +// // endpointOverride forces which OpenAI endpoint to hit ("chat"/"responses"); // pass "" for resolveOpenAIProbeEndpoint's default (Codex OAuth -> responses, // everything else -> chat). -func (e *E2EProber) ProbeProviderWithSDK(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { - mode := testMode - +func (e *E2EProber) probeProviderWithSDK(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { _, wrapProbeHeaders := client.GetProbeHeaders(ctx) var result *E2EData var err error + // maybeCapture wires probe-header + routing-capture round trippers onto a + // client when this is a loopback probe, and returns a func that folds the + // captured routing trace into the result once the call completes. For direct + // probes (no probe headers) it returns a no-op. + maybeCapture := func(c any) func(*E2EData) { + if !wrapProbeHeaders { + return func(*E2EData) {} + } + client.ApplyProbeHeadersToClient(c) + routing := client.ApplyRoutingCaptureToClient(c) + return func(r *E2EData) { + if r != nil { + applyRoutingCapture(r, routing) + } + } + } switch provider.APIStyle { case protocol.APIStyleOpenAI: @@ -333,19 +328,15 @@ func (e *E2EProber) ProbeProviderWithSDK(ctx context.Context, provider *typ.Prov if oc == nil { return nil, fmt.Errorf("failed to get OpenAI client for provider: %s", provider.Name) } - var routing *client.RoutingCapture - if wrapProbeHeaders { - client.ApplyProbeHeadersToClient(oc) - routing = client.ApplyRoutingCaptureToClient(oc) - } + apply := maybeCapture(oc) switch resolveOpenAIProbeEndpoint(endpointOverride, provider) { case "chat": - result, err = probeOpenAIChat(ctx, oc, model, message, mode) + result, err = probeOpenAIChat(ctx, oc, model, message, testMode) case "responses": - result, err = probeOpenAIResponses(ctx, oc, model, message, mode) + result, err = probeOpenAIResponses(ctx, oc, model, message, testMode) } - if err == nil && routing != nil { - applyRoutingCapture(result, routing) + if err == nil { + apply(result) } case protocol.APIStyleAnthropic: @@ -353,14 +344,10 @@ func (e *E2EProber) ProbeProviderWithSDK(ctx context.Context, provider *typ.Prov if ac == nil { return nil, fmt.Errorf("failed to get Anthropic client for provider: %s", provider.Name) } - var routing *client.RoutingCapture - if wrapProbeHeaders { - client.ApplyProbeHeadersToClient(ac) - routing = client.ApplyRoutingCaptureToClient(ac) - } - result, err = probeAnthropicMessages(ctx, ac, model, message, mode) - if err == nil && routing != nil { - applyRoutingCapture(result, routing) + apply := maybeCapture(ac) + result, err = probeAnthropicMessages(ctx, ac, model, message, testMode) + if err == nil { + apply(result) } case protocol.APIStyleGoogle: @@ -368,7 +355,8 @@ func (e *E2EProber) ProbeProviderWithSDK(ctx context.Context, provider *typ.Prov if gc == nil { return nil, fmt.Errorf("failed to get Google client for provider: %s", provider.Name) } - result, err = probeGoogleGenerate(ctx, gc, model, message, mode) + // Google probes are always direct (no loopback route) — no routing capture. + result, err = probeGoogleGenerate(ctx, gc, model, message, testMode) default: return nil, fmt.Errorf("unsupported API style: %s", provider.APIStyle) @@ -404,7 +392,3 @@ func applyRoutingCapture(result *E2EData, cap *client.RoutingCapture) { result.MatchedRuleDesc = cap.MatchedRuleDesc } } - -func (e *E2EProber) probeProviderStream(ctx context.Context, provider *typ.Provider, model, message string, testMode E2EMode, endpointOverride string) (*E2EData, error) { - return e.ProbeProviderWithSDK(ctx, provider, model, message, testMode, endpointOverride) -} diff --git a/internal/probe/light_probe.go b/internal/probe/light_probe.go index b7781fd48..6185b47df 100644 --- a/internal/probe/light_probe.go +++ b/internal/probe/light_probe.go @@ -191,7 +191,7 @@ func (l *LightProber) probeChatEndpoint(ctx context.Context, provider *typ.Provi if err != nil { return endpointReport{false, fmt.Sprintf("Chat endpoint failed: %v", err), responseTime} } - if result != nil && result.Content != "" { + if result != nil && result.Success { return endpointReport{true, "Chat endpoint accessible", responseTime} } return endpointReport{false, "Chat endpoint returned no content", responseTime} @@ -214,7 +214,7 @@ func (l *LightProber) probeResponsesEndpoint(ctx context.Context, provider *typ. if err != nil { return endpointReport{false, fmt.Sprintf("Responses endpoint failed: %v", err), responseTime} } - if result != nil && result.Content != "" { + if result != nil && result.Success { return endpointReport{true, "Responses API endpoint accessible", responseTime} } return endpointReport{false, "Responses endpoint returned no content", responseTime} diff --git a/internal/probe/sdk.go b/internal/probe/sdk.go index 1ec208b44..91294b6dd 100644 --- a/internal/probe/sdk.go +++ b/internal/probe/sdk.go @@ -16,6 +16,7 @@ import ( "github.com/tingly-dev/tingly-box/internal/client" "github.com/tingly-dev/tingly-box/internal/protocol" + "github.com/tingly-dev/tingly-box/internal/protocol/usage" "github.com/tingly-dev/tingly-box/internal/typ" ) @@ -23,6 +24,77 @@ import ( // keep the upstream response minimal. const probeEchoInstruction = "work as `echo` if possible" +// extractToolCallInput unmarshals a JSON arguments/input string into a map. A +// missing or invalid JSON body yields an empty map rather than dropping the +// tool call — the name is still useful diagnostic info. +func extractToolCallInput(raw string) map[string]any { + if raw == "" { + return map[string]any{} + } + var m map[string]any + if err := json.Unmarshal([]byte(raw), &m); err != nil { + return map[string]any{} + } + if m == nil { + return map[string]any{} + } + return m +} + +// toolCallsFromOpenAIChat lifts tool calls out of an OpenAI Chat Completions +// response message. +func toolCallsFromOpenAIChat(msg openai.ChatCompletionMessage) []ToolCall { + var out []ToolCall + for _, choice := range msg.ToolCalls { + tc := choice.AsFunction() + if tc.Function.Name == "" { + continue + } + out = append(out, ToolCall{ + ID: tc.ID, + Name: tc.Function.Name, + Input: extractToolCallInput(tc.Function.Arguments), + }) + } + return out +} + +// toolCallsFromOpenAIResponses lifts function-call items out of an OpenAI +// Responses API output list. +func toolCallsFromOpenAIResponses(output []responses.ResponseOutputItemUnion) []ToolCall { + var out []ToolCall + for _, item := range output { + fc := item.AsFunctionCall() + if fc.Name == "" { + continue + } + out = append(out, ToolCall{ + ID: fc.ID, + Name: fc.Name, + Input: extractToolCallInput(fc.Arguments), + }) + } + return out +} + +// toolCallsFromAnthropic lifts tool_use blocks out of an Anthropic Message +// content list. +func toolCallsFromAnthropic(content []anthropic.ContentBlockUnion) []ToolCall { + var out []ToolCall + for _, block := range content { + tu := block.AsToolUse() + if tu.Name == "" { + continue + } + out = append(out, ToolCall{ + ID: tu.ID, + Name: tu.Name, + Input: extractToolCallInput(string(tu.Input)), + }) + } + return out +} + // The SDK probe helpers below dispatch a minimal request through each client's // real-traffic methods (ChatCompletionsNew, ResponsesNew, MessagesNew, // GenerateContent). Routing probes through the same methods as production @@ -44,6 +116,9 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model params.Tools = getProbeToolsOpenAI() params.ToolChoice = openai.ChatCompletionToolChoiceOptionUnionParam{OfAuto: openai.Opt("auto")} } + // Ask for stream usage so the streaming branch can report real token counts. + // Harmless on the non-streaming path (the provider ignores stream_options). + params.StreamOptions.IncludeUsage = openai.Opt(true) url := oc.GetProvider().APIBase + "/chat/completions" if mode == E2EModeSimple { @@ -52,7 +127,13 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model return nil, err } b, _ := json.Marshal(resp) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false), nil + // Tool calls only appear in the message when the request declared tools + // (tool mode); for simple/streaming probes the slice is empty. + var toolCalls []ToolCall + if len(resp.Choices) > 0 { + toolCalls = toolCallsFromOpenAIChat(resp.Choices[0].Message) + } + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false, usage.FromOpenAIChatCompletion(resp.Usage), toolCalls), nil } stream := oc.ChatCompletionsNewStreaming(ctx, params) @@ -60,15 +141,25 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model return nil, fmt.Errorf("chat streaming not supported by provider") } defer stream.Close() - var chunks []any + var ( + chunks []any + streamUse *protocol.TokenUsage + ) for stream.Next() { - chunks = append(chunks, stream.Current()) + ch := stream.Current() + chunks = append(chunks, ch) + // OpenAI emits the aggregate Usage on the final (empty-choices) chunk + // only when stream_options.include_usage is requested; keep the last + // usage we see so the probe surfaces real token counts. + if ch.JSON.Usage.Valid() { + streamUse = usage.FromOpenAIChatCompletion(ch.Usage) + } } if err := stream.Err(); err != nil { return nil, err } b, _ := json.Marshal(chunks) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true), nil + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true, streamUse, nil), nil } // probeOpenAIResponses builds and dispatches a minimal Responses API probe. @@ -102,7 +193,7 @@ func probeOpenAIResponses(ctx context.Context, oc client.OpenAIClientInterface, return nil, err } b, _ := json.Marshal(resp) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false), nil + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false, usage.FromOpenAIResponses(resp.Usage), toolCallsFromOpenAIResponses(resp.Output)), nil } stream := oc.ResponsesNewStreaming(ctx, params) @@ -110,15 +201,23 @@ func probeOpenAIResponses(ctx context.Context, oc client.OpenAIClientInterface, return nil, fmt.Errorf("responses streaming not supported by provider") } defer stream.Close() - var chunks []any + var ( + chunks []any + streamUse *protocol.TokenUsage + ) for stream.Next() { - chunks = append(chunks, stream.Current()) + ev := stream.Current() + chunks = append(chunks, ev) + // The completed Response event carries the aggregate Usage. + if ev.Type == "response.completed" && ev.Response.Usage.JSON.TotalTokens.Valid() { + streamUse = usage.FromOpenAIResponses(ev.Response.Usage) + } } if err := stream.Err(); err != nil { return nil, err } b, _ := json.Marshal(chunks) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true), nil + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true, streamUse, nil), nil } // probeAnthropicMessages builds and dispatches a minimal Messages probe. @@ -151,7 +250,7 @@ func probeAnthropicMessages(ctx context.Context, ac client.AnthropicClientInterf return nil, err } b, _ := json.Marshal(resp) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false), nil + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false, usage.FromAnthropicMessage(resp.Usage), toolCallsFromAnthropic(resp.Content)), nil } stream := ac.MessagesNewStreaming(ctx, params) @@ -159,15 +258,22 @@ func probeAnthropicMessages(ctx context.Context, ac client.AnthropicClientInterf return nil, fmt.Errorf("messages streaming not supported by provider") } defer stream.Close() + acc := usage.NewAnthropicAccumulator() var chunks []any for stream.Next() { - chunks = append(chunks, stream.Current()) + ev := stream.Current() + acc.Consume(&ev) + chunks = append(chunks, ev) } if err := stream.Err(); err != nil { return nil, err } b, _ := json.Marshal(chunks) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true), nil + var streamUse *protocol.TokenUsage + if acc.HasUsage() { + streamUse = acc.Result() + } + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true, streamUse, nil), nil } // probeGoogleGenerate builds and dispatches a minimal GenerateContent probe. @@ -185,7 +291,7 @@ func probeGoogleGenerate(ctx context.Context, gc *client.GoogleClient, model, me return nil, err } b, _ := json.Marshal(resp) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false), nil + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false, nil, nil), nil } var chunks []any @@ -196,7 +302,7 @@ func probeGoogleGenerate(ctx context.Context, gc *client.GoogleClient, model, me chunks = append(chunks, resp) } b, _ := json.Marshal(chunks) - return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true), nil + return toProbeResult(string(b), time.Since(start).Milliseconds(), url, true, nil, nil), nil } // probeOptions issues a bare OPTIONS request to the provider base URL with the diff --git a/internal/probe/types.go b/internal/probe/types.go index 2c6c8eebe..4120f663c 100644 --- a/internal/probe/types.go +++ b/internal/probe/types.go @@ -3,6 +3,21 @@ // Lightweight strategies, and pure helpers. The Adaptive strategy still // lives in internal/server because it remains coupled to *Server; it will // be moved in a follow-up once that coupling is broken. +// +// Two result types answer two different questions and are deliberately NOT +// unified: +// +// - Result (alias E2EData) — SDK-level truth for one real round-trip through +// the production client methods. Carries normalized token Usage, lifted +// tool calls, and the routing journey. Returned by the E2E prober. +// - LightweightProbeResponseData — a per-endpoint connectivity matrix +// (OPTIONS / models / chat / responses success+latency). Advisory only, +// no usage, never blocks onboarding. Returned by the Lightweight prober. +// +// Both probers share the low-level SDK dispatch helpers (probeOpenAIChat, +// probeOptions, …). A probe never invents a model: if the request omits one +// and the provider record carries none, resolution fails explicitly rather +// than guessing. package probe import ( @@ -21,18 +36,25 @@ type Result struct { Message string `json:"message,omitempty"` Content string `json:"content,omitempty"` LatencyMs int64 `json:"latency_ms"` - ModelsCount int `json:"models_count,omitempty"` ErrorMessage string `json:"error_message,omitempty"` - // Streaming mode indicator + // Streaming mode indicator (true for streaming probes; redundant with the + // caller's test_mode but kept explicit so consumers don't have to infer the + // response shape from Content). Stream bool `json:"stream,omitempty"` - // Token usage - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - - // Tool calls (for tool mode) + // Usage is the normalized token usage for the probe round-trip, parsed via + // internal/protocol/usage from each provider's native usage struct. It uses + // the canonical protocol.TokenUsage shape (input_tokens / output_tokens / + // cache_read_tokens / cache_write_tokens / reasoning_tokens / system_tokens) + // — the same vocabulary the rest of the codebase emits and the frontend + // renders. Nil for cache hits, Google probes (out of scope), and providers + // that don't report usage (notably most streaming responses unless usage is + // requested). + Usage *protocol.TokenUsage `json:"usage,omitempty"` + + // Tool calls lifted out of the response (tool mode only). Empty for + // non-tool probes and for providers whose tool calls couldn't be extracted. ToolCalls []ToolCall `json:"tool_calls,omitempty"` // Request URL (for debugging) @@ -62,43 +84,23 @@ type ToolCall struct { Input map[string]any `json:"input"` } -// toProbeResult builds a Result carrying the raw (JSON-marshaled) -// upstream response for a successful probe. -func toProbeResult(content string, latencyMs int64, requestURL string, isStreaming bool) *Result { +// toProbeResult builds a Result carrying the raw (JSON-marshaled) upstream +// response for a successful probe. latencyMs is the pure upstream round-trip +// time (measured by the SDK probe helper, not the HTTP handler). usage, when +// non-nil, is the normalized token usage (canonical protocol.TokenUsage shape). +// toolCalls carries any tool calls lifted from the response (tool mode). +func toProbeResult(content string, latencyMs int64, requestURL string, isStreaming bool, usage *protocol.TokenUsage, toolCalls []ToolCall) *Result { return &Result{ + Success: true, Content: content, LatencyMs: latencyMs, RequestURL: requestURL, Stream: isStreaming, + Usage: usage, + ToolCalls: toolCalls, } } -// Request represents the request to probe/test a provider and model. -type Request struct { - Provider string `json:"provider" binding:"required" description:"Provider UUID to test against" example:"550e8400-e29b-41d4-a716-446655440000"` - Model string `json:"model" binding:"required" description:"Model name to test against" example:"gpt-4-latest"` -} - -// ProviderRequest represents the request to probe/test a provider's API key and connectivity. -type ProviderRequest struct { - Name string `json:"name" binding:"required" description:"Provider name" example:"openai"` - APIBase string `json:"api_base" binding:"required" description:"API base URL" example:"https://api.openai.com/v1"` - APIStyle string `json:"api_style" binding:"required,oneof=openai anthropic" description:"API style" example:"openai"` - Token string `json:"token" binding:"required" description:"API token to test" example:"sk-..."` -} - -// ProbeProviderResponseData represents the data returned from provider probing. -type ProbeProviderResponseData struct { - Provider string `json:"provider" example:"openai"` - APIBase string `json:"api_base" example:"https://api.openai.com/v1"` - APIStyle string `json:"api_style" example:"openai"` - Valid bool `json:"valid" example:"true"` - Message string `json:"message" example:"API key is valid and accessible"` - TestResult string `json:"test_result" example:"models_endpoint_success"` - ResponseTime int64 `json:"response_time_ms" example:"250"` - ModelsCount int `json:"models_count,omitempty" example:"150"` -} - // LightweightProbeRequest represents a lightweight probe request for key validation. type LightweightProbeRequest struct { Name string `json:"name" binding:"required" description:"Provider name" example:"openai"` @@ -193,18 +195,6 @@ type E2ERequest struct { // keep referring to the historical E2EData name. type E2EData = Result -// E2EResponseChunk represents a streaming response chunk. -type E2EResponseChunk struct { - Type string `json:"type"` // content, error, done - Content string `json:"content,omitempty"` - Error string `json:"error,omitempty"` - LatencyMs int64 `json:"latency_ms,omitempty"` - - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` -} - // ValidationError represents a probe-request validation error. type ValidationError struct { Field string diff --git a/internal/probe/types_test.go b/internal/probe/types_test.go index 27778bc56..ac6e4d513 100644 --- a/internal/probe/types_test.go +++ b/internal/probe/types_test.go @@ -1,10 +1,18 @@ package probe import ( + "encoding/json" "errors" "strings" "testing" + "github.com/anthropics/anthropic-sdk-go" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/responses" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + protocol2 "github.com/tingly-dev/tingly-box/ai" "github.com/tingly-dev/tingly-box/internal/protocol" ) @@ -176,3 +184,79 @@ func TestValidationErrorMessage(t *testing.T) { t.Errorf("ValidationError.Error() = %q", got) } } + +// ---- toProbeResult ---- + +func TestToProbeResult_SetsSuccessAndUsage(t *testing.T) { + // Canonical TokenUsage is passed through unchanged — no derived/renamed + // fields. Input 10, output 5, cache-read 2. + u := protocol2.NewTokenUsageFull(10, 5, 2, 0, 0) + r := toProbeResult("body", 42, "https://x/y", false, u, nil) + + assert.True(t, r.Success, "toProbeResult must set Success=true") + assert.Equal(t, int64(42), r.LatencyMs) + assert.False(t, r.Stream) + assert.Same(t, u, r.Usage, "Usage must be the canonical TokenUsage, passed through") + assert.Equal(t, 10, r.Usage.InputTokens) + assert.Equal(t, 5, r.Usage.OutputTokens) + assert.Equal(t, 2, r.Usage.CacheReadTokens) +} + +func TestToProbeResult_NilUsageStaysNil(t *testing.T) { + r := toProbeResult("body", 1, "url", true, nil, nil) + assert.True(t, r.Success) + assert.Nil(t, r.Usage) + assert.True(t, r.Stream) +} + +// ---- tool-call extractors ---- + +func TestToolCallsFromOpenAIChat(t *testing.T) { + // AsFunction() reads from the union's raw JSON, so construct via unmarshal + // (struct literals don't populate the raw JSON the accessor needs). + raw := `[{"id":"call_1","type":"function", + "function":{"name":"ls","arguments":"{\"dir\":\"/tmp\"}"}}]` + var calls []openai.ChatCompletionMessageToolCallUnion + require.NoError(t, json.Unmarshal([]byte(raw), &calls)) + msg := openai.ChatCompletionMessage{ToolCalls: calls} + + got := toolCallsFromOpenAIChat(msg) + assert.Len(t, got, 1) + assert.Equal(t, ToolCall{ID: "call_1", Name: "ls", Input: map[string]any{"dir": "/tmp"}}, got[0]) +} + +func TestToolCallsFromOpenAIChat_InvalidJSONBecomesEmptyInput(t *testing.T) { + raw := `[{"type":"function","function":{"name":"ls","arguments":"not-json"}}]` + var calls []openai.ChatCompletionMessageToolCallUnion + require.NoError(t, json.Unmarshal([]byte(raw), &calls)) + msg := openai.ChatCompletionMessage{ToolCalls: calls} + + got := toolCallsFromOpenAIChat(msg) + assert.Len(t, got, 1) + assert.Equal(t, "ls", got[0].Name) + assert.Empty(t, got[0].Input) +} + +func TestToolCallsFromOpenAIResponses(t *testing.T) { + // The Responses output-item union uses nested inline wrappers; build it via + // JSON unmarshal, exactly as it arrives from the API. + raw := `[{"type":"function_call","id":"fc_1","name":"get_weather","arguments":"{\"city\":\"SF\"}"}, + {"type":"message","id":"msg_1"}]` + var output []responses.ResponseOutputItemUnion + require.NoError(t, json.Unmarshal([]byte(raw), &output)) + + got := toolCallsFromOpenAIResponses(output) + assert.Len(t, got, 1) + assert.Equal(t, "get_weather", got[0].Name) + assert.Equal(t, "SF", got[0].Input["city"]) +} + +func TestToolCallsFromAnthropic(t *testing.T) { + raw := `[{"type":"tool_use","id":"tu_1","name":"list_dir","input":{"path":"/"}}]` + var content []anthropic.ContentBlockUnion + require.NoError(t, json.Unmarshal([]byte(raw), &content)) + + got := toolCallsFromAnthropic(content) + assert.Len(t, got, 1) + assert.Equal(t, ToolCall{ID: "tu_1", Name: "list_dir", Input: map[string]any{"path": "/"}}, got[0]) +} diff --git a/internal/server/module/probe/handler.go b/internal/server/module/probe/handler.go index c364a8d9f..657a22d2b 100644 --- a/internal/server/module/probe/handler.go +++ b/internal/server/module/probe/handler.go @@ -2,7 +2,6 @@ package probe import ( "net/http" - "time" "github.com/gin-gonic/gin" @@ -74,18 +73,10 @@ func (h *Handler) HandleE2EProbe(c *gin.Context) { } ctx := c.Request.Context() - startTime := time.Now() - - var ( - data *probe.E2EData - err error - ) - switch req.TestMode { - case probe.E2EModeSimple: - data, err = h.e2e.Probe(ctx, &req) - case probe.E2EModeStreaming, probe.E2EModeTool: - data, err = h.e2e.ProbeStream(ctx, &req) - } + + // Probe handles all test modes (simple/streaming/tool); the stream-vs- + // non-stream decision is made inside the SDK helpers from req.TestMode. + data, err := h.e2e.Probe(ctx, &req) if err != nil { c.JSON(http.StatusOK, E2EResponse{ @@ -98,7 +89,8 @@ func (h *Handler) HandleE2EProbe(c *gin.Context) { return } - data.LatencyMs = time.Since(startTime).Milliseconds() + // LatencyMs is owned by the SDK probe (pure upstream round-trip time) — do + // not overwrite it here. c.JSON(http.StatusOK, E2EResponse{Success: true, Data: data}) } diff --git a/internal/server/server_types.go b/internal/server/server_types.go index b5c0d6320..857ff52f7 100644 --- a/internal/server/server_types.go +++ b/internal/server/server_types.go @@ -4,8 +4,6 @@ import ( "github.com/tingly-dev/tingly-box/internal/protocolserver" "strings" "time" - - "github.com/tingly-dev/tingly-box/internal/probe" ) // ============================================= @@ -46,15 +44,6 @@ type OpenAIChatCompletionResponse struct { // Web UI API Models — probe request/data types live in internal/probe // ============================================= -// ProbeProviderResponse represents the response from provider probing. -// The wrapper stays here because it embeds *protocolserver.ErrorDetail (server's global -// error model). The Data shape lives in internal/probe. -type ProbeProviderResponse struct { - Success bool `json:"success" example:"true"` - Error *protocolserver.ErrorDetail `json:"error,omitempty"` - Data *probe.ProbeProviderResponseData `json:"data,omitempty"` -} - // RequestConfig represents a request configuration in defaults response type RequestConfig struct { RequestModel string `json:"request_model" example:"gpt-3.5-turbo"` From 73b922a3cb488bafcfb23256f670171d8aac1cc5 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 18:02:02 +0800 Subject: [PATCH 3/9] feat(frontend,probe): render canonical usage in probe dialog Drop the non-canonical prompt_tokens/completion_tokens/total_tokens fields from the probe result types and read the canonical usage object instead (protocol.TokenUsage shape: input_tokens/output_tokens/cache_read_tokens/ cache_write_tokens). The token chip now derives its total from usage.input_tokens + usage.output_tokens, matching TokenUsage.TotalTokens() on the backend. Updates the MSW mock and the in-dialog demo data to match. Backend model change requires `task codegen` to regenerate schema.d.ts / swagger; the hand-written types/probe.ts is updated as a placeholder. --- frontend/src/components/probe/ProbeDialog.tsx | 42 +++++++++++-------- frontend/src/mocks/handlers.ts | 8 ++-- frontend/src/types/probe.ts | 23 ++++++---- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/frontend/src/components/probe/ProbeDialog.tsx b/frontend/src/components/probe/ProbeDialog.tsx index 291acb1cb..55d77ad43 100644 --- a/frontend/src/components/probe/ProbeDialog.tsx +++ b/frontend/src/components/probe/ProbeDialog.tsx @@ -246,21 +246,28 @@ const StatusBar = memo(({ result }: { result: ProbeResult }) => { }} /> ) : null} - {d?.total_tokens ? ( - } - label={`${d.total_tokens} tokens`} - size="medium" - sx={{ - height: 28, - bgcolor: ok ? 'success.main' : 'error.main', - color: 'common.white', - '& .MuiChip-icon': { + {(() => { + // Canonical TokenUsage total = input + output (cache tracked + // separately, not added). Mirrors protocol.TokenUsage.TotalTokens(). + const total = + (d?.usage?.input_tokens || 0) + (d?.usage?.output_tokens || 0); + if (!total) return null; + return ( + } + label={`${total} tokens`} + size="medium" + sx={{ + height: 28, + bgcolor: ok ? 'success.main' : 'error.main', color: 'common.white', - }, - }} - /> - ) : null} + '& .MuiChip-icon': { + color: 'common.white', + }, + }} + /> + ); + })()} {!ok && result.error && ( = ({ latency_ms: 450, request_url: 'https://api.example.com/v1/chat', stream: mode === 'streaming', - prompt_tokens: 25, - completion_tokens: 18, - total_tokens: 43, + usage: { + input_tokens: 25, + output_tokens: 18, + }, selected_provider: targetName, selected_model: model || 'claude-sonnet-4-20250514', routing_source: 'smart_routing', diff --git a/frontend/src/mocks/handlers.ts b/frontend/src/mocks/handlers.ts index cfd4e2dab..78f67f6a3 100644 --- a/frontend/src/mocks/handlers.ts +++ b/frontend/src/mocks/handlers.ts @@ -1820,9 +1820,11 @@ export const handlers = [ latency_ms: Math.floor(Math.random() * 2200) + 400, request_url: 'http://localhost:12222/tingly/openai/chat/completions', stream: body?.test_mode !== 'simple', - prompt_tokens: 21, - completion_tokens: 14, - total_tokens: 35, + usage: { + input_tokens: 21, + output_tokens: 14, + cache_read_tokens: 0, + }, selected_provider: 'Anthropic', selected_model: 'claude-opus-4-8', routing_source: 'load_balancer', diff --git a/frontend/src/types/probe.ts b/frontend/src/types/probe.ts index f7a52920a..6c51d362d 100644 --- a/frontend/src/types/probe.ts +++ b/frontend/src/types/probe.ts @@ -39,14 +39,25 @@ export interface ProbeToolCall { } // Result payload of POST /api/v2/probe (backend probe.ProbeResult). +export interface ProbeTokenUsage { + input_tokens: number; + output_tokens: number; + cache_read_tokens?: number; + cache_write_tokens?: number; + reasoning_tokens?: number; +} + export interface ProbeResultData { content?: string; latency_ms: number; request_url?: string; stream?: boolean; - prompt_tokens?: number; - completion_tokens?: number; - total_tokens?: number; + // Canonical token usage (same shape as protocol.TokenUsage on the backend): + // input_tokens / output_tokens / cache_read_tokens / cache_write_tokens / + // reasoning_tokens. Present for OpenAI Chat/Responses and Anthropic probes + // (non-stream always; stream when the provider emits a final usage block); + // absent for Google and cache hits. + usage?: ProbeTokenUsage; tool_calls?: ProbeToolCall[]; // Routing trace — populated for TB-loopback probes. selected_provider?: string; @@ -83,10 +94,8 @@ export interface ProbeResponse { request_url?: string; stream?: boolean; - // Token usage (flattened) - prompt_tokens?: number; - completion_tokens?: number; - total_tokens?: number; + // Canonical token usage (protocol.TokenUsage shape). + usage?: ProbeTokenUsage; // Tool calls tool_calls?: ProbeToolCall[]; From 85682492d3e7a2f1fbffb50f5487a8a58f73e484 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 18:02:08 +0800 Subject: [PATCH 4/9] doc(probe): record canonical usage, result fields, and code layout Refresh the stale code-layout block (result.go/e2e.go/sdkprobe.go/ lightweight.go were all renamed) and document the Result field semantics: canonical protocol.TokenUsage (no parallel flat fields), SDK-owned LatencyMs, lifted tool calls, and the routing-trace fields. Note that probeProviderWithSDK reuses internal/protocol/usage rather than re-implementing parsing. --- .design/probe.md | 53 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/.design/probe.md b/.design/probe.md index e15d15398..e8ee65975 100644 --- a/.design/probe.md +++ b/.design/probe.md @@ -137,25 +137,54 @@ Neither round tripper is installed on production clients. `ProbeProviderWithSDK` ``` internal/probe/ - types.go — E2ERequest (incl. Direct field) / E2EData / E2EMode / E2ETarget, ScenarioEndpoint() - result.go — ProbeResult (incl. routing trace fields) - e2e.go — E2EService: resolveTargetToProviderModel, loopbackAPIBase, - ProbeProviderWithSDK, applyRoutingCapture - sdkprobe.go — SDK dispatch helpers: probeOpenAIChat, probeAnthropicMessages, probeGoogleGenerate, … - lightweight.go — LightweightProbeService (HTTP-level, no SDK) - probetools.go — Tool definitions used by E2EModeTool + types.go — Result (= E2EData): Success/Content/Usage/ToolCalls + routing trace; + E2ERequest (incl. Direct field) / E2EMode / E2ETarget, + ScenarioEndpoint(), toProbeResult + e2e_probe.go — E2EProber: resolveTargetToProviderModel, loopbackAPIBase, + probeProviderWithSDK, applyRoutingCapture + sdk.go — SDK dispatch helpers (probeOpenAIChat, probeOpenAIResponses, + probeAnthropicMessages, probeGoogleGenerate, probeOptions) and + the per-provider usage (via internal/protocol/usage) + tool-call + extractors + light_probe.go — LightProber (HTTP-level connectivity matrix, no SDK usage) + probetools.go — Tool definitions used by E2EModeTool + endpoint_probe_cache.go — narrow direct-endpoint capability cache + +internal/protocol/usage/ + extract.go — FromOpenAIChatCompletion / FromOpenAIResponses / FromAnthropicMessage: + the canonical TokenUsage extractors reused by the SDK probes internal/client/ - http.go — probeHeadersKey, WithProbeHeaders, GetProbeHeaders, - probeHeaderRoundTripper, ApplyProbeHeadersToClient - RoutingCapture, captureRoutingRoundTripper, ApplyRoutingCaptureToClient + http.go — probeHeadersKey, WithProbeHeaders, GetProbeHeaders, + probeHeaderRoundTripper, ApplyProbeHeadersToClient + RoutingCapture, captureRoutingRoundTripper, ApplyRoutingCaptureToClient internal/server/ - handlers.go — determineRuleWithScenario: X-Tingly-Probe-Rule / X-Tingly-Probe-Service handling + handlers.go — determineRuleWithScenario: X-Tingly-Probe-Rule / X-Tingly-Probe-Service handling routing/ - simple.go — SelectService: X-Tingly-Probe-Service pin + X-Tingly-Debug-Routing response headers + simple.go — SelectService: X-Tingly-Probe-Service pin + X-Tingly-Debug-Routing response headers ``` +## Result fields + +`Result` (returned as `E2EData`) carries, for one SDK round-trip: + +- `success`, `content` (raw marshaled upstream response — object for non-stream, + chunk array for stream), `latency_ms` (pure upstream call time, measured by the + SDK probe — the HTTP handler does not overwrite it), `error_message`. +- `stream` — true for streaming probes (explicit; the caller's `test_mode` is the + other source of truth). +- `usage` — normalized `*protocol.TokenUsage` parsed via `internal/protocol/usage` + for OpenAI Chat / Responses and Anthropic (non-stream always; stream when the + provider emits a final usage block). Uses the canonical `protocol.TokenUsage` + shape (`input_tokens` / `output_tokens` / `cache_read_tokens` / + `cache_write_tokens` / `reasoning_tokens`) — the same vocabulary the rest of + TB emits and the frontend renders; there are no parallel flat token fields. + `nil` for Google (out of scope) and cache hits. +- `tool_calls` — tool calls lifted out of the response (tool mode), one entry per + function call with `{id, name, input}`. +- Routing trace fields (see below) — populated for TB-loopback probes only. + ## Trade-offs and constraints - **Google probes go direct**: The TB loopback only exposes `/tingly/openai` and `/tingly/anthropic` endpoints. Google uses its own SDK and has no matching loopback route, so `resolveProviderTarget` returns the original provider record for Google. From 5f616339c5ec7ff7835e002a3f8b2735cc485eac Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 19:49:05 +0800 Subject: [PATCH 5/9] refactor(probe): simplify light probe endpoint helpers; fix summary denominator The light probe had four near-identical probeXxxEndpoint methods returning parallel endpointReport/modelsReport structs that Probe() then field-copied into LightweightProbeResponseData one by one. Collapse to focused helpers that write outcomes directly into the target fields: - runOptionsEndpoint (HTTP-level OPTIONS, no SDK) - runChatEndpoint/runResponsesEndpoint over a shared runOpenAIEndpoint body (only the SDK call + success label differ) - runModelsEndpoint with a clean switch over error cases Also fixes a latent bug: the summary message hardcoded "/4 endpoints", so non-OpenAI providers (which skip chat/responses) reported e.g. "2/4". The denominator now reflects the endpoints actually run. --- internal/probe/light_probe.go | 244 ++++++++++++++-------------------- 1 file changed, 101 insertions(+), 143 deletions(-) diff --git a/internal/probe/light_probe.go b/internal/probe/light_probe.go index 6185b47df..6c8584ca5 100644 --- a/internal/probe/light_probe.go +++ b/internal/probe/light_probe.go @@ -33,47 +33,32 @@ func (l *LightProber) Probe(ctx context.Context, provider *typ.Provider) *Lightw APIStyle: string(provider.APIStyle), } - optionsResult := l.probeOptionsEndpoint(ctx, provider) - data.OptionsSuccess = optionsResult.Success - data.OptionsMessage = optionsResult.Message - data.OptionsResponseTime = optionsResult.ResponseTime - - modelsResult := l.probeModelsEndpoint(ctx, provider) - data.ModelsSuccess = modelsResult.Success - data.ModelsMessage = modelsResult.Message - data.ModelsResponseTime = modelsResult.ResponseTime - data.ModelsCount = modelsResult.ModelsCount - data.Warning = modelsResult.Warning + // Each helper writes its outcome directly into data. Track the count of + // endpoints actually run so the summary denominator is correct (non-OpenAI + // providers skip chat/responses). + l.runOptionsEndpoint(ctx, provider, + &data.OptionsSuccess, &data.OptionsMessage, &data.OptionsResponseTime) + l.runModelsEndpoint(ctx, provider, + &data.ModelsSuccess, &data.ModelsMessage, &data.ModelsResponseTime, &data.ModelsCount, &data.Warning) + ran := 2 if provider.APIStyle == protocol.APIStyleOpenAI { - chatResult := l.probeChatEndpoint(ctx, provider) - data.ChatSuccess = chatResult.Success - data.ChatMessage = chatResult.Message - data.ChatResponseTime = chatResult.ResponseTime - - responsesResult := l.probeResponsesEndpoint(ctx, provider) - data.ResponsesSuccess = responsesResult.Success - data.ResponsesMessage = responsesResult.Message - data.ResponsesResponseTime = responsesResult.ResponseTime + l.runChatEndpoint(ctx, provider, + &data.ChatSuccess, &data.ChatMessage, &data.ChatResponseTime) + l.runResponsesEndpoint(ctx, provider, + &data.ResponsesSuccess, &data.ResponsesMessage, &data.ResponsesResponseTime) + ran = 4 } - data.Valid = data.OptionsSuccess || data.ModelsSuccess || data.ChatSuccess || data.ResponsesSuccess - - if data.Valid { - successCount := 0 - if data.OptionsSuccess { - successCount++ - } - if data.ModelsSuccess { - successCount++ - } - if data.ChatSuccess { - successCount++ + successes := 0 + for _, ok := range []bool{data.OptionsSuccess, data.ModelsSuccess, data.ChatSuccess, data.ResponsesSuccess} { + if ok { + successes++ } - if data.ResponsesSuccess { - successCount++ - } - data.Message = fmt.Sprintf("Connection test completed - %d/%d endpoints accessible", successCount, 4) + } + data.Valid = successes > 0 + if data.Valid { + data.Message = fmt.Sprintf("Connection test completed - %d/%d endpoints accessible", successes, ran) } else { data.Message = "Connection test failed - unable to reach any provider endpoint" } @@ -81,141 +66,114 @@ func (l *LightProber) Probe(ctx context.Context, provider *typ.Provider) *Lightw return data } -type endpointReport struct { - Success bool - Message string - ResponseTime int64 +// runOptionsEndpoint issues a bare OPTIONS request (HTTP-level, no SDK) and +// writes the outcome into the target fields. +func (l *LightProber) runOptionsEndpoint(ctx context.Context, provider *typ.Provider, + success *bool, msg *string, rt *int64) { + switch provider.APIStyle { + case protocol.APIStyleOpenAI, protocol.APIStyleAnthropic, protocol.APIStyleGoogle: + // supported below + default: + *success, *msg, *rt = false, fmt.Sprintf("Unsupported API style: %s", provider.APIStyle), 0 + return + } + start := time.Now() + r := probeOptions(ctx, provider) + *rt = time.Since(start).Milliseconds() + if r.Success { + *success, *msg = true, "OPTIONS request successful" + } else { + *success, *msg = false, fmt.Sprintf("OPTIONS failed: %s", r.ErrorMessage) + } } -type modelsReport struct { - Success bool - Message string - ResponseTime int64 - ModelsCount int - Warning string +// runChatEndpoint and runResponsesEndpoint run a minimal SDK round-trip against +// the respective OpenAI endpoint and write the outcome into the target fields. +// They share the timing/client/timeout boilerplate; only the call differs. +func (l *LightProber) runChatEndpoint(ctx context.Context, provider *typ.Provider, + success *bool, msg *string, rt *int64) { + l.runOpenAIEndpoint(ctx, provider, success, msg, rt, "Chat endpoint accessible", + func(c client.OpenAIClientInterface, pctx context.Context) (*Result, error) { + return probeOpenAIChat(pctx, c, "gpt-3.5-turbo", "Hi", E2EModeSimple) + }) } -func (l *LightProber) probeOptionsEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { - startTime := time.Now() +func (l *LightProber) runResponsesEndpoint(ctx context.Context, provider *typ.Provider, + success *bool, msg *string, rt *int64) { + l.runOpenAIEndpoint(ctx, provider, success, msg, rt, "Responses API endpoint accessible", + func(c client.OpenAIClientInterface, pctx context.Context) (*Result, error) { + return probeOpenAIResponses(pctx, c, "gpt-4o", "Hi", E2EModeSimple) + }) +} - switch provider.APIStyle { - case protocol.APIStyleOpenAI, protocol.APIStyleAnthropic, protocol.APIStyleGoogle: - // supported below - default: - return endpointReport{false, fmt.Sprintf("Unsupported API style: %s", provider.APIStyle), 0} +// runOpenAIEndpoint is the shared body for chat/responses connectivity checks. +// okLabel is the success message; call dispatches the actual probe. +func (l *LightProber) runOpenAIEndpoint(ctx context.Context, provider *typ.Provider, + success *bool, msg *string, rt *int64, okLabel string, + call func(c client.OpenAIClientInterface, pctx context.Context) (*Result, error)) { + start := time.Now() + c := l.pool.GetOpenAIClient(context.Background(), provider, "") + if c == nil { + *success, *msg, *rt = false, "Failed to create OpenAI client", 0 + return } + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() - result := probeOptions(ctx, provider) - responseTime := time.Since(startTime).Milliseconds() - if result.Success { - return endpointReport{true, "OPTIONS request successful", responseTime} + res, err := call(c, probeCtx) + *rt = time.Since(start).Milliseconds() + switch { + case err != nil: + *success, *msg = false, fmt.Sprintf("Endpoint failed: %v", err) + case res != nil && res.Success: + *success, *msg = true, okLabel + default: + *success, *msg = false, "Endpoint returned no content" } - return endpointReport{false, fmt.Sprintf("OPTIONS failed: %s", result.ErrorMessage), responseTime} } -func (l *LightProber) probeModelsEndpoint(ctx context.Context, provider *typ.Provider) modelsReport { - startTime := time.Now() +// runModelsEndpoint runs the /models list probe and writes its outcome (incl. +// model count and any warning) into the target fields. +func (l *LightProber) runModelsEndpoint(ctx context.Context, provider *typ.Provider, + success *bool, msg *string, rt *int64, count *int, warning *string) { + start := time.Now() + report := func(ok bool, message string, models int, warn string) { + *success, *msg, *rt, *count, *warning = ok, message, time.Since(start).Milliseconds(), models, warn + } var lister client.ModelLister - switch provider.APIStyle { case protocol.APIStyleOpenAI: c := l.pool.GetOpenAIClient(context.Background(), provider, "") - if c == nil { - return modelsReport{false, "Failed to create OpenAI client", 0, 0, ""} - } lister = c case protocol.APIStyleAnthropic: c := l.pool.GetAnthropicClient(context.Background(), provider, "") - if c == nil { - return modelsReport{false, "Failed to create Anthropic client", 0, 0, ""} - } lister = c case protocol.APIStyleGoogle: c := l.pool.GetGoogleClient(context.Background(), provider, "") - if c == nil { - return modelsReport{false, "Failed to create Google client", 0, 0, ""} - } lister = c default: - return modelsReport{false, fmt.Sprintf("Unsupported API style: %s", provider.APIStyle), 0, 0, ""} + report(false, fmt.Sprintf("Unsupported API style: %s", provider.APIStyle), 0, "") + return + } + if lister == nil { + report(false, fmt.Sprintf("Failed to create %s client", provider.APIStyle), 0, "") + return } probeCtx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() models, err := lister.ListModels(probeCtx) - responseTime := time.Since(startTime).Milliseconds() - - if client.IsModelsEndpointNotSupported(err) { - return modelsReport{ - false, - "Models endpoint not supported for this provider type", - responseTime, - 0, - "This provider does not support the models list endpoint (e.g., OAuth-based providers)", - } - } - - if err != nil { - return modelsReport{false, fmt.Sprintf("Models endpoint failed: %v", err), responseTime, 0, ""} - } - - if len(models) == 0 { - return modelsReport{false, "Models endpoint returned no models", responseTime, 0, ""} - } - - return modelsReport{ - true, - fmt.Sprintf("Models endpoint accessible - %d models found", len(models)), - responseTime, - len(models), - "", - } -} - -func (l *LightProber) probeChatEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { - startTime := time.Now() - - c := l.pool.GetOpenAIClient(context.Background(), provider, "") - if c == nil { - return endpointReport{false, "Failed to create OpenAI client", 0} - } - - probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - result, err := probeOpenAIChat(probeCtx, c, "gpt-3.5-turbo", "Hi", E2EModeSimple) - responseTime := time.Since(startTime).Milliseconds() - - if err != nil { - return endpointReport{false, fmt.Sprintf("Chat endpoint failed: %v", err), responseTime} - } - if result != nil && result.Success { - return endpointReport{true, "Chat endpoint accessible", responseTime} - } - return endpointReport{false, "Chat endpoint returned no content", responseTime} -} - -func (l *LightProber) probeResponsesEndpoint(ctx context.Context, provider *typ.Provider) endpointReport { - startTime := time.Now() - - c := l.pool.GetOpenAIClient(context.Background(), provider, "") - if c == nil { - return endpointReport{false, "Failed to create OpenAI client", 0} - } - - probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - result, err := probeOpenAIResponses(probeCtx, c, "gpt-4o", "Hi", E2EModeSimple) - responseTime := time.Since(startTime).Milliseconds() - - if err != nil { - return endpointReport{false, fmt.Sprintf("Responses endpoint failed: %v", err), responseTime} - } - if result != nil && result.Success { - return endpointReport{true, "Responses API endpoint accessible", responseTime} + switch { + case client.IsModelsEndpointNotSupported(err): + report(false, "Models endpoint not supported for this provider type", 0, + "This provider does not support the models list endpoint (e.g., OAuth-based providers)") + case err != nil: + report(false, fmt.Sprintf("Models endpoint failed: %v", err), 0, "") + case len(models) == 0: + report(false, "Models endpoint returned no models", 0, "") + default: + report(true, fmt.Sprintf("Models endpoint accessible - %d models found", len(models)), len(models), "") } - return endpointReport{false, "Responses endpoint returned no content", responseTime} } From d871a0fae9c1d5602cc81188996216062ff8b922 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 19:52:05 +0800 Subject: [PATCH 6/9] refactor(probe): extract loopbackConfigTarget shared by provider/rule/vmodel paths The three loopback resolution paths (resolveProviderTarget's loopback half, resolveRuleTarget, resolveVModelLoopbackTarget) each built the same provider_config target by hand: {Name, APIBase: loopback, APIStyle, Token: model token, Model}. Extract loopbackConfigTarget so the loopback construction lives in one place; each call site now passes only the name/apiStyle/model that actually differ. --- internal/probe/e2e_probe.go | 38 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/internal/probe/e2e_probe.go b/internal/probe/e2e_probe.go index a494779b1..fe023083c 100644 --- a/internal/probe/e2e_probe.go +++ b/internal/probe/e2e_probe.go @@ -108,13 +108,7 @@ func (e *E2EProber) resolveVModelLoopbackTarget(ctx context.Context, provider *t return nil, "", fmt.Errorf("vmodel probe unsupported for APIStyle %q", provider.APIStyle) } apiBase, apiStyle := loopbackAPIBase(port, scenario) - return e.resolveProviderConfigTarget(ctx, &E2ERequest{ - Name: provider.Name, - APIBase: apiBase, - APIStyle: string(apiStyle), - Token: e.config.GetModelToken(), - Model: model, - }) + return e.loopbackConfigTarget(ctx, provider.Name, apiBase, apiStyle, model) } func (e *E2EProber) resolveProviderTarget(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { @@ -176,13 +170,7 @@ func (e *E2EProber) resolveProviderTarget(ctx context.Context, req *E2ERequest) } logrus.Debugf("[probe-e2e] provider %s -> TB loopback %s (service pin=%s:%s)", provider.UUID, apiBase, req.ProviderUUID, model) - loopbackProvider, loopbackModel, err := e.resolveProviderConfigTarget(ctx, &E2ERequest{ - Name: provider.Name, - APIBase: apiBase, - APIStyle: string(apiStyle), - Token: e.config.GetModelToken(), - Model: model, - }) + loopbackProvider, loopbackModel, err := e.loopbackConfigTarget(ctx, provider.Name, apiBase, apiStyle, model) if err != nil { return nil, "", nil, err } @@ -248,6 +236,20 @@ func (e *E2EProber) resolveProviderConfigTarget(_ context.Context, req *E2EReque return provider, model, nil } +// loopbackConfigTarget builds a provider_config target that points at TB's own +// loopback (using the in-process model token) instead of a real upstream. The +// three loopback paths — provider, rule, and vmodel — share this construction; +// only the name, apiStyle, and model differ. +func (e *E2EProber) loopbackConfigTarget(ctx context.Context, name, apiBase string, apiStyle protocol.APIStyle, model string) (*typ.Provider, string, error) { + return e.resolveProviderConfigTarget(ctx, &E2ERequest{ + Name: name, + APIBase: apiBase, + APIStyle: string(apiStyle), + Token: e.config.GetModelToken(), + Model: model, + }) +} + func (e *E2EProber) resolveRuleTarget(ctx context.Context, req *E2ERequest) (*typ.Provider, string, map[string]string, error) { rule := e.config.GetRuleByUUID(req.RuleUUID) if rule == nil { @@ -279,13 +281,7 @@ func (e *E2EProber) resolveRuleTarget(ctx context.Context, req *E2ERequest) (*ty "X-Tingly-Debug-Routing": "1", } - provider, model, err := e.resolveProviderConfigTarget(ctx, &E2ERequest{ - Name: string(scenario), - APIBase: apiBase, - APIStyle: string(apiStyle), - Token: e.config.GetModelToken(), - Model: rule.RequestModel, - }) + provider, model, err := e.loopbackConfigTarget(ctx, string(scenario), apiBase, apiStyle, rule.RequestModel) if err != nil { return nil, "", nil, err } From e955d333f42ffa7a10dc225ef81ad6a0a0e702d6 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 19:55:19 +0800 Subject: [PATCH 7/9] fix(probe): correct "type:" typo in OpenAI bash tool JSON schema The OpenAI probe tool's function parameters used the JSON key "type:" (colon inside the key) instead of "type", so the schema field was misnamed. Aligns with the other two tool definitions in the same file. --- internal/probe/probetools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/probe/probetools.go b/internal/probe/probetools.go index b91b46578..725029825 100644 --- a/internal/probe/probetools.go +++ b/internal/probe/probetools.go @@ -52,7 +52,7 @@ func getProbeToolsOpenAI() []openai.ChatCompletionToolUnionParam { Name: "bash", Description: param.NewOpt("Execute bash commands for file system operations. Supports commands like: ls, pwd, cat, grep, find, git status, etc."), Parameters: shared.FunctionParameters{ - "type:": "object", + "type": "object", "additionalProperties": false, "properties": map[string]any{ "command": map[string]any{ From c4deed8ff2f7b26f0ed804fb64f9f5bf7cad9e38 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 20:22:51 +0800 Subject: [PATCH 8/9] fix(probe): scope usage option to streams --- internal/probe/sdk.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/probe/sdk.go b/internal/probe/sdk.go index 91294b6dd..138c50876 100644 --- a/internal/probe/sdk.go +++ b/internal/probe/sdk.go @@ -116,10 +116,6 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model params.Tools = getProbeToolsOpenAI() params.ToolChoice = openai.ChatCompletionToolChoiceOptionUnionParam{OfAuto: openai.Opt("auto")} } - // Ask for stream usage so the streaming branch can report real token counts. - // Harmless on the non-streaming path (the provider ignores stream_options). - params.StreamOptions.IncludeUsage = openai.Opt(true) - url := oc.GetProvider().APIBase + "/chat/completions" if mode == E2EModeSimple { resp, err := oc.ChatCompletionsNew(ctx, params) @@ -136,6 +132,9 @@ func probeOpenAIChat(ctx context.Context, oc client.OpenAIClientInterface, model return toProbeResult(string(b), time.Since(start).Milliseconds(), url, false, usage.FromOpenAIChatCompletion(resp.Usage), toolCalls), nil } + // stream_options is valid only for streaming requests. Ask for the final + // aggregate usage block immediately before taking the streaming path. + params.StreamOptions.IncludeUsage = openai.Opt(true) stream := oc.ChatCompletionsNewStreaming(ctx, params) if stream == nil { return nil, fmt.Errorf("chat streaming not supported by provider") From 5c94cbb88c3f97914520e5e0346b885fd0603cb0 Mon Sep 17 00:00:00 2001 From: FFengIll Date: Tue, 4 Aug 2026 20:23:00 +0800 Subject: [PATCH 9/9] docs(probe): distinguish diagnostic paths --- .design/probe.md | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/.design/probe.md b/.design/probe.md index e8ee65975..c3132912f 100644 --- a/.design/probe.md +++ b/.design/probe.md @@ -2,10 +2,22 @@ ## Overview -The probe subsystem performs SDK-level end-to-end connectivity tests for providers and rules. There are two probe strategies: +The probe subsystem provides two diagnostics for different user questions: -- **Lightweight** (`../internal/probe/light_probe.go`): HTTP-level checks (OPTIONS, `/models`, `/chat/completions`) with no SDK. Used during provider onboarding to validate credentials quickly. -- **E2E** (`../internal/probe/e2e_probe.go`): Full SDK round-trip using the same client methods as production traffic (ChatCompletionsNew, ResponsesNew, MessagesNew, GenerateContent). This catches provider quirks that only show up under the real code path. +- **Lightweight** (`../internal/probe/light_probe.go`): the Connect AI **Test Connection** path for an inline, not-yet-saved provider config. It calls the upstream directly and returns an advisory endpoint matrix: OPTIONS, models, and (for OpenAI-style providers) non-streaming Chat and Responses checks. OPTIONS is raw HTTP; models and completion checks reuse the production client/SDK methods. It does not enter `E2EProber`, TB loopback, routing, or rule evaluation. +- **E2E** (`../internal/probe/e2e_probe.go`): the Probe dialog / troubleshooting path. It performs a full SDK round-trip for a saved provider or rule, normally through TB's loopback and production routing path. This catches provider quirks and TB middleware/routing failures that a direct connectivity check cannot distinguish. + +## Product entry points + +| User surface | Frontend call | HTTP endpoint | Backend strategy | Question answered | +|--------------|---------------|---------------|------------------|-------------------| +| Connect AI → Test Connection | `runProviderProbe` → `api.probeProviderLightweight` | `POST /api/v2/probe/lightweight` | `LightProber` | Are these credentials/endpoints reachable enough to continue? | +| Probe dialog / Troubleshoot | `runProbe` | `POST /api/v2/probe` | `E2EProber` | Does a real request work, how did it route, and what came back? | + +The similarly named frontend helper `api.probeProvider` targets the E2E +`provider_config` API, but Connect AI does not call it. Keep the two paths +distinct: onboarding connectivity is advisory and direct; troubleshooting must +exercise the real TB path. ## E2E Target Types @@ -15,7 +27,7 @@ An `E2ERequest` has three `target_type` values: |-------------------|------------------------------------------------------------------------------| | `provider` | A saved provider record by UUID, pinned to a specific model | | `rule` | A rule by UUID — exercises all TB middleware for that rule's scenario | -| `provider_config` | An inline provider config (name, api_base, api_style, token) — used during onboarding before the provider is saved | +| `provider_config` | An inline provider config for direct E2E API callers; it does not represent Connect AI's Test Connection path | ### Direct vs Through-TB (provider probes) @@ -36,7 +48,11 @@ When a through-TB probe fails and a direct probe succeeds, the problem is in TB' |-------------|-----------------------------------------------------| | `simple` | Single non-streaming completion | | `streaming` | Streaming completion (SSE) | -| `tool` | Completion with a tool definition + auto tool choice | +| `tool` | Streaming completion with a tool definition + auto tool choice; tool events remain in the raw chunk array | + +Tool mode intentionally preserves the SDK's raw streamed chunks in `content`. +It does not assemble a parallel normalized tool-call result; the raw response is +the diagnostic artifact. ## TB Loopback Pattern @@ -144,9 +160,9 @@ internal/probe/ probeProviderWithSDK, applyRoutingCapture sdk.go — SDK dispatch helpers (probeOpenAIChat, probeOpenAIResponses, probeAnthropicMessages, probeGoogleGenerate, probeOptions) and - the per-provider usage (via internal/protocol/usage) + tool-call - extractors - light_probe.go — LightProber (HTTP-level connectivity matrix, no SDK usage) + per-provider usage extraction (via internal/protocol/usage) + light_probe.go — LightProber (direct advisory connectivity matrix; reuses + SDK clients for models/chat/responses) probetools.go — Tool definitions used by E2EModeTool endpoint_probe_cache.go — narrow direct-endpoint capability cache @@ -181,8 +197,11 @@ internal/server/ `cache_write_tokens` / `reasoning_tokens`) — the same vocabulary the rest of TB emits and the frontend renders; there are no parallel flat token fields. `nil` for Google (out of scope) and cache hits. -- `tool_calls` — tool calls lifted out of the response (tool mode), one entry per - function call with `{id, name, input}`. +- OpenAI Chat requests set `stream_options.include_usage` only on the streaming + branch. It is a stream-only parameter and must not be sent by non-streaming or + lightweight Chat checks because strict OpenAI-compatible providers may reject it. +- Tool-mode calls are represented in the raw streamed `content`; `tool_calls` is + not assembled as a second representation. - Routing trace fields (see below) — populated for TB-loopback probes only. ## Trade-offs and constraints