diff --git a/docs/WORKBENCH_ASSISTANT.md b/docs/WORKBENCH_ASSISTANT.md index d55e565..bf433b2 100644 --- a/docs/WORKBENCH_ASSISTANT.md +++ b/docs/WORKBENCH_ASSISTANT.md @@ -20,6 +20,10 @@ python -m quant_agent.workbench_assistant --context reviewed-context.json --outp 问题最多4000字，证据最多30条、每条6000字，总JSON最多60KB；重复ID、未知字段、哈希不匹配均拒绝。 模型仅返回`findings`与`next_steps`。每条finding包含`text`、`citations`、`quotes`。 +在线请求显式使用`response_format={"type":"json_object"}`，需要支持该Chat Completions参数的供应商或兼容网关。 +JSON模式仍可能只是网关的提示模拟；客户端继续独立校验，格式失败不自动重试。 +两类助手都接受完整JSON对象，或整个响应仅包含一个`json`/无语言标记代码围栏的JSON对象。 +围栏外的解释文字、多个围栏、额外JSON对象和截断JSON均拒绝；原始响应在去除空白或围栏前检查大小。 引用必须来自输入，逐字片段必须存在于引用来源；无效JSON、超长内容或伪造引用使调用失败，不降级伪装成有效模型结果。 CLI只创建新输出，不覆盖旧建议。失败返回1，日志保留错误类型，不输出可能包含授权头和证据内容的供应商异常全文。 建议仍是待验证判断，引用存在不等于推理正确，也不证明交易策略有效。 diff --git a/src/quant_agent/notebook_advisor.py b/src/quant_agent/notebook_advisor.py index 6905aa0..2972e6e 100644 --- a/src/quant_agent/notebook_advisor.py +++ b/src/quant_agent/notebook_advisor.py @@ -4,7 +4,12 @@ import json import os -from quant_agent.workbench_assistant import advise, validate_answer, validate_context +from quant_agent.workbench_assistant import ( + advise, + parse_model_json, + validate_answer, + validate_context, +) def propose_notebook(context, notebook, *, model=None, invoke=None): @@ -51,12 +56,13 @@ def propose_notebook(context, notebook, *, model=None, invoke=None): raw = ( ChatOpenAI(model=model, temperature=0, timeout=60, max_retries=0) - .invoke([SystemMessage(content=system), HumanMessage(content=serialized)]) + .invoke( + [SystemMessage(content=system), HumanMessage(content=serialized)], + response_format={"type": "json_object"}, + ) .content ) - if not isinstance(raw, str) or len(raw.encode()) > 120_000: - raise ValueError("Invalid model response") - answer = json.loads(raw) + answer = parse_model_json(raw, max_bytes=120_000) if set(answer) != {"findings", "next_steps", "patches"}: raise ValueError("Unexpected proposal fields") validate_answer({k: answer[k] for k in ("findings", "next_steps")}, context) diff --git a/src/quant_agent/workbench_assistant.py b/src/quant_agent/workbench_assistant.py index 26af30d..c0e5bf5 100644 --- a/src/quant_agent/workbench_assistant.py +++ b/src/quant_agent/workbench_assistant.py @@ -76,6 +76,22 @@ def validate_answer(answer, context): return answer +def parse_model_json(response, *, max_bytes): + """Parse one complete object, optionally enclosed in one whole JSON code fence.""" + if not isinstance(response, str) or len(response.encode("utf-8")) > max_bytes: + raise ValueError("Invalid or oversized model response") + text = response.strip() + if text.startswith("```"): + fence = re.fullmatch(r"```(?:json)?[ \t]*\r?\n(.*)\r?\n```", text, re.DOTALL) + if fence is None: + raise ValueError("Expected one complete JSON code fence") + text = fence.group(1) + answer = json.loads(text) + if not isinstance(answer, dict): + raise TypeError("Expected model response object") + return answer + + def advise(context, *, model=None, invoke=None): context = validate_context(context) if not model: @@ -113,12 +129,13 @@ def advise(context, *, model=None, invoke=None): response = ( ChatOpenAI(model=model, temperature=0, timeout=60, max_retries=0) - .invoke([SystemMessage(content=system), HumanMessage(content=request)]) + .invoke( + [SystemMessage(content=system), HumanMessage(content=request)], + response_format={"type": "json_object"}, + ) .content ) - if not isinstance(response, str) or len(response.encode()) > 60000: - raise ValueError("Invalid or oversized model response") - answer = validate_answer(json.loads(response), context) + answer = validate_answer(parse_model_json(response, max_bytes=60000), context) return { **answer, "mode": "model_assisted", diff --git a/tests/test_model_json_boundary.py b/tests/test_model_json_boundary.py new file mode 100644 index 0000000..0a2ae8a --- /dev/null +++ b/tests/test_model_json_boundary.py @@ -0,0 +1,162 @@ +"""Public assistant boundaries accept one complete JSON result, never extract prose.""" + +import hashlib +import json + +import pytest + +from quant_agent.notebook_advisor import propose_notebook +from quant_agent.workbench_assistant import advise + + +def request(): + text = "Synthetic fixture is missing volume." + return { + "question": "What input is missing?", + "evidence": [ + {"id": "E1", "text": text, "sha256": hashlib.sha256(text.encode()).hexdigest()} + ], + } + + +def answer(notebook): + value = { + "findings": [{"text": "Check volume.", "citations": ["E1"], "quotes": ["missing volume"]}], + "next_steps": ["Inspect the input schema."], + } + if notebook: + value["patches"] = [] + return value + + +def call(notebook, **kwargs): + if notebook: + return propose_notebook(request(), {"nbformat": 4, "cells": []}, **kwargs) + return advise(request(), **kwargs) + + +@pytest.mark.parametrize("notebook", [False, True], ids=["workbench", "notebook"]) +@pytest.mark.parametrize("wrapper", ["{}", " \n```json\n{}\n```\t", "```\r\n{}\r\n```"]) +def test_accepts_only_complete_json_result(monkeypatch, notebook, wrapper): + monkeypatch.setenv("QUANT_AGENT_LLM_OK", "1") + expected = answer(notebook) + raw = wrapper.format(json.dumps(expected)) + report = call(notebook, model="test", invoke=lambda *_: raw) + assert report["findings"] == expected["findings"] + assert report["next_steps"] == expected["next_steps"] + assert report["model_invoked"] is False and report["provider"] == "injected_test" + if notebook: + assert report["patches"] == [] + + +@pytest.mark.parametrize("notebook", [False, True], ids=["workbench", "notebook"]) +@pytest.mark.parametrize( + "kind", + [ + "prose", + "suffix", + "two_fences", + "two_objects", + "malformed", + "unclosed", + "wrong_language", + "oversized", + "multibyte_oversized", + "non_object", + "non_text", + "unknown_citation", + "invented_quote", + ], +) +def test_invalid_result_fails_without_retry(monkeypatch, notebook, kind): + monkeypatch.setenv("QUANT_AGENT_LLM_OK", "1") + value = answer(notebook) + if kind == "unknown_citation": + value["findings"][0]["citations"] = ["E2"] + elif kind == "invented_quote": + value["findings"][0]["quotes"] = ["profitable"] + raw = json.dumps(value) + fenced = "```json\n" + raw + "\n```" + invalid = { + "prose": "Here is the answer:\n" + fenced, + "suffix": fenced + "\nPlease run this.", + "two_fences": fenced + "\n" + fenced, + "two_objects": raw + raw, + "malformed": '```json\n{"findings":\n```', + "unclosed": "```json\n" + raw, + "wrong_language": "```python\n" + raw + "\n```", + # A valid body does not permit bypassing the original raw-byte limit with whitespace. + "oversized": " " * (120_000 if notebook else 60_000) + fenced, + "multibyte_oversized": "中" * (40_000 if notebook else 20_000) + fenced, + "non_object": "```json\n[]\n```", + "non_text": {"text": fenced}, + "unknown_citation": fenced, + "invented_quote": fenced, + }[kind] + calls = [] + + def invoke(*_): + calls.append(True) + return invalid + + with pytest.raises(TypeError if kind == "non_object" else ValueError): + call(notebook, model="test", invoke=invoke) + assert calls == [True] + + +@pytest.mark.parametrize("notebook", [False, True], ids=["workbench", "notebook"]) +def test_real_sdk_requests_json_mode_and_keeps_local_validation(monkeypatch, notebook): + import httpx + import langchain_openai + + monkeypatch.setenv("QUANT_AGENT_LLM_OK", "1") + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-test-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://synthetic-provider.invalid/v1") + monkeypatch.setenv("LANGSMITH_TRACING", "false") + monkeypatch.setenv("LANGCHAIN_TRACING_V2", "false") + expected = answer(notebook) + captured, requests = {}, [] + actual_client = langchain_openai.ChatOpenAI + + def transport(request): + assert request.url.path == "/v1/chat/completions" + requests.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "synthetic-completion", + "object": "chat.completion", + "created": 0, + "model": "synthetic-model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": "```json\n" + json.dumps(expected) + "\n```", + }, + } + ], + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport), trust_env=False) as client: + + def factory(**kwargs): + captured.update(kwargs) + return actual_client(**kwargs, http_client=client) + + monkeypatch.setattr(langchain_openai, "ChatOpenAI", factory) + report = call(notebook, model="synthetic-model") + assert captured == { + "model": "synthetic-model", + "temperature": 0, + "timeout": 60, + "max_retries": 0, + } + assert len(requests) == 1 + assert requests[0]["response_format"] == {"type": "json_object"} + assert "tools" not in requests[0] + assert report["findings"] == expected["findings"] + assert report["model_invoked"] is True and report["provider"] == "configured_provider" diff --git a/tests/test_workbench_assistant.py b/tests/test_workbench_assistant.py index f25ef7a..4cbe460 100644 --- a/tests/test_workbench_assistant.py +++ b/tests/test_workbench_assistant.py @@ -108,9 +108,10 @@ class FakeClient: def __init__(self, **kwargs): captured.update(kwargs) - def invoke(self, messages): + def invoke(self, messages, **kwargs): assert len(messages) == 2 assert json.loads(messages[1].content) == context() + assert kwargs == {"response_format": {"type": "json_object"}} return SimpleNamespace(content=json.dumps(answer)) module.ChatOpenAI = FakeClient