diff --git a/poetry.lock b/poetry.lock index 9c0f403c..7361c8af 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,15 +1,15 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. [[package]] name = "aidial-client" -version = "0.8.0" +version = "0.10.0" description = "A Python client library for the AI DIAL API" optional = false python-versions = "<3.14,>=3.10" groups = ["main"] files = [ - {file = "aidial_client-0.8.0-py3-none-any.whl", hash = "sha256:f3e558247925d209bba663d78f9e1544c4119827de910ace9d0699733f61c7f6"}, - {file = "aidial_client-0.8.0.tar.gz", hash = "sha256:e6559669adb2bd3e9a714a6bc7eb90ee3acde6b49d121fb89362ea9f043afbb6"}, + {file = "aidial_client-0.10.0-py3-none-any.whl", hash = "sha256:eaa13b1a18059969f958d1dace4dd9c6d09d38e3b903644696f5d3a3f71a7c2c"}, + {file = "aidial_client-0.10.0.tar.gz", hash = "sha256:748f5cdbe7fa1747ed2db629280ba88d20a1619208b1f4eb8d31de216d36fb8a"}, ] [package.dependencies] @@ -5797,4 +5797,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.13,<3.14" -content-hash = "b4f6183708389002ba75716060857669bb2f4253c11030f9808233ac0b3bd517" +content-hash = "a92627a5e054d30610ca0ead73faabbca4f275e0e6d69d4c16fcfbf8d891360b" diff --git a/pyproject.toml b/pyproject.toml index 1be581af..e7e2301f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] dependencies = [ # Core framework & DI - "aidial-client (>=0.8.0,<0.9.0)", # DIAL API client + "aidial-client (>=0.10.0,<0.11.0)", # DIAL API client "aidial-sdk[telemetry]>=0.32.0,<0.33.0", # DIAL integration SDK "pydantic>=2.12.4,<3.0.0", # Data validation "pydantic-settings>=2.12.0,<3.0.0", # Settings management diff --git a/src/quickapp/common/json_schema_converter.py b/src/quickapp/common/json_schema_converter.py index eb510329..babdb851 100644 --- a/src/quickapp/common/json_schema_converter.py +++ b/src/quickapp/common/json_schema_converter.py @@ -1,7 +1,7 @@ from typing import Any from fastmcp.utilities.json_schema import dereference_refs -from jsonref import replace_refs # type: ignore[import-untyped] +from jsonref import JsonRefError, replace_refs # type: ignore[import-untyped] from quickapp.config.tools.base import ( ConfigurableSchemaArray, @@ -22,6 +22,23 @@ class JsonSchemaConverter: to jsonref.replace_refs() for schemas with circular references. """ + @staticmethod + def _contains_ref(obj: Any) -> bool: + if isinstance(obj, dict): + if "$ref" in obj: + return True + return any(JsonSchemaConverter._contains_ref(v) for v in obj.values()) + if isinstance(obj, list): + return any(JsonSchemaConverter._contains_ref(v) for v in obj) + return False + + @staticmethod + def _dereference_with_jsonref(schema_dict: dict[str, Any]) -> dict[str, Any]: + dereferenced = replace_refs(schema_dict, proxies=False, lazy_load=False) + if not isinstance(dereferenced, dict): + raise TypeError(f"Expected dict from replace_refs, got {type(dereferenced).__name__}") + return dereferenced + @staticmethod def _normalize_type(type_field: str | list[str] | None) -> str | None: """ @@ -206,14 +223,15 @@ def convert_schema_to_properties( try: schema_dict = dereference_refs(schema_dict) except RecursionError: - # Circular inline $ref (e.g. "#/properties/node") causes RecursionError - # in fastmcp 3.x's _strip_discriminator. Fall back to bare jsonref which - # produces circular Python dicts that _convert_properties handles via id(). - dereferenced = replace_refs(schema_dict, proxies=False, lazy_load=False) - if not isinstance(dereferenced, dict): - raise TypeError( - f"Expected dict from replace_refs, got {type(dereferenced).__name__}" - ) - schema_dict = dereferenced + # Circular inline $ref can blow the stack in fastmcp post-processing. + schema_dict = JsonSchemaConverter._dereference_with_jsonref(schema_dict) + else: + if JsonSchemaConverter._contains_ref(schema_dict): + # fastmcp may leave inline circular $ref unresolved; jsonref materializes + # circular Python dicts that _convert_properties handles via id(). + try: + schema_dict = JsonSchemaConverter._dereference_with_jsonref(schema_dict) + except JsonRefError: + pass # broken refs -> ValueError during conversion return JsonSchemaConverter._convert_properties(schema_dict) diff --git a/src/tests/integration_tests/test_documents/integration_context_items.txt b/src/tests/integration_tests/test_documents/integration_context_items.txt new file mode 100644 index 00000000..c1bf6d53 --- /dev/null +++ b/src/tests/integration_tests/test_documents/integration_context_items.txt @@ -0,0 +1,7 @@ +Admin-configured integration test context. + +The three items are: + +- AlphaReader +- BetaIndexer +- GammaMapper diff --git a/src/tests/integration_tests/test_orchestrator_contexts.py b/src/tests/integration_tests/test_orchestrator_contexts.py index ae892a77..f5c680d8 100644 --- a/src/tests/integration_tests/test_orchestrator_contexts.py +++ b/src/tests/integration_tests/test_orchestrator_contexts.py @@ -21,9 +21,9 @@ @pytest.mark.integration @e2e_test( - config_file_set="lazy_admin_context", + config_file_set="no_additional_tools", models_applicable_for_test=_LAZY_CONTEXT_MODELS, - runs=1, + runs=2, include_rest_toolset=False, application_context_files=[ _DOCS / "ontologies.pdf", @@ -36,12 +36,36 @@ similarity_threshold=0.8, ) .add_user_message( - user_message="enlist all tools that you have", - answer=[ - "I have access to internal_attachments_available_context and " - "internal_attachments_get_content tools for listing and loading admin context files.", - "Available tools: internal_attachments_available_context (lists admin files) and " - "internal_attachments_get_content (loads a specific file).", + user_message="enlist tool names that you have", + answer=[""" +Tool names I have access to (with what they do): + +- `functions.internal_attachments_available_context` — list admin-configured context attachments available to load +- `functions.internal_attachments_get_content` — load one attachment by its provided `url` +- `functions.internal_skills_read_skill` — read the full content of an agent skill/instructions + +DIAL file storage tools: + +- `functions.internal_file_list` — list files/folders under a path (optionally recursive to a depth) +- `functions.internal_file_find` — find files by glob pattern (metadata-only walk) +- `functions.internal_file_read_lines` — read specific line ranges from a text file +- `functions.internal_file_search` — search for a substring across a file or folder tree +- `functions.internal_file_write` — create a new UTF-8 text file +- `functions.internal_file_edit` — replace a unique substring in an existing UTF-8 text file +- `functions.internal_file_copy` — copy a file server-side +- `functions.internal_file_move` — move/rename a file +- `functions.internal_file_delete` — delete a file + +Parallel wrapper: + +- `multi_tool_use.parallel` — run multiple tool calls in parallel (only for the developer-defined tools above) + +If you tell me what you want to do (e.g., “search all files for X”, “read a PDF attachment”, “create/edit a report file”), I’ll suggest the best tool(s) to use. +"""], + tool_calls=[ + ToolCall( + ToolNames.INTERNAL_ATTACHMENTS_AVAILABLE_CONTEXT.value, min_calls=0, max_calls=4 + ), ], ) .add_user_message( @@ -71,6 +95,15 @@ 8. TopBraid Composer 9. OntoGraf 10. VOWL""", + """ + 1. Protégé + 2. Topbraid Composer + 3. Ontostudio + 4. Fluent Editor + 5. VocBench + 6. Swoop + 7. Obo-edit + """, ], ), ) diff --git a/src/tests/integration_tests/test_runner/app_test_module.py b/src/tests/integration_tests/test_runner/app_test_module.py index 3c9b95d8..2dc22d86 100644 --- a/src/tests/integration_tests/test_runner/app_test_module.py +++ b/src/tests/integration_tests/test_runner/app_test_module.py @@ -22,6 +22,9 @@ _PyInterpreterSettings, ) from quickapp.mcp_tooling import MCPToolingModule +from quickapp.orchestrator_attachment_strategies.lazy_on_demand.lazy_on_demand_strategy_module import ( + LazyOnDemandStrategyModule, +) from quickapp.predefined_tooling import PredefinedToolingModule from quickapp.rest_api_tooling import RestApiToolingModule from quickapp.shared import shared_module @@ -87,6 +90,7 @@ def get_app(cls, port: int = 8081): SkillsModule(), DialPromptSkillsModule(), DialFilesToolingModule(), + LazyOnDemandStrategyModule(), ] ) dial_settings = DialSettings(url=TestConfig.get_mock_dial_core_url(port)) diff --git a/src/tests/integration_tests/test_runner/cache/gpt-5.2-2025-12-11/test_multi_tool/Image recognition and Web search/927a5c9b28511451d627ee23b611a23e.response b/src/tests/integration_tests/test_runner/cache/gpt-5.2-2025-12-11/test_multi_tool/Image recognition and Web search/927a5c9b28511451d627ee23b611a23e.response new file mode 100644 index 00000000..b410ad79 --- /dev/null +++ b/src/tests/integration_tests/test_runner/cache/gpt-5.2-2025-12-11/test_multi_tool/Image recognition and Web search/927a5c9b28511451d627ee23b611a23e.response @@ -0,0 +1,30 @@ +{ + "body_human_readable": "![The Hitchhiker's Guide to the Galaxy (2005) theatrical release poster](https://upload.wikimedia.org/wikipedia/en/7/73/A_hitchhikers_guide_to_the_galaxy.jpg)\n\nThe 2005 science fiction comedy \"The Hitchhiker's Guide to the Galaxy\" was produced with a budget of approximately $50 million. The film went on to achieve a worldwide box office gross of about $104.5 million.\n\nThe movie's domestic run in the United States and Canada contributed $51.1 million to its total gross. In its opening weekend in North America, the film secured the number-one spot, earning $21.1 million. The international box office added another $53.4 million. While the film was met with a lukewarm reception from some critics, it more than doubled its production budget at the worldwide box office.", + "request": { + "system_message": null, + "model": "gemini-2.5-pro-google-search", + "temperature": null, + "user_message": [ + "sci-fi movie \"42\" robot poster hitchhiker's guide to the galaxy 42 budget worldwide gross", + "The Hitchhiker's Guide to the Galaxy (2005) budget worldwide gross" + ], + "assistant_message": [ + "## Film \"42\" A Box Office Success\n\nThe 2013 biographical sports drama \"42,\" which tells the story of baseball legend Jackie Robinson, proved to be a financial success, grossing a worldwide total of $97.5 million against a production budget of $40 million.\n\nThe film, starring Chadwick Boseman as Robinson and Harrison Ford as Brooklyn Dodgers general manager Branch Rickey, brought in $95 million in the United States and an additional $2.5 million from other territories. The movie's strong domestic performance was highlighted by a record-breaking opening weekend for a baseball film, earning over $27 million. Some sources report a slightly lower production budget of $31 million. Regardless of the exact budget figure, the film was profitable, with a worldwide box office that more than doubled its production cost." + ] + }, + "response": { + "status_code": 200, + "headers": { + "access-control-allow-origin": "*", + "date": "Mon, 22 Jun 2026 09:26:16 GMT", + "server": "uvicorn", + "content-type": "text/event-stream; charset=utf-8", + "x-accel-buffering": "no", + "x-upstream-attempts": "1", + "content-encoding": "gzip", + "transfer-encoding": "chunked" + } + }, + "body": "data: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"content\":\"![The Hitchhiker's Guide to the Galaxy (2005) theatrical release poster](https://\"},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"content\":\"upload.wikimedia.org/wikipedia/en/7/73/A_hitchhikers_guide_to_the_galaxy.jpg)\\n\\nThe 2005 science fiction comedy \\\"The Hitchhiker's Guide to\"},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"content\":\" the Galaxy\\\" was produced with a budget of approximately $50 million. The film went on to achieve a worldwide box office gross of about $104.5 million.\\n\\nThe movie's domestic run in the United States and Canada contributed $51.1 million to its total gross. In its opening weekend in North America, the film secured the number-one spot, earning $21.1 million.\"},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"content\":\" The international box office added another $53.4 million. While the film was met with a lukewarm reception from some critics, it more than doubled its production budget at the worldwide box office.\"},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":0,\"type\":\"text/markdown\",\"title\":\"framerated.co.uk\",\"data\":\"The 2005 science fiction comedy \\\"The Hitchhiker's Guide to the Galaxy\\\" was produced with a budget of approximately $50 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGhE3jSpvaUP1qNSTFmMTx9BrevNDKy-mGe_1pl18lVpyutZEmYBoCT3PN1cOrU_APXuyK_hZExgVK0fBjxGKNdNhJsthCRq2MW_pQQKZ-G7__ZKw05PRLFZJOpMg9XUTzMSb2gnuQeJeM7WjO4H9xf7xzpOnEc\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":1,\"type\":\"text/markdown\",\"title\":\"grokipedia.com\",\"data\":\"The 2005 science fiction comedy \\\"The Hitchhiker's Guide to the Galaxy\\\" was produced with a budget of approximately $50 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGABEwbZR_lMMGaMYKuYcTcAbBd5WLTBhZt7v1kumWiF_Eky8Q4s4Mi2qgVafV0DsWDQ7nnLFZQeKVv9EBlXm3WJy-myGgB1_ooY7O8EPA-HNHLXQK3FsIejZtnw2i6cwB8YFqhhPlxrYyfWfLLP9JnxdSxy5t9YlnUcl_qZ3zB7tDx\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":2,\"type\":\"text/markdown\",\"title\":\"ranker.com\",\"data\":\"The 2005 science fiction comedy \\\"The Hitchhiker's Guide to the Galaxy\\\" was produced with a budget of approximately $50 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGDCTMaKvDO-sgGccDkgrVpO0KR5Ie_4pLiGe4DPuH5eePaPjGjhpbC2aQWmDnWndlUB5zuDO1z35S0ovU9vCT75G4vvz-UW_UN9_otgxHB6TPm-7omTXUmCIM26aS4o_ZIFT_h3BM374SaeD3fyJJ3tw==\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":3,\"type\":\"text/markdown\",\"title\":\"wikipedia.org\",\"data\":\"The film went on to achieve a worldwide box office gross of about $104.5 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQFvbNUlEJ7H0MfRaWxa1PnPlaeAEUCPFQGFue5HbzuLIA3j83GI5Dg6WqPe2ibm-B7p7qTjF4V5gvDNYw8Go9KAlMp4vXy8zvhKpL6lJSxYDFeTFaDolhlI-iJtYeLOt3APZaYQ8BmqC3I6E2KmoptMVijFeO2j0yAvjP82wJ68VU83f_ePUw==\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":4,\"type\":\"text/markdown\",\"title\":\"framerated.co.uk\",\"data\":\"The film went on to achieve a worldwide box office gross of about $104.5 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGhE3jSpvaUP1qNSTFmMTx9BrevNDKy-mGe_1pl18lVpyutZEmYBoCT3PN1cOrU_APXuyK_hZExgVK0fBjxGKNdNhJsthCRq2MW_pQQKZ-G7__ZKw05PRLFZJOpMg9XUTzMSb2gnuQeJeM7WjO4H9xf7xzpOnEc\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":5,\"type\":\"text/markdown\",\"title\":\"grokipedia.com\",\"data\":\"The film went on to achieve a worldwide box office gross of about $104.5 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGABEwbZR_lMMGaMYKuYcTcAbBd5WLTBhZt7v1kumWiF_Eky8Q4s4Mi2qgVafV0DsWDQ7nnLFZQeKVv9EBlXm3WJy-myGgB1_ooY7O8EPA-HNHLXQK3FsIejZtnw2i6cwB8YFqhhPlxrYyfWfLLP9JnxdSxy5t9YlnUcl_qZ3zB7tDx\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":6,\"type\":\"text/markdown\",\"title\":\"grokipedia.com\",\"data\":\"The movie's domestic run in the United States and Canada contributed $51.1 million to its total gross.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGABEwbZR_lMMGaMYKuYcTcAbBd5WLTBhZt7v1kumWiF_Eky8Q4s4Mi2qgVafV0DsWDQ7nnLFZQeKVv9EBlXm3WJy-myGgB1_ooY7O8EPA-HNHLXQK3FsIejZtnw2i6cwB8YFqhhPlxrYyfWfLLP9JnxdSxy5t9YlnUcl_qZ3zB7tDx\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":7,\"type\":\"text/markdown\",\"title\":\"framerated.co.uk\",\"data\":\"In its opening weekend in North America, the film secured the number-one spot, earning $21.1 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGhE3jSpvaUP1qNSTFmMTx9BrevNDKy-mGe_1pl18lVpyutZEmYBoCT3PN1cOrU_APXuyK_hZExgVK0fBjxGKNdNhJsthCRq2MW_pQQKZ-G7__ZKw05PRLFZJOpMg9XUTzMSb2gnuQeJeM7WjO4H9xf7xzpOnEc\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":8,\"type\":\"text/markdown\",\"title\":\"grokipedia.com\",\"data\":\"In its opening weekend in North America, the film secured the number-one spot, earning $21.1 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGABEwbZR_lMMGaMYKuYcTcAbBd5WLTBhZt7v1kumWiF_Eky8Q4s4Mi2qgVafV0DsWDQ7nnLFZQeKVv9EBlXm3WJy-myGgB1_ooY7O8EPA-HNHLXQK3FsIejZtnw2i6cwB8YFqhhPlxrYyfWfLLP9JnxdSxy5t9YlnUcl_qZ3zB7tDx\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":9,\"type\":\"text/markdown\",\"title\":\"grokipedia.com\",\"data\":\"The international box office added another $53.4 million.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGABEwbZR_lMMGaMYKuYcTcAbBd5WLTBhZt7v1kumWiF_Eky8Q4s4Mi2qgVafV0DsWDQ7nnLFZQeKVv9EBlXm3WJy-myGgB1_ooY7O8EPA-HNHLXQK3FsIejZtnw2i6cwB8YFqhhPlxrYyfWfLLP9JnxdSxy5t9YlnUcl_qZ3zB7tDx\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"custom_content\":{\"attachments\":[{\"index\":10,\"type\":\"text/markdown\",\"title\":\"framerated.co.uk\",\"data\":\"While the film was met with a lukewarm reception from some critics, it more than doubled its production budget at the worldwide box office.\",\"reference_url\":\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGhE3jSpvaUP1qNSTFmMTx9BrevNDKy-mGe_1pl18lVpyutZEmYBoCT3PN1cOrU_APXuyK_hZExgVK0fBjxGKNdNhJsthCRq2MW_pQQKZ-G7__ZKw05PRLFZJOpMg9XUTzMSb2gnuQeJeM7WjO4H9xf7xzpOnEc\",\"reference_type\":\"text/markdown\"}]}},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{\"content\":\"\"},\"finish_reason\":null,\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":null}\n\ndata: {\"id\":\"02fc83cf-a312-4115-88a4-fca7c40f3818\",\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\",\"index\":0}],\"created\":1782120378,\"model\":\"gemini-2.5-pro\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":278,\"prompt_tokens\":218,\"total_tokens\":496}}\n\ndata: [DONE]\n\n", + "body_encoded": "" +} \ No newline at end of file diff --git a/src/tests/integration_tests/test_runner/config.py b/src/tests/integration_tests/test_runner/config.py index 1f5ed57d..80e17cab 100644 --- a/src/tests/integration_tests/test_runner/config.py +++ b/src/tests/integration_tests/test_runner/config.py @@ -2,7 +2,6 @@ import logging import os from enum import Enum -from pathlib import Path from pydantic import SecretStr from pydantic.type_adapter import TypeAdapter @@ -13,7 +12,9 @@ from quickapp.config.dial_files import DialFilesConfig from quickapp.config.orchestrator_attachment_strategy import LazyOnDemandAttachmentStrategy from quickapp.config.prompt import PredefinedSystemPromptConfig +from quickapp.config.skill import DialPromptSkillConfig from quickapp.config.toolsets.toolset import ToolSet +from tests.integration_tests.test_runner.paths import TOOLSETS_DIR from tests.integration_tests.test_runner.test_tool_set_rest import TestToolSetRest logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ "integration": ["test_tool_set_chat_hub", "test_tool_set_py_interpreter", "test_mcp_tool"], "integration_simple": ["test_tool_set_chat_hub"], "e2e": ["test_tool_set_chat_hub", "test_tool_set_py_interpreter"], - "lazy_admin_context": [], + "no-additional-tools": [], } @@ -65,6 +66,7 @@ def create_app_configuration( cls, toolsets: list[ToolSet], model: str, + skill_urls: list[str] | None = None, contexts: list[Context] | None = None, attachment_strategy: LazyOnDemandAttachmentStrategy | None = None, ) -> ApplicationConfig: @@ -78,6 +80,8 @@ def create_app_configuration( temperature = 1 template = "gpt_prompt" + skills = [DialPromptSkillConfig(url=url) for url in skill_urls] if skill_urls else None + return ApplicationConfig( orchestrator=OrchestratorConfig( deployment=DialDeploymentConfig( @@ -87,8 +91,9 @@ def create_app_configuration( system_prompt=PredefinedSystemPromptConfig(template=template), attachment_strategy=attachment_strategy, ), - contexts=list(contexts or []), + contexts=contexts or [], tool_sets=toolsets, + skills=skills, features=Features(dial_files=DialFilesConfig()), ) @@ -102,7 +107,7 @@ def load_tools_config( files_list = file_sets.get(config_file_set) or [] tool_set_list: list[ToolSet] = [] for file in files_list: - file_path = Path(__file__).parent / f"{file}.json" + file_path = TOOLSETS_DIR / f"{file}.json" data = json.loads(file_path.read_text()) tool_set: ToolSet = TypeAdapter(ToolSet).validate_python(data) tool_set_list.append(tool_set) diff --git a/src/tests/integration_tests/test_runner/e2e_runner.py b/src/tests/integration_tests/test_runner/e2e_runner.py index 941cd07a..21e58ff1 100644 --- a/src/tests/integration_tests/test_runner/e2e_runner.py +++ b/src/tests/integration_tests/test_runner/e2e_runner.py @@ -3,22 +3,27 @@ import json import logging import mimetypes +import re from pathlib import Path from typing import Any import pytest import uvicorn -from aidial_client import AsyncDial +from aidial_client import AsyncDial, DialException, ResourceNotFoundError +from aidial_client.types.prompt import Prompt from aidial_sdk.chat_completion import Message, Role from pydantic import SecretStr from starlette.testclient import TestClient from quickapp.config.application import ApplicationConfig -from quickapp.config.context import FileContextConfig +from quickapp.config.context import Context, FileContextConfig from quickapp.config.logging_config import LoggingConfig from quickapp.config.logging_settings import LoggingSettings from quickapp.config.orchestrator_attachment_strategy import LazyOnDemandAttachmentStrategy from quickapp.config.utils import bool_env_var +from quickapp.skills._frontmatter import ( # test harness: same parser as production skills + parse_frontmatter, +) from tests.integration_tests.conftest import FailureReason, TestStats, report_test_stats from tests.integration_tests.test_runner.app_test_module import TestApp from tests.integration_tests.test_runner.cache.cache_middleware import ( @@ -26,7 +31,14 @@ CacheMiddlewareConfig, ) from tests.integration_tests.test_runner.config import TestConfig, TestDialCoreConfig -from tests.integration_tests.test_runner.models import Failure, TstCase, check_multiple_alternatives +from tests.integration_tests.test_runner.models import ( + Failure, + SkillFileConfig, + TestContextConfig, + TstCase, + check_multiple_alternatives, +) +from tests.integration_tests.test_runner.paths import SKILLS_DIR from tests.integration_tests.test_runner.utils.string_utils import extract_total_price from tests.integration_tests.test_runner.validators import ResponseValidator @@ -49,6 +61,11 @@ def extract_usage(response_message): API_KEY_HEADER = "Api-Key" CONTENT_TYPE_HEADER = "Content-Type" +INTEGRATION_PROMPTS_FOLDER = "quickapps-integration-tests" + + +def _create_request_headers(api_key: SecretStr) -> dict[str, str]: + return {API_KEY_HEADER: api_key.get_secret_value(), CONTENT_TYPE_HEADER: "application/json"} def _dial_relative_file_url(url: str) -> str: @@ -60,16 +77,34 @@ def _dial_relative_file_url(url: str) -> str: return s -def create_request_headers(api_key: SecretStr, app_config: ApplicationConfig) -> dict[str, str]: - return { - API_KEY_HEADER: api_key.get_secret_value(), - CONTENT_TYPE_HEADER: "application/json", - "X-DIAL-APPLICATION-PROPERTIES": app_config.model_dump_json(), - "X-DIAL-APPLICATION-ID": TestDialCoreConfig.APP_DEPLOYMENT_V2_NAME, - } +def create_request_headers( + api_key: SecretStr, app_config: ApplicationConfig | None = None +) -> dict[str, str]: + headers = _create_request_headers(api_key) + if app_config: + headers.update( + { + "X-DIAL-APPLICATION-PROPERTIES": app_config.model_dump_json(), + "X-DIAL-APPLICATION-ID": TestDialCoreConfig.APP_DEPLOYMENT_V2_NAME, + } + ) + return headers class TestRunner: + @staticmethod + def _resolve_path(path: str | Path, default_parent: Path) -> Path: + candidate = Path(path) + if candidate.is_absolute(): + return candidate + if candidate.parts and candidate.parts[0] == default_parent.name: + return default_parent.parent / candidate + return default_parent / candidate + + @staticmethod + def _sanitize_path_segment(value: str) -> str: + return re.sub(r"[^a-zA-Z0-9._-]+", "-", value).strip("-").lower() + @staticmethod async def start_server(refresh: bool, test_name: str, port: int, model: str, no_cache: bool): logger.debug("Starting middleware server...") @@ -136,7 +171,7 @@ async def _search_file(dial_client: AsyncDial, bucket: str, filename: str) -> st return None @staticmethod - async def get_attachment_url(dial_url: str, headers, attachment: Path): + async def get_attachment_url(dial_url: str, headers: dict[str, str], attachment: Path) -> str: api_key = headers.get(API_KEY_HEADER) dial_client = AsyncDial(api_key=api_key, base_url=dial_url) @@ -157,6 +192,121 @@ async def get_attachment_url(dial_url: str, headers, attachment: Path): return url + @staticmethod + async def upload_dial_prompt_skill( + dial_url: str, + headers: dict[str, str], + skill_file: Path, + ) -> str: + api_key = headers.get(API_KEY_HEADER) + dial_client = AsyncDial(api_key=api_key, base_url=dial_url) + + bucket_resp = await dial_client.bucket.get_raw() + bucket = bucket_resp.appdata or bucket_resp.bucket + + skill_content = skill_file.read_text(encoding="utf-8") + parsed_skill = parse_frontmatter(skill_content, str(skill_file)) + parsed_name = parsed_skill.metadata.name + prompt_name = TestRunner._sanitize_path_segment(parsed_name) + prompt_folder = f"{INTEGRATION_PROMPTS_FOLDER}/" + prompt_url = f"prompts/{bucket}/{INTEGRATION_PROMPTS_FOLDER}/{prompt_name}" + + try: + existing_prompt = await dial_client.prompts.get(prompt_url) + if existing_prompt.content == skill_content: + logger.info("Reusing existing dial-prompt skill: %s", prompt_url) + return prompt_url + except ResourceNotFoundError: + pass + except DialException as exc: + logger.warning( + "Failed to fetch existing skill before upload (%s), proceeding with upload: %s", + exc, + prompt_url, + ) + + prompt = Prompt( + id=prompt_name, + name=prompt_name, + folder_id=prompt_folder, + content=skill_content, + ) + + try: + await dial_client.prompts.save(prompt_url, prompt) + logger.info("Uploaded dial-prompt skill: %s", prompt_url) + return prompt_url + except Exception as e: + raise RuntimeError( + f"Failed to upload dial-prompt skill to DIAL for {skill_file}" + ) from e + + @staticmethod + def _resolve_skill_path(skill: str | Path | SkillFileConfig) -> Path: + if isinstance(skill, SkillFileConfig): + skill_path_value: str | Path = skill.path + else: + skill_path_value = skill + + skill_path_raw = Path(skill_path_value) + if skill_path_raw.parts and skill_path_raw.parts[0] == "skills": + skill_path = SKILLS_DIR / Path(*skill_path_raw.parts[1:]) + else: + skill_path = TestRunner._resolve_path(skill_path_value, SKILLS_DIR) + if skill_path.suffix == "": + skill_path = skill_path.with_suffix(".md") + if not skill_path.exists(): + raise FileNotFoundError(f"Skill file not found: {skill_path}") + return skill_path + + @staticmethod + def resolve_skill_files( + skills: str | Path | SkillFileConfig | list[str | Path | SkillFileConfig] | None, + ) -> list[Path]: + if skills is None: + return [] + if isinstance(skills, list): + return [TestRunner._resolve_skill_path(skill) for skill in skills] + return [TestRunner._resolve_skill_path(skills)] + + @staticmethod + def normalize_context_configs( + contexts: list[str | Path | TestContextConfig] | None, + test_file_dir: Path, + ) -> list[TestContextConfig]: + if not contexts: + return [] + normalized: list[TestContextConfig] = [] + + for context in contexts: + if isinstance(context, TestContextConfig): + raw_path: str | Path = context.path + description = context.description + else: + raw_path = context + description = None + + resolved_path = TestRunner._resolve_path(raw_path, test_file_dir) + if not resolved_path.exists(): + raise FileNotFoundError(f"Context file not found: {resolved_path}") + + normalized.append(TestContextConfig(path=resolved_path, description=description)) + return normalized + + @staticmethod + async def resolve_application_contexts( + dial_url: str, + headers: dict[str, str], + contexts: list[TestContextConfig], + ) -> list[FileContextConfig]: + app_contexts: list[FileContextConfig] = [] + for context in contexts: + url = await TestRunner.get_attachment_url( + dial_url=dial_url, headers=headers, attachment=context.path + ) + app_contexts.append(FileContextConfig(url=url, description=context.description)) + return app_contexts + @staticmethod async def execute_test_case( client: TestClient, test_case: TstCase, ts: TestStats, app_config: ApplicationConfig @@ -344,12 +494,13 @@ def e2e_test( config_file_set: str = "e2e", runs: int = 3, no_cache: bool = False, - application_context_files: list[Path] | None = None, + skills: str | Path | SkillFileConfig | list[str | Path | SkillFileConfig] | None = None, + application_context_files: list[str | Path | TestContextConfig] | None = None, include_rest_toolset: bool = False, attachment_strategy: LazyOnDemandAttachmentStrategy | None = None, ): if refresh is None: - refresh = bool_env_var("REFRESH", default="false") + refresh = bool_env_var("REFRESH", default=False) _application_context_files = application_context_files _include_rest_toolset = include_rest_toolset @@ -425,26 +576,46 @@ async def prepare_and_execute_test( test_stats, run_index, ): - contexts: list[FileContextConfig] = [] - if _application_context_files: - await prepare_contexts(contexts) - tool_sets = TestConfig.load_tools_config( - unique_port, - config_file_set, - include_rest_toolset=_include_rest_toolset, + unique_port, config_file_set, include_rest_toolset=_include_rest_toolset ) + + app = TestApp.get_app(port=unique_port) + client = TestClient(app) + headers = create_request_headers(TestDialCoreConfig.REMOTE_DIAL_API_KEY_SECRET) + + skill_urls: list[str] = [] + for skill_file in TestRunner.resolve_skill_files(skills): + skill_urls.append( + await TestRunner.upload_dial_prompt_skill( + dial_url=TestDialCoreConfig.REMOTE_DIAL_URL, + headers=headers, + skill_file=skill_file, + ) + ) + + if hasattr(request.node, "path"): + test_file_path = Path(str(request.node.path)) + else: + test_file_path = Path(str(request.node.fspath)) + test_file_dir = test_file_path.parent + normalized_contexts = TestRunner.normalize_context_configs( + _application_context_files, test_file_dir=test_file_dir + ) + app_contexts = await TestRunner.resolve_application_contexts( + dial_url=TestDialCoreConfig.REMOTE_DIAL_URL, + headers=headers, + contexts=normalized_contexts, + ) + app_config: ApplicationConfig = TestConfig.create_app_configuration( toolsets=tool_sets, model=execution_model, - contexts=contexts, + skill_urls=skill_urls or None, + contexts=app_contexts, attachment_strategy=_attachment_strategy, ) - app = TestApp.get_app(port=unique_port) - - client = TestClient(app) - # Combine CLI flag with decorator parameter - CLI takes precedence cli_no_cache = bool(request.config.getoption("--no-cache", default=False)) effective_no_cache = cli_no_cache or no_cache @@ -479,7 +650,7 @@ async def prepare_and_execute_test( # TestClient is synchronous and doesn't need async close # Don't shutdown async generators while loop is running - async def prepare_contexts(contexts: list[FileContextConfig]): + async def prepare_contexts(contexts: list[Context]): upload_headers = { API_KEY_HEADER: TestDialCoreConfig.REMOTE_DIAL_API_KEY, CONTENT_TYPE_HEADER: "application/json", diff --git a/src/tests/integration_tests/test_runner/models.py b/src/tests/integration_tests/test_runner/models.py index 744fd10d..ae9266b4 100644 --- a/src/tests/integration_tests/test_runner/models.py +++ b/src/tests/integration_tests/test_runner/models.py @@ -189,6 +189,19 @@ def model_post_init(self, __context: Any) -> None: object.__setattr__(self, "attachment_checks", []) +class SkillFileConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + path: Path + + +class TestContextConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + path: Path + description: str | None = None + + class TstCase: def __init__( self, diff --git a/src/tests/integration_tests/test_runner/paths.py b/src/tests/integration_tests/test_runner/paths.py new file mode 100644 index 00000000..4e4e60f6 --- /dev/null +++ b/src/tests/integration_tests/test_runner/paths.py @@ -0,0 +1,5 @@ +from pathlib import Path + +TEST_RUNNER_ROOT = Path(__file__).parent +TOOLSETS_DIR = TEST_RUNNER_ROOT / "toolsets" +SKILLS_DIR = TEST_RUNNER_ROOT / "skills" diff --git a/src/tests/integration_tests/test_runner/skills/README.md b/src/tests/integration_tests/test_runner/skills/README.md new file mode 100644 index 00000000..e61f55d0 --- /dev/null +++ b/src/tests/integration_tests/test_runner/skills/README.md @@ -0,0 +1,11 @@ +Store integration-test dial-prompt skills as Markdown files in this directory. + +Each file must include YAML frontmatter with `name` and `description` (see +`quickapp/skills/_frontmatter.py`). + +Example usage in tests: + +`@e2e_test(skills="numbered-list-only.md", ...)` + +The test runner uploads skill content to DIAL prompts storage and wires it through +`ApplicationConfig.skills` as `DialPromptSkillConfig`. diff --git a/src/tests/integration_tests/test_runner/skills/numbered-list-only.md b/src/tests/integration_tests/test_runner/skills/numbered-list-only.md new file mode 100644 index 00000000..b29fecb2 --- /dev/null +++ b/src/tests/integration_tests/test_runner/skills/numbered-list-only.md @@ -0,0 +1,16 @@ +--- +name: numbered-list-only +description: Format answer as a numbered list. +--- + +# Numbered list only + +Do not add introductions, list headers, conclusions, or explanations outside the list. +Only valuable data should be in the numbered list. +The list should be in reversed order from what you got. +Use the following format for each row of the list: + +* 1st - +* 2nd - +* 3rd - +... diff --git a/src/tests/integration_tests/test_runner/test_tool_set_rest.py b/src/tests/integration_tests/test_runner/test_tool_set_rest.py index cdd777c7..3667d397 100644 --- a/src/tests/integration_tests/test_runner/test_tool_set_rest.py +++ b/src/tests/integration_tests/test_runner/test_tool_set_rest.py @@ -1,15 +1,15 @@ import json -from pathlib import Path from pydantic.type_adapter import TypeAdapter from quickapp.config.toolsets.rest_api import RestApiToolSet +from tests.integration_tests.test_runner.paths import TOOLSETS_DIR class TestToolSetRest: @staticmethod def get_rest_toolset(port: int): - file_path = Path(__file__).parent / "test_rest_toolset.json" + file_path = TOOLSETS_DIR / "test_rest_toolset.json" data_text = file_path.read_text().replace("", str(port)) data = json.loads(data_text) tool_set: RestApiToolSet = TypeAdapter(RestApiToolSet).validate_python(data) diff --git a/src/tests/integration_tests/test_runner/test_mcp_tool.json b/src/tests/integration_tests/test_runner/toolsets/test_mcp_tool.json similarity index 100% rename from src/tests/integration_tests/test_runner/test_mcp_tool.json rename to src/tests/integration_tests/test_runner/toolsets/test_mcp_tool.json diff --git a/src/tests/integration_tests/test_runner/test_rest_toolset.json b/src/tests/integration_tests/test_runner/toolsets/test_rest_toolset.json similarity index 100% rename from src/tests/integration_tests/test_runner/test_rest_toolset.json rename to src/tests/integration_tests/test_runner/toolsets/test_rest_toolset.json diff --git a/src/tests/integration_tests/test_runner/test_tool_set_chat_hub.json b/src/tests/integration_tests/test_runner/toolsets/test_tool_set_chat_hub.json similarity index 100% rename from src/tests/integration_tests/test_runner/test_tool_set_chat_hub.json rename to src/tests/integration_tests/test_runner/toolsets/test_tool_set_chat_hub.json diff --git a/src/tests/integration_tests/test_runner/test_tool_set_py_interpreter.json b/src/tests/integration_tests/test_runner/toolsets/test_tool_set_py_interpreter.json similarity index 100% rename from src/tests/integration_tests/test_runner/test_tool_set_py_interpreter.json rename to src/tests/integration_tests/test_runner/toolsets/test_tool_set_py_interpreter.json diff --git a/src/tests/integration_tests/test_runner/utils/tool_names.py b/src/tests/integration_tests/test_runner/utils/tool_names.py index 9f3032c6..f384e4c1 100644 --- a/src/tests/integration_tests/test_runner/utils/tool_names.py +++ b/src/tests/integration_tests/test_runner/utils/tool_names.py @@ -4,6 +4,9 @@ class ToolNames(Enum): INTERNAL_ATTACHMENTS_AVAILABLE_CONTEXT = "internal_attachments_available_context" INTERNAL_ATTACHMENTS_GET_CONTENT = "internal_attachments_get_content" + INTERNAL_SKILLS_READ_SKILL = "internal_skills_read_skill" + INTERNAL_FILE_READ_LINES = "internal_file_read_lines" + IMAGE_GENERATION_TOOL = "image_generation_tool" WEB_SEARCH_TOOL = "web_search_tool" RAG_SEARCH_TOOL = "rag_search_tool" diff --git a/src/tests/integration_tests/test_simple_tool.py b/src/tests/integration_tests/test_simple_tool.py index 2636b902..2b545f1e 100644 --- a/src/tests/integration_tests/test_simple_tool.py +++ b/src/tests/integration_tests/test_simple_tool.py @@ -221,6 +221,40 @@ def test_rag_search(client): pass +@pytest.mark.integration +@e2e_test( + runs=2, + config_file_set="no-additional-tools", + skills="numbered-list-only.md", + application_context_files=[ + Path(__file__).parent / "test_documents/integration_context_items.txt" + ], + test_case=TstCase( + "Dial skill format and admin context content", + "Read dial-prompt skill for list format and admin context via internal_file_read_lines", + similarity_threshold=SimilarityThreshold.STRICT.value, + ).add_user_message( + user_message=( + "Read the numbered-list-only skill. Give me the numbered list with data from the available attached file." + ), + tool_calls=[ + ToolCall(ToolNames.INTERNAL_ATTACHMENTS_AVAILABLE_CONTEXT.value, max_calls=1), + ToolCall(ToolNames.INTERNAL_SKILLS_READ_SKILL.value, max_calls=2), + ToolCall(ToolNames.INTERNAL_FILE_READ_LINES.value, max_calls=5).add_soft_argument_check( + "path", + ["files/684f6Lz7ubje66aoCRsa5c/integration_context_items.txt"], + ), + ], + answer=[ + "1st - GammaMapper\n2nd - BetaIndexer\n3rd - AlphaReader", + "* 1st - GammaMapper\n* 2nd - BetaIndexer\n* 3rd - AlphaReader", + ], + ), +) +def test_dial_skill_and_context_numbered_list(client): + pass + + @pytest.mark.integration @e2e_test( config_file_set="integration",