."""
+ c = _make_connector(auth_type=AuthType.BEARER)
+ c.load_credentials({"token": "tok_abc"})
+ assert c._auth_headers == {"Authorization": "Bearer tok_abc"}
+
+ @patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+ def test_basic_auth(self, _dns):
+ """basic should produce an HTTPBasicAuth object."""
+ c = _make_connector(auth_type=AuthType.BASIC)
+ c.load_credentials({"username": "user", "password": "pass"})
+ assert c._basic_auth is not None
+ assert c._basic_auth.username == "user"
+ assert c._basic_auth.password == "pass"
+
+
+# ===================================================================== #
+# 4. Field extraction #
+# ===================================================================== #
+
+class TestFieldExtraction:
+ """Test _extract_field / _extract_field_values dot-notation paths."""
+
+ @patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+ def setup_method(self, method, _dns=None):
+ with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
+ self.connector = _make_connector()
+
+ def test_simple_field(self):
+ """Top-level field extraction."""
+ assert self.connector._extract_field({"title": "Hello"}, "title") == "Hello"
+
+ def test_dot_notation_nested(self):
+ """Dot-notation nested field."""
+ item = {"country": {"name": "Kuwait"}}
+ assert self.connector._extract_field(item, "country.name") == "Kuwait"
+
+ def test_array_wildcard(self):
+ """Wildcard [*] returns all array elements."""
+ item = {"tags": [{"name": "A"}, {"name": "B"}]}
+ result = self.connector._extract_field(item, "tags[*].name")
+ assert result == ["A", "B"]
+
+ def test_missing_field_returns_none(self):
+ """Missing field returns None."""
+ assert self.connector._extract_field({"a": 1}, "nonexistent") is None
+
+ def test_missing_field_with_default(self):
+ """Missing field returns configured default value."""
+ with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
+ c = _make_connector(field_default_values={"missing": "fallback"})
+ result = c._get_typed_field_value("missing", {"other": 1})
+ assert result == "fallback"
+
+ def test_deeply_nested_path(self):
+ """Multi-level dot-notation path."""
+ item = {"a": {"b": {"c": {"d": 42}}}}
+ assert self.connector._extract_field(item, "a.b.c.d") == 42
+
+
+# ===================================================================== #
+# 5. Items array detection #
+# ===================================================================== #
+
+class TestItemsArrayDetection:
+ """Test _extract_items auto-detection of the items array."""
+
+ @patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+ def setup_method(self, method, _dns=None):
+ with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
+ self.connector = _make_connector()
+
+ def test_items_key(self):
+ """Detect 'items' key."""
+ resp = {"items": [{"id": 1}]}
+ assert self.connector._extract_items(resp) == [{"id": 1}]
+
+ def test_results_key(self):
+ """Detect 'results' key."""
+ resp = {"results": [{"id": 2}]}
+ assert self.connector._extract_items(resp) == [{"id": 2}]
+
+ def test_data_key(self):
+ """Detect 'data' key."""
+ resp = {"data": [{"id": 3}]}
+ assert self.connector._extract_items(resp) == [{"id": 3}]
+
+ def test_records_key(self):
+ """Detect 'records' key."""
+ resp = {"records": [{"id": 4}]}
+ assert self.connector._extract_items(resp) == [{"id": 4}]
+
+ def test_custom_key_fallback(self):
+ """Fall back to the first list value in the dict."""
+ resp = {"totalCount": 5, "stories": [{"id": 5}]}
+ assert self.connector._extract_items(resp) == [{"id": 5}]
+
+ def test_response_is_list(self):
+ """Response that is directly a list."""
+ resp = [{"id": 6}, {"id": 7}]
+ assert self.connector._extract_items(resp) == [{"id": 6}, {"id": 7}]
+
+ def test_empty_response(self):
+ """Empty dict returns empty list."""
+ assert self.connector._extract_items({}) == []
+
+ def test_no_list_in_response(self):
+ """Dict with no list values returns empty list."""
+ assert self.connector._extract_items({"count": 0}) == []
+
+
+# ===================================================================== #
+# 6. HTML stripping #
+# ===================================================================== #
+
+class TestHTMLStripping:
+ """Test the _strip_html static method."""
+
+ def test_basic_tag_removal(self):
+ """Remove simple HTML tags."""
+ assert RestAPIConnector._strip_html("Hello
") == "Hello"
+
+ def test_whitespace_collapsing(self):
+ """Multiple whitespace chars collapse to single space."""
+ assert RestAPIConnector._strip_html("Hello
World
") == "Hello World"
+
+ def test_empty_string(self):
+ """Empty input returns empty output."""
+ assert RestAPIConnector._strip_html("") == ""
+
+ def test_plain_text_passthrough(self):
+ """Text without HTML passes through unchanged."""
+ assert RestAPIConnector._strip_html("Hello World") == "Hello World"
+
+ def test_nested_tags(self):
+ """Nested HTML tags are all stripped."""
+ result = RestAPIConnector._strip_html("")
+ assert result == "Bold text"
+
+ def test_html_with_attributes(self):
+ """Tags with attributes are stripped."""
+ result = RestAPIConnector._strip_html('Link ')
+ assert result == "Link"
+
+
+# ===================================================================== #
+# 7. Document creation #
+# ===================================================================== #
+
+class TestDocumentCreation:
+ """Test _item_to_document mapping."""
+
+ @patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+ def setup_method(self, method, _dns=None):
+ with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
+ return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
+ self.connector = _make_connector(
+ id_field="id",
+ content_fields=["title", "body"],
+ metadata_fields=["author"],
+ )
+
+ def test_document_id_from_configured_field(self):
+ """Document ID uses the configured id_field."""
+ item = {"id": "abc", "title": "T", "body": "B", "author": "A"}
+ doc = self.connector._item_to_document(item)
+ assert doc.id is not None and len(doc.id) > 0
+
+ def test_semantic_identifier_from_first_content_field(self):
+ """semantic_identifier comes from the first content field."""
+ item = {"id": "1", "title": "My Title", "body": "Body", "author": "A"}
+ doc = self.connector._item_to_document(item)
+ assert "My Title" in doc.semantic_identifier
+
+ def test_content_blob_contains_all_fields(self):
+ """Blob should contain both content fields."""
+ item = {"id": "1", "title": "Title", "body": "Body text", "author": "A"}
+ doc = self.connector._item_to_document(item)
+ content = doc.blob.decode("utf-8")
+ assert "Title" in content
+ assert "Body text" in content
+
+ def test_metadata_populated(self):
+ """Metadata dict is populated from configured metadata_fields."""
+ item = {"id": "1", "title": "T", "body": "B", "author": "Jane"}
+ doc = self.connector._item_to_document(item)
+ assert doc.metadata is not None
+ assert doc.metadata["author"] == "Jane"
+
+ def test_html_stripped_from_content(self):
+ """HTML tags are removed from content fields."""
+ item = {"id": "1", "title": "T", "body": "Clean
", "author": "A"}
+ doc = self.connector._item_to_document(item)
+ content = doc.blob.decode("utf-8")
+ assert "" not in content
+ assert "Clean" in content
+
+ def test_extension_is_txt(self):
+ """Document extension should be .txt."""
+ item = {"id": "1", "title": "T", "body": "B", "author": "A"}
+ doc = self.connector._item_to_document(item)
+ assert doc.extension == ".txt"
+
+ def test_missing_content_fields_graceful(self):
+ """Missing content fields produce an empty blob gracefully."""
+ item = {"id": "1", "author": "A"}
+ doc = self.connector._item_to_document(item)
+ assert doc.blob == b""
+
+
+# ===================================================================== #
+# 8. Pagination behaviour #
+# ===================================================================== #
+
+class TestPaginationBehavior:
+ """Test pagination iteration with mocked HTTP responses."""
+
+ def test_page_pagination_increments(self):
+ """Page-based pagination should increment the page param."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ page1 = _mock_response({"items": [{"title": "A"}, {"title": "B"}]})
+ page2 = _mock_response({"items": []})
+ mock_rl.get.side_effect = [page1, page2]
+
+ c = _make_paged_connector()
+ items = list(c._iter_items())
+ assert len(items) == 2
+ assert mock_rl.get.call_count == 2
+
+ def test_offset_pagination_increments(self):
+ """Offset-based pagination should increment offset by limit."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ page1 = _mock_response({"items": [{"title": "A"}]})
+ page2 = _mock_response({"items": []})
+ mock_rl.get.side_effect = [page1, page2]
+
+ c = _make_connector(
+ pagination_type=PaginationType.OFFSET,
+ pagination_config={
+ "offset_param": "offset",
+ "limit_param": "limit",
+ "limit": 10,
+ },
+ request_delay=0,
+ )
+ items = list(c._iter_items())
+ assert len(items) == 1
+
+ def test_stops_on_empty_results(self):
+ """Pagination stops when empty items are returned."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({"items": []})
+
+ c = _make_paged_connector()
+ items = list(c._iter_items())
+ assert items == []
+ assert mock_rl.get.call_count == 1
+
+ def test_stops_when_fewer_items_than_page_size(self):
+ """Pagination stops when fewer items than page_size are returned."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ page1 = _mock_response({"items": [{"title": "A"}]})
+ mock_rl.get.return_value = page1
+
+ c = _make_paged_connector(
+ pagination_config={"page_param": "page", "page_size": 10},
+ )
+ items = list(c._iter_items())
+ assert len(items) == 1
+ assert mock_rl.get.call_count == 1
+
+ def test_max_pages_cap(self):
+ """Pagination respects the max_pages safety cap."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response(
+ {"items": [{"title": "A"}, {"title": "B"}]}
+ )
+
+ c = _make_paged_connector(
+ max_pages=3,
+ pagination_config={"page_param": "page", "page_size": 2},
+ )
+ list(c._iter_items())
+ assert mock_rl.get.call_count == 3
+
+ def test_request_delay_applied(self):
+ """request_delay should cause a sleep between pages."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ with patch("common.data_source.rest_api_connector.time.sleep") as mock_sleep:
+ page1 = _mock_response({"items": [{"title": "A"}, {"title": "B"}]})
+ page2 = _mock_response({"items": []})
+ mock_rl.get.side_effect = [page1, page2]
+
+ c = _make_paged_connector(
+ pagination_config={"page_param": "page", "page_size": 2},
+ )
+ c.request_delay = 1.5
+ list(c._iter_items())
+ mock_sleep.assert_called_once_with(1.5)
+
+
+# ===================================================================== #
+# 9. Non-retriable HTTP errors #
+# ===================================================================== #
+
+class TestNonRetriableErrors:
+ """Test that HTTP errors are classified correctly in _fetch_page."""
+
+ def test_401_raises_credential_error(self):
+ """401 should raise ConnectorMissingCredentialError immediately."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({}, status_code=401)
+ c = _make_connector(request_delay=0)
+ c.load_credentials({})
+ with pytest.raises(ConnectorMissingCredentialError):
+ c._fetch_page({})
+
+ def test_403_raises_credential_error(self):
+ """403 should raise ConnectorMissingCredentialError immediately."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({}, status_code=403)
+ c = _make_connector(request_delay=0)
+ c.load_credentials({})
+ with pytest.raises(ConnectorMissingCredentialError):
+ c._fetch_page({})
+
+ def test_404_raises_validation_error(self):
+ """404 should raise ConnectorValidationError (no retry)."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({}, status_code=404)
+ c = _make_connector(request_delay=0)
+ c.load_credentials({})
+ with pytest.raises(ConnectorValidationError, match="non-retriable"):
+ c._fetch_page({})
+
+ def test_400_raises_validation_error(self):
+ """400 should raise ConnectorValidationError (no retry)."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({}, status_code=400)
+ c = _make_connector(request_delay=0)
+ c.load_credentials({})
+ with pytest.raises(ConnectorValidationError, match="non-retriable"):
+ c._fetch_page({})
+
+ def test_500_triggers_retry(self):
+ """500 should raise HTTPError (which the retry decorator catches)."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({}, status_code=500)
+ c = _make_connector(request_delay=0)
+ c.load_credentials({})
+ with pytest.raises(requests.HTTPError):
+ c._fetch_page({})
+
+ def test_429_triggers_retry(self):
+ """429 should raise HTTPError (retriable, not ConnectorValidationError)."""
+ with _mocked_rest_api_requests_and_dns() as mock_rl:
+ mock_rl.get.return_value = _mock_response({}, status_code=429)
+ c = _make_connector(request_delay=0)
+ c.load_credentials({})
+ with pytest.raises(requests.HTTPError):
+ c._fetch_page({})
diff --git a/test/unit_test/rag/conftest.py b/test/unit_test/rag/conftest.py
new file mode 100644
index 00000000000..3ca5e289e9a
--- /dev/null
+++ b/test/unit_test/rag/conftest.py
@@ -0,0 +1,58 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Restore the real ``common.data_source`` package before importing rag unit tests.
+
+``test/unit_test/data_source/conftest.py`` registers a lightweight
+``sys.modules["common.data_source"]`` stub so submodule imports skip the heavy
+package ``__init__.py``. Pytest collection order visits ``data_source/`` before
+``rag/``, so without this hook ``rag.svr.sync_data_source`` fails on
+``from common.data_source import BlobStorageConnector``.
+"""
+
+from __future__ import annotations
+
+import importlib
+import sys
+import types
+
+
+def _restore_common_data_source_package() -> None:
+ mod = sys.modules.get("common.data_source")
+ if mod is None:
+ return
+ # Stub is a bare types.ModuleType with __path__ and no __file__; real package has __init__.py.
+ if getattr(mod, "__file__", None) is not None:
+ return
+ if not isinstance(mod, types.ModuleType) or not getattr(mod, "__path__", None):
+ return
+ keys = [
+ key
+ for key in sys.modules
+ if key == "common.data_source" or key.startswith("common.data_source.")
+ ]
+ for key in keys:
+ del sys.modules[key]
+ importlib.invalidate_caches()
+ try:
+ importlib.import_module("common.data_source")
+ except Exception as exc: # pragma: no cover
+ raise ImportError(
+ "conftest: failed to restore real common.data_source package"
+ ) from exc
+
+
+_restore_common_data_source_package()
diff --git a/web/src/assets/svg/data-source/rest-api.svg b/web/src/assets/svg/data-source/rest-api.svg
new file mode 100644
index 00000000000..f7d3e6d213a
--- /dev/null
+++ b/web/src/assets/svg/data-source/rest-api.svg
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/web/src/components/document-preview/ppt-preview.tsx b/web/src/components/document-preview/ppt-preview.tsx
index 4895ad9fe33..d95e05c46d7 100644
--- a/web/src/components/document-preview/ppt-preview.tsx
+++ b/web/src/components/document-preview/ppt-preview.tsx
@@ -41,7 +41,7 @@ export const PptPreviewer: React.FC = ({
});
pptxPrviewer.preview(arrayBuffer);
}
- } catch (err) {
+ } catch {
message.error('ppt parse failed');
}
};
diff --git a/web/src/components/dynamic-form.tsx b/web/src/components/dynamic-form.tsx
index 5c9fff5eaf4..0ef13df1c13 100644
--- a/web/src/components/dynamic-form.tsx
+++ b/web/src/components/dynamic-form.tsx
@@ -347,7 +347,6 @@ export const RenderField = ({
field: FormFieldConfig;
labelClassName?: string;
}) => {
- const form = useFormContext();
if (field.render) {
if (field.type === FormFieldType.Custom && field.hideLabel) {
return {field.render({})}
;
diff --git a/web/src/components/floating-chat-widget-markdown.tsx b/web/src/components/floating-chat-widget-markdown.tsx
index 3a4e4942c66..4237fa17587 100644
--- a/web/src/components/floating-chat-widget-markdown.tsx
+++ b/web/src/components/floating-chat-widget-markdown.tsx
@@ -296,9 +296,11 @@ const FloatingChatWidgetMarkdown = ({
className="text-sm leading-relaxed space-y-2 prose-sm max-w-full"
components={
{
- p: ({ children, node, ...props }: any) => (
- {children}
- ),
+ p: (props: any) => {
+ const { children, node, ...rest } = props;
+ void node;
+ return {children}
;
+ },
'custom-typography': ({ children }: { children: string }) =>
renderReference(children),
code(props: any) {
diff --git a/web/src/components/ui/audio-button.tsx b/web/src/components/ui/audio-button.tsx
index 313ef387393..15ac5b8c631 100644
--- a/web/src/components/ui/audio-button.tsx
+++ b/web/src/components/ui/audio-button.tsx
@@ -8,7 +8,6 @@ import { getAuthorization } from '@/utils/authorization-util';
import { chain, sum } from 'lodash';
import { Loader2, Mic, Square } from 'lucide-react';
import { useCallback, useEffect, useRef, useState } from 'react';
-import { useIsDarkTheme } from '../theme-provider';
import { Input } from './input';
import { Popover, PopoverContent, PopoverTrigger } from './popover';
@@ -18,7 +17,6 @@ const VoiceVisualizer = ({ isRecording }: { isRecording: boolean }) => {
const analyserRef = useRef(null);
const animationFrameRef = useRef(0);
const streamRef = useRef(null);
- const isDark = useIsDarkTheme();
const draw = useCallback(() => {
const canvas = canvasRef.current;
@@ -273,11 +271,9 @@ export const AudioButton = ({
// throw new Error('ReadableStream not supported in this browser');
// }
- console.log('Response:', response);
const { data, code } = await response.json();
if (code === 0 && data && data.text) {
setTranscript(data.text);
- console.log('Transcript:', data.text);
onOk?.(data.text);
}
setPopoverOpen(false);
@@ -341,7 +337,7 @@ export const AudioButton = ({
{
- setPopoverOpen(true);
+ setPopoverOpen(open);
}}
>
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 0c43e37fdd4..20d5fde23d1 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1365,6 +1365,42 @@ Example: Virtual Hosted Style`,
'Column to use as unique document ID. If not specified, a hash of the content will be used.',
postgresqlTimestampColumnTip:
'Datetime/timestamp column for incremental sync. Only rows modified after the last sync will be fetched.',
+ rest_apiDescription:
+ 'Connect any REST API endpoint as a data source using a flexible, configuration-driven connector.',
+ restApiQueryParamsTip:
+ 'Key=value pairs (one per line) sent as URL query parameters. Use this instead of embedding params in the URL.',
+ restApiHeadersTip:
+ 'Optional JSON object of additional HTTP headers to send with every request.',
+ restApiItemsPathTip:
+ 'Field name or JSONPath to the array of items in the response. Leave empty to auto-detect (tries "items", "results", "data", etc.).',
+ restApiIdFieldTip:
+ 'Field path within each item used to build a stable document ID. Leave empty to auto-generate from content hash.',
+ restApiContentFieldsTip:
+ 'Comma-separated list of item fields to concatenate into the document content.',
+ restApiMetadataFieldsTip:
+ 'Comma-separated list of item fields to store as metadata.',
+ restApiNextCursorPathTip:
+ 'JSONPath expression that resolves to the next-page cursor in the API response.',
+ restApiPollTimestampFieldTip:
+ 'Field path in each item that represents the last updated time, used for incremental sync.',
+ restApiRequestBodyTip:
+ 'Optional JSON body to send for POST requests. Used together with query params and pagination.',
+ restApiRequestDelayTip:
+ 'Delay in seconds between consecutive page requests. Helps avoid rate limiting from the API. Set to 0 to disable.',
+ restApiValidationApiKeyRequired:
+ 'API key is required when Auth Type is API Key (Header).',
+ restApiValidationApiKeyHeaderNameRequired:
+ 'API key header name is required when Auth Type is API Key (Header).',
+ restApiValidationBearerTokenRequired:
+ 'Bearer token is required when Auth Type is Bearer Token.',
+ restApiValidationBasicUsernameRequired:
+ 'Username is required when Auth Type is Basic Auth.',
+ restApiValidationBasicPasswordRequired:
+ 'Password is required when Auth Type is Basic Auth.',
+ restApiTestConnection: 'Test connection',
+ restApiTestSuccess: 'REST API connector validated successfully.',
+ restApiTestFailed:
+ 'REST API connector validation failed. Please check your configuration and logs.',
availableSourcesDescription: 'Select a data source to add',
availableSources: 'Available sources',
datasourceDescription: 'Manage your data source and connections',
diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.ts
index 2ae7cf3a23e..3c0ba20c7bb 100644
--- a/web/src/locales/tr.ts
+++ b/web/src/locales/tr.ts
@@ -341,7 +341,7 @@ Prosedürel Bellek: Öğrenilen beceriler, alışkanlıklar ve otomatik prosedü
config: {
descriptionPlaceholder: 'Belleğinizi açıklayın',
memorySizeTooltip: `Her mesajın içeriği + embedding vektörü için geçerlidir (≈ İçerik + Boyutlar × 8 Bayt).
-Örnek: 1024 boyutlu embedding ile 1 KB\'lık bir mesaj ~9 KB kullanır. 5 MB varsayılan sınır ~500 mesaj tutar.`,
+Örnek: 1024 boyutlu embedding ile 1 KB'lık bir mesaj ~9 KB kullanır. 5 MB varsayılan sınır ~500 mesaj tutar.`,
avatar: 'Avatar',
description: 'Açıklama',
memorySize: 'Bellek boyutu',
@@ -980,9 +980,9 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
systemTip:
'LLM için istemleriniz veya talimatlarınız; rol, yanıtların uzunluğu, tonu ve dili dahil ancak bunlarla sınırlı değildir. Modeliniz doğal olarak akıl yürütmeyi destekliyorsa, akıl yürütmeyi durdurmak için isteme //no_thinking ekleyebilirsiniz.',
topN: 'İlk N',
- topNTip: `Benzerlik eşiğinin üzerindeki tüm parçalar LLM\'ye gönderilmeyecek. Bu, alınanlardan 'İlk N' parçayı seçer.`,
+ topNTip: `Benzerlik eşiğinin üzerindeki tüm parçalar LLM'ye gönderilmeyecek. Bu, alınanlardan 'İlk N' parçayı seçer.`,
variable: 'Değişken',
- variableTip: `RAGFlow\'nun sohbet asistanı yönetim API\'leri ile birlikte kullanılır.`,
+ variableTip: `RAGFlow'nun sohbet asistanı yönetim API'leri ile birlikte kullanılır.`,
add: 'Ekle',
key: 'Anahtar',
optional: 'İsteğe bağlı',
@@ -1154,7 +1154,7 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
"Confluence örneğinizin temel URL'si (örn. https://your-domain.atlassian.net/wiki)",
confluenceSpaceKeyTip:
'İsteğe bağlı: Belirli bir alanla senkronizasyonu sınırlamak için alan anahtarı belirtin.',
- s3PrefixTip: `S3 bucket\'ınızdaki dosyaları almak için klasör yolunu belirtin.`,
+ s3PrefixTip: `S3 bucket'ınızdaki dosyaları almak için klasör yolunu belirtin.`,
S3CompatibleEndpointUrlTip: `S3 uyumlu Depolama Kutusu için zorunludur.`,
S3CompatibleAddressingStyleTip: `S3 uyumlu Depolama Kutusu için zorunludur.`,
addDataSourceModalTitle: '{{name}} bağlayıcınızı oluşturun',
@@ -1509,7 +1509,7 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
'Lütfen Google Cloud Hizmet Hesabı Anahtarını base64 formatında girin',
addGoogleRegion: 'Google Cloud Bölgesi',
GoogleRegionMessage: 'Lütfen Google Cloud Bölgesi girin',
- modelProvidersWarn: `Lütfen önce Ayarlar > Model sağlayıcıları bölümünde hem embedding modelini hem de LLM\'yi ekleyin.`,
+ modelProvidersWarn: `Lütfen önce Ayarlar > Model sağlayıcıları bölümünde hem embedding modelini hem de LLM'yi ekleyin.`,
apiVersion: 'API Sürümü',
apiVersionMessage: 'Lütfen API sürümünü girin',
add: 'Ekle',
@@ -1794,7 +1794,7 @@ En uygun olduğu durumlar: Anlatı bütünlüğünün bitişik paragrafları bir
beginDescription: 'Akışın başladığı yer.',
answerDescription: `İnsan ve bot arasındaki arayüz olarak hizmet eden bir bileşen.`,
retrievalDescription: `Belirtilen datasets içinden bilgi alan bir bileşen.`,
- generateDescription: `LLM\'yi yanıt üretmeye yönlendiren bir bileşen.`,
+ generateDescription: `LLM'yi yanıt üretmeye yönlendiren bir bileşen.`,
categorizeDescription: `Kullanıcı girişlerini önceden tanımlanmış kategorilere sınıflandırmak için LLM kullanan bir bileşen.`,
relevantDescription: `Yukarı akış çıktısının kullanıcının son sorgusuna uygun olup olmadığını değerlendirmek için LLM kullanan bir bileşen.`,
rewriteQuestionDescription: `Önceki diyalogların bağlamına dayanarak Etkileşim bileşeninden bir kullanıcı sorgusunu yeniden yazan bir bileşen.`,
@@ -1802,7 +1802,7 @@ En uygun olduğu durumlar: Anlatı bütünlüğünün bitişik paragrafları bir
'Bu bileşen, önceden tanımlanmış mesaj içeriğiyle birlikte iş akışının nihai veri çıktısını döndürür.',
keywordDescription: `Kullanıcının girdisinden en fazla N arama sonucunu alan bir bileşen.`,
switchDescription: `Önceki bileşenlerin çıktısına göre koşulları değerlendiren ve yürütme akışını yönlendiren bir bileşen.`,
- wikipediaDescription: `wikipedia.org\'dan arama yapan bir bileşen.`,
+ wikipediaDescription: `wikipedia.org'dan arama yapan bir bileşen.`,
promptText: `Lütfen aşağıdaki paragrafları özetleyin. Sayılara dikkat edin, uydurma yapmayın. Paragraflar aşağıdaki gibidir:
{input}
Yukarısı özetlemeniz gereken içeriktir.`,
@@ -1825,7 +1825,7 @@ En uygun olduğu durumlar: Anlatı bütünlüğünün bitişik paragrafları bir
keywordExtract: 'Anahtar kelime',
keywordExtractDescription: `Bir kullanıcı sorgusundan anahtar kelimeler çıkaran bir bileşen.`,
baidu: 'Baidu',
- baiduDescription: `baidu.com\'dan arama yapan bir bileşen.`,
+ baiduDescription: `baidu.com'dan arama yapan bir bileşen.`,
duckDuckGo: 'DuckDuckGo',
duckDuckGoDescription: "duckduckgo.com'dan arama yapan bir bileşen.",
searXNG: 'SearXNG',
@@ -2693,7 +2693,7 @@ Temel Talimatlar:
changeStepModalContent: `
Şu anda bu aşamanın sonuçlarını düzenliyorsunuz.
Daha sonraki bir aşamaya geçerseniz değişiklikleriniz kaybolacak.
- Korumak için lütfen Yeniden Çalıştır\'a tıklayın.
`,
+ Korumak için lütfen Yeniden Çalıştır'a tıklayın.
`,
changeStepModalConfirmText: 'Yine de Geç',
changeStepModalCancelText: 'İptal',
unlinkPipelineModalTitle: 'Alım hattı bağlantısını kes',
diff --git a/web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx b/web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx
index a68bfd4fd9b..31e092ca3ec 100644
--- a/web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx
+++ b/web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx
@@ -369,7 +369,7 @@ export default function VariablePickerMenuPlugin({
const filterStructuredOutput = useGetStructuredOutputByValue();
const testTriggerFn = React.useCallback((text: string) => {
- const triggerRegex = /(^|\s|\()([/]((?:[^/\s\()])*))$/;
+ const triggerRegex = /(^|\s|\()([/]((?:[^/\s()])*))$/;
const match = triggerRegex.exec(text);
if (match !== null) {
diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.ts
index 45f91794937..15b57c5c6e2 100644
--- a/web/src/pages/agent/hooks/use-add-node.ts
+++ b/web/src/pages/agent/hooks/use-add-node.ts
@@ -75,7 +75,7 @@ const GroupStartNodeMap = {
name: Operator.IterationStart,
form: initialIterationStartValues,
},
- extent: 'parent' as 'parent',
+ extent: 'parent' as const,
},
[Operator.Loop]: {
id: `${Operator.LoopStart}:${humanId()}`,
@@ -86,7 +86,7 @@ const GroupStartNodeMap = {
name: Operator.LoopStart,
form: {},
},
- extent: 'parent' as 'parent',
+ extent: 'parent' as const,
},
};
diff --git a/web/src/pages/dataflow-result/components/time-line/index.tsx b/web/src/pages/dataflow-result/components/time-line/index.tsx
index 1e96eb216ec..3c0a3ca678a 100644
--- a/web/src/pages/dataflow-result/components/time-line/index.tsx
+++ b/web/src/pages/dataflow-result/components/time-line/index.tsx
@@ -55,7 +55,6 @@ export interface TimelineDataFlowProps {
const TimelineDataFlow = ({
activeFunc,
activeId,
- data,
timelineNodes,
}: TimelineDataFlowProps) => {
// const [timelineNodeArr,setTimelineNodeArr] = useState()
diff --git a/web/src/pages/next-chats/chat/app-settings/chat-basic-settings.tsx b/web/src/pages/next-chats/chat/app-settings/chat-basic-settings.tsx
index c0715c78466..2be5fc995ee 100644
--- a/web/src/pages/next-chats/chat/app-settings/chat-basic-settings.tsx
+++ b/web/src/pages/next-chats/chat/app-settings/chat-basic-settings.tsx
@@ -27,13 +27,20 @@ export default function ChatBasicSetting() {
const form = useFormContext();
const emptyResponseValue = form.watch('prompt_config.empty_response');
const prologueValue = form.watch('prompt_config.prologue');
- const kbIds = (useWatch({ control: form.control, name: 'dataset_ids' }) ||
- []) as string[];
+ const rawDatasetIds = useWatch({
+ control: form.control,
+ name: 'dataset_ids',
+ });
+ const kbIds = useMemo(
+ () => (rawDatasetIds || []) as string[],
+ [rawDatasetIds],
+ );
const metadataInclude = useWatch({
control: form.control,
name: 'prompt_config.reference_metadata.include',
});
- const { data: metadataKeys } = useFetchKnowledgeMetadataKeys(kbIds);
+ const { data: metadataKeys, loading: metadataKeysLoading } =
+ useFetchKnowledgeMetadataKeys(kbIds);
const metadataFieldOptions = useMemo(() => {
return (metadataKeys || []).map((key) => ({
label: key,
@@ -60,7 +67,7 @@ export default function ChatBasicSetting() {
} else if (!metadataInclude) {
form.setValue('prompt_config.reference_metadata.fields', undefined);
}
- }, [kbIds, metadataKeys, metadataInclude, form]);
+ }, [kbIds, metadataKeys, metadataKeysLoading, metadataInclude, form]);
return (
@@ -176,4 +183,4 @@ export default function ChatBasicSetting() {
)}
);
-}
+}
\ No newline at end of file
diff --git a/web/src/pages/next-search/markdown-content/index.tsx b/web/src/pages/next-search/markdown-content/index.tsx
index a5522b7c7a5..132f34d5df6 100644
--- a/web/src/pages/next-search/markdown-content/index.tsx
+++ b/web/src/pages/next-search/markdown-content/index.tsx
@@ -90,10 +90,13 @@ const MarkdownContent = ({
chunk: IReferenceChunk,
isPdf: boolean = false,
documentUrl?: string,
- ) =>
- () => {
+ ) => {
+ void isPdf;
+ void documentUrl;
+ return () => {
clickDocumentButton?.(documentId, chunk);
- },
+ };
+ },
[clickDocumentButton],
);
@@ -250,9 +253,7 @@ const MarkdownContent = ({
remarkPlugins={[remarkGfm, remarkMath]}
components={
{
- p: ({ children, node, ...props }: any) => (
- {children}
- ),
+ p: ({ children, ...props }: any) => {children}
,
'custom-typography': ({ children }: { children: string }) =>
renderReference(children),
code(props: any) {
diff --git a/web/src/pages/next-search/search-setting.tsx b/web/src/pages/next-search/search-setting.tsx
index c3c812306d5..697e7369e8d 100644
--- a/web/src/pages/next-search/search-setting.tsx
+++ b/web/src/pages/next-search/search-setting.tsx
@@ -217,9 +217,8 @@ const SearchSetting: React.FC = ({
control: formMethods.control,
name: 'search_config.reference_metadata.include',
});
- const { data: metadataKeys } = useFetchKnowledgeMetadataKeys(
- selectedKbIds || [],
- );
+ const { data: metadataKeys, loading: metadataKeysLoading } =
+ useFetchKnowledgeMetadataKeys(selectedKbIds || []);
const metadataFieldOptions = useMemo(() => {
return (metadataKeys || []).map((key) => ({
label: key,
@@ -252,7 +251,13 @@ const SearchSetting: React.FC = ({
undefined,
);
}
- }, [selectedKbIds, metadataKeys, referenceMetadataEnabled, formMethods]);
+ }, [
+ selectedKbIds,
+ metadataKeys,
+ metadataKeysLoading,
+ referenceMetadataEnabled,
+ formMethods,
+ ]);
// Reset top_k to 1024 only when user actively disables rerank (from true to false)
const prevRerankEnabled = useRef(undefined);
diff --git a/web/src/pages/skills/components/markdown-viewer.tsx b/web/src/pages/skills/components/markdown-viewer.tsx
index 12937ed32c8..e50f56289fb 100644
--- a/web/src/pages/skills/components/markdown-viewer.tsx
+++ b/web/src/pages/skills/components/markdown-viewer.tsx
@@ -61,20 +61,20 @@ const MarkdownViewer: React.FC = ({ content }) => {
const language = match ? match[1] : '';
if (language) {
- return (
-
- {String(children).replace(/\n$/, '')}
-
- );
+ return (
+
+ {String(children).replace(/\n$/, '')}
+
+ );
}
return (
diff --git a/web/src/pages/skills/components/skill-detail.tsx b/web/src/pages/skills/components/skill-detail.tsx
index c378a0cb0fb..1b915cb488e 100644
--- a/web/src/pages/skills/components/skill-detail.tsx
+++ b/web/src/pages/skills/components/skill-detail.tsx
@@ -103,9 +103,11 @@ const SkillDetail: React.FC = ({
const [versionFiles, setVersionFiles] = useState([]);
const [versionLoading, setVersionLoading] = useState(false);
- // Check if skill has multiple versions
- const hasVersions = skill?.versions && skill.versions.length > 0;
- const availableVersions = skill?.versions || [];
+ const availableVersions = useMemo(
+ () => skill?.versions ?? [],
+ [skill?.versions],
+ );
+ const hasVersions = availableVersions.length > 0;
// Reset state when skill changes or drawer opens/closes
useEffect(() => {
@@ -128,20 +130,14 @@ const SkillDetail: React.FC = ({
setSelectedFile(null);
setFileContent('');
}
- }, [
- open,
- skill?.id,
- hasVersions,
- skill?.metadata?.version,
- availableVersions,
- ]);
+ }, [open, skill, hasVersions, availableVersions]);
const resolvedVersion = useMemo(() => {
if (!skill) return '';
return (
selectedVersion || skill.metadata?.version || skill.versions?.[0] || ''
);
- }, [selectedVersion, skill?.id, skill?.metadata?.version, skill?.versions]);
+ }, [selectedVersion, skill]);
// Load files when version or skill changes
useEffect(() => {
@@ -204,16 +200,7 @@ const SkillDetail: React.FC = ({
return () => {
isActive = false;
};
- }, [
- skill?.id,
- skill?.source_type,
- skill?.metadata?.version,
- skill?.versions,
- (skill as any)?._folderId,
- skill?.files,
- resolvedVersion,
- getVersionFiles,
- ]);
+ }, [skill, resolvedVersion, getVersionFiles]);
// Use version files if available, otherwise use skill.files
const currentFiles = useMemo(() => {
@@ -248,7 +235,7 @@ const SkillDetail: React.FC = ({
);
setFileContent(content || '');
} catch (error) {
- console.error('Failed to load file content');
+ console.error('Failed to load file content', error);
} finally {
setLoading(false);
}
@@ -274,7 +261,7 @@ const SkillDetail: React.FC = ({
handleSelect({ id: targetFile.path } as TreeDataItem);
}
}
- }, [open, skill?.id, currentFiles.length]);
+ }, [open, skill, currentFiles, handleSelect, selectedFile]);
const renderFileContent = () => {
if (!selectedFile) {
diff --git a/web/src/pages/user-setting/data-source/component/box-token-field.tsx b/web/src/pages/user-setting/data-source/component/box-token-field.tsx
index eccdf2dda54..4ae6bb2cf76 100644
--- a/web/src/pages/user-setting/data-source/component/box-token-field.tsx
+++ b/web/src/pages/user-setting/data-source/component/box-token-field.tsx
@@ -127,7 +127,10 @@ const BoxTokenField = ({ value, onChange }: BoxTokenFieldProps) => {
string,
any
>;
- const { user_id: _userId, code, ...rest } = credentials;
+ const code = credentials.code;
+ const rest = { ...credentials };
+ delete rest.user_id;
+ delete rest.code;
const finalValue: Record = {
...rest,
@@ -173,7 +176,7 @@ const BoxTokenField = ({ value, onChange }: BoxTokenFieldProps) => {
setWebStatus('error');
setWebStatusMessage(errorMessage);
clearWebState();
- } catch (_error) {
+ } catch {
message.error('Unable to retrieve authorization result.');
setWebStatus('error');
setWebStatusMessage('Unable to retrieve authorization result.');
@@ -304,7 +307,7 @@ const BoxTokenField = ({ value, onChange }: BoxTokenFieldProps) => {
} else {
message.error(data.message || 'Failed to start Box authorization.');
}
- } catch (_error) {
+ } catch {
message.error('Failed to start Box authorization.');
} finally {
setSubmitLoading(false);
diff --git a/web/src/pages/user-setting/data-source/component/gmail-token-field.tsx b/web/src/pages/user-setting/data-source/component/gmail-token-field.tsx
index 186281d918a..c347ed649d4 100644
--- a/web/src/pages/user-setting/data-source/component/gmail-token-field.tsx
+++ b/web/src/pages/user-setting/data-source/component/gmail-token-field.tsx
@@ -95,11 +95,7 @@ const withRedirectUri = (credentials: string, redirectUri: string): string => {
});
};
-const GmailTokenField = ({
- value,
- onChange,
- placeholder,
-}: GmailTokenFieldProps) => {
+const GmailTokenField = ({ value, onChange }: GmailTokenFieldProps) => {
const [files, setFiles] = useState([]);
const [pendingCredentials, setPendingCredentials] = useState('');
const [redirectUri, setRedirectUri] = useState('');
@@ -195,7 +191,7 @@ const GmailTokenField = ({
}
message.error(data.message || 'Authorization failed.');
clearWebState();
- } catch (err) {
+ } catch {
message.error('Unable to retrieve authorization result.');
clearWebState();
}
@@ -315,7 +311,7 @@ const GmailTokenField = ({
} else {
message.error(data.message || 'Failed to start browser authorization.');
}
- } catch (err) {
+ } catch {
message.error('Failed to start browser authorization.');
} finally {
setWebAuthLoading(false);
diff --git a/web/src/pages/user-setting/data-source/component/google-drive-token-field.tsx b/web/src/pages/user-setting/data-source/component/google-drive-token-field.tsx
index 8d182fdbbae..53ef6a60bf3 100644
--- a/web/src/pages/user-setting/data-source/component/google-drive-token-field.tsx
+++ b/web/src/pages/user-setting/data-source/component/google-drive-token-field.tsx
@@ -192,7 +192,7 @@ const GoogleDriveTokenField = ({
}
message.error(data.message || 'Authorization failed.');
clearWebState();
- } catch (err) {
+ } catch {
message.error('Unable to retrieve authorization result.');
clearWebState();
}
@@ -312,7 +312,7 @@ const GoogleDriveTokenField = ({
} else {
message.error(data.message || 'Failed to start browser authorization.');
}
- } catch (err) {
+ } catch {
message.error('Failed to start browser authorization.');
} finally {
setWebAuthLoading(false);
diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx
index 0a5eb8c4296..d00db49e166 100644
--- a/web/src/pages/user-setting/data-source/constant/index.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.tsx
@@ -41,6 +41,7 @@ export enum DataSourceKey {
SEAFILE = 'seafile',
MYSQL = 'mysql',
POSTGRESQL = 'postgresql',
+ REST_API = 'rest_api',
RSS = 'rss',
// SHAREPOINT = 'sharepoint',
@@ -202,6 +203,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
description: t(`setting.${DataSourceKey.GMAIL}Description`),
icon: ,
},
+ [DataSourceKey.REST_API]: {
+ name: 'REST API',
+ description: t(`setting.${DataSourceKey.REST_API}Description`),
+ icon: ,
+ },
[DataSourceKey.MOODLE]: {
name: 'Moodle',
description: t(`setting.${DataSourceKey.MOODLE}Description`),
@@ -373,47 +379,6 @@ export const getCommonExtraDefaultValues = () => ({
},
});
-export const getDataSourceFieldsWithExtras = (
- source?: DataSourceKey,
-): FormFieldConfig[] => {
- if (!source) {
- return [];
- }
-
- const sourceFields =
- DataSourceFormFields[source as keyof typeof DataSourceFormFields] || [];
- const extraFields = getCommonExtraFields(source);
-
- if (source !== DataSourceKey.JIRA) {
- return [...sourceFields, ...extraFields];
- }
-
- const modeFieldIndex = sourceFields.findIndex(
- (field) => field.name === 'config.is_cloud',
- );
- if (modeFieldIndex < 0) {
- return [...sourceFields, ...extraFields];
- }
-
- const sharedFields = sourceFields.slice(0, modeFieldIndex);
- const modeFields = sourceFields.slice(modeFieldIndex);
-
- const sharedCheckboxFieldIndex = sharedFields.findIndex(
- (field) => field.type === FormFieldType.Checkbox,
- );
-
- if (sharedCheckboxFieldIndex < 0) {
- return [...sharedFields, ...extraFields, ...modeFields];
- }
-
- return [
- ...sharedFields.slice(0, sharedCheckboxFieldIndex),
- ...sharedFields.slice(sharedCheckboxFieldIndex),
- ...extraFields,
- ...modeFields,
- ];
-};
-
export const DataSourceFormFields = {
[DataSourceKey.RSS]: [
{
@@ -1123,6 +1088,286 @@ export const DataSourceFormFields = {
tooltip: t('setting.postgresqlTimestampColumnTip'),
},
],
+ [DataSourceKey.REST_API]: [
+ // ── Essential fields ──────────────────────────────────────────────
+ {
+ label: 'Base URL',
+ name: 'config.url',
+ type: FormFieldType.Text,
+ required: true,
+ placeholder: 'https://api.example.com/v1/resources',
+ },
+ {
+ label: 'HTTP Method',
+ name: 'config.method',
+ type: FormFieldType.Select,
+ required: true,
+ options: [
+ { label: 'GET', value: 'GET' },
+ { label: 'POST', value: 'POST' },
+ ],
+ defaultValue: 'GET',
+ },
+ {
+ label: 'Query Parameters',
+ name: 'config.query_params',
+ type: FormFieldType.Textarea,
+ required: false,
+ placeholder: `key=value\none_per_line=true`,
+ tooltip: t('setting.restApiQueryParamsTip'),
+ },
+ {
+ label: 'Items Path',
+ name: 'config.items_path',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: '$.items',
+ tooltip: t('setting.restApiItemsPathTip'),
+ },
+ {
+ label: 'ID Field',
+ name: 'config.id_field',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: 'id',
+ tooltip: t('setting.restApiIdFieldTip'),
+ },
+ {
+ label: 'Auth Type',
+ name: 'config.auth_type',
+ type: FormFieldType.Select,
+ required: true,
+ options: [
+ { label: 'None', value: 'none' },
+ { label: 'API Key (Header)', value: 'api_key_header' },
+ { label: 'Bearer Token', value: 'bearer' },
+ { label: 'Basic Auth', value: 'basic' },
+ ],
+ defaultValue: 'none',
+ },
+ {
+ label: 'API Key Header Name',
+ name: 'config.auth_config.header_name',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: 'X-API-Key',
+ shouldRender: (values: any) =>
+ values?.config?.auth_type === 'api_key_header',
+ customValidate: (val: string, values: any) => {
+ if (
+ values?.config?.auth_type === 'api_key_header' &&
+ !(val != null && String(val).trim())
+ ) {
+ return t('setting.restApiValidationApiKeyHeaderNameRequired');
+ }
+ return true;
+ },
+ },
+ {
+ label: 'API Key Value',
+ name: 'config.credentials.api_key',
+ type: FormFieldType.Password,
+ required: false,
+ shouldRender: (values: any) =>
+ values?.config?.auth_type === 'api_key_header',
+ customValidate: (val: string, values: any) => {
+ if (values?.config?.auth_type === 'api_key_header' && !val) {
+ return t('setting.restApiValidationApiKeyRequired');
+ }
+ return true;
+ },
+ },
+ {
+ label: 'Bearer Token',
+ name: 'config.credentials.token',
+ type: FormFieldType.Password,
+ required: false,
+ shouldRender: (values: any) => values?.config?.auth_type === 'bearer',
+ customValidate: (val: string, values: any) => {
+ if (values?.config?.auth_type === 'bearer' && !val) {
+ return t('setting.restApiValidationBearerTokenRequired');
+ }
+ return true;
+ },
+ },
+ {
+ label: 'Username',
+ name: 'config.credentials.username',
+ type: FormFieldType.Text,
+ required: false,
+ shouldRender: (values: any) => values?.config?.auth_type === 'basic',
+ customValidate: (val: string, values: any) => {
+ if (
+ values?.config?.auth_type === 'basic' &&
+ !(val != null && String(val).trim())
+ ) {
+ return t('setting.restApiValidationBasicUsernameRequired');
+ }
+ return true;
+ },
+ },
+ {
+ label: 'Password',
+ name: 'config.credentials.password',
+ type: FormFieldType.Password,
+ required: false,
+ shouldRender: (values: any) => values?.config?.auth_type === 'basic',
+ customValidate: (val: string, values: any) => {
+ if (values?.config?.auth_type === 'basic' && !val) {
+ return t('setting.restApiValidationBasicPasswordRequired');
+ }
+ return true;
+ },
+ },
+ {
+ label: 'Content Fields',
+ name: 'config.content_fields',
+ type: FormFieldType.Text,
+ required: true,
+ placeholder: 'title,body',
+ tooltip: t('setting.restApiContentFieldsTip'),
+ },
+ {
+ label: 'Metadata Fields',
+ name: 'config.metadata_fields',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: 'author,category',
+ tooltip: t('setting.restApiMetadataFieldsTip'),
+ },
+ {
+ label: 'Pagination Type',
+ name: 'config.pagination_type',
+ type: FormFieldType.Select,
+ required: true,
+ options: [
+ { label: 'None', value: 'none' },
+ { label: 'Page', value: 'page' },
+ { label: 'Offset', value: 'offset' },
+ { label: 'Cursor', value: 'cursor' },
+ ],
+ defaultValue: 'none',
+ },
+ {
+ label: 'Start Page',
+ name: 'config.pagination_config.start_page',
+ type: FormFieldType.Number,
+ required: false,
+ defaultValue: 1,
+ shouldRender: (values: any) => values?.config?.pagination_type === 'page',
+ },
+ {
+ label: 'Offset Param',
+ name: 'config.pagination_config.offset_param',
+ type: FormFieldType.Text,
+ required: false,
+ defaultValue: 'offset',
+ shouldRender: (values: any) =>
+ values?.config?.pagination_type === 'offset',
+ },
+ {
+ label: 'Start Offset',
+ name: 'config.pagination_config.start_offset',
+ type: FormFieldType.Number,
+ required: false,
+ defaultValue: 0,
+ shouldRender: (values: any) =>
+ values?.config?.pagination_type === 'offset',
+ },
+ {
+ label: 'Cursor Param',
+ name: 'config.pagination_config.cursor_param',
+ type: FormFieldType.Text,
+ required: false,
+ defaultValue: 'cursor',
+ shouldRender: (values: any) =>
+ values?.config?.pagination_type === 'cursor',
+ },
+ {
+ label: 'Next Cursor JSONPath',
+ name: 'config.pagination_config.next_cursor_path',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: '$.next_cursor',
+ shouldRender: (values: any) =>
+ values?.config?.pagination_type === 'cursor',
+ tooltip: t('setting.restApiNextCursorPathTip'),
+ },
+ // ── Advanced settings toggle ──────────────────────────────────────
+ {
+ label: 'Advanced Settings',
+ name: 'config.show_advanced',
+ type: FormFieldType.Switch,
+ required: false,
+ defaultValue: false,
+ },
+ // ── Advanced fields (hidden until toggled) ────────────────────────
+ {
+ label: 'Custom Headers (JSON)',
+ name: 'config.headers',
+ type: FormFieldType.Textarea,
+ required: false,
+ placeholder: `{"X-Custom-Header": "value"}`,
+ tooltip: t('setting.restApiHeadersTip'),
+ shouldRender: (values: any) => !!values?.config?.show_advanced,
+ },
+ {
+ label: 'Limit Param',
+ name: 'config.pagination_config.limit_param',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: 'limit (leave empty if already in Query Parameters)',
+ shouldRender: (values: any) =>
+ !!values?.config?.show_advanced &&
+ values?.config?.pagination_type === 'offset',
+ },
+ {
+ label: 'Initial Cursor',
+ name: 'config.pagination_config.initial_cursor',
+ type: FormFieldType.Text,
+ required: false,
+ shouldRender: (values: any) =>
+ !!values?.config?.show_advanced &&
+ values?.config?.pagination_type === 'cursor',
+ },
+ {
+ label: 'Max Pages',
+ name: 'config.max_pages',
+ type: FormFieldType.Number,
+ required: false,
+ defaultValue: 1000,
+ shouldRender: (values: any) => !!values?.config?.show_advanced,
+ },
+ {
+ label: 'Request Delay (seconds)',
+ name: 'config.request_delay',
+ type: FormFieldType.Number,
+ required: false,
+ defaultValue: 0.5,
+ placeholder: '0.5',
+ tooltip: t('setting.restApiRequestDelayTip'),
+ shouldRender: (values: any) => !!values?.config?.show_advanced,
+ },
+ {
+ label: 'Poll Timestamp Field',
+ name: 'config.poll_timestamp_field',
+ type: FormFieldType.Text,
+ required: false,
+ placeholder: 'updated_at',
+ tooltip: t('setting.restApiPollTimestampFieldTip'),
+ shouldRender: (values: any) => !!values?.config?.show_advanced,
+ },
+ {
+ label: 'Request Body (POST) JSON',
+ name: 'config.request_body',
+ type: FormFieldType.Textarea,
+ required: false,
+ placeholder: `{"status": "published"}`,
+ tooltip: t('setting.restApiRequestBodyTip'),
+ shouldRender: (values: any) =>
+ !!values?.config?.show_advanced && values?.config?.method === 'POST',
+ },
+ ],
};
export const DataSourceFormDefaultValues = {
@@ -1477,4 +1722,74 @@ export const DataSourceFormDefaultValues = {
},
},
},
+ [DataSourceKey.REST_API]: {
+ name: '',
+ source: DataSourceKey.REST_API,
+ config: {
+ url: '',
+ method: 'GET',
+ query_params: '',
+ headers: '',
+ auth_type: 'none',
+ auth_config: {},
+ items_path: '',
+ id_field: '',
+ content_fields: '',
+ metadata_fields: '',
+ pagination_type: 'none',
+ pagination_config: {},
+ poll_timestamp_field: '',
+ request_body: '',
+ max_pages: 1000,
+ request_delay: 0.5,
+ show_advanced: false,
+ credentials: {
+ api_key: '',
+ token: '',
+ username: '',
+ password: '',
+ },
+ },
+ },
+};
+
+export const getDataSourceFieldsWithExtras = (
+ source?: DataSourceKey,
+): FormFieldConfig[] => {
+ if (!source) {
+ return [];
+ }
+
+ const sourceFields =
+ DataSourceFormFields[source as keyof typeof DataSourceFormFields] || [];
+ const extraFields = getCommonExtraFields(source);
+
+ if (source !== DataSourceKey.JIRA) {
+ return [...sourceFields, ...extraFields];
+ }
+
+ const modeFieldIndex = sourceFields.findIndex(
+ (field) => field.name === 'config.is_cloud',
+ );
+ if (modeFieldIndex < 0) {
+ return [...sourceFields, ...extraFields];
+ }
+
+ const sharedFields = sourceFields.slice(0, modeFieldIndex);
+ const modeFields = sourceFields.slice(modeFieldIndex);
+
+ const sharedCheckboxFieldIndex = sharedFields.findIndex(
+ (field) => field.type === FormFieldType.Checkbox,
+ );
+
+ if (sharedCheckboxFieldIndex < 0) {
+ return [...sharedFields, ...extraFields, ...modeFields];
+ }
+
+ return [
+ ...sharedFields.slice(0, sharedCheckboxFieldIndex),
+ ...sharedFields.slice(sharedCheckboxFieldIndex),
+ ...extraFields,
+ ...modeFields,
+ ];
};
diff --git a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
index 1a4554abeb7..dfeb7e0830e 100644
--- a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
+++ b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
@@ -17,6 +17,7 @@ import { FieldValues } from 'react-hook-form';
import {
DataSourceFormBaseFields,
DataSourceFormDefaultValues,
+ DataSourceKey,
getCommonExtraDefaultValues,
getDataSourceFieldsWithExtras,
mergeDataSourceFormValues,
@@ -26,6 +27,7 @@ import {
useAddDataSource,
useDataSourceResume,
useFetchDataSourceDetail,
+ useTestDataSource,
} from '../hooks';
import { DataSourceLogsTable } from './log-table';
@@ -144,6 +146,7 @@ const SourceDetailPage = () => {
}, [detail, runSchedule]);
const { addLoading, handleAddOk } = useAddDataSource({ isEdit: true });
+ const { loading: testLoading, handleTest } = useTestDataSource();
const onSubmit = useCallback(() => {
formRef?.current?.submit();
@@ -187,7 +190,6 @@ const SourceDetailPage = () => {
detail as FieldValues,
),
};
- console.log('defaultValue', defaultValueTemp);
setDefaultValues(defaultValueTemp);
}
}, [detail, customFields, onSubmit]);
@@ -213,7 +215,18 @@ const SourceDetailPage = () => {
defaultValues={defaultValues}
/>
-
+
+ {detail?.source === DataSourceKey.REST_API && (
+
+ {t('setting.restApiTestConnection')}
+
+ )}
{
);
return { handleRebuild };
};
+
+export const useTestDataSource = () => {
+ const [currentQueryParameters] = useSearchParams();
+ const id = currentQueryParameters.get('id');
+ const [loading, setLoading] = useState(false);
+
+ const handleTest = useCallback(async () => {
+ if (!id) return;
+ setLoading(true);
+ try {
+ const { data } = await testDataSource(id);
+ if (data.code === 0) {
+ message.success(t('setting.restApiTestSuccess'));
+ } else {
+ message.error(data.message || t('setting.restApiTestFailed'));
+ }
+ } catch {
+ message.error(t('setting.restApiTestFailed'));
+ } finally {
+ setLoading(false);
+ }
+ }, [id]);
+
+ return { loading, handleTest };
+};
diff --git a/web/src/services/data-source-service.ts b/web/src/services/data-source-service.ts
index f14b65a111d..7be85dce85a 100644
--- a/web/src/services/data-source-service.ts
+++ b/web/src/services/data-source-service.ts
@@ -37,6 +37,9 @@ export const getDataSourceLogs = (id: string, params?: any) =>
export const featchDataSourceDetail = (id: string) =>
request.get(api.dataSourceDetail(id));
+export const testDataSource = (id: string) =>
+ request.post(api.dataSourceTest(id));
+
export const startGoogleDriveWebAuth = (payload: {
credentials: string;
redirect_uri?: string;
diff --git a/web/src/services/skill-space-service.ts b/web/src/services/skill-space-service.ts
index 04bff34d770..fe549c52897 100644
--- a/web/src/services/skill-space-service.ts
+++ b/web/src/services/skill-space-service.ts
@@ -133,19 +133,12 @@ class SkillSpaceService {
// Create a new skill space
async createSpace(request: CreateSpaceRequest): Promise {
- return await this.request(
- 'POST',
- api.skillSpaces,
- request,
- );
+ return await this.request('POST', api.skillSpaces, request);
}
// Get a skill space by ID
async getSpace(spaceId: string): Promise {
- return await this.request(
- 'GET',
- api.skillSpace(spaceId),
- );
+ return await this.request('GET', api.skillSpace(spaceId));
}
// Update a skill space
@@ -162,20 +155,14 @@ class SkillSpaceService {
// Delete a skill space
async deleteSpace(spaceId: string): Promise {
- await this.request(
- 'DELETE',
- api.skillSpace(spaceId),
- );
+ await this.request('DELETE', api.skillSpace(spaceId));
}
// Get space by folder ID
async getSpaceByFolder(folderId: string): Promise {
- return await this.request(
- 'GET',
- api.skillSpaceByFolder,
- null,
- { folder_id: folderId },
- );
+ return await this.request('GET', api.skillSpaceByFolder, null, {
+ folder_id: folderId,
+ });
}
// ==================== Skill Search Config ====================
@@ -210,11 +197,7 @@ class SkillSpaceService {
// Search skills
async search(request: SearchRequest): Promise {
- return await this.request(
- 'POST',
- api.skillSearch,
- request,
- );
+ return await this.request('POST', api.skillSearch, request);
}
// ==================== Skill Indexing ====================
@@ -235,21 +218,12 @@ class SkillSpaceService {
const params: Record = { skill_id: skillId };
if (spaceId) params.space_id = spaceId;
- await this.request(
- 'DELETE',
- api.skillIndex,
- null,
- params,
- );
+ await this.request('DELETE', api.skillIndex, null, params);
}
// Reindex all skills
async reindex(request: IndexSkillsRequest): Promise {
- return await this.request(
- 'POST',
- api.skillReindex,
- request,
- );
+ return await this.request('POST', api.skillReindex, request);
}
}
diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts
index fbde70b7fc9..f1e67986d35 100644
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -43,6 +43,7 @@ export default {
dataSourceRebuild: (id: string) => `${restAPIv1}/connectors/${id}/rebuild`,
dataSourceLogs: (id: string) => `${restAPIv1}/connectors/${id}/logs`,
dataSourceDetail: (id: string) => `${restAPIv1}/connectors/${id}`,
+ dataSourceTest: (id: string) => `${restAPIv1}/connectors/${id}/test`,
googleWebAuthStart: (type: 'google-drive' | 'gmail') =>
`${restAPIv1}/connectors/google/oauth/web/start?type=${type}`,
googleWebAuthResult: (type: 'google-drive' | 'gmail') =>
From 8c5845f6ca5e7add9c4e02660f24477750467fe7 Mon Sep 17 00:00:00 2001
From: "Ethan T."
Date: Wed, 13 May 2026 21:09:51 +0800
Subject: [PATCH 123/666] fix: use context manager for pdfplumber to prevent
resource leak (#13512)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
- Convert `pdfplumber.open()` to use `with` context manager in
`api/utils/file_utils.py` (`thumbnail_img` function)
- If any exception occurs between `open()` and `close()`, the PDF file
handle leaks
- The rest of the codebase (e.g. `read_potential_broken_pdf` in the same
file) already uses `with pdfplumber.open(...)` correctly
## Test plan
- [x] PDF thumbnail generation works correctly with context manager
- [x] Resources properly cleaned up on exceptions
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude Opus 4.6
---
api/utils/file_utils.py | 32 +++++++++++++++-----------------
1 file changed, 15 insertions(+), 17 deletions(-)
diff --git a/api/utils/file_utils.py b/api/utils/file_utils.py
index 857cf17381d..21b746f8f18 100644
--- a/api/utils/file_utils.py
+++ b/api/utils/file_utils.py
@@ -107,23 +107,21 @@ def thumbnail_img(filename, blob):
if re.match(r".*\.pdf$", filename):
try:
with sys.modules[LOCK_KEY_pdfplumber]:
- pdf = pdfplumber.open(BytesIO(blob))
- if not pdf.pages:
- pdf.close()
- return None
- buffered = BytesIO()
- resolution = 32
- img = None
- for _ in range(10):
- pdf.pages[0].to_image(resolution=resolution).annotated.save(buffered, format="png")
- img = buffered.getvalue()
- if len(img) >= 64000 and resolution >= 2:
- resolution = resolution / 2
- buffered = BytesIO()
- else:
- break
- pdf.close()
- return img
+ with pdfplumber.open(BytesIO(blob)) as pdf:
+ if not pdf.pages:
+ return None
+ buffered = BytesIO()
+ resolution = 32
+ img = None
+ for _ in range(10):
+ pdf.pages[0].to_image(resolution=resolution).annotated.save(buffered, format="png")
+ img = buffered.getvalue()
+ if len(img) >= 64000 and resolution >= 2:
+ resolution = resolution / 2
+ buffered = BytesIO()
+ else:
+ break
+ return img
except Exception:
return None
From 9e0f9767296fb28a8682083d2a67f60d82d9667f Mon Sep 17 00:00:00 2001
From: 47NoahThompson <107704814+47NoahThompson@users.noreply.github.com>
Date: Wed, 13 May 2026 09:13:11 -0400
Subject: [PATCH 124/666] Add widget customization and persistence (#14603)
Introduce comprehensive floating widget customization: add new widget
settings (title, subtitle, footer, colors, mute, streaming) with types
and defaults, and expose them via EmbedDialog UI (split into Embed Setup
and Widget Customization tabs). Persist and load settings through Agent
page by reading/writing globals and wiring an onSaveWidgetSettings
handler to setAgent; show a loading ButtonLoading for saving. Update
embed iframe query params and FloatingChatWidget to honor URL params
(colors, text, mute/streaming) with validation/normalization, color
darkening for gradients, footer link normalization, and improved
styling. Also add copy-to-clipboard in message toolbar, adjust syntax
highlighter layout and Copy button, and add i18n key for muteWidget.
### What problem does this PR solve?
Adds a few fields to the embed widget modal to customize the appearance
of the floating widget when embedded into a page.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---------
Co-authored-by: Noah
---
web/src/components/embed-dialog/index.tsx | 507 ++++++++++++++++----
web/src/components/floating-chat-widget.tsx | 350 +++++++++++---
web/src/locales/en.ts | 1 +
web/src/pages/agent/index.tsx | 53 +-
4 files changed, 742 insertions(+), 169 deletions(-)
diff --git a/web/src/components/embed-dialog/index.tsx b/web/src/components/embed-dialog/index.tsx
index d2656b2ae07..dbb45df2471 100644
--- a/web/src/components/embed-dialog/index.tsx
+++ b/web/src/components/embed-dialog/index.tsx
@@ -1,6 +1,6 @@
import CopyToClipboard from '@/components/copy-to-clipboard';
import { SelectWithSearch } from '@/components/originui/select-with-search';
-import { Button } from '@/components/ui/button';
+import { Button, ButtonLoading } from '@/components/ui/button';
import {
Dialog,
DialogContent,
@@ -17,6 +17,7 @@ import {
} from '@/components/ui/form';
import { Label } from '@/components/ui/label';
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
+import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
import { SharedFrom } from '@/constants/chat';
import {
LanguageAbbreviation,
@@ -48,23 +49,78 @@ const FormSchema = z.object({
locale: z.string(),
embedType: z.enum(['fullscreen', 'widget']),
enableStreaming: z.boolean(),
+ muteWidget: z.boolean(),
theme: z.enum([ThemeEnum.Light, ThemeEnum.Dark]),
userId: z.string().optional(),
+ widgetTitle: z.string(),
+ widgetSubtitle: z.string(),
+ widgetFooterText: z.string(),
+ widgetFooterLink: z.string(),
+ widgetAccentColor: z.string(),
+ widgetBackgroundColor: z.string(),
+ widgetTextColor: z.string(),
+ widgetHeaderTextColor: z.string(),
+ widgetFooterTextColor: z.string(),
});
+export type WidgetSettings = Pick<
+ z.infer,
+ | 'enableStreaming'
+ | 'muteWidget'
+ | 'widgetTitle'
+ | 'widgetSubtitle'
+ | 'widgetFooterText'
+ | 'widgetFooterLink'
+ | 'widgetAccentColor'
+ | 'widgetBackgroundColor'
+ | 'widgetTextColor'
+ | 'widgetHeaderTextColor'
+ | 'widgetFooterTextColor'
+>;
+
+export const defaultWidgetSettings: WidgetSettings = {
+ enableStreaming: false,
+ muteWidget: false,
+ widgetTitle: '',
+ widgetSubtitle: '',
+ widgetFooterText: '',
+ widgetFooterLink: '',
+ widgetAccentColor: '#2563eb',
+ widgetBackgroundColor: '#ffffff',
+ widgetTextColor: '#111827',
+ widgetHeaderTextColor: '#ffffff',
+ widgetFooterTextColor: '#111827',
+};
+
type IProps = IModalProps & {
token: string;
from: SharedFrom;
beta: string;
isAgent: boolean;
+ initialWidgetSettings?: Partial;
+ onSaveWidgetSettings?: (settings: WidgetSettings) => Promise;
+ savingWidgetSettings?: boolean;
+};
+
+const normalizeHexColor = (value: string | undefined, fallback: string) => {
+ const normalizedValue = value?.trim() ?? '';
+ return /^#([0-9a-f]{3}|[0-9a-f]{6})$/i.test(normalizedValue)
+ ? normalizedValue
+ : fallback;
};
+/**
+ * Builds the embed code preview and customization UI for shared chat and agent widgets.
+ */
function EmbedDialog({
hideModal,
token = '',
from,
beta = '',
isAgent,
+ initialWidgetSettings,
+ onSaveWidgetSettings,
+ savingWidgetSettings,
visible,
}: IProps) {
const { t } = useTranslation();
@@ -77,8 +133,9 @@ function EmbedDialog({
published: false,
locale: '',
embedType: 'fullscreen' as const,
- enableStreaming: false,
theme: ThemeEnum.Light,
+ ...defaultWidgetSettings,
+ ...initialWidgetSettings,
},
});
@@ -98,8 +155,18 @@ function EmbedDialog({
locale,
embedType,
enableStreaming,
+ muteWidget,
theme,
userId,
+ widgetTitle,
+ widgetSubtitle,
+ widgetFooterText,
+ widgetFooterLink,
+ widgetAccentColor,
+ widgetBackgroundColor,
+ widgetTextColor,
+ widgetHeaderTextColor,
+ widgetFooterTextColor,
} = values;
const baseRoute =
embedType === 'widget'
@@ -125,6 +192,39 @@ function EmbedDialog({
if (embedType === 'widget') {
src.searchParams.append('mode', 'master');
src.searchParams.append('streaming', String(enableStreaming));
+ src.searchParams.append('muted', String(muteWidget));
+ if (!isEmpty(trim(widgetTitle))) {
+ src.searchParams.append('widget_title', widgetTitle ?? '');
+ }
+ if (!isEmpty(trim(widgetSubtitle))) {
+ src.searchParams.append('widget_subtitle', widgetSubtitle ?? '');
+ }
+ if (!isEmpty(trim(widgetFooterText))) {
+ src.searchParams.append('widget_footer', widgetFooterText ?? '');
+ }
+ if (!isEmpty(trim(widgetFooterLink))) {
+ src.searchParams.append('widget_footer_link', widgetFooterLink ?? '');
+ }
+ src.searchParams.append(
+ 'widget_accent_color',
+ normalizeHexColor(widgetAccentColor, '#2563eb'),
+ );
+ src.searchParams.append(
+ 'widget_background_color',
+ normalizeHexColor(widgetBackgroundColor, '#ffffff'),
+ );
+ src.searchParams.append(
+ 'widget_text_color',
+ normalizeHexColor(widgetTextColor, '#111827'),
+ );
+ src.searchParams.append(
+ 'widget_header_text_color',
+ normalizeHexColor(widgetHeaderTextColor, '#ffffff'),
+ );
+ src.searchParams.append(
+ 'widget_footer_text_color',
+ normalizeHexColor(widgetFooterTextColor, '#111827'),
+ );
}
if (theme && embedType === 'fullscreen') {
src.searchParams.append('theme', theme);
@@ -179,9 +279,29 @@ window.addEventListener('message',e=>{
window.open(iframeSrc, '_blank');
}, [generateIframeSrc]);
+ const handleSaveWidgetSettings = useCallback(async () => {
+ if (!onSaveWidgetSettings) {
+ return;
+ }
+
+ await onSaveWidgetSettings({
+ enableStreaming: values.enableStreaming,
+ muteWidget: values.muteWidget,
+ widgetTitle: values.widgetTitle,
+ widgetSubtitle: values.widgetSubtitle,
+ widgetFooterText: values.widgetFooterText,
+ widgetFooterLink: values.widgetFooterLink,
+ widgetAccentColor: values.widgetAccentColor,
+ widgetBackgroundColor: values.widgetBackgroundColor,
+ widgetTextColor: values.widgetTextColor,
+ widgetHeaderTextColor: values.widgetHeaderTextColor,
+ widgetFooterTextColor: values.widgetFooterTextColor,
+ });
+ }, [onSaveWidgetSettings, values]);
+
return (
-
+
{t('common.embedIntoSite')}
@@ -189,103 +309,274 @@ window.addEventListener('message',e=>{
{t('search.embedCode')}
-
+
+
@@ -293,14 +584,26 @@ window.addEventListener('message',e=>{
-
-
- {t('common.openInNewTab')}
-
+
+ {isAgent && onSaveWidgetSettings && (
+
+ {t('flow.save')} widget settings
+
+ )}
+
+
+ {t('common.openInNewTab')}
+
+
{t(isAgent ? 'flow' : 'chat', { keyPrefix: 'header' })}
ID
diff --git a/web/src/components/floating-chat-widget.tsx b/web/src/components/floating-chat-widget.tsx
index 46fb49482a4..9b5c375cff3 100644
--- a/web/src/components/floating-chat-widget.tsx
+++ b/web/src/components/floating-chat-widget.tsx
@@ -1,3 +1,4 @@
+import CopyToClipboard from '@/components/copy-to-clipboard';
import PdfSheet from '@/components/pdf-drawer';
import { useClickDrawer } from '@/components/pdf-drawer/hooks';
import { MessageType, SharedFrom } from '@/constants/chat';
@@ -14,6 +15,75 @@ import {
} from '../pages/next-chats/hooks/use-send-shared-message';
import FloatingChatWidgetMarkdown from './floating-chat-widget-markdown';
+/**
+ * Normalizes a hex color input and falls back to a safe default when invalid.
+ */
+const normalizeHexColor = (value: string | null, fallback: string) => {
+ return value && /^#([0-9a-f]{3}|[0-9a-f]{6})$/i.test(value)
+ ? value
+ : fallback;
+};
+
+/**
+ * Darkens a hex color to derive hover and gradient variants for the widget chrome.
+ */
+const darkenHexColor = (hexColor: string, amount = 0.12) => {
+ const normalizedHex = hexColor.replace('#', '');
+ const expandedHex =
+ normalizedHex.length === 3
+ ? normalizedHex
+ .split('')
+ .map((char) => `${char}${char}`)
+ .join('')
+ : normalizedHex;
+ const channels = expandedHex.match(/.{2}/g);
+
+ if (!channels) {
+ return hexColor;
+ }
+
+ return `#${channels
+ .map((channel) => {
+ const value = parseInt(channel, 16);
+ const adjustedValue = Math.max(
+ 0,
+ Math.min(255, Math.round(value * (1 - amount))),
+ );
+ return adjustedValue.toString(16).padStart(2, '0');
+ })
+ .join('')}`;
+};
+
+/**
+ * Accepts a footer link from the widget query string and returns a safe HTTP(S) URL.
+ */
+const normalizeWidgetFooterLink = (value: string | null) => {
+ const normalizedValue = value?.trim();
+
+ if (!normalizedValue) {
+ return undefined;
+ }
+
+ const candidate = /^[a-z][a-z\d+.-]*:/i.test(normalizedValue)
+ ? normalizedValue
+ : `https://${normalizedValue}`;
+
+ try {
+ const url = new URL(candidate);
+
+ if (url.protocol === 'http:' || url.protocol === 'https:') {
+ return url.toString();
+ }
+ } catch {
+ return undefined;
+ }
+
+ return undefined;
+};
+
+/**
+ * Renders the embeddable floating chat widget and applies URL-driven widget settings.
+ */
const FloatingChatWidget = () => {
const { t } = useTranslation();
const [isOpen, setIsOpen] = useState(false);
@@ -36,6 +106,34 @@ const FloatingChatWidget = () => {
const urlParams = new URLSearchParams(window.location.search);
const mode = urlParams.get('mode') || 'full'; // 'button', 'window', or 'full'
const enableStreaming = urlParams.get('streaming') === 'true'; // Only enable if explicitly set to true
+ const isMuted = urlParams.get('muted') === 'true';
+ const widgetTitle = urlParams.get('widget_title')?.trim();
+ const widgetSubtitle = urlParams.get('widget_subtitle')?.trim();
+ const widgetFooter = urlParams.get('widget_footer')?.trim();
+ const widgetFooterLink = normalizeWidgetFooterLink(
+ urlParams.get('widget_footer_link'),
+ );
+ const widgetAccentColor = normalizeHexColor(
+ urlParams.get('widget_accent_color'),
+ '#2563eb',
+ );
+ const widgetAccentColorStrong = darkenHexColor(widgetAccentColor);
+ const widgetBackgroundColor = normalizeHexColor(
+ urlParams.get('widget_background_color'),
+ '#ffffff',
+ );
+ const widgetTextColor = normalizeHexColor(
+ urlParams.get('widget_text_color'),
+ '#111827',
+ );
+ const widgetHeaderTextColor = normalizeHexColor(
+ urlParams.get('widget_header_text_color'),
+ '#ffffff',
+ );
+ const widgetFooterTextColor = normalizeHexColor(
+ urlParams.get('widget_footer_text_color'),
+ '#111827',
+ );
const {
handlePressEnter,
@@ -58,6 +156,49 @@ const FloatingChatWidget = () => {
)();
const title = data.title;
+ const displayTitle = widgetTitle || title || t('chat.chatSupport');
+ const displaySubtitle = widgetSubtitle || t('chat.replyInstantly');
+ const displayFooter = widgetFooter || '';
+ const renderFooter = () => {
+ if (!displayFooter) {
+ return null;
+ }
+
+ return (
+
+ );
+ };
+ const bodyContainerStyle: React.CSSProperties = {
+ borderRadius: '0 0 16px 16px',
+ backgroundColor: widgetBackgroundColor,
+ color: widgetTextColor,
+ };
+ const inputStyle: React.CSSProperties = {
+ minHeight: '44px',
+ maxHeight: '120px',
+ color: widgetTextColor,
+ backgroundColor: widgetBackgroundColor,
+ };
const { visible, hideModal, documentId, selectedChunk, clickDocumentButton } =
useClickDrawer();
@@ -69,6 +210,10 @@ const FloatingChatWidget = () => {
// Play sound when opening
const playNotificationSound = useCallback(() => {
+ if (isMuted) {
+ return;
+ }
+
try {
const audioContext = new (
window.AudioContext || (window as any).webkitAudioContext
@@ -94,10 +239,14 @@ const FloatingChatWidget = () => {
console.warn(error);
// Silent fail if audio not supported
}
- }, []);
+ }, [isMuted]);
// Play sound for AI responses (Intercom-style)
const playResponseSound = useCallback(() => {
+ if (isMuted) {
+ return;
+ }
+
try {
const audioContext = new (
window.AudioContext || (window as any).webkitAudioContext
@@ -124,7 +273,7 @@ const FloatingChatWidget = () => {
// Silent fail if audio not supported
}
- }, []);
+ }, [isMuted]);
// Set loaded state and locale
useEffect(() => {
@@ -338,9 +487,10 @@ const FloatingChatWidget = () => {
'*',
);
}}
- className={`w-14 h-14 bg-blue-600 hover:bg-blue-700 text-white rounded-full transition-all duration-300 flex items-center justify-center group ${
+ className={`w-14 h-14 text-white rounded-full transition-all duration-300 flex items-center justify-center group ${
isOpen ? 'scale-95' : 'scale-100 hover:scale-105'
}`}
+ style={{ backgroundColor: widgetAccentColor }}
>
{
{
return (
<>
{/* Header */}
-
+
-
- {title || t('chat.chatSupport')}
+
+ {displayTitle}
-
- {t('chat.replyInstantly')}
+
+ {displaySubtitle}
{/* Messages and Input */}
-
+
{
@@ -447,29 +604,49 @@ const FloatingChatWidget = () => {
className={`flex ${message.role === MessageType.User ? 'justify-end' : 'justify-start'}`}
>
{message.role === MessageType.User ? (
{message.content}
) : (
-
+
+ clickDocumentButton={clickDocumentButton}
+ />
+
+
+
+
)}
@@ -479,14 +656,23 @@ const FloatingChatWidget = () => {
{sendLoading && !enableStreaming && (
@@ -496,7 +682,10 @@ const FloatingChatWidget = () => {
{/* Input Area */}
-
+
@@ -518,11 +707,13 @@ const FloatingChatWidget = () => {
type="button"
onClick={handleSendMessage}
disabled={!inputValue.trim() || sendLoading}
- className="p-3 bg-blue-600 text-white rounded-full hover:bg-blue-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
+ className="p-3 text-white rounded-full disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
+ style={{ backgroundColor: widgetAccentColor }}
>
+ {renderFooter()}
@@ -546,22 +737,31 @@ const FloatingChatWidget = () => {
{/* Chat Widget Container */}
{isOpen && (
{/* Header */}
-
+
-
- {title || t('chat.chatSupport')}
+
+ {displayTitle}
-
- {t('chat.replyInstantly')}
+
+ {displaySubtitle}
@@ -585,10 +785,7 @@ const FloatingChatWidget = () => {
{/* Messages Container */}
{!isMinimized && (
-
+
{
@@ -621,29 +818,49 @@ const FloatingChatWidget = () => {
className={`flex ${message.role === MessageType.User ? 'justify-end' : 'justify-start'}`}
>
{message.role === MessageType.User ? (
{message.content}
) : (
-
+
+ clickDocumentButton={clickDocumentButton}
+ />
+
+
+
+
)}
@@ -686,8 +903,8 @@ const FloatingChatWidget = () => {
onKeyPress={handleKeyPress}
placeholder={t('chat.typeYourMessage')}
rows={1}
- className="w-full resize-none border border-gray-300 rounded-2xl px-4 py-3 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent text-black"
- style={{ minHeight: '44px', maxHeight: '120px' }}
+ className="w-full resize-none border border-gray-300 rounded-2xl px-4 py-3 text-sm focus:outline-none focus:ring-2 focus:border-transparent"
+ style={inputStyle}
disabled={hasError || sendLoading}
/>
@@ -695,11 +912,13 @@ const FloatingChatWidget = () => {
type="button"
onClick={handleSendMessage}
disabled={!inputValue.trim() || sendLoading}
- className="p-3 bg-blue-600 text-white rounded-full hover:bg-blue-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
+ className="p-3 text-white rounded-full disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
+ style={{ backgroundColor: widgetAccentColor }}
>
+ {renderFooter()}
)}
@@ -711,9 +930,10 @@ const FloatingChatWidget = () => {
{
@@ -211,6 +226,37 @@ export default function Agent() {
uploadedFileData,
} = useRunDataflow({ showLogSheet: showPipelineLogSheet, setMessageId });
+ const initialWidgetSettings = useMemo
(() => {
+ const widgetSettings =
+ agentDetail?.dsl?.globals?.[AgentWidgetSettingsGlobalKey];
+
+ return {
+ ...defaultWidgetSettings,
+ ...(widgetSettings && typeof widgetSettings === 'object'
+ ? widgetSettings
+ : {}),
+ };
+ }, [agentDetail]);
+
+ const handleSaveWidgetSettings = useCallback(
+ async (widgetSettings: WidgetSettings) => {
+ const dsl = buildDslData();
+
+ return setAgent({
+ id: id!,
+ title: agentDetail.title,
+ dsl: {
+ ...dsl,
+ globals: {
+ ...dsl.globals,
+ [AgentWidgetSettingsGlobalKey]: widgetSettings,
+ },
+ },
+ });
+ },
+ [agentDetail.title, buildDslData, id, setAgent],
+ );
+
return (
@@ -327,6 +373,9 @@ export default function Agent() {
from={SharedFrom.Agent}
beta={beta}
isAgent
+ initialWidgetSettings={initialWidgetSettings}
+ onSaveWidgetSettings={handleSaveWidgetSettings}
+ savingWidgetSettings={savingWidgetSettings}
>
)}
{versionDialogVisible && (
From cb49f47c38bf583bcd56a711714009388b7aa128 Mon Sep 17 00:00:00 2001
From: writinwaters <93570324+writinwaters@users.noreply.github.com>
Date: Wed, 13 May 2026 21:36:34 +0800
Subject: [PATCH 125/666] Docs: Editorial updates to the v0.25.3 release notes
draft. (#14903)
### What problem does this PR solve?
v0.25.3 release notes. To be continued.
### Type of change
- [x] Documentation Update
---
docs/release_notes.md | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/docs/release_notes.md b/docs/release_notes.md
index 4813089d752..934f3aefb42 100644
--- a/docs/release_notes.md
+++ b/docs/release_notes.md
@@ -18,23 +18,25 @@ Released on May 13, 2026.
- Data source and parsing: Added column-level semantic/metadata control for the spreadsheet file parser; introduced ETag optimization for incremental synchronization of S3 data sources to avoid unnecessary file transfers.
- Enables assigning specific roles like content, metadata, and primary key, to table columns. [#13710](https://github.com/infiniflow/ragflow/pull/13710)
+
## Improvements
- API refactoring and security
- Continues the transition of web APIs to RESTful conventions, ensuring backward compatibility for all legacy endpoints.
- Binds the `user_id` in `POST /api/v1/messages` to the authenticated JWT principal. [#14745](https://github.com/infiniflow/ragflow/pull/14745)
- Secures the sandbox executor against dynamic and indirect code execution bypasses. [#14690](https://github.com/infiniflow/ragflow/pull/14690)
- - LLM request timeout control
- - High concurrency blocking call thread pool
+ - Enforces HTTP request timeouts across the LLM integration layer. [#14313](https://github.com/infiniflow/ragflow/pull/14313)
+- Refactors thread pool lifecycle management in `file_service.py` and `task_executor.py` for more efficient, lightweight resource handling. [#14668](https://github.com/infiniflow/ragflow/pull/14668)
+- Agent: Enables the **Code** component to output and display file-based attachments, such as charts and images, directly in the chat. [#14787](https://github.com/infiniflow/ragflow/pull/14787)
- Reduces ingestion server boot time. [#14894](https://github.com/infiniflow/ragflow/pull/14894)
## Bug fixes
- Images in multi-sheet Excel workbooks were not scoped by sheet, causing images to be incorrectly attributed across different worksheets. [#14120](https://github.com/infiniflow/ragflow/pull/14120)
-- Iteration item alias passing
+- Agent: Splits the **Message** component output into distinct 'waiting' and 'message' states when nested inside a **Iteration** component alongside a **Wait** component. [#14839](https://github.com/infiniflow/ragflow/pull/14839)
+- The **Iteration** component failed to correctly pass array elements to its child components due to a naming mismatch between the expected IterationItem alias and the runtime item variable. [#14146](https://github.com/infiniflow/ragflow/pull/14146)
- Tool parameter template parsing
-- Code execution attachment output
-- Volcano model addition fix
+- Volcengine (Doubao/Ark) endpoints were not visible in the provider list. [#14702](https://github.com/infiniflow/ragflow/pull/14702)
## v0.25.2
From dd76653dc14921988196e64396c38795e3858538 Mon Sep 17 00:00:00 2001
From: plind <59729252+plind-junior@users.noreply.github.com>
Date: Wed, 13 May 2026 06:41:32 -0700
Subject: [PATCH 126/666] feat: add tag management for Agents with filtering
and sorting (#14774) (#14799)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
Closes #14774.
Adds free-form tags on agents (UserCanvas) with full UI + API:
- Stored as comma-separated `tags` column on `UserCanvas` with online
migration.
- New endpoints: `GET /v1/agents/tags` (aggregate counts) and `PUT
/v1/agent//tags` (write). `GET /v1/agents` accepts a `tags=` query.
- "Edit tags" item in agent dropdown opens a chip-style editor dialog;
tags render as badges on each agent card.
- New "Tags" facet in the agents filter bar, with counts.
## Implementation notes
- **Tag matching is exact-token**: the SQL filter wraps stored tags as
`,…,` and matches `,ml,` so `ml` doesn't match `ml-ops`.
- **Server-side normalization** in `UserCanvasService.update_tags`:
dedup (case-insensitive), per-tag cap of 64 chars, total length capped
at 512 chars to fit the column, commas inside tag values are replaced
with spaces.
- **Tenant authorization**: `PUT /v1/agent//tags` gates on
`UserCanvasService.accessible(canvas_id, tenant_id)`.
- **Tag listing scope**: `UserCanvasService.list_tags` follows the same
own + team-shared rule as `get_by_tenant_ids`.
- **i18n**: keys added to `en.ts` and `zh.ts` only (per project
convention; other locales fall back).
- **`HomeCard`** gets a non-breaking `extra?: ReactNode` slot for the
chip row; no `src/components/ui/` files modified.
## Test plan
- [ ] Backend boot runs `migrate_db` → confirm `user_canvas.tags` column
exists (`DESCRIBE user_canvas`).
- [ ] Agents page renders cards normally (no console error from missing
field).
- [ ] `⋯ → Edit tags` opens a dialog that stays open (regression: dialog
was unmounting with the dropdown).
- [ ] Typing a tag without pressing Enter and clicking Save persists it
(regression: last typed tag was being dropped).
- [ ] Chip input supports Enter/comma to commit, Backspace on empty to
remove, `×` to remove individual chip.
- [ ] Tag containing a comma sent via API is stored with the comma
replaced by a space.
- [ ] 20 long tags sent via API does not error (length cap silently
truncates).
- [ ] "Tags" filter in the filter bar shows counts and narrows the list.
- [ ] Filtering by `ml` does **not** return agents tagged `ml-ops`.
- [ ] UI in Chinese shows 编辑标签 / 添加标签以整理和筛选你的智能体 etc.
- [ ] `PUT /v1/agent//tags` returns `Agent not found or
no permission.`
---
api/apps/restful_apis/agent_api.py | 61 ++++++
api/db/db_models.py | 2 +
api/db/services/canvas_service.py | 74 ++++++++
.../test_agents_webhook_unit.py | 3 +-
web/src/components/home-card.tsx | 3 +
web/src/hooks/use-agent-request.ts | 61 ++++++
web/src/interfaces/database/agent.ts | 1 +
web/src/locales/en.ts | 6 +
web/src/locales/zh.ts | 5 +
web/src/pages/agents/agent-card.tsx | 19 ++
web/src/pages/agents/agent-dropdown.tsx | 83 +++++----
web/src/pages/agents/agent-tag-editor.tsx | 176 ++++++++++++++++++
.../pages/agents/hooks/use-selelct-filters.ts | 21 ++-
web/src/services/agent-service.ts | 11 ++
web/src/utils/api.ts | 3 +
15 files changed, 494 insertions(+), 35 deletions(-)
create mode 100644 web/src/pages/agents/agent-tag-editor.tsx
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index 054117d2368..09f284e8953 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -316,6 +316,7 @@ def list_agents(tenant_id):
keywords = request.args.get("keywords", "")
canvas_category = request.args.get("canvas_category")
owner_ids = [item for item in request.args.get("owner_ids", "").strip().split(",") if item]
+ tags = [item for item in request.args.get("tags", "").strip().split(",") if item]
page_number = int(request.args.get("page", 0))
items_per_page = int(request.args.get("page_size", 0))
@@ -347,11 +348,71 @@ def list_agents(tenant_id):
desc,
keywords,
canvas_category,
+ tags,
)
return get_json_result(data={"canvas": canvas, "total": total})
+@manager.route("/agents/tags", methods=["GET"]) # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def list_agent_tags(tenant_id):
+ """Aggregate tag usage counts across agents visible to the caller."""
+ canvas_category = request.args.get("canvas_category")
+ tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
+ joined_ids = list({member["tenant_id"] for member in tenants} | {tenant_id})
+ counts = UserCanvasService.list_tags(joined_ids, tenant_id, canvas_category)
+ logging.info(
+ "list_agent_tags tenant=%s canvas_category=%s tags_count=%d",
+ tenant_id,
+ canvas_category,
+ len(counts),
+ )
+ return get_json_result(data=[{"tag": k, "count": v} for k, v in sorted(counts.items(), key=lambda x: (-x[1], x[0]))])
+
+
+@manager.route("/agents//tags", methods=["PUT"]) # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def update_agent_tags(tenant_id, canvas_id):
+ if not UserCanvasService.accessible(canvas_id, tenant_id):
+ logging.info(
+ "update_agent_tags denied tenant=%s canvas_id=%s reason=no_permission",
+ tenant_id,
+ canvas_id,
+ )
+ return get_json_result(
+ data=False,
+ message="Agent not found or no permission.",
+ code=RetCode.OPERATING_ERROR,
+ )
+ req = await get_request_json()
+ tags = req.get("tags", "")
+ incoming = tags if isinstance(tags, (list, tuple)) else [t for t in str(tags).split(",") if t.strip()]
+ rows_affected = UserCanvasService.update_tags(canvas_id, tags)
+ if rows_affected == 0:
+ logging.info(
+ "update_agent_tags miss tenant=%s canvas_id=%s incoming_count=%d rows=0",
+ tenant_id,
+ canvas_id,
+ len(incoming),
+ )
+ return get_json_result(
+ data=False,
+ message="Agent not found or no permission.",
+ code=RetCode.OPERATING_ERROR,
+ )
+ logging.info(
+ "update_agent_tags ok tenant=%s canvas_id=%s incoming_count=%d rows=%d",
+ tenant_id,
+ canvas_id,
+ len(incoming),
+ rows_affected,
+ )
+ return get_json_result(data=True)
+
+
@manager.route("/agents", methods=["POST"]) # noqa: F821
@login_required
@add_tenant_id_to_kwargs
diff --git a/api/db/db_models.py b/api/db/db_models.py
index 5fe64586c04..3ed32ed3f20 100644
--- a/api/db/db_models.py
+++ b/api/db/db_models.py
@@ -1051,6 +1051,7 @@ class UserCanvas(DataBaseModel):
description = TextField(null=True, help_text="Canvas description")
canvas_type = CharField(max_length=32, null=True, help_text="Canvas type", index=True)
canvas_category = CharField(max_length=32, null=False, default="agent_canvas", help_text="Canvas category: agent_canvas|dataflow_canvas", index=True)
+ tags = CharField(max_length=512, null=False, default="", help_text="Comma-separated tags for organizing agents", index=True)
dsl = JSONField(null=True, default={})
class Meta:
@@ -1647,6 +1648,7 @@ def migrate_db():
alter_db_add_column(migrator, "memory", "tenant_embd_id", IntegerField(null=True, help_text="id in tenant_llm", index=True))
alter_db_add_column(migrator, "memory", "tenant_llm_id", IntegerField(null=True, help_text="id in tenant_llm", index=True))
alter_db_add_column(migrator, "user_canvas_version", "release", BooleanField(null=False, help_text="is released", default=False, index=True))
+ alter_db_add_column(migrator, "user_canvas", "tags", CharField(max_length=512, null=False, default="", help_text="Comma-separated tags for organizing agents", index=True))
alter_db_add_column(migrator, "api_4_conversation", "version_title", CharField(max_length=255, null=True, help_text="canvas version title when session created", index=False))
alter_db_column_type(migrator, "document", "size", BigIntegerField(default=0, index=True))
alter_db_column_type(migrator, "file", "size", BigIntegerField(default=0, index=True))
diff --git a/api/db/services/canvas_service.py b/api/db/services/canvas_service.py
index 1c1583e8f68..8c7fe4748ff 100644
--- a/api/db/services/canvas_service.py
+++ b/api/db/services/canvas_service.py
@@ -16,6 +16,8 @@
import json
import logging
import time
+from functools import reduce
+from operator import or_
from uuid import uuid4
from agent.canvas import Canvas
from api.db import CanvasCategory, TenantPermission
@@ -149,6 +151,7 @@ def get_by_tenant_ids(
desc,
keywords,
canvas_category=None,
+ tags=None,
):
fields = [
cls.model.id,
@@ -161,6 +164,7 @@ def get_by_tenant_ids(
User.avatar.alias('tenant_avatar'),
cls.model.update_time,
cls.model.canvas_category,
+ cls.model.tags,
]
if keywords:
agents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
@@ -173,6 +177,13 @@ def get_by_tenant_ids(
)
if canvas_category:
agents = agents.where(cls.model.canvas_category == canvas_category)
+ if tags:
+ tag_list = [t.strip() for t in tags if t and t.strip()] if isinstance(tags, (list, tuple)) else [t.strip() for t in str(tags).split(",") if t.strip()]
+ if tag_list:
+ # Wrap value with commas so 'ml' doesn't match 'ml-ops'.
+ wrapped = fn.CONCAT(",", cls.model.tags, ",")
+ clauses = [wrapped.contains(f",{t},") for t in tag_list]
+ agents = agents.where(reduce(or_, clauses))
if desc:
agents = agents.order_by(cls.model.getter_by(orderby).desc())
else:
@@ -199,6 +210,69 @@ def get_by_tenant_ids(
return agents_list, count
+ @classmethod
+ @DB.connection_context()
+ def list_tags(cls, joined_tenant_ids, user_id, canvas_category=None):
+ """Return {tag: agent_count} aggregated across agents visible to the user."""
+ query = cls.model.select(cls.model.tags).where(
+ ((cls.model.user_id.in_(joined_tenant_ids)) & (cls.model.permission == TenantPermission.TEAM.value)) | (cls.model.user_id == user_id)
+ )
+ if canvas_category:
+ query = query.where(cls.model.canvas_category == canvas_category)
+
+ counts: dict[str, int] = {}
+ for row in query.dicts():
+ for t in (row.get("tags") or "").split(","):
+ t = t.strip()
+ if t:
+ counts[t] = counts.get(t, 0) + 1
+ logging.info(
+ "UserCanvasService.list_tags user=%s canvas_category=%s tags_count=%d",
+ user_id,
+ canvas_category,
+ len(counts),
+ )
+ return counts
+
+ # Tag storage is a single comma-separated CharField(max_length=512);
+ # commas inside a tag would corrupt the encoding, so strip them on write.
+ TAGS_FIELD_MAX = 512
+ TAG_MAX_LEN = 64
+
+ @classmethod
+ @DB.connection_context()
+ def update_tags(cls, canvas_id, tags):
+ """Persist a normalized comma-separated tag string for the given canvas."""
+ if isinstance(tags, (list, tuple)):
+ cleaned = [str(t).replace(",", " ").strip() for t in tags if t and str(t).strip()]
+ else:
+ cleaned = [t.strip() for t in str(tags or "").split(",") if t.strip()]
+ # Dedupe (case-insensitive, preserve order), cap individual tag length,
+ # then truncate the joined value so it always fits the column.
+ seen = set()
+ normalized = []
+ used = 0
+ for t in cleaned:
+ t = t[: cls.TAG_MAX_LEN]
+ key = t.lower()
+ if key in seen:
+ continue
+ extra = len(t) + (1 if normalized else 0)
+ if used + extra > cls.TAGS_FIELD_MAX:
+ break
+ seen.add(key)
+ normalized.append(t)
+ used += extra
+ value = ",".join(normalized)
+ rows_affected = cls.model.update(tags=value).where(cls.model.id == canvas_id).execute()
+ logging.info(
+ "UserCanvasService.update_tags canvas_id=%s tags_count=%d rows=%d",
+ canvas_id,
+ len(normalized),
+ rows_affected,
+ )
+ return rows_affected
+
@classmethod
@DB.connection_context()
def accessible(cls, canvas_id, tenant_id):
diff --git a/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py b/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py
index 1022a9b45a0..e93c48249ae 100644
--- a/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py
+++ b/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py
@@ -514,7 +514,7 @@ def test_agents_crud_unit_branches(monkeypatch):
captured = {}
- def fake_get_by_tenant_ids(owner_ids, tenant_id, page, page_size, orderby, desc, keywords, canvas_category):
+ def fake_get_by_tenant_ids(owner_ids, tenant_id, page, page_size, orderby, desc, keywords, canvas_category, tags):
captured["owner_ids"] = owner_ids
captured["tenant_id"] = tenant_id
captured["page"] = page
@@ -523,6 +523,7 @@ def fake_get_by_tenant_ids(owner_ids, tenant_id, page, page_size, orderby, desc,
captured["desc"] = desc
captured["keywords"] = keywords
captured["canvas_category"] = canvas_category
+ captured["tags"] = tags
return [{"id": "agent-1"}], 1
monkeypatch.setattr(module.UserCanvasService, "get_by_tenant_ids", fake_get_by_tenant_ids)
diff --git a/web/src/components/home-card.tsx b/web/src/components/home-card.tsx
index cefd9434b6b..d5abbf6851b 100644
--- a/web/src/components/home-card.tsx
+++ b/web/src/components/home-card.tsx
@@ -18,6 +18,7 @@ interface IProps {
icon?: React.ReactNode;
testId?: string;
showReleaseTime?: boolean;
+ extra?: ReactNode;
}
function Time({ time }: { time: string | number | undefined }) {
@@ -31,6 +32,7 @@ export function HomeCard({
icon,
testId,
showReleaseTime = false,
+ extra,
}: IProps) {
const { t } = useTranslation();
@@ -81,6 +83,7 @@ export function HomeCard({
{data.description}
+ {extra}
{showReleaseTime ? (
diff --git a/web/src/hooks/use-agent-request.ts b/web/src/hooks/use-agent-request.ts
index b524ccbc31d..b286e853f42 100644
--- a/web/src/hooks/use-agent-request.ts
+++ b/web/src/hooks/use-agent-request.ts
@@ -29,6 +29,7 @@ import agentService, {
fetchTrace,
fetchWebhookTrace,
updateAgent,
+ updateAgentTags,
uploadAgentFile,
} from '@/services/agent-service';
import { buildMessageListWithUuid } from '@/utils/chat';
@@ -73,6 +74,8 @@ export const enum AgentApiAction {
FetchSessionByIdManually = 'fetchSessionByIdManually',
FetchAgentLog = 'fetchAgentLog',
FetchSharedAgent = 'fetchSharedAgent',
+ FetchAgentTags = 'fetchAgentTags',
+ UpdateAgentTags = 'updateAgentTags',
}
export const useFetchAgentTemplates = () => {
@@ -95,12 +98,14 @@ const buildAgentListParams = ({
keywords,
canvasCategory,
ownerIds,
+ tags,
}: {
page: number;
pageSize: number;
keywords?: string;
canvasCategory?: string;
ownerIds?: string[];
+ tags?: string[];
}) => {
const params: Record = {
page,
@@ -116,6 +121,9 @@ const buildAgentListParams = ({
if (Array.isArray(ownerIds) && ownerIds.length > 0) {
params.owner_ids = ownerIds.join(',');
}
+ if (Array.isArray(tags) && tags.length > 0) {
+ params.tags = tags.join(',');
+ }
return params;
};
@@ -129,6 +137,7 @@ export const useFetchAgentListByPage = () => {
? filterValue.canvasCategory
: [];
const owner = filterValue.owner;
+ const tags = Array.isArray(filterValue.tags) ? filterValue.tags : undefined;
const requestParams = buildAgentListParams({
page: pagination.current,
@@ -136,6 +145,7 @@ export const useFetchAgentListByPage = () => {
keywords: debouncedSearchString,
canvasCategory: canvasCategory.length === 1 ? canvasCategory[0] : undefined,
ownerIds: Array.isArray(owner) ? owner : undefined,
+ tags,
});
const { data, isFetching: loading } = useQuery<{
@@ -264,6 +274,57 @@ export const useDeleteAgent = () => {
return { data, loading, deleteAgent: mutateAsync };
};
+export interface IAgentTagCount {
+ tag: string;
+ count: number;
+}
+
+export const useFetchAgentTags = (canvasCategory?: string) => {
+ const { data, isFetching: loading } = useQuery({
+ queryKey: [AgentApiAction.FetchAgentTags, canvasCategory],
+ initialData: [],
+ gcTime: 0,
+ queryFn: async () => {
+ const { data } = await agentService.listAgentTags(
+ {
+ params: canvasCategory ? { canvas_category: canvasCategory } : {},
+ },
+ true,
+ );
+ return data?.data ?? [];
+ },
+ });
+ return { data, loading };
+};
+
+export const useUpdateAgentTags = () => {
+ const queryClient = useQueryClient();
+ const { isPending: loading, mutateAsync } = useMutation({
+ mutationKey: [AgentApiAction.UpdateAgentTags],
+ mutationFn: async ({
+ agentId,
+ tags,
+ }: {
+ agentId: string;
+ tags: string[];
+ }) => {
+ const { data } = await updateAgentTags(agentId, tags);
+ if (data?.code === 0) {
+ queryClient.invalidateQueries({
+ queryKey: [AgentApiAction.FetchAgentListByPage],
+ });
+ queryClient.invalidateQueries({
+ queryKey: [AgentApiAction.FetchAgentTags],
+ });
+ } else {
+ message.error(data?.message || 'Update failed');
+ }
+ return data?.code === 0;
+ },
+ });
+ return { loading, updateAgentTags: mutateAsync };
+};
+
export const useFetchAgent = (): {
data: IFlow;
loading: boolean;
diff --git a/web/src/interfaces/database/agent.ts b/web/src/interfaces/database/agent.ts
index f548bd6a440..e82d605014e 100644
--- a/web/src/interfaces/database/agent.ts
+++ b/web/src/interfaces/database/agent.ts
@@ -82,6 +82,7 @@ export declare interface IFlow {
release_time?: number;
last_publish_time?: number;
datasets?: Pick[];
+ tags?: string;
}
export interface IFlowTemplate {
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 407c1f960f4..9bfd656017d 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1735,6 +1735,12 @@ Example: Virtual Hosted Style`,
author: 'Author',
sectionTitle: 'Section title',
},
+ editTags: 'Edit tags',
+ editTagsDescription:
+ 'Add tags to organize and filter your agents. Press Enter or comma to add.',
+ tagsPlaceholder: 'Add a tag and press Enter',
+ tagSuggestionsLabel: 'Existing tags',
+ removeTagAriaLabel: 'Remove {{tag}}',
includeHeadingContent: 'Separate parent-heading content',
includeHeadingContentTip:
'When enabled, chunks include only their heading path and content; content immediately following a parent heading is kept as a separate chunk.',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index f9ebe517eb2..dcd8f5871de 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1425,6 +1425,11 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
author: '作者',
sectionTitle: '章节标题',
},
+ editTags: '编辑标签',
+ editTagsDescription: '添加标签以整理和筛选你的智能体。按回车或逗号添加。',
+ tagsPlaceholder: '输入标签后按回车',
+ tagSuggestionsLabel: '现有标签',
+ removeTagAriaLabel: '删除 {{tag}}',
includeHeadingContent: '分离上级标题正文',
includeHeadingContentTip:
'启用后,每个分块仅保留标题路径和自身内容,与上级标题紧挨着的内容将作为一个独立的块保留。',
diff --git a/web/src/pages/agents/agent-card.tsx b/web/src/pages/agents/agent-card.tsx
index 2126475b23d..67304c85504 100644
--- a/web/src/pages/agents/agent-card.tsx
+++ b/web/src/pages/agents/agent-card.tsx
@@ -1,6 +1,7 @@
import { HomeCard } from '@/components/home-card';
import { MoreButton } from '@/components/more-button';
import { SharedBadge } from '@/components/shared-badge';
+import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { AgentCategory } from '@/constants/agent';
import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
@@ -13,6 +14,23 @@ export type DatasetCardProps = {
data: IFlow;
} & Pick, 'showAgentRenameModal'>;
+function AgentTags({ tags }: { tags?: string }) {
+ const list = (tags || '')
+ .split(',')
+ .map((t) => t.trim())
+ .filter(Boolean);
+ if (list.length === 0) return null;
+ return (
+
+ {list.map((tag) => (
+
+ {tag}
+
+ ))}
+
+ );
+}
+
export function AgentCard({ data, showAgentRenameModal }: DatasetCardProps) {
const { navigateToAgent } = useNavigatePage();
@@ -44,6 +62,7 @@ export function AgentCard({ data, showAgentRenameModal }: DatasetCardProps) {
)
}
+ extra={ }
showReleaseTime
/>
);
diff --git a/web/src/pages/agents/agent-dropdown.tsx b/web/src/pages/agents/agent-dropdown.tsx
index 5370f2a39df..8c3e569245a 100644
--- a/web/src/pages/agents/agent-dropdown.tsx
+++ b/web/src/pages/agents/agent-dropdown.tsx
@@ -11,9 +11,10 @@ import {
} from '@/components/ui/dropdown-menu';
import { useDeleteAgent } from '@/hooks/use-agent-request';
import { IFlow } from '@/interfaces/database/agent';
-import { PenLine, Trash2 } from 'lucide-react';
-import { MouseEventHandler, PropsWithChildren, useCallback } from 'react';
+import { PenLine, Tag, Trash2 } from 'lucide-react';
+import { MouseEventHandler, PropsWithChildren, useCallback, useState } from 'react';
import { useTranslation } from 'react-i18next';
+import { AgentTagEditor } from './agent-tag-editor';
import { useRenameAgent } from './use-rename-agent';
export function AgentDropdown({
@@ -26,6 +27,7 @@ export function AgentDropdown({
}) {
const { t } = useTranslation();
const { deleteAgent } = useDeleteAgent();
+ const [tagEditorOpen, setTagEditorOpen] = useState(false);
const handleShowAgentRenameModal: MouseEventHandler =
useCallback(
@@ -36,43 +38,58 @@ export function AgentDropdown({
[agent, showAgentRenameModal],
);
+ const handleEditTags: MouseEventHandler = useCallback((e) => {
+ e.stopPropagation();
+ setTagEditorOpen(true);
+ }, []);
+
const handleDelete: MouseEventHandler = useCallback(() => {
deleteAgent(agent.id);
}, [agent.id, deleteAgent]);
return (
-
- {children}
-
-
- {t('common.rename')}
-
-
-
- ),
- }}
- >
- {
- e.preventDefault();
- }}
- onClick={(e) => {
- e.stopPropagation();
+ <>
+
+ {children}
+
+
+ {t('common.rename')}
+
+
+ {t('flow.editTags')}
+
+
+
+ ),
}}
>
- {t('common.delete')}
-
-
-
-
+ {
+ e.preventDefault();
+ }}
+ onClick={(e) => {
+ e.stopPropagation();
+ }}
+ >
+ {t('common.delete')}
+
+
+
+
+
+ >
);
}
diff --git a/web/src/pages/agents/agent-tag-editor.tsx b/web/src/pages/agents/agent-tag-editor.tsx
new file mode 100644
index 00000000000..9ff8d43a0ad
--- /dev/null
+++ b/web/src/pages/agents/agent-tag-editor.tsx
@@ -0,0 +1,176 @@
+import { Badge } from '@/components/ui/badge';
+import { Button } from '@/components/ui/button';
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from '@/components/ui/dialog';
+import { Input } from '@/components/ui/input';
+import {
+ useFetchAgentTags,
+ useUpdateAgentTags,
+} from '@/hooks/use-agent-request';
+import { IFlow } from '@/interfaces/database/agent';
+import { X } from 'lucide-react';
+import { KeyboardEvent, useEffect, useMemo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+
+interface IProps {
+ agent: IFlow;
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+}
+
+const splitTags = (raw?: string) =>
+ (raw || '')
+ .split(',')
+ .map((t) => t.trim())
+ .filter(Boolean);
+
+export function AgentTagEditor({ agent, open, onOpenChange }: IProps) {
+ const { t } = useTranslation();
+ const { loading, updateAgentTags } = useUpdateAgentTags();
+ const { data: allTags } = useFetchAgentTags();
+ const initial = useMemo(() => splitTags(agent.tags), [agent.tags]);
+ const [tags, setTags] = useState(initial);
+ const [draft, setDraft] = useState('');
+
+ useEffect(() => {
+ if (open) {
+ setTags(initial);
+ setDraft('');
+ }
+ }, [open, initial]);
+
+ const suggestions = useMemo(() => {
+ const taken = new Set(tags.map((t) => t.toLowerCase()));
+ const needle = draft.trim().toLowerCase();
+ return (allTags ?? [])
+ .map((entry) => entry.tag)
+ .filter((tag) => !taken.has(tag.toLowerCase()))
+ .filter((tag) => !needle || tag.toLowerCase().startsWith(needle))
+ .slice(0, 20);
+ }, [allTags, tags, draft]);
+
+ const addTag = (tag: string) => {
+ const next = tag.trim();
+ if (!next) return;
+ if (!tags.some((existing) => existing.toLowerCase() === next.toLowerCase())) {
+ setTags([...tags, next]);
+ }
+ setDraft('');
+ };
+
+ const commitDraft = () => addTag(draft);
+
+ const handleKeyDown = (e: KeyboardEvent) => {
+ if (e.key === 'Enter' || e.key === ',') {
+ e.preventDefault();
+ commitDraft();
+ } else if (e.key === 'Backspace' && !draft && tags.length > 0) {
+ setTags(tags.slice(0, -1));
+ }
+ };
+
+ const removeTag = (tag: string) =>
+ setTags(tags.filter((existing) => existing !== tag));
+
+ const handleSave = async () => {
+ const pending = draft.trim();
+ const alreadyPresent = tags.some(
+ (existing) => existing.toLowerCase() === pending.toLowerCase(),
+ );
+ const finalTags = pending && !alreadyPresent ? [...tags, pending] : tags;
+ setTags(finalTags);
+ setDraft('');
+ const success = await updateAgentTags({
+ agentId: agent.id,
+ tags: finalTags,
+ });
+ if (success) {
+ onOpenChange(false);
+ }
+ };
+
+ return (
+
+ e.stopPropagation()}>
+
+ {t('flow.editTags')}
+
+ {t('flow.editTagsDescription')}
+
+
+
+
+ {tags.map((tag) => (
+
+ {tag}
+ removeTag(tag)}
+ aria-label={t('flow.removeTagAriaLabel', { tag })}
+ className="hover:text-state-error"
+ >
+
+
+
+ ))}
+
+
+ setDraft(e.target.value)}
+ onKeyDown={handleKeyDown}
+ onBlur={commitDraft}
+ placeholder={t('flow.tagsPlaceholder')}
+ />
+
+ {suggestions.length > 0 && (
+
+
+ {t('flow.tagSuggestionsLabel')}
+
+
+ {suggestions.map((tag) => (
+ addTag(tag)}
+ className="text-xs"
+ >
+
+ + {tag}
+
+
+ ))}
+
+
+ )}
+
+
+ onOpenChange(false)}
+ disabled={loading}
+ >
+ {t('common.cancel')}
+
+
+ {t('common.save')}
+
+
+
+
+ );
+}
diff --git a/web/src/pages/agents/hooks/use-selelct-filters.ts b/web/src/pages/agents/hooks/use-selelct-filters.ts
index aa4f4f4ddb2..2f6fb95d714 100644
--- a/web/src/pages/agents/hooks/use-selelct-filters.ts
+++ b/web/src/pages/agents/hooks/use-selelct-filters.ts
@@ -1,10 +1,14 @@
import { FilterCollection } from '@/components/list-filter-bar/interface';
-import { useFetchAgentList } from '@/hooks/use-agent-request';
+import {
+ useFetchAgentList,
+ useFetchAgentTags,
+} from '@/hooks/use-agent-request';
import { buildOwnersFilter, groupListByType } from '@/utils/list-filter-util';
import { useMemo } from 'react';
export function useSelectFilters() {
const { data } = useFetchAgentList({});
+ const { data: tagCounts } = useFetchAgentTags();
const canvasCategory = useMemo(() => {
return groupListByType(
@@ -14,6 +18,16 @@ export function useSelectFilters() {
);
}, [data?.canvas]);
+ const tagList = useMemo(
+ () =>
+ (tagCounts ?? []).map((t) => ({
+ id: t.tag,
+ label: t.tag,
+ count: t.count,
+ })),
+ [tagCounts],
+ );
+
const filters: FilterCollection[] = [
buildOwnersFilter(data?.canvas ?? []),
{
@@ -21,6 +35,11 @@ export function useSelectFilters() {
list: canvasCategory,
label: 'Canvas category',
},
+ {
+ field: 'tags',
+ list: tagList,
+ label: 'Tags',
+ },
];
return filters;
diff --git a/web/src/services/agent-service.ts b/web/src/services/agent-service.ts
index 4a4f59daaf2..92dcfeaf654 100644
--- a/web/src/services/agent-service.ts
+++ b/web/src/services/agent-service.ts
@@ -50,6 +50,10 @@ const methods = {
url: listAgents,
method: 'get',
},
+ listAgentTags: {
+ url: api.listAgentTags,
+ method: 'get',
+ },
resetAgent: {
url: resetAgent,
method: 'post',
@@ -135,6 +139,13 @@ export const updateAgent = (
return request(updateAgentApi(agentId), { method: 'put', data: params });
};
+export const updateAgentTags = (agentId: string, tags: string[]) => {
+ return request(api.updateAgentTags(agentId), {
+ method: 'put',
+ data: { tags: tags.join(',') },
+ });
+};
+
export const fetchTrace = (data: { canvas_id: string; message_id: string }) => {
return request.get(
methods.trace.url({
diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts
index f1e67986d35..5dbfc8e369f 100644
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -189,6 +189,9 @@ export default {
// flow
listAgentTemplate: `${restAPIv1}/agents/templates`,
listAgents: `${restAPIv1}/agents`,
+ listAgentTags: `${restAPIv1}/agents/tags`,
+ updateAgentTags: (agentId: string) =>
+ `${restAPIv1}/agents/${agentId}/tags`,
createAgent: `${restAPIv1}/agents`,
updateAgent: (agentId: string) => `${restAPIv1}/agents/${agentId}`,
deleteAgent: (agentId: string) => `${restAPIv1}/agents/${agentId}`,
From cb01529d8bfaff487c2fed4c9cadccd447b34f80 Mon Sep 17 00:00:00 2001
From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com>
Date: Wed, 13 May 2026 15:46:54 -1000
Subject: [PATCH 127/666] Go: implement provider: Voyage AI (#14811)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Add a Go driver for Voyage AI (https://voyageai.com), one of the
unchecked providers on the umbrella tracking issue #14736. Voyage AI is
**embed + rerank only** — no chat, no streaming, no `/v1/models`
endpoint. It's the first provider in the Go layer of this shape.
Until this PR, a tenant who configured `voyage` as a model provider in
the Go layer fell through to the default branch of
`internal/entity/models/factory.go` and got the dummy driver.
### What this PR includes
- New `internal/entity/models/voyage.go` with a `VoyageModel`
implementing the `ModelDriver` interface.
- New `conf/models/voyage.json` with 6 embedding models (`voyage-3.5`,
`voyage-3.5-lite`, `voyage-3-large`, `voyage-code-3`, `voyage-law-2`,
`voyage-finance-2`) and 2 rerank models (`rerank-2`, `rerank-2-lite`).
- `factory.go`: route `"voyage"` to `NewVoyageModel`.
- `internal/entity/models/voyage_test.go`: 19 unit tests.
### How the driver works
- **Embed**: `POST /v1/embeddings`. Response is OpenAI-shaped (`{data:
[{embedding, index, object, text}], model, usage}`). Driver reorders by
`index`, rejects duplicate / out-of-range / missing slots, and
short-circuits empty input without an HTTP call.
- **Rerank**: `POST /v1/rerank`. Voyage uses **`top_k`** as the request
param name (not `top_n` like Aliyun/SiliconFlow); the driver translates
`RerankConfig.TopN` → `top_k`. Response is Cohere-shaped (`{data:
[{relevance_score, index}], model}`), so the existing
`RerankResponse{Data: []RerankResult{Index, RelevanceScore}}` shape fits
cleanly.
- **`ListModels`**: returns a hardcoded list of `voyageKnownModels`.
Voyage does **not** expose `/v1/models` (probed live, returns 404), so
the driver synthesizes the list from the same set the config ships. New
upstream models are added by extending one slice.
- **`CheckConnection`**: pings a 1-input embed call against
`voyage-3.5`. Without `/v1/models`, this is the cheapest way to verify
the API key + network path before a tenant tries a real workload.
- **`ChatWithMessages` / `ChatStreamlyWithSender` / `Balance` /
`TranscribeAudio` / `AudioSpeech` / `OCRFile`**: all return `"no such
method"`. Voyage does not host any of these surfaces.
No interface change. No new dependencies.
### How was this tested?
**19 unit tests** in `internal/entity/models/voyage_test.go` — all pass
on go 1.25:
```
$ go test -vet=off -run TestVoyage -count=1 ./internal/entity/models/...
ok ragflow/internal/entity/models 0.036s
```
Coverage: Name; Embed (happy path, reorder, empty-input, missing
key/model, duplicate index, out-of-range index, missing slot); Rerank
(happy path with `top_k` assertion, default-to-len-documents, empty
documents, out-of-range index); ListModels (static list, missing key);
CheckConnection (happy, 401); chat methods sentinels; Balance sentinel;
audio/OCR sentinels.
`go build ./internal/entity/models/...` exits 0.
**Live integration test** against `api.voyageai.com`:
```
=== RUN TestVoyageLiveSmoke
[OK] Name() = "voyage"
[OK] ListModels (static): 8 models -> [voyage-3.5 voyage-3.5-lite voyage-3-large voyage-code-3 voyage-law-2 voyage-finance-2 rerank-2 rerank-2-lite]
[OK] CheckConnection
[OK] Embed vectors=3 dim=1024 indices=[0 1 2]
[OK] Embed(empty) -> 0 vectors
[OK] Rerank results=3 scores=[0.8125 0.59765625 0.39453125]
[OK] ChatWithMessages returns voyage, no such method
[OK] Balance returns voyage, no such method
VOYAGE LIVE SMOKE PASSED
--- PASS: TestVoyageLiveSmoke (0.81s)
```
What the live run proves on the wire:
- Auth (`Bearer `) accepted by `api.voyageai.com`.
- Embed `voyage-3.5` on 3 inputs returns 3 vectors at dim 1024 with
`index` field preserved as `[0, 1, 2]` — the reorder-by-index code is
exercised on real data.
- Empty input short-circuits without an HTTP call (mock server would
have been hit if it did).
- Rerank `rerank-2` on 3 docs returns 3 real `relevance_score` floats
`[0.8125, 0.598, 0.395]`. The `top_k` translation works on the live
wire.
- All sentinel methods return the documented `"no such method"` strings.
### Note on PR history
This branch was previously named for LocalAI Embed work which is now
consolidated into PR #14813. The branch was reset to `upstream/main` and
rebuilt for Voyage. Diff against `main` is a clean +838 lines across 4
files.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
Tracking: #14736
---------
Co-authored-by: Jin Hai
---
conf/models/voyage.json | 69 +++++
internal/entity/models/factory.go | 2 +
internal/entity/models/localai.go | 7 +-
internal/entity/models/longcat_test.go | 2 +-
internal/entity/models/novita_test.go | 2 +-
internal/entity/models/voyage.go | 376 +++++++++++++++++++++++
internal/entity/models/voyage_test.go | 399 +++++++++++++++++++++++++
7 files changed, 852 insertions(+), 5 deletions(-)
create mode 100644 conf/models/voyage.json
create mode 100644 internal/entity/models/voyage.go
create mode 100644 internal/entity/models/voyage_test.go
diff --git a/conf/models/voyage.json b/conf/models/voyage.json
new file mode 100644
index 00000000000..65c2272d934
--- /dev/null
+++ b/conf/models/voyage.json
@@ -0,0 +1,69 @@
+{
+ "name": "Voyage",
+ "url": {
+ "default": "https://api.voyageai.com"
+ },
+ "url_suffix": {
+ "embedding": "v1/embeddings",
+ "rerank": "v1/rerank"
+ },
+ "class": "voyage",
+ "models": [
+ {
+ "name": "voyage-3.5",
+ "max_tokens": 327680,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "voyage-3.5-lite",
+ "max_tokens": 1048576,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "voyage-3-large",
+ "max_tokens": 122880,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "voyage-code-3",
+ "max_tokens": 122880,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "voyage-law-2",
+ "max_tokens": 122880,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "voyage-finance-2",
+ "max_tokens": 122880,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "rerank-2",
+ "max_tokens": 4000,
+ "model_types": [
+ "rerank"
+ ]
+ },
+ {
+ "name": "rerank-2-lite",
+ "max_tokens": 2000,
+ "model_types": [
+ "rerank"
+ ]
+ }
+ ]
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index acdb8df9449..581baa51330 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -89,6 +89,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewLongCatModel(baseURL, urlSuffix), nil
case "novita":
return NewNovitaModel(baseURL, urlSuffix), nil
+ case "voyage":
+ return NewVoyageModel(baseURL, urlSuffix), nil
default:
return NewDummyModel(baseURL, urlSuffix), nil
}
diff --git a/internal/entity/models/localai.go b/internal/entity/models/localai.go
index d47d40a91ee..f5fab0df3e7 100644
--- a/internal/entity/models/localai.go
+++ b/internal/entity/models/localai.go
@@ -817,7 +817,8 @@ func (l *LocalAIModel) AudioSpeechWithSender(modelName *string, audioContent *st
return fmt.Errorf("%s, no such method", l.Name())
}
-// OCRFile OCR file
-func (d *LocalAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
- return nil, fmt.Errorf("%s, no such method", d.Name())
+// OCRFile: LocalAI has no OCR pipeline in its OpenAI-compatible surface;
+// document parsing belongs to a different interface entirely.
+func (l *LocalAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", l.Name())
}
diff --git a/internal/entity/models/longcat_test.go b/internal/entity/models/longcat_test.go
index 66e984da236..14870f8f69e 100644
--- a/internal/entity/models/longcat_test.go
+++ b/internal/entity/models/longcat_test.go
@@ -461,7 +461,7 @@ func TestLongCatAudioOCRReturnNoSuchMethod(t *testing.T) {
if _, err := m.AudioSpeech(&model, &model, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
t.Errorf("AudioSpeech: want 'no such method', got %v", err)
}
- if _, err := m.OCRFile(&model, &model, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ if _, err := m.OCRFile(&model, nil, &model, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
t.Errorf("OCRFile: want 'no such method', got %v", err)
}
}
diff --git a/internal/entity/models/novita_test.go b/internal/entity/models/novita_test.go
index 0470918d3af..29cbdace18c 100644
--- a/internal/entity/models/novita_test.go
+++ b/internal/entity/models/novita_test.go
@@ -681,7 +681,7 @@ func TestNovitaAudioOCRReturnNoSuchMethod(t *testing.T) {
if _, err := v.AudioSpeech(&m, &m, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
t.Errorf("AudioSpeech: %v", err)
}
- if _, err := v.OCRFile(&m, &m, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ if _, err := v.OCRFile(&m, nil, &m, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
t.Errorf("OCRFile: %v", err)
}
}
diff --git a/internal/entity/models/voyage.go b/internal/entity/models/voyage.go
new file mode 100644
index 00000000000..41d0237c7ec
--- /dev/null
+++ b/internal/entity/models/voyage.go
@@ -0,0 +1,376 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+)
+
+// VoyageModel implements ModelDriver for Voyage AI.
+//
+// Voyage AI exposes a focused REST API at https://api.voyageai.com/v1
+// with embedding (/embeddings) and reranking (/rerank) only — no chat,
+// no streaming, no /v1/models, no balance. This driver covers Embed
+// and Rerank with real implementations and returns "no such method"
+// for every other ModelDriver method.
+//
+// Wire shape, captured live:
+//
+// Embed response: {object, data:[{object,embedding,index,text}], model, usage}
+// Rerank response: {object, data:[{relevance_score,index}], model, usage}
+//
+// Rerank uses top_k as the request param name (not top_n like
+// Aliyun/SiliconFlow); the driver translates RerankConfig.TopN to
+// top_k on the wire.
+type VoyageModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+// NewVoyageModel creates a new Voyage AI model instance.
+//
+// We clone http.DefaultTransport so we keep Go's defaults for
+// ProxyFromEnvironment, DialContext (with KeepAlive), HTTP/2,
+// TLSHandshakeTimeout, and ExpectContinueTimeout, and only override
+// the connection-pool fields we care about.
+func NewVoyageModel(baseURL map[string]string, urlSuffix URLSuffix) *VoyageModel {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxIdleConns = 100
+ transport.MaxIdleConnsPerHost = 10
+ transport.IdleConnTimeout = 90 * time.Second
+ transport.DisableCompression = false
+ transport.ResponseHeaderTimeout = 60 * time.Second
+
+ return &VoyageModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Transport: transport,
+ },
+ }
+}
+
+func (v *VoyageModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return NewVoyageModel(baseURL, v.URLSuffix)
+}
+
+func (v *VoyageModel) Name() string {
+ return "voyage"
+}
+
+// baseURLForRegion returns the base URL for the given region, or an
+// error if no entry exists. Single-region for Voyage but kept here
+// for consistency with other drivers.
+func (v *VoyageModel) baseURLForRegion(region string) (string, error) {
+ base, ok := v.BaseURL[region]
+ if !ok || base == "" {
+ return "", fmt.Errorf("voyage: no base URL configured for region %q", region)
+ }
+ return base, nil
+}
+
+type voyageEmbeddingData struct {
+ Embedding []float64 `json:"embedding"`
+ Object string `json:"object"`
+ Index int `json:"index"`
+}
+
+type voyageEmbeddingResponse struct {
+ Object string `json:"object"`
+ Data []voyageEmbeddingData `json:"data"`
+ Model string `json:"model"`
+}
+
+// Embed turns a list of texts into embedding vectors using the
+// Voyage AI /v1/embeddings endpoint. Output is one vector per input,
+// in the same order the inputs were given.
+func (v *VoyageModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ if len(texts) == 0 {
+ return []EmbeddingData{}, nil
+ }
+
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := v.baseURLForRegion(region)
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", strings.TrimSuffix(baseURL, "/"), v.URLSuffix.Embedding)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": texts,
+ }
+
+ // Voyage's Matryoshka models (voyage-3.5, voyage-3.5-lite,
+ // voyage-3-large, voyage-code-3) accept output_dimension to
+ // truncate the vector. The wire param is output_dimension
+ // (singular) per https://docs.voyageai.com/reference/embeddings-api;
+ // passing "dimensions" or "output_dimensions" gets rejected with
+ // HTTP 400, so it's worth matching the docs spelling exactly.
+ if embeddingConfig != nil && embeddingConfig.Dimension > 0 {
+ reqBody["output_dimension"] = embeddingConfig.Dimension
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := v.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Voyage embeddings API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var parsed voyageEmbeddingResponse
+ if err = json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ // Reorder by the reported index so the output always lines up with
+ // the input texts. Reject duplicates (silent overwrite would hide
+ // a malformed response) and out-of-range indices (silent panic on
+ // slice growth would mask the bug).
+ embeddings := make([]EmbeddingData, len(texts))
+ filled := make([]bool, len(texts))
+ for _, item := range parsed.Data {
+ if item.Index < 0 || item.Index >= len(texts) {
+ return nil, fmt.Errorf("voyage: response index %d out of range for %d inputs", item.Index, len(texts))
+ }
+ if filled[item.Index] {
+ return nil, fmt.Errorf("voyage: duplicate embedding index %d in response", item.Index)
+ }
+ embeddings[item.Index] = EmbeddingData{
+ Embedding: item.Embedding,
+ Index: item.Index,
+ }
+ filled[item.Index] = true
+ }
+ for i, ok := range filled {
+ if !ok {
+ return nil, fmt.Errorf("voyage: missing embedding for input index %d", i)
+ }
+ }
+
+ return embeddings, nil
+}
+
+type voyageRerankRequest struct {
+ Model string `json:"model"`
+ Query string `json:"query"`
+ Documents []string `json:"documents"`
+ TopK int `json:"top_k"`
+}
+
+type voyageRerankResponse struct {
+ Object string `json:"object"`
+ Data []struct {
+ RelevanceScore float64 `json:"relevance_score"`
+ Index int `json:"index"`
+ } `json:"data"`
+ Model string `json:"model"`
+}
+
+// Rerank calculates similarity scores between a query and a list of
+// documents using Voyage AI's /v1/rerank endpoint. Unlike many other
+// rerank APIs that use `top_n`, Voyage uses `top_k` as the request
+// parameter; the driver translates RerankConfig.TopN -> top_k.
+func (v *VoyageModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ if len(documents) == 0 {
+ return &RerankResponse{}, nil
+ }
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := v.baseURLForRegion(region)
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", strings.TrimSuffix(baseURL, "/"), v.URLSuffix.Rerank)
+
+ topK := len(documents)
+ if rerankConfig != nil && rerankConfig.TopN > 0 {
+ topK = rerankConfig.TopN
+ }
+
+ reqBody := voyageRerankRequest{
+ Model: *modelName,
+ Query: query,
+ Documents: documents,
+ TopK: topK,
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := v.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Voyage rerank API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var parsed voyageRerankResponse
+ if err = json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ // Match Embed's defensive posture: rerank only returns top_k of
+ // len(documents) results, but a duplicate index would still be
+ // a malformed response and should fail loudly.
+ rerankResponse := &RerankResponse{}
+ seen := make(map[int]bool, len(parsed.Data))
+ for _, r := range parsed.Data {
+ if r.Index < 0 || r.Index >= len(documents) {
+ return nil, fmt.Errorf("voyage: rerank result index %d out of range for %d documents", r.Index, len(documents))
+ }
+ if seen[r.Index] {
+ return nil, fmt.Errorf("voyage: duplicate rerank index %d in response", r.Index)
+ }
+ seen[r.Index] = true
+ rerankResponse.Data = append(rerankResponse.Data, RerankResult{
+ Index: r.Index,
+ RelevanceScore: r.RelevanceScore,
+ })
+ }
+
+ return rerankResponse, nil
+}
+
+// ListModels is not exposed by the Voyage AI API. The docs at
+// https://docs.voyageai.com publish embeddings and rerank endpoints
+// only; /v1/models is not documented (live-confirmed: 404). The
+// shipped catalog lives in conf/models/voyage.json; this driver
+// method does not invent a fake one.
+func (v *VoyageModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ return nil, fmt.Errorf("%s, no such method", v.Name())
+}
+
+// CheckConnection is not exposed by the Voyage AI API. With no
+// documented /models or /health endpoint, the only way to verify
+// credentials is to burn an embedding or rerank call against the
+// tenant's quota — which is what this method exists to avoid.
+// Return the documented sentinel rather than pretend.
+func (v *VoyageModel) CheckConnection(apiConfig *APIConfig) error {
+ return fmt.Errorf("%s, no such method", v.Name())
+}
+
+// ChatWithMessages is not exposed by the Voyage AI API.
+func (v *VoyageModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", v.Name())
+}
+
+func (v *VoyageModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", v.Name())
+}
+
+// Balance is not exposed by the Voyage AI API.
+func (v *VoyageModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("%s, no such method", v.Name())
+}
+
+// TranscribeAudio / AudioSpeech / OCRFile: Voyage does not host any of
+// these surfaces.
+func (v *VoyageModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", v.Name())
+}
+
+func (v *VoyageModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", v.Name())
+}
+
+func (v *VoyageModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", v.Name())
+}
+
+func (v *VoyageModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", v.Name())
+}
+
+func (v *VoyageModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", v.Name())
+}
diff --git a/internal/entity/models/voyage_test.go b/internal/entity/models/voyage_test.go
new file mode 100644
index 00000000000..255915bf98a
--- /dev/null
+++ b/internal/entity/models/voyage_test.go
@@ -0,0 +1,399 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func newVoyageServer(t *testing.T, expectedPath string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != expectedPath {
+ t.Errorf("expected path=%s, got %s", expectedPath, r.URL.Path)
+ return
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+ t.Errorf("expected Authorization=Bearer test-key, got %q", got)
+ return
+ }
+ if got := r.Header.Get("Content-Type"); got != "application/json" {
+ t.Errorf("expected Content-Type=application/json, got %q", got)
+ return
+ }
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read body: %v", err)
+ return
+ }
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("unmarshal: %v\nraw=%s", err, string(raw))
+ return
+ }
+ handler(t, body, w)
+ }))
+}
+
+func newVoyageForTest(baseURL string) *VoyageModel {
+ return NewVoyageModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{Embedding: "v1/embeddings", Rerank: "v1/rerank"},
+ )
+}
+
+func TestVoyageName(t *testing.T) {
+ if got := newVoyageForTest("http://unused").Name(); got != "voyage" {
+ t.Errorf("Name()=%q, want %q", got, "voyage")
+ }
+}
+
+func TestVoyageEmbedHappyPath(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "voyage-3.5" {
+ t.Errorf("model=%v", body["model"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "object": "list",
+ "data": []map[string]interface{}{
+ {"object": "embedding", "embedding": []float64{0.1, 0.2}, "index": 0},
+ {"object": "embedding", "embedding": []float64{0.3, 0.4}, "index": 1},
+ {"object": "embedding", "embedding": []float64{0.5, 0.6}, "index": 2},
+ },
+ "model": "voyage-3.5",
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ vecs, err := v.Embed(&model, []string{"a", "b", "c"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+ if len(vecs) != 3 {
+ t.Fatalf("len=%d want 3", len(vecs))
+ }
+ if vecs[1].Embedding[0] != 0.3 || vecs[1].Index != 1 {
+ t.Errorf("vecs[1]=%+v", vecs[1])
+ }
+}
+
+// TestVoyageEmbedPropagatesOutputDimension pins the docs-spelled
+// param name. Voyage 400s on any other key (live-verified — sending
+// "dimensions" returns "Argument 'dimensions' is not supported by our
+// API"), so this name matters and must not regress.
+func TestVoyageEmbedPropagatesOutputDimension(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if got, ok := body["output_dimension"].(float64); !ok || got != 256 {
+ t.Errorf("output_dimension=%v want 256", body["output_dimension"])
+ }
+ for _, wrong := range []string{"dimensions", "output_dimensions", "dimension"} {
+ if _, present := body[wrong]; present {
+ t.Errorf("must not send %q (Voyage rejects unknown fields)", wrong)
+ }
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{{"embedding": []float64{0.1}, "index": 0}},
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ _, err := v.Embed(&model, []string{"x"}, &APIConfig{ApiKey: &apiKey},
+ &EmbeddingConfig{Dimension: 256})
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+}
+
+// And when Dimension is zero/unset, the field MUST be absent — Voyage
+// would default the vector length, but only if we don't send the key
+// at all (sending output_dimension: 0 is a 400).
+func TestVoyageEmbedOmitsOutputDimensionWhenUnset(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if _, present := body["output_dimension"]; present {
+ t.Errorf("output_dimension must be absent when Dimension is unset, got %v", body["output_dimension"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{{"embedding": []float64{0.1}, "index": 0}},
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ _, err := v.Embed(&model, []string{"x"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+}
+
+func TestVoyageEmbedReordersByIndex(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{2}, "index": 2},
+ {"embedding": []float64{0}, "index": 0},
+ {"embedding": []float64{1}, "index": 1},
+ },
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ vecs, err := v.Embed(&model, []string{"a", "b", "c"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+ for i, vec := range vecs {
+ if vec.Index != i || vec.Embedding[0] != float64(i) {
+ t.Errorf("slot %d=%+v", i, vec)
+ }
+ }
+}
+
+func TestVoyageEmbedEmptyInputShortCircuits(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ t.Error("Embed([]) made an unexpected HTTP call")
+ }))
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ vecs, err := v.Embed(&model, []string{}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil || len(vecs) != 0 {
+ t.Errorf("Embed([])=(%v,%v)", vecs, err)
+ }
+}
+
+func TestVoyageEmbedRequiresAPIKey(t *testing.T) {
+ v := newVoyageForTest("http://unused")
+ model := "voyage-3.5"
+ _, err := v.Embed(&model, []string{"a"}, &APIConfig{}, nil)
+ if err == nil || !strings.Contains(err.Error(), "api key is required") {
+ t.Errorf("expected api-key error, got %v", err)
+ }
+}
+
+func TestVoyageEmbedRequiresModelName(t *testing.T) {
+ v := newVoyageForTest("http://unused")
+ apiKey := "test-key"
+ _, err := v.Embed(nil, []string{"a"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("expected model-name error, got %v", err)
+ }
+}
+
+func TestVoyageEmbedRejectsDuplicateIndex(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{1}, "index": 0},
+ {"embedding": []float64{2}, "index": 0},
+ },
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ _, err := v.Embed(&model, []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "duplicate embedding index 0") {
+ t.Errorf("expected duplicate error, got %v", err)
+ }
+}
+
+func TestVoyageEmbedRejectsOutOfRangeIndex(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{1}, "index": 7},
+ },
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ _, err := v.Embed(&model, []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "out of range") {
+ t.Errorf("expected out-of-range error, got %v", err)
+ }
+}
+
+func TestVoyageEmbedRejectsMissingSlot(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{1}, "index": 0},
+ },
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ _, err := v.Embed(&model, []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "missing embedding for input index 1") {
+ t.Errorf("expected missing-slot error, got %v", err)
+ }
+}
+
+func TestVoyageRerankHappyPath(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/rerank", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ // Voyage's request key is top_k (not top_n).
+ if body["top_k"] != float64(3) {
+ t.Errorf("top_k=%v want 3", body["top_k"])
+ }
+ if body["query"] != "x" {
+ t.Errorf("query=%v", body["query"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "object": "list",
+ "data": []map[string]interface{}{
+ {"relevance_score": 0.8, "index": 2},
+ {"relevance_score": 0.5, "index": 0},
+ {"relevance_score": 0.3, "index": 1},
+ },
+ "model": "rerank-2",
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "rerank-2"
+ resp, err := v.Rerank(&model, "x", []string{"a", "b", "c"},
+ &APIConfig{ApiKey: &apiKey}, &RerankConfig{TopN: 3})
+ if err != nil {
+ t.Fatalf("Rerank: %v", err)
+ }
+ if len(resp.Data) != 3 {
+ t.Fatalf("len=%d want 3", len(resp.Data))
+ }
+ want := map[int]float64{0: 0.5, 1: 0.3, 2: 0.8}
+ for _, r := range resp.Data {
+ if got, ok := want[r.Index]; !ok || got != r.RelevanceScore {
+ t.Errorf("unexpected result index=%d score=%v", r.Index, r.RelevanceScore)
+ }
+ }
+}
+
+func TestVoyageRerankTopKDefaultsToLenDocuments(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/rerank", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["top_k"] != float64(4) {
+ t.Errorf("top_k=%v want 4 (len(documents))", body["top_k"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": []map[string]interface{}{}})
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "rerank-2"
+ _, err := v.Rerank(&model, "x", []string{"a", "b", "c", "d"},
+ &APIConfig{ApiKey: &apiKey}, &RerankConfig{})
+ if err != nil {
+ t.Fatalf("Rerank: %v", err)
+ }
+}
+
+func TestVoyageRerankEmptyDocuments(t *testing.T) {
+ v := newVoyageForTest("http://unused")
+ apiKey := "test-key"
+ model := "rerank-2"
+ resp, err := v.Rerank(&model, "x", nil,
+ &APIConfig{ApiKey: &apiKey}, &RerankConfig{TopN: 0})
+ if err != nil {
+ t.Fatalf("Rerank: %v", err)
+ }
+ if len(resp.Data) != 0 {
+ t.Errorf("expected empty Data, got %d", len(resp.Data))
+ }
+}
+
+func TestVoyageRerankRejectsOutOfRangeIndex(t *testing.T) {
+ srv := newVoyageServer(t, "/v1/rerank", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"relevance_score": 0.9, "index": 7},
+ },
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "rerank-2"
+ _, err := v.Rerank(&model, "x", []string{"a", "b"},
+ &APIConfig{ApiKey: &apiKey}, &RerankConfig{TopN: 2})
+ if err == nil || !strings.Contains(err.Error(), "out of range") {
+ t.Errorf("expected out-of-range error, got %v", err)
+ }
+}
+
+func TestVoyageRerankRejectsDuplicateIndex(t *testing.T) {
+ // A duplicate index would silently overwrite an earlier slot, which
+ // is the same failure mode Embed already guards against. Make sure
+ // Rerank fails loudly too.
+ srv := newVoyageServer(t, "/v1/rerank", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"relevance_score": 0.9, "index": 0},
+ {"relevance_score": 0.8, "index": 0},
+ },
+ })
+ })
+ defer srv.Close()
+
+ v := newVoyageForTest(srv.URL)
+ apiKey := "test-key"
+ model := "rerank-2"
+ _, err := v.Rerank(&model, "x", []string{"a", "b"},
+ &APIConfig{ApiKey: &apiKey}, &RerankConfig{TopN: 2})
+ if err == nil || !strings.Contains(err.Error(), "duplicate rerank index 0") {
+ t.Errorf("expected duplicate-index error, got %v", err)
+ }
+}
+
+// TestVoyageEmbedTrimsTrailingSlashInBaseURL guards against a
+// misconfigured baseURL ending in "/" producing a double-slash path
+// (e.g. `.../v1//embeddings`). Rerank already trims, so Embed must
+// trim too; CodeRabbit flagged the inconsistency.
+func TestVoyageEmbedTrimsTrailingSlashInBaseURL(t *testing.T) {
+ var sawPath string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ sawPath = r.URL.Path
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{{"embedding": []float64{1}, "index": 0}},
+ })
+ }))
+ defer srv.Close()
+
+ v := NewVoyageModel(
+ map[string]string{"default": srv.URL + "/"}, // trailing slash
+ URLSuffix{Embedding: "v1/embeddings", Rerank: "v1/rerank"},
+ )
+ apiKey := "test-key"
+ model := "voyage-3.5"
+ if _, err := v.Embed(&model, []string{"x"}, &APIConfig{ApiKey: &apiKey}, nil); err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+ if sawPath != "/v1/embeddings" {
+ t.Errorf("path=%q want %q (no double slash)", sawPath, "/v1/embeddings")
+ }
+}
From e577901388c084b5adb0c1f847d21a323ae36fe1 Mon Sep 17 00:00:00 2001
From: Yingfeng
Date: Thu, 14 May 2026 09:49:45 +0800
Subject: [PATCH 128/666] Fix doc format (#14909)
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
docs/release_notes.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/docs/release_notes.md b/docs/release_notes.md
index 934f3aefb42..dab692c264d 100644
--- a/docs/release_notes.md
+++ b/docs/release_notes.md
@@ -13,13 +13,13 @@ Key features, improvements and bug fixes in the latest releases.
Released on May 13, 2026.
-## New features
+### New features
- Data source and parsing: Added column-level semantic/metadata control for the spreadsheet file parser; introduced ETag optimization for incremental synchronization of S3 data sources to avoid unnecessary file transfers.
- Enables assigning specific roles like content, metadata, and primary key, to table columns. [#13710](https://github.com/infiniflow/ragflow/pull/13710)
-## Improvements
+### Improvements
- API refactoring and security
- Continues the transition of web APIs to RESTful conventions, ensuring backward compatibility for all legacy endpoints.
@@ -30,7 +30,7 @@ Released on May 13, 2026.
- Agent: Enables the **Code** component to output and display file-based attachments, such as charts and images, directly in the chat. [#14787](https://github.com/infiniflow/ragflow/pull/14787)
- Reduces ingestion server boot time. [#14894](https://github.com/infiniflow/ragflow/pull/14894)
-## Bug fixes
+### Bug fixes
- Images in multi-sheet Excel workbooks were not scoped by sheet, causing images to be incorrectly attributed across different worksheets. [#14120](https://github.com/infiniflow/ragflow/pull/14120)
- Agent: Splits the **Message** component output into distinct 'waiting' and 'message' states when nested inside a **Iteration** component alongside a **Wait** component. [#14839](https://github.com/infiniflow/ragflow/pull/14839)
From b2b63600f164e94ce45bdece9b3f61377cc06986 Mon Sep 17 00:00:00 2001
From: sirj0k3r
Date: Thu, 14 May 2026 03:16:24 +0100
Subject: [PATCH 129/666] Adds gpt-5.4-mini and gpt-5.4-nano (#14908)
### What problem does this PR solve?
Includes gpt-5.4-mini and gpt-5.4-nano to the OpenAI model list
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---
conf/llm_factories.json | 14 ++++++++++++++
conf/models/openai.json | 16 ++++++++++++++++
.../test_llm_app/test_llm_list_unit.py | 2 +-
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/conf/llm_factories.json b/conf/llm_factories.json
index 09273fe2455..e8d02ed88eb 100644
--- a/conf/llm_factories.json
+++ b/conf/llm_factories.json
@@ -22,6 +22,20 @@
"model_type": "chat",
"is_tools": true
},
+ {
+ "llm_name": "gpt-5.4-mini",
+ "tags": "LLM,CHAT,400k,IMAGE2TEXT",
+ "max_tokens": 400000,
+ "model_type": "chat",
+ "is_tools": true
+ },
+ {
+ "llm_name": "gpt-5.4-nano",
+ "tags": "LLM,CHAT,400k,IMAGE2TEXT",
+ "max_tokens": 400000,
+ "model_type": "chat",
+ "is_tools": true
+ },
{
"llm_name": "gpt-5.2-pro",
"tags": "LLM,CHAT,400k,IMAGE2TEXT",
diff --git a/conf/models/openai.json b/conf/models/openai.json
index ae252fdccc4..33e4a105061 100644
--- a/conf/models/openai.json
+++ b/conf/models/openai.json
@@ -26,6 +26,22 @@
"vision"
]
},
+ {
+ "name": "gpt-5.4-mini",
+ "max_tokens": 400000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "gpt-5.4-nano",
+ "max_tokens": 400000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
{
"name": "gpt-5.2-pro",
"max_tokens": 400000,
diff --git a/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py b/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py
index e0442e0aa79..1b4dd47a6a8 100644
--- a/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py
+++ b/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py
@@ -269,7 +269,7 @@ def test_openai_catalog_contains_latest_gpt_models_unit():
openai_models = json.load(f)["models"]
model_file_names = {item["name"] for item in openai_models}
- for model_name in ["gpt-5.5", "gpt-5.4"]:
+ for model_name in ["gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano"]:
assert model_name in factory_model_names
assert model_name in model_file_names
From 1c0eaa504bc477ba704f5845b85648624e925046 Mon Sep 17 00:00:00 2001
From: writinwaters <93570324+writinwaters@users.noreply.github.com>
Date: Thu, 14 May 2026 10:57:43 +0800
Subject: [PATCH 130/666] Docs: Finalized v0.25.3 release notes (#14913)
### What problem does this PR solve?
0.25.3 release notes, final.
### Type of change
- [x] Documentation Update
---
docs/release_notes.md | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/docs/release_notes.md b/docs/release_notes.md
index dab692c264d..3f58353ff9d 100644
--- a/docs/release_notes.md
+++ b/docs/release_notes.md
@@ -15,12 +15,11 @@ Released on May 13, 2026.
### New features
-- Data source and parsing: Added column-level semantic/metadata control for the spreadsheet file parser; introduced ETag optimization for incremental synchronization of S3 data sources to avoid unnecessary file transfers.
- Enables assigning specific roles like content, metadata, and primary key, to table columns. [#13710](https://github.com/infiniflow/ragflow/pull/13710)
-
### Improvements
+- S3 connector: Implements ETag-based incremental synchronization for S3 data sources, drastically reduces sync time and AWS egress costs for users with massive-volumn S3-based datasets. [#14628](https://github.com/infiniflow/ragflow/issues/14628)[#14677](https://github.com/infiniflow/ragflow/pull/14677)
- API refactoring and security
- Continues the transition of web APIs to RESTful conventions, ensuring backward compatibility for all legacy endpoints.
- Binds the `user_id` in `POST /api/v1/messages` to the authenticated JWT principal. [#14745](https://github.com/infiniflow/ragflow/pull/14745)
@@ -33,9 +32,9 @@ Released on May 13, 2026.
### Bug fixes
- Images in multi-sheet Excel workbooks were not scoped by sheet, causing images to be incorrectly attributed across different worksheets. [#14120](https://github.com/infiniflow/ragflow/pull/14120)
-- Agent: Splits the **Message** component output into distinct 'waiting' and 'message' states when nested inside a **Iteration** component alongside a **Wait** component. [#14839](https://github.com/infiniflow/ragflow/pull/14839)
-- The **Iteration** component failed to correctly pass array elements to its child components due to a naming mismatch between the expected IterationItem alias and the runtime item variable. [#14146](https://github.com/infiniflow/ragflow/pull/14146)
-- Tool parameter template parsing
+- Agent: Splits the **Message** component output into distinct 'waiting' and 'message' states when nested inside an **Iteration** component alongside a **Wait** component. [#14839](https://github.com/infiniflow/ragflow/pull/14839)
+- Agent: The **Iteration** component failed to correctly pass array elements to its child components due to a naming mismatch between the expected IterationItem alias and the runtime item variable. [#14146](https://github.com/infiniflow/ragflow/pull/14146)
+- Agent: Template strings in tool-type components like **Email** and **Invoke** failed to interpolate; `{{variable}}` placeholders were passed through as raw text. [#14601](https://github.com/infiniflow/ragflow/pull/14601)
- Volcengine (Doubao/Ark) endpoints were not visible in the provider list. [#14702](https://github.com/infiniflow/ragflow/pull/14702)
From b89878c593c059a2106494291cfaf5747ad1cddf Mon Sep 17 00:00:00 2001
From: buua436
Date: Thu, 14 May 2026 10:59:06 +0800
Subject: [PATCH 131/666] Fix: dataset document download route (#14910)
### What problem does this PR solve?
dataset document download route
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/sdk/doc.py | 57 ++-----------------
.../dataset/dataset/dataset-action-cell.tsx | 25 +++++---
web/src/services/file-manager-service.ts | 15 +++++
web/src/utils/api.ts | 2 +
4 files changed, 41 insertions(+), 58 deletions(-)
diff --git a/api/apps/sdk/doc.py b/api/apps/sdk/doc.py
index a71b9016177..85242f77b0d 100644
--- a/api/apps/sdk/doc.py
+++ b/api/apps/sdk/doc.py
@@ -16,9 +16,10 @@
import logging
from io import BytesIO
-from quart import request, send_file
+from quart import send_file
-from api.db.db_models import APIToken, Document, Task
+from api.apps import login_required
+from api.db.db_models import Document, Task
from api.db.joint_services.tenant_model_service import get_model_config_by_id, get_model_config_by_type_and_name, get_tenant_default_model_by_type
from api.db.services.doc_metadata_service import DocMetadataService
from api.db.services.document_service import DocumentService
@@ -27,7 +28,7 @@
from api.db.services.llm_service import LLMBundle
from api.db.services.task_service import TaskService, cancel_all_task_of, queue_tasks
from api.db.services.tenant_llm_service import TenantLLMService
-from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_request_json, get_result, server_error_response, token_required
+from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_request_json, get_result, server_error_response, token_required, add_tenant_id_to_kwargs
from common import settings
from common.constants import LLMType, RetCode, TaskStatus
from common.metadata_utils import convert_conditions, meta_filter
@@ -51,7 +52,8 @@ def _enrich_chunks_with_document_metadata(chunks: list[dict], metadata_fields=No
@manager.route("/datasets//documents/", methods=["GET"]) # noqa: F821
-@token_required
+@login_required
+@add_tenant_id_to_kwargs
async def download(tenant_id, dataset_id, document_id):
"""
Download a document from a dataset.
@@ -90,8 +92,6 @@ async def download(tenant_id, dataset_id, document_id):
"""
if not document_id:
return get_error_data_result(message="Specify document_id please.")
- if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
- return get_error_data_result(message=f"You do not own the dataset {dataset_id}.")
doc = DocumentService.query(kb_id=dataset_id, id=document_id)
if not doc:
return get_error_data_result(message=f"The dataset not own the document {document_id}.")
@@ -110,51 +110,6 @@ async def download(tenant_id, dataset_id, document_id):
)
-@manager.route("/documents/", methods=["GET"]) # noqa: F821
-async def download_doc(document_id):
- token = request.headers.get("Authorization").split()
- if len(token) != 2:
- return get_error_data_result(message="Authorization is not valid!")
- token = token[1]
- logging.info("Beta API token lookup attempted for document download")
- objs = APIToken.query(beta=token)
- if not objs:
- logging.warning("Beta API token lookup failed for document download: invalid API key")
- return get_error_data_result(message='Authentication error: API key is invalid!"')
- if len(objs) > 1:
- logging.error("Beta API token lookup is ambiguous for document download: matches=%s", len(objs))
- return get_error_data_result(message="Authentication error: API key configuration is ambiguous.")
- tenant_id = objs[0].tenant_id
- logging.info("Beta API token authorized for document download: tenant_id=%s", tenant_id)
-
- if not document_id:
- return get_error_data_result(message="Specify document_id please.")
- doc = DocumentService.query(id=document_id)
- if not doc:
- return get_error_data_result(message=f"The dataset not own the document {document_id}.")
- if not KnowledgebaseService.query(id=doc[0].kb_id, tenant_id=tenant_id):
- logging.warning(
- "cross-tenant access denied for document download: tenant_id=%s kb_id=%s document_id=%s",
- tenant_id,
- doc[0].kb_id,
- document_id,
- )
- return get_error_data_result(message="You do not have access to this document.")
- # The process of downloading
- doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address
- file_stream = settings.STORAGE_IMPL.get(doc_id, doc_location)
- if not file_stream:
- return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
- file = BytesIO(file_stream)
- # Use send_file with a proper filename and MIME type
- return await send_file(
- file,
- as_attachment=True,
- attachment_filename=doc[0].name,
- mimetype="application/octet-stream", # Set a default MIME type
- )
-
-
DOC_STOP_PARSING_INVALID_STATE_MESSAGE = "Can't stop parsing document that has not started or already completed"
DOC_STOP_PARSING_INVALID_STATE_ERROR_CODE = "DOC_STOP_PARSING_INVALID_STATE"
diff --git a/web/src/pages/dataset/dataset/dataset-action-cell.tsx b/web/src/pages/dataset/dataset/dataset-action-cell.tsx
index 722fd15ade1..296b519ba5e 100644
--- a/web/src/pages/dataset/dataset/dataset-action-cell.tsx
+++ b/web/src/pages/dataset/dataset/dataset-action-cell.tsx
@@ -8,9 +8,10 @@ import {
import { DocumentType } from '@/constants/knowledge';
import { useRemoveDocument } from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
+import { downloadDatasetDocument } from '@/services/file-manager-service';
import { formatFileSize } from '@/utils/common-util';
import { formatDate } from '@/utils/date';
-import { downloadDocument } from '@/utils/file-util';
+import { downloadFileFromBlob } from '@/utils/file-util';
import { Download, Eye, PenLine, Trash2 } from 'lucide-react';
import { useCallback } from 'react';
import { UseRenameDocumentShowType } from './use-rename-document';
@@ -34,12 +35,22 @@ export function DatasetActionCell({
const { removeDocument } = useRemoveDocument();
- const onDownloadDocument = useCallback(() => {
- downloadDocument({
- id,
- filename: record.name,
- });
- }, [id, record.name]);
+ const onDownloadDocument = useCallback(async () => {
+ try {
+ const ext = record.name.split('.').pop()?.toLowerCase() || 'bin';
+ const response = await downloadDatasetDocument({
+ datasetId: record.dataset_id,
+ docId: id,
+ ext,
+ });
+ const blob = new Blob([response.data], {
+ type: response.data.type,
+ });
+ downloadFileFromBlob(blob, record.name);
+ } catch (error) {
+ console.error('Error downloading document:', error);
+ }
+ }, [id, record.dataset_id, record.name]);
const handleRemove = useCallback(() => {
removeDocument(id);
diff --git a/web/src/services/file-manager-service.ts b/web/src/services/file-manager-service.ts
index ad83a43c46d..aca836d1006 100644
--- a/web/src/services/file-manager-service.ts
+++ b/web/src/services/file-manager-service.ts
@@ -12,6 +12,7 @@ const {
getDocumentFile,
getFile,
moveFile,
+ getDatasetDocumentFileDownload,
getDocumentFileDownload,
} = api;
@@ -67,4 +68,18 @@ export const downloadFile = (data: { docId: string; ext: string }) => {
responseType: 'blob',
});
};
+
+export const downloadDatasetDocument = (data: {
+ datasetId: string;
+ docId: string;
+ ext: string;
+}) => {
+ return request.get(
+ getDatasetDocumentFileDownload(data.datasetId, data.docId),
+ {
+ params: { ext: data.ext },
+ responseType: 'blob',
+ },
+ );
+};
export default fileManagerService;
diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts
index 5dbfc8e369f..f3d6456c478 100644
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -126,6 +126,8 @@ export default {
`${restAPIv1}/datasets/${datasetId}/documents?type=empty`,
documentChangeParser: (datasetId: string, documentId: string) =>
`${restAPIv1}/datasets/${datasetId}/documents/${documentId}`,
+ getDatasetDocumentFileDownload: (datasetId: string, documentId: string) =>
+ `${restAPIv1}/datasets/${datasetId}/documents/${documentId}`,
documentThumbnails: `${restAPIv1}/thumbnails`,
getDocumentFile: `${restAPIv1}/documents`,
getDocumentFileDownload: (docId: string) =>
From f038a341544304a586449a6bdaa33cb17dfd27f3 Mon Sep 17 00:00:00 2001
From: Liu An
Date: Thu, 14 May 2026 11:07:08 +0800
Subject: [PATCH 132/666] Docs: Update version references to v0.25.4 in READMEs
and docs (#14912)
### What problem does this PR solve?
- Update version tags in README files (including translations) from
v0.25.3 to v0.25.4
- Modify Docker image references and documentation to reflect new
version
- Update version badges and image descriptions
- Maintain consistency across all language variants of README files
### Type of change
- [x] Documentation Update
---
README.md | 6 +++---
README_ar.md | 6 +++---
README_fr.md | 6 +++---
README_id.md | 6 +++---
README_ja.md | 6 +++---
README_ko.md | 6 +++---
README_pt_br.md | 6 +++---
README_tr.md | 6 +++---
README_tzh.md | 6 +++---
README_zh.md | 6 +++---
admin/client/README.md | 2 +-
admin/client/pyproject.toml | 2 +-
admin/client/uv.lock | 2 +-
docker/.env | 6 +++---
docker/README.md | 2 +-
docs/administrator/admin/ragflow_cli.md | 4 ++--
.../configurations/configurations.md | 2 +-
docs/administrator/upgrade_ragflow.mdx | 10 +++++-----
docs/develop/build_docker_image.mdx | 2 +-
docs/faq.mdx | 6 +++---
.../guides/dataset/configure_knowledge_base.md | 2 +-
docs/guides/manage_files.md | 2 +-
docs/quickstart.mdx | 6 +++---
helm/values.yaml | 2 +-
pyproject.toml | 2 +-
sdk/python/pyproject.toml | 2 +-
sdk/python/uv.lock | 2 +-
test/README.md | 2 +-
tools/scripts/README.md | 18 +++++++++---------
tools/scripts/db_schema_sync.py | 16 ++++++++--------
uv.lock | 2 +-
31 files changed, 77 insertions(+), 77 deletions(-)
diff --git a/README.md b/README.md
index 89dd6740660..d724690d3c0 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@
-
+
@@ -192,12 +192,12 @@ releases! 🌟
> All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
> If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
-> The command below downloads the `v0.25.3` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.25.3`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
+> The command below downloads the `v0.25.4` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.25.4`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
# This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
diff --git a/README_ar.md b/README_ar.md
index 618fc117861..b3c244f9876 100644
--- a/README_ar.md
+++ b/README_ar.md
@@ -25,7 +25,7 @@
-
+
@@ -192,12 +192,12 @@
> جميع الصور Docker مصممة لمنصات x86. لا نعرض حاليًا صور Docker لـ ARM64.
> إذا كنت تستخدم نظامًا أساسيًا ARM64، فاتبع [هذا الدليل](https://ragflow.io/docs/dev/build_docker_image) لإنشاء صورة Docker متوافقة مع نظامك.
-> يقوم الأمر أدناه بتنزيل إصدار `v0.25.3` من الصورة RAGFlow Docker. راجع الجدول التالي للحصول على أوصاف لإصدارات RAGFlow المختلفة. لتنزيل إصدار RAGFlow مختلف عن `v0.25.3`، قم بتحديث المتغير `RAGFLOW_IMAGE` وفقًا لذلك في **docker/.env** قبل استخدام `docker compose` لبدء تشغيل الخادم.
+> يقوم الأمر أدناه بتنزيل إصدار `v0.25.4` من الصورة RAGFlow Docker. راجع الجدول التالي للحصول على أوصاف لإصدارات RAGFlow المختلفة. لتنزيل إصدار RAGFlow مختلف عن `v0.25.4`، قم بتحديث المتغير `RAGFLOW_IMAGE` وفقًا لذلك في **docker/.env** قبل استخدام `docker compose` لبدء تشغيل الخادم.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
# This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
diff --git a/README_fr.md b/README_fr.md
index ddf2c97a579..354c0e32216 100644
--- a/README_fr.md
+++ b/README_fr.md
@@ -25,7 +25,7 @@
-
+
@@ -189,12 +189,12 @@ Essayez notre service cloud sur [https://cloud.ragflow.io](https://cloud.ragflow
> Toutes les images Docker sont construites pour les plateformes x86. Nous ne proposons pas actuellement d'images Docker pour ARM64.
> Si vous êtes sur une plateforme ARM64, suivez [ce guide](https://ragflow.io/docs/dev/build_docker_image) pour construire une image Docker compatible avec votre système.
-> La commande ci-dessous télécharge l'édition `v0.25.3` de l'image Docker RAGFlow. Consultez le tableau suivant pour les descriptions des différentes éditions de RAGFlow. Pour télécharger une édition de RAGFlow différente de `v0.25.3`, mettez à jour la variable `RAGFLOW_IMAGE` dans **docker/.env** avant d'utiliser `docker compose` pour démarrer le serveur.
+> La commande ci-dessous télécharge l'édition `v0.25.4` de l'image Docker RAGFlow. Consultez le tableau suivant pour les descriptions des différentes éditions de RAGFlow. Pour télécharger une édition de RAGFlow différente de `v0.25.4`, mettez à jour la variable `RAGFLOW_IMAGE` dans **docker/.env** avant d'utiliser `docker compose` pour démarrer le serveur.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# Optionnel : utiliser un tag stable (voir les versions : https://github.com/infiniflow/ragflow/releases)
# Cette étape garantit que le fichier **entrypoint.sh** dans le code correspond à la version de l'image Docker.
diff --git a/README_id.md b/README_id.md
index bc0ba7e2f37..2eeae3f234c 100644
--- a/README_id.md
+++ b/README_id.md
@@ -25,7 +25,7 @@
-
+
@@ -192,12 +192,12 @@ Coba layanan cloud kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
> Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
> Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
-> Perintah di bawah ini mengunduh edisi v0.25.3 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.25.3, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
+> Perintah di bawah ini mengunduh edisi v0.25.4 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.25.4, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases)
# This steps ensures the **entrypoint.sh** file in the code matches the Docker image version.
diff --git a/README_ja.md b/README_ja.md
index f162c6d30ba..5d67babc37b 100644
--- a/README_ja.md
+++ b/README_ja.md
@@ -25,7 +25,7 @@
-
+
@@ -172,12 +172,12 @@
> 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
> ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
-> 以下のコマンドは、RAGFlow Docker イメージの v0.25.3 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.25.3 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
+> 以下のコマンドは、RAGFlow Docker イメージの v0.25.4 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.25.4 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases)
# この手順は、コード内の entrypoint.sh ファイルが Docker イメージのバージョンと一致していることを確認します。
diff --git a/README_ko.md b/README_ko.md
index 68da482bdde..9c46284a1c4 100644
--- a/README_ko.md
+++ b/README_ko.md
@@ -25,7 +25,7 @@
-
+
@@ -174,12 +174,12 @@
> 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
> ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
- > 아래 명령어는 RAGFlow Docker 이미지의 v0.25.3 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.25.3와 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
+ > 아래 명령어는 RAGFlow Docker 이미지의 v0.25.4 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.25.4와 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
# 이 단계는 코드의 entrypoint.sh 파일이 Docker 이미지 버전과 일치하도록 보장합니다.
diff --git a/README_pt_br.md b/README_pt_br.md
index b06bf460925..a03da741efa 100644
--- a/README_pt_br.md
+++ b/README_pt_br.md
@@ -25,7 +25,7 @@
-
+
@@ -192,12 +192,12 @@ Experimente o nosso serviço na nuvem em [https://cloud.ragflow.io](https://clou
> Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
> Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
- > O comando abaixo baixa a edição`v0.25.3` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.25.3`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
+ > O comando abaixo baixa a edição`v0.25.4` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.25.4`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases)
# Esta etapa garante que o arquivo entrypoint.sh no código corresponda à versão da imagem do Docker.
diff --git a/README_tr.md b/README_tr.md
index 95e3ed89e8b..22903d60444 100644
--- a/README_tr.md
+++ b/README_tr.md
@@ -25,7 +25,7 @@
-
+
@@ -190,12 +190,12 @@ Bulut hizmetimizi [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinde
> Tüm Docker imajları x86 platformları için oluşturulmuştur. Şu anda ARM64 için Docker imajı sunmuyoruz.
> ARM64 platformundaysanız, sisteminizle uyumlu bir Docker imajı oluşturmak için [bu kılavuzu](https://ragflow.io/docs/dev/build_docker_image) takip edin.
-> Aşağıdaki komut RAGFlow Docker imajının `v0.25.3` sürümünü indirir. Farklı RAGFlow sürümleri için aşağıdaki tabloya bakın. `v0.25.3` dışında bir sürüm indirmek için, `docker compose` ile sunucuyu başlatmadan önce **docker/.env** dosyasındaki `RAGFLOW_IMAGE` değişkenini güncelleyin.
+> Aşağıdaki komut RAGFlow Docker imajının `v0.25.4` sürümünü indirir. Farklı RAGFlow sürümleri için aşağıdaki tabloya bakın. `v0.25.4` dışında bir sürüm indirmek için, `docker compose` ile sunucuyu başlatmadan önce **docker/.env** dosyasındaki `RAGFLOW_IMAGE` değişkenini güncelleyin.
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# İsteğe bağlı: Kararlı bir etiket kullanın (sürümler: https://github.com/infiniflow/ragflow/releases)
# Bu adım, koddaki **entrypoint.sh** dosyasının Docker imaj sürümüyle eşleşmesini sağlar.
diff --git a/README_tzh.md b/README_tzh.md
index c9452929119..e4b33a19915 100644
--- a/README_tzh.md
+++ b/README_tzh.md
@@ -25,7 +25,7 @@
-
+
@@ -191,12 +191,12 @@
> 所有 Docker 映像檔都是為 x86 平台建置的。目前,我們不提供 ARM64 平台的 Docker 映像檔。
> 如果您使用的是 ARM64 平台,請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
-> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.25.3`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.25.3` 的 Docker 映像,請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
+> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.25.4`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.25.4` 的 Docker 映像,請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# 可選:使用穩定版標籤(查看發佈:https://github.com/infiniflow/ragflow/releases)
# 此步驟確保程式碼中的 entrypoint.sh 檔案與 Docker 映像版本一致。
diff --git a/README_zh.md b/README_zh.md
index 082160a6a47..0e17af24db7 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -25,7 +25,7 @@
-
+
@@ -192,12 +192,12 @@
> 请注意,目前官方提供的所有 Docker 镜像均基于 x86 架构构建,并不提供基于 ARM64 的 Docker 镜像。
> 如果你的操作系统是 ARM64 架构,请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
- > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.25.3`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.25.3` 的 Docker 镜像,请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
+ > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.25.4`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.25.4` 的 Docker 镜像,请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
```bash
$ cd ragflow/docker
- # git checkout v0.25.3
+ # git checkout v0.25.4
# 可选:使用稳定版本标签(查看发布:https://github.com/infiniflow/ragflow/releases)
# 这一步确保代码中的 entrypoint.sh 文件与 Docker 镜像的版本保持一致。
diff --git a/admin/client/README.md b/admin/client/README.md
index fb4e5c4ddca..9274120ad78 100644
--- a/admin/client/README.md
+++ b/admin/client/README.md
@@ -48,7 +48,7 @@ It consists of a server-side Service and a command-line client (CLI), both imple
1. Ensure the Admin Service is running.
2. Install ragflow-cli.
```bash
- pip install ragflow-cli==0.25.3
+ pip install ragflow-cli==0.25.4
```
3. Launch the CLI client:
```bash
diff --git a/admin/client/pyproject.toml b/admin/client/pyproject.toml
index 3a48fc545f1..0d849f40e96 100644
--- a/admin/client/pyproject.toml
+++ b/admin/client/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "ragflow-cli"
-version = "0.25.3"
+version = "0.25.4"
description = "Admin Service's client of [RAGFlow](https://github.com/infiniflow/ragflow). The Admin Service provides user management and system monitoring. "
authors = [{ name = "Lynn", email = "lynn_inf@hotmail.com" }]
license = { text = "Apache License, Version 2.0" }
diff --git a/admin/client/uv.lock b/admin/client/uv.lock
index f4a28e9765d..649ff599b5c 100644
--- a/admin/client/uv.lock
+++ b/admin/client/uv.lock
@@ -188,7 +188,7 @@ wheels = [
[[package]]
name = "ragflow-cli"
-version = "0.25.3"
+version = "0.25.4"
source = { virtual = "." }
dependencies = [
{ name = "beartype" },
diff --git a/docker/.env b/docker/.env
index 85fec3ec5be..9b512c8144f 100644
--- a/docker/.env
+++ b/docker/.env
@@ -159,11 +159,11 @@ GO_ADMIN_PORT=9383
API_PROXY_SCHEME=python # use pure python server deployment
# The RAGFlow Docker image to download. v0.22+ doesn't include embedding models.
-RAGFLOW_IMAGE=infiniflow/ragflow:v0.25.3
+RAGFLOW_IMAGE=infiniflow/ragflow:v0.25.4
# If you cannot download the RAGFlow Docker image:
-# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.25.3
-# RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:v0.25.3
+# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.25.4
+# RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:v0.25.4
#
# - For the `nightly` edition, uncomment either of the following:
# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:nightly
diff --git a/docker/README.md b/docker/README.md
index ff68ca7e501..c681b38ff40 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -79,7 +79,7 @@ The [.env](./.env) file contains important environment variables for Docker.
- `SVR_HTTP_PORT`
The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`.
- `RAGFLOW_IMAGE`
- The Docker image edition. Defaults to `infiniflow/ragflow:v0.25.3`. The RAGFlow Docker image does not include embedding models.
+ The Docker image edition. Defaults to `infiniflow/ragflow:v0.25.4`. The RAGFlow Docker image does not include embedding models.
> [!TIP]
diff --git a/docs/administrator/admin/ragflow_cli.md b/docs/administrator/admin/ragflow_cli.md
index 6bc659efbf0..6c7bc5943c9 100644
--- a/docs/administrator/admin/ragflow_cli.md
+++ b/docs/administrator/admin/ragflow_cli.md
@@ -16,7 +16,7 @@ The RAGFlow CLI is a command-line-based system administration tool that offers a
2. Install ragflow-cli.
```bash
- pip install ragflow-cli==0.25.3
+ pip install ragflow-cli==0.25.4
```
3. Launch the CLI client:
@@ -439,7 +439,7 @@ show_version
+-----------------------+
| version |
+-----------------------+
-| v0.25.3-24-g6f60e9f9e |
+| v0.25.4-24-g6f60e9f9e |
+-----------------------+
```
diff --git a/docs/administrator/configurations/configurations.md b/docs/administrator/configurations/configurations.md
index 58ceba58d05..83a1677d010 100644
--- a/docs/administrator/configurations/configurations.md
+++ b/docs/administrator/configurations/configurations.md
@@ -103,7 +103,7 @@ RAGFlow utilizes MinIO as its object storage solution, leveraging its scalabilit
- `SVR_HTTP_PORT`
The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`.
- `RAGFLOW_IMAGE`
- The Docker image edition. Defaults to `infiniflow/ragflow:v0.25.3` (the RAGFlow Docker image without embedding models).
+ The Docker image edition. Defaults to `infiniflow/ragflow:v0.25.4` (the RAGFlow Docker image without embedding models).
:::tip NOTE
If you cannot download the RAGFlow Docker image, try the following mirrors.
diff --git a/docs/administrator/upgrade_ragflow.mdx b/docs/administrator/upgrade_ragflow.mdx
index 1e8b47c5898..790430958cc 100644
--- a/docs/administrator/upgrade_ragflow.mdx
+++ b/docs/administrator/upgrade_ragflow.mdx
@@ -62,16 +62,16 @@ To upgrade RAGFlow, you must upgrade **both** your code **and** your Docker imag
git pull
```
-3. Switch to the latest, officially published release, e.g., `v0.25.3`:
+3. Switch to the latest, officially published release, e.g., `v0.25.4`:
```bash
- git checkout -f v0.25.3
+ git checkout -f v0.25.4
```
4. Update **ragflow/docker/.env**:
```bash
- RAGFLOW_IMAGE=infiniflow/ragflow:v0.25.3
+ RAGFLOW_IMAGE=infiniflow/ragflow:v0.25.4
```
5. Update the RAGFlow image and restart RAGFlow:
@@ -92,10 +92,10 @@ No, you do not need to. Upgrading RAGFlow in itself will *not* remove your uploa
1. From an environment with Internet access, pull the required Docker image.
2. Save the Docker image to a **.tar** file.
```bash
- docker save -o ragflow.v0.25.3.tar infiniflow/ragflow:v0.25.3
+ docker save -o ragflow.v0.25.4.tar infiniflow/ragflow:v0.25.4
```
3. Copy the **.tar** file to the target server.
4. Load the **.tar** file into Docker:
```bash
- docker load -i ragflow.v0.25.3.tar
+ docker load -i ragflow.v0.25.4.tar
```
diff --git a/docs/develop/build_docker_image.mdx b/docs/develop/build_docker_image.mdx
index 2bf9d879fac..989b409f40b 100644
--- a/docs/develop/build_docker_image.mdx
+++ b/docs/develop/build_docker_image.mdx
@@ -49,7 +49,7 @@ After building the infiniflow/ragflow:nightly image, you are ready to launch a f
1. Edit Docker Compose Configuration
-Open the `docker/.env` file. Find the `RAGFLOW_IMAGE` setting and change the image reference from `infiniflow/ragflow:v0.25.3` to `infiniflow/ragflow:nightly` to use the pre-built image.
+Open the `docker/.env` file. Find the `RAGFLOW_IMAGE` setting and change the image reference from `infiniflow/ragflow:v0.25.4` to `infiniflow/ragflow:nightly` to use the pre-built image.
2. Launch the Service
diff --git a/docs/faq.mdx b/docs/faq.mdx
index dc1739ce922..391b16c5906 100644
--- a/docs/faq.mdx
+++ b/docs/faq.mdx
@@ -147,12 +147,12 @@ When debugging your chat assistant, you can use AI search as a reference to veri
---
-### Get a `Request error 404: undefined` when upgrading to v0.25.3
+### Get a `Request error 404: undefined` when upgrading to v0.25.4
To resolve this issue, do either of the following:
-- Pull the latest source code from the [main branch](https://github.com/infiniflow/ragflow), then pull and start the v0.25.3 image.
-- Update `RAGFLOW_IMAGE` from `infiniflow/ragflow:latest` to `infiniflow/ragflow:v0.25.3` in the [.env file](https://github.com/infiniflow/ragflow/blob/main/docker/.env), then restart the service.
+- Pull the latest source code from the [main branch](https://github.com/infiniflow/ragflow), then pull and start the v0.25.4 image.
+- Update `RAGFLOW_IMAGE` from `infiniflow/ragflow:latest` to `infiniflow/ragflow:v0.25.4` in the [.env file](https://github.com/infiniflow/ragflow/blob/main/docker/.env), then restart the service.
### How to build the RAGFlow image from scratch?
diff --git a/docs/guides/dataset/configure_knowledge_base.md b/docs/guides/dataset/configure_knowledge_base.md
index 694b810b555..50cb24c23fa 100644
--- a/docs/guides/dataset/configure_knowledge_base.md
+++ b/docs/guides/dataset/configure_knowledge_base.md
@@ -135,7 +135,7 @@ See [Run retrieval test](./run_retrieval_test.md) for details.
## Search for dataset
-As of RAGFlow v0.25.3, the search feature is still in a rudimentary form, supporting only dataset search by name.
+As of RAGFlow v0.25.4, the search feature is still in a rudimentary form, supporting only dataset search by name.

diff --git a/docs/guides/manage_files.md b/docs/guides/manage_files.md
index 9171786c341..82a88b5ef84 100644
--- a/docs/guides/manage_files.md
+++ b/docs/guides/manage_files.md
@@ -89,4 +89,4 @@ RAGFlow's file management allows you to download an uploaded file:

-> As of RAGFlow v0.25.3, bulk download is not supported, nor can you download an entire folder.
+> As of RAGFlow v0.25.4, bulk download is not supported, nor can you download an entire folder.
diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx
index 2b9a6572da1..9cbbf0fbdd4 100644
--- a/docs/quickstart.mdx
+++ b/docs/quickstart.mdx
@@ -48,7 +48,7 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
`vm.max_map_count`. This value sets the maximum number of memory map areas a process may have. Its default value is 65530. While most applications require fewer than a thousand maps, reducing this value can result in abnormal behaviors, and the system will throw out-of-memory errors when a process reaches the limitation.
- RAGFlow v0.25.3 uses Elasticsearch or [Infinity](https://github.com/infiniflow/infinity) for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
+ RAGFlow v0.25.4 uses Elasticsearch or [Infinity](https://github.com/infiniflow/infinity) for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
bool:
def version_to_dirname(version: str) -> str:
- """Convert version string to valid directory name (e.g., 'v0.25.3' -> 'v0_25_3')"""
+ """Convert version string to valid directory name (e.g., 'v0.25.4' -> 'v0_25_4')"""
return version.replace('.', '_')
@@ -839,19 +839,19 @@ def main():
epilog="""
Examples:
# List all migrations
- python db_schema_sync.py --list --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.3
+ python db_schema_sync.py --list --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.4
# Create migration from model changes
- python db_schema_sync.py --create --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.3
+ python db_schema_sync.py --create --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.4
# Create migration including dropped fields (destructive!)
- python db_schema_sync.py --create --drop --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.3
+ python db_schema_sync.py --create --drop --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.4
# Run all pending migrations
- python db_schema_sync.py --migrate --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.3
+ python db_schema_sync.py --migrate --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.4
# Show schema differences
- python db_schema_sync.py --diff --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.3
+ python db_schema_sync.py --diff --host localhost --port 3306 --user root --password xxx --database rag_flow --version v0.25.4
"""
)
@@ -864,7 +864,7 @@ def main():
# Version option
parser.add_argument('--version', '-v', type=str, required=True,
- help='Version number in format vxx.xx.xx (e.g., v0.25.3)')
+ help='Version number in format vxx.xx.xx (e.g., v0.25.4)')
# Action options
parser.add_argument('--list', '-l', action='store_true', help='List all migrations')
@@ -882,7 +882,7 @@ def main():
# Validate version format
if not validate_version(args.version):
- logger.error(f"Invalid version format: {args.version}. Expected format: vxx.xx.xx (e.g., v0.25.3)")
+ logger.error(f"Invalid version format: {args.version}. Expected format: vxx.xx.xx (e.g., v0.25.4)")
sys.exit(1)
# Validate at least one action is specified
diff --git a/uv.lock b/uv.lock
index 538a3f0a083..8a66238e96b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6754,7 +6754,7 @@ wheels = [
[[package]]
name = "ragflow"
-version = "0.25.3"
+version = "0.25.4"
source = { virtual = "." }
dependencies = [
{ name = "agentrun-sdk" },
From 851b16b91399880f632124c558aae96c803d889f Mon Sep 17 00:00:00 2001
From: writinwaters <93570324+writinwaters@users.noreply.github.com>
Date: Thu, 14 May 2026 11:24:20 +0800
Subject: [PATCH 133/666] Docs: Added v0.25.4 release notes draft. (#14914)
### What problem does this PR solve?
v0.25.4 release notes draft.
### Type of change
- [x] Documentation Update
---
docs/release_notes.md | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/docs/release_notes.md b/docs/release_notes.md
index 3f58353ff9d..292217887bf 100644
--- a/docs/release_notes.md
+++ b/docs/release_notes.md
@@ -9,6 +9,27 @@ sidebar_custom_props: {
Key features, improvements and bug fixes in the latest releases.
+## v0.25.4
+
+Released on May 14, 2026
+
+### New features
+
+- Adds a generic, configuration-driven RESTful API data source connector.
+
+### Improvements
+
+- Agent tag management with filtering and sorting.
+- Widget customization and persistence.
+
+### Model support
+
+- Adds gpt-5.4-mini and gpt-5.4-nano to the OpenAI model list
+
+### Bug fixes
+
+Fixed dataset document download route.
+
## v0.25.3
Released on May 13, 2026.
@@ -37,7 +58,6 @@ Released on May 13, 2026.
- Agent: Template strings in tool-type components like **Email** and **Invoke** failed to interpolate; `{{variable}}` placeholders were passed through as raw text. [#14601](https://github.com/infiniflow/ragflow/pull/14601)
- Volcengine (Doubao/Ark) endpoints were not visible in the provider list. [#14702](https://github.com/infiniflow/ragflow/pull/14702)
-
## v0.25.2
Released on May 11, 2026.
From 8dc5b1b42d1492c0ab2c5a3b8fd076b38ec99adc Mon Sep 17 00:00:00 2001
From: 07heco <3379248674@qq.com>
Date: Thu, 14 May 2026 11:56:09 +0800
Subject: [PATCH 134/666] fix: optimize reranking module robustness and bug
fixes (#14264)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Description
This PR fixes critical bugs and improves the robustness of the RAG
reranking module while maintaining **100% backward compatibility** with
all existing functionality and providers.
## Key Changes
1. **Network Stability**: Added 30s timeout to all API requests to
prevent service blocking
2. **Boundary Protection**: Added empty query/text validation for all
rerank models
3. **Response Fault Tolerance**: Replaced hardcoded key access with
`.get()` to avoid KeyError crashes
4. **Bug Fixes**:
- Fixed `Ai302Rerank` (completely non-functional before)
- Fixed `GPUStackRerank` incorrect exception catching
- Fixed `_normalize_rank` empty array crash
5. **Code Specification**: Added type annotations, standardized
unimplemented class prompts
## Compatibility
- ✅ No changes to any class/method names
- ✅ All rerank providers (Jina/Cohere/NVIDIA/HuggingFace etc.) work as
before
- ✅ No breaking changes, zero impact on existing workflows
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] Refactoring
---------
Co-authored-by: Kevin Hu
---
rag/llm/rerank_model.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py
index bcf8347e6fc..99801e00a78 100644
--- a/rag/llm/rerank_model.py
+++ b/rag/llm/rerank_model.py
@@ -435,6 +435,7 @@ def post(query: str, texts: list, url: str = "http://127.0.0.1"):
for i in range(0, len(texts), batch_size):
try:
+ # Fix: Add request timeout
res = requests.post(
endpoint, headers={"Content-Type": "application/json"},
json={"query": query, "texts": texts[i:i+batch_size], "raw_scores": False, "truncate": True},
From cc21dc7f007ea753e2c2063922abac6122f7c8fc Mon Sep 17 00:00:00 2001
From: Ricardo-M-L <69202550+Ricardo-M-L@users.noreply.github.com>
Date: Thu, 14 May 2026 12:33:17 +0800
Subject: [PATCH 135/666] fix: replace broken assert with raise ValueError in
variable_assigner and loop (#13906)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
\`assert \"string\"\` always passes in Python because non-empty strings
are truthy. This silently skips input validation:
- **variable_assigner.py line 51**: \`assert \"Variable is not
complete.\"\` → \`raise ValueError(\"Variable is not complete.\")\`
- **loop.py line 59**: \`assert \"Loop Variable is not complete.\"\` →
\`raise ValueError(\"Loop Variable is not complete.\")\`
Without this fix, incomplete variables pass validation silently and
cause a confusing KeyError on the next line.
---
agent/component/loop.py | 2 +-
agent/component/variable_assigner.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/agent/component/loop.py b/agent/component/loop.py
index 484dfae8256..9558e1001ef 100644
--- a/agent/component/loop.py
+++ b/agent/component/loop.py
@@ -56,7 +56,7 @@ def _invoke(self, **kwargs):
for item in self._param.loop_variables:
if any([not item.get("variable"), not item.get("input_mode"), not item.get("value"),not item.get("type")]):
- assert "Loop Variable is not complete."
+ raise ValueError("Loop Variable is not complete.")
if item["input_mode"]=="variable":
self.set_output(item["variable"],self._canvas.get_variable_value(item["value"]))
elif item["input_mode"]=="constant":
diff --git a/agent/component/variable_assigner.py b/agent/component/variable_assigner.py
index dd6182c7ce0..0f782136846 100644
--- a/agent/component/variable_assigner.py
+++ b/agent/component/variable_assigner.py
@@ -48,7 +48,7 @@ def _invoke(self, **kwargs):
else:
for item in self._param.variables:
if any([not item.get("variable"), not item.get("operator"), not item.get("parameter")]):
- assert "Variable is not complete."
+ raise ValueError("Variable is not complete.")
variable=item["variable"]
operator=item["operator"]
parameter=item["parameter"]
From d46bbd30f7f6f03de6ec3c8ba301b246576abfd6 Mon Sep 17 00:00:00 2001
From: Br1an <932039080@qq.com>
Date: Thu, 14 May 2026 13:11:37 +0800
Subject: [PATCH 136/666] Fix: send input and output token usage to Langfuse
(#13294)
### What problem does this PR solve?
Closes #9837
The Langfuse integration currently only sends the output text to
`langfuse_generation.update()` without including token usage
information. This means Langfuse cannot track input/output token
consumption for cost analysis and monitoring.
### Solution
Add the `usage` parameter to `langfuse_generation.update()` with:
- `input`: approximate input token count from `message_fit_in()`
- `output`: approximate output token count from
`num_tokens_from_string(answer)`
- `total`: sum of input and output
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---------
Co-authored-by: Kevin Hu
---
api/db/services/dialog_service.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py
index 6f981efb5e6..07dcd14b5ec 100644
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -809,7 +809,14 @@ def decorate_answer(answer):
if langfuse_tracer and "langfuse_generation" in locals():
langfuse_output = "\n" + re.sub(r"^.*?(### Query:.*)", r"\1", prompt, flags=re.DOTALL)
langfuse_output = {"time_elapsed:": re.sub(r"\n", " \n", langfuse_output), "created_at": time.time()}
- langfuse_generation.update(output=langfuse_output)
+ langfuse_generation.update(
+ output=langfuse_output,
+ usage_details={
+ "input": used_token_count,
+ "output": tk_num,
+ "total": used_token_count + tk_num,
+ },
+ )
langfuse_generation.end()
return {"answer": think + answer, "reference": refs, "prompt": re.sub(r"\n", " \n", prompt), "created_at": time.time()}
From ef46005ef1358bf5b50ee9a72d3014c98ffd6535 Mon Sep 17 00:00:00 2001
From: Haruko386
Date: Thu, 14 May 2026 13:19:31 +0800
Subject: [PATCH 137/666] Go: implement TTS for MiniMax provider and CLI
testing for TTS (#14911)
### What problem does this PR solve?
This PR implement TTS for MiniMax provider and CLI testing for TTS
**The following functionalities are now supported:**
**MiniMax:**
- [x] Chat / Stream Chat
- [x] Embedding
- [x] Rerank
- [x] Model listing
- [x] Provider connection checking
- [x] Text To Speech
- [ ] OCRFile
- [ ] ~~Audio To Text~~
- [ ] ~~Balance~~
**Verified examples from the CLI:**
```plaintext
RAGFlow(user)> tts with 'speech-2.8-hd@test@minimax' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"voice_setting": {"voice_id": "English_radiant_girl", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}'
Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/speech-2.8-hd_output.wav
SUCCESS
RAGFlow(user)> stream tts with 'speech-2.8-hd@test@minimax' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"voice_setting": {"voice_id": "English_radiant_girl", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}'
Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/speech-2.8-hd_output.wav
SUCCESS
```
Set `Play` to play audio in CLI
Set `Save` `PATH_TO_SAVE` to save file
Set `format` to save file in wav or mp3
Set `Param` align with official request body
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---
conf/models/minimax.json | 7 ++
internal/cli/lexer.go | 8 ++
internal/cli/parser.go | 1 +
internal/cli/types.go | 4 +
internal/cli/user_command.go | 139 ++++++++++++++++++++--
internal/cli/user_parser.go | 63 +++++++---
internal/development.md | 8 ++
internal/entity/models/minimax.go | 188 +++++++++++++++++++++++++++++-
internal/entity/models/types.go | 3 +
internal/handler/providers.go | 17 +--
10 files changed, 402 insertions(+), 36 deletions(-)
diff --git a/conf/models/minimax.json b/conf/models/minimax.json
index 31760ac2597..49aa6700a2d 100644
--- a/conf/models/minimax.json
+++ b/conf/models/minimax.json
@@ -99,6 +99,13 @@
"default_value": true,
"clear_thinking": true
}
+ },
+ {
+ "name": "speech-2.8-hd",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
}
]
}
\ No newline at end of file
diff --git a/internal/cli/lexer.go b/internal/cli/lexer.go
index 5f2aadea14f..6a0d1b0ff3b 100644
--- a/internal/cli/lexer.go
+++ b/internal/cli/lexer.go
@@ -455,6 +455,14 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenFatal, Value: ident}
case "PANIC":
return Token{Type: TokenPanic, Value: ident}
+ case "PARAM":
+ return Token{Type: TokenParam, Value: ident}
+ case "PLAY":
+ return Token{Type: TokenPlay, Value: ident}
+ case "FORMAT":
+ return Token{Type: TokenFormat, Value: ident}
+ case "SAVE":
+ return Token{Type: TokenSave, Value: ident}
default:
return Token{Type: TokenIdentifier, Value: ident}
}
diff --git a/internal/cli/parser.go b/internal/cli/parser.go
index 0bba27847b4..035d6b12e57 100644
--- a/internal/cli/parser.go
+++ b/internal/cli/parser.go
@@ -219,6 +219,7 @@ func (p *Parser) parseUserCommand() (*Command, error) {
return p.parseUpdateCommand()
case TokenRemove:
return p.parseRemoveCommand()
+
default:
return nil, fmt.Errorf("unknown command: %s", p.curToken.Value)
}
diff --git a/internal/cli/types.go b/internal/cli/types.go
index a30f26c6ad8..9dd32f55c7f 100644
--- a/internal/cli/types.go
+++ b/internal/cli/types.go
@@ -105,6 +105,10 @@ const (
TokenEmbed
TokenText
TokenQuery
+ TokenFormat
+ TokenParam
+ TokenPlay
+ TokenSave
TokenTop
TokenDimension
TokenAsync
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index f0f025cee20..7a2b2759675 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -27,6 +27,7 @@ import (
"net"
netUrl "net/url"
"os"
+ "os/exec"
"path/filepath"
ce "ragflow/internal/cli/filesystem"
"strings"
@@ -1973,6 +1974,52 @@ func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) {
"text": text,
}
+ ttsConfigPayload := make(map[string]interface{})
+
+ explicitFormat, hasExplicitFormat := cmd.Params["format"].(string)
+
+ if paramStr, ok := cmd.Params["param_str"].(string); ok && paramStr != "" {
+ var dynamicParams map[string]interface{}
+ if err := json.Unmarshal([]byte(paramStr), &dynamicParams); err != nil {
+ return nil, fmt.Errorf("param string must be valid JSON. Error: %w", err)
+ }
+
+ ttsConfigPayload["params"] = dynamicParams
+
+ if !hasExplicitFormat {
+ var findFormat func(map[string]interface{}) string
+ findFormat = func(m map[string]interface{}) string {
+ if val, ok := m["format"]; ok {
+ return fmt.Sprintf("%v", val)
+ }
+ if val, ok := m["response_format"]; ok {
+ return fmt.Sprintf("%v", val)
+ }
+ for _, v := range m {
+ if subMap, ok := v.(map[string]interface{}); ok {
+ if res := findFormat(subMap); res != "" {
+ return res
+ }
+ }
+ }
+ return ""
+ }
+ if ext := findFormat(dynamicParams); ext != "" {
+ explicitFormat = ext
+ }
+ }
+ }
+
+ if explicitFormat != "" {
+ ttsConfigPayload["format"] = explicitFormat
+ } else {
+ explicitFormat = "mp3"
+ }
+
+ if len(ttsConfigPayload) > 0 {
+ payload["tts_config"] = ttsConfigPayload
+ }
+
url := "/audio/speech"
resp, err := c.HTTPClient.Request("POST", url, "web", nil, payload)
@@ -1982,21 +2029,91 @@ func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) {
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to TTS document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
- var result CommonResponse
- if err = json.Unmarshal(resp.Body, &result); err != nil {
+
+ var ttsResult struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ Data struct {
+ Audio string `json:"audio"`
+ } `json:"data"`
+ }
+
+ if err = json.Unmarshal(resp.Body, &ttsResult); err != nil {
return nil, fmt.Errorf("TTS document failed: invalid JSON (%w)", err)
}
- if result.Code != 0 {
- return nil, fmt.Errorf("%s", result.Message)
+
+ if ttsResult.Code != 0 {
+ return nil, fmt.Errorf("%s", ttsResult.Message)
}
- result.Duration = resp.Duration
- // save file
- //err = os.WriteFile(fileToSave, resp.Body, 0644)
- //if err != nil {
- // result.Message += fmt.Sprintf("failed to save file: %s", err.Error())
- // result.Code = 1
- //}
+ // Convert Base64 back to the original audio byte stream
+ audioBytes, err := base64.StdEncoding.DecodeString(ttsResult.Data.Audio)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode audio base64: %w", err)
+ }
+
+ shouldPlay, _ := cmd.Params["play"].(bool)
+ shouldSave, _ := cmd.Params["save"].(bool)
+ saveDir, _ := cmd.Params["save_path"].(string)
+
+
+ fileName := fmt.Sprintf("%s_output.%s", modelName, explicitFormat)
+
+ cwd, err := os.Getwd()
+ if err != nil {
+ cwd = "."
+ }
+ localPath := filepath.Join(cwd, fileName)
+
+ if err := os.WriteFile(localPath, audioBytes, 0644); err != nil {
+ return nil, fmt.Errorf("failed to write local audio file: %w", err)
+ }
+
+ if shouldPlay {
+ cmdExec := exec.Command("aplay", localPath)
+ if err := cmdExec.Run(); err != nil {
+ fmt.Printf("Play error: %v (Hint: did you use 'format: wav' in your params?)\n", err)
+ }
+ }
+
+ var finalMessage string
+ if shouldSave {
+ if saveDir == "" {
+ saveDir = cwd
+ } else {
+ absSaveDir, err := filepath.Abs(saveDir)
+ if err == nil {
+ saveDir = absSaveDir
+ }
+
+ if err := os.MkdirAll(saveDir, 0755); err != nil {
+ return nil, fmt.Errorf("failed to create save directory: %w", err)
+ }
+
+ finalPath := filepath.Join(saveDir, fileName)
+ if err := os.WriteFile(finalPath, audioBytes, 0644); err != nil {
+ return nil, fmt.Errorf("failed to save file to target directory: %w", err)
+ }
+
+ if saveDir != cwd {
+ os.Remove(localPath)
+ }
+
+ finalMessage = fmt.Sprintf("Saved to directory: %s", finalPath)
+ }
+ } else {
+ defer os.Remove(localPath)
+ finalMessage = "TTS Task Completed (Audio not saved)"
+ }
+
+ if finalMessage != "" && shouldSave {
+ fmt.Println(finalMessage)
+ }
+
+ var result SimpleResponse
+ result.Code = 0
+ result.Message = "SUCCESS"
+ result.Duration = resp.Duration
return &result, nil
}
diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go
index b1e2e2ed554..04ebc7e87e9 100644
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -2773,38 +2773,71 @@ func (p *Parser) parseASRCommand() (*Command, error) {
}
func (p *Parser) parseTTSCommand() (*Command, error) {
- p.nextToken() // consume TTS
+ p.nextToken()
+
+ cmd := NewCommand("tts_user_command")
if p.curToken.Type != TokenWith {
- return nil, fmt.Errorf("expected WITH after TTS")
+ return nil, fmt.Errorf("expect 'with' after tts")
}
- p.nextToken() // consume WITH
+ p.nextToken()
- compositeModelName, err := p.parseQuotedString()
- if err != nil {
- return nil, err
+ if p.curToken.Type != TokenQuotedString && p.curToken.Type != TokenIdentifier {
+ return nil, fmt.Errorf("expect model name after 'with'")
}
+ cmd.Params["composite_model_name"] = strings.Trim(p.curToken.Value, "\"'")
p.nextToken()
if p.curToken.Type != TokenText {
- return nil, fmt.Errorf("expected TEXT to TTS")
+ return nil, fmt.Errorf("expect 'text' parameter")
}
- p.nextToken() // consume FILE
+ p.nextToken()
- text, err := p.parseQuotedString()
- if err != nil {
- return nil, err
+ if p.curToken.Type != TokenQuotedString {
+ return nil, fmt.Errorf("expect quoted string after 'text'")
}
+ cmd.Params["text"] = strings.Trim(p.curToken.Value, "\"'")
p.nextToken()
- // Semicolon is optional for UNSET TOKEN
+ for p.curToken.Type != TokenEOF && p.curToken.Type != TokenSemicolon {
+ switch p.curToken.Type {
+ case TokenPlay:
+ p.nextToken()
+ cmd.Params["play"] = true
+ case TokenParam:
+ p.nextToken()
+ if p.curToken.Type != TokenQuotedString {
+ return nil, fmt.Errorf("expect quoted string after 'param'")
+ }
+ cmd.Params["param_str"] = strings.Trim(p.curToken.Value, "\"'")
+ p.nextToken()
+ p.nextToken()
+ case TokenSave:
+ p.nextToken()
+
+ if p.curToken.Type != TokenQuotedString && p.curToken.Type != TokenIdentifier {
+ return nil, fmt.Errorf("expect directory path after 'save'")
+ }
+
+ cmd.Params["save"] = true
+ cmd.Params["save_path"] = strings.Trim(p.curToken.Value, "\"'")
+ p.nextToken()
+ case TokenFormat:
+ p.nextToken()
+ if p.curToken.Type != TokenQuotedString && p.curToken.Type != TokenIdentifier {
+ return nil, fmt.Errorf("expect format string (e.g. 'wav') after 'format'")
+ }
+ cmd.Params["format"] = strings.Trim(p.curToken.Value, "\"'")
+ p.nextToken()
+ default:
+ return nil, fmt.Errorf("unexpected token: %s", p.curToken.Value)
+ }
+ }
+
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
- cmd := NewCommand("tts_user_command")
- cmd.Params["composite_model_name"] = compositeModelName
- cmd.Params["text"] = text
return cmd, nil
}
diff --git a/internal/development.md b/internal/development.md
index 41ff7013ad8..c477e7a7326 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -356,3 +356,11 @@ RAGFlow(user)> list datasets;
| 0 | naive | 1 | embedding-2@ZHIPU-AI | 0abe79f9423311f1ad8d38a74640adcc | English | ccc | aaa | me | 2ba4881420fa11f19e9c38a74640adcc | 0 | 1777375201933 |
+-------------+--------------+----------------+----------------------+----------------------------------+----------+------+----------+------------+----------------------------------+-----------+---------------+
```
+
+### 6.23 Text to Speech
+
+```
+RAGFlow(user)> tts with 'speech-2.8-hd@test@minimax' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"voice_setting": {"voice_id": "English_radiant_girl", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}'
+Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/speech-2.8-hd_output.wav
+SUCCESS
+```
diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go
index e761a228665..1f8afe3b942 100644
--- a/internal/entity/models/minimax.go
+++ b/internal/entity/models/minimax.go
@@ -19,6 +19,7 @@ package models
import (
"bufio"
"bytes"
+ "encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -464,11 +465,194 @@ func (z *MinimaxModel) TranscribeAudioWithSender(modelName *string, file *string
// AudioSpeech convert audio to text
func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("MiniMax API key is missing")
+ }
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("text content is empty")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.TTS)
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "text": audioContent,
+ }
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ reqBody["stream"] = false
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", strings.TrimSpace(*apiConfig.ApiKey)))
+
+ resp, err := z.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("MiniMax TTS API error: status %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ var result struct {
+ BaseResp struct {
+ StatusCode int `json:"status_code"`
+ StatusMsg string `json:"status_msg"`
+ } `json:"base_resp"`
+ Data struct {
+ Audio string `json:"audio"` // HEX
+ } `json:"data"`
+ }
+
+ if err := json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ if result.BaseResp.StatusCode != 0 {
+ return nil, fmt.Errorf("MiniMax TTS returned error: %d - %s", result.BaseResp.StatusCode, result.BaseResp.StatusMsg)
+ }
+
+ // format HEX
+ audioBytes, err := hex.DecodeString(result.Data.Audio)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode MiniMax hex audio: %w", err)
+ }
+
+ return &TTSResponse{
+ Audio: audioBytes,
+ }, nil
}
+// tts with 'speech-2.8-hd@test@minimax' text 'If that day, out position was switched, would our fate, be different?' voice 'English_expressive_narrator' param '{"voice_setting": {"voice_id": "English_expressive_narrator", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}'
func (z *MinimaxModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", z.Name())
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("MiniMax API key is missing")
+ }
+ if audioContent == nil || *audioContent == "" {
+ return fmt.Errorf("text content is empty")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL := strings.TrimSuffix(z.BaseURL[region], "/")
+ if baseURL == "" {
+ baseURL = strings.TrimSuffix(z.BaseURL["default"], "/")
+ }
+ suffix := strings.TrimPrefix(z.URLSuffix.TTS, "/")
+ if suffix == "" {
+ suffix = "v1/t2a_v2"
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, suffix)
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "text": audioContent,
+ }
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ reqBody["stream"] = false
+ reqBody["stream"] = true
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", strings.TrimSpace(*apiConfig.ApiKey)))
+
+ resp, err := z.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("MiniMax stream TTS API error: %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 2*1024*1024)
+
+ for scanner.Scan() {
+ line := scanner.Text()
+
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+
+ dataStr := strings.TrimSpace(line[5:])
+ if dataStr == "" {
+ continue
+ }
+
+ var event struct {
+ Data struct {
+ Audio string `json:"audio"`
+ Status int `json:"status"`
+ } `json:"data"`
+ }
+
+ if err := json.Unmarshal([]byte(dataStr), &event); err != nil {
+ continue
+ }
+
+ if event.Data.Audio != "" {
+ audioBytes, err := hex.DecodeString(event.Data.Audio)
+ if err == nil && len(audioBytes) > 0 {
+ chunk := string(audioBytes)
+ if errSend := sender(&chunk, nil); errSend != nil {
+ return errSend
+ }
+ }
+ }
+
+ if event.Data.Status == 2 {
+ break
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("error reading minimax stream: %w", err)
+ }
+
+ return nil
}
// OCRFile OCR file
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index d4e6e0502e2..3de7ac51587 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -65,6 +65,7 @@ type ASRResponse struct {
}
type TTSResponse struct {
+ Audio []byte `json:"audio"`
}
type OCRResponse struct {
@@ -83,6 +84,7 @@ type URLSuffix struct {
Balance string `json:"balance"`
Files string `json:"files"`
Status string `json:"status"`
+ TTS string `json:"tts"`
}
type ChatConfig struct {
@@ -116,6 +118,7 @@ type ASRConfig struct {
}
type TTSConfig struct {
+ Params map[string]interface{}
}
type OCRConfig struct {
diff --git a/internal/handler/providers.go b/internal/handler/providers.go
index 58dba8352b1..3b060a5097e 100644
--- a/internal/handler/providers.go
+++ b/internal/handler/providers.go
@@ -1166,14 +1166,12 @@ func (h *ProviderHandler) TranscribeAudio(c *gin.Context) {
}
type AudioSpeechRequest struct {
- ProviderName *string `json:"provider_name"`
- InstanceName *string `json:"instance_name"`
- ModelName *string `json:"model_name"`
- Text *string `json:"text"`
- Language []string `json:"language"`
- Voice int `json:"voice"`
- Stream bool `json:"stream"`
- Volume bool `json:"volume"`
+ ProviderName *string `json:"provider_name"`
+ InstanceName *string `json:"instance_name"`
+ ModelName *string `json:"model_name"`
+ Text *string `json:"text"`
+ Stream bool `json:"stream"`
+ TTSConfig *models.TTSConfig `json:"tts_config"`
}
func (h *ProviderHandler) AudioSpeech(c *gin.Context) {
@@ -1219,6 +1217,9 @@ func (h *ProviderHandler) AudioSpeech(c *gin.Context) {
}
ttsConfig := models.TTSConfig{}
+ if req.TTSConfig != nil {
+ ttsConfig = *req.TTSConfig
+ }
// Check if it's a stream request
if req.Stream {
From 4bfdb1e12373e1c901a54d91968c496941bdf7b0 Mon Sep 17 00:00:00 2001
From: Ricardo-M-L <69202550+Ricardo-M-L@users.noreply.github.com>
Date: Thu, 14 May 2026 13:27:04 +0800
Subject: [PATCH 138/666] fix: correct nested path traversal in
set_variable_param_value (#13986)
## Summary
`Graph.set_variable_param_value()` in `agent/canvas.py` has a bug in its
nested path traversal logic. The `for` loop iterates through **all**
keys in the path (including the last one), descending into every level.
After the loop, it then tries to set `cur[keys[-1]] = value`, but `cur`
has already descended one level too deep.
**Example:** For `path = "a.b"`, `value = "hello"`:
- **Before (bug):** `obj["a"]["b"]` becomes `{"b": "hello"}` instead of
`"hello"`
- **After (fix):** `obj["a"]["b"]` becomes `"hello"` as expected
The fix changes `for key in keys:` to `for key in keys[:-1]:`, so the
loop only navigates to the parent dict, and the final key is set
directly. This is consistent with how the read-side counterpart
`get_variable_param_value()` works.
This method is called by `set_variable_value()` when assigning to nested
variable paths (e.g., `component@root.nested.key`), which is used by the
`VariableAssigner` component.
## Test plan
- [ ] Create a canvas with a VariableAssigner that writes to a nested
path (e.g., `component@obj.nested.key`)
- [ ] Verify the value is set correctly at the expected path, not
wrapped in an extra dict layer
- [ ] Verify single-key paths (e.g., `component@key`) still work
correctly
## Summary by CodeRabbit
* **Bug Fixes**
* Fixed a bug in variable parameter assignment where nested structures
were being incorrectly modified, ensuring values are now properly set at
their intended locations without unintended overwrites.
Co-authored-by: Claude Opus 4.6 (1M context)
---
agent/canvas.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/agent/canvas.py b/agent/canvas.py
index ab6d0ba9ff1..bbd06facbb9 100644
--- a/agent/canvas.py
+++ b/agent/canvas.py
@@ -263,7 +263,7 @@ def set_variable_param_value(self, obj: Any, path: str, value) -> Any:
keys = path.split('.')
if not path:
return value
- for key in keys:
+ for key in keys[:-1]:
if key not in cur or not isinstance(cur[key], dict):
cur[key] = {}
cur = cur[key]
From 48b4aa3e93b2a76b89776b0848ab5d60b7271494 Mon Sep 17 00:00:00 2001
From: Ricardo-M-L <69202550+Ricardo-M-L@users.noreply.github.com>
Date: Thu, 14 May 2026 13:28:58 +0800
Subject: [PATCH 139/666] Fix WebDriver resource leak in HTML-to-PDF conversion
(#14310)
### What problem does this PR solve?
In `api/utils/web_utils.py`, `__get_pdf_from_html()` creates a Chrome
WebDriver but only calls `driver.quit()` inside the `TimeoutException`
handler. If the page element becomes stale before the timeout (no
exception raised), the WebDriver is never quit, leaking the Chrome
browser process and returning `None`.
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
### Changes
- Move the PDF printing logic and `driver.quit()` outside the `except`
block so they execute on all code paths
- Use `try/finally` to ensure `driver.quit()` is always called, even if
the `Page.printToPDF` DevTools call fails
Co-authored-by: Claude Opus 4.7
---
api/utils/web_utils.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/api/utils/web_utils.py b/api/utils/web_utils.py
index 23d2421862d..e7c1b48f513 100644
--- a/api/utils/web_utils.py
+++ b/api/utils/web_utils.py
@@ -173,6 +173,9 @@ def __get_pdf_from_html(path: str, timeout: int, install_driver: bool, print_opt
try:
WebDriverWait(driver, timeout).until(staleness_of(driver.find_element(by=By.TAG_NAME, value="html")))
except TimeoutException:
+ pass
+
+ try:
calculated_print_options = {
"landscape": False,
"displayHeaderFooter": False,
@@ -181,8 +184,9 @@ def __get_pdf_from_html(path: str, timeout: int, install_driver: bool, print_opt
}
calculated_print_options.update(print_options)
result = __send_devtools(driver, "Page.printToPDF", calculated_print_options)
- driver.quit()
return base64.b64decode(result["data"])
+ finally:
+ driver.quit()
def is_valid_url(url: str) -> bool:
From 82e06db8c3c099ea11e5bbe28d95967c1bbbba7a Mon Sep 17 00:00:00 2001
From: buua436
Date: Thu, 14 May 2026 13:42:40 +0800
Subject: [PATCH 140/666] Doc: code component output section (#14915)
### What problem does this PR solve?
code component output section
### Type of change
- [x] Documentation Update
---
.../agent/agent_component_reference/code.mdx | 44 ++++++++++++++++++-
1 file changed, 43 insertions(+), 1 deletion(-)
diff --git a/docs/guides/agent/agent_component_reference/code.mdx b/docs/guides/agent/agent_component_reference/code.mdx
index d0af92cc184..fa1de1caabf 100644
--- a/docs/guides/agent/agent_component_reference/code.mdx
+++ b/docs/guides/agent/agent_component_reference/code.mdx
@@ -98,7 +98,49 @@ If you define output variables here, ensure they are also defined in your code i
### Output
-The defined output variable(s) will be auto-populated here.
+The output is split into two parts:
+
+- **Business**: the business output defined in **Return Value**
+- **System**: runtime fields that are populated automatically, such as `content`, `actual_type`, and `attachments`
+
+For example, the following code generates a simple line chart:
+
+```Python
+def main() -> dict:
+ from pathlib import Path
+
+ import matplotlib
+ matplotlib.use("Agg")
+ import matplotlib.pyplot as plt
+
+ artifacts_dir = Path("artifacts")
+ artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+ x = [1, 2, 3, 4, 5]
+ y = [2, 4, 6, 8, 10]
+
+ output_path = artifacts_dir / "simple_plot.png"
+
+ plt.figure(figsize=(6, 4))
+ plt.plot(x, y, marker="o")
+ plt.title("Simple Line Chart")
+ plt.xlabel("X")
+ plt.ylabel("Y")
+ plt.grid(True)
+ plt.tight_layout()
+ plt.savefig(output_path)
+ plt.close()
+
+ return {
+ "result": "plot generated successfully",
+ "file_path": str(output_path),
+ }
+```
+
+
+Business Output shows the return value you defined, while System Output shows the generated `content`, the inferred `actual_type`, and the collected `attachments`.
+
+
## Troubleshooting
From f0122179dd8d13c12f6c8d8a29501f9d2f7a9772 Mon Sep 17 00:00:00 2001
From: buua436
Date: Thu, 14 May 2026 13:46:46 +0800
Subject: [PATCH 141/666] GO: align time units with Python and centralize
timestamp injection in BaseModel (#14875)
### What problem does this PR solve?
align time units with Python and centralize timestamp injection in
BaseModel
### Type of change
- [x] Refactoring
---
internal/admin/service.go | 70 -----------------------
internal/dao/kb.go | 9 ---
internal/dao/migration.go | 26 ++++++++-
internal/dao/skill_search_config.go | 6 --
internal/dao/skill_space.go | 2 -
internal/dao/system_settings.go | 41 +++-----------
internal/entity/base.go | 88 +++++++++++++++++++++++++++++
internal/entity/skill_search.go | 5 +-
internal/entity/skill_space.go | 5 +-
internal/entity/system.go | 15 ++---
internal/service/api_token.go | 6 --
internal/service/chat.go | 14 -----
internal/service/chat_session.go | 11 ----
internal/service/datasets.go | 7 ---
internal/service/document.go | 7 ++-
internal/service/kb.go | 6 --
internal/service/memory.go | 5 --
internal/service/model_service.go | 13 -----
internal/service/skill_space.go | 12 ----
internal/service/user.go | 30 ----------
20 files changed, 135 insertions(+), 243 deletions(-)
diff --git a/internal/admin/service.go b/internal/admin/service.go
index 2b6e282effa..b857ac59aae 100644
--- a/internal/admin/service.go
+++ b/internal/admin/service.go
@@ -213,9 +213,6 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf
loginChannel := "password"
isSuperuser := role == "admin"
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
-
user := &entity.User{
ID: userID,
AccessToken: &accessToken,
@@ -228,12 +225,6 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf
IsAnonymous: "0",
LoginChannel: &loginChannel,
IsSuperuser: &isSuperuser,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
// Start transaction for creating user and related data
@@ -288,12 +279,6 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf
ParserIDs: parserIDs,
Credit: 512,
Status: &tenantStatus,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
if err := tx.Create(tenant).Error; err != nil {
rollbackTx()
@@ -309,12 +294,6 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf
Role: "owner",
InvitedBy: userID,
Status: &userTenantStatus,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
if err := tx.Create(userTenant).Error; err != nil {
rollbackTx()
@@ -345,12 +324,6 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf
Type: "folder",
Size: 0,
Location: &fileLocation,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
if err := tx.Create(file).Error; err != nil {
rollbackTx()
@@ -470,9 +443,6 @@ func (s *Service) getInitTenantLLM(userID string) ([]*entity.TenantLLM, error) {
llmName := llm.LLMName
modelType := llm.ModelType
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
-
tenantLLM := &entity.TenantLLM{
TenantID: userID,
LLMFactory: factoryConfig.Factory,
@@ -482,12 +452,6 @@ func (s *Service) getInitTenantLLM(userID string) ([]*entity.TenantLLM, error) {
APIBase: &apiBase,
MaxTokens: maxTokens,
Status: "1",
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
tenantLLMs = append(tenantLLMs, tenantLLM)
}
@@ -768,8 +732,6 @@ func (s *Service) ChangePassword(username, newPassword string) error {
}
user.Password = &hashedPassword
- now := time.Now().Unix()
- user.UpdateTime = &now
if err := s.userDAO.Update(user); err != nil {
return fmt.Errorf("failed to update user: %w", err)
@@ -807,8 +769,6 @@ func (s *Service) UpdateUserActivateStatus(username string, isActive bool) error
}
user.IsActive = targetStatus
- now := time.Now().Unix()
- user.UpdateTime = &now
if err := s.userDAO.Update(user); err != nil {
return fmt.Errorf("failed to update user: %w", err)
@@ -841,8 +801,6 @@ func (s *Service) GrantAdmin(username string) error {
isSuperuser := true
user.IsSuperuser = &isSuperuser
- now := time.Now().Unix()
- user.UpdateTime = &now
if err := s.userDAO.Update(user); err != nil {
return fmt.Errorf("failed to update user: %w", err)
@@ -875,8 +833,6 @@ func (s *Service) RevokeAdmin(username string) error {
isSuperuser := false
user.IsSuperuser = &isSuperuser
- now := time.Now().Unix()
- user.UpdateTime = &now
if err := s.userDAO.Update(user); err != nil {
return fmt.Errorf("failed to update user: %w", err)
@@ -959,16 +915,12 @@ func (s *Service) GenerateUserAPIToken(username string) (map[string]interface{},
// 3. Generate API token
key := utility.GenerateAPIToken()
beta := utility.GenerateBetaAPIToken(key)
- now := time.Now()
- nowUnix := now.Unix()
apiToken := &entity.APIToken{
TenantID: tenantID,
Token: key,
Beta: &beta,
}
- apiToken.CreateTime = &nowUnix
- apiToken.CreateDate = &now
// 4. Save API token
if err := s.apiTokenDAO.Create(apiToken); err != nil {
@@ -1730,8 +1682,6 @@ func (s *Service) InitDefaultAdmin() error {
}
if len(users) == 0 {
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
userID := utility.GenerateToken()
accessToken := utility.GenerateToken()
status := "1"
@@ -1758,12 +1708,6 @@ func (s *Service) InitDefaultAdmin() error {
IsAnonymous: "0",
LoginChannel: &loginChannel,
IsSuperuser: &isSuperuser,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
if err := dao.DB.Create(user).Error; err != nil {
@@ -1806,8 +1750,6 @@ func (s *Service) InitDefaultAdmin() error {
// addTenantForAdmin add tenant for admin user
func (s *Service) addTenantForAdmin(userID, nickname string) error {
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
status := "1"
role := "owner"
tenantName := nickname + "'s Kingdom"
@@ -1815,12 +1757,6 @@ func (s *Service) addTenantForAdmin(userID, nickname string) error {
tenant := &entity.Tenant{
ID: userID,
Name: &tenantName,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
if err := dao.DB.Create(tenant).Error; err != nil {
@@ -1833,12 +1769,6 @@ func (s *Service) addTenantForAdmin(userID, nickname string) error {
InvitedBy: userID,
Role: role,
Status: &status,
- BaseModel: entity.BaseModel{
- CreateTime: &now,
- CreateDate: &nowDate,
- UpdateTime: &now,
- UpdateDate: &nowDate,
- },
}
return dao.DB.Create(userTenant).Error
diff --git a/internal/dao/kb.go b/internal/dao/kb.go
index 0da2558e675..e025b5e7ce2 100644
--- a/internal/dao/kb.go
+++ b/internal/dao/kb.go
@@ -22,7 +22,6 @@ import (
"strconv"
"strings"
- "time"
)
// KnowledgebaseDAO knowledge base data access object
@@ -314,30 +313,22 @@ func splitNameCounter(name string) (string, int) {
// AtomicIncreaseDocNumByID atomically increments the document count
// This matches the Python atomic_increase_doc_num_by_id method
func (dao *KnowledgebaseDAO) AtomicIncreaseDocNumByID(kbID string) error {
- now := time.Now().Truncate(time.Second)
- updateTime := now.UnixMilli()
return DB.Model(&entity.Knowledgebase{}).
Where("id = ?", kbID).
Updates(map[string]interface{}{
"doc_num": DB.Raw("doc_num + 1"),
- "update_time": updateTime,
- "update_date": now,
}).Error
}
// DecreaseDocumentNum decreases document, chunk, and token counts
// This matches the Python decrease_document_num_in_delete method
func (dao *KnowledgebaseDAO) DecreaseDocumentNum(kbID string, docNum, chunkNum, tokenNum int64) error {
- now := time.Now().Truncate(time.Second)
- updateTime := now.UnixMilli()
return DB.Model(&entity.Knowledgebase{}).
Where("id = ?", kbID).
Updates(map[string]interface{}{
"doc_num": DB.Raw("doc_num - ?", docNum),
"chunk_num": DB.Raw("chunk_num - ?", chunkNum),
"token_num": DB.Raw("token_num - ?", tokenNum),
- "update_time": updateTime,
- "update_date": now,
}).Error
}
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index ca5bd3d06b7..23052f66476 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -345,7 +345,9 @@ func migrateSkillSearchTables(db *gorm.DB) error {
index_version VARCHAR(32) DEFAULT '1.0.0',
status VARCHAR(1) DEFAULT '1',
create_time BIGINT,
- update_time DATETIME,
+ create_date DATETIME,
+ update_time BIGINT,
+ update_date DATETIME,
INDEX idx_tenant_id (tenant_id),
INDEX idx_space_id (space_id),
UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)
@@ -367,6 +369,15 @@ func migrateSkillSearchTables(db *gorm.DB) error {
if err := addColumnIfNotExists(db, "skill_search_configs", "space_id", "VARCHAR(128) NOT NULL DEFAULT 'default'"); err != nil {
return fmt.Errorf("failed to add space_id column to skill_search_configs: %w", err)
}
+ if err := addColumnIfNotExists(db, "skill_search_configs", "create_date", "DATETIME"); err != nil {
+ return fmt.Errorf("failed to add create_date column to skill_search_configs: %w", err)
+ }
+ if err := addColumnIfNotExists(db, "skill_search_configs", "update_date", "DATETIME"); err != nil {
+ return fmt.Errorf("failed to add update_date column to skill_search_configs: %w", err)
+ }
+ if err := db.Exec(`ALTER TABLE skill_search_configs MODIFY COLUMN update_time BIGINT`).Error; err != nil {
+ common.Warn("Failed to modify skill_search_configs.update_time", zap.Error(err))
+ }
// Drop legacy unique index (tenant_id, embd_id) to allow per-space configs.
var legacyIndexExists int64
@@ -411,7 +422,9 @@ func migrateSkillSpaceTables(db *gorm.DB) error {
top_k INT DEFAULT 10,
status VARCHAR(1) DEFAULT '1',
create_time BIGINT,
- update_time DATETIME,
+ create_date DATETIME,
+ update_time BIGINT,
+ update_date DATETIME,
INDEX idx_tenant_id (tenant_id),
UNIQUE INDEX idx_tenant_name_status (tenant_id, name, status)
)
@@ -433,6 +446,15 @@ func migrateSkillSpaceTables(db *gorm.DB) error {
if err := addColumnIfNotExists(db, "skill_spaces", "status", "VARCHAR(1) NOT NULL DEFAULT '1'"); err != nil {
return fmt.Errorf("failed to add status column to skill_spaces: %w", err)
}
+ if err := addColumnIfNotExists(db, "skill_spaces", "create_date", "DATETIME"); err != nil {
+ return fmt.Errorf("failed to add create_date column to skill_spaces: %w", err)
+ }
+ if err := addColumnIfNotExists(db, "skill_spaces", "update_date", "DATETIME"); err != nil {
+ return fmt.Errorf("failed to add update_date column to skill_spaces: %w", err)
+ }
+ if err := db.Exec(`ALTER TABLE skill_spaces MODIFY COLUMN update_time BIGINT`).Error; err != nil {
+ common.Warn("Failed to modify skill_spaces.update_time", zap.Error(err))
+ }
// Migrate index after status column exists
if err := migrateSkillSpaceIndex(db); err != nil {
return fmt.Errorf("failed to migrate skill_space index: %w", err)
diff --git a/internal/dao/skill_search_config.go b/internal/dao/skill_search_config.go
index 6c19964bc21..16c01d56efe 100644
--- a/internal/dao/skill_search_config.go
+++ b/internal/dao/skill_search_config.go
@@ -19,7 +19,6 @@ package dao
import (
"ragflow/internal/entity"
"strings"
- "time"
"github.com/google/uuid"
)
@@ -109,7 +108,6 @@ func (dao *SkillSearchConfigDAO) GetOrCreate(tenantID, spaceID, embdID string) (
// CreateWithTenantSpace creates a new config for tenant+space
func (dao *SkillSearchConfigDAO) CreateWithTenantSpace(tenantID, spaceID, embdID string) (*entity.SkillSearchConfig, error) {
spaceID = normalizeSpaceID(spaceID)
- timestamp := time.Now().UnixMilli()
defaultFieldConfig := entity.DefaultFieldConfig()
fieldConfigMap := entity.JSONMap{
"name": map[string]interface{}{
@@ -140,7 +138,6 @@ func (dao *SkillSearchConfigDAO) CreateWithTenantSpace(tenantID, spaceID, embdID
FieldConfig: fieldConfigMap,
TopK: 10,
Status: "1",
- CreateTime: ×tamp,
}
if err := dao.Create(defaultConfig); err != nil {
@@ -167,20 +164,17 @@ func (dao *SkillSearchConfigDAO) DeleteAllByTenantSpaceExceptID(tenantID, spaceI
// Update updates a skill search config with the given updates map
func (dao *SkillSearchConfigDAO) Update(id string, updates map[string]interface{}) error {
- updates["update_time"] = time.Now()
return DB.Model(&entity.SkillSearchConfig{}).Where("id = ? AND status = ?", id, "1").Updates(updates).Error
}
// UpdateByTenantID updates config by tenant ID
func (dao *SkillSearchConfigDAO) UpdateByTenantID(tenantID, spaceID string, updates map[string]interface{}) error {
- updates["update_time"] = time.Now()
result := DB.Model(&entity.SkillSearchConfig{}).Where("tenant_id = ? AND space_id = ? AND status = ?", tenantID, normalizeSpaceID(spaceID), "1").Updates(updates)
return result.Error
}
// UpdateByTenantAndEmbdID updates config by tenant ID and embedding ID
func (dao *SkillSearchConfigDAO) UpdateByTenantAndEmbdID(tenantID, spaceID, embdID string, updates map[string]interface{}) error {
- updates["update_time"] = time.Now()
result := DB.Model(&entity.SkillSearchConfig{}).Where("tenant_id = ? AND space_id = ? AND embd_id = ? AND status = ?", tenantID, normalizeSpaceID(spaceID), embdID, "1").Updates(updates)
return result.Error
}
diff --git a/internal/dao/skill_space.go b/internal/dao/skill_space.go
index 2c0596f8a33..c8557521fee 100644
--- a/internal/dao/skill_space.go
+++ b/internal/dao/skill_space.go
@@ -19,7 +19,6 @@ package dao
import (
"ragflow/internal/entity"
"strings"
- "time"
"github.com/google/uuid"
)
@@ -101,7 +100,6 @@ func (dao *SkillSpaceDAO) Update(space *entity.SkillSpace) error {
// UpdateByID updates skills space by ID
func (dao *SkillSpaceDAO) UpdateByID(id string, updates map[string]interface{}) error {
- updates["update_time"] = time.Now()
return DB.Model(&entity.SkillSpace{}).Where("id = ?", id).Updates(updates).Error
}
diff --git a/internal/dao/system_settings.go b/internal/dao/system_settings.go
index 2e200ac0491..c224ad0fcde 100644
--- a/internal/dao/system_settings.go
+++ b/internal/dao/system_settings.go
@@ -19,7 +19,6 @@ package dao
import (
"errors"
"ragflow/internal/entity"
- "time"
"gorm.io/gorm"
)
@@ -57,31 +56,18 @@ func (d *SystemSettingsDAO) GetByName(name string) ([]entity.SystemSettings, err
// UpdateByName update system settings by name
// Updates the setting with the given name using the provided data
func (d *SystemSettingsDAO) UpdateByName(name string, setting *entity.SystemSettings) error {
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
-
return DB.Model(&entity.SystemSettings{}).
Where("name = ?", name).
Updates(map[string]interface{}{
- "value": setting.Value,
- "source": setting.Source,
- "data_type": setting.DataType,
- "update_time": now,
- "update_date": nowDate,
+ "value": setting.Value,
+ "source": setting.Source,
+ "data_type": setting.DataType,
}).Error
}
// Create create a new system setting
// Inserts a new system setting record into database
func (d *SystemSettingsDAO) Create(setting *entity.SystemSettings) error {
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
-
- setting.CreateTime = &now
- setting.CreateDate = &nowDate
- setting.UpdateTime = &now
- setting.UpdateDate = &nowDate
-
return DB.Create(setting).Error
}
@@ -96,6 +82,8 @@ func (d *SystemSettingsDAO) SaveOrCreate(name string, value string, source strin
if len(settings) == 1 {
setting := &settings[0]
setting.Value = value
+ setting.Source = source
+ setting.DataType = dataType
return d.UpdateByName(name, setting)
} else if len(settings) > 1 {
return errors.New("can't update more than 1 setting: " + name)
@@ -159,29 +147,16 @@ func (d *SystemSettingsDAO) Transaction(fn func(tx *gorm.DB) error) error {
// CreateWithTx create setting within transaction
func (d *SystemSettingsDAO) CreateWithTx(tx *gorm.DB, setting *entity.SystemSettings) error {
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
-
- setting.CreateTime = &now
- setting.CreateDate = &nowDate
- setting.UpdateTime = &now
- setting.UpdateDate = &nowDate
-
return tx.Create(setting).Error
}
// UpdateByNameWithTx update setting within transaction
func (d *SystemSettingsDAO) UpdateByNameWithTx(tx *gorm.DB, name string, setting *entity.SystemSettings) error {
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
-
return tx.Model(&entity.SystemSettings{}).
Where("name = ?", name).
Updates(map[string]interface{}{
- "value": setting.Value,
- "source": setting.Source,
- "data_type": setting.DataType,
- "update_time": now,
- "update_date": nowDate,
+ "value": setting.Value,
+ "source": setting.Source,
+ "data_type": setting.DataType,
}).Error
}
diff --git a/internal/entity/base.go b/internal/entity/base.go
index 748fea87132..774be466f3d 100644
--- a/internal/entity/base.go
+++ b/internal/entity/base.go
@@ -20,6 +20,8 @@ import (
"database/sql/driver"
"encoding/json"
"time"
+
+ "gorm.io/gorm"
)
// BaseModel base model
@@ -31,6 +33,92 @@ type BaseModel struct {
UpdateDate *time.Time `gorm:"column:update_date;index" json:"update_date,omitempty"`
}
+func autoModelTime() (int64, time.Time) {
+ now := time.Now().Local()
+ return now.UnixMilli(), now.Truncate(time.Second)
+}
+
+func statementHasTimeField(tx *gorm.DB, fieldNames ...string) bool {
+ if tx == nil || tx.Statement == nil {
+ return false
+ }
+
+ switch dest := tx.Statement.Dest.(type) {
+ case map[string]interface{}:
+ for _, fieldName := range fieldNames {
+ if _, ok := dest[fieldName]; ok {
+ return true
+ }
+ }
+ case []map[string]interface{}:
+ for _, item := range dest {
+ for _, fieldName := range fieldNames {
+ if _, ok := item[fieldName]; ok {
+ return true
+ }
+ }
+ }
+ }
+
+ return false
+}
+
+// BeforeCreate injects timestamps for models embedding BaseModel.
+func (m *BaseModel) BeforeCreate(tx *gorm.DB) error {
+ timestamp, dateTime := autoModelTime()
+
+ if m.CreateTime == nil {
+ m.CreateTime = ×tamp
+ }
+ if m.CreateDate == nil {
+ m.CreateDate = &dateTime
+ }
+ if m.UpdateTime == nil {
+ m.UpdateTime = ×tamp
+ }
+ if m.UpdateDate == nil {
+ m.UpdateDate = &dateTime
+ }
+
+ if tx != nil && tx.Statement != nil {
+ if !statementHasTimeField(tx, "create_time", "CreateTime") && m.CreateTime != nil {
+ tx.Statement.SetColumn("CreateTime", *m.CreateTime)
+ }
+ if !statementHasTimeField(tx, "create_date", "CreateDate") && m.CreateDate != nil {
+ tx.Statement.SetColumn("CreateDate", *m.CreateDate)
+ }
+ if !statementHasTimeField(tx, "update_time", "UpdateTime") && m.UpdateTime != nil {
+ tx.Statement.SetColumn("UpdateTime", *m.UpdateTime)
+ }
+ if !statementHasTimeField(tx, "update_date", "UpdateDate") && m.UpdateDate != nil {
+ tx.Statement.SetColumn("UpdateDate", *m.UpdateDate)
+ }
+ }
+ return nil
+}
+
+// BeforeUpdate injects update timestamps for models embedding BaseModel.
+func (m *BaseModel) BeforeUpdate(tx *gorm.DB) error {
+ timestamp, dateTime := autoModelTime()
+
+ if !statementHasTimeField(tx, "update_time", "UpdateTime") {
+ m.UpdateTime = ×tamp
+ }
+ if !statementHasTimeField(tx, "update_date", "UpdateDate") {
+ m.UpdateDate = &dateTime
+ }
+
+ if tx != nil && tx.Statement != nil {
+ if !statementHasTimeField(tx, "update_time", "UpdateTime") && m.UpdateTime != nil {
+ tx.Statement.SetColumn("UpdateTime", *m.UpdateTime)
+ }
+ if !statementHasTimeField(tx, "update_date", "UpdateDate") && m.UpdateDate != nil {
+ tx.Statement.SetColumn("UpdateDate", *m.UpdateDate)
+ }
+ }
+ return nil
+}
+
// JSONMap is a map type that can store JSON data
type JSONMap map[string]interface{}
diff --git a/internal/entity/skill_search.go b/internal/entity/skill_search.go
index 3a31dfb486e..011499bbcbb 100644
--- a/internal/entity/skill_search.go
+++ b/internal/entity/skill_search.go
@@ -56,8 +56,7 @@ type SkillSearchConfig struct {
TenantRerankID *int64 `gorm:"column:tenant_rerank_id" json:"tenant_rerank_id,omitempty"`
TopK int64 `gorm:"column:top_k;default:10" json:"top_k"`
IndexVersion string `gorm:"column:index_version;size:32;default:'1.0.0'" json:"index_version"`
- CreateTime *int64 `gorm:"column:create_time" json:"create_time,omitempty"`
- UpdateTime *time.Time `gorm:"column:update_time" json:"update_time,omitempty"`
+ BaseModel
}
// TableName returns the table name for SkillSearchConfig model
@@ -90,7 +89,7 @@ func (s *SkillSearchConfig) ToMap() map[string]interface{} {
result["create_time"] = s.CreateTime
}
if s.UpdateTime != nil {
- result["update_time"] = s.UpdateTime.Format("2006-01-02 15:04:05")
+ result["update_time"] = time.UnixMilli(*s.UpdateTime).Format("2006-01-02 15:04:05")
}
return result
diff --git a/internal/entity/skill_space.go b/internal/entity/skill_space.go
index 0e90a398171..1df53a9197b 100644
--- a/internal/entity/skill_space.go
+++ b/internal/entity/skill_space.go
@@ -36,8 +36,7 @@ type SkillSpace struct {
RerankID string `gorm:"column:rerank_id;size:128" json:"rerank_id"`
TopK int `gorm:"column:top_k;default:10" json:"top_k"`
Status string `gorm:"column:status;size:1;default:1" json:"status"`
- CreateTime *int64 `gorm:"column:create_time" json:"create_time,omitempty"`
- UpdateTime *time.Time `gorm:"column:update_time" json:"update_time,omitempty"`
+ BaseModel
}
// TableName returns the table name for SkillSpace model
@@ -83,7 +82,7 @@ func (s *SkillSpace) ToMap() map[string]interface{} {
result["create_time"] = s.CreateTime
}
if s.UpdateTime != nil {
- result["update_time"] = s.UpdateTime.Format("2006-01-02 15:04:05")
+ result["update_time"] = time.UnixMilli(*s.UpdateTime).Format("2006-01-02 15:04:05")
}
return result
diff --git a/internal/entity/system.go b/internal/entity/system.go
index 831bb7397f9..49541ec3ef8 100644
--- a/internal/entity/system.go
+++ b/internal/entity/system.go
@@ -16,18 +16,13 @@
package entity
-import "time"
-
// SystemSettings system settings model
type SystemSettings struct {
- Name string `gorm:"column:name;primaryKey;size:128" json:"name"`
- Source string `gorm:"column:source;size:32;not null" json:"source"`
- DataType string `gorm:"column:data_type;size:32;not null" json:"data_type"`
- Value string `gorm:"column:value;type:longtext;not null" json:"value"`
- CreateTime *int64 `gorm:"column:create_time" json:"create_time"`
- CreateDate *time.Time `gorm:"column:create_date" json:"create_date"`
- UpdateTime *int64 `gorm:"column:update_time" json:"update_time"`
- UpdateDate *time.Time `gorm:"column:update_date" json:"update_date"`
+ Name string `gorm:"column:name;primaryKey;size:128" json:"name"`
+ Source string `gorm:"column:source;size:32;not null" json:"source"`
+ DataType string `gorm:"column:data_type;size:32;not null" json:"data_type"`
+ Value string `gorm:"column:value;type:longtext;not null" json:"value"`
+ BaseModel
}
// TableName specify table name
diff --git a/internal/service/api_token.go b/internal/service/api_token.go
index 9f44d740199..667610ae1e5 100644
--- a/internal/service/api_token.go
+++ b/internal/service/api_token.go
@@ -20,7 +20,6 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/utility"
- "time"
)
// TokenResponse token response
@@ -67,9 +66,6 @@ type CreateAPITokenRequest struct {
func (s *SystemService) CreateAPIToken(tenantID string, req *CreateAPITokenRequest) (*TokenResponse, error) {
APITokenDAO := dao.NewAPITokenDAO()
- now := time.Now().Unix()
- nowDate := time.Now()
-
// Generate token and beta values
// token: "ragflow-" + secrets.token_urlsafe(32)
APIToken := utility.GenerateAPIToken()
@@ -81,8 +77,6 @@ func (s *SystemService) CreateAPIToken(tenantID string, req *CreateAPITokenReque
Token: APIToken,
Beta: &betaAPIKey,
}
- APITokenData.CreateDate = &nowDate
- APITokenData.CreateTime = &now
if err := APITokenDAO.Create(APITokenData); err != nil {
return nil, err
diff --git a/internal/service/chat.go b/internal/service/chat.go
index f386d727997..060bd3cc566 100644
--- a/internal/service/chat.go
+++ b/internal/service/chat.go
@@ -21,7 +21,6 @@ import (
"fmt"
"ragflow/internal/entity"
"strings"
- "time"
"unicode/utf8"
"github.com/google/uuid"
@@ -453,10 +452,6 @@ func (s *ChatService) SetDialog(userID string, req *SetDialogRequest) (*SetDialo
newID = newID[:32]
}
- // Get current time
- now := time.Now().Truncate(time.Second)
- createTime := now.UnixMilli()
-
// Set default language
language := "English"
@@ -480,10 +475,6 @@ func (s *ChatService) SetDialog(userID string, req *SetDialogRequest) (*SetDialo
KBIDs: kbIDsJSON,
Status: strPtr("1"),
}
- chat.CreateTime = &createTime
- chat.CreateDate = &now
- chat.UpdateTime = &createTime
- chat.UpdateDate = &now
if err := s.chatDAO.Create(chat); err != nil {
return nil, errors.New("Fail to new a chat")
@@ -498,9 +489,6 @@ func (s *ChatService) SetDialog(userID string, req *SetDialogRequest) (*SetDialo
}, nil
}
- // Update existing chat - also update update_time
- now := time.Now().Truncate(time.Second)
- updateTime := now.UnixMilli()
updateData := map[string]interface{}{
"name": name,
"description": description,
@@ -515,8 +503,6 @@ func (s *ChatService) SetDialog(userID string, req *SetDialogRequest) (*SetDialo
"similarity_threshold": similarityThreshold,
"vector_similarity_weight": vectorSimilarityWeight,
"kb_ids": kbIDsJSON,
- "update_time": updateTime,
- "update_date": now,
}
if err := s.chatDAO.UpdateByID(req.DialogID, updateData); err != nil {
diff --git a/internal/service/chat_session.go b/internal/service/chat_session.go
index 206b6e76b43..50402f9d7c2 100644
--- a/internal/service/chat_session.go
+++ b/internal/service/chat_session.go
@@ -79,8 +79,6 @@ func (s *ChatSessionService) SetChatSession(userID string, req *SetChatSessionRe
updates := map[string]interface{}{
"name": name,
"user_id": userID,
- "update_time": time.Now().UnixMilli(),
- "update_date": time.Now(),
}
if err := s.chatSessionDAO.UpdateByID(req.SessionID, updates); err != nil {
@@ -118,9 +116,6 @@ func (s *ChatSessionService) SetChatSession(userID string, req *SetChatSessionRe
}
}
- now := time.Now().Truncate(time.Second)
- createTime := time.Now().UnixMilli()
-
// Create initial message - store as JSON object with messages array
messagesObj := map[string]interface{}{
"messages": []map[string]interface{}{
@@ -144,10 +139,6 @@ func (s *ChatSessionService) SetChatSession(userID string, req *SetChatSessionRe
UserID: &userID,
Reference: referenceJSON,
}
- session.CreateTime = &createTime
- session.CreateDate = &now
- session.UpdateTime = &createTime
- session.UpdateDate = &now
if err := s.chatSessionDAO.Create(session); err != nil {
return nil, errors.New("Fail to create a chat session")
@@ -459,8 +450,6 @@ func (s *ChatSessionService) updateSessionMessages(session *entity.ChatSession,
updates := map[string]interface{}{
"message": messagesJSON,
"reference": referenceJSON,
- "update_time": time.Now().UnixMilli(),
- "update_date": time.Now(),
}
s.chatSessionDAO.UpdateByID(session.ID, updates)
}
diff --git a/internal/service/datasets.go b/internal/service/datasets.go
index db1320e6ebe..c163f891e45 100644
--- a/internal/service/datasets.go
+++ b/internal/service/datasets.go
@@ -22,7 +22,6 @@ import (
"fmt"
"ragflow/internal/entity"
"strings"
- "time"
"github.com/google/uuid"
"gorm.io/gorm"
@@ -396,8 +395,6 @@ func (s *DatasetsService) CreateDataset(req *CreateDatasetRequest, tenantID stri
return nil, common.CodeServerError, errors.New("Internal server error")
}
- now := time.Now().Truncate(time.Second)
- createTime := now.UnixMilli()
status := string(entity.StatusValid)
// Deduplicate name within tenant
duplicateName, err := common.DuplicateName(func(n, tid string) bool {
@@ -420,10 +417,6 @@ func (s *DatasetsService) CreateDataset(req *CreateDatasetRequest, tenantID stri
EmbdID: embdID,
Status: &status,
}
- kb.CreateTime = &createTime
- kb.UpdateTime = &createTime
- kb.CreateDate = &now
- kb.UpdateDate = &now
if description != nil {
kb.Description = description
diff --git a/internal/service/document.go b/internal/service/document.go
index 9c1fa0a2912..d625bef484a 100644
--- a/internal/service/document.go
+++ b/internal/service/document.go
@@ -223,7 +223,12 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
}
updatedAt := ""
if doc.UpdateTime != nil {
- updatedAt = time.Unix(*doc.UpdateTime, 0).Format("2006-01-02 15:04:05")
+ // Accept both historical second-based values and current millisecond-based values.
+ ts := *doc.UpdateTime
+ if ts > 1000000000000 {
+ ts /= 1000
+ }
+ updatedAt = time.Unix(ts, 0).Format("2006-01-02 15:04:05")
}
return &DocumentResponse{
ID: doc.ID,
diff --git a/internal/service/kb.go b/internal/service/kb.go
index 75916413b60..97776992587 100644
--- a/internal/service/kb.go
+++ b/internal/service/kb.go
@@ -27,7 +27,6 @@ import (
"ragflow/internal/utility"
"strings"
- "time"
)
// KnowledgebaseService service class for managing dataset operations
@@ -213,11 +212,6 @@ func (s *KnowledgebaseService) UpdateKB(req *UpdateKBRequest, userID string) (ma
updates["parser_config"] = req.ParserConfig
}
- now := time.Now().Truncate(time.Second)
- updateTime := now.UnixMilli()
- updates["update_time"] = updateTime
- updates["update_date"] = now
-
// Update in database
if err := s.kbDAO.UpdateByID(req.KBID, updates); err != nil {
return nil, common.CodeServerError, fmt.Errorf("failed to update knowledge base: %w", err)
diff --git a/internal/service/memory.go b/internal/service/memory.go
index 2ab7272b087..882282f4ccb 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -370,8 +370,6 @@ func (s *MemoryService) CreateMemory(tenantID string, req *CreateMemoryRequest)
}
memoryTypeInt := dao.CalculateMemoryType(uniqueMemoryTypes)
- timestamp := time.Now().UnixMilli()
-
systemPrompt := PromptAssembler{}.AssembleSystemPrompt(uniqueMemoryTypes)
newID := common.GenerateUUID()
@@ -402,9 +400,6 @@ func (s *MemoryService) CreateMemory(tenantID string, req *CreateMemoryRequest)
memory.TenantLLMID = &llmID
}
}
- memory.CreateTime = ×tamp
- memory.UpdateTime = ×tamp
-
if err := s.memoryDAO.Create(memory); err != nil {
return nil, errors.New("could not create new memory")
}
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index bd23026d720..f23f962109b 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -25,7 +25,6 @@ import (
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"strings"
- "time"
)
// parseModelName parses a composite model name in format "model@instance@provider" or "model@provider"
@@ -88,17 +87,11 @@ func (m *ModelProviderService) AddModelProvider(providerName, userID string) (co
return common.CodeServerError, errors.New("fail to get UUID")
}
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
tenantModelProvider := &entity.TenantModelProvider{
ID: providerID,
ProviderName: providerName,
TenantID: tenantID,
}
- tenantModelProvider.CreateTime = &now
- tenantModelProvider.UpdateTime = &now
- tenantModelProvider.CreateDate = &nowDate
- tenantModelProvider.UpdateDate = &nowDate
err = m.modelProviderDAO.Create(tenantModelProvider)
if err != nil {
return common.CodeServerError, fmt.Errorf("fail to create model provider: %s", err.Error())
@@ -247,8 +240,6 @@ func (m *ModelProviderService) CreateProviderInstance(providerName, instanceName
}
extraStr := string(extraByte)
- now := time.Now().Unix()
- nowDate := time.Now().Truncate(time.Second)
tenantModelProvider := &entity.TenantModelInstance{
ID: instanceID,
InstanceName: instanceName,
@@ -257,10 +248,6 @@ func (m *ModelProviderService) CreateProviderInstance(providerName, instanceName
Status: "enable",
Extra: extraStr,
}
- tenantModelProvider.CreateTime = &now
- tenantModelProvider.UpdateTime = &now
- tenantModelProvider.CreateDate = &nowDate
- tenantModelProvider.UpdateDate = &nowDate
err = m.modelInstanceDAO.Create(tenantModelProvider)
if err != nil {
diff --git a/internal/service/skill_space.go b/internal/service/skill_space.go
index e40907fec4e..99a89f32550 100644
--- a/internal/service/skill_space.go
+++ b/internal/service/skill_space.go
@@ -126,8 +126,6 @@ func (s *SkillSpaceService) getSkillsFolderID(tenantID string) (string, error) {
// Skills folder not found, create it
common.Info("Creating skills folder", zap.String("tenant_id", tenantID))
folderID := generateSpaceID()
- now := time.Now()
- createTime := now.UnixMilli()
folder := &entity.File{
ID: folderID,
ParentID: rootFolder.ID,
@@ -137,12 +135,6 @@ func (s *SkillSpaceService) getSkillsFolderID(tenantID string) (string, error) {
Type: "folder",
Size: 0,
SourceType: "system",
- BaseModel: entity.BaseModel{
- CreateTime: &createTime,
- UpdateTime: &createTime,
- CreateDate: &now,
- UpdateDate: &now,
- },
}
if err := s.fileDAO.Create(folder); err != nil {
@@ -218,8 +210,6 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int
// Generate space ID and folder ID
spaceID := generateSpaceID()
folderID := generateSpaceID()
- timestamp := time.Now().UnixMilli()
- now := time.Now()
// Create folder for the space under skills folder
folder := &entity.File{
@@ -249,8 +239,6 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int
RerankID: req.RerankID,
TopK: 10,
Status: "1",
- CreateTime: ×tamp,
- UpdateTime: &now,
}
if err := s.spaceDAO.Create(space); err != nil {
diff --git a/internal/service/user.go b/internal/service/user.go
index 6b117697c4d..3503be6e6e5 100644
--- a/internal/service/user.go
+++ b/internal/service/user.go
@@ -150,14 +150,6 @@ func (s *UserService) Register(req *RegisterRequest) (*entity.User, common.Error
IsSuperuser: &isSuperuser,
}
- now := time.Now().Unix()
- user.CreateTime = &now
- user.UpdateTime = &now
- nowDate := time.Now().Truncate(time.Second)
- user.CreateDate = &nowDate
- user.UpdateDate = &nowDate
- user.LastLoginTime = &nowDate
-
tenantName := req.Nickname + "'s Kingdom"
llmID := cfg.UserDefaultLLM.DefaultModels.ChatModel.Name
@@ -192,11 +184,6 @@ func (s *UserService) Register(req *RegisterRequest) (*entity.User, common.Error
ParserIDs: "naive:General,Q&A:Q&A,manual:Manual,table:Table,paper:Research Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag",
Status: &status,
}
- tenant.CreateTime = &now
- tenant.UpdateTime = &now
- tenant.CreateDate = &nowDate
- tenant.UpdateDate = &nowDate
-
userTenantID := utility.GenerateToken()
userTenant := &entity.UserTenant{
ID: userTenantID,
@@ -206,11 +193,6 @@ func (s *UserService) Register(req *RegisterRequest) (*entity.User, common.Error
InvitedBy: userID,
Status: &status,
}
- userTenant.CreateTime = &now
- userTenant.UpdateTime = &now
- userTenant.CreateDate = &nowDate
- userTenant.UpdateDate = &nowDate
-
fileID := utility.GenerateToken()
rootFile := &entity.File{
ID: fileID,
@@ -221,11 +203,6 @@ func (s *UserService) Register(req *RegisterRequest) (*entity.User, common.Error
Type: "folder",
Size: 0,
}
- rootFile.CreateTime = &now
- rootFile.UpdateTime = &now
- rootFile.CreateDate = &nowDate
- rootFile.UpdateDate = &nowDate
-
tenantDAO := dao.NewTenantDAO()
userTenantDAO := dao.NewUserTenantDAO()
fileDAO := dao.NewFileDAO()
@@ -302,9 +279,6 @@ func (s *UserService) Login(req *LoginRequest) (*entity.User, common.ErrorCode,
return nil, common.CodeServerError, fmt.Errorf("failed to update access token: %w", err)
}
- // Update timestamp
- now := time.Now().Unix()
- user.UpdateTime = &now
if err := s.userDAO.Update(user); err != nil {
return nil, common.CodeServerError, fmt.Errorf("failed to update user: %w", err)
}
@@ -340,10 +314,6 @@ func (s *UserService) LoginByEmail(req *EmailLoginRequest) (*entity.User, common
token := utility.GenerateToken()
user.AccessToken = &token
- now := time.Now().Unix()
- user.UpdateTime = &now
- now_date := time.Now().Truncate(time.Second)
- user.UpdateDate = &now_date
if err := s.userDAO.Update(user); err != nil {
return nil, common.CodeServerError, fmt.Errorf("failed to update user: %w", err)
}
From 714f777fa0418b5fd800f254b0c72589082f6c88 Mon Sep 17 00:00:00 2001
From: dale053
Date: Wed, 13 May 2026 22:48:41 -0700
Subject: [PATCH 142/666] Fix: missing authentication on agent file upload and
download endpoints (#14854)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Closes #14853
The `/agents/download` and `/agents//upload` endpoints in the
agent API are missing `@login_required` and `@add_tenant_id_to_kwargs`
decorators, allowing unauthenticated access. This is a security issue —
any user can upload files to or download files from an agent without
being logged in. Additionally, the upload endpoint bypasses canvas
access control (`@_require_canvas_access_async`).
This PR adds the missing authentication and authorization decorators to
both endpoints and replaces the manual `user_id` / `created_by` lookups
with the `tenant_id` provided by the auth middleware, making these
endpoints consistent with the rest of the agent API.
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/restful_apis/agent_api.py | 38 ++++--
.../test_session_sdk_routes_unit.py | 119 +++++++++++++++++-
2 files changed, 143 insertions(+), 14 deletions(-)
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index 09f284e8953..0d545e7649f 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -245,10 +245,12 @@ def delete_agent_session_item(agent_id, session_id, tenant_id):
@manager.route("/agents/download", methods=["GET"]) # noqa: F821
-async def download_agent_file():
+@login_required
+@add_tenant_id_to_kwargs
+async def download_agent_file(tenant_id):
id = request.args.get("id")
- created_by = request.args.get("created_by")
- blob = FileService.get_blob(created_by, id)
+ logging.info("Agent file download requested: tenant_id=%s file_id=%s", tenant_id, id)
+ blob = await thread_pool_exec(FileService.get_blob, tenant_id, id)
return Response(blob)
@@ -482,22 +484,34 @@ async def create_agent(tenant_id):
@manager.route("/agents//upload", methods=["POST"]) # noqa: F821
-async def upload_agent_file(agent_id):
- exists, canvas = UserCanvasService.get_by_canvas_id(agent_id)
- if not exists:
- return get_data_error_result(message="canvas not found.")
-
- user_id = canvas["user_id"]
+@login_required
+@add_tenant_id_to_kwargs
+@_require_canvas_access_async
+async def upload_agent_file(agent_id, tenant_id):
files = await request.files
file_objs = files.getlist("file") if files and files.get("file") else []
+ logging.info(
+ "Agent file upload requested: tenant_id=%s agent_id=%s file_count=%s",
+ tenant_id,
+ agent_id,
+ len(file_objs),
+ )
try:
if len(file_objs) == 1:
- return get_json_result(
- data=FileService.upload_info(user_id, file_objs[0], request.args.get("url"))
+ uploaded = await thread_pool_exec(
+ FileService.upload_info, tenant_id, file_objs[0], request.args.get("url")
)
- results = [FileService.upload_info(user_id, file_obj) for file_obj in file_objs]
+ return get_json_result(data=uploaded)
+ results = await asyncio.gather(
+ *(thread_pool_exec(FileService.upload_info, tenant_id, file_obj) for file_obj in file_objs)
+ )
return get_json_result(data=results)
except Exception as exc:
+ logging.exception(
+ "Agent file upload failed: tenant_id=%s agent_id=%s",
+ tenant_id,
+ agent_id,
+ )
return server_error_response(exc)
diff --git a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py
index 773660fdd48..7a990564552 100644
--- a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py
+++ b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py
@@ -770,7 +770,10 @@ def commit_after_run(**_kwargs):
monkeypatch.setitem(sys.modules, "api.apps.services.canvas_replica_service", canvas_replica_mod)
file_service_mod = ModuleType("api.db.services.file_service")
- file_service_mod.FileService = SimpleNamespace(upload_info=lambda *_args, **_kwargs: {})
+ file_service_mod.FileService = SimpleNamespace(
+ upload_info=lambda *_args, **_kwargs: {},
+ get_blob=lambda *_args, **_kwargs: b"",
+ )
monkeypatch.setitem(sys.modules, "api.db.services.file_service", file_service_mod)
api_service_mod = ModuleType("api.db.services.api_service")
@@ -1213,7 +1216,119 @@ async def _agent_nonstream(*_args, **_kwargs):
"c4": {},
}
assert [item["component_id"] for item in res["data"]["data"]["trace"]] == ["c2", "c3", "c4"]
-
+
+
+class _FakeUploadFileField:
+ def __init__(self, filename: str):
+ self.filename = filename
+
+
+class _FakeRequestFiles:
+ def __init__(self, filenames: list[str]):
+ self._filenames = filenames
+
+ def get(self, key, default=None):
+ if key == "file" and self._filenames:
+ return _FakeUploadFileField(self._filenames[0])
+ return default
+
+ def getlist(self, key):
+ if key == "file":
+ return [_FakeUploadFileField(n) for n in self._filenames]
+ return []
+
+
+@pytest.mark.p2
+def test_agent_file_download_and_upload_unit(monkeypatch):
+ module = _load_agent_api_module(monkeypatch)
+ monkeypatch.setattr(module, "Response", _StubResponse)
+
+ get_blob_calls = []
+
+ def _get_blob(tenant_id, file_id):
+ get_blob_calls.append((tenant_id, file_id))
+ return b"file-bytes"
+
+ monkeypatch.setattr(module.FileService, "get_blob", _get_blob)
+ monkeypatch.setattr(module, "request", SimpleNamespace(args=_Args({"id": "doc-99"})))
+
+ resp = _run(inspect.unwrap(module.download_agent_file)("tenant-1"))
+ assert isinstance(resp, _StubResponse)
+ assert resp.body == b"file-bytes"
+ assert get_blob_calls == [("tenant-1", "doc-99")]
+
+ upload_calls = []
+
+ def _upload_info(tenant_id, file_obj, url=None):
+ upload_calls.append((tenant_id, getattr(file_obj, "filename", None), url))
+ return {"id": tenant_id, "file": getattr(file_obj, "filename", None), "url": url}
+
+ monkeypatch.setattr(module.FileService, "upload_info", _upload_info)
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=_Args({"url": "https://example.com/a.png"}),
+ files=_AwaitableValue(_FakeRequestFiles(["one.png"])),
+ ),
+ )
+ res = _run(
+ inspect.unwrap(module.upload_agent_file)(
+ agent_id="agent-1",
+ tenant_id="tenant-1",
+ )
+ )
+ assert res["code"] == 0
+ assert res["data"]["file"] == "one.png"
+ assert upload_calls == [("tenant-1", "one.png", "https://example.com/a.png")]
+
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=_Args({}),
+ files=_AwaitableValue(_FakeRequestFiles(["a.png", "b.png"])),
+ ),
+ )
+ upload_calls.clear()
+ res = _run(
+ inspect.unwrap(module.upload_agent_file)(
+ agent_id="agent-1",
+ tenant_id="tenant-1",
+ )
+ )
+ assert res["code"] == 0
+ assert len(res["data"]) == 2
+ assert set(upload_calls) == {
+ ("tenant-1", "a.png", None),
+ ("tenant-1", "b.png", None),
+ }
+
+ def _boom(*_a, **_k):
+ raise ValueError("upload failed")
+
+ monkeypatch.setattr(module.FileService, "upload_info", _boom)
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=_Args({}),
+ files=_AwaitableValue(_FakeRequestFiles(["bad.png"])),
+ ),
+ )
+ res = _run(
+ inspect.unwrap(module.upload_agent_file)(
+ agent_id="agent-1",
+ tenant_id="tenant-1",
+ )
+ )
+ assert res["code"] != 0
+
+ monkeypatch.setattr(module.UserCanvasService, "accessible", lambda *_a, **_k: False)
+ res = _run(module.upload_agent_file(agent_id="agent-1"))
+ assert res["code"] == module.RetCode.OPERATING_ERROR
+ assert "permission" in res["message"].lower()
+
@pytest.mark.p2
def test_delete_routes_partial_duplicate_unit(monkeypatch):
From 0450400efd7325f6a26cce058121a6120209a32f Mon Sep 17 00:00:00 2001
From: buua436
Date: Thu, 14 May 2026 14:46:39 +0800
Subject: [PATCH 143/666] Go: fix LastLoginTime update (#14917)
### What problem does this PR solve?
fix LastLoginTime update
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
internal/service/user.go | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/internal/service/user.go b/internal/service/user.go
index 3503be6e6e5..3147ae69c57 100644
--- a/internal/service/user.go
+++ b/internal/service/user.go
@@ -275,10 +275,9 @@ func (s *UserService) Login(req *LoginRequest) (*entity.User, common.ErrorCode,
// Generate new access token
token := utility.GenerateToken()
- if err := s.UpdateUserAccessToken(user, token); err != nil {
- return nil, common.CodeServerError, fmt.Errorf("failed to update access token: %w", err)
- }
-
+ user.AccessToken = &token
+ now := time.Now().Truncate(time.Second)
+ user.LastLoginTime = &now
if err := s.userDAO.Update(user); err != nil {
return nil, common.CodeServerError, fmt.Errorf("failed to update user: %w", err)
}
@@ -313,6 +312,8 @@ func (s *UserService) LoginByEmail(req *EmailLoginRequest) (*entity.User, common
// Generate new access token
token := utility.GenerateToken()
user.AccessToken = &token
+ now := time.Now().Truncate(time.Second)
+ user.LastLoginTime = &now
if err := s.userDAO.Update(user); err != nil {
return nil, common.CodeServerError, fmt.Errorf("failed to update user: %w", err)
From ba8cb9dd4ace50e6aed22bf5732e397eaca2ce90 Mon Sep 17 00:00:00 2001
From: "Ethan T."
Date: Thu, 14 May 2026 14:46:47 +0800
Subject: [PATCH 144/666] fix: replace mutable default arguments with None in
LLM chat models (#13513)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
- Replace `gen_conf={}` with `gen_conf=None` + guard in
`rag/llm/chat_model.py` (12 instances across Base, BaiChuanChat,
LocalLLM, MistralChat, ReplicateChat, BaiduYiyanChat, GoogleChat
classes)
- Replace `doc_ids=[]` with `doc_ids=None` + guard in
`api/db/services/document_service.py` (1 instance)
- Mutable default arguments are shared across all calls, causing
potential cross-request state contamination
- See Python docs:
https://docs.python.org/3/faq/programming.html#why-are-default-values-shared-between-objects
## Test plan
- [x] Verify LLM calls work with and without explicit gen_conf
- [x] No behavior change for existing callers — `None` is replaced with
`{}` at function entry
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude Opus 4.6
Co-authored-by: Jin Hai
Co-authored-by: Yingfeng
Co-authored-by: Kevin Hu
---
api/db/services/document_service.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py
index 2c80e76fc68..44c9995c02d 100644
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -1007,11 +1007,13 @@ def run(cls, tenant_id: str, doc: dict, kb_table_num_map: dict):
queue_tasks(doc, bucket, name, 0)
-def queue_raptor_o_graphrag_tasks(sample_doc, ty, priority, fake_doc_id="", doc_ids=[]):
+def queue_raptor_o_graphrag_tasks(sample_doc, ty, priority, fake_doc_id="", doc_ids=None):
"""
You can provide a fake_doc_id to bypass the restriction of tasks at the knowledgebase level.
Optionally, specify a list of doc_ids to determine which documents participate in the task.
"""
+ if doc_ids is None:
+ doc_ids = []
assert ty in ["graphrag", "raptor", "mindmap"], "type should be graphrag, raptor or mindmap"
chunking_config = DocumentService.get_chunking_config(sample_doc["id"])
From 3c68ad03be642bc23992ad92637426741e98f319 Mon Sep 17 00:00:00 2001
From: buua436
Date: Thu, 14 May 2026 14:47:15 +0800
Subject: [PATCH 145/666] Go: update user settings fields (#14918)
### What problem does this PR solve?
update user settings fields
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
internal/service/user.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/internal/service/user.go b/internal/service/user.go
index 3147ae69c57..dd6b9493048 100644
--- a/internal/service/user.go
+++ b/internal/service/user.go
@@ -774,15 +774,19 @@ func (s *UserService) UpdateUserSettings(user *entity.User, req *UpdateSettingsR
if req.Avatar != nil {
// In Go version, avatar might be stored differently
// For now, just update if field exists
+ user.Avatar = req.Avatar
}
if req.Language != nil {
// Store language preference
+ user.Language = req.Language
}
if req.ColorSchema != nil {
// Store color schema preference
+ user.ColorSchema = req.ColorSchema
}
if req.Timezone != nil {
// Store timezone preference
+ user.Timezone = req.Timezone
}
// Save updated user
From bd99a22661b12ca8d8375f2b28512afb509a1ad3 Mon Sep 17 00:00:00 2001
From: dale053
Date: Wed, 13 May 2026 23:48:52 -0700
Subject: [PATCH 146/666] =?UTF-8?q?fix:=20atomic=20chunk/token=20counter?=
=?UTF-8?q?=20updates=20for=20documents=20and=20knowledge=20b=E2=80=A6=20(?=
=?UTF-8?q?#14867)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Fixes #14866.
Previously, `DocumentService.increment_chunk_num` and
`decrement_chunk_num` updated the `Document` row and its parent
`Knowledgebase` row in two separate, non-transactional statements. If
the second update failed (DB error, connection drop, etc.) after the
first one succeeded, the document and knowledge base chunk/token
counters would drift apart and stay inconsistent.
There was also a behavioral asymmetry between the two methods:
- `increment_chunk_num` only logged a warning when the document row was
missing and returned a value that callers usually treated as success.
- `decrement_chunk_num` raised `LookupError` in the same situation.
This PR makes the counter updates atomic and aligns the missing-document
behavior between the two methods:
- Wrap the `Document` and `Knowledgebase` updates in
`increment_chunk_num` / `decrement_chunk_num` inside a `DB.atomic()`
block so both succeed or both roll back together.
- Raise `LookupError` from `increment_chunk_num` when the target
document no longer exists, matching `decrement_chunk_num`.
- Update `reset_document_for_reparse` in `document_api_service.py` to
catch the new `LookupError` and return a proper "Document not found!"
API error instead of propagating the exception.
No schema changes, no API contract changes for the success path; only
the failure mode for a missing document during reparse is now a clean
error response instead of an uncaught exception.
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/services/document_api_service.py | 17 +++--
api/db/services/document_service.py | 89 +++++++++++++++++++----
2 files changed, 83 insertions(+), 23 deletions(-)
diff --git a/api/apps/services/document_api_service.py b/api/apps/services/document_api_service.py
index 59abbd25072..63c71ff4a24 100644
--- a/api/apps/services/document_api_service.py
+++ b/api/apps/services/document_api_service.py
@@ -122,13 +122,16 @@ def reset_document_for_reparse(doc, tenant_id, parser_id=None, pipeline_id=None)
# Delete chunks from document store
if doc.token_num > 0:
- e = DocumentService.increment_chunk_num(
- doc.id,
- doc.kb_id,
- doc.token_num * -1,
- doc.chunk_num * -1,
- doc.process_duration * -1,
- )
+ try:
+ e = DocumentService.increment_chunk_num(
+ doc.id,
+ doc.kb_id,
+ doc.token_num * -1,
+ doc.chunk_num * -1,
+ doc.process_duration * -1,
+ )
+ except LookupError:
+ return get_error_data_result(message="Document not found!")
if not e:
return get_error_data_result(message="Document not found!")
settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py
index 44c9995c02d..3d9bc09dbbb 100644
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -591,27 +591,84 @@ def get_unfinished_docs(cls):
@classmethod
@DB.connection_context()
def increment_chunk_num(cls, doc_id, kb_id, token_num, chunk_num, duration):
- num = (
- cls.model.update(token_num=cls.model.token_num + token_num, chunk_num=cls.model.chunk_num + chunk_num, process_duration=cls.model.process_duration + duration)
- .where(cls.model.id == doc_id)
- .execute()
- )
- if num == 0:
- logging.warning("Document not found which is supposed to be there")
- num = Knowledgebase.update(token_num=Knowledgebase.token_num + token_num, chunk_num=Knowledgebase.chunk_num + chunk_num).where(Knowledgebase.id == kb_id).execute()
+ """Atomically add chunk/token counters on the document and its knowledge base."""
+ with DB.atomic():
+ num = (
+ cls.model.update(
+ token_num=cls.model.token_num + token_num,
+ chunk_num=cls.model.chunk_num + chunk_num,
+ process_duration=cls.model.process_duration + duration,
+ )
+ .where((cls.model.id == doc_id) & (cls.model.kb_id == kb_id))
+ .execute()
+ )
+ if num == 0:
+ logging.error(
+ "increment_chunk_num: no document matched doc_id=%s kb_id=%s "
+ "token_num=%s chunk_num=%s duration=%s",
+ doc_id,
+ kb_id,
+ token_num,
+ chunk_num,
+ duration,
+ )
+ raise LookupError("Document not found which is supposed to be there")
+ num = (
+ Knowledgebase.update(
+ token_num=Knowledgebase.token_num + token_num,
+ chunk_num=Knowledgebase.chunk_num + chunk_num,
+ )
+ .where(Knowledgebase.id == kb_id)
+ .execute()
+ )
+ if num == 0:
+ logging.error(
+ "increment_chunk_num: no knowledgebase matched kb_id=%s for doc_id=%s "
+ "token_num=%s chunk_num=%s duration=%s",
+ kb_id,
+ doc_id,
+ token_num,
+ chunk_num,
+ duration,
+ )
+ raise LookupError("Knowledgebase not found which is supposed to be there")
return num
@classmethod
@DB.connection_context()
def decrement_chunk_num(cls, doc_id, kb_id, token_num, chunk_num, duration):
- num = (
- cls.model.update(token_num=cls.model.token_num - token_num, chunk_num=cls.model.chunk_num - chunk_num, process_duration=cls.model.process_duration + duration)
- .where(cls.model.id == doc_id)
- .execute()
- )
- if num == 0:
- raise LookupError("Document not found which is supposed to be there")
- num = Knowledgebase.update(token_num=Knowledgebase.token_num - token_num, chunk_num=Knowledgebase.chunk_num - chunk_num).where(Knowledgebase.id == kb_id).execute()
+ """Atomically subtract chunk/token counters on the document and its knowledge base."""
+ with DB.atomic():
+ num = (
+ cls.model.update(
+ token_num=cls.model.token_num - token_num,
+ chunk_num=cls.model.chunk_num - chunk_num,
+ process_duration=cls.model.process_duration + duration,
+ )
+ .where((cls.model.id == doc_id) & (cls.model.kb_id == kb_id))
+ .execute()
+ )
+ if num == 0:
+ raise LookupError("Document not found which is supposed to be there")
+ num = (
+ Knowledgebase.update(
+ token_num=Knowledgebase.token_num - token_num,
+ chunk_num=Knowledgebase.chunk_num - chunk_num,
+ )
+ .where(Knowledgebase.id == kb_id)
+ .execute()
+ )
+ if num == 0:
+ logging.error(
+ "decrement_chunk_num: no knowledgebase matched kb_id=%s for doc_id=%s "
+ "token_num=%s chunk_num=%s duration=%s",
+ kb_id,
+ doc_id,
+ token_num,
+ chunk_num,
+ duration,
+ )
+ raise LookupError("Knowledgebase not found which is supposed to be there")
return num
@classmethod
From 63df01fe3f18a5b54634c8ab68761b0487aee20f Mon Sep 17 00:00:00 2001
From: eviaaaaa <2278596667@qq.com>
Date: Thu, 14 May 2026 15:28:39 +0800
Subject: [PATCH 147/666] fix(agent): handle duplicate MCP tool names (#14217)
### What problem does this PR solve?
When multiple MCP servers expose tools with the same name, the agent
currently registers those tools using their original MCP names. This can
lead to two issues:
- later MCP tools may overwrite earlier ones in the agent tool map
- duplicate function names may be exposed to the LLM
This PR fixes duplicate MCP tool-name handling by applying the same
indexed naming strategy already used for native agent tools. Native
tools are exposed with generated names such as `_` to
avoid collisions, and MCP tools now follow the same convention for
consistency.
Specifically, this PR:
- assigns unique indexed function names to MCP tools exposed to the LLM
- preserves each MCP tool's original server-side name in an
`MCPToolBinding`
- dispatches MCP calls using the original MCP tool name while keeping
the indexed name in the agent tool map
- allows MCP metadata conversion to override only the OpenAI function
name without modifying the original MCP tool metadata
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
### Validation
The validation was performed using two MCP servers. Both servers exposed
a tool with the same name: `mcp0`. Both tools take no input parameters.
**MCP Server One:**
**MCP Server Two:**
**Before the fix:**
When invoking `mcp0`, only the `mcp0` tool from the MCP server injected
later could be called successfully. As shown below, both `mcp0` tools
were present, but only the later-registered one was actually invokable.
**After the fix:**
Both `mcp0` tools can now be invoked correctly.
---
agent/component/agent_with_tools.py | 9 ++++++---
agent/tools/base.py | 14 ++++++++------
common/mcp_tool_call_conn.py | 15 +++++++++++----
3 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/agent/component/agent_with_tools.py b/agent/component/agent_with_tools.py
index 859064046d6..83c3e27e531 100644
--- a/agent/component/agent_with_tools.py
+++ b/agent/component/agent_with_tools.py
@@ -32,7 +32,7 @@
from api.db.services.mcp_server_service import MCPServerService
from api.db.services.tenant_llm_service import TenantLLMService
from common.connection_utils import timeout
-from common.mcp_tool_call_conn import MCPToolCallSession, mcp_tool_metadata_to_openai_tool
+from common.mcp_tool_call_conn import MCPToolBinding, MCPToolCallSession, mcp_tool_metadata_to_openai_tool
from rag.prompts.generator import citation_plus, citation_prompt, full_question, kb_prompt, message_fit_in, structured_output_prompt
@@ -97,13 +97,16 @@ def __init__(self, canvas, id, param: LLMParam):
indexed_meta["function"]["name"] = indexed_name
self.tool_meta.append(indexed_meta)
+ tool_idx = len(self.tools)
for mcp in self._param.mcp:
_, mcp_server = MCPServerService.get_by_id(mcp["mcp_id"])
custom_header = self._param.custom_header
tool_call_session = MCPToolCallSession(mcp_server, mcp_server.variables, custom_header)
for tnm, meta in mcp["tools"].items():
- self.tool_meta.append(mcp_tool_metadata_to_openai_tool(meta))
- self.tools[tnm] = tool_call_session
+ indexed_name = f"{tnm}_{tool_idx}"
+ tool_idx += 1
+ self.tool_meta.append(mcp_tool_metadata_to_openai_tool(meta, function_name=indexed_name))
+ self.tools[indexed_name] = MCPToolBinding(tool_call_session, tnm)
self.callback = partial(self._canvas.tool_use_callback, id)
self.toolcall_session = LLMToolPluginCallSession(self.tools, self.callback)
if self.tool_meta:
diff --git a/agent/tools/base.py b/agent/tools/base.py
index 194b47fceec..dbcb185518f 100644
--- a/agent/tools/base.py
+++ b/agent/tools/base.py
@@ -23,7 +23,7 @@
from agent.component.base import ComponentParamBase, ComponentBase
from common.misc_utils import hash_str2int
from rag.prompts.generator import kb_prompt
-from common.mcp_tool_call_conn import MCPToolCallSession, ToolCallSession
+from common.mcp_tool_call_conn import MCPToolBinding, MCPToolCallSession, ToolCallSession
from timeit import default_timer as timer
@@ -52,16 +52,18 @@ def __init__(self, tools_map: dict[str, object], callback: partial):
self.tools_map = tools_map
self.callback = callback
- def tool_call(self, name: str, arguments: dict[str, Any]) -> Any:
- return asyncio.run(self.tool_call_async(name, arguments))
+ def tool_call(self, name: str, arguments: dict[str, Any], timeout: float | int = 10) -> Any:
+ return asyncio.run(self.tool_call_async(name, arguments, request_timeout=timeout))
- async def tool_call_async(self, name: str, arguments: dict[str, Any]) -> Any:
+ async def tool_call_async(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10) -> Any:
assert name in self.tools_map, f"LLM tool {name} does not exist"
logging.info(f"[ToolCall] invoke name={name} arguments={str(arguments)[:200]}")
st = timer()
tool_obj = self.tools_map[name]
- if isinstance(tool_obj, MCPToolCallSession):
- resp = await thread_pool_exec(tool_obj.tool_call, name, arguments, 60)
+ if isinstance(tool_obj, MCPToolBinding):
+ resp = await thread_pool_exec(tool_obj.session.tool_call, tool_obj.original_name, arguments, request_timeout)
+ elif isinstance(tool_obj, MCPToolCallSession):
+ resp = await thread_pool_exec(tool_obj.tool_call, name, arguments, request_timeout)
elif hasattr(tool_obj, "invoke_async") and asyncio.iscoroutinefunction(tool_obj.invoke_async):
resp = await tool_obj.invoke_async(**arguments)
else:
diff --git a/common/mcp_tool_call_conn.py b/common/mcp_tool_call_conn.py
index 95e3581bb0b..676978d052e 100644
--- a/common/mcp_tool_call_conn.py
+++ b/common/mcp_tool_call_conn.py
@@ -20,6 +20,7 @@
import weakref
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeoutError
+from dataclasses import dataclass
from string import Template
from typing import Any, Literal, Protocol
@@ -36,7 +37,13 @@
class ToolCallSession(Protocol):
- def tool_call(self, name: str, arguments: dict[str, Any]) -> str: ...
+ def tool_call(self, name: str, arguments: dict[str, Any], timeout: float | int = 10) -> str: ...
+
+
+@dataclass(frozen=True)
+class MCPToolBinding:
+ session: ToolCallSession
+ original_name: str
class MCPToolCallSession(ToolCallSession):
@@ -316,12 +323,12 @@ def shutdown_all_mcp_sessions():
logging.info("All MCPToolCallSession instances have been closed.")
-def mcp_tool_metadata_to_openai_tool(mcp_tool: Tool | dict) -> dict[str, Any]:
+def mcp_tool_metadata_to_openai_tool(mcp_tool: Tool | dict, function_name: str | None = None) -> dict[str, Any]:
if isinstance(mcp_tool, dict):
return {
"type": "function",
"function": {
- "name": mcp_tool["name"],
+ "name": function_name or mcp_tool["name"],
"description": mcp_tool["description"],
"parameters": mcp_tool["inputSchema"],
},
@@ -330,7 +337,7 @@ def mcp_tool_metadata_to_openai_tool(mcp_tool: Tool | dict) -> dict[str, Any]:
return {
"type": "function",
"function": {
- "name": mcp_tool.name,
+ "name": function_name or mcp_tool.name,
"description": mcp_tool.description,
"parameters": mcp_tool.inputSchema,
},
From a98994ff918a227771944239409a6846d95ba095 Mon Sep 17 00:00:00 2001
From: wdeveloper16
Date: Thu, 14 May 2026 10:45:44 +0200
Subject: [PATCH 148/666] fix: close db connections reliably in
test_db_connection (#14777)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
- Fixes resource-management bugs in the `POST
/agents/test_db_connection` endpoint where database connections could be
left open on error (part of #14750)
## Changes
- `api/apps/restful_apis/agent_api.py` — `test_db_connection`:
- mysql / mariadb / oceanbase / postgres: replaced bare `db.connect()` /
`db.close()` fallthrough with `with db.connection_context()` and a probe
`SELECT 1` — guaranteed close on both success and exception
- mssql: nested `try/finally` blocks so `cursor.close()` and
`db.close()` are always called even when `cursor.execute()` raises
- trino: wrapped cursor ops in `try/finally` for the same reason
- Removed the `if req["db_type"] != "mssql": db.connect(); db.close()`
shared fallthrough block — each branch now owns its teardown
- Consolidated to a single `return get_json_result(...)` after the
if/elif chain
---
api/apps/restful_apis/agent_api.py | 36 +++++++++++++++++++-----------
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index 0d545e7649f..4d078425922 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -791,6 +791,8 @@ async def test_db_connection():
port=req["port"],
password=req["password"],
)
+ with db.connection_context():
+ db.execute_sql("SELECT 1")
elif req["db_type"] == "oceanbase":
db = MySQLDatabase(
req["database"],
@@ -800,6 +802,8 @@ async def test_db_connection():
password=req["password"],
charset="utf8mb4",
)
+ with db.connection_context():
+ db.execute_sql("SELECT 1")
elif req["db_type"] == "postgres":
db = PostgresqlDatabase(
req["database"],
@@ -808,6 +812,8 @@ async def test_db_connection():
port=req["port"],
password=req["password"],
)
+ with db.connection_context():
+ db.execute_sql("SELECT 1")
elif req["db_type"] == "mssql":
import pyodbc
@@ -819,9 +825,14 @@ async def test_db_connection():
f"PWD={req['password']};"
)
db = pyodbc.connect(connection_string)
- cursor = db.cursor()
- cursor.execute("SELECT 1")
- cursor.close()
+ try:
+ cursor = db.cursor()
+ try:
+ cursor.execute("SELECT 1")
+ finally:
+ cursor.close()
+ finally:
+ db.close()
elif req["db_type"] == "IBM DB2":
import ibm_db
@@ -844,7 +855,6 @@ async def test_db_connection():
stmt = ibm_db.exec_immediate(conn, "SELECT 1 FROM sysibm.sysdummy1")
ibm_db.fetch_assoc(stmt)
ibm_db.close(conn)
- return get_json_result(data="Database Connection Successful!")
elif req["db_type"] == "trino":
import os
import trino
@@ -871,18 +881,18 @@ async def test_db_connection():
http_scheme=http_scheme,
auth=auth,
)
- cur = conn.cursor()
- cur.execute("SELECT 1")
- cur.fetchall()
- cur.close()
- conn.close()
- return get_json_result(data="Database Connection Successful!")
+ try:
+ cur = conn.cursor()
+ try:
+ cur.execute("SELECT 1")
+ cur.fetchall()
+ finally:
+ cur.close()
+ finally:
+ conn.close()
else:
return server_error_response("Unsupported database type.")
- if req["db_type"] != "mssql":
- db.connect()
- db.close()
return get_json_result(data="Database Connection Successful!")
except Exception as exc:
return server_error_response(exc)
From 106f4b777efb8911adc3966497f79a9dfd5dc6b6 Mon Sep 17 00:00:00 2001
From: Haruko386
Date: Thu, 14 May 2026 18:58:00 +0800
Subject: [PATCH 149/666] Go: implement TTS for fishaudio, openrouter and asr
for fishaudio (#14926)
### What problem does this PR solve?
This PR implement TTS for FishAudio and MiniMax provider and ASR for
FishAudio
**The following functionalities are now supported:**
**FishAudio:**
- [x] Text To Speech
- [x] Stream Text To Speech
- [x] Audio To Text
**OpenRouter:**
- [x] Text To Speech
**Verified examples from the CLI:**
```plaintext
**FishAudio**
RAGFlow(user)> tts with 's1@test@fishaudio' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"reference_id": "90e65eaaf50e4470b8e6d43ee6afd7d5", "temperature": 0.7, "top_p": 0.7, "prosody": {"speed": 1, "volume": 0, "normalize_loudness": true}, "chunk_length": 300, "normalize": true, "sample_rate": 44100, "mp3_bitrate": 128, "latency": "normal", "max_new_tokens": 1024, "repetition_penalty": 1.2, "min_chunk_length": 50, "condition_on_previous_chunks": true, "early_stop_threshold": 1}'
Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/s1_output.wav
SUCCESS
RAGFlow(user)> stream tts with 's1@test@fishaudio' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"reference_id": "90e65eaaf50e4470b8e6d43ee6afd7d5", "temperature": 0.7, "top_p": 0.7, "prosody": {"speed": 1, "volume": 0, "normalize_loudness": true}, "chunk_length": 300, "normalize": true, "sample_rate": 44100, "mp3_bitrate": 128, "latency": "normal", "max_new_tokens": 1024, "repetition_penalty": 1.2, "min_chunk_length": 50, "condition_on_previous_chunks": true, "early_stop_threshold": 1}'
Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/s1_output.wav
SUCCESS
RAGFlow(user)> asr with 'transcribe-1@test@fishaudio' audio './internal/test.wav' param '{"language": "en", "ignore_timestamps": true}'
+----------------------------------------------------------------------------------------------------------------------+
| text |
+----------------------------------------------------------------------------------------------------------------------+
| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. |
+----------------------------------------------------------------------------------------------------------------------+
```
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
- [x] Refactoring
---
conf/models/fishaudio.json | 26 ++-
conf/models/openrouter.json | 10 +-
internal/cli/user_command.go | 38 +++-
internal/cli/user_parser.go | 23 ++-
internal/entity/models/fishaudio.go | 272 ++++++++++++++++++++++++++-
internal/entity/models/minimax.go | 17 +-
internal/entity/models/openrouter.go | 59 +++++-
internal/entity/models/types.go | 6 +-
internal/handler/providers.go | 18 +-
9 files changed, 436 insertions(+), 33 deletions(-)
diff --git a/conf/models/fishaudio.json b/conf/models/fishaudio.json
index 585aab33693..aa6beda9647 100644
--- a/conf/models/fishaudio.json
+++ b/conf/models/fishaudio.json
@@ -5,10 +5,32 @@
},
"url_suffix": {
"models": "model",
- "balance": "self/package"
+ "balance": "self/package",
+ "tts": "v1/tts",
+ "asr": "v1/asr"
},
"class": "fishaudio",
"models": [
-
+ {
+ "name": "s2-pro",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "s1",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "transcribe-1",
+ "max_tokens": 8192,
+ "model_types": [
+ "asr"
+ ]
+ }
]
}
\ No newline at end of file
diff --git a/conf/models/openrouter.json b/conf/models/openrouter.json
index 6af1e2d15df..33d0bdadbde 100644
--- a/conf/models/openrouter.json
+++ b/conf/models/openrouter.json
@@ -8,7 +8,8 @@
"models": "models",
"embedding": "embeddings",
"rerank": "rerank",
- "balance": "credits"
+ "balance": "credits",
+ "tts": "audio/speech"
},
"class": "openrouter",
"models": [
@@ -44,6 +45,13 @@
"default_value": true,
"clear_thinking": true
}
+ },
+ {
+ "name": "openai/gpt-audio-mini",
+ "max_tokens": 131072,
+ "model_types": [
+ "tts"
+ ]
}
]
}
\ No newline at end of file
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index 7a2b2759675..f631a252759 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -2013,7 +2013,7 @@ func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) {
if explicitFormat != "" {
ttsConfigPayload["format"] = explicitFormat
} else {
- explicitFormat = "mp3"
+ ttsConfigPayload["format"] = "mp3"
}
if len(ttsConfigPayload) > 0 {
@@ -2056,7 +2056,6 @@ func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) {
shouldSave, _ := cmd.Params["save"].(bool)
saveDir, _ := cmd.Params["save_path"].(string)
-
fileName := fmt.Sprintf("%s_output.%s", modelName, explicitFormat)
cwd, err := os.Getwd()
@@ -2149,14 +2148,27 @@ func (c *RAGFlowClient) ASRUserCommand(cmd *Command) (ResponseIf, error) {
audioFile, ok := cmd.Params["audio_file"].(string)
if !ok {
- return nil, fmt.Errorf("text not provided")
+ return nil, fmt.Errorf("audio file not provided")
}
payload := map[string]interface{}{
"provider_name": providerName,
"instance_name": instanceName,
"model_name": modelName,
- "audio_file": audioFile,
+ "file": audioFile,
+ }
+
+ asrConfigPayload := make(map[string]interface{})
+ if paramStr, ok := cmd.Params["param_str"].(string); ok && paramStr != "" {
+ var dynamicParams map[string]interface{}
+ if err := json.Unmarshal([]byte(paramStr), &dynamicParams); err != nil {
+ return nil, fmt.Errorf("param string must be valid JSON. Error: %w", err)
+ }
+ asrConfigPayload["params"] = dynamicParams
+ }
+
+ if len(asrConfigPayload) > 0 {
+ payload["asr_config"] = asrConfigPayload
}
url := "/audio/transcriptions"
@@ -2168,13 +2180,23 @@ func (c *RAGFlowClient) ASRUserCommand(cmd *Command) (ResponseIf, error) {
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to ASR document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
- var result CommonResponse
- if err = json.Unmarshal(resp.Body, &result); err != nil {
+ var rawResult struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ Data map[string]interface{} `json:"data"`
+ }
+
+ if err = json.Unmarshal(resp.Body, &rawResult); err != nil {
return nil, fmt.Errorf("ASR document failed: invalid JSON (%w)", err)
}
- if result.Code != 0 {
- return nil, fmt.Errorf("%s", result.Message)
+
+ if rawResult.Code != 0 {
+ return nil, fmt.Errorf("%s", rawResult.Message)
}
+
+ var result CommonResponse
+ result.Code = rawResult.Code
+ result.Message = rawResult.Data["text"].(string) // TODO
result.Duration = resp.Duration
return &result, nil
diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go
index 04ebc7e87e9..28f32071141 100644
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -2753,7 +2753,7 @@ func (p *Parser) parseASRCommand() (*Command, error) {
if p.curToken.Type != TokenAudio {
return nil, fmt.Errorf("expected AUDIO to ASR")
}
- p.nextToken() // consume FILE
+ p.nextToken() // consume AUDIO
audioFile, err := p.parseQuotedString()
if err != nil {
@@ -2761,14 +2761,29 @@ func (p *Parser) parseASRCommand() (*Command, error) {
}
p.nextToken()
+ cmd := NewCommand("asr_user_command")
+ cmd.Params["composite_model_name"] = compositeModelName
+ cmd.Params["audio_file"] = audioFile
+
+ for p.curToken.Type != TokenEOF && p.curToken.Type != TokenSemicolon {
+ switch p.curToken.Type {
+ case TokenParam:
+ p.nextToken()
+ if p.curToken.Type != TokenQuotedString {
+ return nil, fmt.Errorf("expect quoted string after 'param'")
+ }
+ cmd.Params["param_str"] = strings.Trim(p.curToken.Value, "\"'")
+ p.nextToken()
+ default:
+ return nil, fmt.Errorf("unexpected token in asr command: %s", p.curToken.Value)
+ }
+ }
+
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
- cmd := NewCommand("asr_user_command")
- cmd.Params["composite_model_name"] = compositeModelName
- cmd.Params["audio_file"] = audioFile
return cmd, nil
}
diff --git a/internal/entity/models/fishaudio.go b/internal/entity/models/fishaudio.go
index 1e9f0aa9d52..0eec1b16539 100644
--- a/internal/entity/models/fishaudio.go
+++ b/internal/entity/models/fishaudio.go
@@ -1,10 +1,17 @@
package models
import (
+ "bufio"
+ "bytes"
+ "encoding/base64"
"encoding/json"
"fmt"
"io"
+ "mime/multipart"
"net/http"
+ "os"
+ "path/filepath"
+ "strconv"
"strings"
"time"
)
@@ -64,20 +71,273 @@ func (f *FishAudioModel) Rerank(modelName *string, query string, documents []str
// TranscribeAudio transcribe audio
func (f *FishAudioModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
- return nil, fmt.Errorf("%s, no such method", f.Name())
+
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("FishAudio API key is missing")
+ }
+
+ if file == nil || *file == "" {
+ return nil, fmt.Errorf("file is missing")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", f.BaseURL[region], f.URLSuffix.ASR)
+
+ var body bytes.Buffer
+ writer := multipart.NewWriter(&body)
+
+ // audio file
+ audioFile, err := os.Open(*file)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open audio file: %w", err)
+ }
+ defer audioFile.Close()
+
+ part, err := writer.CreateFormFile("audio", filepath.Base(*file))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create multipart file: %w", err)
+ }
+
+ if _, err = io.Copy(part, audioFile); err != nil {
+ return nil, fmt.Errorf("failed to copy audio data: %w", err)
+ }
+
+ // extra params
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+
+ var val string
+
+ switch v := value.(type) {
+ case string:
+ val = v
+ case bool:
+ val = strconv.FormatBool(v)
+ case int:
+ val = strconv.Itoa(v)
+ case float64:
+ val = strconv.FormatFloat(v, 'f', -1, 64)
+ default:
+ val = fmt.Sprintf("%v", v)
+ }
+
+ if err := writer.WriteField(key, val); err != nil {
+ return nil, fmt.Errorf("failed to write field %s: %w", key, err)
+ }
+ }
+ }
+
+ if err := writer.Close(); err != nil {
+ return nil, fmt.Errorf("failed to close multipart writer: %w", err)
+ }
+
+ // request
+ req, err := http.NewRequest("POST", url, &body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("Content-Type", writer.FormDataContentType())
+
+ resp, err := f.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ respBody, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf(
+ "FishAudio ASR error: %s - %s",
+ resp.Status,
+ string(respBody),
+ )
+ }
+
+ // result
+ var result struct {
+ Text string `json:"text"`
+ }
+
+ if err := json.Unmarshal(respBody, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+ }
+
+ return &ASRResponse{
+ Text: result.Text,
+ }, nil
}
-func (z *FishAudioModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", z.Name())
+func (f *FishAudioModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", f.Name())
}
// AudioSpeech convert audio to text
func (f *FishAudioModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
- return nil, fmt.Errorf("%s, no such method", f.Name())
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("FishAudio API key is missing")
+ }
+
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("text content is missing")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", f.BaseURL[region], f.URLSuffix.TTS)
+
+ reqBody := map[string]interface{}{
+ "text": *audioContent,
+ }
+
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if asrConfig != nil && asrConfig.Format != "" {
+ reqBody["format"] = asrConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("model", *modelName)
+
+ resp, err := f.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("%s - %s", resp.Status, string(body))
+ }
+
+ return &TTSResponse{Audio: body}, nil
}
-func (z *FishAudioModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", z.Name())
+func (f *FishAudioModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig, sender func(*string, *string) error) error {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("FishAudio API key is missing")
+ }
+
+ if audioContent == nil || *audioContent == "" {
+ return fmt.Errorf("text content is missing")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s/%s", f.BaseURL[region], f.URLSuffix.TTS, "stream/with-timestamp")
+
+ reqBody := map[string]interface{}{
+ "text": *audioContent,
+ }
+
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if asrConfig != nil && asrConfig.Format != "" {
+ reqBody["format"] = asrConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ // Build Request
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("model", *modelName)
+
+ resp, err := f.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ buf := make([]byte, 1024)
+ n, _ := resp.Body.Read(buf)
+ return fmt.Errorf("FishAudio stream API error: %d - %s", resp.StatusCode, string(buf[:n]))
+ }
+
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 8*1024*1024)
+
+ for scanner.Scan() {
+ line := scanner.Text()
+
+ if !strings.HasPrefix(line, "data: ") {
+ continue
+ }
+
+ dataStr := strings.TrimSpace(line[6:])
+ if dataStr == "" {
+ continue
+ }
+
+ var event struct {
+ AudioBase64 string `json:"audio_base64"`
+ }
+
+ if err := json.Unmarshal([]byte(dataStr), &event); err != nil {
+ continue
+ }
+
+ if event.AudioBase64 != "" {
+ audioBytes, err := base64.StdEncoding.DecodeString(event.AudioBase64)
+ if err == nil && len(audioBytes) > 0 {
+ chunk := string(audioBytes)
+ if errSend := sender(&chunk, nil); errSend != nil {
+ return errSend
+ }
+ }
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("error reading FishAudio stream: %w", err)
+ }
+
+ return nil
}
// OCRFile OCR file
diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go
index 1f8afe3b942..683a8dc454e 100644
--- a/internal/entity/models/minimax.go
+++ b/internal/entity/models/minimax.go
@@ -478,7 +478,7 @@ func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiC
}
url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.TTS)
-
+
reqBody := map[string]interface{}{
"model": modelName,
"text": audioContent,
@@ -488,6 +488,11 @@ func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiC
reqBody[key] = value
}
}
+ if asrConfig != nil && asrConfig.Format != "" {
+ reqBody["audio_setting"] = map[string]interface{}{
+ "format": asrConfig.Format,
+ }
+ }
reqBody["stream"] = false
jsonData, err := json.Marshal(reqBody)
@@ -547,7 +552,6 @@ func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiC
}, nil
}
-// tts with 'speech-2.8-hd@test@minimax' text 'If that day, out position was switched, would our fate, be different?' voice 'English_expressive_narrator' param '{"voice_setting": {"voice_id": "English_expressive_narrator", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}'
func (z *MinimaxModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return fmt.Errorf("MiniMax API key is missing")
@@ -581,6 +585,13 @@ func (z *MinimaxModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
}
reqBody["stream"] = false
+
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["audio_setting"] = map[string]interface{}{
+ "format": ttsConfig.Format,
+ }
+ }
+
reqBody["stream"] = true
jsonData, err := json.Marshal(reqBody)
@@ -658,4 +669,4 @@ func (z *MinimaxModel) AudioSpeechWithSender(modelName *string, audioContent *st
// OCRFile OCR file
func (m *MinimaxModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
-}
+}
\ No newline at end of file
diff --git a/internal/entity/models/openrouter.go b/internal/entity/models/openrouter.go
index d401ba02bbc..461a1fe4c38 100644
--- a/internal/entity/models/openrouter.go
+++ b/internal/entity/models/openrouter.go
@@ -545,7 +545,64 @@ func (z *OpenRouterModel) TranscribeAudioWithSender(modelName *string, file *str
// AudioSpeech convert audio to text
func (o *OpenRouterModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
- return nil, fmt.Errorf("%s, no such method", o.Name())
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("OpenRouter API key is missing")
+ }
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("text content is empty")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.TTS)
+
+ // OpenRouter:response Audio bytes stream
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "input": audioContent,
+ }
+
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if asrConfig != nil && asrConfig.Format != "" {
+ reqBody["response_format"] = asrConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := o.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("OpenRouter API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ return &TTSResponse{Audio: body}, nil
}
func (z *OpenRouterModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index 3de7ac51587..b0bc9b46b08 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -62,6 +62,7 @@ type RerankResponse struct {
}
type ASRResponse struct {
+ Text string `json:"text"`
}
type TTSResponse struct {
@@ -85,6 +86,7 @@ type URLSuffix struct {
Files string `json:"files"`
Status string `json:"status"`
TTS string `json:"tts"`
+ ASR string `json:"asr"`
}
type ChatConfig struct {
@@ -115,10 +117,12 @@ type RerankConfig struct {
}
type ASRConfig struct {
+ Params map[string]interface{} `json:"params"`
}
type TTSConfig struct {
- Params map[string]interface{}
+ Format string `json:"format"`
+ Params map[string]interface{} `json:"params"`
}
type OCRConfig struct {
diff --git a/internal/handler/providers.go b/internal/handler/providers.go
index 3b060a5097e..c6d61a2e653 100644
--- a/internal/handler/providers.go
+++ b/internal/handler/providers.go
@@ -1049,13 +1049,14 @@ func (h *ProviderHandler) RerankDocument(c *gin.Context) {
}
type TranscribeAudioRequest struct {
- ProviderName *string `json:"provider_name"`
- InstanceName *string `json:"instance_name"`
- ModelName *string `json:"model_name"`
- File *string `json:"file"`
- Language []string `json:"language"`
- Prompt int `json:"prompt"`
- Stream bool `json:"stream"`
+ ProviderName *string `json:"provider_name"`
+ InstanceName *string `json:"instance_name"`
+ ModelName *string `json:"model_name"`
+ File *string `json:"file"`
+ Language []string `json:"language"`
+ Prompt int `json:"prompt"`
+ Stream bool `json:"stream"`
+ ASRConfig *models.ASRConfig `json:"asr_config"`
}
func (h *ProviderHandler) TranscribeAudio(c *gin.Context) {
@@ -1101,6 +1102,9 @@ func (h *ProviderHandler) TranscribeAudio(c *gin.Context) {
}
asrConfig := models.ASRConfig{}
+ if req.ASRConfig != nil {
+ asrConfig = *req.ASRConfig
+ }
// Check if it's a stream request
if req.Stream {
From 41072ed44d1b8fb10fac040970b9df8da394459a Mon Sep 17 00:00:00 2001
From: balibabu
Date: Thu, 14 May 2026 20:33:11 +0800
Subject: [PATCH 150/666] Feat: This enables SelectWithSearch to search by
label. (#14925)
### What problem does this PR solve?
Feat: This enables SelectWithSearch to search by label.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
Co-authored-by: balibabu
---
.../components/originui/select-with-search.tsx | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/web/src/components/originui/select-with-search.tsx b/web/src/components/originui/select-with-search.tsx
index 8b55d1f8b6f..93f62146ae4 100644
--- a/web/src/components/originui/select-with-search.tsx
+++ b/web/src/components/originui/select-with-search.tsx
@@ -51,6 +51,13 @@ export type SelectWithSearchFlagProps = {
optionTestIdPrefix?: string;
};
+function filterFn(value: string, search: string, keywords?: string[]) {
+ const searchLower = search.toLowerCase();
+ const extendValue = (value + ' ' + (keywords?.join(' ') || '')).toLowerCase();
+ if (extendValue.includes(searchLower)) return 1;
+ return 0;
+}
+
function findLabelWithoutOptions(
options: SelectWithSearchFlagOptionType[],
value: string,
@@ -195,7 +202,7 @@ export const SelectWithSearch = forwardRef<
className="border-border-button w-full min-w-[var(--radix-popper-anchor-width)] p-0"
align="start"
>
-
+
{showSearch && (
Date: Thu, 14 May 2026 21:08:39 +0800
Subject: [PATCH 151/666] Doc: Finalized v0.25.4 release notes (#14929)
### What problem does this PR solve?
v0.25.4 release notes. Final.
### Type of change
- [x] Documentation Update
---
docs/release_notes.md | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/docs/release_notes.md b/docs/release_notes.md
index 292217887bf..293c3910d6a 100644
--- a/docs/release_notes.md
+++ b/docs/release_notes.md
@@ -15,20 +15,20 @@ Released on May 14, 2026
### New features
-- Adds a generic, configuration-driven RESTful API data source connector.
+- Introduces a generic RESTful API connector, enabling configurable data ingestion from niche or enterprise-specific platforms into RAGFlow. [#13545](https://github.com/infiniflow/ragflow/pull/13545)
+- Agent: Implements tag management to help users categorize, filter, and sort their Agent apps. [#14799](https://github.com/infiniflow/ragflow/pull/14799)
### Improvements
-- Agent tag management with filtering and sorting.
-- Widget customization and persistence.
+- Adds widget customization and persistence, allowing users to tailor their chat interface and ensure their settings are retained across sessions. [#14603](https://github.com/infiniflow/ragflow/pull/14603)
### Model support
-- Adds gpt-5.4-mini and gpt-5.4-nano to the OpenAI model list
+- Adds gpt-5.4-mini and gpt-5.4-nano to the OpenAI model list [#14908](https://github.com/infiniflow/ragflow/pull/14908)
### Bug fixes
-Fixed dataset document download route.
+- Corrects the API endpoint for downloading original files from a specified dataset. [#14910](https://github.com/infiniflow/ragflow/pull/14910) See also [Download document](./references/http_api_reference.md#download-document).
## v0.25.3
@@ -40,7 +40,7 @@ Released on May 13, 2026.
### Improvements
-- S3 connector: Implements ETag-based incremental synchronization for S3 data sources, drastically reduces sync time and AWS egress costs for users with massive-volumn S3-based datasets. [#14628](https://github.com/infiniflow/ragflow/issues/14628)[#14677](https://github.com/infiniflow/ragflow/pull/14677)
+- S3 connector: Implements ETag-based incremental synchronization for S3 data sources, drastically reducing sync time and AWS egress costs for users with massive-volumn S3-based datasets. [#14628](https://github.com/infiniflow/ragflow/issues/14628)[#14677](https://github.com/infiniflow/ragflow/pull/14677)
- API refactoring and security
- Continues the transition of web APIs to RESTful conventions, ensuring backward compatibility for all legacy endpoints.
- Binds the `user_id` in `POST /api/v1/messages` to the authenticated JWT principal. [#14745](https://github.com/infiniflow/ragflow/pull/14745)
@@ -54,7 +54,7 @@ Released on May 13, 2026.
- Images in multi-sheet Excel workbooks were not scoped by sheet, causing images to be incorrectly attributed across different worksheets. [#14120](https://github.com/infiniflow/ragflow/pull/14120)
- Agent: Splits the **Message** component output into distinct 'waiting' and 'message' states when nested inside an **Iteration** component alongside a **Wait** component. [#14839](https://github.com/infiniflow/ragflow/pull/14839)
-- Agent: The **Iteration** component failed to correctly pass array elements to its child components due to a naming mismatch between the expected IterationItem alias and the runtime item variable. [#14146](https://github.com/infiniflow/ragflow/pull/14146)
+- Agent: The **Iteration** component failed to correctly pass array elements to its child components due to a naming mismatch between the expected `IterationItem` alias and the runtime `item` variable. [#14146](https://github.com/infiniflow/ragflow/pull/14146)
- Agent: Template strings in tool-type components like **Email** and **Invoke** failed to interpolate; `{{variable}}` placeholders were passed through as raw text. [#14601](https://github.com/infiniflow/ragflow/pull/14601)
- Volcengine (Doubao/Ark) endpoints were not visible in the provider list. [#14702](https://github.com/infiniflow/ragflow/pull/14702)
From 58819f5d3e1a52bded15cd0693092cc7f69aa980 Mon Sep 17 00:00:00 2001
From: buua436
Date: Fri, 15 May 2026 09:36:58 +0800
Subject: [PATCH 152/666] fix: add document download endpoint and refactor
existing download function (#14927)
### What problem does this PR solve?
add document download endpoint and refactor existing download function
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/restful_apis/document_api.py | 2 -
api/apps/sdk/doc.py | 61 +++++++++++++++++++++++++--
2 files changed, 58 insertions(+), 5 deletions(-)
diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.py
index 7300a55a9f7..7547d2f20b5 100644
--- a/api/apps/restful_apis/document_api.py
+++ b/api/apps/restful_apis/document_api.py
@@ -1881,8 +1881,6 @@ async def download_attachment(tenant_id=None, doc_id=None, attachment_id=None):
# Keep backward compatibility with older callers and unit tests that still
# pass `attachment_id` instead of the route parameter name.
doc_id = doc_id or attachment_id
- if not DocumentService.accessible(doc_id, current_user.id):
- return get_data_error_result(message="Document not found!")
ext = request.args.get("ext", "markdown")
data = await thread_pool_exec(settings.STORAGE_IMPL.get, tenant_id, doc_id)
response = await make_response(data)
diff --git a/api/apps/sdk/doc.py b/api/apps/sdk/doc.py
index 85242f77b0d..4498b5f5de9 100644
--- a/api/apps/sdk/doc.py
+++ b/api/apps/sdk/doc.py
@@ -28,7 +28,7 @@
from api.db.services.llm_service import LLMBundle
from api.db.services.task_service import TaskService, cancel_all_task_of, queue_tasks
from api.db.services.tenant_llm_service import TenantLLMService
-from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_request_json, get_result, server_error_response, token_required, add_tenant_id_to_kwargs
+from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_request_json, get_result, server_error_response, token_required
from common import settings
from common.constants import LLMType, RetCode, TaskStatus
from common.metadata_utils import convert_conditions, meta_filter
@@ -53,8 +53,7 @@ def _enrich_chunks_with_document_metadata(chunks: list[dict], metadata_fields=No
@manager.route("/datasets//documents/", methods=["GET"]) # noqa: F821
@login_required
-@add_tenant_id_to_kwargs
-async def download(tenant_id, dataset_id, document_id):
+async def download(dataset_id, document_id):
"""
Download a document from a dataset.
---
@@ -113,6 +112,62 @@ async def download(tenant_id, dataset_id, document_id):
DOC_STOP_PARSING_INVALID_STATE_MESSAGE = "Can't stop parsing document that has not started or already completed"
DOC_STOP_PARSING_INVALID_STATE_ERROR_CODE = "DOC_STOP_PARSING_INVALID_STATE"
+@manager.route("/documents/", methods=["GET"]) # noqa: F821
+@login_required
+async def download_document(document_id):
+ """
+ Download a document.
+ ---
+ tags:
+ - Documents
+ security:
+ - ApiKeyAuth: []
+ produces:
+ - application/octet-stream
+ parameters:
+ - in: path
+ name: dataset_id
+ type: string
+ required: true
+ description: ID of the dataset.
+ - in: path
+ name: document_id
+ type: string
+ required: true
+ description: ID of the document to download.
+ - in: header
+ name: Authorization
+ type: string
+ required: true
+ description: Bearer token for authentication.
+ responses:
+ 200:
+ description: Document file stream.
+ schema:
+ type: file
+ 400:
+ description: Error message.
+ schema:
+ type: object
+ """
+ if not document_id:
+ return get_error_data_result(message="Specify document_id please.")
+ doc = DocumentService.query(id=document_id)
+ if not doc:
+ return get_error_data_result(message=f"The dataset not own the document {document_id}.")
+ # The process of downloading
+ doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address
+ file_stream = settings.STORAGE_IMPL.get(doc_id, doc_location)
+ if not file_stream:
+ return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
+ file = BytesIO(file_stream)
+ # Use send_file with a proper filename and MIME type
+ return await send_file(
+ file,
+ as_attachment=True,
+ attachment_filename=doc[0].name,
+ mimetype="application/octet-stream", # Set a default MIME type
+ )
@manager.route("/datasets//chunks", methods=["POST"]) # noqa: F821
@token_required
From d887b578c5070807cb3d57fa2122a8fc69ecc59d Mon Sep 17 00:00:00 2001
From: Octopus
Date: Fri, 15 May 2026 09:53:35 +0800
Subject: [PATCH 153/666] fix: preserve uploaded file attachments after
subsequent assistant messages (#13993)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Problem
When a user uploads a file attachment in their first message (Q1) and
then sends a follow-up message (Q2) that triggers a backend response,
the uploaded file attachment disappears from Q1 in the chat UI.
Fixes #13959
## Root Cause
In `single-chat-box.tsx`, a `useEffect` hook syncs `derivedMessages`
from `conversation?.messages` whenever the conversation data changes
(e.g., after a new assistant reply arrives):
```typescript
useEffect(() => {
const messages = conversation?.messages;
if (Array.isArray(messages)) {
setDerivedMessages(messages); // ← overwrites local state
}
}, [conversation?.messages, setDerivedMessages]);
```
The problem is that `conversation.messages` comes from the server, which
stores messages as plain JSON. Browser `File` objects (uploaded by the
user) cannot be serialized to JSON, so they are never stored on the
server. Each time the server data is applied to local state, the `files`
array on the user's first message is lost.
## Fix
Instead of replacing the local messages wholesale, preserve any `files`
entries from the previous local state by ID before applying the server
data:
```typescript
useEffect(() => {
const messages = conversation?.messages;
if (Array.isArray(messages)) {
setDerivedMessages((prevMessages) => {
const filesMap = new Map(
prevMessages
.filter((m) => m.files?.length)
.map((m) => [m.id, m.files]),
);
if (filesMap.size === 0) {
return messages;
}
return messages.map((m) => ({
...m,
files: filesMap.get(m.id) ?? m.files,
}));
});
}
}, [conversation?.messages, setDerivedMessages]);
```
This is a minimal, targeted fix: when there are no local files to
preserve the behavior is identical to before (early return with plain
assignment). When local file objects exist they are re-attached to the
corresponding server messages by ID.
## Summary by CodeRabbit
* **Bug Fixes**
* Improved search query processing to properly handle special characters
and apostrophes in search terms and synonyms.
* Fixed chat message file attachments to persist when syncing with
server.
* **Refactor**
* Simplified OCR detection return values by removing timing metadata.
---------
Co-authored-by: ximi
---
.../chat/chat-box/single-chat-box.tsx | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/web/src/pages/next-chats/chat/chat-box/single-chat-box.tsx b/web/src/pages/next-chats/chat/chat-box/single-chat-box.tsx
index 625441d3590..a598d7f3ea1 100644
--- a/web/src/pages/next-chats/chat/chat-box/single-chat-box.tsx
+++ b/web/src/pages/next-chats/chat/chat-box/single-chat-box.tsx
@@ -58,7 +58,23 @@ export function SingleChatBox({
useEffect(() => {
const messages = conversation?.messages;
if (Array.isArray(messages)) {
- setDerivedMessages(messages);
+ setDerivedMessages((prevMessages) => {
+ // Preserve uploaded file objects from local state that the server doesn't
+ // persist (e.g. File instances). Build a map of message id → files from
+ // the current local state so they survive when server data is applied.
+ const filesMap = new Map(
+ prevMessages
+ .filter((m) => m.files?.length)
+ .map((m) => [m.id, m.files]),
+ );
+ if (filesMap.size === 0) {
+ return messages;
+ }
+ return messages.map((m) => ({
+ ...m,
+ files: filesMap.get(m.id) ?? m.files,
+ }));
+ });
}
}, [conversation?.messages, setDerivedMessages]);
From ef2969a46207e050c9f7ebedbd993ea8363027f0 Mon Sep 17 00:00:00 2001
From: sham-sr <35761166+sham-sr@users.noreply.github.com>
Date: Fri, 15 May 2026 07:07:48 +0500
Subject: [PATCH 154/666] fix(llm): Tongyi-Qianwen embeddings use correct
DashScope native API for intl URLs (#14784)
## Summary
- Fixes **Tongyi-Qianwen** (`QWenEmbed`) text embeddings when the
configured `base_url` points at DashScope **international**
(`dashscope-intl.aliyuncs.com`) or **China** (`dashscope.aliyuncs.com`)
hosts, including values copied from Model Studio that use the
**OpenAI-compatible** path (`.../compatible-mode/v1`).
- The `dashscope` Python SDK (`TextEmbedding.call`) expects the
**native** HTTP root (`https:///api/v1`), not the
OpenAI-compatible base URL. Without mapping, international accounts
could hit the wrong host or path.
## Implementation
- Added `_dashscope_native_http_api_url()` to normalize known DashScope
hosts to `.../api/v1`, and wired `QWenEmbed` to set
`dashscope.base_http_api_url` before each embedding call (document and
query).
## Notes
- In-code comments document the Tongyi-Qianwen / DashScope intl vs CN
behavior for future maintainers.
---------
Co-authored-by: Cursor
---
.gitignore | 1 +
rag/llm/embedding_model.py | 92 ++++++++++++++++++++++++++++++++++++--
2 files changed, 89 insertions(+), 4 deletions(-)
diff --git a/.gitignore b/.gitignore
index f65d204fb24..097a885152b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ Cargo.lock
.idea/
.vscode/
+.cursor/settings.json
# Exclude Mac generated files
.DS_Store
diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py
index e1d0409d04d..ccaa8339010 100644
--- a/rag/llm/embedding_model.py
+++ b/rag/llm/embedding_model.py
@@ -17,6 +17,7 @@
import os
import threading
from abc import ABC
+from contextlib import contextmanager
from urllib.parse import urljoin
import dashscope
@@ -32,6 +33,76 @@
import logging
import base64
+logger = logging.getLogger(__name__)
+
+
+def _dashscope_base_url_for_log(base_url: str) -> str:
+ """Log host/path only (no query string) so secrets in URLs are not printed."""
+ return base_url.split("?", 1)[0].strip()[:256]
+
+
+def _dashscope_native_http_api_url(base_url: str | None) -> str | None:
+ """
+ Resolve the DashScope *native* HTTP API root for Tongyi-Qianwen (Qwen) text embeddings.
+
+ RAGFlow often stores an OpenAI-compatible base URL (e.g. ``.../compatible-mode/v1``) for
+ the same provider. The ``dashscope`` Python SDK used by ``TextEmbedding.call`` does *not*
+ use that path; it expects ``https:///api/v1`` instead.
+
+ Users outside mainland China are directed to the international endpoint
+ (``dashscope-intl.aliyuncs.com``); domestic traffic uses ``dashscope.aliyuncs.com``.
+ When ``base_url`` already points at the native API root (ends with ``/api/v1``), it is
+ returned unchanged so custom or regional deployments keep working.
+ """
+ if not base_url:
+ return None
+ u = base_url.strip().rstrip("/")
+ safe = _dashscope_base_url_for_log(u)
+ if u.endswith("/api/v1"):
+ logger.debug("DashScope Tongyi-Qianwen embedding: using native API base as configured (%s)", safe)
+ return u
+ # International (Singapore) DashScope — required for overseas Tongyi-Qianwen accounts.
+ if "dashscope-intl.aliyuncs.com" in u:
+ resolved = "https://dashscope-intl.aliyuncs.com/api/v1"
+ logger.info(
+ "DashScope Tongyi-Qianwen embedding: mapped configured base_url to intl native API (%s -> %s)",
+ safe,
+ resolved,
+ )
+ return resolved
+ # China mainland DashScope default host.
+ if "dashscope.aliyuncs.com" in u:
+ resolved = "https://dashscope.aliyuncs.com/api/v1"
+ logger.info(
+ "DashScope Tongyi-Qianwen embedding: mapped configured base_url to CN native API (%s -> %s)",
+ safe,
+ resolved,
+ )
+ return resolved
+ logger.warning(
+ "DashScope Tongyi-Qianwen embedding: base_url is set but not recognized as a DashScope host; "
+ "using SDK default endpoint (%s)",
+ safe,
+ )
+ return None
+
+
+@contextmanager
+def _dashscope_native_api_url_scope(url: str | None):
+ """
+ Temporarily set ``dashscope.base_http_api_url`` for the duration of a single SDK call,
+ then restore the previous value. Narrows the window where concurrent threads see a mismatch.
+ """
+ if not url:
+ yield
+ return
+ prev = getattr(dashscope, "base_http_api_url", None)
+ dashscope.base_http_api_url = url
+ try:
+ yield
+ finally:
+ dashscope.base_http_api_url = prev
+
class Base(ABC):
def __init__(self, key, model_name, **kwargs):
@@ -197,11 +268,21 @@ def __init__(self, key, model_name="Baichuan-Text-Embedding", base_url="https://
class QWenEmbed(Base):
+ """
+ Embeddings for Alibaba Tongyi-Qianwen via the DashScope ``TextEmbedding`` API.
+
+ ``base_url`` comes from the user's embedding-model configuration (often the same host
+ as the OpenAI-compatible chat endpoint). This class maps known DashScope hosts to the
+ native ``/api/v1`` base URL so international and China endpoints both work.
+ """
+
_FACTORY_NAME = "Tongyi-Qianwen"
- def __init__(self, key, model_name="text_embedding_v2", **kwargs):
+ def __init__(self, key, model_name="text_embedding_v2", base_url=None, **kwargs):
self.key = key
self.model_name = model_name
+ # Native API root for the SDK; None if base_url is absent or not a known DashScope host.
+ self._dashscope_http_api_url = _dashscope_native_http_api_url(base_url)
def encode(self, texts: list):
import time
@@ -214,10 +295,12 @@ def encode(self, texts: list):
texts = [truncate(t, 2048) for t in texts]
for i in range(0, len(texts), batch_size):
retry_max = 5
- resp = dashscope.TextEmbedding.call(model=self.model_name, input=texts[i : i + batch_size], api_key=self.key, text_type="document")
+ with _dashscope_native_api_url_scope(self._dashscope_http_api_url):
+ resp = dashscope.TextEmbedding.call(model=self.model_name, input=texts[i : i + batch_size], api_key=self.key, text_type="document")
while (resp["output"] is None or resp["output"].get("embeddings") is None) and retry_max > 0:
time.sleep(10)
- resp = dashscope.TextEmbedding.call(model=self.model_name, input=texts[i : i + batch_size], api_key=self.key, text_type="document")
+ with _dashscope_native_api_url_scope(self._dashscope_http_api_url):
+ resp = dashscope.TextEmbedding.call(model=self.model_name, input=texts[i : i + batch_size], api_key=self.key, text_type="document")
retry_max -= 1
if retry_max == 0 and (resp["output"] is None or resp["output"].get("embeddings") is None):
if resp.get("message"):
@@ -237,7 +320,8 @@ def encode(self, texts: list):
return np.array(res), token_count
def encode_queries(self, text):
- resp = dashscope.TextEmbedding.call(model=self.model_name, input=text[:2048], api_key=self.key, text_type="query")
+ with _dashscope_native_api_url_scope(self._dashscope_http_api_url):
+ resp = dashscope.TextEmbedding.call(model=self.model_name, input=text[:2048], api_key=self.key, text_type="query")
try:
return np.array(resp["output"]["embeddings"][0]["embedding"]), total_token_count_from_response(resp)
except Exception as _e:
From 4c68a6b86ce0f126b4bbbc5df9f83d74ea8833a4 Mon Sep 17 00:00:00 2001
From: yingjianzh <59633288+yingjianzh@users.noreply.github.com>
Date: Fri, 15 May 2026 10:49:14 +0800
Subject: [PATCH 155/666] fix(agent): pass top_k and fix similarity weight
slider behavior (#14760)
### What problem does this PR solve?
This PR fixes two issues in Agent Retrieval behavior and configuration
UX:
1. `top_k` configured in Agent Retrieval was not passed down to the
backend retriever call, so retrieval could ignore the configured vector
recall limit.
2. Similarity weight slider semantics were confusing in Agent forms
because the Agent field stores `keywords_similarity_weight` while UI
interactions were interpreted as vector weight. This could cause
displayed values and actual behavior to diverge.
This PR ensures Agent retrieval uses configured `top_k`, and makes the
slider behavior consistent and explicit for both vector and keyword
weight modes.
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
agent/tools/retrieval.py | 1 +
.../components/similarity-slider/index.tsx | 26 ++++++++++++++-----
.../pages/agent/form/retrieval-form/next.tsx | 3 ++-
.../form/tool-form/retrieval-form/index.tsx | 3 ++-
web/src/pages/next-search/search-setting.tsx | 2 +-
5 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/agent/tools/retrieval.py b/agent/tools/retrieval.py
index 4496f497aef..0ccf056a292 100644
--- a/agent/tools/retrieval.py
+++ b/agent/tools/retrieval.py
@@ -201,6 +201,7 @@ def _resolve_manual_filter(flt: dict) -> dict:
self._param.top_n,
self._param.similarity_threshold,
1 - self._param.keywords_similarity_weight,
+ top=self._param.top_k,
doc_ids=doc_ids,
aggs=True,
rerank_mdl=rerank_mdl,
diff --git a/web/src/components/similarity-slider/index.tsx b/web/src/components/similarity-slider/index.tsx
index c9506888690..f822bf20d7b 100644
--- a/web/src/components/similarity-slider/index.tsx
+++ b/web/src/components/similarity-slider/index.tsx
@@ -16,7 +16,8 @@ import { NumberInput } from '../ui/input';
interface SimilaritySliderFormFieldProps {
similarityName?: string;
- vectorSimilarityWeightName?: string;
+ similarityWeightName?: string;
+ similarityWeightType?: 'vector' | 'keyword';
isTooltipShown?: boolean;
numberInputClassName?: string;
}
@@ -44,14 +45,21 @@ export const initialVectorSimilarityWeightValue = {
export function SimilaritySliderFormField({
similarityName = 'similarity_threshold',
- vectorSimilarityWeightName = 'vector_similarity_weight',
+ similarityWeightName = 'vector_similarity_weight',
+ similarityWeightType = 'vector',
isTooltipShown,
numberInputClassName,
}: SimilaritySliderFormFieldProps) {
const { t } = useTranslate('knowledgeDetails');
const form = useFormContext();
- const isVector =
- vectorSimilarityWeightName.indexOf('vector_similarity_weight') > -1;
+ const isVector = similarityWeightType === 'vector';
+ const normalizeWeight = (weight: number) => Number(weight.toFixed(2));
+ const getVectorWeight = (weight: number) =>
+ normalizeWeight(isVector ? weight : 1 - weight);
+ const getFullTextWeight = (weight: number) =>
+ normalizeWeight(isVector ? 1 - weight : weight);
+ const getStoredWeight = (vectorWeight: number) =>
+ normalizeWeight(isVector ? vectorWeight : 1 - vectorWeight);
return (
<>
@@ -66,7 +74,7 @@ export function SimilaritySliderFormField({
>
(
- {field.value.toFixed(2)}
+ {getVectorWeight(field.value).toFixed(2)}
@@ -103,12 +111,14 @@ export function SimilaritySliderFormField({
full-text
- {(1 - field.value).toFixed(2)}
+ {getFullTextWeight(field.value).toFixed(2)}
field.onChange(getStoredWeight(value))}
max={1}
step={0.01}
min={0}
@@ -126,6 +136,8 @@ export function SimilaritySliderFormField({
min={0}
step={0.01}
{...field}
+ value={getVectorWeight(field.value)}
+ onChange={(value) => field.onChange(getStoredWeight(value))}
>
diff --git a/web/src/pages/agent/form/retrieval-form/next.tsx b/web/src/pages/agent/form/retrieval-form/next.tsx
index 44a6fe9ad56..6e831cedf2d 100644
--- a/web/src/pages/agent/form/retrieval-form/next.tsx
+++ b/web/src/pages/agent/form/retrieval-form/next.tsx
@@ -169,7 +169,8 @@ function RetrievalForm({ node }: INextOperatorForm) {
{t('flow.advancedSettings')} }>
diff --git a/web/src/pages/agent/form/tool-form/retrieval-form/index.tsx b/web/src/pages/agent/form/tool-form/retrieval-form/index.tsx
index 84cb0896c0c..c84f07efaa2 100644
--- a/web/src/pages/agent/form/tool-form/retrieval-form/index.tsx
+++ b/web/src/pages/agent/form/tool-form/retrieval-form/index.tsx
@@ -48,7 +48,8 @@ const RetrievalForm = () => {
{t('flow.advancedSettings')} }>
diff --git a/web/src/pages/next-search/search-setting.tsx b/web/src/pages/next-search/search-setting.tsx
index 697e7369e8d..cb658eea5e8 100644
--- a/web/src/pages/next-search/search-setting.tsx
+++ b/web/src/pages/next-search/search-setting.tsx
@@ -422,7 +422,7 @@ const SearchSetting: React.FC = ({
{/* Rerank Model */}
From 547b8cf9d84fe14f677fff1175664a44bd3351e0 Mon Sep 17 00:00:00 2001
From: Sebastion
Date: Fri, 15 May 2026 03:58:27 +0100
Subject: [PATCH 156/666] security: always use RestrictedUnpickler in
deserialize_b64 (CWE-502) (#14803)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
Harden `api/utils/configs.deserialize_b64` so that it always routes
pickle data through the existing `RestrictedUnpickler`
(`restricted_loads`) rather than falling back to bare `pickle.loads()`.
- **CWE-502** — Deserialization of Untrusted Data
- **File / function**: `api/utils/configs.py` → `deserialize_b64`
- **Caller**: `SerializedField.python_value` in `api/db/db_models.py`
(invoked by Peewee whenever a pickled DB column is read)
## The issue
Before this change, `deserialize_b64` consulted a
`use_deserialize_safe_module` config flag that **defaults to `False`**
and is not set anywhere in the repository:
```python
use_deserialize_safe_module = get_base_config('use_deserialize_safe_module', False)
if use_deserialize_safe_module:
return restricted_loads(src)
return pickle.loads(src) # <-- default path
```
So the default code path was unrestricted `pickle.loads()` on bytes read
from a MySQL `SerializedField(serialized_type=PICKLE)` column. Any
attacker who can influence those bytes (SQL injection elsewhere,
compromised DB credentials, a backup restored from an untrusted source,
or a compromised replication peer) can craft a pickle payload that
achieves arbitrary code execution on the ragflow application server when
the field is next read.
Today no model in-tree instantiates a `SerializedField` with the default
PICKLE type — only `JsonSerializedField` is used in practice — so the
attack surface is currently **latent** rather than actively reachable
through an HTTP endpoint. But the insecure-by-default behaviour is a
sharp edge: any future field that uses the default PICKLE serialization
would silently inherit RCE-on-read semantics.
## The fix
```diff
- use_deserialize_safe_module = get_base_config(
- 'use_deserialize_safe_module', False)
- if use_deserialize_safe_module:
- return restricted_loads(src)
- return pickle.loads(src)
+ return restricted_loads(src)
```
`restricted_loads` is the existing `RestrictedUnpickler` already defined
in the same file, which limits permitted modules to `numpy` and
`rag_flow`. The config flag (and the now-dead `get_base_config` import)
are removed.
Diff is 1 insertion / 6 deletions, scoped to a single function.
## Testing
- Built a malicious pickle whose `__reduce__` resolves to
`posix.system('id')`. Pre-fix: executes. Post-fix: `restricted_loads`
raises `UnpicklingError: global 'posix.system' is forbidden`.
- Round-tripped a benign `numpy.ndarray` through `serialize_b64` →
`deserialize_b64`. Values preserved bit-for-bit.
- Confirmed `use_deserialize_safe_module` is not set in any config file
in the tree, so removing the flag does not change any operator-facing
knob that was actually in use.
## A note on `restricted_loads` itself
The existing `SECURITY.md` notes that `restricted_loads`'s `numpy`
allow-list can still be reached via `numpy.f2py.diagnose.run_command`.
This PR does **not** attempt to fix that — it is a separate hardening
question about tightening the allow-list to specific symbols rather than
whole modules. The change here strictly improves on the status quo (bare
`pickle.loads`) and brings the default path in line with what the
`restricted_loads` helper was clearly designed for. Happy to follow up
with a separate PR narrowing the allow-list if that direction is
welcome.
## Adversarial review
Before submitting, we tried to argue this finding away. The two
strongest objections are (1) "no field uses PICKLE today, so this is
unreachable" — true, but the default behaviour of a security-sensitive
helper still matters because new fields silently inherit it; and (2)
"the attacker already needs DB write access, which is game over" —
partially true, but pickle-RCE meaningfully escalates *data tampering*
into *code execution on the application host* (filesystem, internal
network, in-process secrets), which is not equivalent. The fix is one
line of real code, has no behavioural cost for legitimate callers, and
removes an insecure default. We decided it was worth filing.
---
_Submitted by Sebastion — autonomous open-source security research
from [Foundation Machines](https://foundationmachines.ai). Free for
public repos via the [Sebastion AI GitHub
App](https://github.com/marketplace/sebastion-ai)._
---
api/utils/configs.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/api/utils/configs.py b/api/utils/configs.py
index 91baa28e36e..c3abc13c37f 100644
--- a/api/utils/configs.py
+++ b/api/utils/configs.py
@@ -18,7 +18,6 @@
import base64
import pickle
from api.utils.common import bytes_to_string, string_to_bytes
-from common.config_utils import get_base_config
safe_module = {
'numpy',
@@ -54,8 +53,4 @@ def deserialize_b64(src):
src = base64.b64decode(
string_to_bytes(src) if isinstance(
src, str) else src)
- use_deserialize_safe_module = get_base_config(
- 'use_deserialize_safe_module', False)
- if use_deserialize_safe_module:
- return restricted_loads(src)
- return pickle.loads(src)
+ return restricted_loads(src)
From 3a5df08c76debe0add51c598f26996a14ce35ebb Mon Sep 17 00:00:00 2001
From: Jin Hai
Date: Fri, 15 May 2026 12:29:52 +0800
Subject: [PATCH 157/666] Go: add file parse command (#14892)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
```
RAGFlow(user)> ocr with 'hunyuanocr@test@gitee' file './picture.png'
+----------------------------------------------------------+
| text |
+----------------------------------------------------------+
| 生活不是等待风暴过去,而是学会在雨中翩翩起舞。
——佚名 |
+----------------------------------------------------------+
RAGFlow(user)> list 'test@gitee' tasks;
+---------+----------------------------------+
| status | task_id |
+---------+----------------------------------+
| success | C3FX4MQNKY5MGC6ZFMIXIAMJKHCEBQB5 |
+---------+----------------------------------+
RAGFlow(user)> show 'test@gitee' task 'C3FX4MQNKY5MGC6ZFMIXIAMJKHCEBQB5';
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
| content | index |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
| # PDF 1: Purpose of RAGFlow
RAGFlow is an open source Retrieval-Augmented Generation (RAG) engine designed to turn raw documents into reliable context for large language models.Its purpose is to make it practical to build an Al assistant that can ans... | 1 |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
```
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---------
Signed-off-by: Jin Hai
---
conf/models/gitee.json | 11 +-
docs/release_notes.md | 1 +
internal/cli/client.go | 6 +
internal/cli/lexer.go | 2 +
internal/cli/response.go | 38 +++
internal/cli/types.go | 1 +
internal/cli/user_command.go | 173 ++++++++++++++
internal/cli/user_parser.go | 125 +++++++++-
internal/engine/elasticsearch/index.go | 3 +-
internal/entity/models/aliyun.go | 20 +-
internal/entity/models/baichuan.go | 15 +-
internal/entity/models/baidu.go | 20 +-
internal/entity/models/cohere.go | 20 +-
internal/entity/models/deepseek.go | 20 +-
internal/entity/models/dummy.go | 20 +-
internal/entity/models/fishaudio.go | 20 +-
internal/entity/models/gitee.go | 311 ++++++++++++++++++++++++-
internal/entity/models/google.go | 20 +-
internal/entity/models/huggingface.go | 20 +-
internal/entity/models/jina.go | 15 +-
internal/entity/models/lmstudio.go | 20 +-
internal/entity/models/localai.go | 15 +-
internal/entity/models/longcat.go | 15 +-
internal/entity/models/minimax.go | 22 +-
internal/entity/models/mistral.go | 15 +-
internal/entity/models/moonshot.go | 20 +-
internal/entity/models/novita.go | 15 +-
internal/entity/models/nvidia.go | 20 +-
internal/entity/models/ollama.go | 20 +-
internal/entity/models/openai.go | 20 +-
internal/entity/models/openrouter.go | 20 +-
internal/entity/models/siliconflow.go | 20 +-
internal/entity/models/stepfun.go | 15 +-
internal/entity/models/types.go | 64 +++--
internal/entity/models/upstage.go | 15 +-
internal/entity/models/vllm.go | 20 +-
internal/entity/models/volcengine.go | 15 +-
internal/entity/models/voyage.go | 15 +-
internal/entity/models/xai.go | 15 +-
internal/entity/models/zhipu-ai.go | 15 +-
internal/handler/memory.go | 10 +-
internal/handler/providers.go | 161 ++++++++++++-
internal/router/router.go | 3 +
internal/server/config.go | 17 ++
internal/service/model_service.go | 242 ++++++++++++++++++-
web/src/utils/api.ts | 3 +-
46 files changed, 1533 insertions(+), 160 deletions(-)
diff --git a/conf/models/gitee.json b/conf/models/gitee.json
index 8940759949f..6b1a0732e85 100644
--- a/conf/models/gitee.json
+++ b/conf/models/gitee.json
@@ -10,7 +10,10 @@
"balance": "tokens/packages/balance",
"embedding": "embedding",
"rerank": "rerank",
- "ocr": "images/ocr"
+ "ocr": "images/ocr",
+ "doc_parse": "async/documents/parse",
+ "tasks": "tasks",
+ "task": "task"
},
"models": [
{
@@ -71,6 +74,12 @@
"model_types": [
"ocr"
]
+ },
+ {
+ "name": "MinerU2.5",
+ "model_types": [
+ "doc_parse"
+ ]
}
]
}
\ No newline at end of file
diff --git a/docs/release_notes.md b/docs/release_notes.md
index 293c3910d6a..feb974f8972 100644
--- a/docs/release_notes.md
+++ b/docs/release_notes.md
@@ -36,6 +36,7 @@ Released on May 13, 2026.
### New features
+- Data source and parsing: Added column-level semantic/metadata control for the spreadsheet file parser; introduced ETag optimization for incremental synchronization of S3 data sources to avoid unnecessary file transfers.
- Enables assigning specific roles like content, metadata, and primary key, to table columns. [#13710](https://github.com/infiniflow/ragflow/pull/13710)
### Improvements
diff --git a/internal/cli/client.go b/internal/cli/client.go
index 0523b36c059..f28b59464e3 100644
--- a/internal/cli/client.go
+++ b/internal/cli/client.go
@@ -273,6 +273,8 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ASRUserCommand(cmd)
case "ocr_user_command":
return c.OCRUserCommand(cmd)
+ case "parse_file_user_command":
+ return c.ParseFileUserCommand(cmd)
case "check_provider_connection":
return c.CheckProviderConnection(cmd)
case "use_model":
@@ -285,6 +287,10 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ResetDefaultModel(cmd)
case "list_user_default_models":
return c.ListDefaultModels(cmd)
+ case "list_tasks_user_command":
+ return c.ListTasksUserCommand(cmd)
+ case "show_task_user_command":
+ return c.ShowTaskUserCommand(cmd)
// Dataset, metadata commands
case "create_dataset_table":
return c.CreateDatasetInDocEngine(cmd)
diff --git a/internal/cli/lexer.go b/internal/cli/lexer.go
index 6a0d1b0ff3b..5f0bf18287f 100644
--- a/internal/cli/lexer.go
+++ b/internal/cli/lexer.go
@@ -437,6 +437,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenRegion, Value: ident}
case "URL":
return Token{Type: TokenURL, Value: ident}
+ case "TASK":
+ return Token{Type: TokenTask, Value: ident}
case "TASKS":
return Token{Type: TokenTasks, Value: ident}
case "LOG":
diff --git a/internal/cli/response.go b/internal/cli/response.go
index b505a7a53f2..ffdd18f1d79 100644
--- a/internal/cli/response.go
+++ b/internal/cli/response.go
@@ -319,6 +319,44 @@ func (r *EmbeddingsResponse) PrintOut() {
}
}
+type SegmentResponse struct {
+ Segments []map[string]interface{} `json:"segments"`
+}
+
+type TaskResponse struct {
+ Code int `json:"code"`
+ Data map[string]interface{} `json:"data"`
+ Message string `json:"message"`
+ Duration float64
+ OutputFormat OutputFormat
+}
+
+func (r *TaskResponse) Type() string {
+ return "task"
+}
+
+func (r *TaskResponse) TimeCost() float64 {
+ return r.Duration
+}
+
+func (r *TaskResponse) SetOutputFormat(format OutputFormat) {
+ r.OutputFormat = format
+}
+
+func (r *TaskResponse) PrintOut() {
+ if r.Code == 0 {
+ segmentsRaw := r.Data["segments"].([]interface{})
+ segments := make([]map[string]interface{}, len(segmentsRaw))
+ for i, v := range segmentsRaw {
+ segments[i] = v.(map[string]interface{})
+ }
+ PrintTableSimpleByFormat(segments, r.OutputFormat)
+ } else {
+ fmt.Println("ERROR")
+ fmt.Printf("%d, %s\n", r.Code, r.Message)
+ }
+}
+
// ==================== ContextEngine Commands ====================
// ContextListResponse represents the response for ls command
diff --git a/internal/cli/types.go b/internal/cli/types.go
index 9dd32f55c7f..bbcf09a432d 100644
--- a/internal/cli/types.go
+++ b/internal/cli/types.go
@@ -152,6 +152,7 @@ const (
TokenTag
TokenRegion
TokenURL
+ TokenTask
TokenTasks
TokenLog
TokenLevel
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index f631a252759..e99912a400c 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -2284,6 +2284,179 @@ func (c *RAGFlowClient) OCRUserCommand(cmd *Command) (ResponseIf, error) {
return &result, nil
}
+func (c *RAGFlowClient) ParseFileUserCommand(cmd *Command) (ResponseIf, error) {
+ if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
+ return nil, fmt.Errorf("API token not set. Please login first")
+ }
+
+ if c.ServerType != "user" {
+ return nil, fmt.Errorf("this command is only allowed in USER mode")
+ }
+
+ var providerName, instanceName, modelName string
+
+ // Check if composite_model_name is provided in command
+ if compositeModelName, ok := cmd.Params["composite_model_name"].(string); ok && compositeModelName != "" {
+ names := strings.Split(compositeModelName, "@")
+ if len(names) != 3 {
+ return nil, fmt.Errorf("model name must be in format 'model@instance@provider'")
+ }
+ providerName = names[2]
+ instanceName = names[1]
+ modelName = names[0]
+ } else if c.CurrentModel != nil {
+ // Use current model if set
+ providerName = c.CurrentModel.Provider
+ instanceName = c.CurrentModel.Instance
+ modelName = c.CurrentModel.Model
+ } else {
+ return nil, fmt.Errorf("model name not provided and no current model set. Use 'use model' command first")
+ }
+
+ var filename string
+ var fileURL string
+ var ok bool
+ var fileContent []byte
+
+ filename, ok = cmd.Params["file"].(string)
+ if ok {
+ // read file and convert to base64
+ var err error
+ fileContent, err = os.ReadFile(filename)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read file: %w", err)
+ }
+ } else {
+ fileURL, ok = cmd.Params["url"].(string)
+ if !ok {
+ return nil, fmt.Errorf("file or url not provided")
+ }
+ }
+
+ payload := map[string]interface{}{
+ "provider_name": providerName,
+ "instance_name": instanceName,
+ "model_name": modelName,
+ }
+
+ if fileContent != nil {
+ payload["content"] = fileContent
+ } else {
+ payload["url"] = fileURL
+ }
+
+ url := "/file/parse"
+
+ resp, err := c.HTTPClient.Request("POST", url, "web", nil, payload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to PARSE document: %w", err)
+ }
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to PARSE document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+ var result CommonDataResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("PARSE document failed: invalid JSON (%w)", err)
+ }
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+ result.Duration = resp.Duration
+
+ return &result, nil
+}
+
+func (c *RAGFlowClient) ListTasksUserCommand(cmd *Command) (ResponseIf, error) {
+ if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
+ return nil, fmt.Errorf("API token not set. Please login first")
+ }
+
+ if c.ServerType != "user" {
+ return nil, fmt.Errorf("this command is only allowed in USER mode")
+ }
+
+ var providerName, instanceName string
+
+ // Check if composite_instance_name is provided in command
+ if compositeModelName, ok := cmd.Params["composite_instance_name"].(string); ok && compositeModelName != "" {
+ names := strings.Split(compositeModelName, "@")
+ if len(names) != 2 {
+ return nil, fmt.Errorf("model name must be in format 'instance@provider'")
+ }
+ providerName = names[1]
+ instanceName = names[0]
+ } else {
+ return nil, fmt.Errorf("no provider name or instance name")
+ }
+
+ url := fmt.Sprintf("/providers/%s/instances/%s/tasks", providerName, instanceName)
+
+ resp, err := c.HTTPClient.Request("GET", url, "web", nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list tasks: %w", err)
+ }
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to list tasks: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+ var result CommonResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("list tasks failed: invalid JSON (%w)", err)
+ }
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+ result.Duration = resp.Duration
+ return &result, nil
+}
+
+func (c *RAGFlowClient) ShowTaskUserCommand(cmd *Command) (ResponseIf, error) {
+ if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
+ return nil, fmt.Errorf("API token not set. Please login first")
+ }
+
+ if c.ServerType != "user" {
+ return nil, fmt.Errorf("this command is only allowed in USER mode")
+ }
+
+ var providerName, instanceName string
+
+ // Check if composite_instance_name is provided in command
+ if compositeModelName, ok := cmd.Params["composite_instance_name"].(string); ok && compositeModelName != "" {
+ names := strings.Split(compositeModelName, "@")
+ if len(names) != 2 {
+ return nil, fmt.Errorf("model name must be in format 'instance@provider'")
+ }
+ providerName = names[1]
+ instanceName = names[0]
+ } else {
+ return nil, fmt.Errorf("no provider name or instance name")
+ }
+
+ taskID, ok := cmd.Params["task_id"].(string)
+ if !ok {
+ return nil, fmt.Errorf("task id not provided")
+ }
+
+ url := fmt.Sprintf("/providers/%s/instances/%s/tasks/%s", providerName, instanceName, taskID)
+
+ resp, err := c.HTTPClient.Request("GET", url, "web", nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get task: %w", err)
+ }
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to get task: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+ var result TaskResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("get task failed: invalid JSON (%w)", err)
+ }
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+ result.Duration = resp.Duration
+ return &result, nil
+}
+
func (c *RAGFlowClient) CheckProviderConnection(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go
index 28f32071141..7b40ee9b2c1 100644
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -163,6 +163,8 @@ func (p *Parser) parseListCommand() (*Command, error) {
return NewCommand("list_user_chats"), nil
case TokenFiles:
return p.parseListFiles()
+ case TokenQuotedString:
+ return p.parseListQuotedStringCommand()
default:
return nil, fmt.Errorf("unknown LIST target: %s", p.curToken.Value)
}
@@ -280,9 +282,57 @@ func (p *Parser) parseListFiles() (*Command, error) {
return cmd, nil
}
+func (p *Parser) parseListQuotedStringCommand() (*Command, error) {
+ str, err := p.parseQuotedString()
+ if err != nil {
+ return nil, err
+ }
+ p.nextToken() // consume str
+ switch p.curToken.Type {
+ case TokenTasks:
+ p.nextToken() // consume TASKS
+ cmd := NewCommand("list_tasks_user_command")
+ cmd.Params["composite_instance_name"] = str
+ return cmd, nil
+ default:
+ return nil, fmt.Errorf("unknown command: %s", str)
+ }
+}
+
+func (p *Parser) parseShowQuotedStringCommand() (*Command, error) {
+ str, err := p.parseQuotedString()
+ if err != nil {
+ return nil, err
+ }
+ p.nextToken() // consume str
+ switch p.curToken.Type {
+ case TokenTask:
+ p.nextToken() // consume TASK
+
+ var taskID string
+ taskID, err = p.parseQuotedString()
+ if err != nil {
+ return nil, fmt.Errorf("expected string: %w", err)
+ }
+ p.nextToken()
+
+ cmd := NewCommand("show_task_user_command")
+ cmd.Params["task_id"] = taskID
+ cmd.Params["composite_instance_name"] = str
+ p.nextToken()
+
+ // Semicolon is optional
+ if p.curToken.Type == TokenSemicolon {
+ p.nextToken()
+ }
+ return cmd, nil
+ default:
+ return nil, fmt.Errorf("unknown command: %s", str)
+ }
+}
+
func (p *Parser) parseShowCommand() (*Command, error) {
p.nextToken() // consume SHOW
-
switch p.curToken.Type {
case TokenVersion:
p.nextToken()
@@ -333,6 +383,10 @@ func (p *Parser) parseShowCommand() (*Command, error) {
return p.parseShowInstance()
case TokenBalance:
return p.parseShowBalance()
+ case TokenTask:
+ return p.parseShowTask()
+ case TokenQuotedString:
+ return p.parseShowQuotedStringCommand()
default:
return nil, fmt.Errorf("unknown SHOW target: %s", p.curToken.Value)
}
@@ -1454,6 +1508,27 @@ func (p *Parser) parseShowBalance() (*Command, error) {
return cmd, nil
}
+// parseShowTask parses SHOW TASK
+func (p *Parser) parseShowTask() (*Command, error) {
+ p.nextToken() // consume TASK
+
+ taskID, err := p.parseQuotedString()
+ if err != nil {
+ return nil, fmt.Errorf("expected string: %w", err)
+ }
+ p.nextToken()
+
+ cmd := NewCommand("show_task_user_command")
+ cmd.Params["task_id"] = taskID
+ p.nextToken()
+
+ // Semicolon is optional
+ if p.curToken.Type == TokenSemicolon {
+ p.nextToken()
+ }
+ return cmd, nil
+}
+
// parseAlterInstance parses ALTER INSTANCE NAME FROM PROVIDER command
func (p *Parser) parseAlterInstance() (*Command, error) {
p.nextToken() // consume INSTANCE
@@ -2900,6 +2975,45 @@ func (p *Parser) parseOCRCommand() (*Command, error) {
return cmd, nil
}
+func (p *Parser) parseModelParseCommand() (*Command, error) {
+ p.nextToken() // consume WITH
+
+ compositeModelName, err := p.parseQuotedString()
+ if err != nil {
+ return nil, err
+ }
+ p.nextToken()
+
+ cmd := NewCommand("parse_file_user_command")
+
+ switch p.curToken.Type {
+ case TokenFile:
+ p.nextToken()
+ var file string
+ file, err = p.parseQuotedString()
+ if err != nil {
+ return nil, err
+ }
+ cmd.Params["file"] = file
+ p.nextToken()
+ case TokenURL:
+ p.nextToken()
+ var url string
+ url, err = p.parseQuotedString()
+ if err != nil {
+ return nil, err
+ }
+ cmd.Params["url"] = url
+ p.nextToken()
+ default:
+ return nil, fmt.Errorf("expected FILE or URL")
+ }
+
+ cmd.Params["composite_model_name"] = compositeModelName
+
+ return cmd, nil
+}
+
func (p *Parser) parseCheckCommand() (*Command, error) {
p.nextToken() // consume CHECK
@@ -2964,11 +3078,14 @@ func (p *Parser) parseUseCommand() (*Command, error) {
func (p *Parser) parseParseCommand() (*Command, error) {
p.nextToken() // consume PARSE
- if p.curToken.Type == TokenDataset {
+ switch p.curToken.Type {
+ case TokenDataset:
return p.parseParseDataset()
+ case TokenWith:
+ return p.parseModelParseCommand()
+ default:
+ return p.parseParseDocs()
}
-
- return p.parseParseDocs()
}
func (p *Parser) parseParseDataset() (*Command, error) {
diff --git a/internal/engine/elasticsearch/index.go b/internal/engine/elasticsearch/index.go
index 7e601acae3f..b2039691073 100644
--- a/internal/engine/elasticsearch/index.go
+++ b/internal/engine/elasticsearch/index.go
@@ -333,7 +333,7 @@ func (e *elasticsearchEngine) CreateMetadata(ctx context.Context, indexName stri
// InsertDataset inserts documents into a dataset index
func (e *elasticsearchEngine) InsertDataset(ctx context.Context, documents []map[string]interface{}, indexName string, knowledgebaseID string) ([]string, error) {
- // TODO
+ // TODO
return []string{}, nil
}
@@ -343,7 +343,6 @@ func (e *elasticsearchEngine) InsertMetadata(ctx context.Context, documents []ma
return []string{}, nil
}
-
// UpdateDataset updates a chunk by condition
func (e *elasticsearchEngine) UpdateDataset(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, tableNamePrefix string, knowledgebaseID string) error {
// TODO
diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.go
index c89a959685f..12ee525ca05 100644
--- a/internal/entity/models/aliyun.go
+++ b/internal/entity/models/aliyun.go
@@ -36,11 +36,6 @@ type AliyunModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (z *AliyunModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewAliyunModel creates a new Aliyun model instance
func NewAliyunModel(baseURL map[string]string, urlSuffix URLSuffix) *AliyunModel {
return &AliyunModel{
@@ -579,7 +574,12 @@ func (z *AliyunModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (z *AliyunModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (z *AliyunModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+// ParseFile parse file
+func (z *AliyunModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
@@ -670,3 +670,11 @@ func (z *AliyunModel) CheckConnection(apiConfig *APIConfig) error {
}
return nil
}
+
+func (z *AliyunModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *AliyunModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/baichuan.go b/internal/entity/models/baichuan.go
index 51ac9ae9d65..98c8f6752fe 100644
--- a/internal/entity/models/baichuan.go
+++ b/internal/entity/models/baichuan.go
@@ -399,7 +399,12 @@ func (z *BaichuanModel) AudioSpeechWithSender(modelName *string, audioContent *s
}
// OCRFile OCR file
-func (z *BaichuanModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (z *BaichuanModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+// ParseFile parse file
+func (z *BaichuanModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
@@ -414,3 +419,11 @@ func (b *BaichuanModel) Balance(apiConfig *APIConfig) (map[string]interface{}, e
func (b *BaichuanModel) CheckConnection(apiConfig *APIConfig) error {
return fmt.Errorf("no such method")
}
+
+func (z *BaichuanModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *BaichuanModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/baidu.go b/internal/entity/models/baidu.go
index 42576c13417..4b99ec7221d 100644
--- a/internal/entity/models/baidu.go
+++ b/internal/entity/models/baidu.go
@@ -18,11 +18,6 @@ type BaiduModel struct {
httpClient *http.Client
}
-func (b *BaiduModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
func (b *BaiduModel) NewInstance(baseURL map[string]string) ModelDriver {
return &BaiduModel{
BaseURL: baseURL,
@@ -626,10 +621,15 @@ func (z *BaiduModel) AudioSpeechWithSender(modelName *string, audioContent *stri
}
// OCRFile OCR file
-func (b *BaiduModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (b *BaiduModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", b.Name())
}
+// ParseFile parse file
+func (z *BaiduModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (b *BaiduModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
@@ -693,3 +693,11 @@ func (b *BaiduModel) CheckConnection(apiConfig *APIConfig) error {
_, err := b.ListModels(apiConfig)
return err
}
+
+func (z *BaiduModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *BaiduModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/cohere.go b/internal/entity/models/cohere.go
index c094bdd3c9e..f078737ecf3 100644
--- a/internal/entity/models/cohere.go
+++ b/internal/entity/models/cohere.go
@@ -17,11 +17,6 @@ type CoHereModel struct {
httpClient *http.Client
}
-func (c *CoHereModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
func (c *CoHereModel) NewInstance(baseURL map[string]string) ModelDriver {
return &CoHereModel{
BaseURL: baseURL,
@@ -504,10 +499,15 @@ func (z *CoHereModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (c *CoHereModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (c *CoHereModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", c.Name())
}
+// ParseFile parse file
+func (z *CoHereModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (c *CoHereModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
@@ -578,3 +578,11 @@ func (c *CoHereModel) CheckConnection(apiConfig *APIConfig) error {
_, err := c.ListModels(apiConfig)
return err
}
+
+func (z *CoHereModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *CoHereModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/deepseek.go b/internal/entity/models/deepseek.go
index 9d9a2406d0f..7c809c71cfb 100644
--- a/internal/entity/models/deepseek.go
+++ b/internal/entity/models/deepseek.go
@@ -36,11 +36,6 @@ type DeepSeekModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (z *DeepSeekModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewDeepSeekModel creates a new DeepSeek model instance
func NewDeepSeekModel(baseURL map[string]string, urlSuffix URLSuffix) *DeepSeekModel {
return &DeepSeekModel{
@@ -609,6 +604,19 @@ func (z *DeepSeekModel) AudioSpeechWithSender(modelName *string, audioContent *s
}
// OCRFile OCR file
-func (d *DeepSeekModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (d *DeepSeekModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", d.Name())
}
+
+// ParseFile parse file
+func (z *DeepSeekModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *DeepSeekModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *DeepSeekModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/dummy.go b/internal/entity/models/dummy.go
index d2fb85095f4..d80616bd04e 100644
--- a/internal/entity/models/dummy.go
+++ b/internal/entity/models/dummy.go
@@ -26,11 +26,6 @@ type DummyModel struct {
URLSuffix URLSuffix
}
-func (d *DummyModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewDummyModel creates a new Dummy AI model instance
func NewDummyModel(baseURL map[string]string, urlSuffix URLSuffix) *DummyModel {
return &DummyModel{
@@ -98,6 +93,19 @@ func (z *DummyModel) AudioSpeechWithSender(modelName *string, audioContent *stri
}
// OCRFile OCR file
-func (d *DummyModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (d *DummyModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", d.Name())
}
+
+// ParseFile parse file
+func (z *DummyModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *DummyModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *DummyModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/fishaudio.go b/internal/entity/models/fishaudio.go
index 0eec1b16539..dc185e8975c 100644
--- a/internal/entity/models/fishaudio.go
+++ b/internal/entity/models/fishaudio.go
@@ -24,11 +24,6 @@ type FishAudioModel struct {
httpClient *http.Client
}
-func (f *FishAudioModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
func NewFishAudioModel(baseURL map[string]string, urlSuffix URLSuffix) *FishAudioModel {
return &FishAudioModel{
BaseURL: baseURL,
@@ -341,10 +336,15 @@ func (f *FishAudioModel) AudioSpeechWithSender(modelName *string, audioContent *
}
// OCRFile OCR file
-func (f *FishAudioModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (f *FishAudioModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", f.Name())
}
+// ParseFile parse file
+func (z *FishAudioModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (f *FishAudioModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
@@ -445,3 +445,11 @@ func (f *FishAudioModel) CheckConnection(apiConfig *APIConfig) error {
_, err := f.ListModels(apiConfig)
return err
}
+
+func (z *FishAudioModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *FishAudioModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/gitee.go b/internal/entity/models/gitee.go
index ad66dcd96e3..70854f78b23 100644
--- a/internal/entity/models/gitee.go
+++ b/internal/entity/models/gitee.go
@@ -37,11 +37,6 @@ type GiteeModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (g *GiteeModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewGiteeModel creates a new Gitee model instance
func NewGiteeModel(baseURL map[string]string, urlSuffix URLSuffix) *GiteeModel {
return &GiteeModel{
@@ -623,7 +618,7 @@ type giteeOCRResponse struct {
}
// OCRFile OCR file
-func (g *GiteeModel) OCRFile(modelName *string, content []byte, imageURL *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (g *GiteeModel) OCRFile(modelName *string, content []byte, imageURL *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
if imageURL == nil && content == nil {
return nil, fmt.Errorf("url or content is required")
}
@@ -709,13 +704,174 @@ func (g *GiteeModel) OCRFile(modelName *string, content []byte, imageURL *string
return nil, fmt.Errorf("failed to parse response: %w", err)
}
- var ocrResponse = OCRResponse{
+ var ocrResponse = OCRFileResponse{
Text: &giteeResponse.Text,
}
return &ocrResponse, nil
}
+type giteeParseFileResponse struct {
+ TaskID string `json:"task_id"`
+ Status string `json:"status"`
+ CreatedAt int64 `json:"created_at"`
+ URLs giteeURLs `json:"urls"`
+}
+
+type giteeURLs struct {
+ Get string `json:"get"`
+ Cancel string `json:"cancel"`
+}
+
+// ParseFile parse file
+func (g *GiteeModel) ParseFile(modelName *string, content []byte, documentURL *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ if documentURL == nil && content == nil {
+ return nil, fmt.Errorf("url or content is required")
+ }
+
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL := g.BaseURL["default"]
+ if region != "default" {
+ if regional, ok := g.BaseURL[region]; ok && regional != "" {
+ baseURL = regional
+ }
+ }
+ if baseURL == "" {
+ return nil, fmt.Errorf("gitee: no base URL configured for default region")
+ }
+
+ url := fmt.Sprintf("%s/%s", strings.TrimSuffix(baseURL, "/"), g.URLSuffix.DocumentParse)
+
+ payload := &bytes.Buffer{}
+ writer := multipart.NewWriter(payload)
+
+ if err := writer.WriteField("model", *modelName); err != nil {
+ return nil, fmt.Errorf("failed to write model field: %w", err)
+ }
+
+ if documentURL != nil {
+ if err := writer.WriteField("file", *documentURL); err != nil {
+ return nil, fmt.Errorf("failed to write file URL: %w", err)
+ }
+ } else if content != nil && len(content) > 0 {
+ part, err := writer.CreateFormFile("file", "file")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create file form file: %w", err)
+ }
+ if _, err = part.Write(content); err != nil {
+ return nil, fmt.Errorf("failed to write file content: %w", err)
+ }
+ } else {
+ return nil, fmt.Errorf("file or file URL is required")
+ }
+
+ writer.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, payload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", writer.FormDataContentType())
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := g.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("gitee OCR API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var giteeParseFileResp giteeParseFileResponse
+ if err = json.Unmarshal(body, &giteeParseFileResp); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ _, err = g.getParseFile(&baseURL, apiConfig.ApiKey, &giteeParseFileResp.TaskID, 5*time.Second, 10)
+ if err != nil {
+ return nil, err
+ }
+
+ var parseFileResponse = ParseFileResponse{}
+
+ return &parseFileResponse, nil
+}
+
+type giteeGetParseFileResponse struct {
+}
+
+func (g *GiteeModel) getParseFile(baseURL *string, apiKey, taskID *string, timeOut time.Duration, count int) (*giteeGetParseFileResponse, error) {
+ url := fmt.Sprintf("%s/task/%s/status", strings.TrimSuffix(*baseURL, "/"), *taskID)
+
+ reqBody := map[string]interface{}{}
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("GET", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiKey))
+
+ var resp *http.Response
+ for i := 0; i < count; i++ {
+ resp, err = g.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ var body []byte
+ body, err = io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ // Parse response
+ var result map[string]interface{}
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ time.Sleep(timeOut)
+ }
+
+ // if resp show the file is ok, download it. otherwise, provide timeout info
+ return nil, nil
+}
+
func (g *GiteeModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig.Region != nil {
@@ -869,3 +1025,144 @@ func (g *GiteeModel) CheckConnection(apiConfig *APIConfig) error {
return nil
}
+
+type giteeTaskListResponse struct {
+ Total int `json:"total"`
+ Items []giteeTaskItem `json:"items"`
+}
+
+type giteeTaskItem struct {
+ TaskID string `json:"task_id"`
+ //Output giteeTaskOutput `json:"output"`
+ Status string `json:"status"`
+ CreatedAt int64 `json:"created_at"`
+ StartedAt int64 `json:"started_at,omitempty"`
+ CompletedAt int64 `json:"completed_at,omitempty"`
+ Price float64 `json:"price"`
+ Currency string `json:"currency"`
+ URLs giteeTaskURLs `json:"urls"`
+}
+
+type giteeTaskOutput struct {
+ Segments []giteeSegment `json:"segments"`
+}
+
+type giteeSegment struct {
+ Index int `json:"index"`
+ Content string `json:"content"`
+}
+type giteeTaskURLs struct {
+ Get string `json:"get"`
+ Cancel string `json:"cancel"`
+}
+
+func (g *GiteeModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ var region = "default"
+ if apiConfig.Region != nil {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", g.BaseURL[region], g.URLSuffix.Tasks)
+
+ // Build request body
+ reqBody := map[string]interface{}{}
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("GET", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := g.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ var body []byte
+ body, err = io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ var giteeTaskList giteeTaskListResponse
+ if err = json.Unmarshal(body, &giteeTaskList); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ taskListResp := []ListTaskStatus{}
+ for _, item := range giteeTaskList.Items {
+ taskListResp = append(taskListResp, ListTaskStatus{
+ TaskID: item.TaskID,
+ Status: item.Status,
+ })
+ }
+ return taskListResp, nil
+}
+
+func (g *GiteeModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ var region = "default"
+ if apiConfig.Region != nil {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s/%s/get", g.BaseURL[region], g.URLSuffix.Task, taskID)
+
+ // Build request body
+ reqBody := map[string]interface{}{}
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("GET", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := g.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var taskOutput giteeTaskOutput
+ if err = json.Unmarshal(body, &taskOutput); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ taskResp := &TaskResponse{}
+
+ for _, segment := range taskOutput.Segments {
+ taskResp.Segments = append(taskResp.Segments, TaskSegment{
+ Index: segment.Index,
+ Content: segment.Content,
+ })
+ }
+
+ return taskResp, nil
+}
diff --git a/internal/entity/models/google.go b/internal/entity/models/google.go
index 2e1f103dae1..5578cdadd79 100644
--- a/internal/entity/models/google.go
+++ b/internal/entity/models/google.go
@@ -77,11 +77,6 @@ type GoogleModel struct {
URLSuffix URLSuffix
}
-func (g *GoogleModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewGoogleModel creates a new Google AI model instance
func NewGoogleModel(baseURL map[string]string, urlSuffix URLSuffix) *GoogleModel {
return &GoogleModel{
@@ -364,6 +359,19 @@ func (z *GoogleModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (g *GoogleModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (g *GoogleModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", g.Name())
}
+
+// ParseFile parse file
+func (z *GoogleModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *GoogleModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *GoogleModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/huggingface.go b/internal/entity/models/huggingface.go
index 41aee10f427..a730e556355 100644
--- a/internal/entity/models/huggingface.go
+++ b/internal/entity/models/huggingface.go
@@ -19,11 +19,6 @@ type HuggingFaceModel struct {
httpClient *http.Client
}
-func (h *HuggingFaceModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewHuggingFaceModel creates a new huggingFace model instance
func NewHuggingFaceModel(baseURL map[string]string, urlSuffix URLSuffix) *HuggingFaceModel {
return &HuggingFaceModel{
@@ -435,10 +430,15 @@ func (z *HuggingFaceModel) AudioSpeechWithSender(modelName *string, audioContent
}
// OCRFile OCR file
-func (h *HuggingFaceModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (h *HuggingFaceModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", h.Name())
}
+// ParseFile parse file
+func (z *HuggingFaceModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (h *HuggingFaceModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig.Region != nil && *apiConfig.Region != "" {
@@ -502,3 +502,11 @@ func (h *HuggingFaceModel) CheckConnection(apiConfig *APIConfig) error {
_, err := h.ListModels(apiConfig)
return err
}
+
+func (z *HuggingFaceModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *HuggingFaceModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.go
index 7a9f1c02c7e..86377fa3303 100644
--- a/internal/entity/models/jina.go
+++ b/internal/entity/models/jina.go
@@ -270,6 +270,19 @@ func (z *JinaModel) AudioSpeechWithSender(modelName *string, audioContent *strin
}
// OCRFile OCR file
-func (z *JinaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (z *JinaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+// ParseFile parse file
+func (z *JinaModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *JinaModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *JinaModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/lmstudio.go b/internal/entity/models/lmstudio.go
index fa2e2dbf959..e9767a511da 100644
--- a/internal/entity/models/lmstudio.go
+++ b/internal/entity/models/lmstudio.go
@@ -20,11 +20,6 @@ type LmStudioModel struct {
httpClient *http.Client
}
-func (l *LmStudioModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewLmStudioModel
func NewLmStudioModel(baseURL map[string]string, urlSuffix URLSuffix) *LmStudioModel {
return &LmStudioModel{
@@ -471,10 +466,15 @@ func (z *LmStudioModel) AudioSpeechWithSender(modelName *string, audioContent *s
}
// OCRFile OCR file
-func (l *LmStudioModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (l *LmStudioModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", l.Name())
}
+// ParseFile parse file
+func (z *LmStudioModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
// ListModels list supported models
func (l *LmStudioModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
@@ -553,3 +553,11 @@ func (l *LmStudioModel) CheckConnection(apiConfig *APIConfig) error {
_, err := l.ListModels(apiConfig)
return err
}
+
+func (z *LmStudioModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *LmStudioModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/localai.go b/internal/entity/models/localai.go
index f5fab0df3e7..bc2b1b0300f 100644
--- a/internal/entity/models/localai.go
+++ b/internal/entity/models/localai.go
@@ -819,6 +819,19 @@ func (l *LocalAIModel) AudioSpeechWithSender(modelName *string, audioContent *st
// OCRFile: LocalAI has no OCR pipeline in its OpenAI-compatible surface;
// document parsing belongs to a different interface entirely.
-func (l *LocalAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (l *LocalAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", l.Name())
}
+
+// ParseFile parse file
+func (z *LocalAIModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *LocalAIModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *LocalAIModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/longcat.go b/internal/entity/models/longcat.go
index b9543188467..b5f194e5a23 100644
--- a/internal/entity/models/longcat.go
+++ b/internal/entity/models/longcat.go
@@ -459,6 +459,19 @@ func (l *LongCatModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile is not exposed by the LongCat API.
-func (l *LongCatModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (l *LongCatModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", l.Name())
}
+
+// ParseFile parse file
+func (z *LongCatModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *LongCatModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *LongCatModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go
index 683a8dc454e..749d0d52e83 100644
--- a/internal/entity/models/minimax.go
+++ b/internal/entity/models/minimax.go
@@ -36,11 +36,6 @@ type MinimaxModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (z *MinimaxModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewMinimaxModel creates a new Minimax model instance
func NewMinimaxModel(baseURL map[string]string, urlSuffix URLSuffix) *MinimaxModel {
return &MinimaxModel{
@@ -667,6 +662,19 @@ func (z *MinimaxModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile OCR file
-func (m *MinimaxModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *MinimaxModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
-}
\ No newline at end of file
+}
+
+// ParseFile parse file
+func (z *MinimaxModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *MinimaxModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *MinimaxModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/mistral.go b/internal/entity/models/mistral.go
index 11ba87ae97b..e1303346fab 100644
--- a/internal/entity/models/mistral.go
+++ b/internal/entity/models/mistral.go
@@ -583,6 +583,19 @@ func (z *MistralModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile OCR file
-func (z *MistralModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (z *MistralModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+// ParseFile parse file
+func (z *MistralModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *MistralModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *MistralModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/moonshot.go b/internal/entity/models/moonshot.go
index 652433b7aad..3156b49cb94 100644
--- a/internal/entity/models/moonshot.go
+++ b/internal/entity/models/moonshot.go
@@ -35,11 +35,6 @@ type MoonshotModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (m *MoonshotModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewMoonshotModel creates a new Moonshot model instance
func NewMoonshotModel(baseURL map[string]string, urlSuffix URLSuffix) *MoonshotModel {
return &MoonshotModel{
@@ -512,6 +507,19 @@ func (z *MoonshotModel) AudioSpeechWithSender(modelName *string, audioContent *s
}
// OCRFile OCR file
-func (m *MoonshotModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *MoonshotModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+
+// ParseFile parse file
+func (z *MoonshotModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *MoonshotModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *MoonshotModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/novita.go b/internal/entity/models/novita.go
index ef48886e872..5eb58efd7c7 100644
--- a/internal/entity/models/novita.go
+++ b/internal/entity/models/novita.go
@@ -658,6 +658,19 @@ func (n *NovitaModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (n *NovitaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (n *NovitaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", n.Name())
}
+
+// ParseFile parse file
+func (z *NovitaModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *NovitaModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *NovitaModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/nvidia.go b/internal/entity/models/nvidia.go
index e1cb9daaba0..bb1e6785c76 100644
--- a/internal/entity/models/nvidia.go
+++ b/internal/entity/models/nvidia.go
@@ -19,11 +19,6 @@ type NvidiaModel struct {
httpClient *http.Client
}
-func (n NvidiaModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewNvidiaModel creates a new Nvidia model instance
func NewNvidiaModel(baseURL map[string]string, urlSuffix URLSuffix) *NvidiaModel {
return &NvidiaModel{
@@ -576,10 +571,15 @@ func (z *NvidiaModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (m *NvidiaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *NvidiaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+// ParseFile parse file
+func (z *NvidiaModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
// ListModels calls /v1/models on the configured NVIDIA NIM base URL
// and returns the list of available model ids. The endpoint is
// OpenAI-compatible, so the parsing follows the same shape used by
@@ -662,3 +662,11 @@ func (n NvidiaModel) CheckConnection(apiConfig *APIConfig) error {
_, err := n.ListModels(apiConfig)
return err
}
+
+func (z *NvidiaModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *NvidiaModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/ollama.go b/internal/entity/models/ollama.go
index 01465c8405e..b4eb1b35d9e 100644
--- a/internal/entity/models/ollama.go
+++ b/internal/entity/models/ollama.go
@@ -20,11 +20,6 @@ type OllamaModel struct {
httpClient *http.Client
}
-func (o *OllamaModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewOllamaModel creates a new Ollama AI model instance
func NewOllamaModel(baseURL map[string]string, urlSuffix URLSuffix) *OllamaModel {
return &OllamaModel{
@@ -469,10 +464,15 @@ func (z *OllamaModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (m *OllamaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *OllamaModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+// ParseFile parse file
+func (z *OllamaModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (o *OllamaModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
@@ -551,3 +551,11 @@ func (o *OllamaModel) CheckConnection(apiConfig *APIConfig) error {
_, err := o.ListModels(apiConfig)
return err
}
+
+func (z *OllamaModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *OllamaModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/openai.go b/internal/entity/models/openai.go
index 0f96386dd92..e46b734bb15 100644
--- a/internal/entity/models/openai.go
+++ b/internal/entity/models/openai.go
@@ -37,11 +37,6 @@ type OpenAIModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (o *OpenAIModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewOpenAIModel creates a new OpenAI model instance.
//
// We clone http.DefaultTransport so we keep Go's defaults for
@@ -618,6 +613,19 @@ func (z *OpenAIModel) AudioSpeechWithSender(modelName *string, audioContent *str
}
// OCRFile OCR file
-func (m *OpenAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *OpenAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+
+// ParseFile parse file
+func (z *OpenAIModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *OpenAIModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *OpenAIModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/openrouter.go b/internal/entity/models/openrouter.go
index 461a1fe4c38..3b885d40f3a 100644
--- a/internal/entity/models/openrouter.go
+++ b/internal/entity/models/openrouter.go
@@ -19,11 +19,6 @@ type OpenRouterModel struct {
httpClient *http.Client
}
-func (o *OpenRouterModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewOpenRouterModel creates a new OpenRouter AI model instance
func NewOpenRouterModel(baseURL map[string]string, urlSuffix URLSuffix) *OpenRouterModel {
return &OpenRouterModel{
@@ -610,10 +605,15 @@ func (z *OpenRouterModel) AudioSpeechWithSender(modelName *string, audioContent
}
// OCRFile OCR file
-func (m *OpenRouterModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *OpenRouterModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+// ParseFile parse file
+func (z *OpenRouterModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (o *OpenRouterModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
@@ -726,3 +726,11 @@ func (o *OpenRouterModel) CheckConnection(apiConfig *APIConfig) error {
_, err := o.Balance(apiConfig)
return err
}
+
+func (z *OpenRouterModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *OpenRouterModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index b31d53898b7..8a823c2ba24 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -36,11 +36,6 @@ type SiliconflowModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (s *SiliconflowModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewSiliconflowModel creates a new Siliconflow model instance
func NewSiliconflowModel(baseURL map[string]string, urlSuffix URLSuffix) *SiliconflowModel {
return &SiliconflowModel{
@@ -745,6 +740,19 @@ func (z *SiliconflowModel) AudioSpeechWithSender(modelName *string, audioContent
}
// OCRFile OCR file
-func (m *SiliconflowModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *SiliconflowModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+
+// ParseFile parse file
+func (z *SiliconflowModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *SiliconflowModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *SiliconflowModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/stepfun.go b/internal/entity/models/stepfun.go
index d632313d986..30d226e2508 100644
--- a/internal/entity/models/stepfun.go
+++ b/internal/entity/models/stepfun.go
@@ -477,6 +477,19 @@ func (z *StepFunModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile OCR file
-func (z *StepFunModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (z *StepFunModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+// ParseFile parse file
+func (z *StepFunModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *StepFunModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *StepFunModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index b0bc9b46b08..e4123e064df 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -17,12 +17,11 @@ type ModelDriver interface {
Name() string
- // ChatWithMessages sends multiple messages with role and content
+ // ChatWithMessages sends multiple messages synchronously
ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error)
- // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
- // messages accepts []Message which supports multimodal content (e.g., [{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}])
+ // ChatStreamlyWithSender sends multiple messages asynchronously
ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error
- // Encode encodes a list of texts into embeddings
+ // Embed a list of texts into embeddings
Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error)
// Rerank calculates similarity scores between query and texts
Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error)
@@ -33,13 +32,19 @@ type ModelDriver interface {
AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error)
AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error
// OCRFile OCR file
- OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error)
+ OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error)
+ // ParseFile parse file
+ ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error)
// ListModels List supported models
ListModels(apiConfig *APIConfig) ([]string, error)
Balance(apiConfig *APIConfig) (map[string]interface{}, error)
CheckConnection(apiConfig *APIConfig) error
+
+ ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error)
+
+ ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error)
}
type ChatResponse struct {
@@ -69,24 +74,44 @@ type TTSResponse struct {
Audio []byte `json:"audio"`
}
-type OCRResponse struct {
+type OCRFileResponse struct {
Text *string `json:"text"`
}
+type ParseFileResponse struct {
+}
+
+type ListTaskStatus struct {
+ TaskID string `json:"task_id"`
+ Status string `json:"status"`
+}
+
+type TaskSegment struct {
+ Index int `json:"index"`
+ Content string `json:"content"`
+}
+
+type TaskResponse struct {
+ Segments []TaskSegment `json:"segments"`
+}
+
// URLSuffix represents the URL suffixes for different API endpoints
type URLSuffix struct {
- Chat string `json:"chat"`
- AsyncChat string `json:"async_chat"`
- AsyncResult string `json:"async_result"`
- Embedding string `json:"embedding"`
- Rerank string `json:"rerank"`
- OCR string `json:"ocr"`
- Models string `json:"models"`
- Balance string `json:"balance"`
- Files string `json:"files"`
- Status string `json:"status"`
- TTS string `json:"tts"`
- ASR string `json:"asr"`
+ Chat string `json:"chat"`
+ AsyncChat string `json:"async_chat"`
+ AsyncResult string `json:"async_result"`
+ Embedding string `json:"embedding"`
+ Rerank string `json:"rerank"`
+ TTS string `json:"tts"`
+ ASR string `json:"asr"`
+ OCR string `json:"ocr"`
+ DocumentParse string `json:"doc_parse"`
+ Models string `json:"models"`
+ Balance string `json:"balance"`
+ Files string `json:"files"`
+ Status string `json:"status"`
+ Tasks string `json:"tasks"`
+ Task string `json:"task"`
}
type ChatConfig struct {
@@ -128,6 +153,9 @@ type TTSConfig struct {
type OCRConfig struct {
}
+type ParseFileConfig struct {
+}
+
// EmbeddingModel wraps a ModelDriver with embedding-specific configuration
type EmbeddingModel struct {
ModelDriver ModelDriver
diff --git a/internal/entity/models/upstage.go b/internal/entity/models/upstage.go
index 2991bb82ce8..7e5596a902f 100644
--- a/internal/entity/models/upstage.go
+++ b/internal/entity/models/upstage.go
@@ -604,6 +604,19 @@ func (z *UpstageModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile OCR file
-func (z *UpstageModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (z *UpstageModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+// ParseFile parse file
+func (z *UpstageModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *UpstageModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *UpstageModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go
index 4933f48fc26..4780fee939a 100644
--- a/internal/entity/models/vllm.go
+++ b/internal/entity/models/vllm.go
@@ -36,11 +36,6 @@ type VllmModel struct {
httpClient *http.Client // Reusable HTTP client with connection pool
}
-func (v *VllmModel) ParseFile() {
- //TODO implement me
- panic("implement me")
-}
-
// NewVllmModel creates a new Vllm AI model instance
func NewVllmModel(baseURL map[string]string, urlSuffix URLSuffix) *VllmModel {
return &VllmModel{
@@ -576,6 +571,19 @@ func (z *VllmModel) AudioSpeechWithSender(modelName *string, audioContent *strin
}
// OCRFile OCR file
-func (m *VllmModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *VllmModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+
+// ParseFile parse file
+func (z *VllmModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *VllmModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *VllmModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/volcengine.go b/internal/entity/models/volcengine.go
index 5ab5f1f107b..4a806883ec7 100644
--- a/internal/entity/models/volcengine.go
+++ b/internal/entity/models/volcengine.go
@@ -529,10 +529,15 @@ func (z *VolcEngine) AudioSpeechWithSender(modelName *string, audioContent *stri
}
// OCRFile OCR file
-func (m *VolcEngine) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *VolcEngine) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+// ParseFile parse file
+func (z *VolcEngine) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
func (z *VolcEngine) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
@@ -627,3 +632,11 @@ func (z *VolcEngine) CheckConnection(apiConfig *APIConfig) error {
return nil
}
+
+func (z *VolcEngine) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *VolcEngine) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/voyage.go b/internal/entity/models/voyage.go
index 41d0237c7ec..14f67dda75c 100644
--- a/internal/entity/models/voyage.go
+++ b/internal/entity/models/voyage.go
@@ -371,6 +371,19 @@ func (v *VoyageModel) AudioSpeechWithSender(modelName *string, audioContent *str
return fmt.Errorf("%s, no such method", v.Name())
}
-func (v *VoyageModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (v *VoyageModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", v.Name())
}
+
+// ParseFile parse file
+func (z *VoyageModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *VoyageModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *VoyageModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/xai.go b/internal/entity/models/xai.go
index 253f1bd195e..88b465fecf3 100644
--- a/internal/entity/models/xai.go
+++ b/internal/entity/models/xai.go
@@ -512,6 +512,19 @@ func (z *XAIModel) AudioSpeechWithSender(modelName *string, audioContent *string
}
// OCRFile OCR file
-func (m *XAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *XAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+
+// ParseFile parse file
+func (z *XAIModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *XAIModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *XAIModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/entity/models/zhipu-ai.go b/internal/entity/models/zhipu-ai.go
index 5a7c88acb5c..fa86c154670 100644
--- a/internal/entity/models/zhipu-ai.go
+++ b/internal/entity/models/zhipu-ai.go
@@ -686,6 +686,19 @@ func (z *ZhipuAIModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile OCR file
-func (m *ZhipuAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
+func (m *ZhipuAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s, no such method", m.Name())
}
+
+// ParseFile parse file
+func (z *ZhipuAIModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *ZhipuAIModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
+
+func (z *ZhipuAIModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", z.Name())
+}
diff --git a/internal/handler/memory.go b/internal/handler/memory.go
index b8e04d06d84..745cd9ddcc6 100644
--- a/internal/handler/memory.go
+++ b/internal/handler/memory.go
@@ -194,7 +194,7 @@ func (h *MemoryHandler) CreateMemory(c *gin.Context) {
// Return success response
c.JSON(http.StatusOK, gin.H{
"code": common.CodeSuccess,
- "message": "success",
+ "message": "success",
"data": result,
})
}
@@ -293,7 +293,7 @@ func (h *MemoryHandler) UpdateMemory(c *gin.Context) {
// Return success response
c.JSON(http.StatusOK, gin.H{
"code": common.CodeSuccess,
- "message": "success",
+ "message": "success",
"data": result,
})
}
@@ -347,7 +347,7 @@ func (h *MemoryHandler) DeleteMemory(c *gin.Context) {
// Return success response
c.JSON(http.StatusOK, gin.H{
"code": common.CodeSuccess,
- "message": "success",
+ "message": "success",
"data": nil,
})
}
@@ -436,7 +436,7 @@ func (h *MemoryHandler) ListMemories(c *gin.Context) {
// Return success response
c.JSON(http.StatusOK, gin.H{
"code": common.CodeSuccess,
- "message": "success",
+ "message": "success",
"data": result,
})
}
@@ -490,7 +490,7 @@ func (h *MemoryHandler) GetMemoryConfig(c *gin.Context) {
// Return success response
c.JSON(http.StatusOK, gin.H{
"code": common.CodeSuccess,
- "message": "success",
+ "message": "success",
"data": result,
})
}
diff --git a/internal/handler/providers.go b/internal/handler/providers.go
index c6d61a2e653..9f7b238fc6e 100644
--- a/internal/handler/providers.go
+++ b/internal/handler/providers.go
@@ -423,6 +423,91 @@ func (h *ProviderHandler) CheckProviderConnection(c *gin.Context) {
})
}
+func (h *ProviderHandler) ListTasks(c *gin.Context) {
+ providerName := c.Param("provider_name")
+ if providerName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Provider name is required",
+ })
+ return
+ }
+
+ instanceName := c.Param("instance_name")
+ if instanceName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Instance name is required",
+ })
+ return
+ }
+
+ userID := c.GetString("user_id")
+
+ // Get tenant ID from user
+ listTaskResponse, errorCode, err := h.modelProviderService.ListTasks(providerName, instanceName, userID)
+ if err != nil {
+ c.JSON(http.StatusOK, gin.H{
+ "code": errorCode,
+ "message": err.Error(),
+ })
+ return
+ }
+
+ c.JSON(http.StatusOK, gin.H{
+ "code": 0,
+ "message": "success",
+ "data": listTaskResponse,
+ })
+}
+
+func (h *ProviderHandler) ShowTask(c *gin.Context) {
+ providerName := c.Param("provider_name")
+ if providerName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Provider name is required",
+ })
+ return
+ }
+
+ instanceName := c.Param("instance_name")
+ if instanceName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Instance name is required",
+ })
+ return
+ }
+
+ taskID := c.Param("task_id")
+ if taskID == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Task id is required",
+ })
+ return
+ }
+
+ userID := c.GetString("user_id")
+
+ // Get tenant ID from user
+ taskResponse, errorCode, err := h.modelProviderService.ShowTask(providerName, instanceName, taskID, userID)
+ if err != nil {
+ c.JSON(http.StatusOK, gin.H{
+ "code": errorCode,
+ "message": err.Error(),
+ })
+ return
+ }
+
+ c.JSON(http.StatusOK, gin.H{
+ "code": 0,
+ "message": "success",
+ "data": taskResponse,
+ })
+}
+
type AlterProviderInstanceRequest struct {
LLMName string `json:"llm_name" binding:"required"`
}
@@ -1341,7 +1426,7 @@ func (h *ProviderHandler) OCRFile(c *gin.Context) {
OCRConfig := models.OCRConfig{}
// Non-stream response
- var response *models.OCRResponse
+ var response *models.OCRFileResponse
var errorCode common.ErrorCode
var err error
@@ -1361,3 +1446,77 @@ func (h *ProviderHandler) OCRFile(c *gin.Context) {
"message": "success",
})
}
+
+type ParseFileRequest struct {
+ ProviderName *string `json:"provider_name"`
+ InstanceName *string `json:"instance_name"`
+ ModelName *string `json:"model_name"`
+ Content []byte `json:"content"`
+ URL *string `json:"url"`
+}
+
+func (h *ProviderHandler) ParseFile(c *gin.Context) {
+ var req ParseFileRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ println("JSON bind error: %v (type: %T)", err, err)
+ c.JSON(http.StatusOK, gin.H{
+ "code": common.CodeBadRequest,
+ "message": err.Error(),
+ })
+ return
+ }
+
+ if req.ProviderName == nil || *req.ProviderName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Provider name is required",
+ })
+ return
+ }
+
+ if req.InstanceName == nil || *req.InstanceName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Instance name is required",
+ })
+ return
+ }
+
+ if req.ModelName == nil || *req.ModelName == "" {
+ c.JSON(http.StatusBadRequest, gin.H{
+ "code": 400,
+ "message": "Model name is required",
+ })
+ return
+ }
+
+ userID := c.GetString("user_id")
+
+ apiConfig := models.APIConfig{
+ ApiKey: nil,
+ Region: nil,
+ }
+
+ parseFileConfig := models.ParseFileConfig{}
+
+ // Non-stream response
+ var response *models.ParseFileResponse
+ var errorCode common.ErrorCode
+ var err error
+
+ response, errorCode, err = h.modelProviderService.ParseFile(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.Content, req.URL, &apiConfig, &parseFileConfig)
+
+ if err != nil {
+ c.JSON(http.StatusOK, gin.H{
+ "code": errorCode,
+ "message": err.Error(),
+ })
+ return
+ }
+
+ c.JSON(http.StatusOK, gin.H{
+ "code": 0,
+ "data": response,
+ "message": "success",
+ })
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index 05a56ff8c8e..4354551b9a1 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -263,6 +263,8 @@ func (r *Router) Setup(engine *gin.Engine) {
provider.GET("/:provider_name/instances/:instance_name", r.providerHandler.ShowProviderInstance)
provider.GET("/:provider_name/instances/:instance_name/balance", r.providerHandler.ShowInstanceBalance)
provider.GET("/:provider_name/instances/:instance_name/connection", r.providerHandler.CheckProviderConnection)
+ provider.GET("/:provider_name/instances/:instance_name/tasks", r.providerHandler.ListTasks)
+ provider.GET("/:provider_name/instances/:instance_name/tasks/:task_id", r.providerHandler.ShowTask)
provider.PUT("/:provider_name/instances/:instance_name", r.providerHandler.AlterProviderInstance)
provider.DELETE("/:provider_name/instances", r.providerHandler.DropProviderInstance)
provider.GET("/:provider_name/instances/:instance_name/models", r.providerHandler.ListInstanceModels)
@@ -275,6 +277,7 @@ func (r *Router) Setup(engine *gin.Engine) {
v1.POST("/audio/transcriptions", r.providerHandler.TranscribeAudio)
v1.POST("/audio/speech", r.providerHandler.AudioSpeech)
v1.POST("/file/ocr", r.providerHandler.OCRFile)
+ v1.POST("/file/parse", r.providerHandler.ParseFile)
}
model := v1.Group("/models")
diff --git a/internal/server/config.go b/internal/server/config.go
index 25f1b41876c..27e97b24720 100644
--- a/internal/server/config.go
+++ b/internal/server/config.go
@@ -723,6 +723,23 @@ func FromConfigFile(configPath string) error {
}
}
+ if v.IsSet("minio_0") {
+ minioConfig := v.Sub("minio_0")
+ if minioConfig != nil {
+ if globalConfig.StorageEngine.Minio == nil {
+ globalConfig.StorageEngine.Minio = &MinioConfig{
+ Host: minioConfig.GetString("host"),
+ User: minioConfig.GetString("user"),
+ Password: minioConfig.GetString("password"),
+ Secure: minioConfig.GetBool("secure"),
+ PrefixPath: minioConfig.GetString("prefix_path"),
+ Verify: minioConfig.GetBool("verify"),
+ Bucket: minioConfig.GetString("bucket"),
+ }
+ }
+ }
+ }
+
if v.IsSet("s3") {
s3Config := v.Sub("s3")
if s3Config != nil {
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index f23f962109b..fe179d54e20 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -460,6 +460,128 @@ func (m *ModelProviderService) CheckProviderConnection(providerName, instanceNam
return common.CodeSuccess, nil
}
+func (m *ModelProviderService) ListTasks(providerName, instanceName, userID string) ([]modelModule.ListTaskStatus, common.ErrorCode, error) {
+
+ // Get tenant ID from user
+ tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner")
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ if len(tenants) == 0 {
+ return nil, common.CodeNotFound, errors.New("user has no tenants")
+ }
+
+ tenantID := tenants[0].TenantID
+
+ // Check if provider exists
+ provider, err := m.modelProviderDAO.GetByTenantIDAndProviderName(tenantID, providerName)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ instance, err := m.modelInstanceDAO.GetByProviderIDAndInstanceName(provider.ID, instanceName)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
+ if providerInfo == nil {
+ return nil, common.CodeServerError, fmt.Errorf("provider %s not found", providerName)
+ }
+
+ var extra map[string]string
+ err = json.Unmarshal([]byte(instance.Extra), &extra)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ apiConfig := &modelModule.APIConfig{
+ ApiKey: nil,
+ Region: nil,
+ }
+
+ region := extra["region"]
+ apiConfig.Region = ®ion
+ apiConfig.ApiKey = &instance.APIKey
+
+ driver := providerInfo.ModelDriver
+ if baseURL, ok := extra["base_url"]; ok && baseURL != "" {
+ newURL := map[string]string{
+ region: baseURL,
+ }
+ driver = driver.NewInstance(newURL)
+ }
+
+ var listTaskResponse []modelModule.ListTaskStatus
+ listTaskResponse, err = driver.ListTasks(apiConfig)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+ return listTaskResponse, common.CodeSuccess, nil
+}
+
+func (m *ModelProviderService) ShowTask(providerName, instanceName, taskID, userID string) (*modelModule.TaskResponse, common.ErrorCode, error) {
+
+ // Get tenant ID from user
+ tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner")
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ if len(tenants) == 0 {
+ return nil, common.CodeNotFound, errors.New("user has no tenants")
+ }
+
+ tenantID := tenants[0].TenantID
+
+ // Check if provider exists
+ provider, err := m.modelProviderDAO.GetByTenantIDAndProviderName(tenantID, providerName)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ instance, err := m.modelInstanceDAO.GetByProviderIDAndInstanceName(provider.ID, instanceName)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
+ if providerInfo == nil {
+ return nil, common.CodeServerError, fmt.Errorf("provider %s not found", providerName)
+ }
+
+ var extra map[string]string
+ err = json.Unmarshal([]byte(instance.Extra), &extra)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ apiConfig := &modelModule.APIConfig{
+ ApiKey: nil,
+ Region: nil,
+ }
+
+ region := extra["region"]
+ apiConfig.Region = ®ion
+ apiConfig.ApiKey = &instance.APIKey
+
+ driver := providerInfo.ModelDriver
+ if baseURL, ok := extra["base_url"]; ok && baseURL != "" {
+ newURL := map[string]string{
+ region: baseURL,
+ }
+ driver = driver.NewInstance(newURL)
+ }
+
+ var taskResponse *modelModule.TaskResponse
+ taskResponse, err = driver.ShowTask(taskID, apiConfig)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+ return taskResponse, common.CodeSuccess, nil
+}
+
func (m *ModelProviderService) AlterProviderInstance(providerName, instanceName, newInstanceName, apiKey, userID string) (common.ErrorCode, error) {
return common.CodeSuccess, nil
}
@@ -1518,7 +1640,7 @@ func (m *ModelProviderService) AudioSpeechStream(providerName, instanceName, mod
return common.CodeServerError, errors.New("model is disabled")
}
-func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, userID string, content []byte, url *string, apiConfig *modelModule.APIConfig, ocrConfig *modelModule.OCRConfig) (*modelModule.OCRResponse, common.ErrorCode, error) {
+func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, userID string, content []byte, url *string, apiConfig *modelModule.APIConfig, ocrConfig *modelModule.OCRConfig) (*modelModule.OCRFileResponse, common.ErrorCode, error) {
if apiConfig == nil {
apiConfig = &modelModule.APIConfig{}
}
@@ -1563,7 +1685,7 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
}
if !model.ModelTypeMap["ocr"] {
- return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not a OCR model", providerName, modelName))
+ return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not an OCR model", providerName, modelName))
}
var extra map[string]string
@@ -1576,7 +1698,7 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
apiConfig.Region = ®ion
apiConfig.ApiKey = &instance.APIKey
- var response *modelModule.OCRResponse
+ var response *modelModule.OCRFileResponse
response, err = providerInfo.ModelDriver.OCRFile(&modelName, content, url, apiConfig, ocrConfig)
if err != nil {
return nil, common.CodeServerError, err
@@ -1589,7 +1711,7 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
}
if modelInfo.Status == "active" {
- if modelInfo.ModelType != "tts" {
+ if modelInfo.ModelType != "ocr" {
return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is an OCR model", modelName, providerName))
}
// For local deployed models
@@ -1613,7 +1735,7 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
}
newProviderInfo := providerInfo.ModelDriver.NewInstance(newURL)
- var response *modelModule.OCRResponse
+ var response *modelModule.OCRFileResponse
response, err = newProviderInfo.OCRFile(&modelName, content, url, apiConfig, ocrConfig)
if err != nil {
return nil, common.CodeServerError, err
@@ -1628,6 +1750,116 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
return nil, common.CodeServerError, errors.New("model is disabled")
}
+func (m *ModelProviderService) ParseFile(providerName, instanceName, modelName, userID string, content []byte, url *string, apiConfig *modelModule.APIConfig, parseFileConfig *modelModule.ParseFileConfig) (*modelModule.ParseFileResponse, common.ErrorCode, error) {
+ if apiConfig == nil {
+ apiConfig = &modelModule.APIConfig{}
+ }
+ if parseFileConfig == nil {
+ parseFileConfig = &modelModule.ParseFileConfig{}
+ }
+
+ // Get tenant ID from user
+ tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner")
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ if len(tenants) == 0 {
+ return nil, common.CodeNotFound, errors.New("user has no tenants")
+ }
+
+ tenantID := tenants[0].TenantID
+
+ // Check if provider exists
+ provider, err := m.modelProviderDAO.GetByTenantIDAndProviderName(tenantID, providerName)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ instance, err := m.modelInstanceDAO.GetByProviderIDAndInstanceName(provider.ID, instanceName)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ modelInfo, err := m.modelDAO.GetModelByProviderIDAndInstanceIDAndModelName(provider.ID, instance.ID, modelName)
+ if err != nil {
+ providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
+ if providerInfo == nil {
+ return nil, common.CodeNotFound, errors.New("provider not found")
+ }
+
+ var model *entity.Model = nil
+ model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
+ if err != nil {
+ return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s not found", providerName, modelName))
+ }
+
+ if !model.ModelTypeMap["doc_parse"] {
+ return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not a Document Parse model", providerName, modelName))
+ }
+
+ var extra map[string]string
+ err = json.Unmarshal([]byte(instance.Extra), &extra)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ region := extra["region"]
+ apiConfig.Region = ®ion
+ apiConfig.ApiKey = &instance.APIKey
+
+ var response *modelModule.ParseFileResponse
+ response, err = providerInfo.ModelDriver.ParseFile(&modelName, content, url, apiConfig, parseFileConfig)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+ if response == nil {
+ return nil, common.CodeServerError, errors.New("empty chat response")
+ }
+
+ return response, common.CodeSuccess, nil
+ }
+
+ if modelInfo.Status == "active" {
+ if modelInfo.ModelType != "doc_parse" {
+ return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is a Document Parse model", modelName, providerName))
+ }
+ // For local deployed models
+ providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
+ if providerInfo == nil {
+ return nil, common.CodeNotFound, errors.New("provider not found")
+ }
+
+ var extra map[string]string
+ err = json.Unmarshal([]byte(instance.Extra), &extra)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+
+ region := extra["region"]
+ apiConfig.Region = ®ion
+ apiConfig.ApiKey = &instance.APIKey
+
+ newURL := map[string]string{
+ region: extra["base_url"],
+ }
+ newProviderInfo := providerInfo.ModelDriver.NewInstance(newURL)
+
+ var response *modelModule.ParseFileResponse
+ response, err = newProviderInfo.ParseFile(&modelName, content, url, apiConfig, parseFileConfig)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+ if response == nil {
+ return nil, common.CodeServerError, errors.New("empty chat response")
+ }
+
+ return response, common.CodeSuccess, nil
+ }
+
+ return nil, common.CodeServerError, errors.New("model is disabled")
+}
+
// GetEmbeddingModel returns an EmbeddingModel wrapper for the given tenant
func (m *ModelProviderService) GetEmbeddingModel(tenantID, compositeModelName string) (*modelModule.EmbeddingModel, error) {
driver, modelName, apiConfig, maxTokens, err := m.getModelConfig(tenantID, compositeModelName)
diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts
index f3d6456c478..32a3d5bfd62 100644
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -192,8 +192,7 @@ export default {
listAgentTemplate: `${restAPIv1}/agents/templates`,
listAgents: `${restAPIv1}/agents`,
listAgentTags: `${restAPIv1}/agents/tags`,
- updateAgentTags: (agentId: string) =>
- `${restAPIv1}/agents/${agentId}/tags`,
+ updateAgentTags: (agentId: string) => `${restAPIv1}/agents/${agentId}/tags`,
createAgent: `${restAPIv1}/agents`,
updateAgent: (agentId: string) => `${restAPIv1}/agents/${agentId}`,
deleteAgent: (agentId: string) => `${restAPIv1}/agents/${agentId}`,
From c9622d0924beb773e9fe5d28a288bcb020538e9b Mon Sep 17 00:00:00 2001
From: plind <59729252+plind-junior@users.noreply.github.com>
Date: Thu, 14 May 2026 21:42:33 -0700
Subject: [PATCH 158/666] fix(agentbot): aggregate structured output in
non-streaming completions (#14848)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## What problem does this PR solve?
Closes #13384.
The `/api/v1/agentbots//completions` non-streaming path
returned the first yielded SSE chunk and exited:
```python
async for answer in agent_completion(objs[0].tenant_id, agent_id, **req):
return get_result(data=answer)
```
That meant structured output, the full assistant message, and reference
data were all dropped when an agent was called with `stream=false`.
Streaming worked because each event was forwarded individually;
non-streaming was returning a raw SSE-formatted string from a single
early event.
The v1 endpoint at
[`agent_api.py:1006-1050`](https://github.com/infiniflow/ragflow/blob/main/api/apps/restful_apis/agent_api.py#L1006-L1050)
already handles this correctly. This PR mirrors that aggregation in the
SDK beta endpoint: parse each SSE line, accumulate `content` from
`message` events, merge `reference`, collect `outputs.structured` from
each `node_finished` event keyed by `component_id`, and attach all of
them to the final response.
## Type of change
- [x] Bug fix (non-breaking change which fixes an issue)
## Test plan
- [ ] Build an agent with a node that emits structured output, call
`POST /api/v1/agentbots//completions` with `stream=false` and
a beta API token, verify `data.structured.` is present in
the response.
- [ ] Same agent with `stream=true` — verify behavior is unchanged.
- [ ] Agent without structured output — verify `data.structured` is
omitted, `content` and `reference` still aggregated correctly.
---
api/apps/sdk/session.py | 48 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 46 insertions(+), 2 deletions(-)
diff --git a/api/apps/sdk/session.py b/api/apps/sdk/session.py
index 7ba6fbd81d2..394ba71e905 100644
--- a/api/apps/sdk/session.py
+++ b/api/apps/sdk/session.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import copy
import json
import re
@@ -293,13 +294,56 @@ async def stream():
return resp
try:
+ full_content = ""
+ reference = {}
+ structured_output = {}
+ final_ans = {}
async for answer in agent_completion(objs[0].tenant_id, agent_id, **req):
- return get_result(data=answer)
+ # agent_completion yields SSE-formatted strings. A single yielded
+ # chunk can contain multiple "data:..." frames separated by "\n\n"
+ # plus blank or comment lines, so parse line-by-line rather than
+ # assuming one frame per chunk.
+ if not isinstance(answer, str):
+ continue
+ for line in answer.splitlines():
+ line = line.strip()
+ if not line.startswith("data:"):
+ continue
+ payload = line[len("data:"):].strip()
+ if not payload:
+ continue
+ try:
+ ans = json.loads(payload)
+ except Exception as e:
+ logging.debug("agent_bot_completions: skipping malformed SSE frame: %s", e)
+ continue
+ event = ans.get("event")
+ if event == "message":
+ full_content += ans.get("data", {}).get("content", "") or ""
+ if ans.get("data", {}).get("reference"):
+ reference.update(ans["data"]["reference"])
+ if event == "node_finished":
+ data = ans.get("data", {})
+ node_out = data.get("outputs") or {}
+ component_id = data.get("component_id")
+ if component_id is not None and "structured" in node_out:
+ structured_output[component_id] = copy.deepcopy(node_out["structured"])
+ final_ans = ans
+
+ if not final_ans:
+ return get_result(data={})
+
+ if "data" not in final_ans or not isinstance(final_ans["data"], dict):
+ final_ans["data"] = {}
+ final_ans["data"]["content"] = full_content
+ final_ans["data"]["reference"] = reference
+ if structured_output:
+ final_ans["data"]["structured"] = structured_output
+ return get_result(data=final_ans)
except Exception as e:
logging.exception(e)
return get_error_data_result(message=str(e) or "Unknown error")
- return None
@manager.route("/agentbots//inputs", methods=["GET"]) # noqa: F821
async def begin_inputs(agent_id):
From eaa5d9921bbc0f0c4c8c601f91d8392837871493 Mon Sep 17 00:00:00 2001
From: Octopus
Date: Fri, 15 May 2026 13:26:31 +0800
Subject: [PATCH 159/666] fix: enable GitHub connector to sync PRs and issues
by default (#14062)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Fixes #13975
## Problem
The GitHub data source connector had both `include_pull_requests` and
`include_issues` defaulting to `false` in both the frontend form and the
backend sync code. This meant that with the default configuration, **no
content was synced at all** from a GitHub repository — silently
producing zero results.
Additionally, the form field labels contained a typo: "Inlcude" instead
of "Include".
## Solution
- Changed `include_pull_requests` default from `false` to `true` in the
frontend form fields and default values
- Changed `include_issues` default from `false` to `true` in the
frontend form fields and default values
- Changed both backend defaults in `sync_data_source.py` from `False` to
`True`
- Fixed label typos: "Inlcude Pull Requests" → "Include Pull Requests"
and "Inlcude Issues" → "Include Issues"
This makes the GitHub connector consistent with the GitLab connector,
which already defaults `include_mrs`, `include_issues`, and
`include_code_files` all to `true`.
## Testing
- The connector now syncs both pull requests and issues by default when
a new GitHub data source is created
- Users who want to exclude PRs or issues can uncheck the corresponding
checkboxes in the form
Co-authored-by: octo-patch
---
rag/svr/sync_data_source.py | 4 ++--
.../user-setting/data-source/constant/index.tsx | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py
index 7d8ccdadc2b..8d397fc2d67 100644
--- a/rag/svr/sync_data_source.py
+++ b/rag/svr/sync_data_source.py
@@ -1235,8 +1235,8 @@ async def _generate(self, task: dict):
self.connector = GithubConnector(
repo_owner=self.conf.get("repository_owner"),
repositories=self.conf.get("repository_name"),
- include_prs=self.conf.get("include_pull_requests", False),
- include_issues=self.conf.get("include_issues", False),
+ include_prs=self.conf.get("include_pull_requests", True),
+ include_issues=self.conf.get("include_issues", True),
)
credentials = self.conf.get("credentials", {})
diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx
index d00db49e166..f1d2bb4aceb 100644
--- a/web/src/pages/user-setting/data-source/constant/index.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.tsx
@@ -856,18 +856,18 @@ export const DataSourceFormFields = {
required: true,
},
{
- label: 'Inlcude Pull Requests',
+ label: 'Include Pull Requests',
name: 'config.include_pull_requests',
type: FormFieldType.Checkbox,
required: false,
- defaultValue: false,
+ defaultValue: true,
},
{
- label: 'Inlcude Issues',
+ label: 'Include Issues',
name: 'config.include_issues',
type: FormFieldType.Checkbox,
required: false,
- defaultValue: false,
+ defaultValue: true,
},
],
[DataSourceKey.IMAP]: [
@@ -1622,8 +1622,8 @@ export const DataSourceFormDefaultValues = {
config: {
repository_owner: '',
repository_name: '',
- include_pull_requests: false,
- include_issues: false,
+ include_pull_requests: true,
+ include_issues: true,
credentials: {
github_access_token: '',
},
From 86bcf9767d3311341502f76a8337480b38a551f9 Mon Sep 17 00:00:00 2001
From: Hunnyboy1217 <110440428+hunnyboy1217@users.noreply.github.com>
Date: Thu, 14 May 2026 22:27:22 -0700
Subject: [PATCH 160/666] Go: implement Rerank in vLLM driver (#14878) (#14880)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Closes #14878.
`VllmModel.Rerank()` in
[internal/entity/models/vllm.go:551](internal/entity/models/vllm.go#L551)
is currently a stub returning `nil, fmt.Errorf("%s, Rerank not
implemented", z.Name())`, and
[conf/models/vllm.json](conf/models/vllm.json) is missing a `rerank`
entry in `url_suffix`. Chat (long-standing) and embeddings (#14688)
already work, so rerank is the last missing leg of the retrieval
pipeline for operators running everything on a single self-hosted vLLM
server — today they have to point rerank at a different provider, which
defeats the point of a fully local deployment.
Upstream vLLM has supported a Jina/Cohere-compatible `POST /v1/rerank`
endpoint since v0.7
([vllm-project/vllm#12376](https://github.com/vllm-project/vllm/pull/12376)).
The request/response shape is essentially identical to the NVIDIA driver
landed in #14778, so this PR mirrors that structure with two
vLLM-specific adjustments.
This PR replaces the stub with a real implementation against vLLM's
`/v1/rerank`:
- `POST {baseURL}/rerank`
- Request body: `{"model": "", "query": "",
"documents": [...], "top_n": }` — documents are a flat `[]string`,
**not** wrapped as `{text: "..."}` like NVIDIA's `/ranking`.
- Response body: `{"results": [{"index": int, "relevance_score": float},
...]}` (Jina-compatible; the optional `document` field is ignored since
callers reconstruct text via `Index`).
- `Authorization: Bearer ` is set **only when `APIConfig.ApiKey`
is non-empty**, matching the existing `Embed`/`ListModels` behaviour in
this file. vLLM is a local driver and can be deployed without an API
key.
The return shape matches the existing `*RerankResponse` contract used by
the NVIDIA ([nvidia.go:461](internal/entity/models/nvidia.go#L461)),
Aliyun ([aliyun.go:507](internal/entity/models/aliyun.go#L507)), and
ZhipuAI ([zhipu-ai.go:554](internal/entity/models/zhipu-ai.go#L554))
drivers, i.e. `Data []RerankResult` carrying `{Index, RelevanceScore}`
in the API's ranking order. Callers that need original-input order sort
by `Index`.
Behaviour requirements from the issue, all covered:
1. Empty `documents` → returns `&RerankResponse{}` without an HTTP call.
2. Missing `modelName` → `"model name is required"` validation error.
3. `rerankConfig.TopN` honored when `0 < TopN < len(documents)`;
otherwise `top_n` defaults to `len(documents)` so callers get a score
per input.
4. Non-200 responses return an error including upstream status and body
(`"vLLM rerank API error: , body: "`).
5. Response `index` values are bounds-checked against `len(documents)`.
**Scope:**
- [internal/entity/models/vllm.go](internal/entity/models/vllm.go) —
replaces the `Rerank` stub at line 551 with a real implementation; adds
`vllmRerankRequest`/`vllmRerankResponse` types for the slim subset of
the payload we need. Region/baseURL resolution, 30s context timeout,
conditional bearer header, and error wrapping all follow the existing
patterns in this file.
- [conf/models/vllm.json](conf/models/vllm.json) — adds `"rerank":
"rerank"` to `url_suffix`, joined to the operator-configured vLLM base
URL the same way the NVIDIA driver joins at
[nvidia.go:485](internal/entity/models/nvidia.go#L485).
-
[internal/entity/models/vllm_rerank_test.go](internal/entity/models/vllm_rerank_test.go)
— adds 7 `httptest`-backed tests mirroring `nvidia_rerank_test.go`:
happy path (out-of-order ranking → Index preservation), `top_n` clamp to
`RerankConfig.TopN`, empty-documents short-circuit, missing-model-name
validation, HTTP error propagation, out-of-range index rejection, and a
vLLM-specific `TestVllmRerankWithoutAPIKey` locking in the optional-auth
behaviour that distinguishes this driver from NVIDIA.
**Out of scope:** no interface change, no DDL, no frontend change. Chat,
embeddings, and balance paths are untouched. No new user-facing docs
required beyond the existing rerank model setup page — vLLM joins the
list of providers whose rerank model can be selected once `/v1/rerank`
is exposed by the server.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---
conf/models/vllm.json | 3 +-
internal/entity/models/vllm.go | 115 +++++++++++-
internal/entity/models/vllm_rerank_test.go | 209 +++++++++++++++++++++
3 files changed, 324 insertions(+), 3 deletions(-)
create mode 100644 internal/entity/models/vllm_rerank_test.go
diff --git a/conf/models/vllm.json b/conf/models/vllm.json
index 9c6a440a87f..d4f074330b3 100644
--- a/conf/models/vllm.json
+++ b/conf/models/vllm.json
@@ -3,7 +3,8 @@
"url_suffix": {
"chat": "chat/completions",
"models": "models",
- "embedding": "embeddings"
+ "embedding": "embeddings",
+ "rerank": "rerank"
},
"class": "local"
}
\ No newline at end of file
diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go
index 4780fee939a..61452c9bcaa 100644
--- a/internal/entity/models/vllm.go
+++ b/internal/entity/models/vllm.go
@@ -547,9 +547,120 @@ func (z *VllmModel) CheckConnection(apiConfig *APIConfig) error {
return err
}
-// Rerank calculates similarity scores between query and documents
+// vllmRerankRequest mirrors vLLM's Jina/Cohere-compatible /v1/rerank
+// payload. Unlike NVIDIA NIM (which wraps each passage as {text: "..."}),
+// vLLM accepts documents as a flat []string.
+type vllmRerankRequest struct {
+ Model string `json:"model"`
+ Query string `json:"query"`
+ Documents []string `json:"documents"`
+ TopN int `json:"top_n"`
+}
+
+// vllmRerankResponse maps the Jina-style results array. The `document`
+// field is intentionally ignored — callers reconstruct text from the
+// original input via Index.
+type vllmRerankResponse struct {
+ Results []struct {
+ Index int `json:"index"`
+ RelevanceScore float64 `json:"relevance_score"`
+ } `json:"results"`
+}
+
+// Rerank scores documents against the query using a vLLM rerank model
+// served at /v1/rerank (stable since vLLM v0.7). Mirrors the contract
+// of NvidiaModel.Rerank: defaults top_n to len(documents) so callers
+// get a score per input, shrinks to RerankConfig.TopN only when set
+// and smaller. Returned RerankResult entries are in the API's ranking
+// order; callers that need original-input order sort by Index. The
+// Authorization header is sent only when APIConfig.ApiKey is non-empty,
+// matching the existing Embed/ListModels behaviour for this local
+// driver.
func (z *VllmModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
- return nil, fmt.Errorf("%s, Rerank not implemented", z.Name())
+ if len(documents) == 0 {
+ return &RerankResponse{}, nil
+ }
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+
+ region := "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL := z.BaseURL[region]
+ if baseURL == "" {
+ baseURL = z.BaseURL["default"]
+ }
+ if baseURL == "" {
+ return nil, fmt.Errorf("missing base URL: please configure the local access address for vLLM (e.g., http://127.0.0.1:8000/v1)")
+ }
+
+ url := fmt.Sprintf("%s/%s", strings.TrimSuffix(baseURL, "/"), z.URLSuffix.Rerank)
+
+ topN := len(documents)
+ if rerankConfig != nil && rerankConfig.TopN > 0 && rerankConfig.TopN < topN {
+ topN = rerankConfig.TopN
+ }
+
+ reqBody := vllmRerankRequest{
+ Model: *modelName,
+ Query: query,
+ Documents: documents,
+ TopN: topN,
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ if apiConfig != nil && apiConfig.ApiKey != nil && *apiConfig.ApiKey != "" {
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ }
+
+ resp, err := z.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("vLLM rerank API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var parsed vllmRerankResponse
+ if err = json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ rerankResponse := RerankResponse{Data: make([]RerankResult, 0, len(parsed.Results))}
+ for _, r := range parsed.Results {
+ if r.Index < 0 || r.Index >= len(documents) {
+ return nil, fmt.Errorf("unexpected rerank index %d for %d inputs", r.Index, len(documents))
+ }
+ rerankResponse.Data = append(rerankResponse.Data, RerankResult{
+ Index: r.Index,
+ RelevanceScore: r.RelevanceScore,
+ })
+ }
+
+ return &rerankResponse, nil
}
// TranscribeAudio transcribe audio
diff --git a/internal/entity/models/vllm_rerank_test.go b/internal/entity/models/vllm_rerank_test.go
new file mode 100644
index 00000000000..42fda948c2f
--- /dev/null
+++ b/internal/entity/models/vllm_rerank_test.go
@@ -0,0 +1,209 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func newVllmRerankServer(t *testing.T, expectAuth string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ t.Errorf("expected POST, got %s", r.Method)
+ return
+ }
+ if r.URL.Path != "/rerank" {
+ t.Errorf("expected path=/rerank, got %s", r.URL.Path)
+ return
+ }
+ if got := r.Header.Get("Authorization"); got != expectAuth {
+ t.Errorf("expected Authorization=%q, got %q", expectAuth, got)
+ return
+ }
+ if got := r.Header.Get("Content-Type"); got != "application/json" {
+ t.Errorf("expected Content-Type=application/json, got %q", got)
+ return
+ }
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("failed to read body: %v", err)
+ return
+ }
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("invalid JSON body: %v\n%s", err, string(raw))
+ return
+ }
+ handler(t, body, w)
+ }))
+}
+
+func newVllmModelForTest(baseURL string) *VllmModel {
+ return NewVllmModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{Rerank: "rerank"},
+ )
+}
+
+func TestVllmRerankHappyPath(t *testing.T) {
+ srv := newVllmRerankServer(t, "Bearer test-key", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "BAAI/bge-reranker-v2-m3" {
+ t.Errorf("expected model=BAAI/bge-reranker-v2-m3, got %v", body["model"])
+ }
+ if body["query"] != "What is RAPTOR?" {
+ t.Errorf("expected query=What is RAPTOR?, got %v", body["query"])
+ }
+ // vLLM differs from NVIDIA: documents is a flat []string, not [{text}].
+ docs, ok := body["documents"].([]interface{})
+ if !ok || len(docs) != 3 {
+ t.Errorf("expected 3 documents, got %v", body["documents"])
+ return
+ }
+ for i, want := range []string{"doc-zero", "doc-one", "doc-two"} {
+ if docs[i] != want {
+ t.Errorf("documents[%d]=%v, want %s", i, docs[i], want)
+ }
+ }
+ if body["top_n"] != float64(3) {
+ t.Errorf("expected top_n=3 (matching len(documents)), got %v", body["top_n"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "results": []map[string]interface{}{
+ {"index": 2, "relevance_score": 0.95},
+ {"index": 0, "relevance_score": 0.42},
+ {"index": 1, "relevance_score": 0.78},
+ },
+ })
+ })
+ defer srv.Close()
+
+ model := newVllmModelForTest(srv.URL)
+ apiKey := "test-key"
+ modelName := "BAAI/bge-reranker-v2-m3"
+ resp, err := model.Rerank(
+ &modelName,
+ "What is RAPTOR?",
+ []string{"doc-zero", "doc-one", "doc-two"},
+ &APIConfig{ApiKey: &apiKey},
+ &RerankConfig{},
+ )
+ if err != nil {
+ t.Fatalf("Rerank failed: %v", err)
+ }
+ if len(resp.Data) != 3 {
+ t.Fatalf("expected 3 results, got %d", len(resp.Data))
+ }
+ want := map[int]float64{0: 0.42, 1: 0.78, 2: 0.95}
+ for _, r := range resp.Data {
+ if got, ok := want[r.Index]; !ok || got != r.RelevanceScore {
+ t.Errorf("unexpected result Index=%d RelevanceScore=%v", r.Index, r.RelevanceScore)
+ }
+ }
+}
+
+func TestVllmRerankTopNClamp(t *testing.T) {
+ srv := newVllmRerankServer(t, "Bearer test-key", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["top_n"] != float64(2) {
+ t.Errorf("expected top_n clamp to RerankConfig.TopN=2, got %v", body["top_n"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"results": []map[string]interface{}{}})
+ })
+ defer srv.Close()
+
+ model := newVllmModelForTest(srv.URL)
+ apiKey := "test-key"
+ modelName := "BAAI/bge-reranker-v2-m3"
+ if _, err := model.Rerank(
+ &modelName, "q",
+ []string{"a", "b", "c", "d"},
+ &APIConfig{ApiKey: &apiKey},
+ &RerankConfig{TopN: 2},
+ ); err != nil {
+ t.Fatalf("Rerank failed: %v", err)
+ }
+}
+
+func TestVllmRerankEmptyDocuments(t *testing.T) {
+ model := newVllmModelForTest("http://unused")
+ apiKey := "test-key"
+ modelName := "BAAI/bge-reranker-v2-m3"
+ resp, err := model.Rerank(&modelName, "q", nil, &APIConfig{ApiKey: &apiKey}, &RerankConfig{})
+ if err != nil {
+ t.Fatalf("expected nil error for empty documents, got %v", err)
+ }
+ if len(resp.Data) != 0 {
+ t.Errorf("expected empty Data, got %d entries", len(resp.Data))
+ }
+}
+
+// vLLM is a local driver; the Authorization header must be omitted when
+// no APIConfig.ApiKey is configured. This diverges from the NVIDIA driver
+// which requires an API key.
+func TestVllmRerankWithoutAPIKey(t *testing.T) {
+ srv := newVllmRerankServer(t, "", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "results": []map[string]interface{}{
+ {"index": 0, "relevance_score": 0.5},
+ },
+ })
+ })
+ defer srv.Close()
+
+ model := newVllmModelForTest(srv.URL)
+ modelName := "BAAI/bge-reranker-v2-m3"
+ resp, err := model.Rerank(&modelName, "q", []string{"a"}, &APIConfig{}, &RerankConfig{})
+ if err != nil {
+ t.Fatalf("Rerank failed without api key: %v", err)
+ }
+ if len(resp.Data) != 1 || resp.Data[0].Index != 0 {
+ t.Errorf("unexpected response: %+v", resp)
+ }
+}
+
+func TestVllmRerankRequiresModelName(t *testing.T) {
+ model := newVllmModelForTest("http://unused")
+ apiKey := "test-key"
+ _, err := model.Rerank(nil, "q", []string{"a"}, &APIConfig{ApiKey: &apiKey}, &RerankConfig{})
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("expected model-name error, got %v", err)
+ }
+}
+
+func TestVllmRerankRejectsHTTPError(t *testing.T) {
+ srv := newVllmRerankServer(t, "Bearer test-key", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusInternalServerError)
+ _, _ = w.Write([]byte(`{"error":"boom"}`))
+ })
+ defer srv.Close()
+
+ model := newVllmModelForTest(srv.URL)
+ apiKey := "test-key"
+ modelName := "BAAI/bge-reranker-v2-m3"
+ _, err := model.Rerank(&modelName, "q", []string{"a"}, &APIConfig{ApiKey: &apiKey}, &RerankConfig{})
+ if err == nil || !strings.Contains(err.Error(), "vLLM rerank API error") {
+ t.Errorf("expected API error, got %v", err)
+ }
+}
+
+func TestVllmRerankRejectsOutOfRangeIndex(t *testing.T) {
+ srv := newVllmRerankServer(t, "Bearer test-key", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "results": []map[string]interface{}{
+ {"index": 5, "relevance_score": 0.9},
+ },
+ })
+ })
+ defer srv.Close()
+
+ model := newVllmModelForTest(srv.URL)
+ apiKey := "test-key"
+ modelName := "BAAI/bge-reranker-v2-m3"
+ _, err := model.Rerank(&modelName, "q", []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, &RerankConfig{})
+ if err == nil || !strings.Contains(err.Error(), "unexpected rerank index") {
+ t.Errorf("expected out-of-range error, got %v", err)
+ }
+}
From 1a25191b138e7fc15c3c27f9c715865ef103eb35 Mon Sep 17 00:00:00 2001
From: "SnakeEye-sudo (Er. Sangam Krishna)"
Date: Fri, 15 May 2026 11:24:09 +0530
Subject: [PATCH 161/666] docs: add FAQ entry for using Ollama with RAGFlow
(#14557)
### What problem does this PR solve?
Users frequently ask how to use Ollama for local LLM inference with
RAGFlow. This FAQ entry provides step-by-step instructions for setting
up Ollama as a local model provider.
### Type of change
- [x] Documentation update
### Description
Adds a new FAQ entry: "How do I use Ollama with RAGFlow for local LLM
inference?"
Covers:
1. Starting Ollama and pulling a model
2. Configuring Ollama as a model provider in RAGFlow Settings
3. Using the Ollama model in an assistant
---
docs/faq.mdx | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/docs/faq.mdx b/docs/faq.mdx
index 391b16c5906..e6a811ccaf8 100644
--- a/docs/faq.mdx
+++ b/docs/faq.mdx
@@ -692,3 +692,26 @@ http://localhost:8080/layout-parsing
| `PADDLEOCR_ACCESS_TOKEN` | Access token for official API | `None` | Only when using official API |
Environment variables can be used for auto-provisioning, but are not required if configuring via UI. When environment variables are set, these values are used to auto-provision a PaddleOCR model for the tenant on first use.
+
+
+### How do I use Ollama with RAGFlow for local LLM inference?
+
+RAGFlow supports Ollama as a local model provider for private, offline inference.
+
+**Step 1: Start Ollama and pull a model**
+
+```bash
+export OLLAMA_HOST=0.0.0.0
+ollama serve
+ollama pull llama3
+```
+
+**Step 2: Add Ollama in RAGFlow**
+
+1. Go to **Settings** > **Model providers** > **Ollama**.
+2. Set the Base URL to `http://host.docker.internal:11434` (Docker) or `http://localhost:11434` (bare-metal).
+3. Enter the model name (e.g., `llama3`) and click **Save**.
+
+**Step 3: Use Ollama in your assistant**
+
+- Open an assistant's **Configuration** page and select the Ollama model under **Chat model**.
From 335dd5a263ef8ff66158509f47c4eb9bc0958f20 Mon Sep 17 00:00:00 2001
From: Jin Hai
Date: Fri, 15 May 2026 14:00:45 +0800
Subject: [PATCH 162/666] Go: add cli command, list dataset documents (#14948)
### What problem does this PR solve?
```
+---------------------+----------------------------------+-------------+-----------------+---------+--------+------+
| created_at | id | meta_fields | name | size | status | type |
+---------------------+----------------------------------+-------------+-----------------+---------+--------+------+
| 2026-05-08 19:35:08 | f6aa38bb4ad111f1ba6338a74640adcc | map[] | abc.pdf | 3387987 | 1 | pdf |
+---------------------+----------------------------------+-------------+-----------------+---------+--------+------+
```
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---------
Signed-off-by: Jin Hai
---
cmd/server_main.go | 8 +--
internal/cli/client.go | 2 +
internal/cli/lexer.go | 2 +
internal/cli/response.go | 36 ++++++++++
internal/cli/types.go | 1 +
internal/cli/user_command.go | 58 ++++++++++++++-
internal/cli/user_parser.go | 27 +++++++
internal/handler/datasets.go | 4 +-
internal/handler/document.go | 76 +++++++++++++-------
internal/router/router.go | 38 +++++-----
internal/service/{datasets.go => dataset.go} | 27 ++++---
internal/service/document.go | 11 ++-
12 files changed, 224 insertions(+), 66 deletions(-)
rename internal/service/{datasets.go => dataset.go} (95%)
diff --git a/cmd/server_main.go b/cmd/server_main.go
index e4a634e72af..ab14dacd873 100644
--- a/cmd/server_main.go
+++ b/cmd/server_main.go
@@ -166,8 +166,7 @@ func startServer(config *server.Config) {
// Initialize service layer
userService := service.NewUserService()
documentService := service.NewDocumentService()
- datasetsService := service.NewDatasetsService()
- kbService := service.NewKnowledgebaseService()
+ datasetsService := service.NewDatasetService()
chunkService := service.NewChunkService()
llmService := service.NewLLMService()
tenantService := service.NewTenantService()
@@ -187,10 +186,9 @@ func startServer(config *server.Config) {
authHandler := handler.NewAuthHandler()
userHandler := handler.NewUserHandler(userService)
tenantHandler := handler.NewTenantHandler(tenantService, userService)
- documentHandler := handler.NewDocumentHandler(documentService)
+ documentHandler := handler.NewDocumentHandler(documentService, datasetsService)
datasetsHandler := handler.NewDatasetsHandler(datasetsService)
systemHandler := handler.NewSystemHandler(systemService)
- kbHandler := handler.NewKnowledgebaseHandler(kbService, userService, documentService)
chunkHandler := handler.NewChunkHandler(chunkService, userService)
llmHandler := handler.NewLLMHandler(llmService, userService)
chatHandler := handler.NewChatHandler(chatService, userService)
@@ -203,7 +201,7 @@ func startServer(config *server.Config) {
providerHandler := handler.NewProviderHandler(userService, modelProviderService)
// Initialize router
- r := router.NewRouter(authHandler, userHandler, tenantHandler, documentHandler, datasetsHandler, systemHandler, kbHandler, chunkHandler, llmHandler, chatHandler, chatSessionHandler, connectorHandler, searchHandler, fileHandler, memoryHandler, skillSearchHandler, providerHandler)
+ r := router.NewRouter(authHandler, userHandler, tenantHandler, documentHandler, datasetsHandler, systemHandler, chunkHandler, llmHandler, chatHandler, chatSessionHandler, connectorHandler, searchHandler, fileHandler, memoryHandler, skillSearchHandler, providerHandler)
// Create Gin engine
ginEngine := gin.New()
diff --git a/internal/cli/client.go b/internal/cli/client.go
index f28b59464e3..679e1d19b55 100644
--- a/internal/cli/client.go
+++ b/internal/cli/client.go
@@ -203,6 +203,8 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.RunBenchmark(cmd)
case "list_datasets":
return c.ListDatasets(cmd)
+ case "list_dataset_documents":
+ return c.ListDatasetDocumentUserCommand(cmd)
case "search_on_datasets":
return c.SearchOnDatasets(cmd)
case "create_token":
diff --git a/internal/cli/lexer.go b/internal/cli/lexer.go
index 5f0bf18287f..6df63fde0c2 100644
--- a/internal/cli/lexer.go
+++ b/internal/cli/lexer.go
@@ -431,6 +431,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenChunks, Value: ident}
case "DOCUMENT":
return Token{Type: TokenDocument, Value: ident}
+ case "DOCUMENTS":
+ return Token{Type: TokenDocuments, Value: ident}
case "TAGS":
return Token{Type: TokenTag, Value: ident}
case "REGION":
diff --git a/internal/cli/response.go b/internal/cli/response.go
index ffdd18f1d79..440345be9d9 100644
--- a/internal/cli/response.go
+++ b/internal/cli/response.go
@@ -85,6 +85,42 @@ func (r *CommonDataResponse) PrintOut() {
}
}
+type ListDocumentsResponse struct {
+ Code int `json:"code"`
+ Data map[string]interface{} `json:"data"`
+ Message string `json:"message"`
+ Duration float64
+ OutputFormat OutputFormat
+}
+
+func (r *ListDocumentsResponse) Type() string {
+ return "list_documents"
+}
+
+func (r *ListDocumentsResponse) TimeCost() float64 {
+ return r.Duration
+}
+
+func (r *ListDocumentsResponse) SetOutputFormat(format OutputFormat) {
+ r.OutputFormat = format
+}
+
+func (r *ListDocumentsResponse) PrintOut() {
+ if r.Code == 0 {
+ total := r.Data["total"].(float64)
+ fmt.Printf("Total: %0.0f\n", total)
+ docs := r.Data["docs"].([]interface{})
+ table := make([]map[string]interface{}, 0)
+ for _, doc := range docs {
+ table = append(table, doc.(map[string]interface{}))
+ }
+ PrintTableSimpleByFormat(table, r.OutputFormat)
+ } else {
+ fmt.Println("ERROR")
+ fmt.Printf("%d, %s\n", r.Code, r.Message)
+ }
+}
+
type SimpleResponse struct {
Code int `json:"code"`
Message string `json:"message"`
diff --git a/internal/cli/types.go b/internal/cli/types.go
index bbcf09a432d..e5bf55d9fce 100644
--- a/internal/cli/types.go
+++ b/internal/cli/types.go
@@ -149,6 +149,7 @@ const (
TokenChunk
TokenChunks
TokenDocument
+ TokenDocuments
TokenTag
TokenRegion
TokenURL
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index e99912a400c..1d4cc970ae2 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -391,7 +391,63 @@ func (c *RAGFlowClient) ListDatasets(cmd *Command) (ResponseIf, error) {
var result CommonResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
- return nil, fmt.Errorf("list users failed: invalid JSON (%w)", err)
+ return nil, fmt.Errorf("list datasets failed: invalid JSON (%w)", err)
+ }
+
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+ result.Duration = resp.Duration
+
+ return &result, nil
+}
+
+// ListDatasetDocumentUserCommand lists dataset documents
+func (c *RAGFlowClient) ListDatasetDocumentUserCommand(cmd *Command) (ResponseIf, error) {
+ if c.ServerType != "user" {
+ return nil, fmt.Errorf("this command is only allowed in USER mode")
+ }
+
+ // Check for benchmark iterations
+ iterations := 1
+ if val, ok := cmd.Params["iterations"].(int); ok && val > 1 {
+ iterations = val
+ }
+
+ // Determine auth kind based on whether API token is being used
+ if c.HTTPClient.LoginToken == "" && !c.HTTPClient.useAPIToken {
+ return nil, fmt.Errorf("no authorization")
+ }
+
+ datasetID, ok := cmd.Params["dataset_id"].(string)
+ if !ok {
+ return nil, fmt.Errorf("no dataset id")
+ }
+
+ page := 1
+ pageSize := 10
+ keywords := ""
+ returnEmptyMetadata := "true"
+ url := fmt.Sprintf("/datasets/%s/documents?page=%d&page_size=%d&keywords=%s&return_empty_metadata=%s", datasetID, page, pageSize, keywords, returnEmptyMetadata)
+
+ if iterations > 1 {
+ // Benchmark mode - return raw result for benchmark stats
+ return c.HTTPClient.RequestWithIterations("GET", url, "web", nil, nil, iterations)
+ }
+
+ // Normal mode
+ resp, err := c.HTTPClient.Request("GET", url, "web", nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list documents: %w", err)
+ }
+
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to list documents: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+
+ var result ListDocumentsResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("list documents failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go
index 7b40ee9b2c1..38b2d2d29d9 100644
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -136,6 +136,8 @@ func (p *Parser) parseListCommand() (*Command, error) {
return NewCommand("list_environments"), nil
case TokenDatasets:
return p.parseListDatasets()
+ case TokenDocuments:
+ return p.parseListDatasetDocuments()
case TokenAgents:
return p.parseListAgents()
case TokenTokens:
@@ -181,6 +183,31 @@ func (p *Parser) parseListDatasets() (*Command, error) {
return cmd, nil
}
+func (p *Parser) parseListDatasetDocuments() (*Command, error) {
+ p.nextToken() // consume DOCUMENTS
+
+ if p.curToken.Type != TokenFrom {
+ return nil, fmt.Errorf("expected FROM")
+ }
+ p.nextToken()
+
+ datasetID, err := p.parseQuotedString()
+ if err != nil {
+ return nil, err
+ }
+ p.nextToken()
+
+ cmd := NewCommand("list_dataset_documents")
+ cmd.Params["dataset_id"] = datasetID
+
+ // Semicolon is optional for UNSET TOKEN
+ if p.curToken.Type == TokenSemicolon {
+ p.nextToken()
+ }
+
+ return cmd, nil
+}
+
func (p *Parser) parseListAgents() (*Command, error) {
p.nextToken() // consume AGENTS
diff --git a/internal/handler/datasets.go b/internal/handler/datasets.go
index f740212329a..aa0d896cb07 100644
--- a/internal/handler/datasets.go
+++ b/internal/handler/datasets.go
@@ -30,7 +30,7 @@ import (
// DatasetsHandler handles the RESTful dataset endpoints.
type DatasetsHandler struct {
- datasetsService *service.DatasetsService
+ datasetsService *service.DatasetService
}
type listDatasetsExt struct {
@@ -40,7 +40,7 @@ type listDatasetsExt struct {
}
// NewDatasetsHandler creates a new datasets handler.
-func NewDatasetsHandler(datasetsService *service.DatasetsService) *DatasetsHandler {
+func NewDatasetsHandler(datasetsService *service.DatasetService) *DatasetsHandler {
return &DatasetsHandler{datasetsService: datasetsService}
}
diff --git a/internal/handler/document.go b/internal/handler/document.go
index a4152c07dc8..fd7e775dfa4 100644
--- a/internal/handler/document.go
+++ b/internal/handler/document.go
@@ -32,12 +32,14 @@ import (
// DocumentHandler document handler
type DocumentHandler struct {
documentService *service.DocumentService
+ datasetService *service.DatasetService
}
// NewDocumentHandler create document handler
-func NewDocumentHandler(documentService *service.DocumentService) *DocumentHandler {
+func NewDocumentHandler(documentService *service.DocumentService, datasetService *service.DatasetService) *DocumentHandler {
return &DocumentHandler{
documentService: documentService,
+ datasetService: datasetService,
}
}
@@ -198,35 +200,22 @@ func (h *DocumentHandler) DeleteDocument(c *gin.Context) {
}
// ListDocuments document list
-// @Summary Document List
-// @Description Get paginated document list
-// @Tags documents
-// @Accept json
-// @Produce json
-// @Param page query int false "page number" default(1)
-// @Param page_size query int false "items per page" default(10)
-// @Success 200 {object} map[string]interface{}
-// @Router /api/v1/document/list [post]
+
func (h *DocumentHandler) ListDocuments(c *gin.Context) {
- _, errorCode, errorMessage := GetUser(c)
- if errorCode != common.CodeSuccess {
- jsonError(c, errorCode, errorMessage)
- return
- }
- kbID := c.Query("kb_id")
- if kbID == "" {
- c.JSON(http.StatusOK, gin.H{
- "code": 1,
- "message": "Lack of KB ID",
- "data": false,
- })
+ datasetID := c.Param("dataset_id")
+ pageStr := c.Query("page")
+ pageSizeStr := c.Query("page_size")
+ page, _ := strconv.Atoi(pageStr)
+ pageSize, _ := strconv.Atoi(pageSizeStr)
+
+ userID := c.GetString("user_id")
+
+ if !h.datasetService.Accessible(datasetID, userID) {
+ jsonError(c, common.CodeAuthenticationError, "No authorization.")
return
}
- page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
- pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "10"))
-
if page < 1 {
page = 1
}
@@ -235,7 +224,7 @@ func (h *DocumentHandler) ListDocuments(c *gin.Context) {
}
// Use kbID to filter documents
- documents, total, err := h.documentService.ListDocumentsByKBID(kbID, page, pageSize)
+ documents, total, err := h.documentService.ListDocumentsByDatasetID(datasetID, page, pageSize)
if err != nil {
c.JSON(http.StatusOK, gin.H{
"code": 1,
@@ -482,4 +471,37 @@ func (h *DocumentHandler) SetMeta(c *gin.Context) {
"message": "success",
"data": true,
})
-}
\ No newline at end of file
+}
+
+type ParseDocumentRequest struct {
+ Documents []string `json:"documents" binding:"required"`
+ DatasetID string `json:"dataset_id" binding:"required"`
+}
+
+func (h *DocumentHandler) ParseDocuments(c *gin.Context) {
+ var req ParseDocumentRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ c.JSON(http.StatusOK, gin.H{
+ "code": common.CodeBadRequest,
+ "message": err.Error(),
+ })
+ return
+ }
+
+ userID := c.GetString("user_id")
+
+ if !h.datasetService.Accessible(req.DatasetID, userID) {
+ jsonError(c, common.CodeAuthenticationError, "No authorization to access the dataset.")
+ return
+ }
+
+ err := h.documentService.ParseDocuments(req.DatasetID, userID, req.Documents)
+ if err != nil {
+ jsonError(c, common.CodeExceptionError, err.Error())
+ return
+ }
+ c.JSON(http.StatusOK, gin.H{
+ "code": 0,
+ "message": "success",
+ })
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index 4354551b9a1..6b858a89f7a 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -50,7 +50,6 @@ func NewRouter(
documentHandler *handler.DocumentHandler,
datasetsHandler *handler.DatasetsHandler,
systemHandler *handler.SystemHandler,
- knowledgebaseHandler *handler.KnowledgebaseHandler,
chunkHandler *handler.ChunkHandler,
llmHandler *handler.LLMHandler,
chatHandler *handler.ChatHandler,
@@ -63,23 +62,22 @@ func NewRouter(
providerHandler *handler.ProviderHandler,
) *Router {
return &Router{
- authHandler: authHandler,
- userHandler: userHandler,
- tenantHandler: tenantHandler,
- documentHandler: documentHandler,
- datasetsHandler: datasetsHandler,
- systemHandler: systemHandler,
- knowledgebaseHandler: knowledgebaseHandler,
- chunkHandler: chunkHandler,
- llmHandler: llmHandler,
- chatHandler: chatHandler,
- chatSessionHandler: chatSessionHandler,
- connectorHandler: connectorHandler,
- searchHandler: searchHandler,
- fileHandler: fileHandler,
- memoryHandler: memoryHandler,
- skillSearchHandler: skillSearchHandler,
- providerHandler: providerHandler,
+ authHandler: authHandler,
+ userHandler: userHandler,
+ tenantHandler: tenantHandler,
+ documentHandler: documentHandler,
+ datasetsHandler: datasetsHandler,
+ systemHandler: systemHandler,
+ chunkHandler: chunkHandler,
+ llmHandler: llmHandler,
+ chatHandler: chatHandler,
+ chatSessionHandler: chatSessionHandler,
+ connectorHandler: connectorHandler,
+ searchHandler: searchHandler,
+ fileHandler: fileHandler,
+ memoryHandler: memoryHandler,
+ skillSearchHandler: skillSearchHandler,
+ providerHandler: providerHandler,
}
}
@@ -159,6 +157,7 @@ func (r *Router) Setup(engine *gin.Engine) {
documents.GET("/:id", r.documentHandler.GetDocumentByID)
documents.PUT("/:id", r.documentHandler.UpdateDocument)
documents.DELETE("/:id", r.documentHandler.DeleteDocument)
+ documents.POST("/parse", r.documentHandler.ParseDocuments)
}
// Chat routes
@@ -177,6 +176,9 @@ func (r *Router) Setup(engine *gin.Engine) {
datasets.POST("", r.datasetsHandler.CreateDataset)
datasets.DELETE("", r.datasetsHandler.DeleteDatasets)
datasets.POST("/search", r.chunkHandler.RetrievalTest)
+
+ // Dataset documents
+ datasets.GET("/:dataset_id/documents", r.documentHandler.ListDocuments)
}
// Search routes
diff --git a/internal/service/datasets.go b/internal/service/dataset.go
similarity index 95%
rename from internal/service/datasets.go
rename to internal/service/dataset.go
index c163f891e45..19be7425258 100644
--- a/internal/service/datasets.go
+++ b/internal/service/dataset.go
@@ -57,8 +57,8 @@ var (
datasetChunkMethodErrorMessage = "Input should be 'naive', 'book', 'email', 'laws', 'manual', 'one', 'paper', 'picture', 'presentation', 'qa', 'resume', 'table' or 'tag'"
)
-// DatasetsService implements the RESTful dataset APIs from dataset_api.py.
-type DatasetsService struct {
+// DatasetService implements the RESTful dataset APIs from dataset_api.py.
+type DatasetService struct {
kbDAO *dao.KnowledgebaseDAO
documentDAO *dao.DocumentDAO
connectorDAO *dao.ConnectorDAO
@@ -66,9 +66,9 @@ type DatasetsService struct {
tenantLLMDAO *dao.TenantLLMDAO
}
-// NewDatasetsService creates a new datasets service.
-func NewDatasetsService() *DatasetsService {
- return &DatasetsService{
+// NewDatasetService creates a new datasets service.
+func NewDatasetService() *DatasetService {
+ return &DatasetService{
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
connectorDAO: dao.NewConnectorDAO(),
@@ -108,7 +108,7 @@ type CreateDatasetRequest struct {
}
// ListDatasets lists datasets with pagination and filtering.
-func (s *DatasetsService) ListDatasets(id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
+func (s *DatasetService) ListDatasets(id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
id = strings.TrimSpace(id)
if id != "" {
normalizedID, err := normalizeDatasetUUID1(id)
@@ -190,7 +190,7 @@ func (s *DatasetsService) ListDatasets(id, name string, page, pageSize int, orde
}
// CreateDataset creates a new dataset.
-func (s *DatasetsService) CreateDataset(req *CreateDatasetRequest, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
+func (s *DatasetService) CreateDataset(req *CreateDatasetRequest, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
if !isValidString(req.Name) {
return nil, common.CodeDataError, errors.New("Dataset name must be string.")
}
@@ -441,7 +441,7 @@ func (s *DatasetsService) CreateDataset(req *CreateDatasetRequest, tenantID stri
}
// DeleteDatasets deletes multiple datasets.
-func (s *DatasetsService) DeleteDatasets(ids []string, deleteAll bool, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
+func (s *DatasetService) DeleteDatasets(ids []string, deleteAll bool, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
normalizedIDs := make([]string, 0, len(ids))
seenIDs := make(map[string]struct{}, len(ids))
@@ -521,7 +521,7 @@ func (s *DatasetsService) DeleteDatasets(ids []string, deleteAll bool, tenantID
}
// GetDataset gets a single dataset with its size and linked connectors.
-func (s *DatasetsService) GetDataset(datasetID, userID string) (map[string]interface{}, common.ErrorCode, error) {
+func (s *DatasetService) GetDataset(datasetID, userID string) (map[string]interface{}, common.ErrorCode, error) {
datasetID = strings.TrimSpace(datasetID)
if datasetID == "" {
return nil, common.CodeDataError, errors.New("Lack of \"Dataset ID\"")
@@ -559,7 +559,12 @@ func (s *DatasetsService) GetDataset(datasetID, userID string) (map[string]inter
return data, common.CodeSuccess, nil
}
-func (s *DatasetsService) deleteDataset(tenantID string, kb *entity.Knowledgebase) error {
+// Accessible checks if a knowledge base is accessible by a user
+func (s *DatasetService) Accessible(kbID, userID string) bool {
+ return s.kbDAO.Accessible(kbID, userID)
+}
+
+func (s *DatasetService) deleteDataset(tenantID string, kb *entity.Knowledgebase) error {
return dao.DB.Transaction(func(tx *gorm.DB) error {
var documents []entity.Document
if err := tx.Where("kb_id = ?", kb.ID).Find(&documents).Error; err != nil {
@@ -706,7 +711,7 @@ func normalizeDatasetUUID1(id string) (string, error) {
return strings.ReplaceAll(parsedUUID.String(), "-", ""), nil
}
-func (s *DatasetsService) verifyEmbeddingAvailability(embdID string, tenantID string) (bool, string) {
+func (s *DatasetService) verifyEmbeddingAvailability(embdID string, tenantID string) (bool, string) {
modelName, _, provider, err := parseModelName(embdID)
if err != nil {
return false, "Embedding model identifier must follow @ format"
diff --git a/internal/service/document.go b/internal/service/document.go
index d625bef484a..29ed2d4b694 100644
--- a/internal/service/document.go
+++ b/internal/service/document.go
@@ -175,8 +175,8 @@ func (s *DocumentService) ListDocuments(page, pageSize int) ([]*DocumentResponse
return responses, total, nil
}
-// ListDocumentsByKBID list documents by knowledge base ID
-func (s *DocumentService) ListDocumentsByKBID(kbID string, page, pageSize int) ([]*DocumentResponse, int64, error) {
+// ListDocumentsByDatasetID list documents by knowledge base ID
+func (s *DocumentService) ListDocumentsByDatasetID(kbID string, page, pageSize int) ([]*DocumentResponse, int64, error) {
offset := (page - 1) * pageSize
documents, total, err := s.documentDAO.ListByKBID(kbID, offset, pageSize)
if err != nil {
@@ -207,6 +207,13 @@ func (s *DocumentService) GetDocumentsByAuthorID(authorID, page, pageSize int) (
return responses, total, nil
}
+func (s *DocumentService) ParseDocuments(datasetID, userID string, docIDs []string) error {
+ // create document parse id
+ // save to task table
+ // send to message queue
+ return nil
+}
+
// toResponse convert model.Document to DocumentResponse
func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
createdAt := ""
From c2863173b070647d9ba49f1ea4eb05dcae84fa30 Mon Sep 17 00:00:00 2001
From: Haruko386
Date: Fri, 15 May 2026 14:03:33 +0800
Subject: [PATCH 163/666] Go: implement TTS, ASR for Siliconflow and TTs for
StepFun (#14944)
### What problem does this PR solve?
This PRimplement TTS, ASR for Siliconflow and TTs for StepFun
**The following functionalities are now supported:**
**SiliConFlow:**
- [x] Text To Speech
- [x] Audio To Text
- [x] Stream Audio To Text
**StrepFun:**
- [x] Audio To Text
- [x] Stream Audio To Text
**Verified examples from the CLI:**
```plaintext
# SiliconFlow
RAGFlow(user)> tts with 'FunAudioLLM/CosyVoice2-0.5B@test@Siliconflow' text 'hello? show yourself' play format 'wav' param '{"voice": "fnlp/MOSS-TTSD-v0.5:alex"}'
SUCCESS
RAGFlow(user)> asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' audio './internal/test.wav' param ''
+----------------------------------------------------------------------------------------------------------------------+
| text |
+----------------------------------------------------------------------------------------------------------------------+
| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. |
+----------------------------------------------------------------------------------------------------------------------+
RAGFlow(user)> stream asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' audio './internal/test.wav' param ''
+----------------------------------------------------------------------------------------------------------------------+
| text |
+----------------------------------------------------------------------------------------------------------------------+
| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. |
+----------------------------------------------------------------------------------------------------------------------+
```
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
---
conf/models/siliconflow.json | 25 ++-
conf/models/stepfun.json | 24 ++-
internal/cli/user_command.go | 9 +-
internal/development.md | 11 +-
internal/entity/models/siliconflow.go | 248 +++++++++++++++++++++++++-
internal/entity/models/stepfun.go | 182 ++++++++++++++++++-
6 files changed, 483 insertions(+), 16 deletions(-)
diff --git a/conf/models/siliconflow.json b/conf/models/siliconflow.json
index 4da3e0dcab8..320d21aba58 100644
--- a/conf/models/siliconflow.json
+++ b/conf/models/siliconflow.json
@@ -8,7 +8,9 @@
"models": "models",
"embedding": "embeddings",
"rerank": "rerank",
- "balance": "user/info"
+ "balance": "user/info",
+ "tts": "audio/speech",
+ "asr": "audio/transcriptions"
},
"models": [
{
@@ -45,6 +47,27 @@
"model_types": [
"embedding"
]
+ },
+ {
+ "name": "fnlp/MOSS-TTSD-v0.5",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "FunAudioLLM/CosyVoice2-0.5B",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "FunAudioLLM/SenseVoiceSmall",
+ "max_tokens": 8192,
+ "model_types": [
+ "asr"
+ ]
}
]
}
diff --git a/conf/models/stepfun.json b/conf/models/stepfun.json
index f13b227a494..34898c5a1b1 100644
--- a/conf/models/stepfun.json
+++ b/conf/models/stepfun.json
@@ -5,7 +5,8 @@
},
"url_suffix": {
"chat": "chat/completions",
- "models": "models"
+ "models": "models",
+ "tts": "audio/speech"
},
"class": "step",
"models": [
@@ -88,6 +89,27 @@
"chat",
"vision"
]
+ },
+ {
+ "name": "step-tts-2 ",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "stepaudio-2.5-tts",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "step-tts-mini",
+ "max_tokens": 8192,
+ "model_types": [
+ "tts"
+ ]
}
]
}
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index 1d4cc970ae2..7ead18285cb 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -2112,7 +2112,10 @@ func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) {
shouldSave, _ := cmd.Params["save"].(bool)
saveDir, _ := cmd.Params["save_path"].(string)
- fileName := fmt.Sprintf("%s_output.%s", modelName, explicitFormat)
+ // format file name
+ safeModelName := strings.ReplaceAll(modelName, "/", "_")
+ safeModelName = strings.ReplaceAll(safeModelName, ":", "-")
+ fileName := fmt.Sprintf("%s_output.%s", safeModelName, explicitFormat)
cwd, err := os.Getwd()
if err != nil {
@@ -2252,7 +2255,9 @@ func (c *RAGFlowClient) ASRUserCommand(cmd *Command) (ResponseIf, error) {
var result CommonResponse
result.Code = rawResult.Code
- result.Message = rawResult.Data["text"].(string) // TODO
+ result.Data = []map[string]interface{}{
+ {"text": rawResult.Data["text"].(string)},
+ }
result.Duration = resp.Duration
return &result, nil
diff --git a/internal/development.md b/internal/development.md
index c477e7a7326..7e31a8df3e0 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -358,9 +358,18 @@ RAGFlow(user)> list datasets;
```
### 6.23 Text to Speech
-
```
RAGFlow(user)> tts with 'speech-2.8-hd@test@minimax' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"voice_setting": {"voice_id": "English_radiant_girl", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}'
Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/speech-2.8-hd_output.wav
SUCCESS
```
+
+### 6.24 Audio to Speech
+```
+RAGFlow(user)> asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' audio './internal/test.wav' param ''
++----------------------------------------------------------------------------------------------------------------------+
+| text |
++----------------------------------------------------------------------------------------------------------------------+
+| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. |
++----------------------------------------------------------------------------------------------------------------------+
+```
\ No newline at end of file
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index 8a823c2ba24..552ef903612 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -22,7 +22,10 @@ import (
"encoding/json"
"fmt"
"io"
+ "mime/multipart"
"net/http"
+ "os"
+ "path/filepath"
"ragflow/internal/common"
"strconv"
"strings"
@@ -723,7 +726,119 @@ func (s *SiliconflowModel) Rerank(modelName *string, query string, documents []s
// TranscribeAudio transcribe audio
func (o *SiliconflowModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
- return nil, fmt.Errorf("%s, no such method", o.Name())
+ if file == nil || *file == "" {
+ return nil, fmt.Errorf("file is missing")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.ASR)
+
+ // multipart body
+ var body bytes.Buffer
+ writer := multipart.NewWriter(&body)
+
+ // open audio file
+ audioFile, err := os.Open(*file)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open audio file: %w", err)
+ }
+ defer audioFile.Close()
+
+ // create multipart file field
+ part, err := writer.CreateFormFile(
+ "file",
+ filepath.Base(*file),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to create multipart file: %w", err)
+ }
+
+ // copy file content
+ if _, err = io.Copy(part, audioFile); err != nil {
+ return nil, fmt.Errorf("failed to copy audio data: %w", err)
+ }
+
+ // model field
+ if err := writer.WriteField("model", *modelName); err != nil {
+ return nil, fmt.Errorf("failed to write model field: %w", err)
+ }
+
+ // extra params
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+
+ var val string
+
+ switch v := value.(type) {
+ case string:
+ val = v
+ case bool:
+ val = strconv.FormatBool(v)
+ case int:
+ val = strconv.Itoa(v)
+ case int64:
+ val = strconv.FormatInt(v, 10)
+ case float32:
+ val = strconv.FormatFloat(float64(v), 'f', -1, 32)
+ case float64:
+ val = strconv.FormatFloat(v, 'f', -1, 64)
+ default:
+ val = fmt.Sprintf("%v", v)
+ }
+
+ if err = writer.WriteField(key, val); err != nil {
+ return nil, fmt.Errorf("failed to write field %s: %w", key, err)
+ }
+ }
+ }
+
+ if err = writer.Close(); err != nil {
+ return nil, fmt.Errorf("failed to close multipart writer: %w", err)
+ }
+
+ // build request
+ req, err := http.NewRequest("POST", url, &body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("Content-Type", writer.FormDataContentType())
+ req.Header.Set("Accept", "application/json")
+
+ // send request
+ resp, err := o.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ respBody, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("SiliconFlow ASR error: %s - %s", resp.Status, string(respBody))
+ }
+
+ // SiliconFlow response
+ var result struct {
+ Text string `json:"text"`
+ }
+
+ if err = json.Unmarshal(respBody, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w, body=%s", err, string(respBody))
+ }
+
+ var res ASRResponse
+ res.Text = result.Text
+
+ return &ASRResponse{Text: result.Text}, nil
}
func (z *SiliconflowModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
@@ -732,11 +847,138 @@ func (z *SiliconflowModel) TranscribeAudioWithSender(modelName *string, file *st
// AudioSpeech convert audio to text
func (o *SiliconflowModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
- return nil, fmt.Errorf("%s, no such method", o.Name())
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("audio content is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.TTS)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": *audioContent,
+ "stream": false,
+ }
+
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if asrConfig != nil && asrConfig.Format != "" {
+ reqBody["response_format"] = asrConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := o.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("%s - %s", resp.Status, string(body))
+ }
+
+ return &TTSResponse{Audio: body}, nil
}
func (z *SiliconflowModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", z.Name())
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("SiliconFlow API key is missing")
+ }
+
+ if audioContent == nil || *audioContent == "" {
+ return fmt.Errorf("audio content is empty")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.TTS)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": *audioContent,
+ "stream": true,
+ }
+
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := z.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("SiliconFlow stream TTS API error: %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ buf := make([]byte, 32*1024)
+
+ for {
+ n, err := resp.Body.Read(buf)
+ if n > 0 {
+ chunk := string(buf[:n])
+ if errSend := sender(&chunk, nil); errSend != nil {
+ return errSend
+ }
+ }
+
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return fmt.Errorf("error reading SiliconFlow binary audio stream: %w", err)
+ }
+ }
+
+ return nil
}
// OCRFile OCR file
diff --git a/internal/entity/models/stepfun.go b/internal/entity/models/stepfun.go
index 30d226e2508..941f815fd25 100644
--- a/internal/entity/models/stepfun.go
+++ b/internal/entity/models/stepfun.go
@@ -20,6 +20,7 @@ import (
"bufio"
"bytes"
"context"
+ "encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -68,6 +69,15 @@ func NewStepFunModel(baseURL map[string]string, urlSuffix URLSuffix) *StepFunMod
}
}
+/*
+
+RAGFlow(user)> tts with 'fnlp/MOSS-TTSD-v0.5@test@siliconflow' text 'He who desires but acts not, breeds pestilence.' play format 'wav' param '{"voice": "fnlp/MOSS-TTSD-v0.5:alex"}'
+SUCCESS
+RAGFlow(user)> stream tts with 'fnlp/MOSS-TTSD-v0.5@test@siliconflow' text 'He who desires but acts not, breeds pestilence.' play format 'wav' param '{"voice": "fnlp/MOSS-TTSD-v0.5:claire"}'
+SUCCESS
+
+*/
+
func (s *StepFunModel) NewInstance(baseURL map[string]string) ModelDriver {
return NewStepFunModel(baseURL, s.URLSuffix)
}
@@ -459,21 +469,177 @@ func (s *StepFunModel) Rerank(modelName *string, query string, documents []strin
}
// TranscribeAudio transcribe audio
-func (z *StepFunModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+func (s *StepFunModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", s.Name())
}
-func (z *StepFunModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", z.Name())
+func (s *StepFunModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", s.Name())
}
// AudioSpeech convert audio to text
-func (z *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+func (s *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+ // TODO Test it
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("audio content is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", s.BaseURL[region], s.URLSuffix.TTS)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": *audioContent,
+ }
+
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if asrConfig != nil && asrConfig.Format != "" {
+ reqBody["response_format"] = asrConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := s.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("%s - %s", resp.Status, string(body))
+ }
+
+ return &TTSResponse{Audio: body}, nil
}
-func (z *StepFunModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", z.Name())
+// AudioSpeechWithSender for Streaming TTS
+func (s *StepFunModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ // TODO Test it
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("StepFun API key is missing")
+ }
+
+ if audioContent == nil || *audioContent == "" {
+ return fmt.Errorf("audio content is empty")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", s.BaseURL[region], s.URLSuffix.TTS)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": *audioContent,
+ "stream_format": "sse",
+ }
+
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
+ reqBody[key] = value
+ }
+ }
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := s.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("StepFun stream TTS API error: %d - %s", resp.StatusCode, string(body))
+ }
+
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 8*1024*1024)
+
+ for scanner.Scan() {
+ line := scanner.Text()
+
+ if !strings.HasPrefix(line, "data: ") {
+ continue
+ }
+
+ dataStr := strings.TrimSpace(line[6:])
+ // [DONE]
+ if dataStr == "" || dataStr == "[DONE]" {
+ continue
+ }
+
+ // Parse
+ var event struct {
+ Type string `json:"type"`
+ Audio string `json:"audio"`
+ }
+
+ if err := json.Unmarshal([]byte(dataStr), &event); err != nil {
+ continue
+ }
+
+ if event.Type == "speech.audio.error" {
+ return fmt.Errorf("StepFun stream encountered an error during generation")
+ }
+
+ // Extract the Base64 string containing the audio and decode it
+ if event.Type == "speech.audio.delta" && event.Audio != "" {
+ audioBytes, err := base64.StdEncoding.DecodeString(event.Audio)
+ if err == nil && len(audioBytes) > 0 {
+ chunk := string(audioBytes)
+ if errSend := sender(&chunk, nil); errSend != nil {
+ return errSend
+ }
+ }
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("error reading StepFun stream: %w", err)
+ }
+
+ return nil
}
// OCRFile OCR file
From cb606e1c38d9466b44098c13ae6d7df9a2fac23a Mon Sep 17 00:00:00 2001
From: Ricardo-M-L <69202550+Ricardo-M-L@users.noreply.github.com>
Date: Fri, 15 May 2026 14:19:41 +0800
Subject: [PATCH 164/666] fix: correct attribute name typo model_speciess to
model_species (#13929)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
- Rename misspelled attribute `model_speciess` to `model_species` across
4 files
- The extra `s` is a typo — `species` is already plural
## Test plan
- [ ] Verify PDF parsing with laws/manual/paper parser types still works
correctly
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: yuj
Co-authored-by: Claude Opus 4.6 (1M context)
---
deepdoc/parser/pdf_parser.py | 4 ++--
rag/app/laws.py | 2 +-
rag/app/manual.py | 2 +-
rag/app/paper.py | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/deepdoc/parser/pdf_parser.py b/deepdoc/parser/pdf_parser.py
index 3a5bd16627b..e409d5556bd 100644
--- a/deepdoc/parser/pdf_parser.py
+++ b/deepdoc/parser/pdf_parser.py
@@ -77,8 +77,8 @@ def __init__(self, **kwargs):
if layout_recognizer_type not in ["onnx", "ascend"]:
raise RuntimeError("Unsupported layout recognizer type.")
- if hasattr(self, "model_speciess"):
- recognizer_domain = "layout." + self.model_speciess
+ if hasattr(self, "model_species"):
+ recognizer_domain = "layout." + self.model_species
else:
recognizer_domain = "layout"
diff --git a/rag/app/laws.py b/rag/app/laws.py
index e2fe885ffa2..46829d23c2e 100644
--- a/rag/app/laws.py
+++ b/rag/app/laws.py
@@ -95,7 +95,7 @@ def __str__(self) -> str:
class Pdf(PdfParser):
def __init__(self):
- self.model_speciess = ParserType.LAWS.value
+ self.model_species = ParserType.LAWS.value
super().__init__()
def __call__(self, filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, zoomin=3, callback=None):
diff --git a/rag/app/manual.py b/rag/app/manual.py
index b3f5f2edc17..c2e17aeb20d 100644
--- a/rag/app/manual.py
+++ b/rag/app/manual.py
@@ -32,7 +32,7 @@
class Pdf(PdfParser):
def __init__(self):
- self.model_speciess = ParserType.MANUAL.value
+ self.model_species = ParserType.MANUAL.value
super().__init__()
def __call__(self, filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, zoomin=3, callback=None):
diff --git a/rag/app/paper.py b/rag/app/paper.py
index 82ddb8bc838..f578a5fc7a8 100644
--- a/rag/app/paper.py
+++ b/rag/app/paper.py
@@ -30,7 +30,7 @@
class Pdf(PdfParser):
def __init__(self):
- self.model_speciess = ParserType.PAPER.value
+ self.model_species = ParserType.PAPER.value
super().__init__()
def __call__(self, filename, binary=None, from_page=0,
From 14c0985182700f11dd568cba9aa75a73e0618fdd Mon Sep 17 00:00:00 2001
From: wdeveloper16
Date: Fri, 15 May 2026 08:40:53 +0200
Subject: [PATCH 165/666] feat: bump Python minimum from 3.12 to 3.13, drop
strenum backport (#14767)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Closes #14753
## What changed
| File | Change |
|---|---|
| `pyproject.toml` | `requires-python` → `>=3.13,<3.15`; remove
`strenum==0.4.15` |
| `Dockerfile` | `uv python install 3.13`, `uv sync --python 3.13` |
| `.github/workflows/tests.yml` | `uv sync --python 3.13` on both matrix
legs |
| `CLAUDE.md` | dev setup command + requirements note updated |
| `deepdoc/parser/mineru_parser.py` | `from strenum import StrEnum` →
`from enum import StrEnum` |
| `agent/tools/code_exec.py` | same |
`StrEnum` has been in the stdlib since Python 3.11 — the `strenum`
backport package is no longer needed once the floor is 3.13.
## Why uv.lock is not regenerated
`uv lock --python 3.13` fails because:
1. The infiniflow/graspologic fork pins `numpy>=1.26.4,<2.0.0`
2. `tensorflow-cpu>=2.20.0` (the first release with cp313 wheels)
depends on `ml-dtypes>=0.5.1`, which requires `numpy>=2.1.0`
3. These two constraints are irreconcilable on Python 3.13
The lockfile regeneration requires loosening the `numpy` upper bound in
the `infiniflow/graspologic` fork. Once that fork commit is updated and
the SHA in `pyproject.toml:49` is bumped, `uv lock --python 3.13` will
succeed.
## RFC corrections
Two claims in the original RFC (#14753) did not hold up under code
review:
- **"graspologic hard-blocks 3.13"** — the infiniflow fork at the pinned
commit has no `<3.13` Python constraint. The blocker is the transitive
`numpy<2.0.0` conflict with tensorflow-cpu's test dependency, not a
direct Python version cap.
- **"free-threading throughput gains for I/O-bound workload"** — Python
3.13 free-threading requires a special `--disable-gil` build and
provides no benefit for async I/O code (the GIL is already released
during I/O). The real motivation is forward compatibility and improved
error messages.
---
.github/workflows/tests.yml | 4 +-
CLAUDE.md | 4 +-
Dockerfile | 4 +-
agent/tools/code_exec.py | 2 +-
api/db/__init__.py | 2 +-
common/constants.py | 2 +-
deepdoc/parser/mineru_parser.py | 2 +-
mcp/server/server.py | 2 +-
pyproject.toml | 31 +-
rag/llm/__init__.py | 2 +-
rag/llm/chat_model.py | 2 +-
.../test_session_sdk_routes_unit.py | 2 +-
uv.lock | 1257 ++---------------
13 files changed, 141 insertions(+), 1175 deletions(-)
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2ff30e628fc..29fbc588679 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -158,7 +158,7 @@ jobs:
- name: Run unit test
run: |
- uv sync --python 3.12 --group test --frozen
+ uv sync --python 3.13 --group test --frozen
source .venv/bin/activate
which pytest || echo "pytest not in PATH"
echo "Start to run unit test"
@@ -222,7 +222,7 @@ jobs:
# Patch entrypoint.sh for coverage
sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
cd ..
- uv sync --python 3.12 --group test --frozen && uv pip install -e sdk/python
+ uv sync --python 3.13 --group test --frozen && uv pip install -e sdk/python
- name: Start ragflow:nightly for Infinity
diff --git a/CLAUDE.md b/CLAUDE.md
index 81888ba3d71..7cb61ad1266 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -51,7 +51,7 @@ RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on d
```bash
# Install Python dependencies
-uv sync --python 3.12 --all-extras
+uv sync --python 3.13 --all-extras
uv run python3 download_deps.py
pre-commit install
@@ -118,7 +118,7 @@ RAGFlow supports switching between Elasticsearch (default) and Infinity:
## Development Environment Requirements
-- Python 3.10-3.12
+- Python 3.10-3.13
- Node.js >=18.20.4
- Docker & Docker Compose
- uv package manager
diff --git a/Dockerfile b/Dockerfile
index fdc5f4c4bba..dd7fcfa8730 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -78,7 +78,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps
tar xzf "/deps/uv-${uv_arch}-unknown-linux-gnu.tar.gz" \
&& cp "uv-${uv_arch}-unknown-linux-gnu/"* /usr/local/bin/ \
&& rm -rf "uv-${uv_arch}-unknown-linux-gnu" \
- && uv python install 3.12
+ && uv python install 3.13
ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 \
UV_HTTP_TIMEOUT=200 \
@@ -147,7 +147,7 @@ RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
else \
sed -i 's|mirrors.aliyun.com/pypi|pypi.org|g' uv.lock; \
fi; \
- uv sync --python 3.12 --frozen && \
+ uv sync --python 3.13 --frozen && \
# Ensure pip is available in the venv for runtime package installation (fixes #12651)
.venv/bin/python3 -m ensurepip --upgrade
diff --git a/agent/tools/code_exec.py b/agent/tools/code_exec.py
index c6f454c2cfd..f748f31b583 100644
--- a/agent/tools/code_exec.py
+++ b/agent/tools/code_exec.py
@@ -24,7 +24,7 @@
from typing import Optional
from pydantic import BaseModel, Field, field_validator
-from strenum import StrEnum
+from enum import StrEnum
from agent.tools.base import ToolBase, ToolMeta, ToolParamBase
from api.db.services.file_service import FileService
diff --git a/api/db/__init__.py b/api/db/__init__.py
index 6d7ed9fcb97..6aa7c5bbf07 100644
--- a/api/db/__init__.py
+++ b/api/db/__init__.py
@@ -15,7 +15,7 @@
#
from enum import IntEnum
-from strenum import StrEnum
+from enum import StrEnum
class UserTenantRole(StrEnum):
diff --git a/common/constants.py b/common/constants.py
index 1e83a770419..c80735255a0 100644
--- a/common/constants.py
+++ b/common/constants.py
@@ -16,7 +16,7 @@
import os
from enum import Enum, IntEnum
-from strenum import StrEnum
+from enum import StrEnum
SERVICE_CONF = "service_conf.yaml"
RAG_FLOW_SERVICE_NAME = "ragflow"
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py
index 2c3f63ae3fd..90c33573cef 100644
--- a/deepdoc/parser/mineru_parser.py
+++ b/deepdoc/parser/mineru_parser.py
@@ -32,7 +32,7 @@
import pdfplumber
import requests
from PIL import Image
-from strenum import StrEnum
+from enum import StrEnum
from deepdoc.parser.pdf_parser import RAGFlowPdfParser
from deepdoc.parser.utils import extract_pdf_outlines
diff --git a/mcp/server/server.py b/mcp/server/server.py
index bc3a362901e..81c8da3f073 100644
--- a/mcp/server/server.py
+++ b/mcp/server/server.py
@@ -32,7 +32,7 @@
from starlette.middleware import Middleware
from starlette.responses import JSONResponse, Response
from starlette.routing import Mount, Route
-from strenum import StrEnum
+from enum import StrEnum
class LaunchMode(StrEnum):
diff --git a/pyproject.toml b/pyproject.toml
index 6c07e0ba6cd..b5a170e4c1e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,8 +5,13 @@ description = "[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-A
authors = [{ name = "Zhichang Yu", email = "yuzhichang@gmail.com" }]
license-files = ["LICENSE"]
readme = "README.md"
-requires-python = ">=3.12,<3.15"
+requires-python = ">=3.13,<3.15"
dependencies = [
+ # discord-py==2.3.2 unconditionally imports audioop in discord/player.py at module-
+ # load time. audioop was removed from the CPython stdlib in Python 3.13 (PEP 594),
+ # so any import of the discord package raises ImportError on Python 3.13 — even in
+ # tests that never use voice features. audioop-lts provides the module as a backport.
+ "audioop-lts>=0.2.1",
"aiosmtplib>=5.0.0",
"akshare>=1.15.78,<2.0.0",
"anthropic==0.34.1",
@@ -74,7 +79,7 @@ dependencies = [
"opencv-python-headless==4.10.0.84",
"opendal>=0.45.0,<0.46.0",
"opensearch-py==2.7.1",
- "ormsgpack==1.5.0",
+ "ormsgpack>=1.5.0",
"pdfplumber==0.10.4",
"pluginlib>=0.10.0",
"psycopg2-binary>=2.9.11,<3.0.0",
@@ -108,7 +113,6 @@ dependencies = [
"agentrun-sdk>=0.0.16,<1.0.0",
"nest-asyncio>=1.6.0,<2.0.0", # Needed for agent/component/message.py
"sqlglotrs==0.9.0",
- "strenum==0.4.15",
"tavily-python==0.5.1",
"tencentcloud-sdk-python==3.0.1478",
"tika==2.6.0",
@@ -179,7 +183,6 @@ test = [
"pycryptodomex==3.20.0",
"pytest-playwright>=0.7.2",
"codecov>=2.1.13",
- "tensorflow-cpu>=2.17.0",
]
[tool.uv]
@@ -187,11 +190,31 @@ constraint-dependencies = [
# CVE-2026-30922: Denial of Service via unbounded recursion in ASN.1 decoding (CVSS 7.5 HIGH)
# pyasn1 < 0.6.3 is vulnerable; pulled in transitively via google-auth / rsa / pyasn1-modules
"pyasn1>=0.6.3",
+ # Python 3.13 added pathlib.PurePath.parser as a public class attribute holding
+ # the posixpath/ntpath module. trio<0.26 introspects all Path class attributes to
+ # generate async forwards and raises TypeError on any non-callable attribute it
+ # encounters (fixed in trio 0.26 by skipping non-callables). Pulled in transitively
+ # via selenium-wire -> trio-websocket -> trio.
+ "trio>=0.26.0",
]
+override-dependencies = [
+ # moodlepy<=0.24.1 pins attrs<23.0.0, but trio>=0.26.0 requires attrs>=23.2.0.
+ # attrs 23.x is backward-compatible; moodlepy works fine at runtime with it.
+ "attrs>=23.2.0",
+]
+# trio 0.26+ (Python 3.13 compatible) is not yet on the Aliyun mirror.
+# Mark PyPI as explicit so it is used only for packages listed in [tool.uv.sources].
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+explicit = true
[[tool.uv.index]]
url = "https://mirrors.aliyun.com/pypi/simple"
+[tool.uv.sources]
+trio = [{ index = "pypi" }]
+
[tool.setuptools]
packages = [
'agent',
diff --git a/rag/llm/__init__.py b/rag/llm/__init__.py
index 8d6db359ce6..4e30c9f91f0 100644
--- a/rag/llm/__init__.py
+++ b/rag/llm/__init__.py
@@ -19,7 +19,7 @@
import importlib
import inspect
-from strenum import StrEnum
+from enum import StrEnum
class SupportedLiteLLMProvider(StrEnum):
diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py
index 45b81a6cc71..43022dd3246 100644
--- a/rag/llm/chat_model.py
+++ b/rag/llm/chat_model.py
@@ -28,7 +28,7 @@
import litellm
import openai
from openai import AsyncOpenAI, OpenAI
-from strenum import StrEnum
+from enum import StrEnum
from common.misc_utils import thread_pool_exec
from common.token_utils import num_tokens_from_string, total_token_count_from_response
diff --git a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py
index 7a990564552..889de4ba1fa 100644
--- a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py
+++ b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py
@@ -123,7 +123,7 @@ def _load_session_module(monkeypatch):
# Mock common.constants module
from enum import Enum
- from strenum import StrEnum
+ from enum import StrEnum
class _StubLLMType(StrEnum):
CHAT = "chat"
diff --git a/uv.lock b/uv.lock
index 8a66238e96b..8a53b108a27 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,29 +1,21 @@
version = 1
revision = 3
-requires-python = ">=3.12, <3.15"
+requires-python = ">=3.13, <3.15"
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'darwin'",
- "python_full_version == '3.13.*' and sys_platform == 'darwin'",
- "python_full_version < '3.13' and sys_platform == 'darwin'",
+ "python_full_version < '3.14' and sys_platform == 'darwin'",
"python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'",
- "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
- "python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'",
+ "python_full_version < '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'",
"(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')",
- "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux')",
- "(python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')",
+ "(python_full_version < '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')",
]
[manifest]
-constraints = [{ name = "pyasn1", specifier = ">=0.6.3" }]
-
-[[package]]
-name = "absl-py"
-version = "2.4.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/64/c7/8de93764ad66968d19329a7e0c147a2bb3c7054c554d4a119111b8f9440f/absl_py-2.4.0.tar.gz", hash = "sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d" },
+constraints = [
+ { name = "pyasn1", specifier = ">=0.6.3" },
+ { name = "trio", specifier = ">=0.26.0", index = "https://pypi.org/simple" },
]
+overrides = [{ name = "attrs", specifier = ">=23.2.0" }]
[[package]]
name = "agentrun-mem0ai"
@@ -103,23 +95,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/50/42/32cf8e7704ceb4481406eb87161349abb46a57fee3f008ba9cb610968646/aiohttp-3.13.3.tar.gz", hash = "sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/a0/be/4fc11f202955a69e0db803a12a062b8379c970c7c84f4882b6da17337cc1/aiohttp-3.13.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/97/2c/621d5b851f94fa0bb7430d6089b3aa970a9d9b75196bc93bb624b0db237a/aiohttp-3.13.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5d/43/4be01406b78e1be8320bb8316dc9c42dbab553d281c40364e0f862d5661c/aiohttp-3.13.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8d/a8/5a35dc56a06a2c90d4742cbf35294396907027f80eea696637945a106f25/aiohttp-3.13.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bf/62/4b9eeb331da56530bf2e198a297e5303e1c1ebdceeb00fe9b568a65c5a0c/aiohttp-3.13.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7c/f6/af16887b5d419e6a367095994c0b1332d154f647e7dc2bd50e61876e8e3d/aiohttp-3.13.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/83/397c634b1bcc24292fa1e0c7822800f9f6569e32934bdeef09dae7992dfb/aiohttp-3.13.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463" },
- { url = "https://mirrors.aliyun.com/pypi/packages/86/f6/a62cbbf13f0ac80a70f71b1672feba90fdb21fd7abd8dbf25c0105fb6fa3/aiohttp-3.13.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0a/87/20a35ad487efdd3fba93d5843efdfaa62d2f1479eaafa7453398a44faf13/aiohttp-3.13.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/de/95/8fd69a66682012f6716e1bc09ef8a1a2a91922c5725cb904689f112309c4/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e5/66/7b94b3b5ba70e955ff597672dad1691333080e37f50280178967aff68657/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/47/71/6f72f77f9f7d74719692ab65a2a0252584bf8d5f301e2ecb4c0da734530a/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fa/b4/75ec16cbbd5c01bdaf4a05b19e103e78d7ce1ef7c80867eb0ace42ff4488/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423" },
- { url = "https://mirrors.aliyun.com/pypi/packages/52/8f/bc518c0eea29f8406dcf7ed1f96c9b48e3bc3995a96159b3fc11f9e08321/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9d/f2/a07a75173124f31f11ea6f863dc44e6f09afe2bca45dd4e64979490deab1/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3c/4a/1a3fee7c21350cac78e5c5cef711bac1b94feca07399f3d406972e2d8fcd/aiohttp-3.13.3-cp312-cp312-win32.whl", hash = "sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d9/b7/76175c7cb4eb73d91ad63c34e29fc4f77c9386bba4a65b53ba8e05ee3c39/aiohttp-3.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57" },
{ url = "https://mirrors.aliyun.com/pypi/packages/97/8a/12ca489246ca1faaf5432844adbfce7ff2cc4997733e0af120869345643a/aiohttp-3.13.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/32/08/de43984c74ed1fca5c014808963cc83cb00d7bb06af228f132d33862ca76/aiohttp-3.13.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/17/f8/8dd2cf6112a5a76f81f81a5130c57ca829d101ad583ce57f889179accdda/aiohttp-3.13.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3" },
@@ -200,7 +175,6 @@ version = "1.4.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "frozenlist" },
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7" }
wheels = [
@@ -523,7 +497,6 @@ version = "4.13.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "idna" },
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc" }
wheels = [
@@ -635,19 +608,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/96/3a/2baa6a2a3319bfcc0bc490a26c9057eba2412502eb6ab16e55533dd511a7/asana-5.2.3-py3-none-any.whl", hash = "sha256:543e928aadf1a0f05769bfab14e1d9dbb7c6183ce75c451aea0fd2196e392e7e" },
]
-[[package]]
-name = "astunparse"
-version = "1.6.3"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "six" },
- { name = "wheel" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8" },
-]
-
[[package]]
name = "atlassian-python-api"
version = "4.0.7"
@@ -668,11 +628,67 @@ wheels = [
[[package]]
name = "attrs"
-version = "22.2.0"
+version = "26.1.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/21/31/3f468da74c7de4fcf9b25591e682856389b3400b4b62f201e65f15ea3e07/attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" }
+sdist = { url = "https://mirrors.aliyun.com/pypi/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/fb/6e/6f83bf616d2becdf333a1640f1d463fef3150e2e926b7010cb0f81c95e88/attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" },
+]
+
+[[package]]
+name = "audioop-lts"
+version = "0.2.2"
+source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
+sdist = { url = "https://mirrors.aliyun.com/pypi/packages/38/53/946db57842a50b2da2e0c1e34bd37f36f5aadba1a929a3971c5d7841dbca/audioop_lts-0.2.2.tar.gz", hash = "sha256:64d0c62d88e67b98a1a5e71987b7aa7b5bcffc7dcee65b635823dbdd0a8dbbd0" }
+wheels = [
+ { url = "https://mirrors.aliyun.com/pypi/packages/de/d4/94d277ca941de5a507b07f0b592f199c22454eeaec8f008a286b3fbbacd6/audioop_lts-0.2.2-cp313-abi3-macosx_10_13_universal2.whl", hash = "sha256:fd3d4602dc64914d462924a08c1a9816435a2155d74f325853c1f1ac3b2d9800" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/f8/5a/656d1c2da4b555920ce4177167bfeb8623d98765594af59702c8873f60ec/audioop_lts-0.2.2-cp313-abi3-macosx_10_13_x86_64.whl", hash = "sha256:550c114a8df0aafe9a05442a1162dfc8fec37e9af1d625ae6060fed6e756f303" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/1b/83/ea581e364ce7b0d41456fb79d6ee0ad482beda61faf0cab20cbd4c63a541/audioop_lts-0.2.2-cp313-abi3-macosx_11_0_arm64.whl", hash = "sha256:9a13dc409f2564de15dd68be65b462ba0dde01b19663720c68c1140c782d1d75" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/b8/3b/e8964210b5e216e5041593b7d33e97ee65967f17c282e8510d19c666dab4/audioop_lts-0.2.2-cp313-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:51c916108c56aa6e426ce611946f901badac950ee2ddaf302b7ed35d9958970d" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/c7/2e/0a1c52faf10d51def20531a59ce4c706cb7952323b11709e10de324d6493/audioop_lts-0.2.2-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47eba38322370347b1c47024defbd36374a211e8dd5b0dcbce7b34fdb6f8847b" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/75/e8/cd95eef479656cb75ab05dfece8c1f8c395d17a7c651d88f8e6e291a63ab/audioop_lts-0.2.2-cp313-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba7c3a7e5f23e215cb271516197030c32aef2e754252c4c70a50aaff7031a2c8" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/5c/1e/a0c42570b74f83efa5cca34905b3eef03f7ab09fe5637015df538a7f3345/audioop_lts-0.2.2-cp313-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def246fe9e180626731b26e89816e79aae2276f825420a07b4a647abaa84becc" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/50/d5/8a0ae607ca07dbb34027bac8db805498ee7bfecc05fd2c148cc1ed7646e7/audioop_lts-0.2.2-cp313-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e160bf9df356d841bb6c180eeeea1834085464626dc1b68fa4e1d59070affdc3" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/12/17/0d28c46179e7910bfb0bb62760ccb33edb5de973052cb2230b662c14ca2e/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4b4cd51a57b698b2d06cb9993b7ac8dfe89a3b2878e96bc7948e9f19ff51dba6" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/84/ba/bd5d3806641564f2024e97ca98ea8f8811d4e01d9b9f9831474bc9e14f9e/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4a53aa7c16a60a6857e6b0b165261436396ef7293f8b5c9c828a3a203147ed4a" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/f9/5e/435ce8d5642f1f7679540d1e73c1c42d933331c0976eb397d1717d7f01a3/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:3fc38008969796f0f689f1453722a0f463da1b8a6fbee11987830bfbb664f623" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/ae/3b/b909e76b606cbfd53875693ec8c156e93e15a1366a012f0b7e4fb52d3c34/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_s390x.whl", hash = "sha256:15ab25dd3e620790f40e9ead897f91e79c0d3ce65fe193c8ed6c26cffdd24be7" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/30/e7/8f1603b4572d79b775f2140d7952f200f5e6c62904585d08a01f0a70393a/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:03f061a1915538fd96272bac9551841859dbb2e3bf73ebe4a23ef043766f5449" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/b5/96/c37846df657ccdda62ba1ae2b6534fa90e2e1b1742ca8dcf8ebd38c53801/audioop_lts-0.2.2-cp313-abi3-win32.whl", hash = "sha256:3bcddaaf6cc5935a300a8387c99f7a7fbbe212a11568ec6cf6e4bc458c048636" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/34/a5/9d78fdb5b844a83da8a71226c7bdae7cc638861085fff7a1d707cb4823fa/audioop_lts-0.2.2-cp313-abi3-win_amd64.whl", hash = "sha256:a2c2a947fae7d1062ef08c4e369e0ba2086049a5e598fda41122535557012e9e" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/34/25/20d8fde083123e90c61b51afb547bb0ea7e77bab50d98c0ab243d02a0e43/audioop_lts-0.2.2-cp313-abi3-win_arm64.whl", hash = "sha256:5f93a5db13927a37d2d09637ccca4b2b6b48c19cd9eda7b17a2e9f77edee6a6f" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/58/a7/0a764f77b5c4ac58dc13c01a580f5d32ae8c74c92020b961556a43e26d02/audioop_lts-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:73f80bf4cd5d2ca7814da30a120de1f9408ee0619cc75da87d0641273d202a09" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/aa/ed/ebebedde1a18848b085ad0fa54b66ceb95f1f94a3fc04f1cd1b5ccb0ed42/audioop_lts-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:106753a83a25ee4d6f473f2be6b0966fc1c9af7e0017192f5531a3e7463dce58" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/cb/6e/11ca8c21af79f15dbb1c7f8017952ee8c810c438ce4e2b25638dfef2b02c/audioop_lts-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fbdd522624141e40948ab3e8cdae6e04c748d78710e9f0f8d4dae2750831de19" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/84/52/0022f93d56d85eec5da6b9da6a958a1ef09e80c39f2cc0a590c6af81dcbb/audioop_lts-0.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:143fad0311e8209ece30a8dbddab3b65ab419cbe8c0dde6e8828da25999be911" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/87/1d/48a889855e67be8718adbc7a01f3c01d5743c325453a5e81cf3717664aad/audioop_lts-0.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfbbc74ec68a0fd08cfec1f4b5e8cca3d3cd7de5501b01c4b5d209995033cde9" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/98/a6/94b7213190e8077547ffae75e13ed05edc488653c85aa5c41472c297d295/audioop_lts-0.2.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfcac6aa6f42397471e4943e0feb2244549db5c5d01efcd02725b96af417f3fe" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/e9/e9/78450d7cb921ede0cfc33426d3a8023a3bda755883c95c868ee36db8d48d/audioop_lts-0.2.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:752d76472d9804ac60f0078c79cdae8b956f293177acd2316cd1e15149aee132" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/4f/e2/cd5439aad4f3e34ae1ee852025dc6aa8f67a82b97641e390bf7bd9891d3e/audioop_lts-0.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83c381767e2cc10e93e40281a04852facc4cd9334550e0f392f72d1c0a9c5753" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/68/4b/9d853e9076c43ebba0d411e8d2aa19061083349ac695a7d082540bad64d0/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c0022283e9556e0f3643b7c3c03f05063ca72b3063291834cca43234f20c60bb" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/58/26/4bae7f9d2f116ed5593989d0e521d679b0d583973d203384679323d8fa85/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:a2d4f1513d63c795e82948e1305f31a6d530626e5f9f2605408b300ae6095093" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/b2/67/a9f4fb3e250dda9e9046f8866e9fa7d52664f8985e445c6b4ad6dfb55641/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c9c8e68d8b4a56fda8c025e538e639f8c5953f5073886b596c93ec9b620055e7" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/70/f7/3de86562db0121956148bcb0fe5b506615e3bcf6e63c4357a612b910765a/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:96f19de485a2925314f5020e85911fb447ff5fbef56e8c7c6927851b95533a1c" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/f1/32/fd772bf9078ae1001207d2df1eef3da05bea611a87dd0e8217989b2848fa/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e541c3ef484852ef36545f66209444c48b28661e864ccadb29daddb6a4b8e5f5" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/4f/41/affea7181592ab0ab560044632571a38edaf9130b84928177823fbf3176a/audioop_lts-0.2.2-cp313-cp313t-win32.whl", hash = "sha256:d5e73fa573e273e4f2e5ff96f9043858a5e9311e94ffefd88a3186a910c70917" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/28/2b/0372842877016641db8fc54d5c88596b542eec2f8f6c20a36fb6612bf9ee/audioop_lts-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9191d68659eda01e448188f60364c7763a7ca6653ed3f87ebb165822153a8547" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/ee/ca/baf2b9cc7e96c179bb4a54f30fcd83e6ecb340031bde68f486403f943768/audioop_lts-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c174e322bb5783c099aaf87faeb240c8d210686b04bd61dfd05a8e5a83d88969" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/5c/73/413b5a2804091e2c7d5def1d618e4837f1cb82464e230f827226278556b7/audioop_lts-0.2.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:f9ee9b52f5f857fbaf9d605a360884f034c92c1c23021fb90b2e39b8e64bede6" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/ae/8c/daa3308dc6593944410c2c68306a5e217f5c05b70a12e70228e7dd42dc5c/audioop_lts-0.2.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:49ee1a41738a23e98d98b937a0638357a2477bc99e61b0f768a8f654f45d9b7a" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/4e/86/c2e0f627168fcf61781a8f72cab06b228fe1da4b9fa4ab39cfb791b5836b/audioop_lts-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5b00be98ccd0fc123dcfad31d50030d25fcf31488cde9e61692029cd7394733b" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/c7/bd/35dce665255434f54e5307de39e31912a6f902d4572da7c37582809de14f/audioop_lts-0.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6d2e0f9f7a69403e388894d4ca5ada5c47230716a03f2847cfc7bd1ecb589d6" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/2d/d2/deeb9f51def1437b3afa35aeb729d577c04bcd89394cb56f9239a9f50b6f/audioop_lts-0.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9b0b8a03ef474f56d1a842af1a2e01398b8f7654009823c6d9e0ecff4d5cfbf" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/76/3b/09f8b35b227cee28cc8231e296a82759ed80c1a08e349811d69773c48426/audioop_lts-0.2.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b267b70747d82125f1a021506565bdc5609a2b24bcb4773c16d79d2bb260bbd" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/0b/15/05b48a935cf3b130c248bfdbdea71ce6437f5394ee8533e0edd7cfd93d5e/audioop_lts-0.2.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0337d658f9b81f4cd0fdb1f47635070cc084871a3d4646d9de74fdf4e7c3d24a" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/83/80/186b7fce6d35b68d3d739f228dc31d60b3412105854edb975aa155a58339/audioop_lts-0.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:167d3b62586faef8b6b2275c3218796b12621a60e43f7e9d5845d627b9c9b80e" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/49/89/c78cc5ac6cb5828f17514fb12966e299c850bc885e80f8ad94e38d450886/audioop_lts-0.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0d9385e96f9f6da847f4d571ce3cb15b5091140edf3db97276872647ce37efd7" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/4c/4b/6401888d0c010e586c2ca50fce4c903d70a6bb55928b16cfbdfd957a13da/audioop_lts-0.2.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:48159d96962674eccdca9a3df280e864e8ac75e40a577cc97c5c42667ffabfc5" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/de/f8/c874ca9bb447dae0e2ef2e231f6c4c2b0c39e31ae684d2420b0f9e97ee68/audioop_lts-0.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8fefe5868cd082db1186f2837d64cfbfa78b548ea0d0543e9b28935ccce81ce9" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/3e/c0/0323e66f3daebc13fd46b36b30c3be47e3fc4257eae44f1e77eb828c703f/audioop_lts-0.2.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:58cf54380c3884fb49fdd37dfb7a772632b6701d28edd3e2904743c5e1773602" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/98/6b/acc7734ac02d95ab791c10c3f17ffa3584ccb9ac5c18fd771c638ed6d1f5/audioop_lts-0.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:088327f00488cdeed296edd9215ca159f3a5a5034741465789cad403fcf4bec0" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/13/c3/c3dc3f564ce6877ecd2a05f8d751b9b27a8c320c2533a98b0c86349778d0/audioop_lts-0.2.2-cp314-cp314t-win32.whl", hash = "sha256:068aa17a38b4e0e7de771c62c60bbca2455924b67a8814f3b0dee92b5820c0b3" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/72/bb/b4608537e9ffcb86449091939d52d24a055216a36a8bf66b936af8c3e7ac/audioop_lts-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a5bf613e96f49712073de86f20dbdd4014ca18efd4d34ed18c75bd808337851b" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/f6/22/91616fe707a5c5510de2cac9b046a30defe7007ba8a0c04f9c08f27df312/audioop_lts-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:b492c3b040153e68b9fdaff5913305aaaba5bb433d8a7f73d5cf6a64ed3cc1dd" },
]
[[package]]
@@ -837,15 +853,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/9d/61/c59a849bd457c8a1b408ae828dbcc15e674962b5a29705e869e15b32bf25/biopython-1.86.tar.gz", hash = "sha256:93a50b586a4d2cec68ab2f99d03ef583c5761d8fba5535cb8e81da781d0d92ff" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/98/e2/199b8ccbd4b9bf234157db0668177b5b7784d62f29d9096fd0d3a70e3b86/biopython-1.86-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f8d372aae21d79b11613751c6ae23c88db0e94d25b7567b1f67aa0304fb61667" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d8/2f/1a7da2a55212b3d0a03866d22213f91273fee3722b5364575419fbe574a5/biopython-1.86-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:baf19d9237aaaa387a68f8f055f978af5c80338d7e037ab028e8d768928f1250" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5b/e9/4057d4c2aa22ca25c180ecbed2ce9e7d65bf787999778bc63b41df0d03b5/biopython-1.86-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f9abdf6cbf0087850de5f8148da0d420c4cb87905bf4de3145ad24a8d55dcd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a7/b2/3e6862720d7c51f0fbe7d6d25be72a95486779d9d98122283b4e8032fb40/biopython-1.86-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:187c3c24dd2255e7328f3e0523ab5d6350b73ff562517de0c1922385617101d2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d7/cb/61877367bf08670573d62513b239dc65cf2b7488dc74322cc6051da2e55e/biopython-1.86-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1859830b8262785c6b59dfe0c82cddb643974f63b9d2779bb9f3e2c47c0a95da" },
- { url = "https://mirrors.aliyun.com/pypi/packages/84/1a/3182a77776b76f3f5c64825ee1acf9355f665bed72ee9e8ff49e48f25d98/biopython-1.86-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfd906c47b6fb38e3abb9f52e0c06822e6e82a043d38c2000773692c29db1ed8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1a/22/828b08fac8dbc8c1dbc1ad03815137cebc9c78303ec7d21b568544028119/biopython-1.86-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a6ab2c60742f1c8494cfbbe3b7a8b45f0400c8f2b36b686b895d5e4d625f04e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/36/7a/122aea7653fa93d7eb72978928e80759082efffa70afe0c25a17e18521da/biopython-1.86-cp312-cp312-win32.whl", hash = "sha256:192c61bc3d782c171b7d50bb7d8189d84790d6e3c4b24fd41d1d7ffc7d303efe" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a9/13/00db03b01e54070d5b0ec9c71eef86e61afa733d9af76e5b9b09f5dc9165/biopython-1.86-cp312-cp312-win_amd64.whl", hash = "sha256:35a6b9c5dcdfb5c2631a313a007f3f41a7d72573ba2b68c962e10ea92096ff3b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fd/6e/84d6c66ab93095aa7adb998a8eef045328470eafd36b9237c4db213e587c/biopython-1.86-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fb3a11a98e49428720dca227e2a5bdd57c973ee7c4df3cf6734c0aa13fd134c7" },
{ url = "https://mirrors.aliyun.com/pypi/packages/12/75/60386f2640f13765b1651f2f26d8b4f893c46ee663df3ca76eda966d4f6a/biopython-1.86-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e161f3d3b6e65fbfd1ce22a01c3e9fa9da789adde4972fd0cc2370795ea5357b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/dd/de/a39adb98a0552a257219503c236ef17f007598af55326c0d143db52e5a92/biopython-1.86-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aa8c9e92ee6fe59dfe0d2c2daf9a9eec6b812c78328caad038f79163c500218" },
@@ -899,13 +906,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/d0/d0/d8cc8c9a4488a787e7fa430f6055e5bd1ddb22c340a751d9e901b82e2efe/blis-1.3.3.tar.gz", hash = "sha256:034d4560ff3cc43e8aa37e188451b0440e3261d989bb8a42ceee865607715ecd" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/16/d1/429cf0cf693d4c7dc2efed969bd474e315aab636e4a95f66c4ed7264912d/blis-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a1c74e100665f8e918ebdbae2794576adf1f691680b5cdb8b29578432f623ef" },
- { url = "https://mirrors.aliyun.com/pypi/packages/11/69/363c8df8d98b3cc97be19aad6aabb2c9c53f372490d79316bdee92d476e7/blis-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3f6c595185176ce021316263e1a1d636a3425b6c48366c1fd712d08d0b71849a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/96/2a/fbf65d906d823d839076c5150a6f8eb5ecbc5f9135e0b6510609bda1e6b7/blis-1.3.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d734b19fba0be7944f272dfa7b443b37c61f9476d9ab054a9ac53555ceadd2e0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d5/ad/58deaa3ad856dd3cc96493e40ffd2ed043d18d4d304f85a65cde1ccbf644/blis-1.3.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ef6d6e2b599a3a2788eb6d9b443533961265aa4ec49d574ed4bb846e548dcdb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/78/82/816a7adfe1f7acc8151f01ec86ef64467a3c833932d8f19f8e06613b8a4e/blis-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8c888438ae99c500422d50698e3028b65caa8ebb44e24204d87fda2df64058f7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1e/e2/0e93b865f648b5519360846669a35f28ee8f4e1d93d054f6850d8afbabde/blis-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8177879fd3590b5eecdd377f9deafb5dc8af6d684f065bd01553302fb3fcf9a7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/20/07/fb43edc2ff0a6a367e4a94fc39eb3b85aa1e55e24cc857af2db145ce9f0d/blis-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:f20f7ad69aaffd1ce14fe77de557b6df9b61e0c9e582f75a843715d836b5c8af" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e6/f7/d26e62d9be3d70473a63e0a5d30bae49c2fe138bebac224adddcdef8a7ce/blis-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1e647341f958421a86b028a2efe16ce19c67dba2a05f79e8f7e80b1ff45328aa" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4a/78/750d12da388f714958eb2f2fd177652323bbe7ec528365c37129edd6eb84/blis-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d563160f874abb78a57e346f07312c5323f7ad67b6370052b6b17087ef234a8e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e8/36/eac4199c5b200a5f3e93cad197da8d26d909f218eb444c4f552647c95240/blis-1.3.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:30b8a5b90cb6cb81d1ada9ae05aa55fb8e70d9a0ae9db40d2401bb9c1c8f14c4" },
@@ -969,16 +969,6 @@ version = "1.2.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/f7/16/c92ca344d646e71a43b8bb353f0a6490d7f6e06210f8554c8f874e454285/brotli-1.2.0.tar.gz", hash = "sha256:e310f77e41941c13340a95976fe66a8a95b01e783d430eeaf7a2f87e0a57dd0a" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/11/ee/b0a11ab2315c69bb9b45a2aaed022499c9c24a205c3a49c3513b541a7967/brotli-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:35d382625778834a7f3061b15423919aa03e4f5da34ac8e02c074e4b75ab4f84" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e1/2f/29c1459513cd35828e25531ebfcbf3e92a5e49f560b1777a9af7203eb46e/brotli-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a61c06b334bd99bc5ae84f1eeb36bfe01400264b3c352f968c6e30a10f9d08b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3d/6f/feba03130d5fceadfa3a1bb102cb14650798c848b1df2a808356f939bb16/brotli-1.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acec55bb7c90f1dfc476126f9711a8e81c9af7fb617409a9ee2953115343f08d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/38/f3abb554eee089bd15471057ba85f47e53a44a462cfce265d9bf7088eb09/brotli-1.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:260d3692396e1895c5034f204f0db022c056f9e2ac841593a4cf9426e2a3faca" },
- { url = "https://mirrors.aliyun.com/pypi/packages/03/a7/03aa61fbc3c5cbf99b44d158665f9b0dd3d8059be16c460208d9e385c837/brotli-1.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:072e7624b1fc4d601036ab3f4f27942ef772887e876beff0301d261210bca97f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/21/1b/0374a89ee27d152a5069c356c96b93afd1b94eae83f1e004b57eb6ce2f10/brotli-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adedc4a67e15327dfdd04884873c6d5a01d3e3b6f61406f99b1ed4865a2f6d28" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/57/69d4fe84a67aef4f524dcd075c6eee868d7850e85bf01d778a857d8dbe0a/brotli-1.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7a47ce5c2288702e09dc22a44d0ee6152f2c7eda97b3c8482d826a1f3cfc7da7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d5/3b/39e13ce78a8e9a621c5df3aeb5fd181fcc8caba8c48a194cd629771f6828/brotli-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af43b8711a8264bb4e7d6d9a6d004c3a2019c04c01127a868709ec29962b6036" },
- { url = "https://mirrors.aliyun.com/pypi/packages/62/28/4d00cb9bd76a6357a66fcd54b4b6d70288385584063f4b07884c1e7286ac/brotli-1.2.0-cp312-cp312-win32.whl", hash = "sha256:e99befa0b48f3cd293dafeacdd0d191804d105d279e0b387a32054c1180f3161" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1c/4e/bc1dcac9498859d5e353c9b153627a3752868a9d5f05ce8dedd81a2354ab/brotli-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:b35c13ce241abdd44cb8ca70683f20c0c079728a36a996297adb5334adfc1c44" },
{ url = "https://mirrors.aliyun.com/pypi/packages/6c/d4/4ad5432ac98c73096159d9ce7ffeb82d151c2ac84adcc6168e476bb54674/brotli-1.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e5825ba2c9998375530504578fd4d5d1059d09621a02065d1b6bfc41a8e05ab" },
{ url = "https://mirrors.aliyun.com/pypi/packages/91/9f/9cc5bd03ee68a85dc4bc89114f7067c056a3c14b3d95f171918c088bf88d/brotli-1.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0cf8c3b8ba93d496b2fae778039e2f5ecc7cff99df84df337ca31d8f2252896c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/2e/b6/fe84227c56a865d16a6614e2c4722864b380cb14b13f3e6bef441e73a85a/brotli-1.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8565e3cdc1808b1a34714b553b262c5de5fbda202285782173ec137fd13709f" },
@@ -1064,13 +1054,6 @@ version = "5.9.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/bd/cb/09939728be094d155b5d4ac262e39877875f5f7e36eea66beb359f647bd0/cbor2-5.9.0.tar.gz", hash = "sha256:85c7a46279ac8f226e1059275221e6b3d0e370d2bb6bd0500f9780781615bcea" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ee/39/72d8a5a4b06565561ec28f4fcb41aff7bb77f51705c01f00b8254a2aca4f/cbor2-5.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1f223dffb1bcdd2764665f04c1152943d9daa4bc124a576cd8dee1cad4264313" },
- { url = "https://mirrors.aliyun.com/pypi/packages/09/fd/7ddf3d3153b54c69c3be77172b8d9aa3a9d74f62a7fbde614d53eaeed9a4/cbor2-5.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae6c706ac1d85a0b3cb3395308fd0c4d55e3202b4760773675957e93cdff45fc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/db/9d/7ede2cc42f9bb4260492e7d29d2aab781eacbbcfb09d983de1e695077199/cbor2-5.9.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4cd43d8fc374b31643b2830910f28177a606a7bc84975a62675dd3f2e320fc7b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/9d/588ebc7c5bc5843f609b05fe07be8575c7dec987735b0bbc908ac9c1264a/cbor2-5.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4aa07b392cc3d76fb31c08a46a226b58c320d1c172ff3073e864409ced7bc50f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f7/a1/6fc8f4b15c6a27e7fbb7966c30c2b4b18c274a3221fa2f5e6235502d34bc/cbor2-5.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:971d425b3a23b75953d8853d5f9911bdeefa09d759ee3b5e6b07b5ff3cbd9073" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/20/9a22cfe08be16ddfeef2542cf4eeed1b29f3f57ddbba0b42f7e0bb8331fd/cbor2-5.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:34a6cb15e6ab6a8eae94ad2041731cd3ef786af43a8df99f847969af5b902ee7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c6/9e/695f92d09006614034e25a9f5b10620f3b219f79c1bec3c37b7c6f27a7a9/cbor2-5.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:7d1ddc4541e7367ac58c2470cc0df847f7137167fe4f5729e2d3cc0b993d7da4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/81/c5/4901e21a8afe9448fd947b11e8f383903207cd6dd0800e5f5a386838de5b/cbor2-5.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fbb06f34aa645b4deca66643bba3d400d20c15312d1fe88d429be60c1ab50f27" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1b/10/df643a381aebc3f05486de4813662bc58accb640fc3275cb276a75e89694/cbor2-5.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac684fe195c39821fca70d18afbf748f728aefbfbf88456018d299e559b8cae0" },
{ url = "https://mirrors.aliyun.com/pypi/packages/c6/0c/8aa6b766059ae4a0ca1ec3ff96fe3823a69a7be880dba2e249f7fbe2700b/cbor2-5.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a54fbb32cb828c214f7f333a707e4aec61182e7efdc06ea5d9596d3ecee624a" },
@@ -1106,18 +1089,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037" },
- { url = "https://mirrors.aliyun.com/pypi/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba" },
- { url = "https://mirrors.aliyun.com/pypi/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b" },
@@ -1169,22 +1140,6 @@ version = "3.4.6"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/e5/62/c0815c992c9545347aeea7859b50dc9044d147e2e7278329c6e02ac9a616/charset_normalizer-3.4.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a8/37/bdca6613c2e3c58c7421891d80cc3efa1d32e882f7c4a7ee6039c3fc951a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6c/92/9934d1bbd69f7f398b38c5dae1cbf9cc672e7c34a4adf7b17c0a9c17d15d/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/af/90/25f6ab406659286be929fd89ab0e78e38aa183fc374e03aa3c12d730af8a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4e/ef/79a463eb0fff7f96afa04c1d4c51f8fc85426f918db467854bfb6a569ce3/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f7/72/d0426afec4b71dc159fa6b4e68f868cd5a3ecd918fec5813a15d292a7d10/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bf/18/c82b06a68bfcb6ce55e508225d210c7e6a4ea122bfc0748892f3dc4e8e11/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/44/d6/0c25979b92f8adafdbb946160348d8d44aa60ce99afdc27df524379875cb/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2e/3d/7fea3e8fe84136bebbac715dd1221cc25c173c57a699c030ab9b8900cbb7/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/57/8a/d6f7fd5cb96c58ef2f681424fbca01264461336d2a7fc875e4446b1f1346/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/16/50/478cdda782c8c9c3fb5da3cc72dd7f331f031e7f1363a893cdd6ca0f8de0/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/75/fc/cc2fcac943939c8e4d8791abfa139f685e5150cae9f94b60f12520feaa9b/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a8/b7/a4add1d9a5f68f3d037261aecca83abdb0ab15960a3591d340e829b37298/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6c/18/c094561b5d64a24277707698e54b7f67bd17a4f857bbfbb1072bba07c8bf/charset_normalizer-3.4.6-cp312-cp312-win32.whl", hash = "sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ab/20/0567efb3a8fd481b8f34f739ebddc098ed062a59fed41a8d193a61939e8f/charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/15/57/28d79b44b51933119e21f65479d0864a8d5893e494cf5daab15df0247c17/charset_normalizer-3.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1e/1d/4fdabeef4e231153b6ed7567602f3b68265ec4e5b76d6024cf647d43d981/charset_normalizer-3.4.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/47/7b/20e809b89c69d37be748d98e84dce6820bf663cf19cf6b942c951a3e8f41/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843" },
{ url = "https://mirrors.aliyun.com/pypi/packages/37/a6/4f8d27527d59c039dce6f7622593cdcd3d70a8504d87d09eb11e9fdc6062/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf" },
@@ -1382,17 +1337,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69" },
- { url = "https://mirrors.aliyun.com/pypi/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc" },
{ url = "https://mirrors.aliyun.com/pypi/packages/68/35/0167aad910bbdb9599272bd96d01a9ec6852f36b9455cf2ca67bd4cc2d23/contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5" },
{ url = "https://mirrors.aliyun.com/pypi/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286" },
@@ -1445,21 +1389,6 @@ version = "7.13.5"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422" },
- { url = "https://mirrors.aliyun.com/pypi/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376" },
- { url = "https://mirrors.aliyun.com/pypi/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256" },
- { url = "https://mirrors.aliyun.com/pypi/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09" },
- { url = "https://mirrors.aliyun.com/pypi/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de" },
{ url = "https://mirrors.aliyun.com/pypi/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3" },
{ url = "https://mirrors.aliyun.com/pypi/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26" },
@@ -1529,21 +1458,6 @@ version = "2.11.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/14/12/34bf6e840a79130dfd0da7badfb6f7810b8fcfd60e75b0539372667b41b6/cramjam-2.11.0.tar.gz", hash = "sha256:5c82500ed91605c2d9781380b378397012e25127e89d64f460fea6aeac4389b4" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/0b/0d/7c84c913a5fae85b773a9dcf8874390f9d68ba0fcc6630efa7ff1541b950/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:dba5c14b8b4f73ea1e65720f5a3fe4280c1d27761238378be8274135c60bbc6e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/cc/4f6d185d8a744776f53035e72831ff8eefc2354f46ab836f4bd3c4f6c138/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:11eb40722b3fcf3e6890fba46c711bf60f8dc26360a24876c85e52d76c33b25b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1c/a8/626c76263085c6d5ded0e71823b411e9522bfc93ba6cc59855a5869296e7/cramjam-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aeb26e2898994b6e8319f19a4d37c481512acdcc6d30e1b5ecc9d8ec57e835cb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e9/52/0851a16a62447532e30ba95a80e638926fdea869a34b4b5b9d0a020083ba/cramjam-2.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f8d82081ed7d8fe52c982bd1f06e4c7631a73fe1fb6d4b3b3f2404f87dc40fe" },
- { url = "https://mirrors.aliyun.com/pypi/packages/98/76/122e444f59dbc216451d8e3d8282c9665dc79eaf822f5f1470066be1b695/cramjam-2.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:092a3ec26e0a679305018380e4f652eae1b6dfe3fc3b154ee76aa6b92221a17c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a3/bc/3a0189aef1af2b29632c039c19a7a1b752bc21a4053582a5464183a0ad3d/cramjam-2.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:529d6d667c65fd105d10bd83d1cd3f9869f8fd6c66efac9415c1812281196a92" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2e/80/8a6343b13778ce52d94bb8d5365a30c3aa951276b1857201fe79d7e2ad25/cramjam-2.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:555eb9c90c450e0f76e27d9ff064e64a8b8c6478ab1a5594c91b7bc5c82fd9f0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/df/6b/cd1778a207c29eda10791e3dfa018b588001928086e179fc71254793c625/cramjam-2.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5edf4c9e32493035b514cf2ba0c969d81ccb31de63bd05490cc8bfe3b431674e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/dc/f0/5c2a5cd5711032f3b191ca50cb786c17689b4a9255f9f768866e6c9f04d9/cramjam-2.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa2fe41f48c4d58d923803383b0737f048918b5a0d10390de9628bb6272b107" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f9/8b/b363a5fb2c3347504fe9a64f8d0f1e276844f0e532aa7162c061cd1ffee4/cramjam-2.11.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9ca14cf1cabdb0b77d606db1bb9e9ca593b1dbd421fcaf251ec9a5431ec449f3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/78/7b/d83dad46adb6c988a74361f81ad9c5c22642be53ad88616a19baedd06243/cramjam-2.11.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:309e95bf898829476bccf4fd2c358ec00e7ff73a12f95a3cdeeba4bb1d3683d5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1a/be/60d9be4cb33d8740a4aa94c7513f2ef3c4eba4fd13536f086facbafade71/cramjam-2.11.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:86dca35d2f15ef22922411496c220f3c9e315d5512f316fe417461971cc1648d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/11/b0/4a595f01a243aec8ad272b160b161c44351190c35d98d7787919d962e9e5/cramjam-2.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:193c6488bd2f514cbc0bef5c18fad61a5f9c8d059dd56edf773b3b37f0e85496" },
- { url = "https://mirrors.aliyun.com/pypi/packages/38/47/7776659aaa677046b77f527106e53ddd47373416d8fcdb1e1a881ec5dc06/cramjam-2.11.0-cp312-cp312-win32.whl", hash = "sha256:514e2c008a8b4fa823122ca3ecab896eac41d9aa0f5fc881bd6264486c204e32" },
- { url = "https://mirrors.aliyun.com/pypi/packages/75/b1/d53002729cfd94c5844ddfaf1233c86d29f2dbfc1b764a6562c41c044199/cramjam-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:53fed080476d5f6ad7505883ec5d1ec28ba36c2273db3b3e92d7224fe5e463db" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0a/8b/406c5dc0f8e82385519d8c299c40fd6a56d97eca3fcd6f5da8dad48de75b/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2c289729cc1c04e88bafa48b51082fb462b0a57dbc96494eab2be9b14dca62af" },
{ url = "https://mirrors.aliyun.com/pypi/packages/00/ad/4186884083d6e4125b285903e17841827ab0d6d0cffc86216d27ed91e91d/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:045201ee17147e36cf43d8ae2fa4b4836944ac672df5874579b81cf6d40f1a1f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/54/01/91b485cf76a7efef638151e8a7d35784dae2c4ff221b1aec2c083e4b106d/cramjam-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:619cd195d74c9e1d2a3ad78d63451d35379c84bd851aec552811e30842e1c67a" },
@@ -1641,15 +1555,6 @@ version = "2.8"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/b6/36/fd18ef23c42926b79c7003e16cb0f79043b5b179c633521343d3b499e996/crc32c-2.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:572ffb1b78cce3d88e8d4143e154d31044a44be42cb3f6fbbf77f1e7a941c5ab" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7f/b8/c584958e53f7798dd358f5bdb1bbfc97483134f053ee399d3eeb26cca075/crc32c-2.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf827b3758ee0c4aacd21ceca0e2da83681f10295c38a10bfeb105f7d98f7a68" },
- { url = "https://mirrors.aliyun.com/pypi/packages/62/e6/6f2af0ec64a668a46c861e5bc778ea3ee42171fedfc5440f791f470fd783/crc32c-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:106fbd79013e06fa92bc3b51031694fcc1249811ed4364ef1554ee3dd2c7f5a2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/17/8b/4a04bd80a024f1a23978f19ae99407783e06549e361ab56e9c08bba3c1d3/crc32c-2.8-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6dde035f91ffbfe23163e68605ee5a4bb8ceebd71ed54bb1fb1d0526cdd125a2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/21/8f/01c7afdc76ac2007d0e6a98e7300b4470b170480f8188475b597d1f4b4c6/crc32c-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e41ebe7c2f0fdcd9f3a3fd206989a36b460b4d3f24816d53e5be6c7dba72c5e1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/32/2b/8f78c5a8cc66486be5f51b6f038fc347c3ba748d3ea68be17a014283c331/crc32c-2.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ecf66cf90266d9c15cea597d5cc86c01917cd1a238dc3c51420c7886fa750d7e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/db/86/fad1a94cdeeeb6b6e2323c87f970186e74bfd6fbfbc247bf5c88ad0873d5/crc32c-2.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:59eee5f3a69ad0793d5fa9cdc9b9d743b0cd50edf7fccc0a3988a821fef0208c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d5/db/1a7cb6757a1e32376fa2dfce00c815ea4ee614a94f9bff8228e37420c183/crc32c-2.8-cp312-cp312-win32.whl", hash = "sha256:a73d03ce3604aa5d7a2698e9057a0eef69f529c46497b27ee1c38158e90ceb76" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bf/8e/2024de34399b2e401a37dcb54b224b56c747b0dc46de4966886827b4d370/crc32c-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:56b3b7d015247962cf58186e06d18c3d75a1a63d709d3233509e1c50a2d36aa2" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395" },
{ url = "https://mirrors.aliyun.com/pypi/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b" },
@@ -1776,14 +1681,6 @@ version = "2.0.13"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/c0/8f/2f0fbb32535c3731b7c2974c569fb9325e0a38ed5565a08e1139a3b71e82/cymem-2.0.13.tar.gz", hash = "sha256:1c91a92ae8c7104275ac26bd4d29b08ccd3e7faff5893d3858cb6fadf1bc1588" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/c9/52/478a2911ab5028cb710b4900d64aceba6f4f882fcb13fd8d40a456a1b6dc/cymem-2.0.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8afbc5162a0fe14b6463e1c4e45248a1b2fe2cbcecc8a5b9e511117080da0eb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f9/71/f0f8adee945524774b16af326bd314a14a478ed369a728a22834e6785a18/cymem-2.0.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9251d889348fe79a75e9b3e4d1b5fa651fca8a64500820685d73a3acc21b6a8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/62/6d/159780fe162ff715d62b809246e5fc20901cef87ca28b67d255a8d741861/cymem-2.0.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:742fc19764467a49ed22e56a4d2134c262d73a6c635409584ae3bf9afa092c33" },
- { url = "https://mirrors.aliyun.com/pypi/packages/eb/12/678d16f7aa1996f947bf17b8cfb917ea9c9674ef5e2bd3690c04123d5680/cymem-2.0.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f190a92fe46197ee64d32560eb121c2809bb843341733227f51538ce77b3410d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/31/5d/0dd8c167c08cd85e70d274b7235cfe1e31b3cebc99221178eaf4bbb95c6f/cymem-2.0.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d670329ee8dbbbf241b7c08069fe3f1d3a1a3e2d69c7d05ea008a7010d826298" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/c9/d6514a412a1160aa65db539836b3d47f9b59f6675f294ec34ae32f867c82/cymem-2.0.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a84ba3178d9128b9ffb52ce81ebab456e9fe959125b51109f5b73ebdfc6b60d6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/dd/fe/3ee37d02ca4040f2fb22d34eb415198f955862b5dd47eee01df4c8f5454c/cymem-2.0.13-cp312-cp312-win_amd64.whl", hash = "sha256:2ff1c41fd59b789579fdace78aa587c5fc091991fa59458c382b116fc36e30dc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/94/fb/1b681635bfd5f2274d0caa8f934b58435db6c091b97f5593738065ddb786/cymem-2.0.13-cp312-cp312-win_arm64.whl", hash = "sha256:6bbd701338df7bf408648191dff52472a9b334f71bcd31a21a41d83821050f67" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ce/0f/95a4d1e3bebfdfa7829252369357cf9a764f67569328cd9221f21e2c952e/cymem-2.0.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:891fd9030293a8b652dc7fb9fdc79a910a6c76fc679cd775e6741b819ffea476" },
{ url = "https://mirrors.aliyun.com/pypi/packages/bf/a0/8fc929cc29ae466b7b4efc23ece99cbd3ea34992ccff319089c624d667fd/cymem-2.0.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89c4889bd16513ce1644ccfe1e7c473ba7ca150f0621e66feac3a571bde09e7e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4a/b3/deeb01354ebaf384438083ffe0310209ef903db3e7ba5a8f584b06d28387/cymem-2.0.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:45dcaba0f48bef9cc3d8b0b92058640244a95a9f12542210b51318da97c2cf28" },
@@ -1858,10 +1755,6 @@ version = "1.8.20"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/e0/b7/cd8080344452e4874aae67c40d8940e2b4d47b01601a8fd9f44786c757c7/debugpy-1.8.20.tar.gz", hash = "sha256:55bc8701714969f1ab89a6d5f2f3d40c36f91b2cbe2f65d98bf8196f6a6a2c33" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/14/57/7f34f4736bfb6e00f2e4c96351b07805d83c9a7b33d28580ae01374430f7/debugpy-1.8.20-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:4ae3135e2089905a916909ef31922b2d733d756f66d87345b3e5e52b7a55f13d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ab/78/b193a3975ca34458f6f0e24aaf5c3e3da72f5401f6054c0dfd004b41726f/debugpy-1.8.20-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:88f47850a4284b88bd2bfee1f26132147d5d504e4e86c22485dfa44b97e19b4b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c1/55/f14deb95eaf4f30f07ef4b90a8590fc05d9e04df85ee379712f6fb6736d7/debugpy-1.8.20-cp312-cp312-win32.whl", hash = "sha256:4057ac68f892064e5f98209ab582abfee3b543fb55d2e87610ddc133a954d390" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a1/39/2bef246368bd42f9bd7cba99844542b74b84dacbdbea0833e610f384fee8/debugpy-1.8.20-cp312-cp312-win_amd64.whl", hash = "sha256:a1a8f851e7cf171330679ef6997e9c579ef6dd33c9098458bd9986a0f4ca52e3" },
{ url = "https://mirrors.aliyun.com/pypi/packages/15/e2/fc500524cc6f104a9d049abc85a0a8b3f0d14c0a39b9c140511c61e5b40b/debugpy-1.8.20-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:5dff4bb27027821fdfcc9e8f87309a28988231165147c31730128b1c983e282a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/90/83/fb33dcea789ed6018f8da20c5a9bc9d82adc65c0c990faed43f7c955da46/debugpy-1.8.20-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:84562982dd7cf5ebebfdea667ca20a064e096099997b175fe204e86817f64eaf" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a6/25/b1e4a01bfb824d79a6af24b99ef291e24189080c93576dfd9b1a2815cd0f/debugpy-1.8.20-cp313-cp313-win32.whl", hash = "sha256:da11dea6447b2cadbf8ce2bec59ecea87cc18d2c574980f643f2d2dfe4862393" },
@@ -2019,19 +1912,6 @@ name = "editdistance"
version = "0.8.1"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz", hash = "sha256:d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/cb/4c/7f195588949b4e72436dc7fc902632381f96e586af829685b56daebb38b8/editdistance-0.8.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04af61b3fcdd287a07c15b6ae3b02af01c5e3e9c3aca76b8c1d13bd266b6f57" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8d/82/31dc1640d830cd7d36865098329f34e4dad3b77f31cfb9404b347e700196/editdistance-0.8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:18fc8b6eaae01bfd9cf999af726c1e8dcf667d120e81aa7dbd515bea7427f62f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ea/2a/6b823e71cef694d6f070a1d82be2842706fa193541aab8856a8f42044cd0/editdistance-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a87839450a5987028738d061ffa5ef6a68bac2ddc68c9147a8aae9806629c7f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e1/31/bfb8e590f922089dc3471ed7828a6da2fc9453eba38c332efa9ee8749fd7/editdistance-0.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24b5f9c9673c823d91b5973d0af8b39f883f414a55ade2b9d097138acd10f31e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a9/c7/57423942b2f847cdbbb46494568d00cd8a45500904ea026f0aad6ca01bc7/editdistance-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59248eabfad603f0fba47b0c263d5dc728fb01c2b6b50fb6ca187cec547fdb3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1b/05/dfa4cdcce063596cbf0d7a32c46cd0f4fa70980311b7da64d35f33ad02a0/editdistance-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e239d88ff52821cf64023fabd06a1d9a07654f364b64bf1284577fd3a79d0e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0e/14/39608ff724a9523f187c4e28926d78bc68f2798f74777ac6757981108345/editdistance-0.8.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2f7f71698f83e8c83839ac0d876a0f4ef996c86c5460aebd26d85568d4afd0db" },
- { url = "https://mirrors.aliyun.com/pypi/packages/df/92/4a1c61d72da40dedfd0ff950fdc71ae83f478330c58a8bccfd776518bd67/editdistance-0.8.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:04e229d6f4ce0c12abc9f4cd4023a5b5fa9620226e0207b119c3c2778b036250" },
- { url = "https://mirrors.aliyun.com/pypi/packages/47/3d/9877566e724c8a37f2228a84ec5cbf66dbfd0673515baf68a0fe07caff40/editdistance-0.8.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e16721636da6d6b68a2c09eaced35a94f4a4a704ec09f45756d4fd5e128ed18d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d2/f5/8c50757d198b8ca30ddb91e8b8f0247a8dca04ff2ec30755245f0ab1ff0c/editdistance-0.8.1-cp312-cp312-win32.whl", hash = "sha256:87533cf2ebc3777088d991947274cd7e1014b9c861a8aa65257bcdc0ee492526" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/f0/65101e51dc7c850e7b7581a5d8fa8721a1d7479a0dca6c08386328e19882/editdistance-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:09f01ed51746d90178af7dd7ea4ebb41497ef19f53c7f327e864421743dffb0a" },
-]
[[package]]
name = "elastic-transport"
@@ -2102,9 +1982,6 @@ wheels = [
name = "exceptiongroup"
version = "1.3.1"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
-]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219" }
wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598" },
@@ -2160,12 +2037,6 @@ version = "1.12.1"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/65/8b/fa2d3287fd2267be6261d0177c6809a7fa12c5600ddb33490c8dc29e77b2/fastavro-1.12.1.tar.gz", hash = "sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/7c/f0/10bd1a3d08667fa0739e2b451fe90e06df575ec8b8ba5d3135c70555c9bd/fastavro-1.12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167" },
- { url = "https://mirrors.aliyun.com/pypi/packages/78/ad/0d985bc99e1fa9e74c636658000ba38a5cd7f5ab2708e9c62eaf736ecf1a/fastavro-1.12.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0d/9e/b4951dc84ebc34aac69afcbfbb22ea4a91080422ec2bfd2c06076ff1d419/fastavro-1.12.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34" },
- { url = "https://mirrors.aliyun.com/pypi/packages/af/f8/5a8df450a9f55ca8441f22ea0351d8c77809fc121498b6970daaaf667a21/fastavro-1.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/99/b2/40f25299111d737e58b85696e91138a66c25b7334f5357e7ac2b0e8966f8/fastavro-1.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e0/07/85157a7c57c5f8b95507d7829b5946561e5ee656ff80e9dd9a757f53ddaf/fastavro-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/bb/57/26d5efef9182392d5ac9f253953c856ccb66e4c549fd3176a1e94efb05c9/fastavro-1.12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:78df838351e4dff9edd10a1c41d1324131ffecbadefb9c297d612ef5363c049a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/33/cb/8ab55b21d018178eb126007a56bde14fd01c0afc11d20b5f2624fe01e698/fastavro-1.12.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:780476c23175d2ae457c52f45b9ffa9d504593499a36cd3c1929662bf5b7b14b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fe/03/9c94ec9bf873eb1ffb0aa694f4e71940154e6e9728ddfdc46046d7e8ced4/fastavro-1.12.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0714b285160fcd515eb0455540f40dd6dac93bdeacdb03f24e8eac3d8aa51f8d" },
@@ -2201,15 +2072,6 @@ dependencies = [
{ name = "pandas" },
]
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/97/9c/f2c018807cab35716df732be6c09ec017ad9ee40dc2e876b10ed5d9a963e/fastparquet-2026.3.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:c413adcea221c11e8a14d096d825b42d4f0b4b6621f64d6c13f4a433574906e6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/93/bf/6470b62e3eabb46e5abc6ad4e0c13587e1448f2365f7c35079fe4d6602ab/fastparquet-2026.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4623c12e5dc05f6164cad7a2f6962c1e8f69f4670abd6b19fe7b1f13b4f4937d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e8/ef/78a5db203e2e1d19249286f52ecb5531b8863e56a346d9d193633c3030fd/fastparquet-2026.3.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:41def5d94abf44830e58b4c2ed137b71f2f0e068c6241a1d2524595178880851" },
- { url = "https://mirrors.aliyun.com/pypi/packages/61/99/e43283ac6cc83269c8214b8ee57e7773ea5f39016a8e8fcfe4529fa2cc30/fastparquet-2026.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a5ddb40b58b62ef660ea9f0774d3b3cfe6d0b88c20b44b986e500439290de81" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cd/9a/f5aada3af89dcd3027e543fa39756f67790daef0c31f03973bb97c6171c9/fastparquet-2026.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6cc397aa8ca5bb2e84670270b46a89e6d6e426f8bfce5437d028a90cd2d8b3d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/07/92/bdd4d8dfd59a6ae92b33eab0f583fd5099188c8065d875d22782f26b79e0/fastparquet-2026.3.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:46c60acc4b5752cd883bd0cc9076a01698b1b5f28cb7f94449fba68f40758316" },
- { url = "https://mirrors.aliyun.com/pypi/packages/70/7d/d46abd9713f53d90ebc47c373d78ddb34c24e5fa6a02c5a974370f8a57b0/fastparquet-2026.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c61d734cff4d29f16bf1c813b4d1725dec3676cb82a2f617713a894b4e97546d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/16/6dda2bf60e830feb1a1bdecb01e8aa33b011058ee767418cef4bc68a1249/fastparquet-2026.3.0-cp312-cp312-win32.whl", hash = "sha256:8835b763f1843ecde3f7e8bc9deda4a7dc317b65b1dfc9a10e7e4f26eac73ce4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/45/5b/7cc76aa44962280e496f35715f172afbd6476fcde5ecfa8fdc1c30416b03/fastparquet-2026.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:db0698e4e34788baadb4d8871f93409c9803bc661b7d58d90f616ded889289bb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/af/aa/3dbde9b0592a7aca0489edefa368b861a7d85df1ec51d7f5f05d83c4ad0f/fastparquet-2026.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4e0f5464bc0661b345e26aa7feab34bd21c9ca2d3c4f411278f50c76e7adb7f2" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b5/f1/d81496c2887f166ea7222ef81d489dcc139ff3dc0f4b0393c0d201bdfb47/fastparquet-2026.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97d48ea111b0cc09bf99b97c2218c5fd24abac8b53879b4ce73eea55d5484a55" },
{ url = "https://mirrors.aliyun.com/pypi/packages/6d/1d/dba2033c57087d74ec463fbf9fc23b57a1bd731db38877f2b002d8b8c05b/fastparquet-2026.3.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ab0c62c1890def8a40f3d878fc75fbf725a21df4e3676da74a56195346824bb0" },
@@ -2245,17 +2107,6 @@ version = "0.14.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/02/a2/e78fcc5df65467f0d207661b7ef86c5b7ac62eea337c0c0fcedbeee6fb13/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/b3/c846f933f22f581f558ee63f81f29fa924acd971ce903dab1a9b6701816e/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/54/ea/682551030f8c4fa9a769d9825570ad28c0c71e30cf34020b85c1f7ee7382/fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070" },
- { url = "https://mirrors.aliyun.com/pypi/packages/14/dd/5927f0a523d8e6a76b70968e6004966ee7df30322f5fc9b6cdfb0276646a/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796" },
- { url = "https://mirrors.aliyun.com/pypi/packages/16/6e/c0fb547eef61293153348f12e0f75a06abb322664b34a1573a7760501336/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2d/b1/b9c75e03b768f61cf2e84ee193dc18601aeaf89a4684b20f2f0e9f52b62c/fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fc/fa/f7395fdac07c7a54f18f801744573707321ca0cee082e638e36452355a9d/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741" },
- { url = "https://mirrors.aliyun.com/pypi/packages/66/49/c9fd06a4a0b1f0f048aacb6599e7d96e5d6bc6fa680ed0d46bf111929d1b/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057" },
- { url = "https://mirrors.aliyun.com/pypi/packages/be/9c/909e8c95b494e8e140e8be6165d5fc3f61fdc46198c1554df7b3e1764471/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/90/eb/d29d17521976e673c55ef7f210d4cdd72091a9ec6755d0fd4710d9b3c871/fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cc/fc/f5c799a6ea6d877faec0472d0b27c079b47c86b1cdc577720a5386483b36/fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a5/83/ae12dd39b9a39b55d7f90abb8971f1a5f3c321fd72d5aa83f90dc67fe9ed/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021" },
{ url = "https://mirrors.aliyun.com/pypi/packages/53/b0/a4b03ff5d00f563cc7546b933c28cb3f2a07344b2aec5834e874f7d44143/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9c/6d/64aee0a0f6a58eeabadd582e55d0d7d70258ffdd01d093b30c53d668303b/fastuuid-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5" },
@@ -2411,14 +2262,6 @@ version = "4.62.1"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/9a/08/7012b00a9a5874311b639c3920270c36ee0c445b69d9989a85e5c92ebcb0/fonttools-4.62.1.tar.gz", hash = "sha256:e54c75fd6041f1122476776880f7c3c3295ffa31962dc6ebe2543c00dca58b5d" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/47/d4/dbacced3953544b9a93088cc10ef2b596d348c983d5c67a404fa41ec51ba/fonttools-4.62.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:90365821debbd7db678809c7491ca4acd1e0779b9624cdc6ddaf1f31992bf974" },
- { url = "https://mirrors.aliyun.com/pypi/packages/66/9e/a769c8e99b81e5a87ab7e5e7236684de4e96246aae17274e5347d11ebd78/fonttools-4.62.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12859ff0b47dd20f110804c3e0d0970f7b832f561630cd879969011541a464a9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/69/64/f19a9e3911968c37e1e620e14dfc5778299e1474f72f4e57c5ec771d9489/fonttools-4.62.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c125ffa00c3d9003cdaaf7f2c79e6e535628093e14b5de1dccb08859b680936" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9b/8a/99c8b3c3888c5c474c08dbfd7c8899786de9604b727fcefb055b42c84bba/fonttools-4.62.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:149f7d84afca659d1a97e39a4778794a2f83bf344c5ee5134e09995086cc2392" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d1/c6/0f904540d3e6ab463c1243a0d803504826a11604c72dd58c2949796a1762/fonttools-4.62.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0aa72c43a601cfa9273bb1ae0518f1acadc01ee181a6fc60cd758d7fdadffc04" },
- { url = "https://mirrors.aliyun.com/pypi/packages/29/0b/5cbef6588dc9bd6b5c9ad6a4d5a8ca384d0cea089da31711bbeb4f9654a6/fonttools-4.62.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:19177c8d96c7c36359266e571c5173bcee9157b59cfc8cb0153c5673dc5a3a7d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4a/47/b3a5342d381595ef439adec67848bed561ab7fdb1019fa522e82101b7d9c/fonttools-4.62.1-cp312-cp312-win32.whl", hash = "sha256:a24decd24d60744ee8b4679d38e88b8303d86772053afc29b19d23bb8207803c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/b1/0c2ab56a16f409c6c8a68816e6af707827ad5d629634691ff60a52879792/fonttools-4.62.1-cp312-cp312-win_amd64.whl", hash = "sha256:9e7863e10b3de72376280b515d35b14f5eeed639d1aa7824f4cf06779ec65e42" },
{ url = "https://mirrors.aliyun.com/pypi/packages/3b/56/6f389de21c49555553d6a5aeed5ac9767631497ac836c4f076273d15bd72/fonttools-4.62.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c22b1014017111c401469e3acc5433e6acf6ebcc6aa9efb538a533c800971c79" },
{ url = "https://mirrors.aliyun.com/pypi/packages/03/c5/0e3966edd5ec668d41dfe418787726752bc07e2f5fd8c8f208615e61fa89/fonttools-4.62.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68959f5fc58ed4599b44aad161c2837477d7f35f5f79402d97439974faebfebe" },
{ url = "https://mirrors.aliyun.com/pypi/packages/52/94/e6ac4b44026de7786fe46e3bfa0c87e51d5d70a841054065d49cd62bb909/fonttools-4.62.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef46db46c9447103b8f3ff91e8ba009d5fe181b1920a83757a5762551e32bb68" },
@@ -2471,22 +2314,6 @@ version = "1.8.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29" },
- { url = "https://mirrors.aliyun.com/pypi/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa" },
- { url = "https://mirrors.aliyun.com/pypi/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd" },
{ url = "https://mirrors.aliyun.com/pypi/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40" },
@@ -2572,15 +2399,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216" },
]
-[[package]]
-name = "gast"
-version = "0.7.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/91/f6/e73969782a2ecec280f8a176f2476149dd9dba69d5f8779ec6108a7721e6/gast-0.7.0.tar.gz", hash = "sha256:0bb14cd1b806722e91ddbab6fb86bba148c22b40e7ff11e248974e04c8adfdae" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/1d/33/f1c6a276de27b7d7339a34749cc33fa87f077f921969c47185d34a887ae2/gast-0.7.0-py3-none-any.whl", hash = "sha256:99cbf1365633a74099f69c59bd650476b96baa5ef196fec88032b00b31ba36f7" },
-]
-
[[package]]
name = "gensim"
version = "4.4.0"
@@ -2592,11 +2410,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/1a/80/fe9d2e1ace968041814dbcfce4e8499a643a36c41267fa4b6c4f54cce420/gensim-4.4.0.tar.gz", hash = "sha256:a3f5b626da5518e79a479140361c663089fe7998df8ba52d56e1ded71ac5bdf5" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/4f/65/d5285865ca54b93d41ccd8683c2d79952434957c76b411283c7a6c66ca69/gensim-4.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0845b2fa039dbea5667fb278b5414e70f6d48fd208ef51f33e84a78444288d8d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/32/59/f0ea443cbfb3b06e1d2e060217bb91f954845f6df38cbc9c5468b6c9c638/gensim-4.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1853fc5be730f692c444a826041fef9a2fc8d74c73bb59748904b2e3221daa86" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f0/b8/9b0ba15756e41ccfdd852f9c65cd2b552f240c201dc3237ad8c178642e80/gensim-4.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23a2a4260f01c8f71bae5dd0e8a01bb247a2c789480c033e0eaba100b0ad4239" },
- { url = "https://mirrors.aliyun.com/pypi/packages/97/2c/c29701826c963b04a43d5d7b87573a74040387ab9219e65b10f377d22b5b/gensim-4.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b73ff30af6ddd0d2ddf9473b1eb44603cd79ec14c87d93b75291802b991916c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fd/f2/9ec6863143888bf390cdc5261f6d9e71d79bc95d98fb815679dba478d5f6/gensim-4.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b3a3f9bc8d4178b01d114e1c58c5ab2333f131c7415fb3d8ec8f1ecfe4c5b544" },
{ url = "https://mirrors.aliyun.com/pypi/packages/80/6c/4e522973e07ca491d33cc7829996b9e8c8663a16b3f87f580cbdc2732d97/gensim-4.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8961b7a2bb5190b46bc6cd26c29d5bfea22f99123ed5f506ebd0aaf65996758" },
{ url = "https://mirrors.aliyun.com/pypi/packages/cc/6a/593107ee98331128ed20e5d074865587558a0766659be787a40550ab66df/gensim-4.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:59d0d29099a76dd97d4563e002f3488a43e51f99d46387025da38007ebfeeff9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d9/ef/1675e1a3a04f7d0293a21082f57f4a6a8bf0a9e387da58b71db648b663de/gensim-4.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3bec3e6a1ecaa6439b21a3e42ceb0ca67ffabc114b646f89b1aab5fe69a39ffc" },
@@ -2728,11 +2541,6 @@ version = "1.8.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113" },
- { url = "https://mirrors.aliyun.com/pypi/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411" },
- { url = "https://mirrors.aliyun.com/pypi/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454" },
- { url = "https://mirrors.aliyun.com/pypi/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa" },
@@ -2766,18 +2574,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/84/de/7d3ee9c94b74c3578ea4f88d45e8de9405902f857932334d81e89bce3dfa/google_genai-1.68.0-py3-none-any.whl", hash = "sha256:a1bc9919c0e2ea2907d1e319b65471d3d6d58c54822039a249fe1323e4178d15" },
]
-[[package]]
-name = "google-pasta"
-version = "0.2.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "six" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed" },
-]
-
[[package]]
name = "google-resumable-media"
version = "2.8.0"
@@ -2865,15 +2661,6 @@ version = "3.3.2"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/a3/51/1664f6b78fc6ebbd98019a1fd730e83fa78f2db7058f72b1463d3612b8db/greenlet-3.3.2.tar.gz", hash = "sha256:2eaf067fc6d886931c7962e8c6bede15d2f01965560f3359b27c80bde2d151f2" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9b/40/cc802e067d02af8b60b6771cea7d57e21ef5e6659912814babb42b864713/greenlet-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:34308836d8370bddadb41f5a7ce96879b72e2fdfb4e87729330c6ab52376409f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/58/2e/fe7f36ff1982d6b10a60d5e0740c759259a7d6d2e1dc41da6d96de32fff6/greenlet-3.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:d3a62fa76a32b462a97198e4c9e99afb9ab375115e74e9a83ce180e7a496f643" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986" },
{ url = "https://mirrors.aliyun.com/pypi/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92" },
@@ -2928,16 +2715,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/06/8a/3d098f35c143a89520e568e6539cc098fcd294495910e359889ce8741c84/grpcio-1.78.0.tar.gz", hash = "sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/4e/f4/7384ed0178203d6074446b3c4f46c90a22ddf7ae0b3aee521627f54cfc2a/grpcio-1.78.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97" },
- { url = "https://mirrors.aliyun.com/pypi/packages/81/ed/be1caa25f06594463f685b3790b320f18aea49b33166f4141bfdc2bfb236/grpcio-1.78.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/24/a7/f06d151afc4e64b7e3cc3e872d331d011c279aaab02831e40a81c691fb65/grpcio-1.78.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8a/a8/4482922da832ec0082d0f2cc3a10976d84a7424707f25780b82814aafc0a/grpcio-1.78.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/54/bf/f4a3b9693e35d25b24b0b39fa46d7d8a3c439e0a3036c3451764678fec20/grpcio-1.78.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c7/b9/521875265cc99fe5ad4c5a17010018085cae2810a928bf15ebe7d8bcd9cc/grpcio-1.78.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383" },
- { url = "https://mirrors.aliyun.com/pypi/packages/05/86/296a82844fd40a4ad4a95f100b55044b4f817dece732bf686aea1a284147/grpcio-1.78.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f3/e4/ea3c0caf5468537f27ad5aab92b681ed7cc0ef5f8c9196d3fd42c8c2286b/grpcio-1.78.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d7/47/7f05f81e4bb6b831e93271fb12fd52ba7b319b5402cbc101d588f435df00/grpcio-1.78.0-cp312-cp312-win32.whl", hash = "sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ad/e7/d6914822c88aa2974dbbd10903d801a28a19ce9cd8bad7e694cbbcf61528/grpcio-1.78.0-cp312-cp312-win_amd64.whl", hash = "sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/05/a9/8f75894993895f361ed8636cd9237f4ab39ef87fd30db17467235ed1c045/grpcio-1.78.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:ce3a90455492bf8bfa38e56fbbe1dbd4f872a3d8eeaf7337dc3b1c8aa28c271b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/55/06/0b78408e938ac424100100fd081189451b472236e8a3a1f6500390dc4954/grpcio-1.78.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:2bf5e2e163b356978b23652c4818ce4759d40f4712ee9ec5a83c4be6f8c23a3a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/88/93/b59fe7832ff6ae3c78b813ea43dac60e295fa03606d14d89d2e0ec29f4f3/grpcio-1.78.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8f2ac84905d12918e4e55a16da17939eb63e433dc11b677267c35568aa63fc84" },
@@ -2985,16 +2762,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/ad/9a/edfefb47f11ef6b0f39eea4d8f022c5bb05ac1d14fcc7058e84a51305b73/grpcio_tools-1.71.2.tar.gz", hash = "sha256:b5304d65c7569b21270b568e404a5a843cf027c66552a6a0978b23f137679c09" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/9c/d3/3ed30a9c5b2424627b4b8411e2cd6a1a3f997d3812dbc6a8630a78bcfe26/grpcio_tools-1.71.2-cp312-cp312-linux_armv7l.whl", hash = "sha256:bfc0b5d289e383bc7d317f0e64c9dfb59dc4bef078ecd23afa1a816358fb1473" },
- { url = "https://mirrors.aliyun.com/pypi/packages/54/61/e0b7295456c7e21ef777eae60403c06835160c8d0e1e58ebfc7d024c51d3/grpcio_tools-1.71.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b4669827716355fa913b1376b1b985855d5cfdb63443f8d18faf210180199006" },
- { url = "https://mirrors.aliyun.com/pypi/packages/75/d7/7bcad6bcc5f5b7fab53e6bce5db87041f38ef3e740b1ec2d8c49534fa286/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:d4071f9b44564e3f75cdf0f05b10b3e8c7ea0ca5220acbf4dc50b148552eef2f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b2/8a/e4c1c4cb8c9ff7f50b7b2bba94abe8d1e98ea05f52a5db476e7f1c1a3c70/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a28eda8137d587eb30081384c256f5e5de7feda34776f89848b846da64e4be35" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fd/aa/95bc77fda5c2d56fb4a318c1b22bdba8914d5d84602525c99047114de531/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b19c083198f5eb15cc69c0a2f2c415540cbc636bfe76cea268e5894f34023b40" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c9/ff/ca11f930fe1daa799ee0ce1ac9630d58a3a3deed3dd2f465edb9a32f299d/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:784c284acda0d925052be19053d35afbf78300f4d025836d424cf632404f676a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/64/10/c6fc97914c7e19c9bb061722e55052fa3f575165da9f6510e2038d6e8643/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:381e684d29a5d052194e095546eef067201f5af30fd99b07b5d94766f44bf1ae" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e5/d6/965f36cfc367c276799b730d5dd1311b90a54a33726e561393b808339b04/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3e4b4801fabd0427fc61d50d09588a01b1cfab0ec5e8a5f5d515fbdd0891fd11" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8d/f0/c05d5c3d0c1d79ac87df964e9d36f1e3a77b60d948af65bec35d3e5c75a3/grpcio_tools-1.71.2-cp312-cp312-win32.whl", hash = "sha256:84ad86332c44572305138eafa4cc30040c9a5e81826993eae8227863b700b490" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e2/e9/c84c1078f0b7af7d8a40f5214a9bdd8d2a567ad6c09975e6e2613a08d29d/grpcio_tools-1.71.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e1108d37eecc73b1c4a27350a6ed921b5dda25091700c1da17cfe30761cd462" },
{ url = "https://mirrors.aliyun.com/pypi/packages/60/9c/bdf9c5055a1ad0a09123402d73ecad3629f75b9cf97828d547173b328891/grpcio_tools-1.71.2-cp313-cp313-linux_armv7l.whl", hash = "sha256:b0f0a8611614949c906e25c225e3360551b488d10a366c96d89856bcef09f729" },
{ url = "https://mirrors.aliyun.com/pypi/packages/49/d0/6aaee4940a8fb8269c13719f56d69c8d39569bee272924086aef81616d4a/grpcio_tools-1.71.2-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:7931783ea7ac42ac57f94c5047d00a504f72fbd96118bf7df911bb0e0435fc0f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d9/11/50a471dcf301b89c0ed5ab92c533baced5bd8f796abfd133bbfadf6b60e5/grpcio_tools-1.71.2-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:d188dc28e069aa96bb48cb11b1338e47ebdf2e2306afa58a8162cc210172d7a8" },
@@ -3029,49 +2796,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd" },
]
-[[package]]
-name = "h5py"
-version = "3.16.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "numpy" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/c8/c0/5d4119dba94093bbafede500d3defd2f5eab7897732998c04b54021e530b/h5py-3.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c5313566f4643121a78503a473f0fb1e6dcc541d5115c44f05e037609c565c4d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b0/42/c84efcc1d4caebafb1ecd8be4643f39c85c47a80fe254d92b8b43b1eadaf/h5py-3.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42b012933a83e1a558c673176676a10ce2fd3759976a0fedee1e672d1e04fc9d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/89/84/06281c82d4d1686fde1ac6b0f307c50918f1c0151062445ab3b6fa5a921d/h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/e9/1a19e42cd43cc1365e127db6aae85e1c671da1d9a5d746f4d34a50edb577/h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/8e/9790c1655eabeb85b92b1ecab7d7e62a2069e53baefd58c98f0909c7a948/h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794" },
- { url = "https://mirrors.aliyun.com/pypi/packages/51/d7/ab693274f1bd7e8c5f9fdd6c7003a88d59bedeaf8752716a55f532924fbb/h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074" },
- { url = "https://mirrors.aliyun.com/pypi/packages/03/c1/0976b235cf29ead553e22f2fb6385a8252b533715e00d0ae52ed7b900582/h5py-3.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:96b422019a1c8975c2d5dadcf61d4ba6f01c31f92bbde6e4649607885fe502d6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/14/d9/866b7e570b39070f92d47b0ff1800f0f8239b6f9e45f02363d7112336c1f/h5py-3.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:39c2838fb1e8d97bcf1755e60ad1f3dd76a7b2a475928dc321672752678b96db" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0f/9e/6142ebfda0cb6e9349c091eae73c2e01a770b7659255248d637bec54a88b/h5py-3.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:370a845f432c2c9619db8eed334d1e610c6015796122b0e57aa46312c22617d9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42108e93326c50c2810025aade9eac9d6827524cdccc7d4b75a546e5ab308edb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/da/1e/6172269e18cc5a484e2913ced33339aad588e02ba407fafd00d369e22ef3/h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bc/81/5b62d760039eed64348c98129d17061fdfc7839fc9c04eaaad6dee1004e4/h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/c4/532123bcd9080e250696779c927f2cb906c8bf3447df98f5ceb8dcded539/h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c3/d9/a27997f84341fc0dfcdd1fe4179b6ba6c32a7aa880fdb8c514d4dad6fba3/h5py-3.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:18f2bbcd545e6991412253b98727374c356d67caa920e68dc79eab36bf5fedad" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a5/23/bb8647521d4fd770c30a76cfc6cb6a2f5495868904054e92f2394c5a78ff/h5py-3.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:656f00e4d903199a1d58df06b711cf3ca632b874b4207b7dbec86185b5c8c7d4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/48/3c/7fcd9b4c9eed82e91fb15568992561019ae7a829d1f696b2c844355d95dd/h5py-3.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9c9d307c0ef862d1cd5714f72ecfafe0a5d7529c44845afa8de9f46e5ba8bd65" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6a/b7/9366ed44ced9b7ef357ab48c94205280276db9d7f064aa3012a97227e966/h5py-3.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c1eff849cdd53cbc73c214c30ebdb6f1bb8b64790b4b4fc36acdb5e43570210" },
- { url = "https://mirrors.aliyun.com/pypi/packages/58/a5/4964bc0e91e86340c2bbda83420225b2f770dcf1eb8a39464871ad769436/h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f1/16/d905e7f53e661ce2c24686c38048d8e2b750ffc4350009d41c4e6c6c9826/h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4b/f2/58f34cb74af46d39f4cd18ea20909a8514960c5a3e5b92fd06a28161e0a8/h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/ca/934a39c24ce2e2db017268c08da0537c20fa0be7e1549be3e977313fc8f5/h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3e/14/615a450205e1b56d16c6783f5ccd116cde05550faad70ae077c955654a75/h5py-3.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:fa48993a0b799737ba7fd21e2350fa0a60701e58180fae9f2de834bc39a147ab" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7b/48/a6faef5ed632cae0c65ac6b214a6614a0b510c3183532c521bdb0055e117/h5py-3.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:1897a771a7f40d05c262fc8f37376ec37873218544b70216872876c627640f63" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5d/32/0c8bb8aedb62c772cf7c1d427c7d1951477e8c2835f872bc0a13d1f85f86/h5py-3.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15922e485844f77c0b9d275396d435db3baa58292a9c2176a386e072e0cf2491" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1d/1f/fcc5977d32d6387c5c9a694afee716a5e20658ac08b3ff24fdec79fb05f2/h5py-3.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:df02dd29bd247f98674634dfe41f89fd7c16ba3d7de8695ec958f58404a4e618" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f5/a1/af87f64b9f986889884243643621ebbd4ac72472ba8ec8cec891ac8e2ca1/h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cc/d0/146f5eaff3dc246a9c7f6e5e4f42bd45cc613bce16693bcd4d1f7c958bf5/h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a1/9d/12a13424f1e604fc7df9497b73c0356fb78c2fb206abd7465ce47226e8fd/h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/41/8c/bbe98f813722b4873818a8db3e15aa3e625b59278566905ac439725e8070/h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725" },
- { url = "https://mirrors.aliyun.com/pypi/packages/32/9e/87e6705b4d6890e7cecdf876e2a7d3e40654a2ae37482d79a6f1b87f7b92/h5py-3.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4c6ab014ab704b4feaa719ae783b86522ed0bf1f82184704ed3c9e4e3228796e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/96/91/9fad90cfc5f9b2489c7c26ad897157bce82f0e9534a986a221b99760b23b/h5py-3.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:faca8fb4e4319c09d83337adc80b2ca7d5c5a343c2d6f1b6388f32cfecca13c1" },
-]
-
[[package]]
name = "hanziconv"
version = "0.3.2"
@@ -3343,17 +3067,6 @@ version = "3.5.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/f4/57/60d1a6a512f2f0508d0bc8b4f1cc5616fd3196619b66bd6a01f9155a1292/ijson-3.5.0.tar.gz", hash = "sha256:94688760720e3f5212731b3cb8d30267f9a045fb38fb3870254e7b9504246f31" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/17/9c63c7688025f3a8c47ea717b8306649c8c7244e49e20a2be4e3515dc75c/ijson-3.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1ebefbe149a6106cc848a3eaf536af51a9b5ccc9082de801389f152dba6ab755" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6f/dd/e15c2400244c117b06585452ebc63ae254f5a6964f712306afd1422daae0/ijson-3.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:19e30d9f00f82e64de689c0b8651b9cfed879c184b139d7e1ea5030cec401c21" },
- { url = "https://mirrors.aliyun.com/pypi/packages/77/a9/bf4fe3538a0c965f16b406f180a06105b875da83f0743e36246be64ef550/ijson-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a04a33ee78a6f27b9b8528c1ca3c207b1df3b8b867a4cf2fcc4109986f35c227" },
- { url = "https://mirrors.aliyun.com/pypi/packages/31/76/6f91bdb019dd978fce1bc5ea1cd620cfc096d258126c91db2c03a20a7f34/ijson-3.5.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d48dc2984af02eb3c56edfb3f13b3f62f2f3e4fe36f058c8cfc75d93adf4fed" },
- { url = "https://mirrors.aliyun.com/pypi/packages/11/be/bbc983059e48a54b0121ee60042979faed7674490bbe7b2c41560db3f436/ijson-3.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1e73a44844d9adbca9cf2c4132cd875933e83f3d4b23881fcaf82be83644c7d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6d/81/2fee58f9024a3449aee83edfa7167fb5ccd7e1af2557300e28531bb68e16/ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7389a56b8562a19948bdf1d7bae3a2edc8c7f86fb59834dcb1c4c722818e645a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c7/56/f1706761fcc096c9d414b3dcd000b1e6e5c24364c21cfba429837f98ee8d/ijson-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3176f23f8ebec83f374ed0c3b4e5a0c4db7ede54c005864efebbed46da123608" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d9/6e/ee0d9c875a0193b632b3e9ccd1b22a50685fb510256ad57ba483b6529f77/ijson-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6babd88e508630c6ef86c9bebaaf13bb2fb8ec1d8f8868773a03c20253f599bc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d2/bf/f9d4399d0e6e3fd615035290a71e97c843f17f329b43638c0a01cf112d73/ijson-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dc1b3836b174b6db2fa8319f1926fb5445abd195dc963368092103f8579cb8ed" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b2/71/a7254a065933c0e2ffd3586f46187d84830d3d7b6f41cfa5901820a4f87d/ijson-3.5.0-cp312-cp312-win32.whl", hash = "sha256:6673de9395fb9893c1c79a43becd8c8fbee0a250be6ea324bfd1487bb5e9ee4c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8f/7b/2edca79b359fc9f95d774616867a03ecccdf333797baf5b3eea79733918c/ijson-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4f7fabd653459dcb004175235f310435959b1bb5dfa8878578391c6cc9ad944" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a2/71/d67e764a712c3590627480643a3b51efcc3afa4ef3cb54ee4c989073c97e/ijson-3.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e9cedc10e40dd6023c351ed8bfc7dcfce58204f15c321c3c1546b9c7b12562a4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1a/39/f1c299371686153fa3cf5c0736b96247a87a1bee1b7145e6d21f359c505a/ijson-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3647649f782ee06c97490b43680371186651f3f69bebe64c6083ee7615d185e5" },
{ url = "https://mirrors.aliyun.com/pypi/packages/16/94/b1438e204d75e01541bebe3e668fe3e68612d210e9931ae1611062dd0a56/ijson-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:90e74be1dce05fce73451c62d1118671f78f47c9f6be3991c82b91063bf01fc9" },
@@ -3569,19 +3282,6 @@ version = "0.13.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/0d/5e/4ec91646aee381d01cdb9974e30882c9cd3b8c5d1079d6b5ff4af522439a/jiter-0.13.0.tar.gz", hash = "sha256:f2839f9c2c7e2dffc1bc5929a510e14ce0a946be9365fd1219e7ef342dae14f4" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/2e/30/7687e4f87086829955013ca12a9233523349767f69653ebc27036313def9/jiter-0.13.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0a2bd69fc1d902e89925fc34d1da51b2128019423d7b339a45d9e99c894e0663" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c3/27/e57f9a783246ed95481e6749cc5002a8a767a73177a83c63ea71f0528b90/jiter-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f917a04240ef31898182f76a332f508f2cc4b57d2b4d7ad2dbfebbfe167eb505" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/52/e5719a60ac5d4d7c5995461a94ad5ef962a37c8bf5b088390e6fad59b2ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e2b199f446d3e82246b4fd9236d7cb502dc2222b18698ba0d986d2fecc6152" },
- { url = "https://mirrors.aliyun.com/pypi/packages/61/db/c1efc32b8ba4c740ab3fc2d037d8753f67685f475e26b9d6536a4322bcdd/jiter-0.13.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04670992b576fa65bd056dbac0c39fe8bd67681c380cb2b48efa885711d9d726" },
- { url = "https://mirrors.aliyun.com/pypi/packages/55/8a/fb75556236047c8806995671a18e4a0ad646ed255276f51a20f32dceaeec/jiter-0.13.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a1aff1fbdb803a376d4d22a8f63f8e7ccbce0b4890c26cc7af9e501ab339ef0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7e/16/43512e6ee863875693a8e6f6d532e19d650779d6ba9a81593ae40a9088ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b3fb8c2053acaef8580809ac1d1f7481a0a0bdc012fd7f5d8b18fb696a5a089" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f8/4c/09b93e30e984a187bc8aaa3510e1ec8dcbdcd71ca05d2f56aac0492453aa/jiter-0.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaba7d87e66f26a2c45d8cbadcbfc4bf7884182317907baf39cfe9775bb4d93" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1a/1b/46c5e349019874ec5dfa508c14c37e29864ea108d376ae26d90bee238cd7/jiter-0.13.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b88d649135aca526da172e48083da915ec086b54e8e73a425ba50999468cc08" },
- { url = "https://mirrors.aliyun.com/pypi/packages/15/9e/26184760e85baee7162ad37b7912797d2077718476bf91517641c92b3639/jiter-0.13.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e404ea551d35438013c64b4f357b0474c7abf9f781c06d44fcaf7a14c69ff9e2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e9/34/2c9355247d6debad57a0a15e76ab1566ab799388042743656e566b3b7de1/jiter-0.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f4748aad1b4a93c8bdd70f604d0f748cdc0e8744c5547798acfa52f10e79228" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ac/4a/9f2c23255d04a834398b9c2e0e665382116911dc4d06b795710503cdad25/jiter-0.13.0-cp312-cp312-win32.whl", hash = "sha256:0bf670e3b1445fc4d31612199f1744f67f889ee1bbae703c4b54dc097e5dd394" },
- { url = "https://mirrors.aliyun.com/pypi/packages/09/ee/f0ae675a957ae5a8f160be3e87acea6b11dc7b89f6b7ab057e77b2d2b13a/jiter-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:15db60e121e11fe186c0b15236bd5d18381b9ddacdcf4e659feb96fc6c969c92" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1b/02/ae611edf913d3cbf02c97cdb90374af2082c48d7190d74c1111dde08bcdd/jiter-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:41f92313d17989102f3cb5dd533a02787cdb99454d494344b0361355da52fcb9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/91/9c/7ee5a6ff4b9991e1a45263bfc46731634c4a2bde27dfda6c8251df2d958c/jiter-0.13.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1f8a55b848cbabf97d861495cd65f1e5c590246fabca8b48e1747c4dfc8f85bf" },
{ url = "https://mirrors.aliyun.com/pypi/packages/7c/02/be5b870d1d2be5dd6a91bdfb90f248fbb7dcbd21338f092c6b89817c3dbf/jiter-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f556aa591c00f2c45eb1b89f68f52441a016034d18b65da60e2d2875bbbf344a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/da/92/b25d2ec333615f5f284f3a4024f7ce68cfa0604c322c6808b2344c7f5d2b/jiter-0.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7e1d61da332ec412350463891923f960c3073cf1aae93b538f0bb4c8cd46efb" },
@@ -3625,10 +3325,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/47/66/eea81dfff765ed66c68fd2ed8c96245109e13c896c2a5015c7839c92367e/jiter-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:24dc96eca9f84da4131cdf87a95e6ce36765c3b156fc9ae33280873b1c32d5f6" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ff/32/4ac9c7a76402f8f00d00842a7f6b83b284d0cf7c1e9d4227bc95aa6d17fa/jiter-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0a8d76c7524087272c8ae913f5d9d608bd839154b62c4322ef65723d2e5bb0b8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024" },
- { url = "https://mirrors.aliyun.com/pypi/packages/80/60/e50fa45dd7e2eae049f0ce964663849e897300433921198aef94b6ffa23a/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:3d744a6061afba08dd7ae375dcde870cffb14429b7477e10f67e9e6d68772a0a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d2/73/a009f41c5eed71c49bec53036c4b33555afcdee70682a18c6f66e396c039/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:ff732bd0a0e778f43d5009840f20b935e79087b4dc65bd36f1cd0f9b04b8ff7f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c4/10/528b439290763bff3d939268085d03382471b442f212dca4ff5f12802d43/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab44b178f7981fcaea7e0a5df20e773c663d06ffda0198f1a524e91b2fde7e59" },
- { url = "https://mirrors.aliyun.com/pypi/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19" },
]
[[package]]
@@ -3700,46 +3396,12 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/4a/4a/cf14bf3b1f5ffb13c69cf5f0ea78031247790558ee88984a8bdd22fae60d/kaitaistruct-0.11-py2.py3-none-any.whl", hash = "sha256:5c6ce79177b4e193a577ecd359e26516d1d6d000a0bffd6e1010f2a46a62a561" },
]
-[[package]]
-name = "keras"
-version = "3.14.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "absl-py" },
- { name = "h5py" },
- { name = "ml-dtypes" },
- { name = "namex" },
- { name = "numpy" },
- { name = "optree" },
- { name = "packaging" },
- { name = "rich" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/88/ce/47874047a49eedc2a5d3b41bc4f1f572bb637f51e4351ef3538e49a63800/keras-3.14.0.tar.gz", hash = "sha256:86fcf8249a25264a566ac393c287c7ad657000e5e62615dcaad4b3472a17aeda" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/c0/20/78d26f81115d570bdf0e57d19b81de9ad8aa55ddb68eb10c8f0699fccb63/keras-3.14.0-py3-none-any.whl", hash = "sha256:19ce94b798caaba4d404ab6ef4753b44219170e5c2868156de8bb0494a260114" },
-]
-
[[package]]
name = "kiwisolver"
version = "1.5.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/4d/b2/818b74ebea34dabe6d0c51cb1c572e046730e64844da6ed646d5298c40ce/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bf/d9/405320f8077e8e1c5c4bd6adc45e1e6edf6d727b6da7f2e2533cf58bff71/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588" },
- { url = "https://mirrors.aliyun.com/pypi/packages/99/9f/795fedf35634f746151ca8839d05681ceb6287fbed6cc1c9bf235f7887c2/kiwisolver-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c8/2f/cebfcdb60fd6a9b0f6b47a9337198bcbad6fbe15e68189b7011fd914911f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f2/0d/9b782923aada3fafb1d6b84e13121954515c669b18af0c26e7d21f579855/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/27/70/83241b6634b04fe44e892688d5208332bde130f38e610c0418f9ede47ded/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e4/db/30ed226fb271ae1a6431fc0fe0edffb2efe23cadb01e798caeb9f2ceae8f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ec/bd/c314595208e4c9587652d50959ead9e461995389664e490f4dce7ff0f782/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c1/43/0499cec932d935229b5543d073c2b87c9c22846aab48881e9d8d6e742a2d/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3d/6f/79b0d760907965acfd9d61826a3d41f8f093c538f55cd2633d3f0db269f6/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ab/31/01d0537c41cb75a551a438c3c7a80d0c60d60b81f694dac83dd436aec0d0/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e4/34/8aefdd0be9cfd00a44509251ba864f5caf2991e36772e61c408007e7f417/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ad/cf/0348374369ca588f8fe9c338fae49fa4e16eeb10ffb3d012f23a54578a9e/kiwisolver-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/26/192b26196e2316e2bd29deef67e37cdf9870d9af8e085e521afff0fed526/kiwisolver-1.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9d/69/024d6711d5ba575aa65d5538042e99964104e97fa153a9f10bc369182bc2/kiwisolver-1.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:fd40bb9cd0891c4c3cb1ddf83f8bbfa15731a248fdc8162669405451e2724b09" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ce/48/adbb40df306f587054a348831220812b9b1d787aff714cfbc8556e38fccd/kiwisolver-1.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0e1403fd7c26d77c1f03e096dc58a5c726503fa0db0456678b8668f76f521e3" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a8/3a/d0a972b34e1c63e2409413104216cd1caa02c5a37cb668d1687d466c1c45/kiwisolver-1.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dda366d548e89a90d88a86c692377d18d8bd64b39c1fb2b92cb31370e2896bbd" },
@@ -3799,10 +3461,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/07/18/43a5f24608d8c313dd189cf838c8e68d75b115567c6279de7796197cfb6a/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646" },
{ url = "https://mirrors.aliyun.com/pypi/packages/3b/b5/98222136d839b8afabcaa943b09bd05888c2d36355b7e448550211d1fca4/kiwisolver-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681" },
{ url = "https://mirrors.aliyun.com/pypi/packages/99/a2/ca7dc962848040befed12732dff6acae7fb3c4f6fc4272b3f6c9a30b8713/kiwisolver-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1c/fa/2910df836372d8761bb6eff7d8bdcb1613b5c2e03f260efe7abe34d388a7/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0f/41/c5f71f9f00aabcc71fee8b7475e3f64747282580c2fe748961ba29b18385/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fa/06/7399a607f434119c6e1fdc8ec89a8d51ccccadf3341dee4ead6bd14caaf5/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57" },
]
[[package]]
@@ -3833,23 +3491,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12" },
]
-[[package]]
-name = "libclang"
-version = "18.1.1"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/4b/49/f5e3e7e1419872b69f6f5e82ba56e33955a74bd537d8a1f5f1eff2f3668a/libclang-18.1.1-1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:0b2e143f0fac830156feb56f9231ff8338c20aecfe72b4ffe96f19e5a1dbb69a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e2/e5/fc61bbded91a8830ccce94c5294ecd6e88e496cc85f6704bf350c0634b70/libclang-18.1.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6f14c3f194704e5d09769108f03185fce7acaf1d1ae4bbb2f30a72c2400cb7c5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/db/ed/1df62b44db2583375f6a8a5e2ca5432bbdc3edb477942b9b7c848c720055/libclang-18.1.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:83ce5045d101b669ac38e6da8e58765f12da2d3aafb3b9b98d88b286a60964d8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0b/2d/3f480b1e1d31eb3d6de5e3ef641954e5c67430d5ac93b7fa7e07589576c7/libclang-18.1.1-py2.py3-none-win_amd64.whl", hash = "sha256:4dd2d3b82fab35e2bf9ca717d7b63ac990a3519c7e312f19fa8e86dcc712f7fb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/71/cf/e01dc4cc79779cd82d77888a88ae2fa424d93b445ad4f6c02bfc18335b70/libclang-18.1.1-py2.py3-none-win_arm64.whl", hash = "sha256:3f0e1f49f04d3cd198985fea0511576b0aee16f9ff0e0f0cad7f9c57ec3c20e8" },
-]
-
[[package]]
name = "litellm"
version = "1.82.6"
@@ -3879,10 +3520,6 @@ version = "0.46.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/74/cd/08ae687ba099c7e3d21fe2ea536500563ef1943c5105bf6ab4ee3829f68e/llvmlite-0.46.0.tar.gz", hash = "sha256:227c9fd6d09dce2783c18b754b7cd9d9b3b3515210c46acc2d3c5badd9870ceb" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/f8/4db016a5e547d4e054ff2f3b99203d63a497465f81ab78ec8eb2ff7b2304/llvmlite-0.46.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b9588ad4c63b4f0175a3984b85494f0c927c6b001e3a246a3a7fb3920d9a137" },
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/85/4890a7c14b4fa54400945cb52ac3cd88545bbdb973c440f98ca41591cdc5/llvmlite-0.46.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3535bd2bb6a2d7ae4012681ac228e5132cdb75fefb1bcb24e33f2f3e0c865ed4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6a/07/3d31d39c1a1a08cd5337e78299fca77e6aebc07c059fbd0033e3edfab45c/llvmlite-0.46.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4cbfd366e60ff87ea6cc62f50bc4cd800ebb13ed4c149466f50cf2163a473d1e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2a/6b/d139535d7590a1bba1ceb68751bef22fadaa5b815bbdf0e858e3875726b2/llvmlite-0.46.0-cp312-cp312-win_amd64.whl", hash = "sha256:398b39db462c39563a97b912d4f2866cd37cba60537975a09679b28fbbc0fb38" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e6/ff/3eba7eb0aed4b6fca37125387cd417e8c458e750621fce56d2c541f67fa8/llvmlite-0.46.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:30b60892d034bc560e0ec6654737aaa74e5ca327bd8114d82136aa071d611172" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0e/54/737755c0a91558364b9200702c3c9c15d70ed63f9b98a2c32f1c2aa1f3ba/llvmlite-0.46.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6cc19b051753368a9c9f31dc041299059ee91aceec81bd57b0e385e5d5bf1a54" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e6/91/14f32e1d70905c1c0aa4e6609ab5d705c3183116ca02ac6df2091868413a/llvmlite-0.46.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bca185892908f9ede48c0acd547fe4dc1bafefb8a4967d47db6cf664f9332d12" },
@@ -3899,23 +3536,6 @@ version = "5.4.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/76/3d/14e82fc7c8fb1b7761f7e748fd47e2ec8276d137b6acfe5a4bb73853e08f/lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/f8/4c/d101ace719ca6a4ec043eb516fcfcb1b396a9fccc4fcd9ef593df34ba0d5/lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/11/84/beddae0cec4dd9ddf46abf156f0af451c13019a0fa25d7445b655ba5ccb7/lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d0/25/d0d93a4e763f0462cccd2b8a665bf1e4343dd788c76dcfefa289d46a38a9/lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779" },
- { url = "https://mirrors.aliyun.com/pypi/packages/31/ce/1df18fb8f7946e7f3388af378b1f34fcf253b94b9feedb2cec5969da8012/lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4e/62/f4a6c60ae7c40d43657f552f3045df05118636be1165b906d3423790447f/lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/aa/04f00009e1e3a77838c7fc948f161b5d2d5de1136b2b81c712a263829ea4/lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c9/1f/e0b2f61fa2404bf0f1fdf1898377e5bd1b74cc9b2cf2c6ba8509b8f27990/lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/24/a2/8263f351b4ffe0ed3e32ea7b7830f845c795349034f912f490180d88a877/lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/05/00/41db052f279995c0e35c79d0f0fc9f8122d5b5e9630139c592a0b58c71b4/lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1d/be/ee99e6314cdef4587617d3b3b745f9356d9b7dd12a9663c5f3b5734b64ba/lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ad/36/239820114bf1d71f38f12208b9c58dec033cbcf80101cde006b9bde5cffd/lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d4/e1/1b795cc0b174efc9e13dbd078a9ff79a58728a033142bc6d70a1ee8fc34d/lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20" },
- { url = "https://mirrors.aliyun.com/pypi/packages/72/48/3c198455ca108cec5ae3662ae8acd7fd99476812fd712bb17f1b39a0b589/lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d6/10/5bf51858971c51ec96cfc13e800a9951f3fd501686f4c18d7d84fe2d6352/lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/11/06710dd809205377da380546f91d2ac94bad9ff735a72b64ec029f706c85/lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f5/b0/15b6217834b5e3a59ebf7f53125e08e318030e8cc0d7310355e6edac98ef/lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/91/1e/05ddcb57ad2f3069101611bd5f5084157d90861a2ef460bf42f45cced944/lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2" },
{ url = "https://mirrors.aliyun.com/pypi/packages/87/cb/2ba1e9dd953415f58548506fa5549a7f373ae55e80c61c9041b7fd09a38a/lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b5/3e/6602a4dca3ae344e8609914d6ab22e52ce42e3e1638c10967568c5c1450d/lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4c/72/bf00988477d3bb452bef9436e45aeea82bb40cdfb4684b83c967c53909c7/lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76" },
@@ -3958,14 +3578,6 @@ version = "4.4.5"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/1b/ac/016e4f6de37d806f7cc8f13add0a46c9a7cfc41a5ddc2bc831d7954cf1ce/lz4-4.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:df5aa4cead2044bab83e0ebae56e0944cc7fcc1505c7787e9e1057d6d549897e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8d/df/0fadac6e5bd31b6f34a1a8dbd4db6a7606e70715387c27368586455b7fc9/lz4-4.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d0bf51e7745484d2092b3a51ae6eb58c3bd3ce0300cf2b2c14f76c536d5697a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/17/34e36cc49bb16ca73fb57fbd4c5eaa61760c6b64bce91fcb4e0f4a97f852/lz4-4.4.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b62f94b523c251cf32aa4ab555f14d39bd1a9df385b72443fd76d7c7fb051f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/90/1c/b1d8e3741e9fc89ed3b5f7ef5f22586c07ed6bb04e8343c2e98f0fa7ff04/lz4-4.4.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c3ea562c3af274264444819ae9b14dbbf1ab070aff214a05e97db6896c7597e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/55/d9/e3867222474f6c1b76e89f3bd914595af69f55bf2c1866e984c548afdc15/lz4-4.4.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24092635f47538b392c4eaeff14c7270d2c8e806bf4be2a6446a378591c5e69e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b2/e7/d667d337367686311c38b580d1ca3d5a23a6617e129f26becd4f5dc458df/lz4-4.4.5-cp312-cp312-win32.whl", hash = "sha256:214e37cfe270948ea7eb777229e211c601a3e0875541c1035ab408fbceaddf50" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a5/0b/a54cd7406995ab097fceb907c7eb13a6ddd49e0b231e448f1a81a50af65c/lz4-4.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:713a777de88a73425cf08eb11f742cd2c98628e79a8673d6a52e3c5f0c116f33" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6a/7e/dc28a952e4bfa32ca16fa2eb026e7a6ce5d1411fcd5986cd08c74ec187b9/lz4-4.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:a88cbb729cc333334ccfb52f070463c21560fca63afcf636a9f160a55fac3301" },
{ url = "https://mirrors.aliyun.com/pypi/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d" },
@@ -4053,17 +3665,6 @@ version = "3.0.3"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219" },
{ url = "https://mirrors.aliyun.com/pypi/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6" },
@@ -4127,13 +3728,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/8a/76/d3c6e3a13fe484ebe7718d14e269c9569c4eb0020a968a327acb3b9a8fe6/matplotlib-3.10.8.tar.gz", hash = "sha256:2299372c19d56bcd35cf05a2738308758d32b9eaed2371898d8f5bd33f084aa3" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/67/f997cdcbb514012eb0d10cd2b4b332667997fb5ebe26b8d41d04962fa0e6/matplotlib-3.10.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:64fcc24778ca0404ce0cb7b6b77ae1f4c7231cdd60e6778f999ee05cbd581b9a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7e/65/07d5f5c7f7c994f12c768708bd2e17a4f01a2b0f44a1c9eccad872433e2e/matplotlib-3.10.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9a5ca4ac220a0cdd1ba6bcba3608547117d30468fefce49bb26f55c1a3d5c58" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3e/f3/c5195b1ae57ef85339fd7285dfb603b22c8b4e79114bae5f4f0fcf688677/matplotlib-3.10.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3ab4aabc72de4ff77b3ec33a6d78a68227bf1123465887f9905ba79184a1cc04" },
- { url = "https://mirrors.aliyun.com/pypi/packages/00/f9/7638f5cc82ec8a7aa005de48622eecc3ed7c9854b96ba15bd76b7fd27574/matplotlib-3.10.8-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24d50994d8c5816ddc35411e50a86ab05f575e2530c02752e02538122613371f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/57/61/78cd5920d35b29fd2a0fe894de8adf672ff52939d2e9b43cb83cd5ce1bc7/matplotlib-3.10.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:99eefd13c0dc3b3c1b4d561c1169e65fe47aab7b8158754d7c084088e2329466" },
- { url = "https://mirrors.aliyun.com/pypi/packages/30/4e/c10f171b6e2f44d9e3a2b96efa38b1677439d79c99357600a62cc1e9594e/matplotlib-3.10.8-cp312-cp312-win_amd64.whl", hash = "sha256:dd80ecb295460a5d9d260df63c43f4afbdd832d725a531f008dad1664f458adf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f1/76/934db220026b5fef85f45d51a738b91dea7d70207581063cd9bd8fafcf74/matplotlib-3.10.8-cp312-cp312-win_arm64.whl", hash = "sha256:3c624e43ed56313651bc18a47f838b60d7b8032ed348911c54906b130b20071b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/3d/b9/15fd5541ef4f5b9a17eefd379356cf12175fe577424e7b1d80676516031a/matplotlib-3.10.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3f2e409836d7f5ac2f1c013110a4d50b9f7edc26328c108915f9075d7d7a91b6" },
{ url = "https://mirrors.aliyun.com/pypi/packages/8d/a0/2ba3473c1b66b9c74dc7107c67e9008cb1782edbe896d4c899d39ae9cf78/matplotlib-3.10.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56271f3dac49a88d7fca5060f004d9d22b865f743a12a23b1e937a0be4818ee1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/75/97/a471f1c3eb1fd6f6c24a31a5858f443891d5127e63a7788678d14e249aea/matplotlib-3.10.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a0a7f52498f72f13d4a25ea70f35f4cb60642b466cbb0a9be951b5bc3f45a486" },
@@ -4249,21 +3843,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/9b/f7/4a5e785ec9fbd65146a27b6b70b6cdc161a66f2024e4b04ac06a67f5578b/mistune-3.2.0-py3-none-any.whl", hash = "sha256:febdc629a3c78616b94393c6580551e0e34cc289987ec6c35ed3f4be42d0eee1" },
]
-[[package]]
-name = "ml-dtypes"
-version = "0.4.1"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "numpy" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/fd/15/76f86faa0902836cc133939732f7611ace68cf54148487a99c539c272dc8/ml_dtypes-0.4.1.tar.gz", hash = "sha256:fad5f2de464fd09127e49b7fd1252b9006fb43d2edc1ff112d390c324af5ca7a" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ba/1a/99e924f12e4b62139fbac87419698c65f956d58de0dbfa7c028fa5b096aa/ml_dtypes-0.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:827d3ca2097085cf0355f8fdf092b888890bb1b1455f52801a2d7756f056f54b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8f/8c/7b610bd500617854c8cc6ed7c8cfb9d48d6a5c21a1437a36a4b9bc8a3598/ml_dtypes-0.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:772426b08a6172a891274d581ce58ea2789cc8abc1c002a27223f314aaf894e7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c7/c6/f89620cecc0581dc1839e218c4315171312e46c62a62da6ace204bda91c0/ml_dtypes-0.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:126e7d679b8676d1a958f2651949fbfa182832c3cd08020d8facd94e4114f3e9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ae/11/a742d3c31b2cc8557a48efdde53427fd5f9caa2fa3c9c27d826e78a66f51/ml_dtypes-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:df0fb650d5c582a9e72bb5bd96cfebb2cdb889d89daff621c8fbc60295eba66c" },
-]
-
[[package]]
name = "moodlepy"
version = "0.24.1"
@@ -4319,14 +3898,6 @@ version = "0.20.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/ea/9c/bfbd12955a49180cbd234c5d29ec6f74fe641698f0cd9df154a854fc8a15/msgspec-0.20.0.tar.gz", hash = "sha256:692349e588fde322875f8d3025ac01689fead5901e7fb18d6870a44519d62a29" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/d9/6f/1e25eee957e58e3afb2a44b94fa95e06cebc4c236193ed0de3012fff1e19/msgspec-0.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2aba22e2e302e9231e85edc24f27ba1f524d43c223ef5765bd8624c7df9ec0a5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7f/ee/af51d090ada641d4b264992a486435ba3ef5b5634bc27e6eb002f71cef7d/msgspec-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:716284f898ab2547fedd72a93bb940375de9fbfe77538f05779632dc34afdfde" },
- { url = "https://mirrors.aliyun.com/pypi/packages/49/d6/9709ee093b7742362c2934bfb1bbe791a1e09bed3ea5d8a18ce552fbfd73/msgspec-0.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:558ed73315efa51b1538fa8f1d3b22c8c5ff6d9a2a62eff87d25829b94fc5054" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5c/a2/488517a43ccf5a4b6b6eca6dd4ede0bd82b043d1539dd6bb908a19f8efd3/msgspec-0.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:509ac1362a1d53aa66798c9b9fd76872d7faa30fcf89b2fba3bcbfd559d56eb0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d5/e8/49b832808aa23b85d4f090d1d2e48a4e3834871415031ed7c5fe48723156/msgspec-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1353c2c93423602e7dea1aa4c92f3391fdfc25ff40e0bacf81d34dbc68adb870" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9f/56/1dc2fa53685dca9c3f243a6cbecd34e856858354e455b77f47ebd76cf5bf/msgspec-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cb33b5eb5adb3c33d749684471c6a165468395d7aa02d8867c15103b81e1da3e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5a/51/aba940212c23b32eedce752896205912c2668472ed5b205fc33da28a6509/msgspec-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:fb1d934e435dd3a2b8cf4bbf47a8757100b4a1cfdc2afdf227541199885cdacb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/41/ad/3b9f259d94f183daa9764fef33fdc7010f7ecffc29af977044fa47440a83/msgspec-0.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:00648b1e19cf01b2be45444ba9dc961bd4c056ffb15706651e64e5d6ec6197b7" },
{ url = "https://mirrors.aliyun.com/pypi/packages/8a/d1/b902d38b6e5ba3bdddbec469bba388d647f960aeed7b5b3623a8debe8a76/msgspec-0.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9c1ff8db03be7598b50dd4b4a478d6fe93faae3bd54f4f17aa004d0e46c14c46" },
{ url = "https://mirrors.aliyun.com/pypi/packages/57/b6/eff0305961a1d9447ec2b02f8c73c8946f22564d302a504185b730c9a761/msgspec-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f6532369ece217fd37c5ebcfd7e981f2615628c21121b7b2df9d3adcf2fd69b8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/99/93/f2ec1ae1de51d3fdee998a1ede6b2c089453a2ee82b5c1b361ed9095064a/msgspec-0.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9a1697da2f85a751ac3cc6a97fceb8e937fc670947183fb2268edaf4016d1ee" },
@@ -4372,24 +3943,6 @@ version = "6.7.1"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75" },
- { url = "https://mirrors.aliyun.com/pypi/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961" },
- { url = "https://mirrors.aliyun.com/pypi/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582" },
- { url = "https://mirrors.aliyun.com/pypi/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511" },
- { url = "https://mirrors.aliyun.com/pypi/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf" },
{ url = "https://mirrors.aliyun.com/pypi/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23" },
{ url = "https://mirrors.aliyun.com/pypi/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2" },
{ url = "https://mirrors.aliyun.com/pypi/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445" },
@@ -4494,14 +4047,6 @@ version = "1.0.15"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/23/2e/88c147931ea9725d634840d538622e94122bceaf346233349b7b5c62964b/murmurhash-1.0.15.tar.gz", hash = "sha256:58e2b27b7847f9e2a6edf10b47a8c8dd70a4705f45dccb7bf76aeadacf56ba01" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/b6/46/be8522d3456fdccf1b8b049c6d82e7a3c1114c4fc2cfe14b04cba4b3e701/murmurhash-1.0.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d37e3ae44746bca80b1a917c2ea625cf216913564ed43f69d2888e5df97db0cb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ed/cc/630449bf4f6178d7daf948ce46ad00b25d279065fc30abd8d706be3d87e0/murmurhash-1.0.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0861cb11039409eaf46878456b7d985ef17b6b484103a6fc367b2ecec846891d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ff/30/ea8f601a9bf44db99468696efd59eb9cff1157cd55cb586d67116697583f/murmurhash-1.0.15-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5a301decfaccfec70fe55cb01dde2a012c3014a874542eaa7cc73477bb749616" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c9/de/c40ce8c0877d406691e735b8d6e9c815f36a82b499d358313db5dbe219d7/murmurhash-1.0.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32c6fde7bd7e9407003370a07b5f4addacabe1556ad3dc2cac246b7a2bba3400" },
- { url = "https://mirrors.aliyun.com/pypi/packages/47/84/bd49963ecd84ebab2fe66595e2d1ed41d5e8b5153af5dc930f0bd827007c/murmurhash-1.0.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d8b43a7011540dc3c7ce66f2134df9732e2bc3bbb4a35f6458bc755e48bde26" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4f/7c/2530769c545074417c862583f05f4245644599f1e9ff619b3dfe2969aafc/murmurhash-1.0.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:43bf4541892ecd95963fcd307bf1c575fc0fee1682f41c93007adee71ca2bb40" },
- { url = "https://mirrors.aliyun.com/pypi/packages/84/a4/b249b042f5afe34d14ada2dc4afc777e883c15863296756179652e081c44/murmurhash-1.0.15-cp312-cp312-win_amd64.whl", hash = "sha256:f4ac15a2089dc42e6eb0966622d42d2521590a12c92480aafecf34c085302cca" },
- { url = "https://mirrors.aliyun.com/pypi/packages/13/bf/028179259aebc18fd4ba5cae2601d1d47517427a537ab44336446431a215/murmurhash-1.0.15-cp312-cp312-win_arm64.whl", hash = "sha256:4a70ca4ae19e600d9be3da64d00710e79dde388a4d162f22078d64844d0ebdda" },
{ url = "https://mirrors.aliyun.com/pypi/packages/29/2f/ba300b5f04dae0409202d6285668b8a9d3ade43a846abee3ef611cb388d5/murmurhash-1.0.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fe50dc70e52786759358fd1471e309b94dddfffb9320d9dfea233c7684c894ba" },
{ url = "https://mirrors.aliyun.com/pypi/packages/34/02/29c19d268e6f4ea1ed2a462c901eed1ed35b454e2cbc57da592fad663ac6/murmurhash-1.0.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1349a7c23f6092e7998ddc5bd28546cc31a595afc61e9fdb3afc423feec3d7ad" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e2/63/58e2de2b5232cd294c64092688c422196e74f9fa8b3958bdf02d33df24b9/murmurhash-1.0.15-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3ba6d05de2613535b5a9227d4ad8ef40a540465f64660d4a8800634ae10e04f" },
@@ -4563,11 +4108,6 @@ version = "9.6.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/6f/6e/c89babc7de3df01467d159854414659c885152579903a8220c8db02a3835/mysql_connector_python-9.6.0.tar.gz", hash = "sha256:c453bb55347174d87504b534246fb10c589daf5d057515bf615627198a3c7ef1" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/8f/d9/2a4b4d90b52f4241f0f71618cd4bd8779dd6d18db8058b0a4dd83ec0541c/mysql_connector_python-9.6.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9664e217c72dd6fb700f4c8512af90261f72d2f5d7c00c4e13e4c1e09bfa3d5e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/33/91/2495835733a054e716a17dc28404748b33f2dc1da1ae4396fb45574adf40/mysql_connector_python-9.6.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:1ed4b5c4761e5333035293e746683890e4ef2e818e515d14023fd80293bc31fa" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7a/69/e83abbbbf7f8eed855b5a5ff7285bc0afb1199418ac036c7691edf41e154/mysql_connector_python-9.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5095758dcb89a6bce2379f349da336c268c407129002b595c5dba82ce387e2a5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/82/44/67bb61c71f398fbc739d07e8dcadad94e2f655874cb32ae851454066bea0/mysql_connector_python-9.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ae4e7780fad950a4f267dea5851048d160f5b71314a342cdbf30b154f1c74f7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ba/39/994c4f7e9c59d3ca534a831d18442ac4c529865db20aeaa4fd94e2af5efd/mysql_connector_python-9.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c180e0b4100d7402e03993bfac5c97d18e01d7ca9d198d742fffc245077f8ffe" },
{ url = "https://mirrors.aliyun.com/pypi/packages/2f/58/9521aa678708ec6cebfd40524c14c3d151e4f29e3774e6086aa0a30d203b/mysql_connector_python-9.6.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e86e45a7b540ca09af8a18ecfa761e0cdeccfdb62818331614ec030ae44bfd26" },
{ url = "https://mirrors.aliyun.com/pypi/packages/39/8d/b108f9bcce9780f6a1f91decb2af54defdaf845e237ddc42f2b4578f1cd7/mysql_connector_python-9.6.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:8d3e9252384e1b7f95b07020664f2673d9c29c5e95eeda2e048b3331e190b9d4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d6/28/735cd93d16e76dc2feb4abb3f1229a1d9475af34d80c26712fec6abe1d70/mysql_connector_python-9.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:0fa18ead33cb699ea92005695077cef09aa494eebf51164ee30c891c3eaea90c" },
@@ -4581,15 +4121,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/15/dd/b3250826c29cee7816de4409a2fe5e469a68b9a89f6bfaa5eed74f05532c/mysql_connector_python-9.6.0-py2.py3-none-any.whl", hash = "sha256:44b0fb57207ebc6ae05b5b21b7968a9ed33b29187fe87b38951bad2a334d75d5" },
]
-[[package]]
-name = "namex"
-version = "0.1.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c" },
-]
-
[[package]]
name = "nest-asyncio"
version = "1.6.0"
@@ -4633,10 +4164,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/23/c9/a0fb41787d01d621046138da30f6c2100d80857bf34b3390dd68040f27a3/numba-0.64.0.tar.gz", hash = "sha256:95e7300af648baa3308127b1955b52ce6d11889d16e8cfe637b4f85d2fca52b1" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/70/a6/9fc52cb4f0d5e6d8b5f4d81615bc01012e3cf24e1052a60f17a68deb8092/numba-0.64.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:69440a8e8bc1a81028446f06b363e28635aa67bd51b1e498023f03b812e0ce68" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9b/89/1a74ea99b180b7a5587b0301ed1b183a2937c4b4b67f7994689b5d36fc34/numba-0.64.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f13721011f693ba558b8dd4e4db7f2640462bba1b855bdc804be45bbeb55031a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/91/e1/583c647404b15f807410510fec1eb9b80cb8474165940b7749f026f21cbc/numba-0.64.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0b180b1133f2b5d8b3f09d96b6d7a9e51a7da5dda3c09e998b5bcfac85d222c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/85/23/0fce5789b8a5035e7ace21216a468143f3144e02013252116616c58339aa/numba-0.64.0-cp312-cp312-win_amd64.whl", hash = "sha256:e63dc94023b47894849b8b106db28ccb98b49d5498b98878fac1a38f83ac007a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/52/80/2734de90f9300a6e2503b35ee50d9599926b90cbb7ac54f9e40074cd07f1/numba-0.64.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3bab2c872194dcd985f1153b70782ec0fbbe348fffef340264eacd3a76d59fd6" },
{ url = "https://mirrors.aliyun.com/pypi/packages/42/e8/14b5853ebefd5b37723ef365c5318a30ce0702d39057eaa8d7d76392859d/numba-0.64.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:703a246c60832cad231d2e73c1182f25bf3cc8b699759ec8fe58a2dbc689a70c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/8a/a2/f60dc6c96d19b7185144265a5fbf01c14993d37ff4cd324b09d0212aa7ce/numba-0.64.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2e49a7900ee971d32af7609adc0cfe6aa7477c6f6cccdf6d8138538cf7756f" },
@@ -4652,16 +4179,6 @@ name = "numpy"
version = "1.26.4"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218" },
- { url = "https://mirrors.aliyun.com/pypi/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110" },
- { url = "https://mirrors.aliyun.com/pypi/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818" },
-]
[[package]]
name = "oauthlib"
@@ -4739,11 +4256,6 @@ dependencies = [
{ name = "sympy" },
]
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321" },
- { url = "https://mirrors.aliyun.com/pypi/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c0/b4/569d298f9fc4d286c11c45e85d9ffa9e877af12ace98af8cab52396e8f46/onnxruntime-1.23.2-cp312-cp312-win_amd64.whl", hash = "sha256:25de5214923ce941a3523739d34a520aac30f21e631de53bba9174dc9c004435" },
{ url = "https://mirrors.aliyun.com/pypi/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b" },
@@ -4766,8 +4278,6 @@ dependencies = [
{ name = "sympy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
]
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/6c/d9/b7140a4f1615195938c7e358c0804bb84271f0d6886b5cbf105c6cb58aae/onnxruntime_gpu-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f2d1f720685d729b5258ec1b36dee1de381b8898189908c98cbeecdb2f2b5c2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/87/da/2685c79e5ea587beddebe083601fead0bdf3620bc2f92d18756e7de8a636/onnxruntime_gpu-1.23.2-cp312-cp312-win_amd64.whl", hash = "sha256:fe925a84b00e291e0ad3fac29bfd8f8e06112abc760cdc82cb711b4f3935bd95" },
{ url = "https://mirrors.aliyun.com/pypi/packages/03/05/40d561636e4114b54aa06d2371bfbca2d03e12cfdf5d4b85814802f18a75/onnxruntime_gpu-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e8f75af5da07329d0c3a5006087f4051d8abd133b4be7c9bae8cdab7bea4c26" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b6/3b/418300438063d403384c79eaef1cb13c97627042f2247b35a887276a355a/onnxruntime_gpu-1.23.2-cp313-cp313-win_amd64.whl", hash = "sha256:7f1b3f49e5e126b99e23ec86b4203db41c2a911f6165f7624f2bc8267aaca767" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b8/dc/80b145e3134d7eba31309b3299a2836e37c76e4c419a261ad9796f8f8d65/onnxruntime_gpu-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20959cd4ae358aab6579ab9123284a7b1498f7d51ec291d429a5edc26511306f" },
@@ -4960,108 +4470,12 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/b2/37/cc6a55e448deaa9b27377d087da8615a3416d8ad523d5960b78dbeadd02a/opentelemetry_semantic_conventions-0.61b0-py3-none-any.whl", hash = "sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2" },
]
-[[package]]
-name = "opt-einsum"
-version = "3.4.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd" },
-]
-
-[[package]]
-name = "optree"
-version = "0.19.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "typing-extensions" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/3d/63/7b078bc36d5a206c21b03565a818ede38ff0fbf014e92085ec467ef10adb/optree-0.19.0.tar.gz", hash = "sha256:bc1991a948590756409e76be4e29efd4a487a185056d35db6c67619c19ea27a1" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/2d/bf/5cbbf61a27f94797c3d9786f6230223023a943b60f5e893d52368f10b8b1/optree-0.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ec4b2ce49622c6be2c8634712b6c63cc274835bac89a56e3ab2ca863a32ff4b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/00/9e/65899e6470f5df289ccdbe9e228fb0cd0ae45ccda8e32c92d6efae1530ef/optree-0.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0978603623b4b1f794f05f6bbed0645cb7e219f4a5a349b2a2bd4514d84ac82" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d1/dc/f4826835be660181f1b4444ac92b51dda96d4634d3c2271e14598da7bf2a/optree-0.19.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c9e52c50ed3f3f8b1cf4e47a20a7c5e77175b4f84b2ecf390a76f0d1dd91da6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/b0/89283ac1dd1ead3aa3d7a6b45a26846f457bded79a83b6828fc1ed9a6db3/optree-0.19.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:3fe3e5f7a30a7d08ddba0a34e48f5483f6c4d7bb710375434ad3633170c73c48" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2a/a2/47f620f87b0544b2e0eb0b3c661682bd0ea1c79f6e38f9147bc0f835c973/optree-0.19.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8315527e1f14a91173fe6871847da7b949048ec61ff8b3e507fc286e75b0aa3c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/84/e9/b9ae18404135de53809fb994b754ac0eac838d8c4dfa8a10a811d8dec91d/optree-0.19.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:938fb15d140ab65148f4e6975048facbef83a9210353fbedd471ac39e7544339" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0a/e5/a77df15a62b37bb14c81b5757e2a0573f57e7c06d125a410ad2cd7cefb72/optree-0.19.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b8209570340135a7e586c90f393f3c6359e8a49c40d783196721cc487e51d9c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8c/43/1aa431cee19cd98c4229e468767021f9a92195d9431857e28198a3a3ce2f/optree-0.19.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:1397dc925026917531a43fda32054ae1e77e5ed9bf8284bcae6354c19c26e14a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5b/b9/b94fd3a116b80951d692a82f4135ae84b3d78bd1b092250aff76a3366138/optree-0.19.0-cp312-cp312-win32.whl", hash = "sha256:68f58e8f8b75c76c51e61e3dc2d9e94609bafb0e1a6459e6d525ced905cd9a74" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/7f/31fa1b2311038bfc355ad6e4e4e63d028719cb67fb3ebe6fb76ff2124105/optree-0.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:5c44ca0f579ed3e0ca777a5711d4a6c1b374feacf1bb4fe9cfe85297b0c8d237" },
- { url = "https://mirrors.aliyun.com/pypi/packages/09/86/863bc3f42f83113f5c6a5beaf4fec3c3481a76872f3244d0e64fb9ebd3b0/optree-0.19.0-cp312-cp312-win_arm64.whl", hash = "sha256:0461f796b4ade3fab519d821b0fa521f07e2af70206b76aac75fcfdc2e051fca" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ee/61/d79c7eeb87e98d08bc8d95ed08dee83bedb4e55371a7d2ae3c874ec02608/optree-0.19.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:1eea5b7be833c6d555d08ff68046d3dd2112dfb39e6f1eb09887ab6c617a6d64" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2d/ed/e80504f65e7e80fdcd129258428d7976ea9f03bf9dad56a5293c44d563ad/optree-0.19.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4d9cf9dfa0ac051e0ed82869d782f0affdbdb1daa5f2e851d37ea8625c60071a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/65/e5/d1926a2f0e0240f6800ff385c8486879f7da0a5a030b7aa5d84e44e9c9ca/optree-0.19.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:43c4f8ba5755d56d046be2cb1380cbc362234ad93fd9933384c6dd7fdebe6c4a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/61/88/9c598325e89bbed29b37a381ebb2b94f1d9d769c973b879b3e9766b4b16d/optree-0.19.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36b1134680ee3f9768ede290da653e1604a8083bce69fef8fb4e46863346d5c8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6b/d2/fcba2a1826d362a64cb36ec9f675ed6dcddee47099948913122b0aafbe44/optree-0.19.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c9f7e7e7bf2ef011d0be1c2e87c96f5dc543dad1ac34430c2f606938c9ec5135" },
- { url = "https://mirrors.aliyun.com/pypi/packages/eb/43/5e6d51d8c203a79cff084efa9f04a745b8ef5cf4c86dbb127e7b192f14d9/optree-0.19.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bb5752f17afa017b08b0cbac8a383d4bb90035b353bef7a25fe03cda69a21d33" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4b/dc/dc09347136876287b463b8599239d6fa338298fd322ac629817bd2f4def4/optree-0.19.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e9b6245993494b1aa54529eb7356aeefa6704c8b436e6e5f20b25c30f7af7620" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ee/cc/5d2c9cf906bd3ae357e7221450bacefd0321d7b94e6171dec39552b346e6/optree-0.19.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7351a24b30568c963a92b19f543c9562b36b3222caed2a5ac3209ef910972bec" },
- { url = "https://mirrors.aliyun.com/pypi/packages/64/7f/75b10f88da994fc3da3dc1ab7d54bab7bd3a6fa5eb81b586f13f8bd6ab0e/optree-0.19.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2c6610a1d1d74af0f53c9bbabb7c265679a9a07e03783c8cc4a678ba3bb6f9a5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/78/fc/753bf69b907652d54b7c6012ccb320d8c1a3161454e415331058b6f04246/optree-0.19.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37e07a5233be64329cbf41e20ab07c50da53bdc374109a2b376be49c4a34a37f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e2/a8/70640f9998438f50a0a1c57f2a12aac856cd937f2c4c4feef5a3cfe8e9c7/optree-0.19.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:c23a25caff6b096b62379adb99e2c401805141497ebb8131f271a4c93f5ed5dc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ad/05/0b8bf4abf5d1a7cd9a19ba680e1ec64ad38eec3204e4e16a769e8aeaa4a2/optree-0.19.0-cp313-cp313-win32.whl", hash = "sha256:045cf112adaebc76c9c7cabde857c01babfc9fae8aa0a28d48f7c565fadf0cb9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b1/c7/9ce83f115d7f4a47741827a037067b9026c29996ad7913bc40277924c773/optree-0.19.0-cp313-cp313-win_amd64.whl", hash = "sha256:bc0c6c9f99fb90e3a20a8b94c219e6b03e585f65ab9a11c9acd1511a5f885f79" },
- { url = "https://mirrors.aliyun.com/pypi/packages/17/fd/97c27d6e51c8b958b29f5c7b4cdcae4f2e7c9ef5b5465be459811a48876b/optree-0.19.0-cp313-cp313-win_arm64.whl", hash = "sha256:48f492363fa0f9ffe5029d0ecafd2fa30ffe0d5d52c8dd414123f47b743bd42e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/46/45/9a2f05b5d033482b58ca36df6f41b0b28af3ccfa43267a82254c973dcd14/optree-0.19.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d6362b9e9a0f4dd7c5b88debe182a90541aba7f1ad02d00922d01c4df4b3c933" },
- { url = "https://mirrors.aliyun.com/pypi/packages/20/b7/5d0a013c5461e0933ce7385a06eed625358de12216c80da935138e6af205/optree-0.19.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:381096a293d385fd3135e5c707bb7e58c584bc9bd50f458237b49da21a621df3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d6/2c/d3f2674411c8e3338e91e7446af239597ae6efd23f14e2039f29ced3d73e/optree-0.19.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9675007cc54371be544bb33fd7eb07b0773d88deacf8aa4cc72fa735c4a4d33" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e9/e9/009964734f19d6996291e77f2c1da5d35a743defc4e89aefb01260e2f9d6/optree-0.19.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:406b355d6f29f99535efa97ea16eda70414968271a894c99f48cd91848723706" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/4c/96706f855c6b623259e754f751020acfb3452e412f7c85330629ab4b9ecc/optree-0.19.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d05e5bf6ce30258cda643ea50cc424038e5107905e9fc11d19a04453a8d2ee27" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b5/e4/9b23a27c9bd211d22a2e55a5a66e62afe5c75ff98b81fc7d000d879e75e6/optree-0.19.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b6e11479d98690fc9efd15d65195af37608269bb1e176b5a836b066440f9c52f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/15/3b/462582f0050508f1ce0734f1dffd19078fb013fa12ccf0761c208ab6f756/optree-0.19.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d523ffc6d3e22851ed25bec806a6c78d68340259e79941059752209b07a75ec" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d6/c6/843c6a33b700ef88407bd5840813e53c6986b6130d94c75c49ff7a2e31f9/optree-0.19.0-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:ca148527b6e5d59c25c733e66d4165fbcf85102f4ea10f096370fda533fe77d1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e3/ed/13f938444de70bec2ff0edef8917a08160d41436a3cad976e541d21747f5/optree-0.19.0-cp313-cp313t-win32.whl", hash = "sha256:40d067cf87e76ad21b8ee2e6ba0347c517c88c2ce7190d666b30b4057e4de5ba" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e1/a2/5074dedbc1be5deca76fe57285ec3e7d5d475922572f92a90f3b3a4f21c5/optree-0.19.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b133e1b9a30ec0bca3f875cfa68c2ce88c0b9e08b21f97f687bb669266411f4a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/49/3a/ea23a29f63d8eadab4e030ebc1329906d44f631076cd1da4751388649960/optree-0.19.0-cp313-cp313t-win_arm64.whl", hash = "sha256:45184b3c73e2147b26b139f34f15c2111cde54b8893b1104a00281c3f283b209" },
- { url = "https://mirrors.aliyun.com/pypi/packages/81/46/643ea3d06c24d351888edfef387e611e550b64a14758169eaeb1d285e658/optree-0.19.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:adf611b95d3159209c5d1eafcb2eb669733aaf75f9b6754f92d2d8b749192579" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d7/10/8717b93d93fcc3c42a6ee0e0a1a222fe25bc749b32a9e353b039dab836ce/optree-0.19.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:bad7bb78baa83f950bb3c59b09d7ca93d30f6bb975a1a7ce8c5f3dfe65fc834d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a1/5e/8263600ef51ae2decb3e31776c810b8c6b5f8927697046c4434b17346d9d/optree-0.19.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:73f122e8acf2f1fd346e9c08f771bc1f7394359793fe632a8e1040733bdbcbec" },
- { url = "https://mirrors.aliyun.com/pypi/packages/04/3c/40774378ebf423d7f074dfd7169f0466eb9de734f0ea5fbb368eddcb1e49/optree-0.19.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:36e426e96b3e1773e879189b12c306b58ae70052efc4087e3f14545701c7ac35" },
- { url = "https://mirrors.aliyun.com/pypi/packages/08/67/2e19866a03a6e75eb62194a5b55e1e3154ca1517478c300232b0229f8c2a/optree-0.19.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d22b947603be4768c2bd73a59652c94d63465f928b3099e9035f9c48dfc61953" },
- { url = "https://mirrors.aliyun.com/pypi/packages/45/a5/7c059f643bc34c70cc5ebe63c82ae6c33b6b746219f96757d840ea1e2dcd/optree-0.19.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14cc72d0c3a3c0d0b13c66801f2adc6583a01f8499fd151caaa649aabb7f99b9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/67/1a/2c5041cf476fb4b2a27f6644934ac2d079e3e4491f609cba411b3d890291/optree-0.19.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5369ac9584ef3fbb703699be694e84dbc78b730bd6d00c48c0c5a588617a1980" },
- { url = "https://mirrors.aliyun.com/pypi/packages/40/a0/abcd7bc3218e1108d253d6783f3e610f0ac3d0e63b2720bff94eb4ed4689/optree-0.19.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:80b3dca5607f04316a9dcb2bb46df2f04abf4da71731bd4a53a1559c0bee6181" },
- { url = "https://mirrors.aliyun.com/pypi/packages/82/49/7983e66210c78965bc75e386c329ec34854370d337a9ebdc4c8aede3a0b3/optree-0.19.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1bb36da9b95b165c7b77fd3ff0af36a30b802cd1c020da3bcdc8aa029991c4ea" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fe/16/00261f20f467b9e8950a76ec1749f01359bf47f2fc3dac5e206de99835c0/optree-0.19.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb220bb85128c8de71aeffb9c38be817569e4bca413b38d5e0de11ba6471ef4a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/18/31/5e78a451ba9a6ed4b0903b10080dc028e3c9b9c5797cce0ca73990fb5604/optree-0.19.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:5d2b83a37f150f827b8b0bc2c486056f9b2203e7b0bee699d2ee96a36c090f3a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9c/03/1516cb4fdb753cd76e5dc595217f84df48372bdabe1a7fb740a5b2530f5c/optree-0.19.0-cp314-cp314-win32.whl", hash = "sha256:b0c23d50b7f6a7c80f642307c87eee841cf513239706f2f60bd9480304170054" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7e/c3/587cc9aa8d4742cd690da79460081e7d834499e07e8b2bd2ccc4c66928df/optree-0.19.0-cp314-cp314-win_amd64.whl", hash = "sha256:ff773c852122cef6dcae68b5e252a20aaf5d2986f78e278d747e226e7829d44e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e6/9b/c17c74ef6b85ad1a2687de8a08d1b56e3a27154b4db6c3ef1e9c2c53a96c/optree-0.19.0-cp314-cp314-win_arm64.whl", hash = "sha256:259ac2a426816d53d576c143b8dca87176af45fc8efd5dfe09db50d74a2fa0a5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ef/4c/e881fb840cef2cead7582ee36c0e0348e66730cb2a2af1938338c72b1bf3/optree-0.19.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:428fdc8cf5dc43fa32496be6aa84fc0d8f549f899062dd9dd0aa7e3aa7f77ae9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b6/6b/0a8538815abe28e4307dd98385d4991d36555b841b060df3295a8408b856/optree-0.19.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d1b497032b5823a09625b118fd4df84199fb0895afb78af536d638ce7645beb6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/71/0c/d70a513fa93dbaa0e3e8c9b218b3805efb7083369cd14e1340bd2c0bc910/optree-0.19.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5f05fecbca17b48451ba3455198cec9db20802c0ffbbba51eaeb421bd846a1c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/77/04/bd30c9f4e694f7b6585f333208ac7894578c1fa30dc5c938f22155df7859/optree-0.19.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a51d0ad4e9dd089f317c94d95b7fa360e87491324e2bfa83d9c4f18dd928d4e1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e5/17/aba83aa0e8bf31c00cdd3863c2a05854ce414426a69c094ae51210b76677/optree-0.19.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:108ab83937d91658ef96c4f70a6c76b36038754f4779907ee8f127780575740f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e0/da/52e684c42dc29d3b4d52f2029545742ef43e151cea112d9093d2ad164f53/optree-0.19.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a39fdd614f46bcaf810b2bb1ed940e82b8a19e654bc325df0cc6554e25c3b7eb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2d/f7/0d41edf484e11ba5357f91dba8d85ce06ca9d840ac7d95e58b856a49b13b/optree-0.19.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfc1bcba22f182f39f1a80ae3ac511ebfa4daea62c3058edd021ce7a5cda3009" },
- { url = "https://mirrors.aliyun.com/pypi/packages/79/5e/a8f49cfd6c3ae0e59dcb1155cd49f1e5ba41889c9388360264c8369589c6/optree-0.19.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:afe595a052cc45d3addb6045f04a3ca7e1fb664de032ecbbb2bfd76dfe1fcb61" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9a/1b/4105e562d86b2de7eb3f240164a7dd3948e268878a9ee8925bfe1ad1da4f/optree-0.19.0-cp314-cp314t-win32.whl", hash = "sha256:b15ab972e2133e70570259386684624a17128daab7fb353a0a7435e9dd2c7354" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c4/43/bbc4c7a1f37f1a0ed6efe07a5c44b2835e81d1f6ce1cca6a395a2339e60f/optree-0.19.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c90c15a80c325c2c6e03e20c95350df5db4591d35e8e4a35a40d2f865c260193" },
- { url = "https://mirrors.aliyun.com/pypi/packages/62/12/6758b43dbddc6911e3225a15ca686c913959fb63c267840b54f0002be503/optree-0.19.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a1e7b358df8fc4b97a05380d446e87b08eac899c1f34d9846b9afa0be7f96bc7" },
-]
-
[[package]]
name = "orjson"
version = "3.10.18"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/81/0b/fea456a3ffe74e70ba30e01ec183a9b26bec4d497f61dcfce1b601059c60/orjson-3.10.18.tar.gz", hash = "sha256:e8da3947d92123eda795b68228cafe2724815621fe35e8e320a9e9593a4bcd53" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/21/1a/67236da0916c1a192d5f4ccbe10ec495367a726996ceb7614eaa687112f2/orjson-3.10.18-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:50c15557afb7f6d63bc6d6348e0337a880a04eaa9cd7c9d569bcb4e760a24753" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b3/bc/c7f1db3b1d094dc0c6c83ed16b161a16c214aaa77f311118a93f647b32dc/orjson-3.10.18-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:356b076f1662c9813d5fa56db7d63ccceef4c271b1fb3dd522aca291375fcf17" },
- { url = "https://mirrors.aliyun.com/pypi/packages/af/84/664657cd14cc11f0d81e80e64766c7ba5c9b7fc1ec304117878cc1b4659c/orjson-3.10.18-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:559eb40a70a7494cd5beab2d73657262a74a2c59aff2068fdba8f0424ec5b39d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9a/bb/f50039c5bb05a7ab024ed43ba25d0319e8722a0ac3babb0807e543349978/orjson-3.10.18-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3c29eb9a81e2fbc6fd7ddcfba3e101ba92eaff455b8d602bf7511088bbc0eae" },
- { url = "https://mirrors.aliyun.com/pypi/packages/93/8c/ee74709fc072c3ee219784173ddfe46f699598a1723d9d49cbc78d66df65/orjson-3.10.18-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6612787e5b0756a171c7d81ba245ef63a3533a637c335aa7fcb8e665f4a0966f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6a/37/e6d3109ee004296c80426b5a62b47bcadd96a3deab7443e56507823588c5/orjson-3.10.18-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ac6bd7be0dcab5b702c9d43d25e70eb456dfd2e119d512447468f6405b4a69c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4f/5d/387dafae0e4691857c62bd02839a3bf3fa648eebd26185adfac58d09f207/orjson-3.10.18-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9f72f100cee8dde70100406d5c1abba515a7df926d4ed81e20a9730c062fe9ad" },
- { url = "https://mirrors.aliyun.com/pypi/packages/27/6f/875e8e282105350b9a5341c0222a13419758545ae32ad6e0fcf5f64d76aa/orjson-3.10.18-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dca85398d6d093dd41dc0983cbf54ab8e6afd1c547b6b8a311643917fbf4e0c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/48/b2/73a1f0b4790dcb1e5a45f058f4f5dcadc8a85d90137b50d6bbc6afd0ae50/orjson-3.10.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:22748de2a07fcc8781a70edb887abf801bb6142e6236123ff93d12d92db3d406" },
- { url = "https://mirrors.aliyun.com/pypi/packages/56/f5/7ed133a5525add9c14dbdf17d011dd82206ca6840811d32ac52a35935d19/orjson-3.10.18-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:3a83c9954a4107b9acd10291b7f12a6b29e35e8d43a414799906ea10e75438e6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/11/7c/439654221ed9c3324bbac7bdf94cf06a971206b7b62327f11a52544e4982/orjson-3.10.18-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:303565c67a6c7b1f194c94632a4a39918e067bd6176a48bec697393865ce4f06" },
- { url = "https://mirrors.aliyun.com/pypi/packages/48/e7/d58074fa0cc9dd29a8fa2a6c8d5deebdfd82c6cfef72b0e4277c4017563a/orjson-3.10.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:86314fdb5053a2f5a5d881f03fca0219bfdf832912aa88d18676a5175c6916b5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/57/4d/fe17581cf81fb70dfcef44e966aa4003360e4194d15a3f38cbffe873333a/orjson-3.10.18-cp312-cp312-win32.whl", hash = "sha256:187ec33bbec58c76dbd4066340067d9ece6e10067bb0cc074a21ae3300caa84e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e6/22/469f62d25ab5f0f3aee256ea732e72dc3aab6d73bac777bd6277955bceef/orjson-3.10.18-cp312-cp312-win_amd64.whl", hash = "sha256:f9f94cf6d3f9cd720d641f8399e390e7411487e493962213390d1ae45c7814fc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/10/b0/1040c447fac5b91bc1e9c004b69ee50abb0c1ffd0d24406e1350c58a7fcb/orjson-3.10.18-cp312-cp312-win_arm64.whl", hash = "sha256:3d600be83fe4514944500fa8c2a0a77099025ec6482e8087d7659e891f23058a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/04/f0/8aedb6574b68096f3be8f74c0b56d36fd94bcf47e6c7ed47a7bd1474aaa8/orjson-3.10.18-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:69c34b9441b863175cc6a01f2935de994025e773f814412030f269da4f7be147" },
{ url = "https://mirrors.aliyun.com/pypi/packages/bc/f7/7118f965541aeac6844fcb18d6988e111ac0d349c9b80cda53583e758908/orjson-3.10.18-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:1ebeda919725f9dbdb269f59bc94f861afbe2a27dce5608cdba2d92772364d1c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fb/d9/839637cc06eaf528dd8127b36004247bf56e064501f68df9ee6fd56a88ee/orjson-3.10.18-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5adf5f4eed520a4959d29ea80192fa626ab9a20b2ea13f8f6dc58644f6927103" },
@@ -5081,15 +4495,32 @@ wheels = [
[[package]]
name = "ormsgpack"
-version = "1.5.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/c5/70/11a6ab33136c2f98bb64e96743a55c7a87b87bae0413460cab7cc5764951/ormsgpack-1.5.0.tar.gz", hash = "sha256:00c0743ebaa8d21f1c868fbb609c99151ea79e67fec98b51a29077efd91ce348" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/47/19/df1626f9c149a20d2273eecf97ae913a026be2730264db86126ac3e594db/ormsgpack-1.5.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:a921b0d54b5fb5ba1ea4e87c65caa8992736224f1fc5ce8f46a882e918c8e22d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/82/cc/bad6d4a237ff0943cb1c8c4a12fe95bcd7ff81c0f8bca26340efd599aa1d/ormsgpack-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6d423668e2c3abdbc474562b1c73360ff7326f06cb9532dcb73254b5b63dae4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c7/d1/3ed38a54923fe04eace750c0f0adbc149fb2b028375c71e864aee5e2d6d6/ormsgpack-1.5.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eeb2dd4ed3e503a8266dcbfbb8d810a36baa34e4bb4229e90e9c213058a06d74" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9b/52/0261a80de2486793b4844c2668b17f49d03a20aba13a8d3d975831b1d866/ormsgpack-1.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f13bd643df1324e8797caba4c5c0168a87524df8424e8413ba29723e89a586a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/f0/2ebda08824d4f658c5ad048bcbe64e352b637b661b4d26c51d7403d30569/ormsgpack-1.5.0-cp312-none-win_amd64.whl", hash = "sha256:e016da381a126478c4bafab0ae19d3a2537f6471341ecced4bb61471e8841cad" },
+version = "1.12.2"
+source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
+sdist = { url = "https://mirrors.aliyun.com/pypi/packages/12/0c/f1761e21486942ab9bb6feaebc610fa074f7c5e496e6962dea5873348077/ormsgpack-1.12.2.tar.gz", hash = "sha256:944a2233640273bee67521795a73cf1e959538e0dfb7ac635505010455e53b33" }
+wheels = [
+ { url = "https://mirrors.aliyun.com/pypi/packages/eb/29/bb0eba3288c0449efbb013e9c6f58aea79cf5cb9ee1921f8865f04c1a9d7/ormsgpack-1.12.2-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5ea60cb5f210b1cfbad8c002948d73447508e629ec375acb82910e3efa8ff355" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/6e/31/5efa31346affdac489acade2926989e019e8ca98129658a183e3add7af5e/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3601f19afdbea273ed70b06495e5794606a8b690a568d6c996a90d7255e51c1" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/eb/56/d0087278beef833187e0167f8527235ebe6f6ffc2a143e9de12a98b1ce87/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29a9f17a3dac6054c0dce7925e0f4995c727f7c41859adf9b5572180f640d172" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/1c/a2/072343e1413d9443e5a252a8eb591c2d5b1bffbe5e7bfc78c069361b92eb/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39c1bd2092880e413902910388be8715f70b9f15f20779d44e673033a6146f2d" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/a2/8b/a0da3b98a91d41187a63b02dda14267eefc2a74fcb43cc2701066cf1510e/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:50b7249244382209877deedeee838aef1542f3d0fc28b8fe71ca9d7e1896a0d7" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/19/bb/6d226bc4cf9fc20d8eb1d976d027a3f7c3491e8f08289a2e76abe96a65f3/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:5af04800d844451cf102a59c74a841324868d3f1625c296a06cc655c542a6685" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/fb/f1/bb2c7223398543dedb3dbf8bb93aaa737b387de61c5feaad6f908841b782/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cec70477d4371cd524534cd16472d8b9cc187e0e3043a8790545a9a9b296c258" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/7b/e8/0fb45f57a2ada1fed374f7494c8cd55e2f88ccd0ab0a669aa3468716bf5f/ormsgpack-1.12.2-cp313-cp313-win_amd64.whl", hash = "sha256:21f4276caca5c03a818041d637e4019bc84f9d6ca8baa5ea03e5cc8bf56140e9" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/7a/d4/0cfeea1e960d550a131001a7f38a5132c7ae3ebde4c82af1f364ccc5d904/ormsgpack-1.12.2-cp313-cp313-win_arm64.whl", hash = "sha256:baca4b6773d20a82e36d6fd25f341064244f9f86a13dead95dd7d7f996f51709" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/94/16/24d18851334be09c25e87f74307c84950f18c324a4d3c0b41dabdbf19c29/ormsgpack-1.12.2-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bc68dd5915f4acf66ff2010ee47c8906dc1cf07399b16f4089f8c71733f6e36c" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/b5/a2/88b9b56f83adae8032ac6a6fa7f080c65b3baf9b6b64fd3d37bd202991d4/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46d084427b4132553940070ad95107266656cb646ea9da4975f85cb1a6676553" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/a9/80/43e4555963bf602e5bdc79cbc8debd8b6d5456c00d2504df9775e74b450b/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c010da16235806cf1d7bc4c96bf286bfa91c686853395a299b3ddb49499a3e13" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/78/e1/7cfbf28de8bca6efe7e525b329c31277d1b64ce08dcba723971c241a9d60/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18867233df592c997154ff942a6503df274b5ac1765215bceba7a231bea2745d" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/95/f8/30ae5716e88d792a4e879debee195653c26ddd3964c968594ddef0a3cc7e/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b009049086ddc6b8f80c76b3955df1aa22a5fbd7673c525cd63bf91f23122ede" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/dc/81/aee5b18a3e3a0e52f718b37ab4b8af6fae0d9d6a65103036a90c2a8ffb5d/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1dcc17d92b6390d4f18f937cf0b99054824a7815818012ddca925d6e01c2e49e" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/bd/17/71c9ba472d5d45f7546317f467a5fc941929cd68fb32796ca3d13dcbaec2/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f04b5e896d510b07c0ad733d7fce2d44b260c5e6c402d272128f8941984e4285" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/2e/a6/ac99cd7fe77e822fed5250ff4b86fa66dd4238937dd178d2299f10b69816/ormsgpack-1.12.2-cp314-cp314-win_amd64.whl", hash = "sha256:ae3aba7eed4ca7cb79fd3436eddd29140f17ea254b91604aa1eb19bfcedb990f" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/3a/67/339872846a1ae4592535385a1c1f93614138566d7af094200c9c3b45d1e5/ormsgpack-1.12.2-cp314-cp314-win_arm64.whl", hash = "sha256:118576ea6006893aea811b17429bfc561b4778fad393f5f538c84af70b01260c" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/49/c2/6feb972dc87285ad381749d3882d8aecbde9f6ecf908dd717d33d66df095/ormsgpack-1.12.2-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7121b3d355d3858781dc40dafe25a32ff8a8242b9d80c692fd548a4b1f7fd3c8" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/a3/9a/900a6b9b413e0f8a471cf07830f9cf65939af039a362204b36bd5b581d8b/ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ee766d2e78251b7a63daf1cddfac36a73562d3ddef68cacfb41b2af64698033" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/87/4c/27a95466354606b256f24fad464d7c97ab62bce6cc529dd4673e1179b8fb/ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292410a7d23de9b40444636b9b8f1e4e4b814af7f1ef476e44887e52a123f09d" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/73/cd/29cee6007bddf7a834e6cd6f536754c0535fcb939d384f0f37a38b1cddb8/ormsgpack-1.12.2-cp314-cp314t-win_amd64.whl", hash = "sha256:837dd316584485b72ef451d08dd3e96c4a11d12e4963aedb40e08f89685d8ec2" },
]
[[package]]
@@ -5139,13 +4570,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35" },
- { url = "https://mirrors.aliyun.com/pypi/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98" },
- { url = "https://mirrors.aliyun.com/pypi/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084" },
- { url = "https://mirrors.aliyun.com/pypi/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713" },
{ url = "https://mirrors.aliyun.com/pypi/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d" },
@@ -5269,17 +4693,6 @@ version = "12.2.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421" },
- { url = "https://mirrors.aliyun.com/pypi/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005" },
- { url = "https://mirrors.aliyun.com/pypi/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2" },
{ url = "https://mirrors.aliyun.com/pypi/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c" },
@@ -5457,13 +4870,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/42/8b/5f939eaf1fbeb7ff914fe540d659486951a056e5537b8f454362045b6c72/pot-0.9.6.post1.tar.gz", hash = "sha256:9b6cc14a8daecfe1268268168cf46548f9130976b22b24a9e8ec62a734be6c43" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/b9/28/13622807461f9f6082a8cd6768f9b4a810bc3a8fda474b81572da94b4d23/pot-0.9.6.post1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f7c542fc20662e35c24dd82eeff8a737220757434d7f0038664a7322221452f7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c6/5c/b4e017560531f53d06798c681b0d0a9488bb8116bc98da9d399a3d096391/pot-0.9.6.post1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c1755516a7354cbd6110ad2e5f341b98b9968240c2f0f67b0ff5e3ebcb3105bd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/07/9f/57e49b3f7173359741053c5e2766a45dcf649d767c2e967ef93526c9045f/pot-0.9.6.post1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3207362d3e3b5aaa783f452aa85f66e83edbefb5764f34662860af54ac72ee6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/30/60/fa72dd6094f7dbe6b38e2c6907af8cd0f18c6bd107e0cf4874deddaba883/pot-0.9.6.post1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f6659c5657e6d7e9f98f4a82e0ed64f88e9fce69b2e557416d156343919ba3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2f/3f/cc519c1176116271b6282268a705162fa042c16cc922bc56039445c9d697/pot-0.9.6.post1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f1b0148ae17bec0ed12264c6da3a05e13913b716e2a8c9043242b5d8349d8df" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f5/01/0132c94404cd0b1b2f21c4a49698db9dcd6107c47c02b22df1ed38206b2a/pot-0.9.6.post1-cp312-cp312-win32.whl", hash = "sha256:571e543cc2b0a462365002203595baf2b89c3d064cce4fce70fd1231e832c21f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c1/6d/23229c0e198a4f7fb27750b3ef8497e6ebed23fe531ed64b5194da8b2b02/pot-0.9.6.post1-cp312-cp312-win_amd64.whl", hash = "sha256:b1d8bd9a334c72baa37f9a2b268de5366c23c0f9c9e3d6dc25d150137ec2823c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/53/17/e4aebb8deef58b0d40ac339d952d12c63559801b50ae43c622d49bebda7e/pot-0.9.6.post1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:659fff750a162f58b52b33a64c4ac358f4ff44e9dff0841052c088e1b6a54430" },
{ url = "https://mirrors.aliyun.com/pypi/packages/f7/b9/3646c153b13f999ac30112dcf85c5f233af79b0d98c37b52dda9a624c91b/pot-0.9.6.post1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4f54830e9f9cb78b1ff7abd5c5bf162625ed6aea903241267c64ea9f0fb73ddb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/53/e9/c7092f7aec8cb32739ad66ba1f1259626546e4893b61b905ce2da3987235/pot-0.9.6.post1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e9fd4b1fafacd37debdb984687ddb26f5c43d1429401847d388a6f1bd1f10e98" },
@@ -5483,14 +4889,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/43/75/fe6b7bbd0dea530a001b0e24c331b21a0be2786e402abf3c57f5dce43d4b/preshed-3.0.13.tar.gz", hash = "sha256:d75f718bbfd97e992f7827e0fa7faf6a91bdd9c922d5baa4b50d62731396cb89" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/39/fb/ccff23c44c04088c248539005fcda78b9014512a34d170c5360f02ad908b/preshed-3.0.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5d14eea14bd01291388928991d7df7d60b9fd19ae970e55006eb4d29b0c1e8eb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8e/ce/cad5a8145881a771e6c0d002f2e585fc19b962f120860b54d32af5baa342/preshed-3.0.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f05b08ce92399c0655b5e0eb5a1cc1f9e295703ed3aabdfaf6538dfa8ae23d57" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a7/a2/c5fed4fb3e946699259d11e4036a3cfdd8c89b3e542e3077d46781642425/preshed-3.0.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:62cf7f3113132891d6bba70ff547ad81c6fe50a31930bbbb8499f1d47cd122b7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/51/94/8c9bc48a6ea4903f53a1a0031ce8e35687526949f25821762ef21493c007/preshed-3.0.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8b8de3f58043070a354477995acdd98626ce43e4193c708ebd0f694e467f5155" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b6/df/ecd2f40055ff52527ca117ffbfafb888c1a3079b59fbabe03c5b8f9b7240/preshed-3.0.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:183b339956a9e1d7a4a00038a3b9587a734db9e8bd915939a49791bd1b372156" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e6/88/bdb244e40284ded3632a9f88c23bc80230bd7b2ae4a8b7f2cc91adead7a8/preshed-3.0.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e77bed56aded7cbe5d28d6bd2178bc5b13eda0e0e464dab205fb578fa915000" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a0/c9/c91ea56342e6c364fc69b444a1ac5432327857199c44032c9cc9dc4c3a23/preshed-3.0.13-cp312-cp312-win_amd64.whl", hash = "sha256:04d8f13f2986e5d11af5ac51f55ce3106c70c41b483d20ea392e6180bdd0f870" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b2/0b/6a99d99619fd83b14c696e2489caed7070647488d4d3ac0b723d35db2de0/preshed-3.0.13-cp312-cp312-win_arm64.whl", hash = "sha256:19318dc1cd8cac6663c6c830bf7e0002d2de853769fb03e056774e97c21bedfd" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0e/2a/401158195d6dc7f6aef0b354d74d0e95c9da124499448c2b3dbb95b71204/preshed-3.0.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0d0c14187dc0078d8a63bf190ec045a4d13e7748b6caeb557a7d575e411410b" },
{ url = "https://mirrors.aliyun.com/pypi/packages/88/8f/e20e64573988528785447a6893b2e7ab287ecfd85b3888e978b28812fd20/preshed-3.0.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7770987c2e57497cd26124a9be5f652b5b3ccd0def89859ab0da8bca6144a3de" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b9/72/18168f881359c4482d312f8dc196371bdd61c1583a52b34390da4c88bbea/preshed-3.0.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4a7bc48220de579be6bdb0a8715482cf36e2a625a6fd5ad26c9f43485a4a23b5" },
@@ -5591,21 +4989,6 @@ version = "0.4.1"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/9e/da/e9fc233cf63743258bff22b3dfa7ea5baef7b5bc324af47a0ad89b8ffc6f/propcache-0.4.1.tar.gz", hash = "sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/a2/0f/f17b1b2b221d5ca28b4b876e8bb046ac40466513960646bda8e1853cdfa2/propcache-0.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/76/47/8ccf75935f51448ba9a16a71b783eb7ef6b9ee60f5d14c7f8a8a79fbeed7/propcache-0.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0a/b6/5c9a0e42df4d00bfb4a3cbbe5cf9f54260300c88a0e9af1f47ca5ce17ac0/propcache-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/d3/6c7ee328b39a81ee877c962469f1e795f9db87f925251efeb0545e0020d0/propcache-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72" },
- { url = "https://mirrors.aliyun.com/pypi/packages/01/5d/1c53f4563490b1d06a684742cc6076ef944bc6457df6051b7d1a877c057b/propcache-0.4.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367" },
- { url = "https://mirrors.aliyun.com/pypi/packages/20/e1/ce4620633b0e2422207c3cb774a0ee61cac13abc6217763a7b9e2e3f4a12/propcache-0.4.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/46/4b/3aae6835b8e5f44ea6a68348ad90f78134047b503765087be2f9912140ea/propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6e/a5/8a5e8678bcc9d3a1a15b9a29165640d64762d424a16af543f00629c87338/propcache-0.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f1/63/b7b215eddeac83ca1c6b934f89d09a625aa9ee4ba158338854c87210cc36/propcache-0.4.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778" },
- { url = "https://mirrors.aliyun.com/pypi/packages/57/74/f580099a58c8af587cac7ba19ee7cb418506342fbbe2d4a4401661cca886/propcache-0.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c4/ee/542f1313aff7eaf19c2bb758c5d0560d2683dac001a1c96d0774af799843/propcache-0.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8f/18/9c6b015dd9c6930f6ce2229e1f02fb35298b847f2087ea2b436a5bfa7287/propcache-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75" },
- { url = "https://mirrors.aliyun.com/pypi/packages/80/9e/e7b85720b98c45a45e1fca6a177024934dc9bc5f4d5dd04207f216fc33ed/propcache-0.4.1-cp312-cp312-win32.whl", hash = "sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/54/09/d19cff2a5aaac632ec8fc03737b223597b1e347416934c1b3a7df079784c/propcache-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db" },
- { url = "https://mirrors.aliyun.com/pypi/packages/68/ab/6b5c191bb5de08036a8c697b265d4ca76148efb10fa162f14af14fb5f076/propcache-0.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/bf/df/6d9c1b6ac12b003837dde8a10231a7344512186e87b36e855bef32241942/propcache-0.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf" },
{ url = "https://mirrors.aliyun.com/pypi/packages/8b/e8/677a0025e8a2acf07d3418a2e7ba529c9c33caf09d3c1f25513023c1db56/propcache-0.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311" },
{ url = "https://mirrors.aliyun.com/pypi/packages/89/a4/92380f7ca60f99ebae761936bc48a72a639e8a47b29050615eef757cb2a7/propcache-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74" },
@@ -5729,17 +5112,6 @@ version = "2.9.11"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/ac/6c/8767aaa597ba424643dc87348c6f1754dd9f48e80fdc1b9f7ca5c3a7c213/psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/d8/91/f870a02f51be4a65987b45a7de4c2e1897dd0d01051e2b559a38fa634e3e/psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/27/fa/cae40e06849b6c9a95eb5c04d419942f00d9eaac8d81626107461e268821/psycopg2_binary-2.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6f/a0/567f7ea38b6e1c62aafd58375665a547c00c608a471620c0edc364733e13/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3c/94/c1777c355bc560992af848d98216148be5f1be001af06e06fc49cbded578/psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bd/42/c9a21edf0e3daa7825ed04a4a8588686c6c14904344344a039556d78aa58/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/12/22/dedfbcfa97917982301496b6b5e5e6c5531d1f35dd2b488b08d1ebc52482/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/66/ea/d3390e6696276078bd01b2ece417deac954dfdd552d2edc3d03204416c0c/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34" },
- { url = "https://mirrors.aliyun.com/pypi/packages/12/9a/0402ded6cbd321da0c0ba7d34dc12b29b14f5764c2fc10750daa38e825fc/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b1/d2/99b55e85832ccde77b211738ff3925a5d73ad183c0b37bcbbe5a8ff04978/psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee" },
{ url = "https://mirrors.aliyun.com/pypi/packages/11/32/b2ffe8f3853c181e88f0a157c5fb4e383102238d73c52ac6d93a5c8bffe6/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0" },
@@ -5804,13 +5176,6 @@ version = "22.0.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/30/53/04a7fdc63e6056116c9ddc8b43bc28c12cdd181b85cbeadb79278475f3ae/pyarrow-22.0.0.tar.gz", hash = "sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/af/63/ba23862d69652f85b615ca14ad14f3bcfc5bf1b99ef3f0cd04ff93fdad5a/pyarrow-22.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b1/d0/f9ad86fe809efd2bcc8be32032fa72e8b0d112b01ae56a053006376c5930/pyarrow-22.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b4/a8/f910afcb14630e64d673f15904ec27dd31f1e009b77033c365c84e8c1e1d/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/13/95/aec81f781c75cd10554dc17a25849c720d54feafb6f7847690478dcf5ef8/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bb/d4/74ac9f7a54cfde12ee42734ea25d5a3c9a45db78f9def949307a92720d37/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2e/71/fedf2499bf7a95062eafc989ace56572f3343432570e1c54e6599d5b88da/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/68/ed/b202abd5a5b78f519722f3d29063dda03c114711093c1995a33b8e2e0f4b/pyarrow-22.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a6/d6/d0fac16a2963002fc22c8fa75180a838737203d558f0ed3b564c4a54eef5/pyarrow-22.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/c6/9c/1d6357347fbae062ad3f17082f9ebc29cc733321e892c0d2085f42a2212b/pyarrow-22.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ff/c0/782344c2ce58afbea010150df07e3a2f5fdad299cd631697ae7bd3bac6e3/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691" },
@@ -5868,12 +5233,6 @@ version = "1.4.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/f6/21/3c06205bb407e1f79b73b7b4dfb3950bd9537c4f625a68ab5cc41177f5bc/pyclipper-1.4.0.tar.gz", hash = "sha256:9882bd889f27da78add4dd6f881d25697efc740bf840274e749988d25496c8e1" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/90/1b/7a07b68e0842324d46c03e512d8eefa9cb92ba2a792b3b4ebf939dafcac3/pyclipper-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:222ac96c8b8281b53d695b9c4fedc674f56d6d4320ad23f1bdbd168f4e316140" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6b/dd/8bd622521c05d04963420ae6664093f154343ed044c53ea260a310c8bb4d/pyclipper-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f3672dbafbb458f1b96e1ee3e610d174acb5ace5bd2ed5d1252603bb797f2fc6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7a/06/6e3e241882bf7d6ab23d9c69ba4e85f1ec47397cbbeee948a16cf75e21ed/pyclipper-1.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d1f807e2b4760a8e5c6d6b4e8c1d71ef52b7fe1946ff088f4fa41e16a881a5ca" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/f4/3418c1cd5eea640a9fa2501d4bc0b3655fa8d40145d1a4f484b987990a75/pyclipper-1.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce1f83c9a4e10ea3de1959f0ae79e9a5bd41346dff648fee6228ba9eaf8b3872" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ac/94/c85401d24be634af529c962dd5d781f3cb62a67cd769534df2cb3feee97a/pyclipper-1.4.0-cp312-cp312-win32.whl", hash = "sha256:3ef44b64666ebf1cb521a08a60c3e639d21b8c50bfbe846ba7c52a0415e936f4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/97/77/dfea08e3b230b82ee22543c30c35d33d42f846a77f96caf7c504dd54fab1/pyclipper-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:d1e5498d883b706a4ce636247f0d830c6eb34a25b843a1b78e2c969754ca9037" },
{ url = "https://mirrors.aliyun.com/pypi/packages/67/d0/cbce7d47de1e6458f66a4d999b091640134deb8f2c7351eab993b70d2e10/pyclipper-1.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d49df13cbb2627ccb13a1046f3ea6ebf7177b5504ec61bdef87d6a704046fd6e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ce/cc/742b9d69d96c58ac156947e1b56d0f81cbacbccf869e2ac7229f2f86dc4e/pyclipper-1.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37bfec361e174110cdddffd5ecd070a8064015c99383d95eb692c253951eee8a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/db/48/dd301d62c1529efdd721b47b9e5fb52120fcdac5f4d3405cfc0d2f391414/pyclipper-1.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:14c8bdb5a72004b721c4e6f448d2c2262d74a7f0c9e3076aeff41e564a92389f" },
@@ -5973,20 +5332,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69" },
- { url = "https://mirrors.aliyun.com/pypi/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75" },
- { url = "https://mirrors.aliyun.com/pypi/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05" },
- { url = "https://mirrors.aliyun.com/pypi/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3" },
{ url = "https://mirrors.aliyun.com/pypi/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34" },
{ url = "https://mirrors.aliyun.com/pypi/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0" },
@@ -6029,10 +5374,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa" },
{ url = "https://mirrors.aliyun.com/pypi/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008" },
- { url = "https://mirrors.aliyun.com/pypi/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b" },
]
[[package]]
@@ -6213,15 +5554,6 @@ version = "5.3.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/8f/85/44b10070a769a56bd910009bb185c0c0a82daff8d567cd1a116d7d730c7d/pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/f5/0c/7ecf8077f4b932a5d25896699ff5c394ffc2a880a9c2c284d6a3e6ea5949/pyodbc-5.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ebf6b5d989395efe722b02b010cb9815698a4d681921bf5db1c0e1195ac1bde" },
- { url = "https://mirrors.aliyun.com/pypi/packages/03/78/9fbde156055d88c1ef3487534281a5b1479ee7a2f958a7e90714968749ac/pyodbc-5.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:197bb6ddafe356a916b8ee1b8752009057fce58e216e887e2174b24c7ab99269" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9f/f9/8c106dcd6946e95fee0da0f1ba58cd90eb872eebe8968996a2ea1f7ac3c1/pyodbc-5.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6ccb5315ec9e081f5cbd66f36acbc820ad172b8fa3736cf7f993cdf69bd8a96" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4b/30/2c70f47a76a4fafa308d148f786aeb35a4d67a01d41002f1065b465d9994/pyodbc-5.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5dd3d5e469f89a3112cf8b0658c43108a4712fad65e576071e4dd44d2bd763c7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7d/b2/0631d84731606bfe40d3b03a436b80cbd16b63b022c7b13444fb30761ca8/pyodbc-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b180bc5e49b74fd40a24ef5b0fe143d0c234ac1506febe810d7434bf47cb925b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/74/b9/707c5314cca9401081b3757301241c167a94ba91b4bd55c8fa591bf35a4a/pyodbc-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e3c39de3005fff3ae79246f952720d44affc6756b4b85398da4c5ea76bf8f506" },
- { url = "https://mirrors.aliyun.com/pypi/packages/97/7c/893036c8b0c8d359082a56efdaa64358a38dda993124162c3faa35d1924d/pyodbc-5.3.0-cp312-cp312-win32.whl", hash = "sha256:d32c3259762bef440707098010035bbc83d1c73d81a434018ab8c688158bd3bb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c0/70/5e61b216cc13c7f833ef87f4cdeab253a7873f8709253f5076e9bb16c1b3/pyodbc-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe77eb9dcca5fc1300c9121f81040cc9011d28cff383e2c35416e9ec06d4bc95" },
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/85/e7d0629c9714a85eb4f85d21602ce6d8a1ec0f313fde8017990cf913e3b4/pyodbc-5.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:afe7c4ac555a8d10a36234788fc6cfc22a86ce37fc5ba88a1f75b3e6696665dc" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0c/1d/9e74cbcc1d4878553eadfd59138364b38656369eb58f7e5b42fb344c0ce7/pyodbc-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e9ab0b91de28a5ab838ac4db0253d7cc8ce2452efe4ad92ee6a57b922bf0c24" },
{ url = "https://mirrors.aliyun.com/pypi/packages/37/c7/27d83f91b3144d3e275b5b387f0564b161ddbc4ce1b72bb3b3653e7f4f7a/pyodbc-5.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6132554ffbd7910524d643f13ce17f4a72f3a6824b0adef4e9a7f66efac96350" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1b/33/2bb24e7fc95e98a7b11ea5ad1f256412de35d2e9cc339be198258c1d9a76/pyodbc-5.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1629af4706e9228d79dabb4863c11cceb22a6dab90700db0ef449074f0150c0d" },
@@ -6257,7 +5589,6 @@ version = "25.1.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "cryptography" },
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/04/8c/cd89ad05804f8e3c17dea8f178c3f40eeab5694c30e0c9f5bcd49f576fc3/pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b" }
wheels = [
@@ -6360,7 +5691,6 @@ version = "1.3.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "pytest" },
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5" }
wheels = [
@@ -6428,20 +5758,6 @@ version = "0.6.2"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/01/18/e1e53ade001b30a3c6642d876e5defe8431da8c31fb7798909e6c8ab8c34/python_calamine-0.6.2.tar.gz", hash = "sha256:2c90e5224c5e92db9fcd8f22b6085ce63b935cfe7a893ac9a1c3c56793bafd9d" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/7a/ec/e111c1a3a4c138ebc41e416e33730ee6d7c54e714af21c2a4e59b41715a5/python_calamine-0.6.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:857e4cddadba9b55c76dc583c58c5dc101a6cd5320190c10f8b2ab98d66c9040" },
- { url = "https://mirrors.aliyun.com/pypi/packages/53/26/fe4c2138ff21542e2f1130a4d83c330d7f9486b62775196e998b88a03de6/python_calamine-0.6.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cd89d6a53e4b22328cd685fc054c31d359cb3ae67bd24bc57e1c1db62a4cfc97" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6a/b0/bfeaf45ac5e2f6553723dd2fbe127d1d17c6f26496db5781de42a933776a/python_calamine-0.6.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d6c9af39db39e0c70710ae79cd1b5d980f9c0aea55fc16d194460c1561a0c6a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fe/6e/81106aa80609075015d400584030605b05f5e12931717160dcc58fdc4980/python_calamine-0.6.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a2382dbc410dd48c99d89ee460662cc70892fe1b2901ab982604b923e8eb8f6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4d/ba/6311b24f9889246be63b664630c5601039ef771f7ed04c8f51aace39b7a9/python_calamine-0.6.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ebb93255709874ede5b5e62828cb5758e60097e5390b6c9a3eb7751b617b12e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/23/e4/027a1b046d30768872307ebe808dc4cdc5357295cdcda98b30b3ea924904/python_calamine-0.6.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:837bca19bd945cb83aded433f4cf76e80d70a5400404d876400ca7e88e5ea311" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5c/4d/da8716a1b3a66938aaabe36873f6fa210fa063bab1b20c2ec236013de6b3/python_calamine-0.6.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723990a47668cb819f307ccc634741370d3cd3804a0ee8cda392a522ae6d5016" },
- { url = "https://mirrors.aliyun.com/pypi/packages/36/40/9521e8da5496cbc4b18027626a40018301f546b3e9802ca2f3a6cb5b4739/python_calamine-0.6.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b067630d693e1d7de41e3d44a99c7dd3feebb52db8dda8636ac3f70d8b6a4ad6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cd/b0/7a63963512c5ba7e9539b7452e2b1561625e63e4e29c044e487e2e93dcbe/python_calamine-0.6.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6ab09c9da53a2b33633e9f940aed11c08e083810a0fd6885826cdc52ba4f86a5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/22/81/e2bc38a5cf9629f656adcdabe8e134028f60c236e4bb96375dda90db3fdd/python_calamine-0.6.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:ae08e1308a0d0c6b8b4cc0a039ed8a85fc9ee2f8a3ca9ea57b1af9f97ed68fe4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b1/ea/513117015fd5903ca6dde9c8fb8502af60af6965642f4e3311623943e673/python_calamine-0.6.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c441a20c7aff0e904ca01b5cdc1e5be2c6d4a41a24a0ea4d5ea6d211343bb95f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a2/14/8846478dacf31535f5f15448ade3bc688b51f3183f1b52844451aa27b0e6/python_calamine-0.6.2-cp312-cp312-win32.whl", hash = "sha256:39cae8e66f8bce499f5f965f4575ddf61e30184cc97f02e1c7031a57abe0903b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bb/e2/2d2dcf4ec7e5ec08e33bf966ab010a7be178a4b623bd5f7601d47f2c734c/python_calamine-0.6.2-cp312-cp312-win_amd64.whl", hash = "sha256:1617efa24532f2420934a8cf77e6d33ff1740cae1d39355cab4f4cf141fdab49" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f1/eb/2f50f3395c0435e6186cab56c36d04c06581ba827264bca1f1acae523aa3/python_calamine-0.6.2-cp312-cp312-win_arm64.whl", hash = "sha256:c2b378db494740e540e8157a7e5fe61dadae69ad2d988a7c80f9583f434acf07" },
{ url = "https://mirrors.aliyun.com/pypi/packages/15/db/f409c3ffa5d452b8184978c94440b48c933c79232c5e40fe9ce3608ff06d/python_calamine-0.6.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:4c6e68c233841604fa3f63899d13bd2e47cddf0787c4b4b8188f74c3be452045" },
{ url = "https://mirrors.aliyun.com/pypi/packages/66/fe/8cf4309a00ad5628c45e69f13352d6a1e0e0a3148a2fc28d7a43a8cefec9/python_calamine-0.6.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0fd5bcbd904d05f8b9f127a93706fdbb0a5934efdc9677b402a82d91e6e3f920" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b8/cc/c5edfb89a99d19c66b029e2e6dc0db052709888753fc0a771bf28343c5e5/python_calamine-0.6.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cef6454aa1b3b2137d7a202c9f84b87dffdd187ff218f2cee459480c102c20a3" },
@@ -6595,9 +5911,6 @@ name = "pywin32"
version = "311"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d" },
{ url = "https://mirrors.aliyun.com/pypi/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a" },
@@ -6612,16 +5925,6 @@ version = "6.0.3"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196" },
- { url = "https://mirrors.aliyun.com/pypi/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28" },
- { url = "https://mirrors.aliyun.com/pypi/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea" },
- { url = "https://mirrors.aliyun.com/pypi/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c" },
@@ -6765,6 +6068,7 @@ dependencies = [
{ name = "arxiv" },
{ name = "asana" },
{ name = "atlassian-python-api" },
+ { name = "audioop-lts" },
{ name = "azure-identity" },
{ name = "azure-storage-file-datalake" },
{ name = "beartype" },
@@ -6864,7 +6168,6 @@ dependencies = [
{ name = "socksio" },
{ name = "spacy" },
{ name = "sqlglotrs" },
- { name = "strenum" },
{ name = "tavily-python" },
{ name = "tencentcloud-sdk-python" },
{ name = "tika" },
@@ -6898,8 +6201,6 @@ test = [
{ name = "reportlab" },
{ name = "requests" },
{ name = "requests-toolbelt" },
- { name = "tensorflow-cpu", version = "2.18.0", source = { registry = "https://mirrors.aliyun.com/pypi/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "tensorflow-cpu", version = "2.18.1", source = { registry = "https://mirrors.aliyun.com/pypi/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
]
[package.metadata]
@@ -6912,6 +6213,7 @@ requires-dist = [
{ name = "arxiv", specifier = "==2.1.3" },
{ name = "asana", specifier = ">=5.2.2" },
{ name = "atlassian-python-api", specifier = "==4.0.7" },
+ { name = "audioop-lts", specifier = ">=0.2.1" },
{ name = "azure-identity", specifier = ">=1.25.3" },
{ name = "azure-storage-file-datalake", specifier = "==12.16.0" },
{ name = "beartype", specifier = ">=0.20.0,<1.0.0" },
@@ -6977,7 +6279,7 @@ requires-dist = [
{ name = "opencv-python-headless", specifier = "==4.10.0.84" },
{ name = "opendal", specifier = ">=0.45.0,<0.46.0" },
{ name = "opensearch-py", specifier = "==2.7.1" },
- { name = "ormsgpack", specifier = "==1.5.0" },
+ { name = "ormsgpack", specifier = ">=1.5.0" },
{ name = "pdfplumber", specifier = "==0.10.4" },
{ name = "peewee", specifier = ">=3.17.1,<4.0.0" },
{ name = "pluginlib", specifier = ">=0.10.0" },
@@ -7011,7 +6313,6 @@ requires-dist = [
{ name = "socksio", specifier = "==1.0.0" },
{ name = "spacy", specifier = "==3.8.14" },
{ name = "sqlglotrs", specifier = "==0.9.0" },
- { name = "strenum", specifier = "==0.4.15" },
{ name = "tavily-python", specifier = "==0.5.1" },
{ name = "tencentcloud-sdk-python", specifier = "==3.0.1478" },
{ name = "tika", specifier = "==2.6.0" },
@@ -7045,7 +6346,6 @@ test = [
{ name = "reportlab", specifier = ">=4.4.1" },
{ name = "requests", specifier = ">=2.32.2" },
{ name = "requests-toolbelt", specifier = ">=1.0.0" },
- { name = "tensorflow-cpu", specifier = ">=2.17.0" },
]
[[package]]
@@ -7123,7 +6423,6 @@ source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "attrs" },
{ name = "rpds-py" },
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8" }
wheels = [
@@ -7136,22 +6435,6 @@ version = "2026.2.28"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/8b/71/41455aa99a5a5ac1eaf311f5d8efd9ce6433c03ac1e0962de163350d0d97/regex-2026.2.28.tar.gz", hash = "sha256:a729e47d418ea11d03469f321aaf67cdee8954cde3ff2cf8403ab87951ad10f2" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/07/42/9061b03cf0fc4b5fa2c3984cbbaed54324377e440a5c5a29d29a72518d62/regex-2026.2.28-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fcf26c3c6d0da98fada8ae4ef0aa1c3405a431c0a77eb17306d38a89b02adcd7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/77/83/0c8a5623a233015595e3da499c5a1c13720ac63c107897a6037bb97af248/regex-2026.2.28-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:02473c954af35dd2defeb07e44182f5705b30ea3f351a7cbffa9177beb14da5d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/06/3ef1ac6910dc3295ebd71b1f9bfa737e82cfead211a18b319d45f85ddd09/regex-2026.2.28-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9b65d33a17101569f86d9c5966a8b1d7fbf8afdda5a8aa219301b0a80f58cf7d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/dd/c9/8cc8d850b35ab5650ff6756a1cb85286e2000b66c97520b29c1587455344/regex-2026.2.28-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71dcecaa113eebcc96622c17692672c2d104b1d71ddf7adeda90da7ddeb26fc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e9/5d/57702597627fc23278ebf36fbb497ac91c0ce7fec89ac6c81e420ca3e38c/regex-2026.2.28-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:481df4623fa4969c8b11f3433ed7d5e3dc9cec0f008356c3212b3933fb77e3d8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/02/6d/f3ecad537ca2811b4d26b54ca848cf70e04fcfc138667c146a9f3157779c/regex-2026.2.28-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:64e7c6ad614573e0640f271e811a408d79a9e1fe62a46adb602f598df42a818d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/40/bb226f203caa22c1043c1ca79b36340156eca0f6a6742b46c3bb222a3a57/regex-2026.2.28-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6b08a06976ff4fb0d83077022fde3eca06c55432bb997d8c0495b9a4e9872f4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/44/7c/c6d91d8911ac6803b45ca968e8e500c46934e58c0903cbc6d760ee817a0a/regex-2026.2.28-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:864cdd1a2ef5716b0ab468af40139e62ede1b3a53386b375ec0786bb6783fc05" },
- { url = "https://mirrors.aliyun.com/pypi/packages/dc/8d/4a9368d168d47abd4158580b8c848709667b1cd293ff0c0c277279543bd0/regex-2026.2.28-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:511f7419f7afab475fd4d639d4aedfc54205bcb0800066753ef68a59f0f330b5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cc/bf/2c72ab5d8b7be462cb1651b5cc333da1d0068740342f350fcca3bca31947/regex-2026.2.28-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b42f7466e32bf15a961cf09f35fa6323cc72e64d3d2c990b10de1274a5da0a59" },
- { url = "https://mirrors.aliyun.com/pypi/packages/7c/f4/6b65c979bb6d09f51bb2d2a7bc85de73c01ec73335d7ddd202dcb8cd1c8f/regex-2026.2.28-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8710d61737b0c0ce6836b1da7109f20d495e49b3809f30e27e9560be67a257bf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8e/32/29ea5e27400ee86d2cc2b4e80aa059df04eaf78b4f0c18576ae077aeff68/regex-2026.2.28-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4390c365fd2d45278f45afd4673cb90f7285f5701607e3ad4274df08e36140ae" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1d/91/3233d03b5f865111cd517e1c95ee8b43e8b428d61fa73764a80c9bb6f537/regex-2026.2.28-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cb3b1db8ff6c7b8bf838ab05583ea15230cb2f678e569ab0e3a24d1e8320940b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/76/92/abc706c1fb03b4580a09645b206a3fc032f5a9f457bc1a8038ac555658ab/regex-2026.2.28-cp312-cp312-win32.whl", hash = "sha256:f8ed9a5d4612df9d4de15878f0bc6aa7a268afbe5af21a3fdd97fa19516e978c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fa/06/2a6f7dff190e5fa9df9fb4acf2fdf17a1aa0f7f54596cba8de608db56b3a/regex-2026.2.28-cp312-cp312-win_amd64.whl", hash = "sha256:01d65fd24206c8e1e97e2e31b286c59009636c022eb5d003f52760b0f42155d4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/f0/58a2484851fadf284458fdbd728f580d55c1abac059ae9f048c63b92f427/regex-2026.2.28-cp312-cp312-win_arm64.whl", hash = "sha256:c0b5ccbb8ffb433939d248707d4a8b31993cb76ab1a0187ca886bf50e96df952" },
{ url = "https://mirrors.aliyun.com/pypi/packages/87/f6/dc9ef48c61b79c8201585bf37fa70cd781977da86e466cd94e8e95d2443b/regex-2026.2.28-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6d63a07e5ec8ce7184452cb00c41c37b49e67dc4f73b2955b5b8e782ea970784" },
{ url = "https://mirrors.aliyun.com/pypi/packages/95/c8/c20390f2232d3f7956f420f4ef1852608ad57aa26c3dd78516cb9f3dc913/regex-2026.2.28-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e59bc8f30414d283ae8ee1617b13d8112e7135cb92830f0ec3688cb29152585a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d2/a6/ba1068a631ebd71a230e7d8013fcd284b7c89c35f46f34a7da02082141b1/regex-2026.2.28-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:de0cf053139f96219ccfabb4a8dd2d217c8c82cb206c91d9f109f3f552d6b43d" },
@@ -7341,21 +6624,6 @@ version = "0.30.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05" },
- { url = "https://mirrors.aliyun.com/pypi/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51" },
- { url = "https://mirrors.aliyun.com/pypi/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf" },
- { url = "https://mirrors.aliyun.com/pypi/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4" },
@@ -7472,16 +6740,6 @@ version = "0.2.15"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/ea/97/60fda20e2fb54b83a61ae14648b0817c8f5d84a3821e40bfbdae1437026a/ruamel_yaml_clib-0.2.15.tar.gz", hash = "sha256:46e4cc8c43ef6a94885f72512094e482114a8a706d3c555a34ed4b0d20200600" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/72/4b/5fde11a0722d676e469d3d6f78c6a17591b9c7e0072ca359801c4bd17eee/ruamel_yaml_clib-0.2.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb15a2e2a90c8475df45c0949793af1ff413acfb0a716b8b94e488ea95ce7cff" },
- { url = "https://mirrors.aliyun.com/pypi/packages/85/82/4d08ac65ecf0ef3b046421985e66301a242804eb9a62c93ca3437dc94ee0/ruamel_yaml_clib-0.2.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:64da03cbe93c1e91af133f5bec37fd24d0d4ba2418eaf970d7166b0a26a148a2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b9/cb/22366d68b280e281a932403b76da7a988108287adff2bfa5ce881200107a/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f6d3655e95a80325b84c4e14c080b2470fe4f33b6846f288379ce36154993fb1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/71/73/81230babf8c9e33770d43ed9056f603f6f5f9665aea4177a2c30ae48e3f3/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71845d377c7a47afc6592aacfea738cc8a7e876d586dfba814501d8c53c1ba60" },
- { url = "https://mirrors.aliyun.com/pypi/packages/61/62/150c841f24cda9e30f588ef396ed83f64cfdc13b92d2f925bb96df337ba9/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11e5499db1ccbc7f4b41f0565e4f799d863ea720e01d3e99fa0b7b5fcd7802c9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/30/93/e79bd9cbecc3267499d9ead919bd61f7ddf55d793fb5ef2b1d7d92444f35/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b293a37dc97e2b1e8a1aec62792d1e52027087c8eea4fc7b5abd2bdafdd6642" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8d/06/1eb640065c3a27ce92d76157f8efddb184bd484ed2639b712396a20d6dce/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:512571ad41bba04eac7268fe33f7f4742210ca26a81fe0c75357fa682636c690" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a5/21/ee353e882350beab65fcc47a91b6bdc512cace4358ee327af2962892ff16/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5e9f630c73a490b758bf14d859a39f375e6999aea5ddd2e2e9da89b9953486a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/57/34/cc1b94057aa867c963ecf9ea92ac59198ec2ee3a8d22a126af0b4d4be712/ruamel_yaml_clib-0.2.15-cp312-cp312-win32.whl", hash = "sha256:f4421ab780c37210a07d138e56dd4b51f8642187cdfb433eb687fe8c11de0144" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b3/e5/8925a4208f131b218f9a7e459c0d6fcac8324ae35da269cb437894576366/ruamel_yaml_clib-0.2.15-cp312-cp312-win_amd64.whl", hash = "sha256:2b216904750889133d9222b7b873c199d48ecbb12912aca78970f84a5aa1a4bc" },
{ url = "https://mirrors.aliyun.com/pypi/packages/17/5e/2f970ce4c573dc30c2f95825f2691c96d55560268ddc67603dc6ea2dd08e/ruamel_yaml_clib-0.2.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dcec721fddbb62e60c2801ba08c87010bd6b700054a09998c4d09c08147b8fb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d6/03/a1baa5b94f71383913f21b96172fb3a2eb5576a4637729adbf7cd9f797f8/ruamel_yaml_clib-0.2.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:65f48245279f9bb301d1276f9679b82e4c080a1ae25e679f682ac62446fac471" },
{ url = "https://mirrors.aliyun.com/pypi/packages/dc/19/40d676802390f85784235a05788fd28940923382e3f8b943d25febbb98b7/ruamel_yaml_clib-0.2.15-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:46895c17ead5e22bea5e576f1db7e41cb273e8d062c04a6a49013d9f60996c25" },
@@ -7551,12 +6809,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76" },
- { url = "https://mirrors.aliyun.com/pypi/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4" },
- { url = "https://mirrors.aliyun.com/pypi/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809" },
- { url = "https://mirrors.aliyun.com/pypi/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57" },
@@ -7592,16 +6844,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086" },
- { url = "https://mirrors.aliyun.com/pypi/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21" },
- { url = "https://mirrors.aliyun.com/pypi/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3" },
{ url = "https://mirrors.aliyun.com/pypi/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d" },
@@ -7724,14 +6966,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/24/c0/f3b6453cf2dfa99adc0ba6675f9aaff9e526d2224cbd7ff9c1a879238693/shapely-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe2533caae6a91a543dec62e8360fe86ffcdc42a7c55f9dfd0128a977a896b94" },
- { url = "https://mirrors.aliyun.com/pypi/packages/86/07/59dee0bc4b913b7ab59ab1086225baca5b8f19865e6101db9ebb7243e132/shapely-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ba4d1333cc0bc94381d6d4308d2e4e008e0bd128bdcff5573199742ee3634359" },
- { url = "https://mirrors.aliyun.com/pypi/packages/26/29/a5397e75b435b9895cd53e165083faed5d12fd9626eadec15a83a2411f0f/shapely-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0bd308103340030feef6c111d3eb98d50dc13feea33affc8a6f9fa549e9458a3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b9/37/e781683abac55dde9771e086b790e554811a71ed0b2b8a1e789b7430dd44/shapely-2.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e7d4d7ad262a48bb44277ca12c7c78cb1b0f56b32c10734ec9a1d30c0b0c54b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d8/f3/9876b64d4a5a321b9dc482c92bb6f061f2fa42131cba643c699f39317cb9/shapely-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9eddfe513096a71896441a7c37db72da0687b34752c4e193577a145c71736fc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d1/a0/704c7292f7014c7e74ec84eddb7b109e1fbae74a16deae9c1504b1d15565/shapely-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:980c777c612514c0cf99bc8a9de6d286f5e186dcaf9091252fcd444e5638193d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/53/46/319c9dc788884ad0785242543cdffac0e6530e4d0deb6c4862bc4143dcf3/shapely-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9111274b88e4d7b54a95218e243282709b330ef52b7b86bc6aaf4f805306f454" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ec/bf/cb6c1c505cb31e818e900b9312d514f381fbfa5c4363edfce0fcc4f8c1a4/shapely-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:743044b4cfb34f9a67205cee9279feaf60ba7d02e69febc2afc609047cb49179" },
{ url = "https://mirrors.aliyun.com/pypi/packages/c3/90/98ef257c23c46425dc4d1d31005ad7c8d649fe423a38b917db02c30f1f5a/shapely-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b510dda1a3672d6879beb319bc7c5fd302c6c354584690973c838f46ec3e0fa8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/6d/ab/0bee5a830d209adcd3a01f2d4b70e587cdd9fd7380d5198c064091005af8/shapely-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8cff473e81017594d20ec55d86b54bc635544897e13a7cfc12e36909c5309a2a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/2d/5e/7d7f54ba960c13302584c73704d8c4d15404a51024631adb60b126a4ae88/shapely-2.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe7b77dc63d707c09726b7908f575fc04ff1d1ad0f3fb92aec212396bc6cfe5e" },
@@ -7876,14 +7110,6 @@ dependencies = [
{ name = "weasel" },
]
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/0c/78/e4f2ae19a791cae756cd0e801204953eaec4e9ab75a60ad39f671dbb8d5a/spacy-3.8.14-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:726f02c60a2c6b0029167370d22d51731172a053d29c7e2ea6190db6de3ab483" },
- { url = "https://mirrors.aliyun.com/pypi/packages/06/df/178bbab47fa209c8baf2f1e609cbddc6b18a985200be1ceee22bd5b89beb/spacy-3.8.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e3ebe50b93f2d40e8ec3451255528bb622ccb12be39fd140bb87668ce8d1075b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ff/e8/048d83b73b28686307bd9a60878a58de7b7b21b562ca4de8b5bd558031e9/spacy-3.8.14-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:daeb64b048f12c059997281aed53eb8776d26416dd313cf17ad6f63124b2b564" },
- { url = "https://mirrors.aliyun.com/pypi/packages/8e/3f/1799af5f4ccc8eb7500e4a20ca301488134429dba08cda5be68ce6ab2992/spacy-3.8.14-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d45715a24446f23b98ec3f09409a1d4111983d1d64613250ee38c3270e21853" },
- { url = "https://mirrors.aliyun.com/pypi/packages/78/07/81ab9acd0ec64bfdd7339acfc4cf35f5fb74bbbb0b2be7e64d717c416bac/spacy-3.8.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1069a8be34940809f8462eb69f09a3f0ce59bf8b9cb82475f2a8e3580f50ece0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/74/a5/b081b5bd3cedb2634c23eb470b5e24c65c894c57646567f47627291c2b3f/spacy-3.8.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2dfa77aec7fdebac0455d8afd4ce1d92d6f868b03d507ed1976179a63db7b374" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5f/55/4371413a6dfc1fa837282a365498165f828c2f3fe018dfb35336acc869e0/spacy-3.8.14-cp312-cp312-win_amd64.whl", hash = "sha256:9def18c76a4472b326cb91a195623c9ca38a2b86999ad2df9e00b49ba8c63734" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f3/5e/12ac876017da6c1e6b72afcc3c8b309996227fd3aa15382cd3311aee21b8/spacy-3.8.14-cp312-cp312-win_arm64.whl", hash = "sha256:d6257133357e4801c9c5d011925af5439b0a015aacf3c16528aa0009982431c7" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1b/e5/822bbdfa459fee863ef2e9879a34b0ae5db7cd1e3eb76d32c766f19222e9/spacy-3.8.14-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b4f60fa8b9641a5e93e7a96db0cdd106d05d61756bf1d0ddcd1705ad347909a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/7e/de/0e512154113e1f341567f2b9341835775e4180c180221e60faedaebb2f65/spacy-3.8.14-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0860c57220c633ccb20468bcd64bfb0d28908990c371a8857951d093a148dc8e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0c/4f/29c7e56afc7db07348a9e0efe0243b5eef465d5dc3d56433f164378c3fa6/spacy-3.8.14-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c24620b7dba879c69cebc51ef3b1107d4d4e44a1e0d4baa439372887d00c3fd9" },
@@ -8029,13 +7255,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/1f/73/b4a9737255583b5fa858e0bb8e116eb94b88c910164ed2ed719147bde3de/sqlalchemy-2.0.48.tar.gz", hash = "sha256:5ca74f37f3369b45e1f6b7b06afb182af1fd5dde009e4ffd831830d98cbe5fe7" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ef/91/a42ae716f8925e9659df2da21ba941f158686856107a61cc97a95e7647a3/sqlalchemy-2.0.48-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:348174f228b99f33ca1f773e85510e08927620caa59ffe7803b37170df30332b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b9/52/f75f516a1f3888f027c1cfb5d22d4376f4b46236f2e8669dcb0cddc60275/sqlalchemy-2.0.48-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53667b5f668991e279d21f94ccfa6e45b4e3f4500e7591ae59a8012d0f010dcb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/37/9a/0c28b6371e0cdcb14f8f1930778cb3123acfcbd2c95bb9cf6b4a2ba0cce3/sqlalchemy-2.0.48-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34634e196f620c7a61d18d5cf7dc841ca6daa7961aed75d532b7e58b309ac894" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1c/46/0aee8f3ff20b1dcbceb46ca2d87fcc3d48b407925a383ff668218509d132/sqlalchemy-2.0.48-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:546572a1793cc35857a2ffa1fe0e58571af1779bcc1ffa7c9fb0839885ed69a9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ce/8c/a957bc91293b49181350bfd55e6dfc6e30b7f7d83dc6792d72043274a390/sqlalchemy-2.0.48-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:07edba08061bc277bfdc772dd2a1a43978f5a45994dd3ede26391b405c15221e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4b/44/1d257d9f9556661e7bdc83667cc414ba210acfc110c82938cb3611eea58f/sqlalchemy-2.0.48-cp312-cp312-win32.whl", hash = "sha256:908a3fa6908716f803b86896a09a2c4dde5f5ce2bb07aacc71ffebb57986ce99" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f2/af/c3c7e1f3a2b383155a16454df62ae8c62a30dd238e42e68c24cebebbfae6/sqlalchemy-2.0.48-cp312-cp312-win_amd64.whl", hash = "sha256:68549c403f79a8e25984376480959975212a670405e3913830614432b5daa07a" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d1/c6/569dc8bf3cd375abc5907e82235923e986799f301cd79a903f784b996fca/sqlalchemy-2.0.48-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e3070c03701037aa418b55d36532ecb8f8446ed0135acb71c678dbdf12f5b6e4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/6d/ff/f4e04a4bd5a24304f38cb0d4aa2ad4c0fb34999f8b884c656535e1b2b74c/sqlalchemy-2.0.48-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2645b7d8a738763b664a12a1542c89c940daa55196e8d73e55b169cc5c99f65f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fe/88/cb59509e4668d8001818d7355d9995be90c321313078c912420603a7cb95/sqlalchemy-2.0.48-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b19151e76620a412c2ac1c6f977ab1b9fa7ad43140178345136456d5265b32ed" },
@@ -8085,16 +7304,6 @@ version = "0.9.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/e0/c1/de7ee4729d49d15339717d6c4cc9aac06382c1161a8212dfdd266d51ffe5/sqlglotrs-0.9.0.tar.gz", hash = "sha256:72f61561d63607a8d88f5da608c11e21b2a57773ca631e6b89a4eed668da2db5" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/28/a2/c898fe0dffea8ea988fdd7a15bdb414488eca2f9c7def679bf69c490a0f6/sqlglotrs-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1ae7b3b1fedd7b99f6a2c7d7ad1f2b23e433d69ed6e2a5ededa26fc9d74da626" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a8/17/344e5e600b61d177a7e535f078f04466097666120059a4a016d21fa1290c/sqlglotrs-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:938723a4ee7647f2a858ac581ac6cbbfe40320b843f9826f6b0d204579781466" },
- { url = "https://mirrors.aliyun.com/pypi/packages/da/0f/39d33a403416dc608c0dba31f1b8be5c6476ab7795043e73be4350974adf/sqlglotrs-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:816cdd9b5838c4df5c5206180508a87e6f2ef1860f9bc4655c8125257ef51484" },
- { url = "https://mirrors.aliyun.com/pypi/packages/39/c9/9971b2dd27c9781bec09c5c29676bf0c70cbf0345f1bc4c2315c1fcf68ab/sqlglotrs-0.9.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:080d58c906673c8905965af640cab16203b1e991f8f52a468c371e5f75b1ea04" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bb/8b/3f61abd5844b65cab7085e4c9af3af0e01f7a21e9786125498d901a87a40/sqlglotrs-0.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5241de862190e0c01830833d42bc58a479821d8bd07c51f1e74b5bddc0eb51b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ea/58/bd10f0ebd55f4d043922792dc1eb4b55ecbe9be323e749cd40586d3d6b0f/sqlglotrs-0.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:830198b4de0409e07fa82d2d515cb3b6f8e9627a966aacceb2c538e2bd4d2ceb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/60/34/7d2972e0c41747296b1ff29a671eac7ae6584cd1e29c012edbc4082b7ca7/sqlglotrs-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61011f8b28cb4b23abcc780c6a622aacd6b7acc546363c24501891e29a1950c7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/06/ce/37cf36d3765ecea1e5d22b1f107a3022ae5032bf319f805f3b918abdddeb/sqlglotrs-0.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:78eed1e668109ebc61771c0163bf9ff2d8073eea24034ba012edf71ba0759bf0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b6/a6/faea946e386e29f066a476cbcadc091369ac356f9b24b3e2c7e539d8800b/sqlglotrs-0.9.0-cp312-cp312-win32.whl", hash = "sha256:136a5001e43401b81b678e6f3433edc317cba08af3e7098e0228deef87f23562" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2b/e2/9264dd3b2a4369fbcb7b911f5ddaa0bed73ab5ae2d910b4fa14b0f56879e/sqlglotrs-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1c54ed249f16676fe8270738c8f05f08b1516d8b2975387b45bd67aa6f3b3a5" },
{ url = "https://mirrors.aliyun.com/pypi/packages/f3/27/6d42c98f2f33fc6dbbc7d669bf99ea6f7898d8bcd0aaf87aa1a4c96cc9c9/sqlglotrs-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e2a5a697dbfc9cfee5434433a4d698a26df94277e0916bbfc25e1e72436cd0c0" },
{ url = "https://mirrors.aliyun.com/pypi/packages/50/53/d1f8f42ec14d69d8ba249036d83dcb4d6b51fe5b3ddb357499c737ae2a99/sqlglotrs-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3a22d4064e923bbe07750f6e4b4b338e5b9fa0cbc2073bd503cc4b1c9280c2ac" },
{ url = "https://mirrors.aliyun.com/pypi/packages/52/e0/a2aa5e533427af4b64f9a630000cfee3cbbf877f58dcd79bb931963adf8a/sqlglotrs-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fbf6f211d4b0d091855984279be7a9d57b89a43db07aeaf6cabee075c08ac80" },
@@ -8126,14 +7335,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/2b/db/f794f219a6c788b881252d2536a8c4a97d2bdaadc690391e1cb53d123d71/srsly-2.5.3.tar.gz", hash = "sha256:08f98dbecbff3a31466c4ae7c833131f59d3655a0ad8ac749e6e2c149e2b0680" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/02/cc/e9f7fcec4cc92ad8bad6316c4241638b8cf7380382d4489d94ec6c436452/srsly-2.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:71e51c046ccbeefb86524c6b1e17574f579c6ac4dc8ea4a09437d3e8f88342d3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/21/e4/fea4512e9785f58509b2cf67d993323848e583161b5fcfdc7dd9d7c1f3df/srsly-2.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f73c0db911552e94fe2016e1759d261d2f47926f68826664cada3723c87006a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/20/b1/53591681b6ff2699a4f97b2d5552ba196eaa6a979b0873605f4c04b5f7ee/srsly-2.5.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c1ac27ae5f4bb9163c7d2c45fc8ec173aac3d92e32086d9472b326c5c6e570e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4e/c9/741e29f534919a944a16da4184924b1d3404c4bf60716ab2b91be771d1e3/srsly-2.5.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:99026bcd9cbd3211cc36517400b04ca0fc5d3e412b14daf84ee6e65f67d9a2d8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/89/57/5554f786eccf78b2750d6ac63be126e1b67badec2cb409dd611cf6f8c52b/srsly-2.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:07d682679e639eb46ff7e6da4a92714f4d5ffe351d088ee66f221e9b1f8865bb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/eb/95/9b4f73b1be3692f86d72ccc131c8e50f26f824d5c8830a59390bcc5b60ef/srsly-2.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8e0542d85d6b55cf2934050d6ffcb1cd76c768dcf9572e7467002cf087bb366d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5a/de/89ca640ca1953c4612279ce515d0af35658df3c06cdb324329bc91b4a7e1/srsly-2.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:598f1e494c18cacb978299d77125415a586417081959f8ec3f068b32d97f8933" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6d/4f/7ab6d49e36d9cc72ee15746cabd116eb6f338be8a06c1882968ee9d6c7d7/srsly-2.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:4b1b721cd3ad1a9b2343519aadc786a4d09d5c0666962d49852eb12d6ec3fe26" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9d/5c/12901e3794f4158abc6da750725aad6c2afddb1e4227b300fe7c71f66957/srsly-2.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e67b6bbacbfadea5e100266d2797f2d4cec9883ea4dc84a5537673850036a8d8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/04/61/181c26370995f96f56f1b64b801e3ca1e0d703fc36506ae28606d62369fb/srsly-2.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:348c231b4477d8fe86603131d0f166d2feac9c372704dfc4398be71cc5b6fb07" },
{ url = "https://mirrors.aliyun.com/pypi/packages/77/c6/35876c78889f8ffe11ed3521644e666c3aef20ea31527b70f47456cf35c2/srsly-2.5.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b0938c2978c91ae1ef9c1f2ba35abb86330e198fb23469e356eba311e02233ee" },
@@ -8179,7 +7380,6 @@ version = "1.0.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "anyio" },
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149" }
wheels = [
@@ -8199,12 +7399,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/25/ce/308e5e5da57515dd7cab3ec37ea2d5b8ff50bef1fcc8e6d31456f9fae08e/statsmodels-0.14.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe76140ae7adc5ff0e60a3f0d56f4fffef484efa803c3efebf2fcd734d72ecb5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/05/30/affbabf3c27fb501ec7b5808230c619d4d1a4525c07301074eb4bda92fa9/statsmodels-0.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26d4f0ed3b31f3c86f83a92f5c1f5cbe63fc992cd8915daf28ca49be14463a1c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/48/f5/3a73b51e6450c31652c53a8e12e24eac64e3824be816c0c2316e7dbdcb7d/statsmodels-0.14.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8c00a42863e4f4733ac9d078bbfad816249c01451740e6f5053ecc7db6d6368" },
- { url = "https://mirrors.aliyun.com/pypi/packages/81/68/dddd76117df2ef14c943c6bbb6618be5c9401280046f4ddfc9fb4596a1b8/statsmodels-0.14.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b58cf7474aa9e7e3b0771a66537148b2df9b5884fbf156096c0e6c1ff0469d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/56/4a/dce451c74c4050535fac1ec0c14b80706d8fc134c9da22db3c8a0ec62c33/statsmodels-0.14.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:81e7dcc5e9587f2567e52deaff5220b175bf2f648951549eae5fc9383b62bc37" },
- { url = "https://mirrors.aliyun.com/pypi/packages/60/15/3daba2df40be8b8a9a027d7f54c8dedf24f0d81b96e54b52293f5f7e3418/statsmodels-0.14.6-cp312-cp312-win_amd64.whl", hash = "sha256:b5eb07acd115aa6208b4058211138393a7e6c2cf12b6f213ede10f658f6a714f" },
{ url = "https://mirrors.aliyun.com/pypi/packages/81/59/a5aad5b0cc266f5be013db8cde563ac5d2a025e7efc0c328d83b50c72992/statsmodels-0.14.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:47ee7af083623d2091954fa71c7549b8443168f41b7c5dce66510274c50fd73e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/53/dd/d8cfa7922fc6dc3c56fa6c59b348ea7de829a94cd73208c6f8202dd33f17/statsmodels-0.14.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa60d82e29fcd0a736e86feb63a11d2380322d77a9369a54be8b0965a3985f71" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ee/77/0ec96803eba444efd75dba32f2ef88765ae3e8f567d276805391ec2c98c6/statsmodels-0.14.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ee7d595f5939cc20bf946faedcb5137d975f03ae080f300ebb4398f16a5bd4" },
@@ -8232,15 +7426,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/5c/92/d0c83f63d3518e5f0b8a311937c31347349ec9a47b209ddc17f7566f58fc/stone-3.3.1-py3-none-any.whl", hash = "sha256:e15866fad249c11a963cce3bdbed37758f2e88c8ff4898616bc0caeb1e216047" },
]
-[[package]]
-name = "strenum"
-version = "0.4.15"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/85/ad/430fb60d90e1d112a62ff57bdd1f286ec73a2a0331272febfddd21f330e1/StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659" },
-]
-
[[package]]
name = "sympy"
version = "1.14.0"
@@ -8343,150 +7528,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/c5/db/daa85799b9af2aa50539b27eeb0d6a2a0ac35465f62683107847830dbe4d/tencentcloud_sdk_python-3.0.1478-py2.py3-none-any.whl", hash = "sha256:10ddee1c1348f49e2b54af606f978d4cb17fca656639e8d99b6527e6e4793833" },
]
-[[package]]
-name = "tensorboard"
-version = "2.18.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "absl-py" },
- { name = "grpcio" },
- { name = "markdown" },
- { name = "numpy" },
- { name = "packaging" },
- { name = "protobuf" },
- { name = "setuptools" },
- { name = "six" },
- { name = "tensorboard-data-server" },
- { name = "werkzeug" },
-]
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/b1/de/021c1d407befb505791764ad2cbd56ceaaa53a746baed01d2e2143f05f18/tensorboard-2.18.0-py3-none-any.whl", hash = "sha256:107ca4821745f73e2aefa02c50ff70a9b694f39f790b11e6f682f7d326745eab" },
-]
-
-[[package]]
-name = "tensorboard-data-server"
-version = "0.7.2"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60" },
- { url = "https://mirrors.aliyun.com/pypi/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530" },
-]
-
-[[package]]
-name = "tensorflow-cpu"
-version = "2.18.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-resolution-markers = [
- "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')",
- "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux')",
- "(python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')",
-]
-dependencies = [
- { name = "absl-py", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "astunparse", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "flatbuffers", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "gast", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "google-pasta", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "grpcio", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "h5py", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "keras", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "libclang", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "ml-dtypes", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "opt-einsum", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "packaging", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "protobuf", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "requests", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "six", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "tensorboard", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "tensorflow-intel", marker = "sys_platform == 'win32'" },
- { name = "termcolor", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "wrapt", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
-]
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ca/3f/2ed163140237aefa72c761d56af8ba3fa5cb0fe37a9f53b14ad8bcd7ef87/tensorflow_cpu-2.18.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39bd421ad125e4163d6e2d41ab0e158b583fb5c6f9254522fb87635b0e70b891" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0e/7a/1c99bb2bb7d24238b748f9f0244a198ee15d23782bb56dbf4e7b93a29c6a/tensorflow_cpu-2.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:0b093b727c2f2a8cf4ee4f2c7352c8e958a2a1d27a452961b8d5f43a0798dcd2" },
-]
-
-[[package]]
-name = "tensorflow-cpu"
-version = "2.18.1"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-resolution-markers = [
- "python_full_version >= '3.14' and sys_platform == 'darwin'",
- "python_full_version == '3.13.*' and sys_platform == 'darwin'",
- "python_full_version < '3.13' and sys_platform == 'darwin'",
- "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'",
- "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
- "python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'",
-]
-dependencies = [
- { name = "absl-py", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "astunparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "flatbuffers", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "gast", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "google-pasta", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "grpcio", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "h5py", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "keras", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "libclang", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "ml-dtypes", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "numpy", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "opt-einsum", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "packaging", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "protobuf", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "requests", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "setuptools", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "six", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "tensorboard", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "termcolor", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "typing-extensions", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
- { name = "wrapt", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or sys_platform == 'darwin'" },
-]
-
-[[package]]
-name = "tensorflow-intel"
-version = "2.18.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "absl-py", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "astunparse", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "flatbuffers", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "gast", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "google-pasta", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "grpcio", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "h5py", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "keras", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "libclang", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "ml-dtypes", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "opt-einsum", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "packaging", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "protobuf", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "requests", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "six", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "tensorboard", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "termcolor", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
- { name = "wrapt", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
-]
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/ae/4e/44ce609139065035c56fe570fe7f0ee8d06180c99a424bac588472052c5d/tensorflow_intel-2.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:a5818043f565cf74179b67eb52fc060587ccecb9540141c39d84fbcb37ecff8c" },
-]
-
-[[package]]
-name = "termcolor"
-version = "3.3.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5" },
-]
-
[[package]]
name = "text-unidecode"
version = "1.3"
@@ -8529,14 +7570,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/13/46/76df95f2c327f9a9cef30c1523bf285627897097163584dcf5f77b2ebce2/thinc-8.3.13.tar.gz", hash = "sha256:68e658549fc1eb3ff92aed5147fcbb9c15d6e9cc0e623b4d0998d16522ffb4f9" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/3e/af/f7c1ebfe92eb5d27d7f2f3da67a11e2eb57bc30ab1553279af6dc65b65a8/thinc-8.3.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:77a41f66285321d20aaedaea1e87d7cd48dca6d2427bed1867ec7cba7109fc8d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/45/8f/69d7338575d98df85d0b54c0f5fc277dba72587fe9ab846ecdd12a998bcb/thinc-8.3.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3710d318b4e5460cf366a6f7b5ddbefb5d39dbd4cfa408222750fdc6c27c4411" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4b/a5/21d010c81e81e1589e5ccb4950e521804d13726e541e87f644c51815673b/thinc-8.3.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a08c87143a6d20177652dca1ec0dc815d88216d8fc62594a57e8bc45bf5ed49" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f9/ff/6914bf370bd1d604d89e6dfb46b97d10cd9b00d42ff8c036283e92314a8c/thinc-8.3.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b5ec9ff313819e7d8667794a3559463fa89ff45aaa73e3fd8d6273b1e0d7a7f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f3/3d/5572b47fa155fb3388c071515b74024fa17a6efd1df9406da378f0aa84ef/thinc-8.3.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5c9a48f2bc1e04f138240ed5f9b815a9141a5de26accd0f08fa0137fcefed258" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f0/f0/a8d77c7bac089697c6df302cc3c936a1ab36a4720deae889e6f1dbcbd0eb/thinc-8.3.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:79a29a44d76bd02f5ac0624268c6e42b3576ae472c791a8ae9c2d813ae789b59" },
- { url = "https://mirrors.aliyun.com/pypi/packages/21/82/5651bb1f904d04220fc7670035ada921bf0638e2cff6444d67c12887a968/thinc-8.3.13-cp312-cp312-win_amd64.whl", hash = "sha256:ed1dc709ac4f2f03b710457889e4e02f05de51bc8456980c241d0b28798bc7cb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/94/8d/683703de021ffbe46833d722b70f49ffbbca8e5bd6876256977555d92d7d/thinc-8.3.13-cp312-cp312-win_arm64.whl", hash = "sha256:c6a049703a6011c8fe26ee41af7e70272145594140d82f79bb23de619c6a6525" },
{ url = "https://mirrors.aliyun.com/pypi/packages/af/b9/7b46942176df459d1804a9e77b0976f7c56f3abf3ec7485d0e5f836a0382/thinc-8.3.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c2811dfd8d46d8b5d3b39051b23e64006b2994a5143b1978b436938018792af8" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a7/79/53085a72cd8f4fc4e6e313d05ea5aa98e870684f4a0fb318a9875fc0a964/thinc-8.3.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5593e6300cb1ebe0c0e546e9c9fb49e7c2627a0aa688795cd4f995a8b820d2ec" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9e/3e/d61b462b16da95ac6885f95bb395e672040ee594833e571a6edcffd234f5/thinc-8.3.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f697174d3fb474966ce50b430bbafa101a6d2f7ffb559dac4b5c59389ef72d22" },
@@ -8590,13 +7623,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad" },
- { url = "https://mirrors.aliyun.com/pypi/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5" },
- { url = "https://mirrors.aliyun.com/pypi/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3" },
- { url = "https://mirrors.aliyun.com/pypi/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd" },
{ url = "https://mirrors.aliyun.com/pypi/packages/00/61/441588ee21e6b5cdf59d6870f86beb9789e532ee9718c251b391b70c68d6/tiktoken-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3" },
{ url = "https://mirrors.aliyun.com/pypi/packages/1f/05/dcf94486d5c5c8d34496abe271ac76c5b785507c8eae71b3708f1ad9b45a/tiktoken-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160" },
{ url = "https://mirrors.aliyun.com/pypi/packages/a0/70/5163fe5359b943f8db9946b62f19be2305de8c3d78a16f629d4165e2f40e/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa" },
@@ -8692,8 +7718,8 @@ wheels = [
[[package]]
name = "trio"
-version = "0.24.0"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
+version = "0.33.0"
+source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "attrs" },
{ name = "cffi", marker = "(implementation_name != 'pypy' and os_name == 'nt' and platform_machine != 'aarch64' and sys_platform == 'linux') or (implementation_name != 'pypy' and os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" },
@@ -8702,9 +7728,9 @@ dependencies = [
{ name = "sniffio" },
{ name = "sortedcontainers" },
]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/8a/f3/07c152213222c615fe2391b8e1fea0f5af83599219050a549c20fcbd9ba2/trio-0.24.0.tar.gz", hash = "sha256:ffa09a74a6bf81b84f8613909fb0beaee84757450183a7a2e0b47b455c0cac5d" }
+sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/14/fb/9299cf74953f473a15accfdbe2c15218e766bae8c796f2567c83bae03e98/trio-0.24.0-py3-none-any.whl", hash = "sha256:c3bd3a4e3e3025cd9a2241eae75637c43fe0b9e88b4c97b9161a55b9e54cd72c" },
+ { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" },
]
[[package]]
@@ -8999,15 +8025,6 @@ version = "16.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79" },
- { url = "https://mirrors.aliyun.com/pypi/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39" },
- { url = "https://mirrors.aliyun.com/pypi/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1" },
- { url = "https://mirrors.aliyun.com/pypi/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89" },
- { url = "https://mirrors.aliyun.com/pypi/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea" },
{ url = "https://mirrors.aliyun.com/pypi/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230" },
{ url = "https://mirrors.aliyun.com/pypi/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c" },
@@ -9050,18 +8067,6 @@ wheels = [
{ url = "https://mirrors.aliyun.com/pypi/packages/7f/b2/0bba9bbb4596d2d2f285a16c2ab04118f6b957d8441566e1abb892e6a6b2/werkzeug-3.1.7-py3-none-any.whl", hash = "sha256:4b314d81163a3e1a169b6a0be2a000a0e204e8873c5de6586f453c55688d422f" },
]
-[[package]]
-name = "wheel"
-version = "0.46.3"
-source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
-dependencies = [
- { name = "packaging" },
-]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/89/24/a2eb353a6edac9a0303977c4cb048134959dd2a51b48a269dfc9dde00c8a/wheel-0.46.3.tar.gz", hash = "sha256:e3e79874b07d776c40bd6033f8ddf76a7dad46a7b8aa1b2787a83083519a1803" }
-wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/87/22/b76d483683216dde3d67cba61fb2444be8d5be289bf628c13fc0fd90e5f9/wheel-0.46.3-py3-none-any.whl", hash = "sha256:4b399d56c9d9338230118d705d9737a2a468ccca63d5e813e2a4fc7815d8bc4d" },
-]
-
[[package]]
name = "wikipedia"
version = "1.4.0"
@@ -9090,16 +8095,6 @@ version = "1.17.3"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba" },
- { url = "https://mirrors.aliyun.com/pypi/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396" },
- { url = "https://mirrors.aliyun.com/pypi/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe" },
- { url = "https://mirrors.aliyun.com/pypi/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6" },
{ url = "https://mirrors.aliyun.com/pypi/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0" },
{ url = "https://mirrors.aliyun.com/pypi/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77" },
{ url = "https://mirrors.aliyun.com/pypi/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7" },
@@ -9195,21 +8190,6 @@ version = "3.6.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490" },
- { url = "https://mirrors.aliyun.com/pypi/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0" },
- { url = "https://mirrors.aliyun.com/pypi/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae" },
- { url = "https://mirrors.aliyun.com/pypi/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c" },
- { url = "https://mirrors.aliyun.com/pypi/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829" },
{ url = "https://mirrors.aliyun.com/pypi/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec" },
{ url = "https://mirrors.aliyun.com/pypi/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6" },
@@ -9283,24 +8263,6 @@ dependencies = [
]
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069" },
- { url = "https://mirrors.aliyun.com/pypi/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25" },
- { url = "https://mirrors.aliyun.com/pypi/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072" },
- { url = "https://mirrors.aliyun.com/pypi/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51" },
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67" },
- { url = "https://mirrors.aliyun.com/pypi/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760" },
- { url = "https://mirrors.aliyun.com/pypi/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86" },
- { url = "https://mirrors.aliyun.com/pypi/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34" },
- { url = "https://mirrors.aliyun.com/pypi/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d" },
- { url = "https://mirrors.aliyun.com/pypi/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/9a/4b/a0a6e5d0ee8a2f3a373ddef8a4097d74ac901ac363eea1440464ccbe0898/yarl-1.23.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e" },
{ url = "https://mirrors.aliyun.com/pypi/packages/67/b6/8925d68af039b835ae876db5838e82e76ec87b9782ecc97e192b809c4831/yarl-1.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5" },
{ url = "https://mirrors.aliyun.com/pypi/packages/ae/50/06d511cc4b8e0360d3c94af051a768e84b755c5eb031b12adaaab6dec6e5/yarl-1.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b" },
@@ -9429,8 +8391,6 @@ version = "0.1.10"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/35/3e/dd482d5bf99d1dabcce0a20a479859cb7a6bd8a365b07b41ebf46b3c0f3d/zlib_state-0.1.10.tar.gz", hash = "sha256:c29b6b93cea1b80025fbc96fa91ceed8b5e7b54ef08f16d6e4c7f8fb56aad777" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/9e/c9/318a8fa73d41b94810816815e38372d75a8c83c02c9d10dd796443b74ccd/zlib_state-0.1.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d4f3196f84a4d504f4c04147ec7fd9132651883830f6f07be3702d82731f99e" },
- { url = "https://mirrors.aliyun.com/pypi/packages/38/d8/89a7e7fbea33b20dcdefa122afde7e79a9fdbe75cf5b48e13a110a2c8c8e/zlib_state-0.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:8465b3ddb7fc11e30a49f38615426e369dd1ac5d3d780d89e759e731dfc7bbf4" },
{ url = "https://mirrors.aliyun.com/pypi/packages/70/0c/2b0803cb9f30bddbc9eda87d251d958d21cfdde826bc1deb1e19ca0ff320/zlib_state-0.1.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dfecba070cdeeab073573ac721459727d60e0b8ef7b38dac3c965459781b0eeb" },
{ url = "https://mirrors.aliyun.com/pypi/packages/b1/d2/74ff59bb480801eae2731523f98be198eec135a9d37e27791b635f2c9124/zlib_state-0.1.10-cp313-cp313-win_amd64.whl", hash = "sha256:72e354f09c942055677ba59d76ca8c311a8129dfc98c3b44db33302843090204" },
{ url = "https://mirrors.aliyun.com/pypi/packages/e1/b2/83cfa28037f152d623c1cf716013e5938513d414e8ac3c0312e1b839928f/zlib_state-0.1.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c86d39c50e046547e23d2f0170556444f1f385c251ce0d5cc00c9d7ed6c0ef1e" },
@@ -9443,23 +8403,6 @@ version = "0.25.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00" },
- { url = "https://mirrors.aliyun.com/pypi/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea" },
- { url = "https://mirrors.aliyun.com/pypi/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb" },
- { url = "https://mirrors.aliyun.com/pypi/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a" },
- { url = "https://mirrors.aliyun.com/pypi/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902" },
- { url = "https://mirrors.aliyun.com/pypi/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f" },
- { url = "https://mirrors.aliyun.com/pypi/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b" },
- { url = "https://mirrors.aliyun.com/pypi/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6" },
- { url = "https://mirrors.aliyun.com/pypi/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91" },
- { url = "https://mirrors.aliyun.com/pypi/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708" },
- { url = "https://mirrors.aliyun.com/pypi/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512" },
- { url = "https://mirrors.aliyun.com/pypi/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd" },
- { url = "https://mirrors.aliyun.com/pypi/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01" },
- { url = "https://mirrors.aliyun.com/pypi/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9" },
{ url = "https://mirrors.aliyun.com/pypi/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94" },
{ url = "https://mirrors.aliyun.com/pypi/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1" },
{ url = "https://mirrors.aliyun.com/pypi/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f" },
From bf41d35729872088273b5b1a8b6d8f9822cb5fd9 Mon Sep 17 00:00:00 2001
From: Haruko386
Date: Fri, 15 May 2026 18:41:43 +0800
Subject: [PATCH 166/666] Go: implement PaddleOCR provider and implement ASR
for CoHere (#14954)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
This PR implement implement OCR for Baidu and Mistral, implement
PaddleOCR provider and implement ASR for CoHere
**Verified examples from the CLI:**
```
RAGFlow(user)> ocr with 'mistral-ocr-2512@test@mistral' file './internal/text.jpg'
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| text |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Parallel to these organizational innovations there were significant complementary technical innovations (e.g., improved methods of manufacturing cast-iron pipe and of coating interiors for pressure maintenance, and newer paving and construction material... |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
RAGFlow(user)> ocr with 'paddleocr-vl-0.9b@test@baidu' file './internal/text.jpg'
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| text |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Parallel to these organizational innovations there were significant complementary technical innovations (e.g., improved methods of manufacturing cast-iron pipe and of coating interiors for pressure maintenance, and newer paving and construction material... |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
# PaddleOCR
RAGFlow(user)> ocr with 'PaddleOCR-VL-1.5@test@paddleocr' file './internal/test.pdf'
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| text |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| # Repurposing Diffusion-Based Image Generators for Monocular Depth Estimation
Bingxin Ke
Nando Metzger
Photogra
Anton Obukhov
Rodrigo Caye Daudt
netry and Remote Sensing,
Shengyu Huang
Konrad Schindler
ETH Zürich
asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' audio './
+----------------------------------------------------------------------------------------------------------------------+
| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. |
+----------------------------------------------------------------------------------------------------------------------+
+```
+
+### 6.25 Optical Character Recognition\
+```
+RAGFlow(user)> ocr with 'paddleocr-vl-0.9b@test@baidu' file './internal/text.jpg'
++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| text |
++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| Parallel to these organizational innovations there were significant complementary technical innovations (e.g., improved methods of manufacturing cast-iron pipe and of coating interiors for pressure maintenance, and newer paving and construction material... |
++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
```
\ No newline at end of file
diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.go
index 12ee525ca05..79a01b02f70 100644
--- a/internal/entity/models/aliyun.go
+++ b/internal/entity/models/aliyun.go
@@ -564,8 +564,8 @@ func (z *AliyunModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *AliyunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *AliyunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/baichuan.go b/internal/entity/models/baichuan.go
index 98c8f6752fe..d139b8a672a 100644
--- a/internal/entity/models/baichuan.go
+++ b/internal/entity/models/baichuan.go
@@ -389,8 +389,8 @@ func (z *BaichuanModel) TranscribeAudioWithSender(modelName *string, file *strin
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *BaichuanModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *BaichuanModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/baidu.go b/internal/entity/models/baidu.go
index 4b99ec7221d..b2bcf84393c 100644
--- a/internal/entity/models/baidu.go
+++ b/internal/entity/models/baidu.go
@@ -3,6 +3,8 @@ package models
import (
"bufio"
"bytes"
+ "context"
+ "encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -611,8 +613,8 @@ func (z *BaiduModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (b *BaiduModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (b *BaiduModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", b.Name())
}
@@ -621,13 +623,99 @@ func (z *BaiduModel) AudioSpeechWithSender(modelName *string, audioContent *stri
}
// OCRFile OCR file
-func (b *BaiduModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
- return nil, fmt.Errorf("%s, no such method", b.Name())
+type qianfanOCRResponse struct {
+ Id string `json:"id"`
+ Result struct {
+ LayoutParsingResults []struct {
+ Markdown struct {
+ Text string `json:"text"`
+ } `json:"markdown"`
+ } `json:"layoutParsingResults"`
+ } `json:"result"`
}
-// ParseFile parse file
-func (z *BaiduModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+func (b *BaiduModel) OCRFile(modelName *string, content []byte, fileURL *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ if (fileURL == nil || *fileURL == "") && (content == nil || len(content) == 0) {
+ return nil, fmt.Errorf("image url or content is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", b.BaseURL[region], b.URLSuffix.OCR)
+
+ reqData := map[string]interface{}{
+ "model": *modelName,
+ }
+
+ if fileURL != nil && *fileURL != "" {
+ reqData["file"] = *fileURL
+ if strings.HasSuffix(strings.ToLower(*fileURL), ".pdf") {
+ reqData["fileType"] = 0 // PDF
+ } else {
+ reqData["fileType"] = 1 // img
+ }
+ } else if content != nil && len(content) > 0 {
+ reqData["file"] = base64.StdEncoding.EncodeToString(content)
+
+ mimeType := http.DetectContentType(content)
+ if strings.Contains(mimeType, "pdf") {
+ reqData["fileType"] = 0 // PDF
+ } else {
+ reqData["fileType"] = 1 // img
+ }
+ }
+
+ jsonData, err := json.Marshal(reqData)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal json payload: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := b.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var apiResponse qianfanOCRResponse
+ if err = json.Unmarshal(body, &apiResponse); err != nil {
+ return nil, fmt.Errorf("failed to parse response json: %w", err)
+ }
+
+ var extractedText string
+ if len(apiResponse.Result.LayoutParsingResults) > 0 {
+ extractedText = apiResponse.Result.LayoutParsingResults[0].Markdown.Text
+ } else {
+ return nil, fmt.Errorf("no parsing results returned from API")
+ }
+
+ var ocrResponse = OCRFileResponse{
+ Text: &extractedText,
+ }
+
+ return &ocrResponse, nil
}
func (b *BaiduModel) ListModels(apiConfig *APIConfig) ([]string, error) {
@@ -694,10 +782,14 @@ func (b *BaiduModel) CheckConnection(apiConfig *APIConfig) error {
return err
}
+func (z *BaiduModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("no such method", z.Name())
+}
+
func (z *BaiduModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+ return nil, fmt.Errorf("no such method", z.Name())
}
func (z *BaiduModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+ return nil, fmt.Errorf("no such method", z.Name())
}
diff --git a/internal/entity/models/cohere.go b/internal/entity/models/cohere.go
index f078737ecf3..1d16ece24dd 100644
--- a/internal/entity/models/cohere.go
+++ b/internal/entity/models/cohere.go
@@ -6,7 +6,11 @@ import (
"encoding/json"
"fmt"
"io"
+ "mime/multipart"
"net/http"
+ "os"
+ "path/filepath"
+ "strconv"
"strings"
"time"
)
@@ -482,15 +486,118 @@ func (c *CoHereModel) Rerank(modelName *string, query string, documents []string
// TranscribeAudio transcribe audio
func (c *CoHereModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
- return nil, fmt.Errorf("%s, no such method", c.Name())
+ if file == nil || *file == "" {
+ return nil, fmt.Errorf("file is missing")
+ }
+
+ region := "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", c.BaseURL[region], c.URLSuffix.ASR)
+
+ // multipart body
+
+ var body bytes.Buffer
+ writer := multipart.NewWriter(&body)
+
+ // open audio file
+ audioFile, err := os.Open(*file)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open audio file: %w", err)
+ }
+ defer audioFile.Close()
+
+ // create multipart file field
+
+ if err = writer.WriteField("model", *modelName); err != nil {
+ return nil, fmt.Errorf("failed to write model name: %w", err)
+ }
+ // extra params
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+
+ var val string
+
+ switch v := value.(type) {
+ case string:
+ val = v
+ case bool:
+ val = strconv.FormatBool(v)
+ case int:
+ val = strconv.Itoa(v)
+ case int64:
+ val = strconv.FormatInt(v, 10)
+ case float32:
+ val = strconv.FormatFloat(float64(v), 'f', -1, 32)
+ case float64:
+ val = strconv.FormatFloat(v, 'f', -1, 64)
+ default:
+ val = fmt.Sprintf("%v", v)
+ }
+
+ if err = writer.WriteField(key, val); err != nil {
+ return nil, fmt.Errorf("failed to write field %s: %w", key, err)
+ }
+ }
+ }
+
+ // all form fields (model, language) must appear before the file part in the multipart body
+ part, err := writer.CreateFormFile("file", filepath.Base(*file))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create form file: %w", err)
+ }
+
+ if _, err := io.Copy(part, audioFile); err != nil {
+ return nil, fmt.Errorf("failed to copy audio file: %w", err)
+ }
+
+ if err = writer.Close(); err != nil {
+ return nil, fmt.Errorf("failed to close writer: %w", err)
+ }
+
+ // build request
+ req, err := http.NewRequest("POST", url, &body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("Content-Type", writer.FormDataContentType())
+
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ respBody, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Cohere ASR API error: status %d, body: %s", resp.StatusCode, string(respBody))
+ }
+
+ var result struct {
+ Text string `json:"text"`
+ }
+
+ if err = json.Unmarshal(respBody, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+ }
+
+ return &ASRResponse{Text: result.Text}, nil
}
func (z *CoHereModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (c *CoHereModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (c *CoHereModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", c.Name())
}
@@ -514,15 +621,7 @@ func (c *CoHereModel) ListModels(apiConfig *APIConfig) ([]string, error) {
region = *apiConfig.Region
}
- baseURL := c.BaseURL[region]
- if baseURL == "" {
- baseURL = c.BaseURL["default"]
- }
- if baseURL == "" {
- baseURL = "https://api.cohere.com"
- }
- suffix := c.URLSuffix.Models
- url := fmt.Sprintf("%s/%s", strings.TrimSuffix(baseURL, "/"), strings.TrimPrefix(suffix, "/"))
+ url := fmt.Sprintf("%s/%s", c.BaseURL[region], c.URLSuffix.Models)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
diff --git a/internal/entity/models/deepseek.go b/internal/entity/models/deepseek.go
index 7c809c71cfb..78f5607c424 100644
--- a/internal/entity/models/deepseek.go
+++ b/internal/entity/models/deepseek.go
@@ -594,8 +594,8 @@ func (z *DeepSeekModel) TranscribeAudioWithSender(modelName *string, file *strin
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (d *DeepSeekModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (d *DeepSeekModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", d.Name())
}
diff --git a/internal/entity/models/dummy.go b/internal/entity/models/dummy.go
index d80616bd04e..718dd4bca1b 100644
--- a/internal/entity/models/dummy.go
+++ b/internal/entity/models/dummy.go
@@ -83,8 +83,8 @@ func (z *DummyModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (d *DummyModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (d *DummyModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", d.Name())
}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index 581baa51330..2df11aec484 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -91,6 +91,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewNovitaModel(baseURL, urlSuffix), nil
case "voyage":
return NewVoyageModel(baseURL, urlSuffix), nil
+ case "paddleocr":
+ return NewPaddleOCRModel(baseURL, urlSuffix), nil
default:
return NewDummyModel(baseURL, urlSuffix), nil
}
diff --git a/internal/entity/models/fishaudio.go b/internal/entity/models/fishaudio.go
index dc185e8975c..70f67076211 100644
--- a/internal/entity/models/fishaudio.go
+++ b/internal/entity/models/fishaudio.go
@@ -176,8 +176,8 @@ func (f *FishAudioModel) TranscribeAudioWithSender(modelName *string, file *stri
return fmt.Errorf("%s, no such method", f.Name())
}
-// AudioSpeech convert audio to text
-func (f *FishAudioModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (f *FishAudioModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("FishAudio API key is missing")
}
@@ -197,13 +197,13 @@ func (f *FishAudioModel) AudioSpeech(modelName *string, audioContent *string, ap
"text": *audioContent,
}
- if asrConfig != nil && asrConfig.Params != nil {
- for key, value := range asrConfig.Params {
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
reqBody[key] = value
}
}
- if asrConfig != nil && asrConfig.Format != "" {
- reqBody["format"] = asrConfig.Format
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["format"] = ttsConfig.Format
}
jsonData, err := json.Marshal(reqBody)
@@ -238,7 +238,7 @@ func (f *FishAudioModel) AudioSpeech(modelName *string, audioContent *string, ap
return &TTSResponse{Audio: body}, nil
}
-func (f *FishAudioModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig, sender func(*string, *string) error) error {
+func (f *FishAudioModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return fmt.Errorf("FishAudio API key is missing")
}
@@ -258,13 +258,13 @@ func (f *FishAudioModel) AudioSpeechWithSender(modelName *string, audioContent *
"text": *audioContent,
}
- if asrConfig != nil && asrConfig.Params != nil {
- for key, value := range asrConfig.Params {
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
reqBody[key] = value
}
}
- if asrConfig != nil && asrConfig.Format != "" {
- reqBody["format"] = asrConfig.Format
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["format"] = ttsConfig.Format
}
jsonData, err := json.Marshal(reqBody)
diff --git a/internal/entity/models/gitee.go b/internal/entity/models/gitee.go
index 70854f78b23..6a493906bd4 100644
--- a/internal/entity/models/gitee.go
+++ b/internal/entity/models/gitee.go
@@ -603,8 +603,8 @@ func (z *GiteeModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (g *GiteeModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (g *GiteeModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", g.Name())
}
diff --git a/internal/entity/models/google.go b/internal/entity/models/google.go
index 5578cdadd79..2702a04384e 100644
--- a/internal/entity/models/google.go
+++ b/internal/entity/models/google.go
@@ -349,8 +349,8 @@ func (z *GoogleModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (g *GoogleModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (g *GoogleModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", g.Name())
}
diff --git a/internal/entity/models/huggingface.go b/internal/entity/models/huggingface.go
index a730e556355..87e41242bfb 100644
--- a/internal/entity/models/huggingface.go
+++ b/internal/entity/models/huggingface.go
@@ -420,8 +420,8 @@ func (z *HuggingFaceModel) TranscribeAudioWithSender(modelName *string, file *st
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (h *HuggingFaceModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (h *HuggingFaceModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", h.Name())
}
diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.go
index 86377fa3303..fffca661285 100644
--- a/internal/entity/models/jina.go
+++ b/internal/entity/models/jina.go
@@ -260,8 +260,8 @@ func (z *JinaModel) TranscribeAudioWithSender(modelName *string, file *string, a
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *JinaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *JinaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/lmstudio.go b/internal/entity/models/lmstudio.go
index e9767a511da..d5d60efb37f 100644
--- a/internal/entity/models/lmstudio.go
+++ b/internal/entity/models/lmstudio.go
@@ -456,8 +456,8 @@ func (z *LmStudioModel) TranscribeAudioWithSender(modelName *string, file *strin
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *LmStudioModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *LmStudioModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/localai.go b/internal/entity/models/localai.go
index bc2b1b0300f..b218709ba19 100644
--- a/internal/entity/models/localai.go
+++ b/internal/entity/models/localai.go
@@ -808,8 +808,8 @@ func (l *LocalAIModel) TranscribeAudioWithSender(modelName *string, file *string
return fmt.Errorf("%s, no such method", l.Name())
}
-// AudioSpeech (TTS): same story as TranscribeAudio above.
-func (l *LocalAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (l *LocalAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", l.Name())
}
diff --git a/internal/entity/models/longcat.go b/internal/entity/models/longcat.go
index b5f194e5a23..e35dea19669 100644
--- a/internal/entity/models/longcat.go
+++ b/internal/entity/models/longcat.go
@@ -450,7 +450,7 @@ func (l *LongCatModel) TranscribeAudioWithSender(modelName *string, file *string
}
// AudioSpeech (TTS) is not exposed by the LongCat API.
-func (l *LongCatModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+func (l *LongCatModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", l.Name())
}
diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go
index 749d0d52e83..9ec953b9068 100644
--- a/internal/entity/models/minimax.go
+++ b/internal/entity/models/minimax.go
@@ -458,8 +458,8 @@ func (z *MinimaxModel) TranscribeAudioWithSender(modelName *string, file *string
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("MiniMax API key is missing")
}
@@ -478,14 +478,14 @@ func (z *MinimaxModel) AudioSpeech(modelName *string, audioContent *string, apiC
"model": modelName,
"text": audioContent,
}
- if asrConfig != nil && asrConfig.Params != nil {
- for key, value := range asrConfig.Params {
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
reqBody[key] = value
}
}
- if asrConfig != nil && asrConfig.Format != "" {
+ if ttsConfig != nil && ttsConfig.Format != "" {
reqBody["audio_setting"] = map[string]interface{}{
- "format": asrConfig.Format,
+ "format": ttsConfig.Format,
}
}
reqBody["stream"] = false
diff --git a/internal/entity/models/mistral.go b/internal/entity/models/mistral.go
index e1303346fab..1b526a87763 100644
--- a/internal/entity/models/mistral.go
+++ b/internal/entity/models/mistral.go
@@ -20,6 +20,7 @@ import (
"bufio"
"bytes"
"context"
+ "encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -573,8 +574,8 @@ func (z *MistralModel) TranscribeAudioWithSender(modelName *string, file *string
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *MistralModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *MistralModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
@@ -583,19 +584,99 @@ func (z *MistralModel) AudioSpeechWithSender(modelName *string, audioContent *st
}
// OCRFile OCR file
-func (z *MistralModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+func (z *MistralModel) OCRFile(modelName *string, content []byte, urls *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ if (urls == nil || *urls == "") && (content == nil || len(content) == 0) {
+ return nil, fmt.Errorf("file url or content is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.OCR)
+
+ var docURL string
+ if urls != nil && *urls != "" {
+ docURL = *urls
+ } else {
+ mimeType := http.DetectContentType(content)
+ base64Str := base64.StdEncoding.EncodeToString(content)
+ docURL = fmt.Sprintf("data:%s;base64,%s", mimeType, base64Str)
+ }
+
+ reqData := map[string]interface{}{
+ "model": *modelName,
+ "document": map[string]interface{}{
+ "type": "document_url",
+ "document_url": docURL,
+ },
+ }
+
+ jsonData, err := json.Marshal(reqData)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal json payload: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := z.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Mistral OCR API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var mistralResp struct {
+ Pages []struct {
+ Index int `json:"index"`
+ Markdown string `json:"markdown"`
+ } `json:"pages"`
+ }
+
+ if err = json.Unmarshal(body, &mistralResp); err != nil {
+ return nil, fmt.Errorf("failed to parse response json: %w", err)
+ }
+
+ var fullMarkdown strings.Builder
+ for _, page := range mistralResp.Pages {
+ fullMarkdown.WriteString(page.Markdown)
+ fullMarkdown.WriteString("\n\n")
+ }
+
+ resultText := strings.TrimSpace(fullMarkdown.String())
+
+ return &OCRFileResponse{
+ Text: &resultText,
+ }, nil
}
-// ParseFile parse file
func (z *MistralModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+ //TODO implement me
+ panic("implement me")
}
func (z *MistralModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+ return nil, fmt.Errorf("no such method", z.Name())
}
func (z *MistralModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
- return nil, fmt.Errorf("%s, no such method", z.Name())
+ return nil, fmt.Errorf("no such method", z.Name())
}
diff --git a/internal/entity/models/moonshot.go b/internal/entity/models/moonshot.go
index 3156b49cb94..fa1ad76ec4e 100644
--- a/internal/entity/models/moonshot.go
+++ b/internal/entity/models/moonshot.go
@@ -497,8 +497,8 @@ func (z *MoonshotModel) TranscribeAudioWithSender(modelName *string, file *strin
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *MoonshotModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *MoonshotModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/novita.go b/internal/entity/models/novita.go
index 5eb58efd7c7..9eb10ee9874 100644
--- a/internal/entity/models/novita.go
+++ b/internal/entity/models/novita.go
@@ -649,7 +649,7 @@ func (n *NovitaModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", n.Name())
}
-func (n *NovitaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+func (n *NovitaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", n.Name())
}
diff --git a/internal/entity/models/nvidia.go b/internal/entity/models/nvidia.go
index bb1e6785c76..998196253c5 100644
--- a/internal/entity/models/nvidia.go
+++ b/internal/entity/models/nvidia.go
@@ -561,8 +561,8 @@ func (z *NvidiaModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (n *NvidiaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (n *NvidiaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", n.Name())
}
diff --git a/internal/entity/models/ollama.go b/internal/entity/models/ollama.go
index b4eb1b35d9e..d95e9e8c734 100644
--- a/internal/entity/models/ollama.go
+++ b/internal/entity/models/ollama.go
@@ -454,8 +454,8 @@ func (z *OllamaModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *OllamaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *OllamaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", o.Name())
}
diff --git a/internal/entity/models/openai.go b/internal/entity/models/openai.go
index e46b734bb15..c19751231e3 100644
--- a/internal/entity/models/openai.go
+++ b/internal/entity/models/openai.go
@@ -603,8 +603,8 @@ func (z *OpenAIModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *OpenAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *OpenAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", o.Name())
}
diff --git a/internal/entity/models/openrouter.go b/internal/entity/models/openrouter.go
index 3b885d40f3a..4e99ef5bea4 100644
--- a/internal/entity/models/openrouter.go
+++ b/internal/entity/models/openrouter.go
@@ -538,8 +538,8 @@ func (z *OpenRouterModel) TranscribeAudioWithSender(modelName *string, file *str
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *OpenRouterModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *OpenRouterModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("OpenRouter API key is missing")
}
@@ -560,13 +560,13 @@ func (o *OpenRouterModel) AudioSpeech(modelName *string, audioContent *string, a
"input": audioContent,
}
- if asrConfig != nil && asrConfig.Params != nil {
- for key, value := range asrConfig.Params {
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
reqBody[key] = value
}
}
- if asrConfig != nil && asrConfig.Format != "" {
- reqBody["response_format"] = asrConfig.Format
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
}
jsonData, err := json.Marshal(reqBody)
diff --git a/internal/entity/models/paddleocr.go b/internal/entity/models/paddleocr.go
new file mode 100644
index 00000000000..25445ca38da
--- /dev/null
+++ b/internal/entity/models/paddleocr.go
@@ -0,0 +1,300 @@
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "strings"
+ "time"
+)
+
+type PaddleOCRModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewPaddleOCRModel(baseURL map[string]string, urlSuffix URLSuffix) *PaddleOCRModel {
+ return &PaddleOCRModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Timeout: 120 * time.Second,
+ Transport: &http.Transport{
+ MaxIdleConns: 100,
+ MaxIdleConnsPerHost: 10,
+ IdleConnTimeout: 90 * time.Second,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (p PaddleOCRModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return &PaddleOCRModel{
+ BaseURL: baseURL,
+ URLSuffix: p.URLSuffix,
+ httpClient: &http.Client{
+ Timeout: 120 * time.Second,
+ Transport: &http.Transport{
+ MaxIdleConns: 100,
+ MaxIdleConnsPerHost: 10,
+ IdleConnTimeout: 90 * time.Second,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (p *PaddleOCRModel) Name() string {
+ return "paddle_ocr"
+}
+
+func (p *PaddleOCRModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("no such method", p.Name())
+}
+
+type paddleSubmitResponse struct {
+ Data struct {
+ JobId string `json:"jobId"`
+ } `json:"data"`
+}
+
+type paddlePollResponse struct {
+ Data struct {
+ State string `json:"state"`
+ ErrorMsg string `json:"errorMsg"`
+ ResultUrl struct {
+ JsonUrl string `json:"jsonUrl"`
+ } `json:"resultUrl"`
+ } `json:"data"`
+}
+
+type paddleJsonlLine struct {
+ Result struct {
+ LayoutParsingResults []struct {
+ Markdown struct {
+ Text string `json:"text"`
+ } `json:"markdown"`
+ } `json:"layoutParsingResults"`
+ } `json:"result"`
+}
+
+func (p *PaddleOCRModel) OCRFile(modelName *string, content []byte, fileURL *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ if (content == nil || len(content) == 0) && (fileURL == nil || *fileURL == "") {
+ return nil, fmt.Errorf("content and fileURL cannot be both empty")
+ }
+
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", p.BaseURL[region], p.URLSuffix.OCR)
+
+ optionalPayload := map[string]bool{
+ "useDocOrientationClassify": false,
+ "useDocUnwarping": false,
+ "useChartRecognition": false,
+ }
+ optBytes, _ := json.Marshal(optionalPayload)
+
+ var req *http.Request
+ var err error
+
+ if fileURL != nil && strings.HasPrefix(*fileURL, "http") {
+ reqData := map[string]interface{}{
+ "fileUrl": *fileURL,
+ "model": *modelName,
+ "optionalPayload": optionalPayload,
+ }
+ jsonData, err := json.Marshal(reqData)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal json: %w", err)
+ }
+ req, err = http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ req.Header.Set("Content-Type", "application/json")
+ } else {
+ body := &bytes.Buffer{}
+ writer := multipart.NewWriter(body)
+
+ _ = writer.WriteField("model", *modelName)
+ _ = writer.WriteField("optionalPayload", string(optBytes))
+
+ part, err := writer.CreateFormFile("file", "document.pdf")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create form file: %w", err)
+ }
+ part.Write(content)
+ writer.Close()
+
+ req, err = http.NewRequest("POST", url, body)
+ req.Header.Set("Content-Type", writer.FormDataContentType())
+ }
+
+ req.Header.Set("Authorization", fmt.Sprintf("bearer %s", *apiConfig.ApiKey))
+
+ resp, err := p.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to submit job: %w", err)
+ }
+ defer resp.Body.Close()
+
+ respBody, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("submit job failed: %s", string(respBody))
+ }
+
+ var submitResp paddleSubmitResponse
+ if err := json.Unmarshal(respBody, &submitResp); err != nil {
+ return nil, fmt.Errorf("failed to parse submit response: %w", err)
+ }
+
+ jobId := submitResp.Data.JobId
+ if jobId == "" {
+ return nil, fmt.Errorf("failed to get jobId from response")
+ }
+
+ pollUrl := fmt.Sprintf("%s/%s", url, jobId)
+ var jsonlUrl string
+
+ for {
+ time.Sleep(3 * time.Second)
+
+ pollReq, _ := http.NewRequest("GET", pollUrl, nil)
+ pollReq.Header.Set("Authorization", fmt.Sprintf("bearer %s", *apiConfig.ApiKey))
+
+ pollResp, err := p.httpClient.Do(pollReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to poll job status: %w", err)
+ }
+
+ pollBody, _ := io.ReadAll(pollResp.Body)
+ pollResp.Body.Close()
+
+ if pollResp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("poll job failed: %s", string(pollBody))
+ }
+
+ var pollData paddlePollResponse
+ if err = json.Unmarshal(pollBody, &pollData); err != nil {
+ return nil, fmt.Errorf("failed to parse poll response: %w", err)
+ }
+
+ // end if 'done' or 'failed'
+ state := pollData.Data.State
+ if state == "done" {
+ jsonlUrl = pollData.Data.ResultUrl.JsonUrl
+ break
+ } else if state == "failed" {
+ return nil, fmt.Errorf("ocr job failed on server: %s", pollData.Data.ErrorMsg)
+ }
+ }
+
+ if jsonlUrl == "" {
+ return nil, fmt.Errorf("job done but jsonl url is empty")
+ }
+
+ resReq, err := http.NewRequest("GET", jsonlUrl, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request for jsonl: %w", err)
+ }
+
+ resResp, err := p.httpClient.Do(resReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to download jsonl result: %w", err)
+ }
+ defer resResp.Body.Close()
+
+ if resResp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to download jsonl, status: %d", resResp.StatusCode)
+ }
+
+ var fullMarkdown strings.Builder
+ scanner := bufio.NewScanner(resResp.Body)
+
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" {
+ continue
+ }
+
+ var lineData paddleJsonlLine
+ if err := json.Unmarshal([]byte(line), &lineData); err != nil {
+ continue
+ }
+
+ for _, layoutRes := range lineData.Result.LayoutParsingResults {
+ fullMarkdown.WriteString(layoutRes.Markdown.Text)
+ fullMarkdown.WriteString("\n\n")
+ }
+ }
+
+ if err = scanner.Err(); err != nil {
+ return nil, fmt.Errorf("error reading jsonl: %w", err)
+ }
+
+ extractedText := strings.TrimSpace(fullMarkdown.String())
+
+ return &OCRFileResponse{Text: &extractedText}, nil
+}
+
+func (p *PaddleOCRModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) CheckConnection(apiConfig *APIConfig) error {
+ return fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
+
+func (p *PaddleOCRModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("no such method", p.Name())
+}
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index 552ef903612..11ba909365c 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -731,7 +731,7 @@ func (o *SiliconflowModel) TranscribeAudio(modelName *string, file *string, apiC
}
region := "default"
- if apiConfig.Region != nil && *apiConfig.Region != "" {
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -835,9 +835,6 @@ func (o *SiliconflowModel) TranscribeAudio(modelName *string, file *string, apiC
return nil, fmt.Errorf("failed to unmarshal response: %w, body=%s", err, string(respBody))
}
- var res ASRResponse
- res.Text = result.Text
-
return &ASRResponse{Text: result.Text}, nil
}
@@ -845,8 +842,8 @@ func (z *SiliconflowModel) TranscribeAudioWithSender(modelName *string, file *st
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *SiliconflowModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *SiliconflowModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
if audioContent == nil || *audioContent == "" {
return nil, fmt.Errorf("audio content is empty")
}
@@ -864,13 +861,13 @@ func (o *SiliconflowModel) AudioSpeech(modelName *string, audioContent *string,
"stream": false,
}
- if asrConfig != nil && asrConfig.Params != nil {
- for key, value := range asrConfig.Params {
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
reqBody[key] = value
}
}
- if asrConfig != nil && asrConfig.Format != "" {
- reqBody["response_format"] = asrConfig.Format
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
}
jsonData, err := json.Marshal(reqBody)
diff --git a/internal/entity/models/stepfun.go b/internal/entity/models/stepfun.go
index 941f815fd25..a4cce4bb9eb 100644
--- a/internal/entity/models/stepfun.go
+++ b/internal/entity/models/stepfun.go
@@ -477,8 +477,8 @@ func (s *StepFunModel) TranscribeAudioWithSender(modelName *string, file *string
return fmt.Errorf("%s, no such method", s.Name())
}
-// AudioSpeech convert audio to text
-func (s *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (s *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
// TODO Test it
if audioContent == nil || *audioContent == "" {
return nil, fmt.Errorf("audio content is empty")
@@ -496,13 +496,13 @@ func (s *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiC
"input": *audioContent,
}
- if asrConfig != nil && asrConfig.Params != nil {
- for key, value := range asrConfig.Params {
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ for key, value := range ttsConfig.Params {
reqBody[key] = value
}
}
- if asrConfig != nil && asrConfig.Format != "" {
- reqBody["response_format"] = asrConfig.Format
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
}
jsonData, err := json.Marshal(reqBody)
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index e4123e064df..d791ac04cb3 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -28,8 +28,8 @@ type ModelDriver interface {
// TranscribeAudio transcribe audio
TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error)
TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error
- // AudioSpeech convert audio to text
- AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error)
+ // AudioSpeech convert text to audio
+ AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error)
AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error
// OCRFile OCR file
OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error)
diff --git a/internal/entity/models/upstage.go b/internal/entity/models/upstage.go
index 7e5596a902f..1b492b60e2b 100644
--- a/internal/entity/models/upstage.go
+++ b/internal/entity/models/upstage.go
@@ -594,8 +594,8 @@ func (z *UpstageModel) TranscribeAudioWithSender(modelName *string, file *string
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (z *UpstageModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (z *UpstageModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go
index 61452c9bcaa..c8dbcdab7c0 100644
--- a/internal/entity/models/vllm.go
+++ b/internal/entity/models/vllm.go
@@ -672,8 +672,8 @@ func (z *VllmModel) TranscribeAudioWithSender(modelName *string, file *string, a
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *VllmModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *VllmModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", o.Name())
}
diff --git a/internal/entity/models/volcengine.go b/internal/entity/models/volcengine.go
index 4a806883ec7..8f3133aa416 100644
--- a/internal/entity/models/volcengine.go
+++ b/internal/entity/models/volcengine.go
@@ -519,8 +519,8 @@ func (z *VolcEngine) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *VolcEngine) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *VolcEngine) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", o.Name())
}
diff --git a/internal/entity/models/voyage.go b/internal/entity/models/voyage.go
index 14f67dda75c..c17a0e80815 100644
--- a/internal/entity/models/voyage.go
+++ b/internal/entity/models/voyage.go
@@ -363,7 +363,7 @@ func (v *VoyageModel) TranscribeAudioWithSender(modelName *string, file *string,
return fmt.Errorf("%s, no such method", v.Name())
}
-func (v *VoyageModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+func (v *VoyageModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", v.Name())
}
diff --git a/internal/entity/models/xai.go b/internal/entity/models/xai.go
index 88b465fecf3..b19f93ca7dc 100644
--- a/internal/entity/models/xai.go
+++ b/internal/entity/models/xai.go
@@ -502,8 +502,8 @@ func (z *XAIModel) TranscribeAudioWithSender(modelName *string, file *string, ap
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *XAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *XAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", o.Name())
}
diff --git a/internal/entity/models/zhipu-ai.go b/internal/entity/models/zhipu-ai.go
index fa86c154670..d90d63559be 100644
--- a/internal/entity/models/zhipu-ai.go
+++ b/internal/entity/models/zhipu-ai.go
@@ -676,8 +676,8 @@ func (z *ZhipuAIModel) TranscribeAudioWithSender(modelName *string, file *string
return fmt.Errorf("%s, no such method", z.Name())
}
-// AudioSpeech convert audio to text
-func (o *ZhipuAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+// AudioSpeech convert text to audio
+func (o *ZhipuAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", o.Name())
}
From 09d45046e5f46fb2300f8860a3819568d71916d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B0=8F=E7=86=8A?= <1114202297@qq.com>
Date: Fri, 15 May 2026 22:29:44 +0800
Subject: [PATCH 167/666] Feat/web markdown UI updates (#14214)
### What problem does this PR solve?
LLM/chat and search UIs render Markdown in several places (document
preview, floating chat widget, next-search, etc.). Plugin lists and
behavior were duplicated or inconsistent, and single newlines in model
output were not always rendered as visible line breaks, which hurts
readability for chat-style content.
This PR centralizes shared **remark/rehype** configuration (including
**`remark-breaks`** for newline handling) and wires the main Markdown
surfaces to use it, so behavior is consistent and easier to maintain.
### Type of change
- [x] Refactoring
---------
Co-authored-by: Yingfeng Zhang
---
web/package-lock.json | 30 +++++++++++++++++++
web/package.json | 1 +
.../components/document-preview/md/index.tsx | 6 ++--
.../floating-chat-widget-markdown.tsx | 5 ++--
.../components/highlight-markdown/index.tsx | 5 ++--
web/src/components/markdown-content/index.tsx | 5 ++--
.../next-markdown-content/index.tsx | 5 ++--
web/src/constants/markdown-remark-plugins.ts | 17 +++++++++++
.../next-search/markdown-content/index.tsx | 5 ++--
9 files changed, 62 insertions(+), 17 deletions(-)
create mode 100644 web/src/constants/markdown-remark-plugins.ts
diff --git a/web/package-lock.json b/web/package-lock.json
index bfb0aee4f27..38407cfbe33 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -100,6 +100,7 @@
"recharts": "^2.12.4",
"rehype-katex": "^7.0.1",
"rehype-raw": "^7.0.0",
+ "remark-breaks": "^4.0.0",
"remark-gfm": "^4.0.0",
"remark-math": "^6.0.0",
"sonner": "^1.7.4",
@@ -18439,6 +18440,20 @@
"url": "https://opencollective.com/unified"
}
},
+ "node_modules/mdast-util-newline-to-break": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmmirror.com/mdast-util-newline-to-break/-/mdast-util-newline-to-break-2.0.0.tgz",
+ "integrity": "sha512-MbgeFca0hLYIEx/2zGsszCSEJJ1JSCdiY5xQxRcLDDGa8EPvlLPupJ4DSajbMPAnC0je8jfb9TiUATnxxrHUog==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/mdast": "^4.0.0",
+ "mdast-util-find-and-replace": "^3.0.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/unified"
+ }
+ },
"node_modules/mdast-util-phrasing": {
"version": "4.1.0",
"resolved": "https://registry.npmmirror.com/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz",
@@ -22539,6 +22554,21 @@
"url": "https://opencollective.com/unified"
}
},
+ "node_modules/remark-breaks": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmmirror.com/remark-breaks/-/remark-breaks-4.0.0.tgz",
+ "integrity": "sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/mdast": "^4.0.0",
+ "mdast-util-newline-to-break": "^2.0.0",
+ "unified": "^11.0.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/unified"
+ }
+ },
"node_modules/remark-gfm": {
"version": "4.0.1",
"resolved": "https://registry.npmmirror.com/remark-gfm/-/remark-gfm-4.0.1.tgz",
diff --git a/web/package.json b/web/package.json
index 4e0485c6d98..6dbed295b41 100644
--- a/web/package.json
+++ b/web/package.json
@@ -122,6 +122,7 @@
"recharts": "^2.12.4",
"rehype-katex": "^7.0.1",
"rehype-raw": "^7.0.0",
+ "remark-breaks": "^4.0.0",
"remark-gfm": "^4.0.0",
"remark-math": "^6.0.0",
"sonner": "^1.7.4",
diff --git a/web/src/components/document-preview/md/index.tsx b/web/src/components/document-preview/md/index.tsx
index bdc30f91bc1..13f1af3c2f4 100644
--- a/web/src/components/document-preview/md/index.tsx
+++ b/web/src/components/document-preview/md/index.tsx
@@ -1,10 +1,10 @@
import { Authorization } from '@/constants/authorization';
+import { MarkdownRemarkPluginsLite } from '@/constants/markdown-remark-plugins';
import { cn } from '@/lib/utils';
import FileError from '@/pages/document-viewer/file-error';
import { getAuthorization } from '@/utils/authorization-util';
import React, { useEffect, useState } from 'react';
import ReactMarkdown from 'react-markdown';
-import remarkGfm from 'remark-gfm';
interface MdProps {
// filePath: string;
@@ -34,7 +34,9 @@ export const Md: React.FC = ({ url, className }) => {
style={{ padding: 4, overflow: 'scroll' }}
className={cn(className, 'markdown-body h-[calc(100vh - 200px)]')}
>
- {content}
+
+ {content}
+
);
};
diff --git a/web/src/components/floating-chat-widget-markdown.tsx b/web/src/components/floating-chat-widget-markdown.tsx
index 4237fa17587..51912d72afb 100644
--- a/web/src/components/floating-chat-widget-markdown.tsx
+++ b/web/src/components/floating-chat-widget-markdown.tsx
@@ -36,8 +36,7 @@ import {
} from 'react-syntax-highlighter/dist/esm/styles/prism';
import rehypeKatex from 'rehype-katex';
import rehypeRaw from 'rehype-raw';
-import remarkGfm from 'remark-gfm';
-import remarkMath from 'remark-math';
+import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
import { visitParents } from 'unist-util-visit-parents';
import styles from './floating-chat-widget-markdown.module.less';
import { useIsDarkTheme } from './theme-provider';
@@ -292,7 +291,7 @@ const FloatingChatWidgetMarkdown = ({
{children}
,
diff --git a/web/src/components/next-markdown-content/index.tsx b/web/src/components/next-markdown-content/index.tsx
index ebedb0eed67..1bab4440a77 100644
--- a/web/src/components/next-markdown-content/index.tsx
+++ b/web/src/components/next-markdown-content/index.tsx
@@ -10,8 +10,7 @@ import Markdown from 'react-markdown';
import SyntaxHighlighter from 'react-syntax-highlighter';
import rehypeKatex from 'rehype-katex';
import rehypeRaw from 'rehype-raw';
-import remarkGfm from 'remark-gfm';
-import remarkMath from 'remark-math';
+import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
import { visitParents } from 'unist-util-visit-parents';
import { useTranslation } from 'react-i18next';
@@ -355,7 +354,7 @@ function MarkdownContent({
{children}
,
diff --git a/web/src/constants/markdown-remark-plugins.ts b/web/src/constants/markdown-remark-plugins.ts
new file mode 100644
index 00000000000..9d797082bbd
--- /dev/null
+++ b/web/src/constants/markdown-remark-plugins.ts
@@ -0,0 +1,17 @@
+import remarkBreaks from 'remark-breaks';
+import remarkGfm from 'remark-gfm';
+import remarkMath from 'remark-math';
+
+/**
+ * GFM + line breaks only (no TeX). For surfaces that do not wire rehype-katex
+ * (e.g. uploaded document preview).
+ */
+export const MarkdownRemarkPluginsLite = [remarkGfm, remarkBreaks];
+
+/**
+ * Shared Markdown pipeline for assistant-style content:
+ * - remark-gfm: GFM tables, task lists, strikethrough, autolinks, etc.
+ * - remark-math: TeX ($...$ / $$...$$); pair with rehype-katex on render.
+ * - remark-breaks: treat single newlines as hard breaks (common in LLM chat).
+ */
+export const MarkdownRemarkPlugins = [remarkGfm, remarkMath, remarkBreaks];
diff --git a/web/src/pages/next-search/markdown-content/index.tsx b/web/src/pages/next-search/markdown-content/index.tsx
index 132f34d5df6..7149e2b8f0a 100644
--- a/web/src/pages/next-search/markdown-content/index.tsx
+++ b/web/src/pages/next-search/markdown-content/index.tsx
@@ -8,8 +8,7 @@ import Markdown from 'react-markdown';
import SyntaxHighlighter from 'react-syntax-highlighter';
import rehypeKatex from 'rehype-katex';
import rehypeRaw from 'rehype-raw';
-import remarkGfm from 'remark-gfm';
-import remarkMath from 'remark-math';
+import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
import { visitParents } from 'unist-util-visit-parents';
import { useTranslation } from 'react-i18next';
@@ -250,7 +249,7 @@ const MarkdownContent = ({
>
{children}
,
From ff318aba7a9f6a7b7eec5f47747b1cf51b3095b9 Mon Sep 17 00:00:00 2001
From: Ricardo-M-L <69202550+Ricardo-M-L@users.noreply.github.com>
Date: Mon, 18 May 2026 09:58:45 +0800
Subject: [PATCH 168/666] fix: correct literal_eval dispatch and bool
isinstance ordering in agent components (#13988)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
This PR fixes 3 bugs in agent components:
### Bug 1: `DataOperations._invoke()` dispatches `"literal_eval"` to
wrong handler
**File:** `agent/component/data_operations.py`, line 76
The `_invoke()` method compares `self._param.operations` against
`"recursive_eval"` (line 76), but the valid value defined in
`DataOperationsParam.__init__()` (line 29) and validated in `check()`
(line 43) is `"literal_eval"`. This means selecting the `literal_eval`
operation from the frontend would never match, and the method
`_literal_eval()` would never be called.
**Fix:** Change `"recursive_eval"` to `"literal_eval"` in the dispatch
condition.
### Bug 2: `VariableAssigner._clear()` — `bool` branch unreachable
**File:** `agent/component/variable_assigner.py`, lines 95–100
In Python, `bool` is a subclass of `int` (`True` is `isinstance(True,
int) == True`). The `isinstance(variable, int)` check on line 95 catches
boolean values before the `isinstance(variable, bool)` check on line 99,
making the bool branch unreachable. A boolean variable would be cleared
to `0` instead of `False`.
**Fix:** Move the `isinstance(variable, bool)` check before
`isinstance(variable, int)`.
### Bug 3: `LoopItem.evaluate_condition()` — `bool` branch unreachable
**File:** `agent/component/loopitem.py`, lines 67–93
Same issue as Bug 2: `isinstance(var, (int, float))` on line 67 catches
boolean values before `isinstance(var, bool)` on line 85. Boolean
variables would be evaluated with numeric operators (`=`, `≠`, `>`,
etc.) instead of boolean operators (`is`, `is not`).
**Fix:** Move the `isinstance(var, bool)` check before `isinstance(var,
(int, float))`.
## Test plan
- [ ] Verify `DataOperations` with `literal_eval` operation correctly
invokes `_literal_eval()`
- [ ] Verify `VariableAssigner._clear()` returns `False` for boolean
variables (not `0`)
- [ ] Verify `LoopItem.evaluate_condition()` uses boolean operators for
`True`/`False` values
🤖 Generated with [Claude Code](https://claude.com/claude-code)
## Summary by CodeRabbit
* **Bug Fixes**
* Fixed operation routing logic to correctly dispatch the "literal_eval"
operation to its handler.
* **Refactor**
* Reorganized conditional branch ordering in agent components to improve
code structure and maintainability without affecting functional
behavior.
Co-authored-by: Claude Opus 4.6 (1M context)
---
agent/component/data_operations.py | 2 +-
agent/component/loopitem.py | 20 ++++++++++----------
agent/component/variable_assigner.py | 4 ++--
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/agent/component/data_operations.py b/agent/component/data_operations.py
index 60e65f88121..9cf5c55335b 100644
--- a/agent/component/data_operations.py
+++ b/agent/component/data_operations.py
@@ -73,7 +73,7 @@ def _invoke(self, **kwargs):
continue
if self._param.operations == "select_keys":
self._select_keys()
- elif self._param.operations == "recursive_eval":
+ elif self._param.operations == "literal_eval":
self._literal_eval()
elif self._param.operations == "combine":
self._combine()
diff --git a/agent/component/loopitem.py b/agent/component/loopitem.py
index b656ea78948..0cfb500850d 100644
--- a/agent/component/loopitem.py
+++ b/agent/component/loopitem.py
@@ -64,6 +64,16 @@ def evaluate_condition(self,var, operator, value):
elif operator == "not empty":
return var != ""
+ elif isinstance(var, bool):
+ if operator == "is":
+ return var is value
+ elif operator == "is not":
+ return var is not value
+ elif operator == "empty":
+ return var is None
+ elif operator == "not empty":
+ return var is not None
+
elif isinstance(var, (int, float)):
if operator == "=":
return var == value
@@ -82,16 +92,6 @@ def evaluate_condition(self,var, operator, value):
elif operator == "not empty":
return var is not None
- elif isinstance(var, bool):
- if operator == "is":
- return var is value
- elif operator == "is not":
- return var is not value
- elif operator == "empty":
- return var is None
- elif operator == "not empty":
- return var is not None
-
elif isinstance(var, dict):
if operator == "empty":
return len(var) == 0
diff --git a/agent/component/variable_assigner.py b/agent/component/variable_assigner.py
index 0f782136846..5b5e39a8259 100644
--- a/agent/component/variable_assigner.py
+++ b/agent/component/variable_assigner.py
@@ -92,12 +92,12 @@ def _clear(self,variable):
return ""
elif isinstance(variable,dict):
return {}
+ elif isinstance(variable,bool):
+ return False
elif isinstance(variable,int):
return 0
elif isinstance(variable,float):
return 0.0
- elif isinstance(variable,bool):
- return False
else:
return None
From e194027b01112f3f7ad91558c19830dcec7881ed Mon Sep 17 00:00:00 2001
From: 07heco <3379248674@qq.com>
Date: Mon, 18 May 2026 10:00:18 +0800
Subject: [PATCH 169/666] refactor: optimize BaseTitleChunker to improve RAG
document chunk quality (#14247)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## RAG Optimization Description
Optimize the core `BaseTitleChunker` in
`rag/flow/chunker/title_chunker/common.py` to improve RAG document
chunking quality and retrieval accuracy.
## Key Changes
1. **Format-branched text processing**: Preserve original whitespace &
indentation for Markdown/HTML payloads to maintain document semantics
and chunk fidelity; only perform full whitespace cleaning on plain text
content.
2. **Empty chunk filtering**: Thoroughly filter invalid pure-blank lines
to reduce noisy data in vector database.
3. **Code deduplication**: Unified markdown/text/html payload extraction
logic, removed redundant repeated code blocks.
4. **None serialization fix**: Avoid converting `None` value into
literal `"None"` string in chunk text fields.
5. **Production logging**: Added input/output line count logging for
filter logic, observable in online environment.
6. **100% backward compatible**: No changes to chunking hierarchy rules,
output format and all existing workflows.
## RAG Business Value
- Preserves document format fidelity for structured Markdown/HTML files
- Reduces invalid noisy chunks → improves RAG retrieval precision
- Cleans plain text data → optimizes vector embedding quality
- Improves code maintainability with no breaking changes
- Provides observable logging for chunk filtering behavior
## Compatibility
- ✅ No API changes
- ✅ No chunk logic modifications
- ✅ All document parsing/chunking workflows unaffected
- ✅ All pre-checks passed, no code conflicts
### Type of change
- [x] Refactoring
- [x] Performance Improvement
---
rag/flow/chunker/title_chunker/common.py | 59 +++++++++++++++++++-----
1 file changed, 47 insertions(+), 12 deletions(-)
diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.py
index 89981a83de5..0ca6549a960 100644
--- a/rag/flow/chunker/title_chunker/common.py
+++ b/rag/flow/chunker/title_chunker/common.py
@@ -73,25 +73,61 @@ async def invoke(self):
def extract_line_records(self):
- # Normalize all upstream payloads into an ordered record stream.
- # Level resolution and chunk construction operate on this stream only,
- # so strategy code does not depend on source-specific output layouts.
+ """
+ Normalize all upstream input payloads into a unified ordered record stream.
+ All level resolution and chunk construction logic operates on this standard stream,
+ decoupling downstream chunking strategies from different upstream output formats.
+ """
+ import logging
+ logger = logging.getLogger(__name__)
+
+ payload = None
+ # Extract raw content payload based on upstream output format type
if self.from_upstream.output_format == "markdown":
payload = self.from_upstream.markdown_result or ""
- return [{"text": line, "doc_type_kwd": "text", "img_id": None, "layout": "", PDF_POSITIONS_KEY: []} for line in payload.split("\n") if line]
-
- if self.from_upstream.output_format == "text":
+ elif self.from_upstream.output_format == "text":
payload = self.from_upstream.text_result or ""
- return [{"text": line, "doc_type_kwd": "text", "img_id": None, "layout": "", PDF_POSITIONS_KEY: []} for line in payload.split("\n") if line]
-
- if self.from_upstream.output_format == "html":
+ elif self.from_upstream.output_format == "html":
payload = self.from_upstream.html_result or ""
- return [{"text": line, "doc_type_kwd": "text", "img_id": None, "layout": "", PDF_POSITIONS_KEY: []} for line in payload.split("\n") if line]
+
+ # Boundary robustness fix: explicit None check to distinguish `None` and empty string ""
+ # Prevents empty payload from unexpectedly falling through to structured chunk branch
+ if payload is not None:
+ lines = payload.split("\n")
+ input_line_count = len(lines)
+
+ # Format-branched text processing to preserve original document semantics
+ # Plain text: perform full whitespace stripping and invalid empty line filtering
+ if self.from_upstream.output_format == "text":
+ clean_lines = [line.strip() for line in lines if line.strip()]
+ # Markdown & HTML: retain original indentation/spacing, only filter pure blank lines
+ else:
+ clean_lines = [line for line in lines if line.strip()]
+
+ output_line_count = len(clean_lines)
+ # Production observability log: added format dimension per project coding guidelines
+ logger.info(
+ f"payload filter: format={self.from_upstream.output_format} before={input_line_count} after={output_line_count}"
+ )
+
+ return [
+ {
+ "text": line,
+ "doc_type_kwd": "text",
+ "img_id": None,
+ "layout": "",
+ PDF_POSITIONS_KEY: []
+ }
+ for line in clean_lines
+ ]
+ # Return empty array directly for null payload to block invalid branch fallthrough
+ return []
items = self.from_upstream.chunks if self.from_upstream.output_format == "chunks" else self.from_upstream.json_result
return [
{
- "text": str(item.get("text") or ""),
+ # Serialization fix: avoid None value being converted into literal "None" string
+ "text": item.get("text") or "",
"doc_type_kwd": str(item.get("doc_type_kwd") or "text"),
"img_id": item.get("img_id"),
"layout": "{} {}".format(item.get("layout_type", ""), item.get("layoutno", "")).strip(),
@@ -100,7 +136,6 @@ def extract_line_records(self):
for item in items or []
]
-
def extract_outlines(self):
file = self.from_upstream.file or {}
source = (
From 9d94527b1d98f18b8c09a32e24f0721c9a2cc5b1 Mon Sep 17 00:00:00 2001
From: qinling0210 <88864212+qinling0210@users.noreply.github.com>
Date: Mon, 18 May 2026 10:25:59 +0800
Subject: [PATCH 170/666] Bump to infinity v0.7.0 (#14968)
### What problem does this PR solve?
Upgrade infinity
### Type of change
- [x] Refactoring
---
docker/docker-compose-base.yml | 2 +-
helm/values.yaml | 2 +-
pyproject.toml | 2 +-
uv.lock | 8 ++++----
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/docker/docker-compose-base.yml b/docker/docker-compose-base.yml
index 12dbe5eef69..1122fe7a7c6 100644
--- a/docker/docker-compose-base.yml
+++ b/docker/docker-compose-base.yml
@@ -72,7 +72,7 @@ services:
infinity:
profiles:
- infinity
- image: infiniflow/infinity:v0.7.0-dev7
+ image: infiniflow/infinity:v0.7.0
volumes:
- infinity_data:/var/infinity
- ./infinity_conf.toml:/infinity_conf.toml
diff --git a/helm/values.yaml b/helm/values.yaml
index 133beb8073b..948f0702548 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -124,7 +124,7 @@ ragflow:
infinity:
image:
repository: infiniflow/infinity
- tag: v0.7.0-dev7
+ tag: v0.7.0
pullPolicy: IfNotPresent
pullSecrets: []
storage:
diff --git a/pyproject.toml b/pyproject.toml
index b5a170e4c1e..fcb27283d23 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -55,7 +55,7 @@ dependencies = [
"groq==0.9.0",
"grpcio-status==1.67.1",
"html-text==0.6.2",
- "infinity-sdk==0.7.0-dev7",
+ "infinity-sdk==0.7.0",
"infinity-emb>=0.0.66,<0.0.67",
"jira==3.10.5",
"json-repair==0.35.0",
diff --git a/uv.lock b/uv.lock
index 8a53b108a27..63a118689e0 100644
--- a/uv.lock
+++ b/uv.lock
@@ -3150,7 +3150,7 @@ wheels = [
[[package]]
name = "infinity-sdk"
-version = "0.7.0.dev7"
+version = "0.7.0"
source = { registry = "https://mirrors.aliyun.com/pypi/simple" }
dependencies = [
{ name = "datrie" },
@@ -3167,9 +3167,9 @@ dependencies = [
{ name = "sqlglot", extra = ["rs"] },
{ name = "thrift" },
]
-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/73/65/8c8a4bb3abfd21e13e295a2e16166e6ea3c9c1e0e7af937d91946ea147bc/infinity_sdk-0.7.0.dev7.tar.gz", hash = "sha256:5a11ce8ded7181240842ecde47ab5a0d19c52e1246bbf509b392da6c195df8e7" }
+sdist = { url = "https://mirrors.aliyun.com/pypi/packages/46/c9/0bb4981acdaca79b864b26d998201f652ef6024fce7d0c39da9c11605890/infinity_sdk-0.7.0.tar.gz", hash = "sha256:42ba8c6acd4fad918b1ed189fab3383023e4750d46ef1bf1c9465ffcf3ff8335" }
wheels = [
- { url = "https://mirrors.aliyun.com/pypi/packages/cf/7e/0e8b3f418bceb290bbfdbe7e7f260d249f819512692b9972cd25b01d936c/infinity_sdk-0.7.0.dev7-py3-none-any.whl", hash = "sha256:cb0c3bad29ed01a75a75fae0a17ad45108aa22f2f10e8e2203e3fb31decc0568" },
+ { url = "https://mirrors.aliyun.com/pypi/packages/27/56/01a0b4b816c70595a83ad2d7ec387d1d991f0ea4607d77799bac010be27c/infinity_sdk-0.7.0-py3-none-any.whl", hash = "sha256:4772fded64ff733eb3dff36df3a2c1c867049acf3c6e54d4e40b5a915c0f3c18" },
]
[[package]]
@@ -6254,7 +6254,7 @@ requires-dist = [
{ name = "grpcio-status", specifier = "==1.67.1" },
{ name = "html-text", specifier = "==0.6.2" },
{ name = "infinity-emb", specifier = ">=0.0.66,<0.0.67" },
- { name = "infinity-sdk", specifier = "==0.7.0.dev7" },
+ { name = "infinity-sdk", specifier = "==0.7.0" },
{ name = "jira", specifier = "==3.10.5" },
{ name = "json-repair", specifier = "==0.35.0" },
{ name = "langfuse", specifier = ">=4.0.1" },
From e98f3e5c0d81528c798f77b7ab822cb56358e531 Mon Sep 17 00:00:00 2001
From: Idriss Sbaaoui <112825897+6ba3i@users.noreply.github.com>
Date: Mon, 18 May 2026 11:14:27 +0800
Subject: [PATCH 171/666] Fix session deletion leaking chat-upload blobs
(#14969)
### What problem does this PR solve?
This fixes a bug where files uploaded in chat were left in storage after
the session was deleted. It now removes those chat-uploaded blobs during
session deletion. fixes #14965
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/restful_apis/chat_api.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/api/apps/restful_apis/chat_api.py b/api/apps/restful_apis/chat_api.py
index 9a4d5b14180..baba98ae288 100644
--- a/api/apps/restful_apis/chat_api.py
+++ b/api/apps/restful_apis/chat_api.py
@@ -47,6 +47,7 @@
)
from api.utils.tenant_utils import ensure_tenant_model_id_for_params
from common.constants import LLMType, RetCode, StatusEnum
+from common import settings
from common.misc_utils import get_uuid, thread_pool_exec
from rag.prompts.generator import chunks_format
from rag.prompts.template import load_prompt
@@ -794,6 +795,17 @@ async def delete_sessions(chat_id):
if not ConversationService.query(id=sid, dialog_id=chat_id):
errors.append(f"The chat doesn't own the session {sid}")
continue
+ ok, conv = ConversationService.get_by_id(sid)
+ if ok:
+ for msg in conv.message or []:
+ for file in msg.get("files") or []:
+ file_id = file.get("id")
+ if not file_id:
+ continue
+ try:
+ settings.STORAGE_IMPL.rm(f"{current_user.id}-downloads", file_id)
+ except Exception:
+ logging.warning("Failed to delete chat upload blob %s/%s", current_user.id, file_id)
ConversationService.delete_by_id(sid)
success_count += 1
all_errors = errors + duplicate_messages
From 6794ad2f70f0b738c9456be278b69582c9687cf9 Mon Sep 17 00:00:00 2001
From: Panda Dev <56657208+pandadev66@users.noreply.github.com>
Date: Mon, 18 May 2026 06:02:28 +0200
Subject: [PATCH 172/666] Go: implement Embed (embeddings) in Novita driver
(#14895)
### What problem does this PR solve?
Fixes #14893
The Novita Go driver landed in #14850 and shipped a stub `Embed` method
that returned `"novita, no such method"`, so Novita could not be used as
an embedding provider in RAGFlow. This PR fills that gap.
Novita exposes a public embeddings endpoint at `POST
https://api.novita.ai/v3/embeddings` that accepts the standard
OpenAI-compatible request shape (`{model, input}`) with `Authorization:
Bearer `. Two embedding models are documented in Novita's model
library: `baai/bge-m3` (multilingual, 8192 tokens) and
`baai/bge-large-en-v1.5`.
### Changes
- `internal/entity/models/novita.go`: implement `NovitaModel.Embed`.
- Validate inputs (api key, model name) and short-circuit on empty
texts.
- Resolve region with the existing `baseURLForRegion` helper.
- Build URL from `URLSuffix.Embedding` (the embeddings path lives under
`/v3/`, separate from the chat path under `/openai/v1/`).
- Send `{model, input}` POST body, add `dimensions` when
`embeddingConfig.Dimension > 0` (matches the pattern in #14735).
- Bearer auth + JSON content type, mirroring the chat path.
- Parse `{data: [{embedding, index}]}` and reorder by `index`, rejecting
out-of-range indices, duplicates, and missing entries so the output
always lines up with the input. Same shape as the merged Mistral and
Upstage Embed implementations.
- `conf/models/novita.json`:
- Add `"embedding": "v3/embeddings"` to `url_suffix`.
- Add default embedding model entries for `baai/bge-m3` and
`baai/bge-large-en-v1.5` so they appear in the model picker.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---------
Co-authored-by: Jin Hai
---
conf/models/novita.json | 10 ++-
internal/entity/models/novita.go | 106 ++++++++++++++++++++++++++++++-
2 files changed, 113 insertions(+), 3 deletions(-)
diff --git a/conf/models/novita.json b/conf/models/novita.json
index c5b3839e140..f95e6849291 100644
--- a/conf/models/novita.json
+++ b/conf/models/novita.json
@@ -5,7 +5,8 @@
},
"url_suffix": {
"chat": "openai/v1/chat/completions",
- "models": "openai/v1/models"
+ "models": "openai/v1/models",
+ "embedding": "openai/v1/embeddings"
},
"class": "novita",
"models": [
@@ -57,6 +58,13 @@
"model_types": [
"chat"
]
+ },
+ {
+ "name": "baai/bge-m3",
+ "max_tokens": 8192,
+ "model_types": [
+ "embedding"
+ ]
}
]
}
diff --git a/internal/entity/models/novita.go b/internal/entity/models/novita.go
index 9eb10ee9874..33e945f6134 100644
--- a/internal/entity/models/novita.go
+++ b/internal/entity/models/novita.go
@@ -626,9 +626,111 @@ func (n *NovitaModel) CheckConnection(apiConfig *APIConfig) error {
return err
}
-// Embed is not exposed on Novita's OpenAI-compatible surface yet.
+type novitaEmbeddingData struct {
+ Embedding []float64 `json:"embedding"`
+ Object string `json:"object"`
+ Index int `json:"index"`
+}
+
+type novitaEmbeddingResponse struct {
+ Data []novitaEmbeddingData `json:"data"`
+ Model string `json:"model"`
+ Object string `json:"object"`
+}
+
+// Embed turns a list of texts into embedding vectors using the Novita
+// /v3/embeddings endpoint. The output has one vector per input, in the
+// same order the inputs were given.
func (n *NovitaModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
- return nil, fmt.Errorf("%s, no such method", n.Name())
+ if len(texts) == 0 {
+ return []EmbeddingData{}, nil
+ }
+
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := n.baseURLForRegion(region)
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, n.URLSuffix.Embedding)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": texts,
+ }
+ if embeddingConfig != nil && embeddingConfig.Dimension > 0 {
+ reqBody["dimensions"] = embeddingConfig.Dimension
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := n.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Novita embeddings API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var parsed novitaEmbeddingResponse
+ if err = json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ embeddings := make([]EmbeddingData, len(texts))
+ filled := make([]bool, len(texts))
+ for _, item := range parsed.Data {
+ if item.Index < 0 || item.Index >= len(texts) {
+ return nil, fmt.Errorf("novita: response index %d out of range for %d inputs", item.Index, len(texts))
+ }
+ if filled[item.Index] {
+ return nil, fmt.Errorf("novita: duplicate embedding index %d in response", item.Index)
+ }
+ embeddings[item.Index] = EmbeddingData{
+ Embedding: item.Embedding,
+ Index: item.Index,
+ }
+ filled[item.Index] = true
+ }
+ for i, ok := range filled {
+ if !ok {
+ return nil, fmt.Errorf("novita: missing embedding for input index %d", i)
+ }
+ }
+
+ return embeddings, nil
}
// Rerank is not exposed by the Novita API.
From fe1433d1ff2e784736498743b5b86286e48f3ab8 Mon Sep 17 00:00:00 2001
From: Jake Armstrong <65635253+jakearmstrong59@users.noreply.github.com>
Date: Sun, 17 May 2026 18:03:12 -1000
Subject: [PATCH 173/666] Go: add Jina chat completions support (#14935)
### What problem does this PR solve?
This PR adds non-streaming chat support for the Jina Go model provider.
The Jina provider was added with embedding, rerank, model listing, and
connection checking, but `ChatWithMessages` still returned a
not-implemented error even though Jina exposes an OpenAI-compatible
`/v1/chat/completions` endpoint.
Closes #14933
**The following functionalities are now supported:**
### **Jina:**
- [x] Chat
- [ ] Stream Chat
- [x] Embedding
- [x] Rerank
- [x] Model listing
- [x] Provider connection checking
- [ ] Balance
### **Implementation details:**
- Implements `JinaModel.ChatWithMessages`
- Sends `Authorization: Bearer ` and JSON chat completion
requests
- Validates API key, model name, messages, and configured region before
making requests
- Forwards supported chat config fields: `max_tokens`, `temperature`,
`top_p`, and `stop`
- Parses the first chat completion choice into `ChatResponse.Answer`
- Adds `jina-ai/jina-vlm` as a chat-capable model in
`conf/models/jina.json`
- Adds focused unit tests for request construction, auth, response
parsing, validation errors, provider errors, and region handling
**Verification:**
```plaintext
docker run --rm -v $PWD:/repo -w /repo golang:1.25 sh -c '/usr/local/go/bin/gofmt -w internal/entity/models/jina.go internal/entity/models/jina_test.go && /usr/local/go/bin/go test -vet=off ./internal/entity/models -run TestJina -count=1'
ok ragflow/internal/entity/models 0.037s
```
Note: `go test ./internal/entity/models -run TestJina -count=1`
currently hits unrelated existing vet findings in other provider files,
so the focused Jina tests were run with `-vet=off`.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---------
Co-authored-by: Jin Hai
---
conf/models/jina.json | 9 +-
internal/entity/models/jina.go | 122 +++++++++++++-
internal/entity/models/jina_test.go | 240 ++++++++++++++++++++++++++++
3 files changed, 368 insertions(+), 3 deletions(-)
create mode 100644 internal/entity/models/jina_test.go
diff --git a/conf/models/jina.json b/conf/models/jina.json
index 07463b6edf5..97069a4b9a8 100644
--- a/conf/models/jina.json
+++ b/conf/models/jina.json
@@ -12,6 +12,13 @@
},
"class": "jina",
"models": [
+ {
+ "name": "jina-vlm",
+ "max_tokens": 32768,
+ "model_types": [
+ "chat"
+ ]
+ },
{
"name": "jina-reranker-v3",
"max_tokens": 134144,
@@ -97,4 +104,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.go
index fffca661285..fca88664a1b 100644
--- a/internal/entity/models/jina.go
+++ b/internal/entity/models/jina.go
@@ -2,6 +2,7 @@ package models
import (
"bytes"
+ "context"
"encoding/json"
"fmt"
"io"
@@ -39,9 +40,126 @@ func (j *JinaModel) Name() string {
return "jina"
}
+func (j *JinaModel) baseURLForRegion(region string) (string, error) {
+ base, ok := j.BaseURL[region]
+ if !ok || base == "" {
+ return "", fmt.Errorf("jina: no base URL configured for region %q", region)
+ }
+ return base, nil
+}
+
func (j *JinaModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
- //TODO implement me: https://api.jina.ai/docs#/Search%20Foundation%20Models/chat_completions_v1_chat_completions_post
- return nil, fmt.Errorf("jina does not implement ChatWithMessages(not available for now)")
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+ if modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := j.baseURLForRegion(region)
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, j.URLSuffix.Chat)
+
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": false,
+ }
+
+ if chatModelConfig != nil {
+ if chatModelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *chatModelConfig.MaxTokens
+ }
+ if chatModelConfig.Temperature != nil {
+ reqBody["temperature"] = *chatModelConfig.Temperature
+ }
+ if chatModelConfig.TopP != nil {
+ reqBody["top_p"] = *chatModelConfig.TopP
+ }
+ if chatModelConfig.Stop != nil {
+ reqBody["stop"] = *chatModelConfig.Stop
+ }
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := j.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Jina chat API error: status %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ var result map[string]interface{}
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ choices, ok := result["choices"].([]interface{})
+ if !ok || len(choices) == 0 {
+ return nil, fmt.Errorf("no choices in response")
+ }
+
+ firstChoice, ok := choices[0].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("invalid choice format")
+ }
+
+ messageMap, ok := firstChoice["message"].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("invalid message format")
+ }
+
+ content, ok := messageMap["content"].(string)
+ if !ok {
+ return nil, fmt.Errorf("invalid content format")
+ }
+
+ reasonContent := ""
+ return &ChatResponse{
+ Answer: &content,
+ ReasonContent: &reasonContent,
+ }, nil
}
func (j *JinaModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
diff --git a/internal/entity/models/jina_test.go b/internal/entity/models/jina_test.go
new file mode 100644
index 00000000000..2ae8a7be86d
--- /dev/null
+++ b/internal/entity/models/jina_test.go
@@ -0,0 +1,240 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func newJinaServer(t *testing.T, expectedPath string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != expectedPath {
+ t.Errorf("expected path=%s, got %s", expectedPath, r.URL.Path)
+ return
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+ t.Errorf("expected Authorization=Bearer test-key, got %q", got)
+ return
+ }
+ if got := r.Header.Get("Content-Type"); got != "application/json" {
+ t.Errorf("expected Content-Type=application/json, got %q", got)
+ return
+ }
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("failed to read body: %v", err)
+ return
+ }
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("invalid JSON body: %v\n%s", err, string(raw))
+ return
+ }
+ handler(t, body, w)
+ }))
+}
+
+func newJinaForTest(baseURL string) *JinaModel {
+ return NewJinaModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{
+ Chat: "chat/completions",
+ Models: "models",
+ Embedding: "embeddings",
+ Rerank: "rerank",
+ },
+ )
+}
+
+func TestJinaChatHappyPath(t *testing.T) {
+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "jina-vlm" {
+ t.Errorf("expected model=jina-vlm, got %v", body["model"])
+ }
+ if body["stream"] != false {
+ t.Errorf("expected stream=false, got %v", body["stream"])
+ }
+ msgs, ok := body["messages"].([]interface{})
+ if !ok || len(msgs) != 1 {
+ t.Errorf("expected 1 message, got %v", body["messages"])
+ return
+ }
+ msg, ok := msgs[0].(map[string]interface{})
+ if !ok || msg["role"] != "user" || msg["content"] != "ping" {
+ t.Errorf("unexpected message payload: %v", msgs[0])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{
+ {"message": map[string]interface{}{"content": "pong"}},
+ },
+ })
+ })
+ defer srv.Close()
+
+ j := newJinaForTest(srv.URL)
+ apiKey := "test-key"
+ resp, err := j.ChatWithMessages("jina-vlm", []Message{{Role: "user", Content: "ping"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if resp.Answer == nil || *resp.Answer != "pong" {
+ t.Errorf("answer=%v, want pong", resp.Answer)
+ }
+ if resp.ReasonContent == nil || *resp.ReasonContent != "" {
+ t.Errorf("expected empty reason content, got %v", resp.ReasonContent)
+ }
+}
+
+func TestJinaChatPropagatesConfig(t *testing.T) {
+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["max_tokens"] != float64(128) {
+ t.Errorf("max_tokens=%v want 128", body["max_tokens"])
+ }
+ if body["temperature"] != 0.2 {
+ t.Errorf("temperature=%v want 0.2", body["temperature"])
+ }
+ if body["top_p"] != 0.8 {
+ t.Errorf("top_p=%v want 0.8", body["top_p"])
+ }
+ stop, ok := body["stop"].([]interface{})
+ if !ok || len(stop) != 1 || stop[0] != "END" {
+ t.Errorf("stop=%v want [END]", body["stop"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{{"message": map[string]interface{}{"content": "ok"}}},
+ })
+ })
+ defer srv.Close()
+
+ j := newJinaForTest(srv.URL)
+ apiKey := "test-key"
+ maxTokens := 128
+ temperature := 0.2
+ topP := 0.8
+ stop := []string{"END"}
+ _, err := j.ChatWithMessages("jina-vlm", []Message{{Role: "user", Content: "ping"}},
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{MaxTokens: &maxTokens, Temperature: &temperature, TopP: &topP, Stop: &stop},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+}
+
+func TestJinaChatValidation(t *testing.T) {
+ j := newJinaForTest("http://unused")
+ apiKey := "test-key"
+ emptyKey := ""
+
+ tests := []struct {
+ name string
+ modelName string
+ messages []Message
+ apiConfig *APIConfig
+ want string
+ }{
+ {
+ name: "missing api config",
+ modelName: "jina-vlm",
+ messages: []Message{{Role: "user", Content: "x"}},
+ want: "api key is required",
+ },
+ {
+ name: "missing api key",
+ modelName: "jina-vlm",
+ messages: []Message{{Role: "user", Content: "x"}},
+ apiConfig: &APIConfig{},
+ want: "api key is required",
+ },
+ {
+ name: "empty api key",
+ modelName: "jina-vlm",
+ messages: []Message{{Role: "user", Content: "x"}},
+ apiConfig: &APIConfig{ApiKey: &emptyKey},
+ want: "api key is required",
+ },
+ {
+ name: "missing model",
+ messages: []Message{{Role: "user", Content: "x"}},
+ apiConfig: &APIConfig{ApiKey: &apiKey},
+ want: "model name is required",
+ },
+ {
+ name: "missing messages",
+ modelName: "jina-vlm",
+ apiConfig: &APIConfig{ApiKey: &apiKey},
+ want: "messages is empty",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := j.ChatWithMessages(tt.modelName, tt.messages, tt.apiConfig, nil)
+ if err == nil || !strings.Contains(err.Error(), tt.want) {
+ t.Fatalf("expected %q error, got %v", tt.want, err)
+ }
+ })
+ }
+}
+
+func TestJinaChatRejectsHTTPError(t *testing.T) {
+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"detail":"invalid api key"}`))
+ })
+ defer srv.Close()
+
+ j := newJinaForTest(srv.URL)
+ apiKey := "test-key"
+ _, err := j.ChatWithMessages("jina-vlm", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "status 401") {
+ t.Errorf("expected 401 propagated, got %v", err)
+ }
+}
+
+func TestJinaChatRejectsMalformedResponse(t *testing.T) {
+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"choices": []map[string]interface{}{}})
+ })
+ defer srv.Close()
+
+ j := newJinaForTest(srv.URL)
+ apiKey := "test-key"
+ _, err := j.ChatWithMessages("jina-vlm", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "no choices in response") {
+ t.Errorf("expected malformed-response error, got %v", err)
+ }
+}
+
+func TestJinaChatRejectsUnknownRegion(t *testing.T) {
+ j := newJinaForTest("http://unused")
+ apiKey := "test-key"
+ region := "eu"
+ _, err := j.ChatWithMessages("jina-vlm", []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey, Region: ®ion}, nil)
+ if err == nil || !strings.Contains(err.Error(), "no base URL configured for region") {
+ t.Errorf("expected region error, got %v", err)
+ }
+}
+
+func TestJinaChatFallsBackToDefaultOnEmptyRegion(t *testing.T) {
+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{{"message": map[string]interface{}{"content": "ok"}}},
+ })
+ })
+ defer srv.Close()
+
+ j := newJinaForTest(srv.URL)
+ apiKey := "test-key"
+ emptyRegion := ""
+ _, err := j.ChatWithMessages("jina-vlm", []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey, Region: &emptyRegion}, nil)
+ if err != nil {
+ t.Errorf("empty Region: expected fallback to default, got %v", err)
+ }
+}
From 2eba2c4d7590d595076aa9180b73fdc9eee84eb8 Mon Sep 17 00:00:00 2001
From: carlos4s <71615127+carlos4s@users.noreply.github.com>
Date: Sun, 17 May 2026 18:03:33 -1000
Subject: [PATCH 174/666] Add Anthropic Go model provider (#14940)
### What problem does this PR solve?
Adds the missing Anthropic provider implementation for the Go model
provider layer.
Closes #14939
### What changed
- Add `conf/models/anthropic.json` with Anthropic Claude chat/vision
models and API endpoints.
- Add `internal/entity/models/anthropic.go` implementing non-streaming
Messages API chat, model listing, and connection checking.
- Register `anthropic` in the Go model factory.
- Add httptest coverage for headers, payload mapping, response parsing,
validation errors, provider errors, model listing, connection checking,
factory registration, and unsupported methods.
### Notes
Streaming chat is left as an explicit `no such method` follow-up because
this initial provider focuses on non-streaming chat and connection
checking.
### Tests
- `docker run --rm -v
/home/ubuntu/Documents/gitTensor_repos/carlos/ragflow:/work -v
/tmp/ragflow-go-cache:/go/pkg/mod -v
/tmp/ragflow-go-build:/root/.cache/go-build -w /work golang:1.25 go test
-vet=off ./internal/entity/models -run Anthropic -count=1 -v`
- `docker run --rm -v
/home/ubuntu/Documents/gitTensor_repos/carlos/ragflow:/work -v
/tmp/ragflow-go-cache:/go/pkg/mod -v
/tmp/ragflow-go-build:/root/.cache/go-build -w /work golang:1.25 go test
-vet=off ./internal/entity -count=1`
- `git diff --check`
- `jq . conf/models/anthropic.json >/dev/null`
Plain `go test ./internal/entity/models` currently hits pre-existing
unrelated vet findings in other provider files (`baidu.go`, `cohere.go`,
`fishaudio.go`, `openrouter.go`).
---------
Co-authored-by: Jin Hai
---
conf/models/anthropic.json | 85 ++++
internal/entity/models/anthropic.go | 491 +++++++++++++++++++++++
internal/entity/models/anthropic_test.go | 383 ++++++++++++++++++
internal/entity/models/factory.go | 2 +
4 files changed, 961 insertions(+)
create mode 100644 conf/models/anthropic.json
create mode 100644 internal/entity/models/anthropic.go
create mode 100644 internal/entity/models/anthropic_test.go
diff --git a/conf/models/anthropic.json b/conf/models/anthropic.json
new file mode 100644
index 00000000000..93cd6b27975
--- /dev/null
+++ b/conf/models/anthropic.json
@@ -0,0 +1,85 @@
+{
+ "name": "Anthropic",
+ "url": {
+ "default": "https://api.anthropic.com"
+ },
+ "url_suffix": {
+ "chat": "v1/messages",
+ "models": "v1/models"
+ },
+ "class": "anthropic",
+ "models": [
+ {
+ "name": "claude-opus-4-5-20251101",
+ "max_tokens": 128000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-opus-4-1-20250805",
+ "max_tokens": 128000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-opus-4-20250514",
+ "max_tokens": 128000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-sonnet-4-5-20250929",
+ "max_tokens": 64000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-sonnet-4-20250514",
+ "max_tokens": 64000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-haiku-4-5-20251001",
+ "max_tokens": 64000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-3-7-sonnet-20250219",
+ "max_tokens": 64000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-3-5-sonnet-20241022",
+ "max_tokens": 8192,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-3-5-haiku-20241022",
+ "max_tokens": 8192,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ }
+ ]
+}
diff --git a/internal/entity/models/anthropic.go b/internal/entity/models/anthropic.go
new file mode 100644
index 00000000000..8777712f7d9
--- /dev/null
+++ b/internal/entity/models/anthropic.go
@@ -0,0 +1,491 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+)
+
+const anthropicVersion = "2023-06-01"
+
+// AnthropicModel implements ModelDriver for Claude models through the
+// Anthropic Messages API.
+type AnthropicModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewAnthropicModel(baseURL map[string]string, urlSuffix URLSuffix) *AnthropicModel {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxIdleConns = 100
+ transport.MaxIdleConnsPerHost = 10
+ transport.IdleConnTimeout = 90 * time.Second
+ transport.ResponseHeaderTimeout = 60 * time.Second
+
+ return &AnthropicModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Transport: transport,
+ },
+ }
+}
+
+func (a *AnthropicModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return NewAnthropicModel(baseURL, a.URLSuffix)
+}
+
+func (a *AnthropicModel) Name() string {
+ return "anthropic"
+}
+
+func (a *AnthropicModel) baseURLForRegion(region string) (string, error) {
+ base, ok := a.BaseURL[region]
+ if !ok || strings.TrimSpace(base) == "" {
+ return "", fmt.Errorf("anthropic: no base URL configured for region %q", region)
+ }
+ return strings.TrimRight(base, "/"), nil
+}
+
+func (a *AnthropicModel) region(apiConfig *APIConfig) string {
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ return *apiConfig.Region
+ }
+ return "default"
+}
+
+func (a *AnthropicModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ apiKey, err := anthropicAPIKey(apiConfig)
+ if err != nil {
+ return nil, err
+ }
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ apiMessages, systemPrompt, err := anthropicMessages(messages)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL, err := a.baseURLForRegion(a.region(apiConfig))
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(a.URLSuffix.Chat, "/"))
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "max_tokens": 1024,
+ }
+ if systemPrompt != "" {
+ reqBody["system"] = systemPrompt
+ }
+ applyAnthropicChatConfig(reqBody, chatModelConfig)
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ setAnthropicHeaders(req, apiKey)
+
+ resp, err := a.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Anthropic messages API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ answer, reasoning, err := parseAnthropicChatResponse(body)
+ if err != nil {
+ return nil, err
+ }
+ return &ChatResponse{
+ Answer: &answer,
+ ReasonContent: &reasoning,
+ }, nil
+}
+
+func anthropicAPIKey(apiConfig *APIConfig) (string, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || strings.TrimSpace(*apiConfig.ApiKey) == "" {
+ return "", fmt.Errorf("api key is required")
+ }
+ return strings.TrimSpace(*apiConfig.ApiKey), nil
+}
+
+func applyAnthropicChatConfig(reqBody map[string]interface{}, chatModelConfig *ChatConfig) {
+ if chatModelConfig == nil {
+ return
+ }
+ if chatModelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *chatModelConfig.MaxTokens
+ }
+ if chatModelConfig.Temperature != nil {
+ reqBody["temperature"] = *chatModelConfig.Temperature
+ }
+ if chatModelConfig.TopP != nil {
+ reqBody["top_p"] = *chatModelConfig.TopP
+ }
+ if chatModelConfig.Stop != nil {
+ reqBody["stop_sequences"] = *chatModelConfig.Stop
+ }
+}
+
+func setAnthropicHeaders(req *http.Request, apiKey string) {
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("x-api-key", apiKey)
+ req.Header.Set("anthropic-version", anthropicVersion)
+}
+
+func anthropicMessages(messages []Message) ([]map[string]interface{}, string, error) {
+ apiMessages := make([]map[string]interface{}, 0, len(messages))
+ systemPrompts := make([]string, 0)
+ for _, msg := range messages {
+ role := strings.ToLower(strings.TrimSpace(msg.Role))
+ content, err := anthropicContent(msg.Content)
+ if err != nil {
+ return nil, "", err
+ }
+ switch role {
+ case "system":
+ if text, ok := anthropicSystemText(content); ok && text != "" {
+ systemPrompts = append(systemPrompts, text)
+ }
+ case "user", "assistant":
+ apiMessages = append(apiMessages, map[string]interface{}{
+ "role": role,
+ "content": content,
+ })
+ default:
+ return nil, "", fmt.Errorf("anthropic: unsupported message role %q", msg.Role)
+ }
+ }
+ if len(apiMessages) == 0 {
+ return nil, "", fmt.Errorf("messages is empty")
+ }
+ return apiMessages, strings.Join(systemPrompts, "\n\n"), nil
+}
+
+func anthropicSystemText(content interface{}) (string, bool) {
+ switch value := content.(type) {
+ case string:
+ return value, true
+ case []map[string]interface{}:
+ parts := make([]string, 0, len(value))
+ for _, block := range value {
+ if block["type"] == "text" {
+ if text, ok := block["text"].(string); ok {
+ parts = append(parts, text)
+ }
+ }
+ }
+ return strings.Join(parts, "\n"), true
+ default:
+ return "", false
+ }
+}
+
+func anthropicContent(content interface{}) (interface{}, error) {
+ switch value := content.(type) {
+ case string:
+ return value, nil
+ case []interface{}:
+ return anthropicContentBlocks(value)
+ case []map[string]interface{}:
+ blocks := make([]interface{}, 0, len(value))
+ for _, block := range value {
+ blocks = append(blocks, block)
+ }
+ return anthropicContentBlocks(blocks)
+ default:
+ return nil, fmt.Errorf("anthropic: unsupported message content type %T", content)
+ }
+}
+
+func anthropicContentBlocks(blocks []interface{}) ([]map[string]interface{}, error) {
+ apiBlocks := make([]map[string]interface{}, 0, len(blocks))
+ for _, item := range blocks {
+ block, ok := item.(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("anthropic: invalid content block %T", item)
+ }
+ converted, err := anthropicContentBlock(block)
+ if err != nil {
+ return nil, err
+ }
+ apiBlocks = append(apiBlocks, converted)
+ }
+ return apiBlocks, nil
+}
+
+func anthropicContentBlock(block map[string]interface{}) (map[string]interface{}, error) {
+ blockType, _ := block["type"].(string)
+ switch blockType {
+ case "text":
+ text, ok := block["text"].(string)
+ if !ok {
+ return nil, fmt.Errorf("anthropic: text block missing or invalid text field %T", block["text"])
+ }
+ return map[string]interface{}{"type": "text", "text": text}, nil
+ case "image":
+ return validateAnthropicImageBlock(block)
+ case "image_url":
+ return anthropicImageURLBlock(block)
+ default:
+ return nil, fmt.Errorf("anthropic: unsupported content block type %q", blockType)
+ }
+}
+
+func validateAnthropicImageBlock(block map[string]interface{}) (map[string]interface{}, error) {
+ source, ok := block["source"].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("anthropic: image block missing source object")
+ }
+ sourceType, ok := source["type"].(string)
+ if !ok || sourceType == "" {
+ return nil, fmt.Errorf("anthropic: image source missing type")
+ }
+ switch sourceType {
+ case "url":
+ if url, ok := source["url"].(string); !ok || url == "" {
+ return nil, fmt.Errorf("anthropic: image url source missing url")
+ }
+ case "base64":
+ mediaType, ok := source["media_type"].(string)
+ if !ok || mediaType == "" {
+ return nil, fmt.Errorf("anthropic: image base64 source missing media_type")
+ }
+ data, ok := source["data"].(string)
+ if !ok || data == "" {
+ return nil, fmt.Errorf("anthropic: image base64 source missing data")
+ }
+ if _, err := base64.StdEncoding.DecodeString(data); err != nil {
+ return nil, fmt.Errorf("anthropic: invalid base64 image data: %w", err)
+ }
+ default:
+ return nil, fmt.Errorf("anthropic: unsupported image source type %q", sourceType)
+ }
+ return block, nil
+}
+
+func anthropicImageURLBlock(block map[string]interface{}) (map[string]interface{}, error) {
+ imageURL, ok := block["image_url"].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("anthropic: image_url block missing image_url object")
+ }
+ url, _ := imageURL["url"].(string)
+ if url == "" {
+ return nil, fmt.Errorf("anthropic: image_url block missing url")
+ }
+ source := map[string]interface{}{
+ "type": "url",
+ "url": url,
+ }
+ if strings.HasPrefix(url, "data:") {
+ mediaType, data, err := parseDataImageURL(url)
+ if err != nil {
+ return nil, err
+ }
+ source = map[string]interface{}{
+ "type": "base64",
+ "media_type": mediaType,
+ "data": data,
+ }
+ }
+ return map[string]interface{}{
+ "type": "image",
+ "source": source,
+ }, nil
+}
+
+func parseDataImageURL(url string) (string, string, error) {
+ const marker = ";base64,"
+ if !strings.HasPrefix(url, "data:") || !strings.Contains(url, marker) {
+ return "", "", fmt.Errorf("anthropic: unsupported data image url")
+ }
+ trimmed := strings.TrimPrefix(url, "data:")
+ parts := strings.SplitN(trimmed, marker, 2)
+ if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+ return "", "", fmt.Errorf("anthropic: invalid data image url")
+ }
+ if _, err := base64.StdEncoding.DecodeString(parts[1]); err != nil {
+ return "", "", fmt.Errorf("anthropic: invalid base64 image data: %w", err)
+ }
+ return parts[0], parts[1], nil
+}
+
+func parseAnthropicChatResponse(body []byte) (string, string, error) {
+ var result struct {
+ Content []struct {
+ Type string `json:"type"`
+ Text string `json:"text"`
+ Thinking string `json:"thinking"`
+ } `json:"content"`
+ }
+ if err := json.Unmarshal(body, &result); err != nil {
+ return "", "", fmt.Errorf("failed to parse response: %w", err)
+ }
+ if len(result.Content) == 0 {
+ return "", "", fmt.Errorf("no content in Anthropic response")
+ }
+
+ var answer strings.Builder
+ var reasoning strings.Builder
+ for _, block := range result.Content {
+ switch block.Type {
+ case "text":
+ answer.WriteString(block.Text)
+ case "thinking":
+ reasoning.WriteString(block.Thinking)
+ }
+ }
+ if answer.Len() == 0 {
+ return "", "", fmt.Errorf("no text content in Anthropic response")
+ }
+ return answer.String(), reasoning.String(), nil
+}
+
+func (a *AnthropicModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ apiKey, err := anthropicAPIKey(apiConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL, err := a.baseURLForRegion(a.region(apiConfig))
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(a.URLSuffix.Models, "/"))
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ setAnthropicHeaders(req, apiKey)
+
+ resp, err := a.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Anthropic models API error: %s, body: %s", resp.Status, string(body))
+ }
+
+ var result struct {
+ Data []struct {
+ ID string `json:"id"`
+ } `json:"data"`
+ }
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+ models := make([]string, 0, len(result.Data))
+ for _, item := range result.Data {
+ if item.ID != "" {
+ models = append(models, item.ID)
+ }
+ }
+ return models, nil
+}
+
+func (a *AnthropicModel) CheckConnection(apiConfig *APIConfig) error {
+ _, err := a.ListModels(apiConfig)
+ return err
+}
+
+func (a *AnthropicModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
+
+func (a *AnthropicModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", a.Name())
+}
diff --git a/internal/entity/models/anthropic_test.go b/internal/entity/models/anthropic_test.go
new file mode 100644
index 00000000000..e2ad0768d23
--- /dev/null
+++ b/internal/entity/models/anthropic_test.go
@@ -0,0 +1,383 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func newAnthropicServer(t *testing.T, expectedPath string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != expectedPath {
+ t.Errorf("expected path=%s, got %s", expectedPath, r.URL.Path)
+ return
+ }
+ if got := r.Header.Get("x-api-key"); got != "test-key" {
+ t.Errorf("expected x-api-key=test-key, got %q", got)
+ return
+ }
+ if got := r.Header.Get("anthropic-version"); got != anthropicVersion {
+ t.Errorf("expected anthropic-version=%s, got %q", anthropicVersion, got)
+ return
+ }
+ if got := r.Header.Get("Content-Type"); !strings.HasPrefix(got, "application/json") {
+ t.Errorf("expected Content-Type to start with application/json, got %q", got)
+ return
+ }
+ if r.Method == http.MethodPost {
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read body: %v", err)
+ return
+ }
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("unmarshal: %v\nraw=%s", err, string(raw))
+ return
+ }
+ handler(t, body, w)
+ return
+ }
+ handler(t, nil, w)
+ }))
+}
+
+func newAnthropicForTest(baseURL string) *AnthropicModel {
+ return NewAnthropicModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{Chat: "v1/messages", Models: "v1/models"},
+ )
+}
+
+func TestAnthropicName(t *testing.T) {
+ if got := newAnthropicForTest("http://unused").Name(); got != "anthropic" {
+ t.Errorf("Name()=%q, want anthropic", got)
+ }
+}
+
+func TestAnthropicChatHappyPath(t *testing.T) {
+ srv := newAnthropicServer(t, "/v1/messages", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "claude-sonnet-4-5-20250929" {
+ t.Errorf("model=%v", body["model"])
+ }
+ if body["max_tokens"] != float64(1024) {
+ t.Errorf("max_tokens=%v want 1024", body["max_tokens"])
+ }
+ msgs, ok := body["messages"].([]interface{})
+ if !ok || len(msgs) != 1 {
+ t.Errorf("messages=%v, want one message", body["messages"])
+ return
+ }
+ msg, ok := msgs[0].(map[string]interface{})
+ if !ok || msg["role"] != "user" || msg["content"] != "ping" {
+ t.Errorf("message=%v, want user ping", msgs[0])
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "content": []map[string]interface{}{
+ {"type": "thinking", "thinking": "reasoning"},
+ {"type": "text", "text": "pong"},
+ },
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ resp, err := newAnthropicForTest(srv.URL).ChatWithMessages(
+ "claude-sonnet-4-5-20250929",
+ []Message{{Role: "user", Content: "ping"}},
+ &APIConfig{ApiKey: &apiKey},
+ nil,
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if resp.Answer == nil || *resp.Answer != "pong" {
+ t.Errorf("answer=%v, want pong", resp.Answer)
+ }
+ if resp.ReasonContent == nil || *resp.ReasonContent != "reasoning" {
+ t.Errorf("reason=%v, want reasoning", resp.ReasonContent)
+ }
+}
+
+func TestAnthropicChatMapsSystemConfigAndImages(t *testing.T) {
+ srv := newAnthropicServer(t, "/v1/messages", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["system"] != "be concise" {
+ t.Errorf("system=%v, want be concise", body["system"])
+ }
+ if body["max_tokens"] != float64(64) {
+ t.Errorf("max_tokens=%v want 64", body["max_tokens"])
+ }
+ if body["temperature"] != 0.25 {
+ t.Errorf("temperature=%v want 0.25", body["temperature"])
+ }
+ if body["top_p"] != 0.8 {
+ t.Errorf("top_p=%v want 0.8", body["top_p"])
+ }
+ stop, ok := body["stop_sequences"].([]interface{})
+ if !ok || len(stop) != 1 || stop[0] != "END" {
+ t.Errorf("stop_sequences=%v want [END]", body["stop_sequences"])
+ }
+ msgs, ok := body["messages"].([]interface{})
+ if !ok || len(msgs) == 0 {
+ t.Errorf("messages=%v, want non-empty array", body["messages"])
+ return
+ }
+ first, ok := msgs[0].(map[string]interface{})
+ if !ok {
+ t.Errorf("first message=%v, want object", msgs[0])
+ return
+ }
+ content, ok := first["content"].([]interface{})
+ if !ok || len(content) < 2 {
+ t.Errorf("content=%v, want at least 2 blocks", first["content"])
+ return
+ }
+ image, ok := content[1].(map[string]interface{})
+ if !ok {
+ t.Errorf("image block=%v, want object", content[1])
+ return
+ }
+ source, ok := image["source"].(map[string]interface{})
+ if !ok {
+ t.Errorf("image source=%v, want object", image["source"])
+ return
+ }
+ if image["type"] != "image" || source["type"] != "url" || source["url"] != "https://example.com/cat.png" {
+ t.Errorf("image block=%v", image)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "content": []map[string]interface{}{{"type": "text", "text": "ok"}},
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ maxTokens := 64
+ temperature := 0.25
+ topP := 0.8
+ stop := []string{"END"}
+ _, err := newAnthropicForTest(srv.URL).ChatWithMessages(
+ "claude-opus-4-5-20251101",
+ []Message{
+ {Role: "system", Content: "be concise"},
+ {Role: "user", Content: []interface{}{
+ map[string]interface{}{"type": "text", "text": "what is this?"},
+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "https://example.com/cat.png"}},
+ }},
+ },
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{MaxTokens: &maxTokens, Temperature: &temperature, TopP: &topP, Stop: &stop},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+}
+
+func TestAnthropicChatMapsDataImageURL(t *testing.T) {
+ srv := newAnthropicServer(t, "/v1/messages", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ msgs, ok := body["messages"].([]interface{})
+ if !ok || len(msgs) == 0 {
+ t.Errorf("messages=%v, want non-empty array", body["messages"])
+ return
+ }
+ first, ok := msgs[0].(map[string]interface{})
+ if !ok {
+ t.Errorf("first message=%v, want object", msgs[0])
+ return
+ }
+ content, ok := first["content"].([]interface{})
+ if !ok || len(content) == 0 {
+ t.Errorf("content=%v, want non-empty array", first["content"])
+ return
+ }
+ image, ok := content[0].(map[string]interface{})
+ if !ok {
+ t.Errorf("image block=%v, want object", content[0])
+ return
+ }
+ source, ok := image["source"].(map[string]interface{})
+ if !ok {
+ t.Errorf("source=%v, want object", image["source"])
+ return
+ }
+ if source["type"] != "base64" || source["media_type"] != "image/png" || source["data"] != "aGVsbG8=" {
+ t.Errorf("source=%v", source)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "content": []map[string]interface{}{{"type": "text", "text": "ok"}},
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ _, err := newAnthropicForTest(srv.URL).ChatWithMessages(
+ "claude-sonnet-4-5-20250929",
+ []Message{{Role: "user", Content: []interface{}{
+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,aGVsbG8="}},
+ }}},
+ &APIConfig{ApiKey: &apiKey},
+ nil,
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+}
+
+func TestAnthropicChatValidationErrors(t *testing.T) {
+ m := newAnthropicForTest("http://unused")
+ apiKey := "test-key"
+ if _, err := m.ChatWithMessages("claude", []Message{{Role: "user", Content: "x"}}, nil, nil); err == nil || !strings.Contains(err.Error(), "api key is required") {
+ t.Errorf("nil api config: got %v", err)
+ }
+ if _, err := m.ChatWithMessages("claude", nil, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "messages is empty") {
+ t.Errorf("empty messages: got %v", err)
+ }
+ if _, err := m.ChatWithMessages("claude", []Message{{Role: "tool", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "unsupported message role") {
+ t.Errorf("bad role: got %v", err)
+ }
+ if _, err := m.ChatWithMessages("claude", []Message{{Role: "user", Content: []interface{}{map[string]interface{}{"type": "video_url"}}}}, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "unsupported content block type") {
+ t.Errorf("bad block: got %v", err)
+ }
+ if _, err := m.ChatWithMessages("claude", []Message{{Role: "user", Content: []interface{}{map[string]interface{}{"type": "text", "text": 42}}}}, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "invalid text field") {
+ t.Errorf("bad text block: got %v", err)
+ }
+ if _, err := m.ChatWithMessages("claude", []Message{{Role: "user", Content: []interface{}{map[string]interface{}{"type": "image"}}}}, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "image block missing source") {
+ t.Errorf("bad image block: got %v", err)
+ }
+}
+
+func TestAnthropicChatRejectsHTTPError(t *testing.T) {
+ srv := newAnthropicServer(t, "/v1/messages", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error":{"message":"bad key"}}`))
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ _, err := newAnthropicForTest(srv.URL).ChatWithMessages("claude", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "401") || !strings.Contains(err.Error(), "bad key") {
+ t.Errorf("expected provider error, got %v", err)
+ }
+}
+
+func TestAnthropicChatRejectsMalformedResponse(t *testing.T) {
+ srv := newAnthropicServer(t, "/v1/messages", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"content": []map[string]interface{}{{"type": "tool_use"}}})
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ _, err := newAnthropicForTest(srv.URL).ChatWithMessages("claude", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "no text content") {
+ t.Errorf("expected no-text error, got %v", err)
+ }
+}
+
+func TestAnthropicListModelsAndCheckConnection(t *testing.T) {
+ var calls int
+ srv := newAnthropicServer(t, "/v1/models", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ calls++
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"id": "claude-sonnet-4-5-20250929"},
+ {"id": "claude-haiku-4-5-20251001"},
+ },
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ m := newAnthropicForTest(srv.URL)
+ models, err := m.ListModels(&APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ if strings.Join(models, ",") != "claude-sonnet-4-5-20250929,claude-haiku-4-5-20251001" {
+ t.Errorf("models=%v", models)
+ }
+ if err := m.CheckConnection(&APIConfig{ApiKey: &apiKey}); err != nil {
+ t.Errorf("CheckConnection: %v", err)
+ }
+ if calls != 2 {
+ t.Errorf("calls=%d, want 2", calls)
+ }
+}
+
+func TestAnthropicListModelsRejectsProviderError(t *testing.T) {
+ srv := newAnthropicServer(t, "/v1/models", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"error":"forbidden"}`))
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ _, err := newAnthropicForTest(srv.URL).ListModels(&APIConfig{ApiKey: &apiKey})
+ if err == nil || !strings.Contains(err.Error(), "403") {
+ t.Errorf("expected 403 error, got %v", err)
+ }
+}
+
+func TestAnthropicFactoryRegistration(t *testing.T) {
+ driver, err := NewModelFactory().CreateModelDriver("Anthropic", map[string]string{"default": "http://unused"}, URLSuffix{})
+ if err != nil {
+ t.Fatalf("CreateModelDriver: %v", err)
+ }
+ if _, ok := driver.(*AnthropicModel); !ok {
+ t.Fatalf("driver type=%T, want *AnthropicModel", driver)
+ }
+}
+
+func TestAnthropicUnsupportedMethods(t *testing.T) {
+ m := newAnthropicForTest("http://unused")
+ apiKey := "test-key"
+ modelName := "claude"
+ checks := []struct {
+ name string
+ err error
+ }{
+ {"stream", m.ChatStreamlyWithSender(modelName, []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil, func(*string, *string) error { return nil })},
+ }
+ for _, check := range checks {
+ if check.err == nil || !strings.Contains(check.err.Error(), "no such method") {
+ t.Errorf("%s: want no such method, got %v", check.name, check.err)
+ }
+ }
+ if _, err := m.Embed(&modelName, []string{"x"}, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Embed: got %v", err)
+ }
+ if _, err := m.Rerank(&modelName, "q", []string{"d"}, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Rerank: got %v", err)
+ }
+ if _, err := m.Balance(&APIConfig{ApiKey: &apiKey}); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Balance: got %v", err)
+ }
+ if _, err := m.TranscribeAudio(&modelName, &modelName, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("TranscribeAudio: got %v", err)
+ }
+ if err := m.TranscribeAudioWithSender(&modelName, &modelName, &APIConfig{ApiKey: &apiKey}, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("TranscribeAudioWithSender: got %v", err)
+ }
+ if _, err := m.AudioSpeech(&modelName, &modelName, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("AudioSpeech: got %v", err)
+ }
+ if err := m.AudioSpeechWithSender(&modelName, &modelName, &APIConfig{ApiKey: &apiKey}, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("AudioSpeechWithSender: got %v", err)
+ }
+ if _, err := m.OCRFile(&modelName, nil, &modelName, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("OCRFile: got %v", err)
+ }
+ if _, err := m.ParseFile(&modelName, nil, &modelName, &APIConfig{ApiKey: &apiKey}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("ParseFile: got %v", err)
+ }
+ if _, err := m.ListTasks(&APIConfig{ApiKey: &apiKey}); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("ListTasks: got %v", err)
+ }
+ if _, err := m.ShowTask("task-id", &APIConfig{ApiKey: &apiKey}); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("ShowTask: got %v", err)
+ }
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index 2df11aec484..b1ca708051c 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -33,6 +33,8 @@ func NewModelFactory() *ModelFactory {
func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string]string, urlSuffix URLSuffix) (ModelDriver, error) {
providerLower := strings.ToLower(providerName)
switch providerLower {
+ case "anthropic":
+ return NewAnthropicModel(baseURL, urlSuffix), nil
case "zhipu-ai":
return NewZhipuAIModel(baseURL, urlSuffix), nil
case "deepseek":
From 9f2fb4611fc8734238c188294b5f21d821583bf9 Mon Sep 17 00:00:00 2001
From: Rene Arredondo <120709323+Rene0422@users.noreply.github.com>
Date: Sun, 17 May 2026 23:11:54 -0700
Subject: [PATCH 175/666] Fix: guard empty/whitespace embedding inputs in
LLMBundle (#14428) (#14924)
Closes #14428
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/db/services/llm_service.py | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/api/db/services/llm_service.py b/api/db/services/llm_service.py
index 60090bb0409..9b6b5bd4f1e 100644
--- a/api/db/services/llm_service.py
+++ b/api/db/services/llm_service.py
@@ -97,7 +97,24 @@ def encode(self, texts: list):
generation = self.langfuse.start_observation(trace_context=self.trace_context, as_type="generation", name="encode", model=self.model_config["llm_name"], input={"texts": texts})
safe_texts = []
- for text in texts:
+ for idx, text in enumerate(texts):
+ # Embedding APIs (OpenAI-compatible, Zhipu, etc.) reject empty or
+ # whitespace-only inputs with errors like "Input at index N cannot
+ # be empty or whitespace only". Upstream parsers can produce such
+ # chunks — e.g. when OCR/vision on an embedded DOCX image returns
+ # nothing, or a table has only empty cells — so coerce to a safe
+ # placeholder here, at the single boundary every embedding path
+ # funnels through.
+ if text is None or not str(text).strip():
+ marker = "None" if text is None else "whitespace-only"
+ logging.warning(
+ "LLMBundle.encode: empty input at index %d (%s) coerced to placeholder 'None' for model %s",
+ idx,
+ marker,
+ self.model_config["llm_name"],
+ )
+ safe_texts.append("None")
+ continue
token_size = num_tokens_from_string(text)
if token_size > self.max_length:
target_len = int(self.max_length * 0.95)
@@ -121,6 +138,14 @@ def encode_queries(self, query: str):
if self.langfuse:
generation = self.langfuse.start_observation(trace_context=self.trace_context, as_type="generation", name="encode_queries", model=self.model_config["llm_name"], input={"query": query})
+ if query is None or not str(query).strip():
+ marker = "None" if query is None else "whitespace-only"
+ logging.warning(
+ "LLMBundle.encode_queries: empty query (%s) coerced to placeholder 'None' for model %s",
+ marker,
+ self.model_config["llm_name"],
+ )
+ query = "None"
emd, used_tokens = self.mdl.encode_queries(query)
if self.model_config["llm_factory"] == "Builtin":
logging.info("LLMBundle.encode_queries query: {}, emd len: {}, used_tokens: {}. Builtin model don't need to update token usage".format(query, len(emd), used_tokens))
From 7cdc74bbe5e0e9e0ea3cbdd8b4264ca7f16b49b2 Mon Sep 17 00:00:00 2001
From: Kevin Hu
Date: Mon, 18 May 2026 14:21:56 +0800
Subject: [PATCH 176/666] Refactor: Drop the vector fetch for ES (#14970)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
- Stop pulling chunk vectors (`q_*_vec`) back from Elasticsearch in the
main retrieval path. ES already knows them; shipping them was pure
bandwidth/memory overhead.
- Recover the per-chunk cosine similarity via a second KNN-only ES call
filtered by the candidate chunk ids. The new `_score` is merged with
locally computed term similarity using the user-configured
`vector_similarity_weight`.
- Lazily fetch the chunk embedding only for the chunks
`insert_citations` actually needs.
## Details
**`rag/nlp/search.py`**
- `Dealer.search`: no longer appends `q_*_vec` to the ES select list.
OceanBase still gets it (its rerank path is unchanged).
- New `Dealer._knn_scores(sres, idx_names, kb_ids)`: a `MatchDenseExpr`
over the cached query vector filtered by `id IN sres.ids`, returning
`{chunk_id: cosine_score}` via ES `_score`.
- New `Dealer.rerank_with_knn(...)`: term similarity from
`qryr.token_similarity` plus the ES-supplied KNN score, combined with
`tkweight`/`vtweight` and the existing rank-feature bonus.
- New `Dealer.fetch_chunk_vectors(chunk_ids, tenant_ids, kb_ids, dim)`:
on-demand vector fetch for citation use.
- `Dealer.retrieval` routes Infinity → unchanged, OceanBase → existing
local `rerank`, ES → new KNN-score path.
**`common/doc_store/es_conn_base.py`**
- New `get_scores(res)` helper returning `{_id: _score}` directly from
hit headers (ES doesn't surface `_score` through `get_fields`).
**`api/db/services/dialog_service.py`**
- New top-level `_hydrate_chunk_vectors(...)` helper. On ES it
back-fills `ck["vector"]` from `fetch_chunk_vectors` right before
`insert_citations`. No-op on Infinity / OB (their chunks already carry
vectors).
- Both `decorate_answer` closures became `async` and are `await`-ed at
all call sites in `async_chat` and `async_ask`.
## Backend behavior
| Backend | Returns chunk vec in main search | Sim source | Vectors for
citations |
|---|---|---|---|
| ES | No | second KNN call (`_score`) merged with term sim | fetched on
demand |
| Infinity | No (unchanged) | normalized `_score` | already on chunks |
| OceanBase | Yes (kept) | local hybrid rerank | already on chunks |
## Test plan
---
api/db/services/dialog_service.py | 63 +++++++++++--
common/doc_store/es_conn_base.py | 15 ++++
rag/nlp/search.py | 142 +++++++++++++++++++++++++++++-
rag/utils/es_conn.py | 3 +-
4 files changed, 212 insertions(+), 11 deletions(-)
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py
index 07dcd14b5ec..f7a5befc3f3 100644
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -65,6 +65,53 @@ def _chunk_kb_id_for_doc(row_dict, kb_ids, doc_id):
return kb_ids[0]
return row_dict.get("kb_id") or row_dict.get("kb_id_kwd")
+
+async def _hydrate_chunk_vectors(retriever, chunks, tenant_ids, kb_ids):
+ """
+ Citation prep: on the ES backend the main retrieval call deliberately
+ skips fetching the chunk embedding. insert_citations needs it, so we
+ pull the vectors for just the candidate chunks right before computing
+ answer-vs-chunk similarity. Chunks without an ES chunk_id (e.g. web
+ search results) keep whatever placeholder they were given. Other
+ backends still carry vectors in the chunk, so we skip the round-trip.
+ """
+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE:
+ return
+ if not chunks:
+ return
+ dim = 0
+ for ck in chunks:
+ v = ck.get("vector")
+ if isinstance(v, list) and v:
+ dim = len(v)
+ break
+ if not dim:
+ return
+ # Skip chunks that already have a non-zero vector (e.g. parent chunks
+ # produced by retrieval_by_children copy the child vector inline).
+ chunk_ids = []
+ for ck in chunks:
+ cid = ck.get("chunk_id")
+ if not cid:
+ continue
+ v = ck.get("vector") or []
+ if any(x for x in v):
+ continue
+ chunk_ids.append(cid)
+ if not chunk_ids:
+ return
+ try:
+ vectors = await retriever.fetch_chunk_vectors(chunk_ids, tenant_ids, kb_ids, dim)
+ except Exception as e: # noqa: BLE001 - degrade gracefully on hydrate failure
+ logger.warning("fetch_chunk_vectors failed; citations will use placeholders: %s", e)
+ return
+ if not vectors:
+ return
+ for ck in chunks:
+ cid = ck.get("chunk_id")
+ if cid and cid in vectors:
+ ck["vector"] = vectors[cid]
+
def _normalize_internet_flag(value):
if isinstance(value, bool):
return value
@@ -735,7 +782,7 @@ async def callback(msg: str):
if "max_tokens" in gen_conf:
gen_conf["max_tokens"] = min(gen_conf["max_tokens"], max_tokens - used_token_count)
- def decorate_answer(answer):
+ async def decorate_answer(answer):
nonlocal embd_mdl, prompt_config, knowledges, kwargs, kbinfos, prompt, retrieval_ts, questions, langfuse_tracer
refs = []
@@ -749,6 +796,9 @@ def decorate_answer(answer):
idx = set([])
normalized_answer = normalize_arabic_digits(answer) or ""
if embd_mdl and not CITATION_MARKER_PATTERN.search(normalized_answer):
+ # Main retrieval no longer ships chunk vectors back from ES.
+ # Pull them on demand for the chunks we are about to cite.
+ await _hydrate_chunk_vectors(retriever, kbinfos.get("chunks", []), tenant_ids, dialog.kb_ids)
answer, idx = retriever.insert_citations(
answer,
[ck["content_ltks"] for ck in kbinfos["chunks"]],
@@ -841,7 +891,7 @@ def decorate_answer(answer):
yield {"answer": value, "reference": {}, "audio_binary": tts(tts_mdl, value), "final": False}
full_answer = last_state.full_text if last_state else ""
if full_answer:
- final = decorate_answer(_extract_visible_answer(thought + full_answer))
+ final = await decorate_answer(_extract_visible_answer(thought + full_answer))
final["final"] = True
final["audio_binary"] = None
yield final
@@ -852,7 +902,7 @@ def decorate_answer(answer):
answer = await chat_mdl.async_chat(prompt + prompt4citation, msg[1:], gen_conf, images=image_files)
user_content = msg[-1].get("content", "[content not available]")
logging.debug("User: {}|Assistant: {}".format(user_content, answer))
- res = decorate_answer(answer)
+ res = await decorate_answer(answer)
res["audio_binary"] = tts(tts_mdl, answer)
yield res
@@ -1542,8 +1592,11 @@ async def async_ask(question, kb_ids, tenant_id, chat_llm_name=None, search_conf
msg = [{"role": "user", "content": question}]
- def decorate_answer(answer):
+ async def decorate_answer(answer):
nonlocal knowledges, kbinfos, sys_prompt
+ # Main retrieval no longer ships chunk vectors back from ES. Pull
+ # them on demand for the chunks we are about to cite.
+ await _hydrate_chunk_vectors(retriever, kbinfos.get("chunks", []), tenant_ids, kb_ids)
answer, idx = retriever.insert_citations(answer, [ck["content_ltks"] for ck in kbinfos["chunks"]], [ck["vector"] for ck in kbinfos["chunks"]], embd_mdl, tkweight=0.7, vtweight=0.3)
idx = set([kbinfos["chunks"][int(i)]["doc_id"] for i in idx])
recall_docs = [d for d in kbinfos["doc_aggs"] if d["doc_id"] in idx]
@@ -1570,7 +1623,7 @@ def decorate_answer(answer):
continue
yield {"answer": value, "reference": {}, "final": False}
full_answer = last_state.full_text if last_state else ""
- final = decorate_answer(_extract_visible_answer(full_answer))
+ final = await decorate_answer(_extract_visible_answer(full_answer))
final["final"] = True
yield final
diff --git a/common/doc_store/es_conn_base.py b/common/doc_store/es_conn_base.py
index 88615649f5f..daa5f17770e 100644
--- a/common/doc_store/es_conn_base.py
+++ b/common/doc_store/es_conn_base.py
@@ -302,6 +302,21 @@ def get_total(self, res):
def get_doc_ids(self, res):
return [d["_id"] for d in res["hits"]["hits"]]
+ def get_scores(self, res) -> dict[str, float]:
+ """
+ Map hit `_id` to its raw `_score`. Used to recover the cosine
+ similarity returned by a KNN-only search without pulling the
+ chunk vectors out of the index.
+ """
+ out = {}
+ for d in res.get("hits", {}).get("hits", []):
+ doc_id = d.get("_id")
+ if doc_id is None:
+ continue
+ score = d.get("_score")
+ out[doc_id] = float(score) if score is not None else 0.0
+ return out
+
def _get_source(self, res):
rr = []
for d in res["hits"]["hits"]:
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 87c1c6682a5..980dba04d93 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -180,7 +180,13 @@ async def search(self, req, idx_names: str | list[str],
else:
matchDense = await self.get_vector(qst, emb_mdl, topk, req.get("similarity", 0.1))
q_vec = matchDense.embedding_data
- if not settings.DOC_ENGINE_INFINITY:
+ # ES path no longer fetches chunk vectors here. The clean
+ # cosine score is recovered later via a second KNN-only call
+ # in retrieval(); chunk vectors are fetched on demand for
+ # citations (see Dealer.fetch_chunk_vectors). OceanBase
+ # still relies on local rerank against chunk vectors, so
+ # keep pulling them for that backend.
+ if settings.DOC_ENGINE_OCEANBASE:
src.append(f"q_{len(q_vec)}_vec")
fusionExpr = FusionExpr("weighted_sum", topk, {"weights": "0.05,0.95"})
@@ -358,6 +364,113 @@ def _rank_feature_scores(self, query_rfea, search_res):
rank_fea.append(nor / np.sqrt(denor) / q_denor)
return np.array(rank_fea) * 10. + pageranks
+ async def _knn_scores(self, sres: "Dealer.SearchResult",
+ idx_names: str | list[str],
+ kb_ids: list[str]) -> dict[str, float]:
+ """
+ Second-pass ES call that returns the cosine similarity between the
+ query embedding and each candidate chunk's embedding, filtered to the
+ chunk ids the original search already surfaced. We rely on ES to do
+ the vector math so the chunk vectors never leave the engine.
+ """
+ if not sres.ids or not sres.query_vector:
+ return {}
+ dim = len(sres.query_vector)
+ matchDense = MatchDenseExpr(
+ f"q_{dim}_vec",
+ sres.query_vector,
+ "float",
+ "cosine",
+ len(sres.ids),
+ {"similarity": 0.0},
+ )
+ condition = {"id": list(sres.ids)}
+ res = await thread_pool_exec(
+ self.dataStore.search,
+ [], # no _source fields needed; we only want _id and _score
+ [],
+ condition,
+ [matchDense],
+ OrderByExpr(),
+ 0,
+ len(sres.ids),
+ idx_names,
+ kb_ids,
+ )
+ return self.dataStore.get_scores(res)
+
+ async def fetch_chunk_vectors(self, chunk_ids: list[str],
+ tenant_ids: str | list[str],
+ kb_ids: list[str],
+ dim: int) -> dict[str, list[float]]:
+ """
+ Citation-time helper: fetch only the embedding vectors for an
+ explicit set of chunk ids. Used by callers that need to compute
+ answer-vs-chunk similarity locally (e.g. insert_citations) so the
+ main retrieval path can keep skipping vector transport.
+ """
+ if not chunk_ids:
+ return {}
+ if isinstance(tenant_ids, str):
+ idx_names = [index_name(tid) for tid in tenant_ids.split(",")]
+ else:
+ idx_names = [index_name(tid) for tid in tenant_ids]
+ vec_field = f"q_{dim}_vec"
+ res = await thread_pool_exec(
+ self.dataStore.search,
+ [vec_field],
+ [],
+ {"id": list(chunk_ids)},
+ [],
+ OrderByExpr(),
+ 0,
+ len(chunk_ids),
+ idx_names,
+ kb_ids,
+ )
+ fields = self.dataStore.get_fields(res, [vec_field])
+ out: dict[str, list[float]] = {}
+ zero = [0.0] * dim
+ for cid, doc in fields.items():
+ v = doc.get(vec_field)
+ if isinstance(v, str):
+ v = [get_float(x) for x in v.split("\t")]
+ if not isinstance(v, list) or len(v) != dim:
+ v = zero
+ out[cid] = v
+ return out
+
+ def rerank_with_knn(self, sres, query, knn_scores: dict[str, float],
+ tkweight=0.3, vtweight=0.7,
+ cfield="content_ltks",
+ rank_feature: dict | None = None):
+ """
+ Merge ES-side KNN cosine similarity with locally computed term
+ similarity using the user-configured weights. Replaces the older
+ local-only rerank() for the ES path, which depended on shipping
+ chunk vectors back to the application.
+ """
+ _, keywords = self.qryr.question(query)
+
+ for i in sres.ids:
+ if isinstance(sres.field[i].get("important_kwd", []), str):
+ sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]
+ ins_tw = []
+ for i in sres.ids:
+ content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))
+ title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]
+ question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]
+ important_kwd = sres.field[i].get("important_kwd", [])
+ tks = content_ltks + title_tks * 2 + important_kwd * 5 + question_tks * 6
+ ins_tw.append(tks)
+
+ tksim = np.array(self.qryr.token_similarity(keywords, ins_tw), dtype=np.float64)
+ vtsim = np.array([knn_scores.get(chunk_id, 0.0) for chunk_id in sres.ids],
+ dtype=np.float64)
+ rank_fea = self._rank_feature_scores(rank_feature, sres)
+ sim = tkweight * tksim + vtweight * vtsim + rank_fea
+ return sim, tksim, vtsim
+
def rerank(self, sres, query, tkweight=0.3,
vtweight=0.7, cfield="content_ltks",
rank_feature: dict | None = None
@@ -491,7 +604,8 @@ async def retrieval(
if isinstance(tenant_ids, str):
tenant_ids = tenant_ids.split(",")
- sres = await self.search(req, [index_name(tid) for tid in tenant_ids], kb_ids, embd_mdl, highlight,
+ idx_names = [index_name(tid) for tid in tenant_ids]
+ sres = await self.search(req, idx_names, kb_ids, embd_mdl, highlight,
rank_feature=rank_feature)
# Temporary retrieval-side guard: prune chunks whose parent document no
# longer exists before reranking and returning results.
@@ -516,8 +630,9 @@ async def retrieval(
sim = [s if s is not None else 0.0 for s in sim]
tsim = sim
vsim = sim
- else:
- # ElasticSearch doesn't normalize each way score before fusion.
+ elif settings.DOC_ENGINE_OCEANBASE:
+ # OceanBase still returns chunk vectors in the result; use
+ # the historical local rerank that depends on them.
sim, tsim, vsim = self.rerank(
sres,
question,
@@ -525,6 +640,20 @@ async def retrieval(
vector_similarity_weight,
rank_feature=rank_feature,
)
+ else:
+ # ES path: ask ES for the clean cosine score via a second
+ # KNN-only call filtered by the candidate ids, then merge it
+ # with locally computed term similarity using the user's
+ # weight. Chunk vectors stay in the index.
+ knn_scores = await self._knn_scores(sres, idx_names, kb_ids)
+ sim, tsim, vsim = self.rerank_with_knn(
+ sres,
+ question,
+ knn_scores,
+ 1 - vector_similarity_weight,
+ vector_similarity_weight,
+ rank_feature=rank_feature,
+ )
sim_np = np.array(sim, dtype=np.float64)
if sim_np.size == 0:
@@ -559,6 +688,11 @@ async def retrieval(
did = chunk.get("doc_id", "")
position_int = chunk.get("position_int", [])
+ # Chunk vectors are no longer fetched during the main retrieval
+ # call. Fall back to whatever the chunk happens to carry (Infinity
+ # path) and otherwise emit a zero placeholder so the downstream
+ # shape stays stable. Citation callers refill this via
+ # Dealer.fetch_chunk_vectors when needed.
d = {
"chunk_id": id,
"content_ltks": chunk["content_ltks"],
diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.py
index 1c80515d682..eed2e67c27c 100644
--- a/rag/utils/es_conn.py
+++ b/rag/utils/es_conn.py
@@ -69,8 +69,7 @@ def _es_search_once(self, index_names: list[str], query: dict, track_total_hits:
index=index_names,
body=query,
timeout="600s",
- track_total_hits=track_total_hits,
- _source=True,
+ track_total_hits=track_total_hits
)
def _search_with_search_after(self, index_names: list[str], query: dict, offset: int, limit: int):
From f1d238357258b792dc8937845e0ab6cfd0473982 Mon Sep 17 00:00:00 2001
From: qinling0210 <88864212+qinling0210@users.noreply.github.com>
Date: Mon, 18 May 2026 14:22:04 +0800
Subject: [PATCH 177/666] Push metadata filters down to Infinity (#14974)
### What problem does this PR solve?
Push metadata filters down to Infinity
### Type of change
- [x] Refactoring
---
api/apps/services/dataset_api_service.py | 1 +
api/db/services/doc_metadata_service.py | 163 +++--
common/metadata_infinity_filter.py | 296 ++++++++
common/metadata_utils.py | 106 +--
rag/prompts/generator.py | 22 +
.../common/test_metadata_es_filter.py | 473 -------------
test/unit_test/common/test_metadata_filter.py | 659 ++++++++++++++++++
7 files changed, 1148 insertions(+), 572 deletions(-)
create mode 100644 common/metadata_infinity_filter.py
delete mode 100644 test/unit_test/common/test_metadata_es_filter.py
create mode 100644 test/unit_test/common/test_metadata_filter.py
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 74b081add30..5927b780ec6 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -1344,6 +1344,7 @@ async def search_datasets(tenant_id: str, req: dict):
chat_mdl = LLMBundle(tenant_id, chat_model_config)
if meta_data_filter:
+ logging.debug(f"Metadata filter: {meta_data_filter}, question: {question}, chat_mdl={'None' if chat_mdl is None else chat_mdl.llm_name}")
local_doc_ids = await apply_meta_data_filter(
meta_data_filter,
None,
diff --git a/api/db/services/doc_metadata_service.py b/api/db/services/doc_metadata_service.py
index 34258c69f56..fbe32f9e5b7 100644
--- a/api/db/services/doc_metadata_service.py
+++ b/api/db/services/doc_metadata_service.py
@@ -404,7 +404,7 @@ def insert_document_metadata(cls, doc_id: str, meta_fields: Dict) -> bool:
)
else:
logging.debug(f"Backend {type(settings.docStoreConn).__name__} has no refresh_idx; skipping")
-
+
logging.debug(f"Successfully inserted metadata for document {doc_id}")
return True
@@ -448,7 +448,8 @@ def update_document_metadata(cls, doc_id: str, meta_fields: Dict) -> bool:
# Post-process to split combined values
processed_meta = cls._split_combined_values(meta_fields)
- logging.debug(f"[update_document_metadata] Updating doc_id: {doc_id}, kb_id: {kb_id}, meta_fields: {processed_meta}")
+ logging.debug(
+ f"[update_document_metadata] Updating doc_id: {doc_id}, kb_id: {kb_id}, meta_fields: {processed_meta}")
# For Elasticsearch, use efficient partial update
if not settings.DOC_ENGINE_INFINITY and not settings.DOC_ENGINE_OCEANBASE:
@@ -456,7 +457,8 @@ def update_document_metadata(cls, doc_id: str, meta_fields: Dict) -> bool:
index_exists = settings.docStoreConn.index_exist(index_name, "")
if not index_exists:
# Index doesn't exist - create it and insert directly
- logging.debug(f"[update_document_metadata] Index {index_name} does not exist, creating and inserting")
+ logging.debug(
+ f"[update_document_metadata] Index {index_name} does not exist, creating and inserting")
result = settings.docStoreConn.create_doc_meta_idx(index_name)
if result is False:
logging.error(f"Failed to create metadata index {index_name}")
@@ -477,7 +479,8 @@ def update_document_metadata(cls, doc_id: str, meta_fields: Dict) -> bool:
# to a backend-provided scripted assignment that fully overwrites it.
replace_meta_fields = getattr(settings.docStoreConn, "replace_meta_fields", None)
if callable(replace_meta_fields) and replace_meta_fields(index_name, doc_id, processed_meta):
- logging.debug(f"Successfully updated metadata for document {doc_id} via {type(settings.docStoreConn).__name__}.replace_meta_fields")
+ logging.debug(
+ f"Successfully updated metadata for document {doc_id} via {type(settings.docStoreConn).__name__}.replace_meta_fields")
return True
logging.warning(
f"replace_meta_fields unavailable or failed on backend "
@@ -537,7 +540,8 @@ def delete_document_metadata(cls, doc_id: str, kb_id: str, tenant_id: str = None
# Check if metadata table exists before attempting deletion
# This is the key optimization - no table = no metadata = nothing to delete
if not settings.docStoreConn.index_exist(index_name, ""):
- logging.debug(f"Metadata table {index_name} does not exist, skipping metadata deletion for document {doc_id}")
+ logging.debug(
+ f"Metadata table {index_name} does not exist, skipping metadata deletion for document {doc_id}")
return True # No metadata to delete is considered success
# Try to get the metadata to confirm it exists before deleting
@@ -627,7 +631,8 @@ def _drop_empty_metadata_table(cls, index_name: str, tenant_id: str) -> None:
if isinstance(results, tuple) and len(results) == 2:
# Infinity returns (DataFrame, int)
df, total = results
- logging.debug(f"[DROP EMPTY TABLE] Infinity format - total: {total}, df length: {len(df) if hasattr(df, '__len__') else 'N/A'}")
+ logging.debug(
+ f"[DROP EMPTY TABLE] Infinity format - total: {total}, df length: {len(df) if hasattr(df, '__len__') else 'N/A'}")
is_empty = (total == 0 or (hasattr(df, '__len__') and len(df) == 0))
elif hasattr(results, 'get') and 'hits' in results:
# ES format - MUST check this before hasattr(results, '__len__')
@@ -791,52 +796,33 @@ def get_flatted_meta_by_kbs(cls, kb_ids: List[str]) -> Dict:
@classmethod
def filter_doc_ids_by_meta_pushdown(
- cls,
- kb_ids: List[str],
- filters: List[Dict],
- logic: str = "and",
- limit: int = 10000,
+ cls,
+ kb_ids: List[str],
+ filters: List[Dict],
+ logic: str = "and",
+ limit: int = 10000,
) -> Optional[List[str]]:
- """Run a metadata filter directly against ES, returning matching doc IDs.
+ """Run a metadata filter directly against ES or Infinity, returning matching doc IDs.
Returns ``None`` to signal "push-down not viable, use the in-memory
``meta_filter`` fallback". Reasons for ``None``:
- - Active doc store is not Elasticsearch (Infinity / OceanBase have
- different filter semantics for the JSON ``meta_fields`` column).
- - One of the user filters cannot be expressed in ES DSL.
- - The ES request itself failed (network, mapping, missing index).
+ - kb_ids or filters is empty
+ - One of the user filters cannot be expressed in ES DSL or Infinity SQL
+ - The request itself failed (network, mapping, missing index)
On success returns the deduplicated, ordered list of document IDs the
- ES query matched. Callers can union or intersect this with their own
+ query matched. Callers can union or intersect this with their own
base ``doc_ids`` rather than fetching the entire metadata table.
"""
- from common.metadata_es_filter import (
- UnsupportedMetaFilter,
- build_meta_filter_query,
- extract_doc_ids,
- is_pushdown_supported,
- )
-
- if not kb_ids:
- return []
-
- if settings.DOC_ENGINE_INFINITY:
- # Infinity stores ``meta_fields`` as a JSON column without dotted
- # field access; the in-memory path is still the reliable answer.
- return None
-
- es_client = getattr(settings.docStoreConn, "es", None)
- if es_client is None:
- return None
-
- if not is_pushdown_supported(filters):
+ if not kb_ids or not filters:
+ logging.debug("Metadata filter skipped: empty kb_ids or filters")
return None
try:
kb = Knowledgebase.get_by_id(kb_ids[0])
except Exception as e:
- logging.warning(f"[meta_pushdown] cannot resolve tenant for kb {kb_ids[0]}: {e}")
+ logging.warning(f"Metadata filter cannot resolve tenant for kb {kb_ids[0]}: {e}")
return None
if not kb:
return None
@@ -844,24 +830,48 @@ def filter_doc_ids_by_meta_pushdown(
tenant_id = kb.tenant_id
index_name = cls._get_doc_meta_index_name(tenant_id)
- try:
- if not settings.docStoreConn.index_exist(index_name, ""):
- # No metadata index → no metadata-filtered docs. Returning an
- # empty list (rather than ``None``) so callers don't bounce
- # back to the in-memory path and re-query MySQL for nothing.
- return []
- except Exception as e:
- logging.warning(f"[meta_pushdown] index_exist check failed for {index_name}: {e}")
+ if not settings.docStoreConn.index_exist(index_name, ""):
+ return []
+
+ if settings.DOC_ENGINE_INFINITY:
+ return cls._filter_doc_ids_by_metadata_infinity(
+ index_name, kb_ids, filters, logic
+ )
+ else:
+ return cls._filter_doc_ids_by_metadata_es(
+ index_name, kb_ids, filters, logic, limit
+ )
+
+ @classmethod
+ def _filter_doc_ids_by_metadata_es(
+ cls,
+ index_name: str,
+ kb_ids: List[str],
+ filters: List[Dict],
+ logic: str,
+ limit: int,
+ ) -> Optional[List[str]]:
+ """ES push-down path for metadata filtering."""
+ from common.metadata_es_filter import (
+ UnsupportedMetaFilter,
+ build_meta_filter_query,
+ extract_doc_ids,
+ is_pushdown_supported,
+ )
+
+ es_client = getattr(settings.docStoreConn, "es", None)
+ if es_client is None:
+ return None
+
+ if not is_pushdown_supported(filters):
return None
try:
query_body = build_meta_filter_query(filters, logic, kb_ids)
except UnsupportedMetaFilter as e:
- logging.debug(f"[meta_pushdown] falling back to in-memory: {e.reason}")
+ logging.error(f"ES build query failed: {e.reason}, filters={filters}")
return None
- # Only the doc id is needed downstream; trimming ``_source`` keeps the
- # response small when the metadata blob is large.
request_body = {
**query_body,
"size": limit,
@@ -871,12 +881,10 @@ def filter_doc_ids_by_meta_pushdown(
try:
response = es_client.search(index=index_name, body=request_body)
except Exception as e:
- logging.warning(f"[meta_pushdown] ES query failed for {index_name}: {e}")
+ logging.error(f"ES metadata filter failed for {index_name}: {e}")
return None
doc_ids = extract_doc_ids(response if isinstance(response, dict) else dict(response))
- # Preserve order while removing duplicates so caller-side de-dupe stays
- # cheap.
seen: set[str] = set()
unique: List[str] = []
for did in doc_ids:
@@ -887,12 +895,52 @@ def filter_doc_ids_by_meta_pushdown(
if len(unique) >= limit:
logging.warning(
- f"[meta_pushdown] hit limit {limit} for KBs {kb_ids}; some matches may be missing"
+ f"ES metadata filter hit limit {limit} for KBs {kb_ids}"
)
- logging.debug(f"[meta_pushdown] {len(unique)} matches for KBs {kb_ids}")
+ logging.debug(f"ES metadata filter returned {len(unique)} matches for KBs {kb_ids}")
return unique
+ @classmethod
+ def _filter_doc_ids_by_metadata_infinity(
+ cls,
+ index_name: str,
+ kb_ids: List[str],
+ filters: List[Dict],
+ logic: str,
+ ) -> Optional[List[str]]:
+ """Infinity push-down path for metadata filtering."""
+ from common.metadata_infinity_filter import (
+ build_infinity_filter,
+ extract_doc_ids,
+ is_pushdown_supported,
+ )
+
+ if not is_pushdown_supported(filters):
+ return None
+
+ try:
+ sql_filter = build_infinity_filter(filters, logic)
+ escaped_kb_ids = [k.replace("'", "''") for k in kb_ids]
+ kb_filter = "kb_id IN (" + ", ".join([f"'{k}'" for k in escaped_kb_ids]) + ")"
+ where_clause = f"{kb_filter} AND {sql_filter}"
+ logging.debug(f"Infinity metadata filter: {where_clause}")
+
+ inf_conn = settings.docStoreConn.connPool.get_conn()
+ try:
+ db_instance = inf_conn.get_database(settings.docStoreConn.dbName)
+ table_instance = db_instance.get_table(index_name)
+ df, _ = table_instance.output(["id"]).filter(where_clause).to_df()
+ doc_ids = extract_doc_ids(df)
+ logging.debug(
+ f"Infinity metadata filter returned {len(doc_ids)} doc IDs for kb_ids={kb_ids}, logic={logic}")
+ return doc_ids
+ finally:
+ settings.docStoreConn.connPool.release_conn(inf_conn)
+ except Exception:
+ logging.warning("Metadata filter push-down failed; falling back to in-memory filter", exc_info=True)
+ return None
+
@classmethod
def get_metadata_keys_by_kbs(cls, kb_ids: List[str]) -> List[str]:
"""
@@ -955,7 +1003,8 @@ def get_metadata_for_documents(cls, doc_ids: Optional[List[str]], kb_id: str) ->
if doc_meta:
meta_mapping[doc_id] = doc_meta
- logging.debug(f"[get_metadata_for_documents] Found metadata for {len(meta_mapping)}/{len(doc_ids) if doc_ids else 'all'} documents")
+ logging.debug(
+ f"[get_metadata_for_documents] Found metadata for {len(meta_mapping)}/{len(doc_ids) if doc_ids else 'all'} documents")
return meta_mapping
except Exception as e:
@@ -981,6 +1030,7 @@ def get_metadata_summary(cls, kb_id: str, doc_ids=None) -> Dict:
}
}
"""
+
def _is_time_string(value: str) -> bool:
"""Check if a string value is an ISO 8601 datetime (e.g., '2026-02-03T00:00:00')."""
if not isinstance(value, str):
@@ -1220,7 +1270,8 @@ def _apply_deletes(meta):
doc_ids_set = set(doc_ids)
missing_doc_ids = doc_ids_set - found_doc_ids
if missing_doc_ids and updates:
- logging.debug(f"[batch_update_metadata] Inserting new metadata for documents without metadata rows: {missing_doc_ids}")
+ logging.debug(
+ f"[batch_update_metadata] Inserting new metadata for documents without metadata rows: {missing_doc_ids}")
for doc_id in missing_doc_ids:
# Apply updates to create new metadata
meta = {}
diff --git a/common/metadata_infinity_filter.py b/common/metadata_infinity_filter.py
new file mode 100644
index 00000000000..076cc2e23e1
--- /dev/null
+++ b/common/metadata_infinity_filter.py
@@ -0,0 +1,296 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Translate RAGflow document-metadata filter lists into Infinity SQL filter expressions.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from typing import Any, Dict, List, Sequence
+
+_KEY_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+
+def _validate_key(key: str, flt: Dict[str, Any]) -> None:
+ if not _KEY_PATTERN.match(key):
+ raise ValueError(f"invalid key format (must be identifier-like): {flt}")
+
+SUPPORTED_OPERATORS: frozenset[str] = frozenset(
+ {
+ "=",
+ "≠",
+ ">",
+ "<",
+ "≥",
+ "≤",
+ "in",
+ "not in",
+ "contains",
+ "not contains",
+ "start with",
+ "end with",
+ "empty",
+ "not empty",
+ }
+)
+
+_RANGE_OPS: Dict[str, str] = {
+ ">": ">",
+ "<": "<",
+ "≥": ">=",
+ "≤": "<=",
+}
+
+class MetaFilterTranslator:
+ """Translate one user filter clause at a time into Infinity SQL filter strings."""
+
+ def translate(self, flt: Dict[str, Any]) -> str:
+ op = flt.get("op")
+ key = flt.get("key")
+ value = flt.get("value")
+
+ if not key or not isinstance(key, str):
+ raise ValueError(f"filter is missing a string key: {flt}")
+ _validate_key(key, flt)
+ if op not in SUPPORTED_OPERATORS:
+ raise ValueError(f"unknown operator: {op!r}, filter: {flt}")
+
+ if op == "empty":
+ return self._translate_empty(key)
+ if op == "not empty":
+ return self._translate_not_empty(key)
+ if op == "=":
+ return self._translate_equal(key, value, flt)
+ if op == "≠":
+ return self._translate_not_equal(key, value, flt)
+ if op in _RANGE_OPS:
+ return self._translate_range(key, op, value, flt)
+ if op == "in":
+ return self._translate_in(key, value, flt)
+ if op == "not in":
+ return self._translate_not_in(key, value, flt)
+ if op == "contains":
+ return self._translate_contains(key, value, flt)
+ if op == "not contains":
+ return self._translate_not_contains(key, value, flt)
+ if op == "start with":
+ return self._translate_start_with(key, value, flt)
+ if op == "end with":
+ return self._translate_end_with(key, value, flt)
+
+ raise ValueError(f"no handler for operator: {op!r}, filter: {flt}")
+
+ def _translate_empty(self, key: str) -> str:
+ return f"JSON_EXTRACT_STRING(meta_fields, '$.{key}') = '\"\"'"
+
+ def _translate_not_empty(self, key: str) -> str:
+ return f"JSON_EXTRACT_STRING(meta_fields, '$.{key}') != '\"\"'"
+
+ def _translate_equal(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ coerced = _coerce_scalar(value, flt)
+ if isinstance(coerced, str):
+ escaped = _escape_sql_string(coerced)
+ return f"JSON_CONTAINS(meta_fields, '$.{key}', '\"{escaped}\"')"
+ return f"JSON_CONTAINS(meta_fields, '$.{key}', {coerced})"
+
+ def _translate_not_equal(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ coerced = _coerce_scalar(value, flt)
+ if isinstance(coerced, str):
+ escaped = _escape_sql_string(coerced)
+ return f"NOT JSON_CONTAINS(meta_fields, '$.{key}', '\"{escaped}\"')"
+ return f"NOT JSON_CONTAINS(meta_fields, '$.{key}', {coerced})"
+
+ def _translate_range(self, key: str, op: str, value: Any, flt: Dict[str, Any]) -> str:
+ coerced = _coerce_range_value(value, flt)
+ sql_op = _RANGE_OPS.get(op, op)
+ if isinstance(coerced, str):
+ escaped = _escape_sql_string(coerced)
+ return f"JSON_EXTRACT_STRING(meta_fields, '$.{key}') {sql_op} '{escaped}'"
+ return f"JSON_EXTRACT_DOUBLE(meta_fields, '$.{key}') {sql_op} {coerced}"
+
+ def _translate_in(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ members = _csv_or_list(value, flt)
+ string_parts = []
+ num_parts = []
+ for m in members:
+ # Use same coercion as range operators to detect numeric values
+ coerced = _coerce_range_value(m, flt)
+ if isinstance(coerced, (int, float)):
+ num_parts.append(f"JSON_CONTAINS(meta_fields, '$.{key}', {coerced})")
+ else:
+ escaped = _escape_sql_string(coerced)
+ string_parts.append(f"JSON_CONTAINS(meta_fields, '$.{key}', '\"{escaped}\"')")
+ conditions = []
+ if string_parts:
+ conditions.append("(" + " OR ".join(string_parts) + ")")
+ if num_parts:
+ conditions.append("(" + " OR ".join(num_parts) + ")")
+ return "(" + " OR ".join(conditions) + ")"
+
+ def _translate_not_in(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ members = _csv_or_list(value, flt)
+ string_parts = []
+ num_parts = []
+ for m in members:
+ # Use same coercion as range operators to detect numeric values
+ coerced = _coerce_range_value(m, flt)
+ if isinstance(coerced, (int, float)):
+ num_parts.append(f"NOT JSON_CONTAINS(meta_fields, '$.{key}', {coerced})")
+ else:
+ escaped = _escape_sql_string(coerced)
+ string_parts.append(f"NOT JSON_CONTAINS(meta_fields, '$.{key}', '\"{escaped}\"')")
+ conditions = []
+ if string_parts:
+ conditions.append("(" + " AND ".join(string_parts) + ")")
+ if num_parts:
+ conditions.append("(" + " AND ".join(num_parts) + ")")
+ return " AND ".join(conditions)
+
+ def _translate_contains(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ if not value and value != 0:
+ raise ValueError(f"contains value is empty: {flt}")
+ # Use same coercion as range operators to detect numeric values
+ coerced = _coerce_range_value(value, flt)
+ if isinstance(coerced, (int, float)):
+ return f"JSON_CONTAINS(meta_fields, '$.{key}', {coerced})"
+ escaped = _escape_sql_string(str(value))
+ return f"JSON_CONTAINS(meta_fields, '$.{key}', '\"{escaped}\"')"
+
+ def _translate_not_contains(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ text = _coerce_string(value, flt)
+ escaped = _escape_sql_string(text)
+ # Use Infinity's JSON_CONTAINS to check if value does NOT exist in JSON array
+ return f"NOT JSON_CONTAINS(meta_fields, '$.{key}', '\"{escaped}\"')"
+
+ def _translate_start_with(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ text = _coerce_string(value, flt)
+ escaped = _escape_sql_string(_escape_likeWildcards(text))
+ return f"JSON_EXTRACT_STRING(meta_fields, '$.{key}') LIKE '{escaped}%'"
+
+ def _translate_end_with(self, key: str, value: Any, flt: Dict[str, Any]) -> str:
+ text = _coerce_string(value, flt)
+ escaped = _escape_sql_string(_escape_likeWildcards(text))
+ return f"JSON_EXTRACT_STRING(meta_fields, '$.{key}') LIKE '%{escaped}'"
+
+
+def plan_pushdown(filters: Sequence[Dict[str, Any]], logic: str) -> List[str]:
+ if logic not in {"and", "or"}:
+ raise ValueError(f"unknown logic {logic!r}")
+ translator = MetaFilterTranslator()
+ return [translator.translate(flt) for flt in filters]
+
+
+def build_infinity_filter(filters: Sequence[Dict[str, Any]], logic: str) -> str:
+ if not filters:
+ return "1=1"
+ fragments = plan_pushdown(filters, logic)
+ joiner = " AND " if logic == "and" else " OR "
+ result = "(" + joiner.join(fragments) + ")"
+ return result
+
+
+def is_pushdown_supported(filters: Sequence[Dict[str, Any]]) -> bool:
+ for flt in filters:
+ op = flt.get("op")
+ if op not in SUPPORTED_OPERATORS:
+ return False
+ if not isinstance(flt.get("key"), str) or not flt.get("key"):
+ return False
+ return True
+
+
+def extract_doc_ids(df) -> List[str]:
+ if df is None or not hasattr(df, "iterrows"):
+ return []
+ return [str(row["id"]) for _, row in df.iterrows() if "id" in row]
+
+
+# ---------------------------------------------------------------------------
+# Value coercion helpers
+# ---------------------------------------------------------------------------
+
+
+def _coerce_scalar(value: Any, flt: Dict[str, Any]) -> Any:
+ if value is None:
+ raise ValueError(f"scalar comparison value is None: {flt}")
+ if isinstance(value, (list, dict)):
+ raise ValueError(f"scalar comparison value is non-scalar: {flt}")
+ try:
+ parsed = ast.literal_eval(str(value).strip())
+ if isinstance(parsed, (int, float, bool)):
+ return parsed
+ except Exception:
+ pass
+ return str(value)
+
+
+def _coerce_range_value(value: Any, flt: Dict[str, Any]) -> Any:
+ if value is None:
+ raise ValueError(f"range comparison value is None: {flt}")
+ try:
+ parsed = ast.literal_eval(str(value).strip())
+ if isinstance(parsed, (int, float)):
+ return parsed
+ except Exception:
+ pass
+ return str(value)
+
+
+def _coerce_string(value: Any, flt: Dict[str, Any]) -> str:
+ if value is None:
+ raise ValueError(f"string-operator value is None: {flt}")
+ if isinstance(value, (list, dict)):
+ raise ValueError(f"string-operator value must be a scalar: {flt}")
+ s = str(value)
+ if not s:
+ raise ValueError(f"string-operator value is empty: {flt}")
+ return s
+
+
+def _csv_or_list(value: Any, flt: Dict[str, Any]) -> List[Any]:
+ if value is None:
+ raise ValueError(f"membership value is None: {flt}")
+ if isinstance(value, (list, tuple)):
+ members = list(value)
+ elif isinstance(value, str):
+ try:
+ parsed = ast.literal_eval(value)
+ except Exception:
+ parsed = value
+ if isinstance(parsed, (list, tuple)):
+ members = list(parsed)
+ else:
+ members = [m.strip() for m in value.split(",") if m.strip()]
+ else:
+ members = [value]
+ if not members:
+ raise ValueError(f"membership value resolved to empty list: {flt}")
+ normalised: List[Any] = []
+ for m in members:
+ if isinstance(m, str):
+ normalised.append(m.lower().strip())
+ else:
+ normalised.append(m)
+ return normalised
+
+
+def _escape_sql_string(s: str) -> str:
+ return s.replace("'", "''")
+
+
+def _escape_likeWildcards(text: str) -> str:
+ return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
\ No newline at end of file
diff --git a/common/metadata_utils.py b/common/metadata_utils.py
index c2fc90b5414..53af2b4eaf3 100644
--- a/common/metadata_utils.py
+++ b/common/metadata_utils.py
@@ -19,6 +19,7 @@
import json_repair
+
def convert_conditions(metadata_condition):
if metadata_condition is None:
metadata_condition = {}
@@ -60,21 +61,21 @@ def filter_out(v2docs, operator, value):
# Strict date format detection: YYYY-MM-DD (must be 10 chars with correct format)
is_input_date = (
- len(input_str) == 10 and
- input_str[4] == '-' and
- input_str[7] == '-' and
- input_str[:4].isdigit() and
- input_str[5:7].isdigit() and
- input_str[8:10].isdigit()
+ len(input_str) == 10 and
+ input_str[4] == '-' and
+ input_str[7] == '-' and
+ input_str[:4].isdigit() and
+ input_str[5:7].isdigit() and
+ input_str[8:10].isdigit()
)
is_value_date = (
- len(value_str) == 10 and
- value_str[4] == '-' and
- value_str[7] == '-' and
- value_str[:4].isdigit() and
- value_str[5:7].isdigit() and
- value_str[8:10].isdigit()
+ len(value_str) == 10 and
+ value_str[4] == '-' and
+ value_str[7] == '-' and
+ value_str[:4].isdigit() and
+ value_str[5:7].isdigit() and
+ value_str[8:10].isdigit()
)
if is_value_date:
@@ -109,17 +110,23 @@ def filter_out(v2docs, operator, value):
matched = False
try:
if operator == "contains":
- matched = str(input).find(value) >= 0 if not isinstance(input, list) else any(str(i).find(value) >= 0 for i in input)
+ matched = str(input).find(value) >= 0 if not isinstance(input, list) else any(
+ str(i).find(value) >= 0 for i in input)
elif operator == "not contains":
- matched = str(input).find(value) == -1 if not isinstance(input, list) else all(str(i).find(value) == -1 for i in input)
+ matched = str(input).find(value) == -1 if not isinstance(input, list) else all(
+ str(i).find(value) == -1 for i in input)
elif operator == "in":
matched = input in value if not isinstance(input, list) else all(i in value for i in input)
elif operator == "not in":
matched = input not in value if not isinstance(input, list) else all(i not in value for i in input)
elif operator == "start with":
- matched = str(input).lower().startswith(str(value).lower()) if not isinstance(input, list) else "".join([str(i).lower() for i in input]).startswith(str(value).lower())
+ matched = str(input).lower().startswith(str(value).lower()) if not isinstance(input,
+ list) else "".join(
+ [str(i).lower() for i in input]).startswith(str(value).lower())
elif operator == "end with":
- matched = str(input).lower().endswith(str(value).lower()) if not isinstance(input, list) else "".join([str(i).lower() for i in input]).endswith(str(value).lower())
+ matched = str(input).lower().endswith(str(value).lower()) if not isinstance(input,
+ list) else "".join(
+ [str(i).lower() for i in input]).endswith(str(value).lower())
elif operator == "empty":
matched = not input
elif operator == "not empty":
@@ -158,21 +165,23 @@ def filter_out(v2docs, operator, value):
if logic == "and":
doc_ids = doc_ids & set(ids)
if not doc_ids:
+ logging.debug(f"meta_filter filters={filters}, logic={logic}, early return []")
return []
else:
doc_ids = doc_ids | set(ids)
+ logging.debug(f"meta_filter filters={filters}, logic={logic}, returning doc_ids={list(doc_ids)}")
return list(doc_ids)
async def apply_meta_data_filter(
- meta_data_filter: dict | None,
- metas: dict | None = None,
- question: str = "",
- chat_mdl: Any = None,
- base_doc_ids: list[str] | None = None,
- manual_value_resolver: Callable[[dict], dict] | None = None,
- kb_ids: list[str] | None = None,
- metas_loader: Callable[[], dict] | None = None,
+ meta_data_filter: dict | None,
+ metas: dict | None = None,
+ question: str = "",
+ chat_mdl: Any = None,
+ base_doc_ids: list[str] | None = None,
+ manual_value_resolver: Callable[[dict], dict] | None = None,
+ kb_ids: list[str] | None = None,
+ metas_loader: Callable[[], dict] | None = None,
) -> list[str] | None:
"""
Apply metadata filtering rules and return the filtered doc_ids.
@@ -182,12 +191,11 @@ async def apply_meta_data_filter(
- semi_auto: generate conditions using selected metadata keys only
- manual: directly filter based on provided conditions
- When ``kb_ids`` is supplied and the active doc store is Elasticsearch the
- generated filter conditions are pushed down to ES via
- ``DocMetadataService.filter_doc_ids_by_meta_pushdown`` instead of being
- evaluated in Python over ``metas``. The in-memory ``meta_filter`` path
- remains the fallback so callers without a KB scope, or backends without
- push-down support, behave exactly as before.
+ When ``kb_ids`` is supplied, metadata filters are pushed down to the doc metadata
+ index (ES/Infinity) via ``DocMetadataService.filter_doc_ids_by_metadata`` instead
+ of being evaluated in Python over ``metas``. The in-memory ``meta_filter`` path
+ remains the fallback so callers without a KB scope, or backends without push-down
+ support, behave exactly as before.
``metas`` may be supplied eagerly or via ``metas_loader``. The loader is
only invoked when the metadata dict is actually needed — i.e. for the LLM
@@ -200,7 +208,7 @@ async def apply_meta_data_filter(
list of doc_ids, ["-999"] when manual filters yield no result, or None
when auto/semi_auto filters return empty.
"""
- from rag.prompts.generator import gen_meta_filter # move from the top of the file to avoid circular import
+ from rag.prompts.generator import gen_meta_filter # move from the top of the file to avoid circular import
doc_ids = list(base_doc_ids) if base_doc_ids else []
@@ -220,17 +228,26 @@ def _get_metas() -> dict:
cached_metas = metas_loader() if metas_loader else {}
return cached_metas
- def _evaluate(conditions: list[dict], logic: str) -> list[str]:
- """Run conditions through ES push-down when possible, in-memory otherwise."""
+ def _run_metadata_filter(conditions: list[dict], logic: str) -> list[str]:
+ """Run conditions through ES/Infinity push-down when possible, in-memory otherwise."""
if conditions and kb_ids:
- pushed = _try_meta_pushdown(kb_ids, conditions, logic)
- if pushed is not None:
- return pushed
+ try:
+ from api.db.services.doc_metadata_service import DocMetadataService
+ doc_ids = DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)
+ logging.debug(f"Doc ids filtered by metadata: {doc_ids}")
+ if doc_ids is not None:
+ return doc_ids
+ except Exception as e:
+ logging.error(f"Metadata filter push down errored: {e}")
+
+ # In-memory fallback
+ logging.debug("Metadata filter falls back to in-memory filter")
return meta_filter(_get_metas(), conditions, logic)
if method == "auto":
filters: dict = await gen_meta_filter(chat_mdl, _get_metas(), question)
- doc_ids.extend(_evaluate(filters["conditions"], filters.get("logic", "and")))
+ logging.debug(f"Metadata filter(auto) generated: {filters}")
+ doc_ids.extend(_run_metadata_filter(filters["conditions"], filters.get("logic", "and")))
if not doc_ids:
return None
elif method == "semi_auto":
@@ -251,24 +268,27 @@ def _evaluate(conditions: list[dict], logic: str) -> list[str]:
filtered_metas = {key: current_metas[key] for key in selected_keys if key in current_metas}
if filtered_metas:
filters: dict = await gen_meta_filter(chat_mdl, filtered_metas, question, constraints=constraints)
- doc_ids.extend(_evaluate(filters["conditions"], filters.get("logic", "and")))
+ logging.debug(f"Metadata filter(semi_auto) generated: {filters}")
+ doc_ids.extend(_run_metadata_filter(filters["conditions"], filters.get("logic", "and")))
if not doc_ids:
return None
elif method == "manual":
filters = meta_data_filter.get("manual", [])
if manual_value_resolver:
filters = [manual_value_resolver(flt) for flt in filters]
- doc_ids.extend(_evaluate(filters, meta_data_filter.get("logic", "and")))
+ logging.debug(f"Metadata filter(manual): {filters}")
+ doc_ids.extend(_run_metadata_filter(filters, meta_data_filter.get("logic", "and")))
if filters and not doc_ids:
doc_ids = ["-999"]
+ logging.debug(f"apply_meta_data_filter meta_filter={meta_data_filter}, returning doc_ids={doc_ids}")
return doc_ids
def _try_meta_pushdown(
- kb_ids: list[str],
- conditions: list[dict],
- logic: str,
+ kb_ids: list[str],
+ conditions: list[dict],
+ logic: str,
) -> list[str] | None:
"""Attempt the ES push-down path; return ``None`` to fall back in-memory.
@@ -335,7 +355,7 @@ def update_metadata_to(metadata, meta):
return metadata
-def metadata_schema(metadata: dict|list|None) -> Dict[str, Any]:
+def metadata_schema(metadata: dict | list | None) -> Dict[str, Any]:
if not metadata:
return {}
properties = {}
diff --git a/rag/prompts/generator.py b/rag/prompts/generator.py
index b55e7a4c912..fc4999dbe45 100644
--- a/rag/prompts/generator.py
+++ b/rag/prompts/generator.py
@@ -494,6 +494,28 @@ async def rank_memories_async(chat_mdl, goal: str, sub_goal: str, tool_call_summ
async def gen_meta_filter(chat_mdl, meta_data: dict, query: str, constraints: dict = None) -> dict:
+ """Generate metadata filter conditions from a user query using an LLM.
+
+ Args:
+ chat_mdl: LLM bundle for generating filters
+ meta_data: Dict of {key: set of values} - e.g. {"character": {"Caocao", "Liubei"}, "year": {2026}}
+ query: User question (e.g. "Caocao in 2026")
+ constraints: Optional dict of {key: operator} to constrain which op to use for a key
+
+ Returns:
+ Dict with "logic" ("and"/"or") and "conditions" list.
+ Example return value:
+ {
+ "logic": "and",
+ "conditions": [
+ {"key": "year", "value": "2026", "op": "="},
+ {"key": "character", "value": "Caocao", "op": "="}
+ ]
+ }
+
+ The LLM is prompted with the available metadata keys and values, and is asked to
+ generate filter conditions that match the user's query semantics.
+ """
meta_data_structure = {}
for key, values in meta_data.items():
meta_data_structure[key] = list(values.keys()) if isinstance(values, dict) else values
diff --git a/test/unit_test/common/test_metadata_es_filter.py b/test/unit_test/common/test_metadata_es_filter.py
deleted file mode 100644
index eb8217909e3..00000000000
--- a/test/unit_test/common/test_metadata_es_filter.py
+++ /dev/null
@@ -1,473 +0,0 @@
-"""Unit tests for the Elasticsearch push-down translator.
-
-These tests cover the public surface of ``common.metadata_es_filter`` without
-touching the live ES cluster. They verify the shape of the produced query DSL
-operator-by-operator and confirm that the parity rules with the in-memory
-``meta_filter`` (lower-casing, list-membership coercion, date detection) hold.
-"""
-
-import pytest
-
-from common.metadata_es_filter import (
- META_FIELDS_PREFIX,
- MetaFilterPushdownPlan,
- MetaFilterTranslator,
- SUPPORTED_OPERATORS,
- UnsupportedMetaFilter,
- build_meta_filter_query,
- extract_doc_ids,
- is_pushdown_supported,
- plan_pushdown,
-)
-
-
-# ---------------------------------------------------------------------------
-# Fixtures
-# ---------------------------------------------------------------------------
-
-
-@pytest.fixture
-def translator() -> MetaFilterTranslator:
- return MetaFilterTranslator()
-
-
-def _field(key: str) -> str:
- return f"{META_FIELDS_PREFIX}.{key}"
-
-
-# ---------------------------------------------------------------------------
-# Translator: per-operator shape
-# ---------------------------------------------------------------------------
-
-
-def test_equal_translates_to_term_with_lowercased_value(translator):
- """String equality runs against ``.keyword`` so multi-word phrases match.
-
- Querying the analyzed parent field with ``term`` only matches docs whose
- inverted index contains the literal phrase token, which never happens for
- multi-word values. The ``.keyword`` sub-field stores the unmodified string,
- and ``case_insensitive: true`` keeps the lower-cased compare semantics from
- the in-memory ``meta_filter``.
- """
- clauses = translator.translate({"key": "tag", "op": "=", "value": "Alpha"}).to_clauses()
- assert clauses == [
- {"term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}}
- ]
-
-
-def test_equal_parses_numeric_literal(translator):
- """Numeric values stay on the parent path — no ``.keyword`` sub-field exists for ``long``."""
- clauses = translator.translate({"key": "score", "op": "=", "value": "5"}).to_clauses()
- assert clauses == [{"term": {_field("score"): 5}}]
-
-
-def test_equal_multiword_uses_keyword_subfield(translator):
- """Regression for qinling0210's report: multi-word string values must match.
-
- Before the keyword-routing fix this emitted
- ``term: meta_fields.author = "alice wonderland"`` against an analyzed text
- field, which never matched (inverted index only contained per-token
- entries). Routing through ``.keyword`` preserves the full phrase.
- """
- clauses = translator.translate(
- {"key": "author", "op": "=", "value": "Alice Wonderland"}
- ).to_clauses()
- assert clauses == [
- {
- "term": {
- _field("author") + ".keyword": {
- "value": "alice wonderland",
- "case_insensitive": True,
- }
- }
- }
- ]
-
-
-def test_not_equal_requires_field_to_exist(translator):
- clauses = translator.translate({"key": "tag", "op": "≠", "value": "alpha"}).to_clauses()
- assert clauses == [
- {
- "bool": {
- "must": [{"exists": {"field": _field("tag")}}],
- "must_not": [
- {"term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}}
- ],
- }
- }
- ]
-
-
-@pytest.mark.parametrize(
- "op,es_key",
- [(">", "gt"), ("<", "lt"), ("≥", "gte"), ("≤", "lte")],
-)
-def test_range_operator_translation(translator, op, es_key):
- # Multi-clause positive filters wrap into a single bool so OR-logic
- # parents can't match on just the ``exists`` half of the range.
- clauses = translator.translate({"key": "score", "op": op, "value": "10"}).to_clauses()
- assert clauses == [
- {
- "bool": {
- "must": [
- {"exists": {"field": _field("score")}},
- {"range": {_field("score"): {es_key: 10}}},
- ]
- }
- }
- ]
-
-
-def test_range_passes_iso_date_through_unparsed(translator):
- clauses = translator.translate({"key": "published", "op": "≥", "value": "2025-01-15"}).to_clauses()
- range_clause = clauses[0]["bool"]["must"][1]
- assert range_clause == {"range": {_field("published"): {"gte": "2025-01-15"}}}
-
-
-def _string_terms_should(field_path: str, members):
- """``in``/``not in`` over string members expands per-element so each ``term``
- can carry ``case_insensitive`` (``terms`` does not accept that flag)."""
- return {
- "bool": {
- "should": [
- {"term": {field_path + ".keyword": {"value": m, "case_insensitive": True}}}
- for m in members
- ],
- "minimum_should_match": 1,
- }
- }
-
-
-def test_in_operator_csv_value_lowercased(translator):
- clauses = translator.translate({"key": "status", "op": "in", "value": "Active,Pending"}).to_clauses()
- assert clauses == [_string_terms_should(_field("status"), ["active", "pending"])]
-
-
-def test_in_operator_python_list_literal(translator):
- clauses = translator.translate({"key": "status", "op": "in", "value": "['Open', 'Closed']"}).to_clauses()
- assert clauses == [_string_terms_should(_field("status"), ["open", "closed"])]
-
-
-def test_in_operator_numeric_members_keep_terms(translator):
- """All-numeric member lists keep the cheaper ``terms`` form on the parent path."""
- clauses = translator.translate({"key": "year", "op": "in", "value": "[2024, 2025]"}).to_clauses()
- assert clauses == [{"terms": {_field("year"): [2024, 2025]}}]
-
-
-def test_not_in_negates_with_existence_guard(translator):
- clauses = translator.translate({"key": "status", "op": "not in", "value": "active,pending"}).to_clauses()
- assert clauses == [
- {
- "bool": {
- "must": [{"exists": {"field": _field("status")}}],
- "must_not": [_string_terms_should(_field("status"), ["active", "pending"])],
- }
- }
- ]
-
-
-def test_contains_uses_case_insensitive_wildcard(translator):
- clauses = translator.translate({"key": "version", "op": "contains", "value": "earth"}).to_clauses()
- assert clauses == [
- {
- "wildcard": {
- _field("version") + ".keyword": {
- "value": "*earth*",
- "case_insensitive": True,
- }
- }
- }
- ]
-
-
-def test_contains_escapes_user_wildcards(translator):
- clauses = translator.translate({"key": "title", "op": "contains", "value": "a*b?c"}).to_clauses()
- pattern = clauses[0]["wildcard"][_field("title") + ".keyword"]["value"]
- assert pattern == "*a\\*b\\?c*"
-
-
-def test_not_contains_negates_with_exists(translator):
- clauses = translator.translate({"key": "version", "op": "not contains", "value": "earth"}).to_clauses()
- assert clauses == [
- {
- "bool": {
- "must": [{"exists": {"field": _field("version")}}],
- "must_not": [
- {
- "wildcard": {
- _field("version") + ".keyword": {
- "value": "*earth*",
- "case_insensitive": True,
- }
- }
- }
- ],
- }
- }
- ]
-
-
-def test_start_with_uses_prefix(translator):
- clauses = translator.translate({"key": "name", "op": "start with", "value": "pre"}).to_clauses()
- assert clauses == [
- {"prefix": {_field("name") + ".keyword": {"value": "pre", "case_insensitive": True}}}
- ]
-
-
-def test_end_with_uses_trailing_wildcard(translator):
- clauses = translator.translate({"key": "file", "op": "end with", "value": ".pdf"}).to_clauses()
- pattern = clauses[0]["wildcard"][_field("file") + ".keyword"]["value"]
- assert pattern == "*.pdf"
-
-
-def test_empty_matches_missing_or_blank(translator):
- clauses = translator.translate({"key": "notes", "op": "empty", "value": ""}).to_clauses()
- assert clauses == [
- {
- "bool": {
- "should": [
- {"bool": {"must_not": [{"exists": {"field": _field("notes")}}]}},
- {"term": {_field("notes") + ".keyword": ""}},
- ],
- "minimum_should_match": 1,
- }
- }
- ]
-
-
-def test_not_empty_requires_exists_and_excludes_blank(translator):
- clauses = translator.translate({"key": "notes", "op": "not empty", "value": ""}).to_clauses()
- assert clauses == [
- {
- "bool": {
- "must": [{"exists": {"field": _field("notes")}}],
- "must_not": [{"term": {_field("notes") + ".keyword": ""}}],
- }
- }
- ]
-
-
-# ---------------------------------------------------------------------------
-# Translator: validation paths
-# ---------------------------------------------------------------------------
-
-
-def test_unknown_operator_raises(translator):
- with pytest.raises(UnsupportedMetaFilter) as exc:
- translator.translate({"key": "tag", "op": "regex", "value": "^foo"})
- assert "regex" in exc.value.reason
-
-
-def test_missing_key_raises(translator):
- with pytest.raises(UnsupportedMetaFilter):
- translator.translate({"op": "=", "value": "x"})
-
-
-def test_scalar_op_with_list_value_raises(translator):
- with pytest.raises(UnsupportedMetaFilter):
- translator.translate({"key": "tag", "op": "=", "value": ["a", "b"]})
-
-
-def test_string_op_with_empty_value_raises(translator):
- with pytest.raises(UnsupportedMetaFilter):
- translator.translate({"key": "tag", "op": "contains", "value": ""})
-
-
-def test_membership_with_empty_csv_raises(translator):
- with pytest.raises(UnsupportedMetaFilter):
- translator.translate({"key": "tag", "op": "in", "value": ""})
-
-
-def test_supported_operator_set_matches_documentation():
- expected = {
- "=",
- "≠",
- ">",
- "<",
- "≥",
- "≤",
- "in",
- "not in",
- "contains",
- "not contains",
- "start with",
- "end with",
- "empty",
- "not empty",
- }
- assert SUPPORTED_OPERATORS == expected
-
-
-# ---------------------------------------------------------------------------
-# Plan composition
-# ---------------------------------------------------------------------------
-
-
-def test_plan_emits_must_clauses_for_and_logic():
- plan = plan_pushdown(
- [
- {"key": "tag", "op": "=", "value": "alpha"},
- {"key": "score", "op": ">", "value": "5"},
- ],
- logic="and",
- )
- assert isinstance(plan, MetaFilterPushdownPlan)
- body = plan.to_query(["kb1"])
- bool_root = body["query"]["bool"]
- assert bool_root["filter"][0] == {"terms": {"kb_id": ["kb1"]}}
- inner = bool_root["filter"][1]["bool"]
- assert "must" in inner
- # Each translated filter contributes exactly one clause to the parent bool:
- # ``=`` is a single ``term``; ``>`` is wrapped into one atomic ``bool``.
- assert len(inner["must"]) == 2
- expected_tag_term = {
- "term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}
- }
- assert expected_tag_term in inner["must"]
- range_wrap = {
- "bool": {
- "must": [
- {"exists": {"field": _field("score")}},
- {"range": {_field("score"): {"gt": 5}}},
- ]
- }
- }
- assert range_wrap in inner["must"]
-
-
-def test_range_filter_under_or_stays_atomic():
- """An OR'd range must not split into independent ``exists`` + ``range`` should branches."""
- body = build_meta_filter_query(
- [
- {"key": "tag", "op": "=", "value": "alpha"},
- {"key": "score", "op": ">", "value": "5"},
- ],
- logic="or",
- kb_ids=["kb1"],
- )
- should = body["query"]["bool"]["filter"][1]["bool"]["should"]
- # Two filters → two should branches, not three or four.
- assert len(should) == 2
- assert {
- "term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}
- } in should
-
-
-def test_plan_emits_should_clauses_for_or_logic():
- plan = plan_pushdown(
- [
- {"key": "tag", "op": "=", "value": "alpha"},
- {"key": "tag", "op": "=", "value": "beta"},
- ],
- logic="or",
- )
- inner = plan.to_query(["kb1"])["query"]["bool"]["filter"][1]["bool"]
- assert inner["minimum_should_match"] == 1
- assert len(inner["should"]) == 2
-
-
-def test_unknown_logic_rejected():
- with pytest.raises(UnsupportedMetaFilter):
- plan_pushdown([{"key": "k", "op": "=", "value": "v"}], logic="xor")
-
-
-def test_empty_filter_list_returns_kb_only_query():
- body = build_meta_filter_query([], "and", ["kb1", "kb2"])
- assert body == {"query": {"bool": {"filter": [{"terms": {"kb_id": ["kb1", "kb2"]}}]}}}
-
-
-def test_negative_filter_in_or_logic_keeps_negation_scope():
- """Wrapping ``≠`` in an OR should not let the ``must_not`` swallow other branches.
-
- ``≠`` is rejected by :func:`is_pushdown_supported` for multi-value safety, so
- this test exercises the translator directly to confirm the per-filter
- wrapping invariant. The same shape protects ``not contains`` (which IS
- pushed down) from leaking its ``must_not`` into a parent should.
- """
- body = build_meta_filter_query(
- [
- {"key": "tag", "op": "=", "value": "alpha"},
- {"key": "tag", "op": "≠", "value": "beta"},
- ],
- logic="or",
- kb_ids=["kb1"],
- )
- inner = body["query"]["bool"]["filter"][1]["bool"]
- should = inner["should"]
- assert should[0] == {
- "term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}
- }
- # The ≠ branch is wrapped so its must_not does not bleed into the OR set.
- assert "bool" in should[1]
- assert "must_not" in should[1]["bool"]
-
-
-# ---------------------------------------------------------------------------
-# is_pushdown_supported pre-check
-# ---------------------------------------------------------------------------
-
-
-def test_pushdown_check_accepts_known_ops():
- assert is_pushdown_supported(
- [
- {"key": "tag", "op": "=", "value": "v"},
- {"key": "tag", "op": "contains", "value": "x"},
- ]
- )
-
-
-def test_pushdown_check_rejects_unknown_op():
- assert not is_pushdown_supported([{"key": "tag", "op": "regex", "value": "^v"}])
-
-
-def test_pushdown_check_rejects_missing_key():
- assert not is_pushdown_supported([{"op": "=", "value": "v"}])
-
-
-@pytest.mark.parametrize("op", ["≠", "not in"])
-def test_pushdown_check_rejects_multivalue_unsafe_negatives(op):
- """Negatives that diverge on multi-valued fields force the in-memory fallback."""
- assert not is_pushdown_supported([{"key": "tag", "op": op, "value": "x"}])
-
-
-def test_pushdown_check_one_unsafe_op_rejects_whole_request():
- """Mixing one unsafe op with safe ones still falls back, preserving correctness."""
- assert not is_pushdown_supported(
- [
- {"key": "tag", "op": "=", "value": "v"},
- {"key": "tag", "op": "≠", "value": "w"},
- ]
- )
-
-
-def test_pushdown_check_accepts_not_contains():
- """``not contains`` stays in push-down; ``all(not contains)`` ≡ ``not any(contains)``."""
- assert is_pushdown_supported([{"key": "tag", "op": "not contains", "value": "x"}])
-
-
-# ---------------------------------------------------------------------------
-# extract_doc_ids
-# ---------------------------------------------------------------------------
-
-
-def test_extract_doc_ids_from_dict_response():
- response = {
- "hits": {
- "hits": [
- {"_id": "doc1", "_source": {"id": "doc1"}},
- {"_id": "doc2", "_source": {"id": "doc2"}},
- ]
- }
- }
- assert extract_doc_ids(response) == ["doc1", "doc2"]
-
-
-def test_extract_doc_ids_falls_back_to_source_id():
- response = {"hits": {"hits": [{"_source": {"id": "src-id"}}]}}
- assert extract_doc_ids(response) == ["src-id"]
-
-
-def test_extract_doc_ids_empty_response():
- assert extract_doc_ids({}) == []
- assert extract_doc_ids({"hits": {}}) == []
- assert extract_doc_ids({"hits": {"hits": []}}) == []
diff --git a/test/unit_test/common/test_metadata_filter.py b/test/unit_test/common/test_metadata_filter.py
new file mode 100644
index 00000000000..d48b30fb6cd
--- /dev/null
+++ b/test/unit_test/common/test_metadata_filter.py
@@ -0,0 +1,659 @@
+"""Unit tests for the metadata filter push-down translators (ES and Infinity).
+
+Verifies the shape of the produced filter expressions for both ES DSL and
+Infinity SQL, and confirms that coercion rules (lower-casing, list-membership,
+date detection) are consistent between the two backends.
+"""
+
+import pytest
+
+pytestmark = pytest.mark.p2
+
+from common.metadata_es_filter import MetaFilterTranslator as ESMetaFilterTranslator
+from common.metadata_infinity_filter import (
+ MetaFilterTranslator as InfinityMetaFilterTranslator,
+ SUPPORTED_OPERATORS,
+ build_infinity_filter,
+ is_pushdown_supported,
+ plan_pushdown,
+ extract_doc_ids,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def es_translator() -> ESMetaFilterTranslator:
+ return ESMetaFilterTranslator()
+
+
+@pytest.fixture
+def infinity_translator() -> InfinityMetaFilterTranslator:
+ return InfinityMetaFilterTranslator()
+
+
+# ---------------------------------------------------------------------------
+# Shared: is_pushdown_supported pre-check (same logic for both backends)
+# ---------------------------------------------------------------------------
+
+
+def test_pushdown_check_accepts_known_ops():
+ assert is_pushdown_supported(
+ [
+ {"key": "tag", "op": "=", "value": "v"},
+ {"key": "tag", "op": "contains", "value": "x"},
+ ]
+ )
+
+
+def test_pushdown_check_rejects_unknown_op():
+ assert not is_pushdown_supported([{"key": "tag", "op": "regex", "value": "^v"}])
+
+
+def test_pushdown_check_rejects_missing_key():
+ assert not is_pushdown_supported([{"op": "=", "value": "v"}])
+
+
+def test_pushdown_check_accepts_not_contains():
+ assert is_pushdown_supported([{"key": "tag", "op": "not contains", "value": "x"}])
+
+
+# ---------------------------------------------------------------------------
+# Shared: plan_pushdown (same logic for both backends)
+# ---------------------------------------------------------------------------
+
+
+def test_plan_pushdown_and_logic():
+ fragments = plan_pushdown(
+ [
+ {"key": "tag", "op": "=", "value": "alpha"},
+ {"key": "score", "op": ">", "value": "5"},
+ ],
+ logic="and",
+ )
+ assert len(fragments) == 2
+
+
+def test_plan_pushdown_or_logic():
+ fragments = plan_pushdown(
+ [
+ {"key": "tag", "op": "=", "value": "alpha"},
+ {"key": "tag", "op": "=", "value": "beta"},
+ ],
+ logic="or",
+ )
+ assert len(fragments) == 2
+
+
+def test_unknown_logic_rejected():
+ with pytest.raises(ValueError):
+ plan_pushdown([{"key": "k", "op": "=", "value": "v"}], logic="xor")
+
+
+# ---------------------------------------------------------------------------
+# Shared: extract_doc_ids (same implementation)
+# ---------------------------------------------------------------------------
+
+
+def test_extract_doc_ids_from_dataframe():
+ import pandas as pd
+
+ df = pd.DataFrame({"id": ["doc1", "doc2", "doc3"]})
+ assert extract_doc_ids(df) == ["doc1", "doc2", "doc3"]
+
+
+def test_extract_doc_ids_empty_dataframe():
+ import pandas as pd
+
+ df = pd.DataFrame({"id": []})
+ assert extract_doc_ids(df) == []
+
+
+def test_extract_doc_ids_none_input():
+ assert extract_doc_ids(None) == []
+
+
+def test_extract_doc_ids_non_dataframe():
+ assert extract_doc_ids("not a dataframe") == []
+
+
+# ---------------------------------------------------------------------------
+# Shared: SUPPORTED_OPERATORS
+# ---------------------------------------------------------------------------
+
+
+def test_supported_operator_set_matches_documentation():
+ expected = {
+ "=",
+ "≠",
+ ">",
+ "<",
+ "≥",
+ "≤",
+ "in",
+ "not in",
+ "contains",
+ "not contains",
+ "start with",
+ "end with",
+ "empty",
+ "not empty",
+ }
+ assert SUPPORTED_OPERATORS == expected
+
+
+# ===========================================================================
+# ES-only tests
+# ===========================================================================
+
+
+def test_equal_translates_to_term_with_lowercased_value(es_translator):
+ """String equality runs against ``.keyword`` so multi-word phrases match."""
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "tag", "op": "=", "value": "Alpha"}).to_clauses()
+ assert clauses == [
+ {"term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}}
+ ]
+
+
+def test_equal_parses_numeric_literal(es_translator):
+ """Numeric values stay on the parent path — no ``.keyword`` sub-field exists for ``long``."""
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "score", "op": "=", "value": "5"}).to_clauses()
+ assert clauses == [{"term": {_field("score"): 5}}]
+
+
+def test_equal_multiword_uses_keyword_subfield(es_translator):
+ """Regression: multi-word string values must match via .keyword sub-field."""
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate(
+ {"key": "author", "op": "=", "value": "Alice Wonderland"}
+ ).to_clauses()
+ assert clauses == [
+ {
+ "term": {
+ _field("author") + ".keyword": {
+ "value": "alice wonderland",
+ "case_insensitive": True,
+ }
+ }
+ }
+ ]
+
+
+def test_not_equal_requires_field_to_exist(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "tag", "op": "≠", "value": "alpha"}).to_clauses()
+ assert clauses == [
+ {
+ "bool": {
+ "must": [{"exists": {"field": _field("tag")}}],
+ "must_not": [
+ {"term": {_field("tag") + ".keyword": {"value": "alpha", "case_insensitive": True}}}
+ ],
+ }
+ }
+ ]
+
+
+@pytest.mark.parametrize(
+ "op,es_key",
+ [(">", "gt"), ("<", "lt"), ("≥", "gte"), ("≤", "lte")],
+)
+def test_range_operator_translation(es_translator, op, es_key):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "score", "op": op, "value": "10"}).to_clauses()
+ assert clauses == [
+ {
+ "bool": {
+ "must": [
+ {"exists": {"field": _field("score")}},
+ {"range": {_field("score"): {es_key: 10}}},
+ ]
+ }
+ }
+ ]
+
+
+def test_range_passes_iso_date_through_unparsed(es_translator):
+ clauses = es_translator.translate({"key": "published", "op": "≥", "value": "2025-01-15"}).to_clauses()
+ range_clause = clauses[0]["bool"]["must"][1]
+ assert range_clause == {"range": {"meta_fields.published": {"gte": "2025-01-15"}}}
+
+
+def test_in_operator_csv_value_lowercased(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ def _string_terms_should(field_path: str, members):
+ return {
+ "bool": {
+ "should": [
+ {"term": {field_path + ".keyword": {"value": m, "case_insensitive": True}}}
+ for m in members
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+
+ clauses = es_translator.translate({"key": "status", "op": "in", "value": "Active,Pending"}).to_clauses()
+ assert clauses == [_string_terms_should(_field("status"), ["active", "pending"])]
+
+
+def test_in_operator_python_list_literal(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ def _string_terms_should(field_path: str, members):
+ return {
+ "bool": {
+ "should": [
+ {"term": {field_path + ".keyword": {"value": m, "case_insensitive": True}}}
+ for m in members
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+
+ clauses = es_translator.translate({"key": "status", "op": "in", "value": "['Open', 'Closed']"}).to_clauses()
+ assert clauses == [_string_terms_should(_field("status"), ["open", "closed"])]
+
+
+def test_in_operator_numeric_members_keep_terms(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "year", "op": "in", "value": "[2024, 2025]"}).to_clauses()
+ assert clauses == [{"terms": {_field("year"): [2024, 2025]}}]
+
+
+def test_not_in_negates_with_existence_guard(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ def _string_terms_should(field_path: str, members):
+ return {
+ "bool": {
+ "should": [
+ {"term": {field_path + ".keyword": {"value": m, "case_insensitive": True}}}
+ for m in members
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+
+ clauses = es_translator.translate({"key": "status", "op": "not in", "value": "active,pending"}).to_clauses()
+ assert clauses == [
+ {
+ "bool": {
+ "must": [{"exists": {"field": _field("status")}}],
+ "must_not": [_string_terms_should(_field("status"), ["active", "pending"])],
+ }
+ }
+ ]
+
+
+def test_contains_uses_case_insensitive_wildcard(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "version", "op": "contains", "value": "earth"}).to_clauses()
+ assert clauses == [
+ {
+ "wildcard": {
+ _field("version") + ".keyword": {
+ "value": "*earth*",
+ "case_insensitive": True,
+ }
+ }
+ }
+ ]
+
+
+def test_contains_escapes_user_wildcards(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "title", "op": "contains", "value": "a*b?c"}).to_clauses()
+ pattern = clauses[0]["wildcard"][_field("title") + ".keyword"]["value"]
+ assert pattern == "*a\\*b\\?c*"
+
+
+def test_not_contains_negates_with_exists(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "version", "op": "not contains", "value": "earth"}).to_clauses()
+ assert clauses == [
+ {
+ "bool": {
+ "must": [{"exists": {"field": _field("version")}}],
+ "must_not": [
+ {
+ "wildcard": {
+ _field("version") + ".keyword": {
+ "value": "*earth*",
+ "case_insensitive": True,
+ }
+ }
+ }
+ ],
+ }
+ }
+ ]
+
+
+def test_start_with_uses_prefix(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "name", "op": "start with", "value": "pre"}).to_clauses()
+ assert clauses == [
+ {"prefix": {_field("name") + ".keyword": {"value": "pre", "case_insensitive": True}}}
+ ]
+
+
+def test_end_with_uses_trailing_wildcard(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "file", "op": "end with", "value": ".pdf"}).to_clauses()
+ pattern = clauses[0]["wildcard"][_field("file") + ".keyword"]["value"]
+ assert pattern == "*.pdf"
+
+
+def test_empty_matches_missing_or_blank(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "notes", "op": "empty", "value": ""}).to_clauses()
+ assert clauses == [
+ {
+ "bool": {
+ "should": [
+ {"bool": {"must_not": [{"exists": {"field": _field("notes")}}]}},
+ {"term": {_field("notes") + ".keyword": ""}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ ]
+
+
+def test_not_empty_requires_exists_and_excludes_blank(es_translator):
+ from common.metadata_es_filter import META_FIELDS_PREFIX
+
+ def _field(key: str) -> str:
+ return f"{META_FIELDS_PREFIX}.{key}"
+
+ clauses = es_translator.translate({"key": "notes", "op": "not empty", "value": ""}).to_clauses()
+ assert clauses == [
+ {
+ "bool": {
+ "must": [{"exists": {"field": _field("notes")}}],
+ "must_not": [{"term": {_field("notes") + ".keyword": ""}}],
+ }
+ }
+ ]
+
+
+def test_unknown_operator_raises(es_translator):
+ from common.metadata_es_filter import UnsupportedMetaFilter
+
+ with pytest.raises(UnsupportedMetaFilter) as exc:
+ es_translator.translate({"key": "tag", "op": "regex", "value": "^foo"})
+ assert "regex" in exc.value.reason
+
+
+def test_missing_key_raises(es_translator):
+ from common.metadata_es_filter import UnsupportedMetaFilter
+
+ with pytest.raises(UnsupportedMetaFilter):
+ es_translator.translate({"op": "=", "value": "x"})
+
+
+def test_scalar_op_with_list_value_raises(es_translator):
+ from common.metadata_es_filter import UnsupportedMetaFilter
+
+ with pytest.raises(UnsupportedMetaFilter):
+ es_translator.translate({"key": "tag", "op": "=", "value": ["a", "b"]})
+
+
+def test_string_op_with_empty_value_raises(es_translator):
+ from common.metadata_es_filter import UnsupportedMetaFilter
+
+ with pytest.raises(UnsupportedMetaFilter):
+ es_translator.translate({"key": "tag", "op": "contains", "value": ""})
+
+
+def test_membership_with_empty_csv_raises(es_translator):
+ from common.metadata_es_filter import UnsupportedMetaFilter
+
+ with pytest.raises(UnsupportedMetaFilter):
+ es_translator.translate({"key": "tag", "op": "in", "value": ""})
+
+
+# ===========================================================================
+# Infinity-only tests
+# ===========================================================================
+
+
+def test_build_infinity_filter_and_logic():
+ body = build_infinity_filter(
+ [
+ {"key": "tag", "op": "=", "value": "alpha"},
+ {"key": "score", "op": ">", "value": "5"},
+ ],
+ logic="and",
+ )
+ assert " AND " in body
+ assert "alpha" in body
+
+
+def test_build_infinity_filter_or_logic():
+ body = build_infinity_filter(
+ [
+ {"key": "tag", "op": "=", "value": "alpha"},
+ {"key": "tag", "op": "=", "value": "beta"},
+ ],
+ logic="or",
+ )
+ assert " OR " in body
+ assert "alpha" in body
+ assert "beta" in body
+
+
+def test_empty_filter_list_returns_1eq1():
+ body = build_infinity_filter([], "and")
+ assert body == "1=1"
+
+
+def test_infinity_equal_string_uses_lowercase(infinity_translator):
+ cond = infinity_translator.translate({"key": "tag", "op": "=", "value": "Alpha"})
+ assert cond == "JSON_CONTAINS(meta_fields, '$.tag', '\"Alpha\"')"
+
+
+def test_infinity_equal_numeric_keeps_number(infinity_translator):
+ cond = infinity_translator.translate({"key": "score", "op": "=", "value": "5"})
+ assert cond == "JSON_CONTAINS(meta_fields, '$.score', 5)"
+
+
+def test_infinity_equal_date_passes_unparsed(infinity_translator):
+ cond = infinity_translator.translate({"key": "published", "op": "=", "value": "2025-01-15"})
+ assert cond == "JSON_CONTAINS(meta_fields, '$.published', '\"2025-01-15\"')"
+
+
+def test_infinity_not_equal_string(infinity_translator):
+ cond = infinity_translator.translate({"key": "tag", "op": "≠", "value": "alpha"})
+ assert "JSON_CONTAINS" in cond
+ assert "alpha" in cond
+ assert "NOT" in cond
+
+
+def test_infinity_not_equal_numeric(infinity_translator):
+ cond = infinity_translator.translate({"key": "score", "op": "≠", "value": "5"})
+ assert "JSON_CONTAINS" in cond and "NOT" in cond and "5" in cond
+
+
+@pytest.mark.parametrize("op,sql_op", [(">", ">"), ("<", "<"), ("≥", ">="), ("≤", "<=")])
+def test_infinity_range_operators(infinity_translator, op, sql_op):
+ cond = infinity_translator.translate({"key": "score", "op": op, "value": "10"})
+ assert sql_op in cond
+ assert "JSON_EXTRACT_DOUBLE(meta_fields, '$.score')" in cond
+
+
+def test_infinity_range_string_value(infinity_translator):
+ cond = infinity_translator.translate({"key": "published", "op": "≥", "value": "2025-01-15"})
+ assert ">=" in cond
+ assert "2025-01-15" in cond
+
+
+def test_infinity_in_csv_lowercased(infinity_translator):
+ cond = infinity_translator.translate({"key": "status", "op": "in", "value": "Active,Pending"})
+ assert "JSON_CONTAINS" in cond
+ assert "active" in cond
+ assert "pending" in cond
+
+
+def test_infinity_in_python_list(infinity_translator):
+ cond = infinity_translator.translate({"key": "status", "op": "in", "value": "['Open', 'Closed']"})
+ assert "JSON_CONTAINS" in cond
+ assert "open" in cond
+ assert "closed" in cond
+
+
+def test_infinity_in_numeric_members(infinity_translator):
+ cond = infinity_translator.translate({"key": "year", "op": "in", "value": "[2024, 2025]"})
+ assert "JSON_CONTAINS" in cond
+ assert "2024" in cond
+ assert "2025" in cond
+
+
+def test_infinity_not_in_csv(infinity_translator):
+ cond = infinity_translator.translate({"key": "status", "op": "not in", "value": "active,pending"})
+ assert "NOT JSON_CONTAINS" in cond
+
+
+def test_infinity_contains_uses_JSON_CONTAINS(infinity_translator):
+ """Infinity 'contains' uses JSON_CONTAINS for JSON array membership."""
+ cond = infinity_translator.translate({"key": "version", "op": "contains", "value": "earth"})
+ assert "JSON_CONTAINS" in cond
+ assert "earth" in cond
+
+
+def test_infinity_contains_escapes_quotes(infinity_translator):
+ """Special characters in contains value are escaped for JSON_CONTAINS."""
+ cond = infinity_translator.translate({"key": "title", "op": "contains", "value": "a%b_c"})
+ assert "JSON_CONTAINS" in cond
+ assert "a%b_c" in cond
+
+
+def test_infinity_not_contains_uses_JSON_CONTAINS(infinity_translator):
+ """Infinity 'not contains' uses JSON_CONTAINS with NOT."""
+ cond = infinity_translator.translate({"key": "version", "op": "not contains", "value": "earth"})
+ assert "JSON_CONTAINS" in cond
+ assert "NOT" in cond or "not" in cond.lower()
+
+
+def test_infinity_start_with(infinity_translator):
+ cond = infinity_translator.translate({"key": "name", "op": "start with", "value": "pre"})
+ assert "LIKE" in cond
+ assert "'pre%" in cond
+
+
+def test_infinity_end_with(infinity_translator):
+ """Infinity 'end with' uses LIKE with trailing wildcard."""
+ cond = infinity_translator.translate({"key": "file", "op": "end with", "value": ".pdf"})
+ assert "LIKE" in cond
+ assert "%.pdf" in cond
+
+
+def test_infinity_empty(infinity_translator):
+ cond = infinity_translator.translate({"key": "notes", "op": "empty", "value": ""})
+ assert "JSON_EXTRACT_STRING" in cond
+ assert '""' in cond
+
+
+def test_infinity_not_empty(infinity_translator):
+ cond = infinity_translator.translate({"key": "notes", "op": "not empty", "value": ""})
+ assert "JSON_EXTRACT_STRING" in cond
+ assert "!=" in cond
+
+
+def test_infinity_unknown_operator_raises(infinity_translator):
+ with pytest.raises(ValueError) as exc:
+ infinity_translator.translate({"key": "tag", "op": "regex", "value": "^foo"})
+ assert "regex" in str(exc.value)
+
+
+def test_infinity_missing_key_raises(infinity_translator):
+ with pytest.raises(ValueError):
+ infinity_translator.translate({"op": "=", "value": "x"})
+
+
+def test_infinity_invalid_key_format_raises(infinity_translator):
+ with pytest.raises(ValueError, match="invalid key format"):
+ infinity_translator.translate({"key": "a;b", "op": "=", "value": "x"})
+
+
+def test_infinity_key_with_brace_raises(infinity_translator):
+ with pytest.raises(ValueError, match="invalid key format"):
+ infinity_translator.translate({"key": "field$}", "op": "=", "value": "x"})
+
+
+def test_infinity_scalar_op_with_list_value_raises(infinity_translator):
+ with pytest.raises(ValueError):
+ infinity_translator.translate({"key": "tag", "op": "=", "value": ["a", "b"]})
+
+
+def test_infinity_string_op_with_empty_value_raises(infinity_translator):
+ with pytest.raises(ValueError):
+ infinity_translator.translate({"key": "tag", "op": "contains", "value": ""})
+
+
+def test_infinity_membership_with_empty_csv_raises(infinity_translator):
+ with pytest.raises(ValueError):
+ infinity_translator.translate({"key": "tag", "op": "in", "value": ""})
\ No newline at end of file
From b09da6e347ffc19927ad3cc5d9e0720d2385e9d9 Mon Sep 17 00:00:00 2001
From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com>
Date: Sun, 17 May 2026 20:31:16 -1000
Subject: [PATCH 178/666] Go: implement provider: CometAPI (#14930)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Adds the Go model provider driver for CometAPI, which is listed as
unchecked in the Go provider tracking issue #14736 and requested in
#14804. Without this, the Go layer falls back to the dummy driver for
the `cometapi` provider.
Fixes #14804
### What this PR includes
- New `internal/entity/models/cometapi.go` implementing `ModelDriver`
for CometAPI.
- New `conf/models/cometapi.json` with CometAPI base URLs and
representative chat / embedding models from the public catalog.
- `factory.go`: route `"cometapi"` to `NewCometAPIModel`.
- Unit tests in `internal/entity/models/cometapi_test.go`.
### Method coverage
- `ChatWithMessages`: `POST /v1/chat/completions`.
- `ChatStreamlyWithSender`: SSE streaming on the same endpoint.
- `Embed`: `POST /v1/embeddings`, including optional `dimensions`.
- `ListModels`: `GET /api/models` public catalog.
- `Balance`: `GET https://query.cometapi.com/user/quota?key=...`.
- `CheckConnection`: delegates to the quota query to verify the key.
- `Rerank`, ASR, TTS, OCR: return `no such method` for now.
No ModelDriver interface change. No new dependencies.
### How was this tested?
```bash
go test -vet=off -run TestCometAPI -count=1 ./internal/entity/models/...
go test -vet=off -count=1 ./internal/entity/models/...
```
---------
Signed-off-by: dependabot[bot]
Signed-off-by: Jin Hai
Signed-off-by: majiayu000 <1835304752@qq.com>
Co-authored-by: 加帆
Co-authored-by: Kevin Hu
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: bulexu
Co-authored-by: xubh
Co-authored-by: Jin Hai
Co-authored-by: Carve_ <75568342+Rynzie02@users.noreply.github.com>
Co-authored-by: Paul Y Hui
Co-authored-by: LIRUI YU <128563231+LiruiYu33@users.noreply.github.com>
Co-authored-by: yun.kou
Co-authored-by: Yun.kou
Co-authored-by: Ahmad Intisar <168020872+ahmadintisar@users.noreply.github.com>
Co-authored-by: Ahmad Intisar
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: Syed Shahmeer Ali
Co-authored-by: Octopus
Co-authored-by: lif <1835304752@qq.com>
---
conf/models/cometapi.json | 110 ++++
internal/entity/models/cometapi.go | 655 +++++++++++++++++++++
internal/entity/models/cometapi_test.go | 744 ++++++++++++++++++++++++
internal/entity/models/factory.go | 2 +
4 files changed, 1511 insertions(+)
create mode 100644 conf/models/cometapi.json
create mode 100644 internal/entity/models/cometapi.go
create mode 100644 internal/entity/models/cometapi_test.go
diff --git a/conf/models/cometapi.json b/conf/models/cometapi.json
new file mode 100644
index 00000000000..c53a5a59228
--- /dev/null
+++ b/conf/models/cometapi.json
@@ -0,0 +1,110 @@
+{
+ "name": "CometAPI",
+ "url": {
+ "default": "https://api.cometapi.com"
+ },
+ "url_suffix": {
+ "chat": "v1/chat/completions",
+ "models": "api/models",
+ "embedding": "v1/embeddings",
+ "balance": "https://query.cometapi.com/user/quota"
+ },
+ "class": "cometapi",
+ "models": [
+ {
+ "name": "gpt-5.5",
+ "max_tokens": 400000,
+ "model_types": [
+ "chat",
+ "vision"
+ ],
+ "thinking": {
+ "default_value": true,
+ "clear_thinking": true
+ }
+ },
+ {
+ "name": "gpt-5.4-mini",
+ "max_tokens": 400000,
+ "model_types": [
+ "chat",
+ "vision"
+ ],
+ "thinking": {
+ "default_value": true,
+ "clear_thinking": true
+ }
+ },
+ {
+ "name": "gpt-5",
+ "max_tokens": 400000,
+ "model_types": [
+ "chat",
+ "vision"
+ ],
+ "thinking": {
+ "default_value": true,
+ "clear_thinking": true
+ }
+ },
+ {
+ "name": "gpt-4o",
+ "max_tokens": 128000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "claude-sonnet-4-6",
+ "max_tokens": 200000,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "gemini-3-pro-preview",
+ "max_tokens": 1048576,
+ "model_types": [
+ "chat",
+ "vision"
+ ]
+ },
+ {
+ "name": "deepseek-v3.2",
+ "max_tokens": 128000,
+ "model_types": [
+ "chat"
+ ]
+ },
+ {
+ "name": "qwen3-235b-a22b",
+ "max_tokens": 128000,
+ "model_types": [
+ "chat"
+ ]
+ },
+ {
+ "name": "text-embedding-3-small",
+ "max_tokens": 8191,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "text-embedding-3-large",
+ "max_tokens": 8191,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "text-embedding-ada-002",
+ "max_tokens": 8191,
+ "model_types": [
+ "embedding"
+ ]
+ }
+ ]
+}
diff --git a/internal/entity/models/cometapi.go b/internal/entity/models/cometapi.go
new file mode 100644
index 00000000000..ef8014847b3
--- /dev/null
+++ b/internal/entity/models/cometapi.go
@@ -0,0 +1,655 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+)
+
+// CometAPIModel implements ModelDriver for CometAPI AI.
+//
+// CometAPI exposes OpenAI-compatible chat and embeddings under
+// https://api.cometapi.com/v1, a public model catalog under
+// https://api.cometapi.com/api/models, and account quota data through the
+// separate query service at https://query.cometapi.com/user/quota.
+type CometAPIModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+// NewCometAPIModel creates a new CometAPI model instance.
+//
+// We clone http.DefaultTransport so we keep Go's defaults for
+// ProxyFromEnvironment, DialContext (with KeepAlive), HTTP/2,
+// TLSHandshakeTimeout, and ExpectContinueTimeout, and only override
+// the connection-pool fields we care about.
+//
+// The Client itself has no Timeout. http.Client.Timeout would also
+// cap the time spent reading the response body, which would cut off
+// long-lived SSE streams in ChatStreamlyWithSender. Non-streaming
+// callers wrap each request with context.WithTimeout instead.
+func NewCometAPIModel(baseURL map[string]string, urlSuffix URLSuffix) *CometAPIModel {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxIdleConns = 100
+ transport.MaxIdleConnsPerHost = 10
+ transport.IdleConnTimeout = 90 * time.Second
+ transport.DisableCompression = false
+ transport.ResponseHeaderTimeout = 60 * time.Second
+
+ return &CometAPIModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Transport: transport,
+ },
+ }
+}
+
+func (m *CometAPIModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return NewCometAPIModel(baseURL, m.URLSuffix)
+}
+
+func (m *CometAPIModel) Name() string {
+ return "cometapi"
+}
+
+func validateCometAPIAPIKey(apiConfig *APIConfig) (string, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return "", fmt.Errorf("api key is required")
+ }
+ return *apiConfig.ApiKey, nil
+}
+
+func validateCometAPIModelName(modelName string) error {
+ if strings.TrimSpace(modelName) == "" {
+ return fmt.Errorf("model name is required")
+ }
+ return nil
+}
+
+func cometapiRegion(apiConfig *APIConfig) string {
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ return *apiConfig.Region
+ }
+ return "default"
+}
+
+// baseURLForRegion returns the base URL for the given region, or an
+// error if no entry exists. This makes a misconfigured region fail
+// fast with a clear message, instead of silently producing a relative
+// URL that the HTTP transport then rejects.
+func (m *CometAPIModel) baseURLForRegion(region string) (string, error) {
+ base, ok := m.BaseURL[region]
+ if !ok || base == "" {
+ return "", fmt.Errorf("cometapi: no base URL configured for region %q", region)
+ }
+ return strings.TrimRight(base, "/"), nil
+}
+
+func (m *CometAPIModel) endpointURL(region, suffix string) (string, error) {
+ baseURL, err := m.baseURLForRegion(region)
+ if err != nil {
+ return "", err
+ }
+ return fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(suffix, "/")), nil
+}
+
+func (m *CometAPIModel) balanceURL(apiKey string) string {
+ rawURL := strings.TrimSpace(m.URLSuffix.Balance)
+ if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
+ rawURL = fmt.Sprintf("https://query.cometapi.com/%s", strings.TrimLeft(rawURL, "/"))
+ }
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return rawURL
+ }
+ query := parsed.Query()
+ query.Set("key", apiKey)
+ parsed.RawQuery = query.Encode()
+ return parsed.String()
+}
+
+type cometapiChatRequest struct {
+ Model string `json:"model"`
+ Messages []cometapiAPIMessage `json:"messages"`
+ Stream bool `json:"stream"`
+ MaxTokens *int `json:"max_tokens,omitempty"`
+ Temperature *float64 `json:"temperature,omitempty"`
+ TopP *float64 `json:"top_p,omitempty"`
+ Stop *[]string `json:"stop,omitempty"`
+}
+
+type cometapiAPIMessage struct {
+ Role string `json:"role"`
+ Content interface{} `json:"content"`
+}
+
+func buildCometAPIChatRequest(modelName string, messages []Message, stream bool, chatModelConfig *ChatConfig) cometapiChatRequest {
+ apiMessages := make([]cometapiAPIMessage, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = cometapiAPIMessage{
+ Role: msg.Role,
+ Content: msg.Content,
+ }
+ }
+
+ reqBody := cometapiChatRequest{
+ Model: modelName,
+ Messages: apiMessages,
+ Stream: stream,
+ }
+ if chatModelConfig != nil {
+ reqBody.MaxTokens = chatModelConfig.MaxTokens
+ reqBody.Temperature = chatModelConfig.Temperature
+ reqBody.TopP = chatModelConfig.TopP
+ reqBody.Stop = chatModelConfig.Stop
+ }
+ return reqBody
+}
+
+func newCometAPIJSONRequest(ctx context.Context, method string, endpoint string, payload interface{}, apiKey string) (*http.Request, error) {
+ jsonData, err := json.Marshal(payload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ if apiKey != "" {
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+ }
+ return req, nil
+}
+
+type cometapiHTTPResponse struct {
+ StatusCode int
+ Status string
+ Body []byte
+}
+
+func (m *CometAPIModel) doCometAPIRequest(req *http.Request) (*cometapiHTTPResponse, error) {
+ resp, err := m.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ return &cometapiHTTPResponse{
+ StatusCode: resp.StatusCode,
+ Status: resp.Status,
+ Body: body,
+ }, nil
+}
+
+type cometapiChatResponsePayload struct {
+ Choices []cometapiChatChoice `json:"choices"`
+}
+
+type cometapiChatChoice struct {
+ Message cometapiChatMessage `json:"message"`
+ Delta cometapiChatDelta `json:"delta"`
+ FinishReason string `json:"finish_reason"`
+}
+
+type cometapiChatMessage struct {
+ Content *string `json:"content"`
+ ReasoningContent string `json:"reasoning_content"`
+}
+
+type cometapiChatDelta struct {
+ Content string `json:"content"`
+ ReasoningContent string `json:"reasoning_content"`
+}
+
+func parseCometAPIChatResponse(body []byte) (*ChatResponse, error) {
+ var parsed cometapiChatResponsePayload
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+ if len(parsed.Choices) == 0 {
+ return nil, fmt.Errorf("no choices in response")
+ }
+ if parsed.Choices[0].Message.Content == nil {
+ return nil, fmt.Errorf("invalid content format")
+ }
+
+ content := *parsed.Choices[0].Message.Content
+ reasonContent := strings.TrimLeft(parsed.Choices[0].Message.ReasoningContent, "\n")
+ return &ChatResponse{
+ Answer: &content,
+ ReasonContent: &reasonContent,
+ }, nil
+}
+
+func parseCometAPIStreamEvent(data string) (content string, reasonContent string, terminal bool, ok bool) {
+ var event cometapiChatResponsePayload
+ if err := json.Unmarshal([]byte(data), &event); err != nil {
+ return "", "", false, false
+ }
+ if len(event.Choices) == 0 {
+ return "", "", false, false
+ }
+ choice := event.Choices[0]
+ return choice.Delta.Content, choice.Delta.ReasoningContent, choice.FinishReason != "", true
+}
+
+type cometapiModelCatalogResponse struct {
+ Data []cometapiModelCatalogItem `json:"data"`
+}
+
+type cometapiModelCatalogItem struct {
+ ID string `json:"id"`
+}
+
+func parseCometAPIModelCatalog(body []byte) ([]string, error) {
+ var parsed cometapiModelCatalogResponse
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ models := make([]string, 0, len(parsed.Data))
+ for _, model := range parsed.Data {
+ if model.ID != "" {
+ models = append(models, model.ID)
+ }
+ }
+ return models, nil
+}
+
+// ChatWithMessages sends multiple messages with roles and returns the response.
+func (m *CometAPIModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ apiKey, err := validateCometAPIAPIKey(apiConfig)
+ if err != nil {
+ return nil, err
+ }
+ if err := validateCometAPIModelName(modelName); err != nil {
+ return nil, err
+ }
+
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ url, err := m.endpointURL(cometapiRegion(apiConfig), m.URLSuffix.Chat)
+ if err != nil {
+ return nil, err
+ }
+
+ // Note: do NOT propagate chatModelConfig.Stream into the request body
+ // here. ChatWithMessages parses a single JSON response, so stream must
+ // always be off for this code path.
+ reqBody := buildCometAPIChatRequest(modelName, messages, false, chatModelConfig)
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := newCometAPIJSONRequest(ctx, "POST", url, reqBody, apiKey)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := m.doCometAPIRequest(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(resp.Body))
+ }
+ return parseCometAPIChatResponse(resp.Body)
+}
+
+// ChatStreamlyWithSender sends messages and streams the response via the
+// sender function. The CometAPI SSE stream uses the same shape as OpenAI:
+// "data:" lines carrying JSON events, with a final "[DONE]" line.
+func (m *CometAPIModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
+ if sender == nil {
+ return fmt.Errorf("sender is required")
+ }
+
+ if err := validateCometAPIModelName(modelName); err != nil {
+ return err
+ }
+
+ if len(messages) == 0 {
+ return fmt.Errorf("messages is empty")
+ }
+
+ apiKey, err := validateCometAPIAPIKey(apiConfig)
+ if err != nil {
+ return err
+ }
+
+ url, err := m.endpointURL(cometapiRegion(apiConfig), m.URLSuffix.Chat)
+ if err != nil {
+ return err
+ }
+
+ if chatModelConfig != nil {
+ // Refuse to run if the caller explicitly asked for stream=false.
+ // The body of this method only knows how to read SSE, so a
+ // non-SSE JSON response would be parsed as if it were a stream
+ // and produce no chunks. Better to fail clearly.
+ if chatModelConfig.Stream != nil && !*chatModelConfig.Stream {
+ return fmt.Errorf("stream must be true in ChatStreamlyWithSender")
+ }
+ }
+ reqBody := buildCometAPIChatRequest(modelName, messages, true, chatModelConfig)
+
+ // Use an explicit background context. SSE streams are long-lived
+ // so we do not attach a hard deadline here; the transport's
+ // ResponseHeaderTimeout caps the connection-establishment phase.
+ req, err := newCometAPIJSONRequest(context.Background(), "POST", url, reqBody, apiKey)
+ if err != nil {
+ return err
+ }
+ resp, err := m.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ // SSE parsing: bump the scanner buffer from the 64KB default to 1MB
+ // so we never silently truncate a long data: line.
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+ sawTerminal := false
+ for scanner.Scan() {
+ line := scanner.Text()
+
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+
+ data := strings.TrimSpace(line[5:])
+
+ if data == "[DONE]" {
+ sawTerminal = true
+ break
+ }
+
+ content, reasoningContent, terminal, ok := parseCometAPIStreamEvent(data)
+ if !ok {
+ continue
+ }
+
+ if reasoningContent != "" {
+ if err := sender(nil, &reasoningContent); err != nil {
+ return err
+ }
+ }
+
+ if content != "" {
+ if err := sender(&content, nil); err != nil {
+ return err
+ }
+ }
+
+ if terminal {
+ sawTerminal = true
+ break
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("failed to scan response body: %w", err)
+ }
+ if !sawTerminal {
+ return fmt.Errorf("cometapi: stream ended before [DONE] or finish_reason")
+ }
+
+ endOfStream := "[DONE]"
+ if err := sender(&endOfStream, nil); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+type cometapiEmbeddingData struct {
+ Embedding []float64 `json:"embedding"`
+ Object string `json:"object"`
+ Index int `json:"index"`
+}
+
+type cometapiEmbeddingResponse struct {
+ Data []cometapiEmbeddingData `json:"data"`
+ Model string `json:"model"`
+ Object string `json:"object"`
+}
+
+type cometapiEmbeddingRequest struct {
+ Model string `json:"model"`
+ Input []string `json:"input"`
+ Dimensions int `json:"dimensions,omitempty"`
+}
+
+// Embed turns a list of texts into embedding vectors using the
+// CometAPI /v1/embeddings endpoint. The output has one vector per input,
+// in the same order the inputs were given.
+func (m *CometAPIModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ if len(texts) == 0 {
+ return []EmbeddingData{}, nil
+ }
+
+ apiKey, err := validateCometAPIAPIKey(apiConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ if modelName == nil || strings.TrimSpace(*modelName) == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+
+ url, err := m.endpointURL(cometapiRegion(apiConfig), m.URLSuffix.Embedding)
+ if err != nil {
+ return nil, err
+ }
+
+ reqBody := cometapiEmbeddingRequest{
+ Model: *modelName,
+ Input: texts,
+ }
+ if embeddingConfig != nil && embeddingConfig.Dimension > 0 {
+ reqBody.Dimensions = embeddingConfig.Dimension
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := newCometAPIJSONRequest(ctx, "POST", url, reqBody, apiKey)
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := m.doCometAPIRequest(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("CometAPI embeddings API error: %s, body: %s", resp.Status, string(resp.Body))
+ }
+
+ var parsed cometapiEmbeddingResponse
+ if err = json.Unmarshal(resp.Body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ // Reorder the returned vectors by their reported index so the output
+ // always lines up with the input texts, even if the upstream API ever
+ // returns items out of order. A nil slot at the end indicates the
+ // upstream did not return an embedding for that input.
+ embeddings := make([]EmbeddingData, len(texts))
+ filled := make([]bool, len(texts))
+ for _, item := range parsed.Data {
+ if item.Index < 0 || item.Index >= len(texts) {
+ return nil, fmt.Errorf("cometapi: response index %d out of range for %d inputs", item.Index, len(texts))
+ }
+ if filled[item.Index] {
+ // A malformed response that repeats the same index would
+ // silently overwrite the earlier vector. Fail loudly so
+ // the caller never uses ambiguous output.
+ return nil, fmt.Errorf("cometapi: duplicate embedding index %d in response", item.Index)
+ }
+ embeddings[item.Index] = EmbeddingData{
+ Embedding: item.Embedding,
+ Index: item.Index,
+ }
+ filled[item.Index] = true
+ }
+ for i, ok := range filled {
+ if !ok {
+ return nil, fmt.Errorf("cometapi: missing embedding for input index %d", i)
+ }
+ }
+
+ return embeddings, nil
+}
+
+// ListModels returns the public CometAPI model catalog.
+func (m *CometAPIModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ url, err := m.endpointURL(cometapiRegion(apiConfig), m.URLSuffix.Models)
+ if err != nil {
+ return nil, err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ resp, err := m.doCometAPIRequest(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(resp.Body))
+ }
+ return parseCometAPIModelCatalog(resp.Body)
+}
+
+// Balance queries CometAPI's quota service. Unlike model requests, this
+// endpoint authenticates with the key query parameter on query.cometapi.com.
+func (m *CometAPIModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+ if strings.TrimSpace(m.URLSuffix.Balance) == "" {
+ return nil, fmt.Errorf("balance URL is required")
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "GET", m.balanceURL(*apiConfig.ApiKey), nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ resp, err := m.doCometAPIRequest(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("CometAPI quota API error: %s, body: %s", resp.Status, string(resp.Body))
+ }
+
+ var result map[string]interface{}
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ return result, nil
+}
+
+// CheckConnection runs a quota query to verify the API key.
+func (m *CometAPIModel) CheckConnection(apiConfig *APIConfig) error {
+ _, err := m.Balance(apiConfig)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// Rerank calculates similarity scores between query and documents. CometAPI
+// does not expose a public rerank API, so this returns "no such method".
+func (m *CometAPIModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("no such method")
+}
+
+// TranscribeAudio transcribe audio
+func (m *CometAPIModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", m.Name())
+}
+
+func (m *CometAPIModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", m.Name())
+}
+
+// AudioSpeech synthesizes speech audio from text.
+func (m *CometAPIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", m.Name())
+}
+
+func (m *CometAPIModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", m.Name())
+}
+
+// OCRFile OCR file
+func (m *CometAPIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", m.Name())
+}
+
+func (m *CometAPIModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", m.Name())
+}
+
+func (m *CometAPIModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", m.Name())
+}
+
+func (m *CometAPIModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", m.Name())
+}
diff --git a/internal/entity/models/cometapi_test.go b/internal/entity/models/cometapi_test.go
new file mode 100644
index 00000000000..34cfe9c6ce5
--- /dev/null
+++ b/internal/entity/models/cometapi_test.go
@@ -0,0 +1,744 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+// newCometAPIServer stands up an httptest server that asserts the
+// request shape and lets the caller decide what to return.
+func newCometAPIServer(t *testing.T, expectedPath string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != expectedPath {
+ t.Errorf("expected path=%s, got %s", expectedPath, r.URL.Path)
+ return
+ }
+ if r.Method != http.MethodGet && r.Header.Get("Authorization") != "Bearer test-key" {
+ got := r.Header.Get("Authorization")
+ t.Errorf("expected Authorization=Bearer test-key, got %q", got)
+ return
+ }
+ if r.Method == http.MethodPost {
+ if got := r.Header.Get("Content-Type"); got != "application/json" {
+ t.Errorf("expected Content-Type=application/json, got %q", got)
+ return
+ }
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("failed to read body: %v", err)
+ return
+ }
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("invalid JSON body: %v\n%s", err, string(raw))
+ return
+ }
+ handler(t, body, w)
+ return
+ }
+ // GET path: no body
+ handler(t, nil, w)
+ }))
+}
+
+func newCometAPIForTest(baseURL string) *CometAPIModel {
+ return NewCometAPIModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{
+ Chat: "v1/chat/completions",
+ Models: "api/models",
+ Embedding: "v1/embeddings",
+ Balance: "user/quota",
+ },
+ )
+}
+
+func TestCometAPIName(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ if got := m.Name(); got != "cometapi" {
+ t.Errorf("Name()=%q, want %q", got, "cometapi")
+ }
+}
+
+func TestCometAPIFactoryRoute(t *testing.T) {
+ driver, err := NewModelFactory().CreateModelDriver("cometapi", map[string]string{"default": "http://unused"}, URLSuffix{})
+ if err != nil {
+ t.Fatalf("CreateModelDriver: %v", err)
+ }
+ if _, ok := driver.(*CometAPIModel); !ok {
+ t.Fatalf("driver type=%T, want *CometAPIModel", driver)
+ }
+}
+
+func TestCometAPIChatHappyPath(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "gpt-5" {
+ t.Errorf("expected model=gpt-5, got %v", body["model"])
+ }
+ if body["stream"] != false {
+ t.Errorf("expected stream=false, got %v", body["stream"])
+ }
+ msgs, ok := body["messages"].([]interface{})
+ if !ok || len(msgs) != 1 {
+ t.Errorf("expected 1 message, got %v", body["messages"])
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{
+ {"message": map[string]interface{}{"content": "pong"}},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ resp, err := m.ChatWithMessages("gpt-5", []Message{
+ {Role: "user", Content: "ping"},
+ }, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if resp.Answer == nil || *resp.Answer != "pong" {
+ t.Errorf("answer=%v, want pong", resp.Answer)
+ }
+ if resp.ReasonContent == nil || *resp.ReasonContent != "" {
+ t.Errorf("expected empty reason content, got %v", resp.ReasonContent)
+ }
+}
+
+func TestCometAPIChatPropagatesConfig(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["max_tokens"] != float64(64) {
+ t.Errorf("max_tokens=%v want 64", body["max_tokens"])
+ }
+ if body["temperature"] != 0.3 {
+ t.Errorf("temperature=%v want 0.3", body["temperature"])
+ }
+ if body["top_p"] != 0.9 {
+ t.Errorf("top_p=%v want 0.9", body["top_p"])
+ }
+ stop, ok := body["stop"].([]interface{})
+ if !ok || len(stop) != 1 || stop[0] != "END" {
+ t.Errorf("stop=%v want [END]", body["stop"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{{"message": map[string]interface{}{"content": "ok"}}},
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ mt := 64
+ temp := 0.3
+ topP := 0.9
+ stop := []string{"END"}
+ _, err := m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "ping"}},
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{MaxTokens: &mt, Temperature: &temp, TopP: &topP, Stop: &stop},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+}
+
+func TestCometAPIChatReturnsReasoningContent(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{
+ {"message": map[string]interface{}{"content": "answer", "reasoning_content": "\nreason"}},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ resp, err := m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "ping"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if resp.ReasonContent == nil || *resp.ReasonContent != "reason" {
+ t.Errorf("reason=%v want reason", resp.ReasonContent)
+ }
+}
+
+func TestCometAPIChatRequiresAPIKey(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ _, err := m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "x"}}, &APIConfig{}, nil)
+ if err == nil || !strings.Contains(err.Error(), "api key is required") {
+ t.Errorf("expected api-key error, got %v", err)
+ }
+ emptyKey := ""
+ _, err = m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &emptyKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "api key is required") {
+ t.Errorf("empty key: expected api-key error, got %v", err)
+ }
+}
+
+func TestCometAPIChatRequiresModelName(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ apiKey := "test-key"
+ _, err := m.ChatWithMessages("", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("expected model-name error, got %v", err)
+ }
+ err = m.ChatStreamlyWithSender(" ", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil, func(*string, *string) error { return nil })
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("stream: expected model-name error, got %v", err)
+ }
+}
+
+func TestCometAPIChatRequiresMessages(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ apiKey := "test-key"
+ _, err := m.ChatWithMessages("gpt-5", nil, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "messages is empty") {
+ t.Errorf("expected messages-empty error, got %v", err)
+ }
+}
+
+func TestCometAPIChatRejectsHTTPError(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error":"unauthorized"}`))
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ _, err := m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "401") {
+ t.Errorf("expected 401 propagated, got %v", err)
+ }
+}
+
+func TestCometAPIChatFallsBackToDefaultOnEmptyRegion(t *testing.T) {
+ // Empty *Region pointer must fall back to the "default" entry, not
+ // be treated as an explicit "" region (which would miss the lookup).
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{{"message": map[string]interface{}{"content": "ok"}}},
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ emptyRegion := ""
+ _, err := m.ChatWithMessages("gpt-5",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey, Region: &emptyRegion}, nil)
+ if err != nil {
+ t.Errorf("empty Region: expected fallback to default, got %v", err)
+ }
+}
+
+func TestCometAPIListModelsFallsBackToDefaultOnEmptyRegion(t *testing.T) {
+ srv := newCometAPIServer(t, "/api/models", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": []map[string]interface{}{{"id": "x"}}})
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ emptyRegion := ""
+ if _, err := m.ListModels(&APIConfig{ApiKey: &apiKey, Region: &emptyRegion}); err != nil {
+ t.Errorf("empty Region: expected fallback to default, got %v", err)
+ }
+}
+
+func TestCometAPIStreamRequiresSender(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ apiKey := "test-key"
+ err := m.ChatStreamlyWithSender("gpt-5",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err == nil || !strings.Contains(err.Error(), "sender is required") {
+ t.Errorf("expected sender-required error, got %v", err)
+ }
+}
+
+func TestCometAPIChatRejectsUnknownRegion(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ apiKey := "test-key"
+ region := "eu"
+ _, err := m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey, Region: ®ion}, nil)
+ if err == nil || !strings.Contains(err.Error(), "no base URL configured for region") {
+ t.Errorf("expected region error, got %v", err)
+ }
+}
+
+func TestCometAPIBaseURLNormalizesSlashes(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ run func(*CometAPIModel, *APIConfig) error
+ }{
+ {
+ name: "Chat",
+ path: "/v1/chat/completions",
+ run: func(m *CometAPIModel, apiConfig *APIConfig) error {
+ _, err := m.ChatWithMessages("gpt-5", []Message{{Role: "user", Content: "x"}}, apiConfig, nil)
+ return err
+ },
+ },
+ {
+ name: "Stream",
+ path: "/v1/chat/completions",
+ run: func(m *CometAPIModel, apiConfig *APIConfig) error {
+ return m.ChatStreamlyWithSender("gpt-5", []Message{{Role: "user", Content: "x"}}, apiConfig, nil, func(*string, *string) error { return nil })
+ },
+ },
+ {
+ name: "Embed",
+ path: "/v1/embeddings",
+ run: func(m *CometAPIModel, apiConfig *APIConfig) error {
+ model := "text-embedding-3-small"
+ _, err := m.Embed(&model, []string{"x"}, apiConfig, nil)
+ return err
+ },
+ },
+ {
+ name: "ListModels",
+ path: "/api/models",
+ run: func(m *CometAPIModel, apiConfig *APIConfig) error {
+ _, err := m.ListModels(apiConfig)
+ return err
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ srv := newCometAPIServer(t, tt.path, func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ switch tt.name {
+ case "Chat":
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"choices": []map[string]interface{}{{"message": map[string]interface{}{"content": "ok"}}}})
+ case "Stream":
+ w.Header().Set("Content-Type", "text/event-stream")
+ _, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}]}`+"\n")
+ case "Embed":
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": []map[string]interface{}{{"embedding": []float64{1}, "index": 0}}})
+ case "ListModels":
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": []map[string]interface{}{{"id": "gpt-5"}}})
+ }
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL + "/")
+ m.URLSuffix.Chat = "/v1/chat/completions"
+ m.URLSuffix.Models = "/api/models"
+ m.URLSuffix.Embedding = "/v1/embeddings"
+ apiKey := "test-key"
+ if err := tt.run(m, &APIConfig{ApiKey: &apiKey}); err != nil {
+ t.Fatalf("%s: %v", tt.name, err)
+ }
+ })
+ }
+}
+
+func TestCometAPIStreamHappyPath(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["stream"] != true {
+ t.Errorf("expected stream=true, got %v", body["stream"])
+ }
+ w.Header().Set("Content-Type", "text/event-stream")
+ w.WriteHeader(http.StatusOK)
+ // Two content chunks then finish_reason terminator, then [DONE].
+ _, _ = io.WriteString(w,
+ `data: {"choices":[{"delta":{"content":"Hello "}}]}`+"\n"+
+ `data: {"choices":[{"delta":{"content":"world"}}]}`+"\n"+
+ `data: {"choices":[{"delta":{},"finish_reason":"stop"}]}`+"\n"+
+ `data: [DONE]`+"\n",
+ )
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ var chunks []string
+ var sawDone int32
+ err := m.ChatStreamlyWithSender("gpt-5",
+ []Message{{Role: "user", Content: "hi"}},
+ &APIConfig{ApiKey: &apiKey}, nil,
+ func(content *string, _ *string) error {
+ if content == nil {
+ return nil
+ }
+ if *content == "[DONE]" {
+ atomic.StoreInt32(&sawDone, 1)
+ return nil
+ }
+ chunks = append(chunks, *content)
+ return nil
+ },
+ )
+ if err != nil {
+ t.Fatalf("stream: %v", err)
+ }
+ if strings.Join(chunks, "") != "Hello world" {
+ t.Errorf("chunks=%v want [\"Hello \" \"world\"]", chunks)
+ }
+ if atomic.LoadInt32(&sawDone) != 1 {
+ t.Error("expected sender to receive [DONE] sentinel")
+ }
+}
+
+func TestCometAPIStreamRejectsExplicitFalse(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ apiKey := "test-key"
+ stream := false
+ err := m.ChatStreamlyWithSender("gpt-5",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{Stream: &stream},
+ func(*string, *string) error { return nil },
+ )
+ if err == nil || !strings.Contains(err.Error(), "stream must be true") {
+ t.Errorf("expected stream-true guard, got %v", err)
+ }
+}
+
+func TestCometAPIStreamFailsWithoutTerminal(t *testing.T) {
+ // Body closes before [DONE] or a finish_reason -> driver must complain
+ // instead of pretending the stream finished cleanly.
+ srv := newCometAPIServer(t, "/v1/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusOK)
+ _, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"half"}}]}`+"\n")
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ err := m.ChatStreamlyWithSender("gpt-5",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &apiKey}, nil,
+ func(*string, *string) error { return nil },
+ )
+ if err == nil || !strings.Contains(err.Error(), "stream ended before") {
+ t.Errorf("expected stream-truncation error, got %v", err)
+ }
+}
+
+func TestCometAPIListModelsHappyPath(t *testing.T) {
+ srv := newCometAPIServer(t, "/api/models", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"id": "gpt-5"},
+ {"id": "gpt-4o-mini"},
+ {"id": "text-embedding-3-small"},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ ids, err := m.ListModels(&APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ if len(ids) != 3 || ids[0] != "gpt-5" || ids[2] != "text-embedding-3-small" {
+ t.Errorf("ids=%v, want [gpt-5 gpt-4o-mini text-embedding-3-small]", ids)
+ }
+}
+
+func TestCometAPIListModelsAllowsNilAPIConfig(t *testing.T) {
+ srv := newCometAPIServer(t, "/api/models", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": []map[string]interface{}{{"id": "gpt-5"}}})
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ ids, err := m.ListModels(nil)
+ if err != nil {
+ t.Fatalf("ListModels(nil): %v", err)
+ }
+ if len(ids) != 1 || ids[0] != "gpt-5" {
+ t.Errorf("ids=%v want [gpt-5]", ids)
+ }
+}
+
+func TestCometAPICheckConnectionDelegatesToBalance(t *testing.T) {
+ // 200 -> CheckConnection succeeds; 401 -> CheckConnection propagates.
+ okSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/user/quota" {
+ t.Errorf("path=%s want /user/quota", r.URL.Path)
+ }
+ if got := r.URL.Query().Get("key"); got != "test-key" {
+ t.Errorf("key query=%q want test-key", got)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"total_quota": 10.0})
+ }))
+ defer okSrv.Close()
+ failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ defer failSrv.Close()
+
+ apiKey := "test-key"
+ mOK := newCometAPIForTest(okSrv.URL)
+ mOK.URLSuffix.Balance = okSrv.URL + "/user/quota"
+ if err := mOK.CheckConnection(&APIConfig{ApiKey: &apiKey}); err != nil {
+ t.Errorf("CheckConnection(ok): %v", err)
+ }
+ mFail := newCometAPIForTest(failSrv.URL)
+ mFail.URLSuffix.Balance = failSrv.URL + "/user/quota"
+ if err := mFail.CheckConnection(&APIConfig{ApiKey: &apiKey}); err == nil {
+ t.Error("CheckConnection(fail): expected error, got nil")
+ }
+}
+
+func TestCometAPIBalanceHappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/user/quota" {
+ t.Errorf("path=%s want /user/quota", r.URL.Path)
+ }
+ if got := r.URL.Query().Get("key"); got != "test-key" {
+ t.Errorf("key query=%q want test-key", got)
+ }
+ if got := r.Header.Get("Authorization"); got != "" {
+ t.Errorf("Authorization=%q want empty", got)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "username": "tester",
+ "total_quota": 20.5,
+ "total_used_quota": 1.25,
+ "request_count": 7,
+ })
+ }))
+ defer srv.Close()
+
+ m := newCometAPIForTest("http://unused")
+ m.URLSuffix.Balance = srv.URL + "/user/quota"
+ apiKey := "test-key"
+ balance, err := m.Balance(&APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("Balance: %v", err)
+ }
+ if balance["username"] != "tester" || balance["total_quota"] != 20.5 {
+ t.Errorf("balance=%v", balance)
+ }
+}
+
+func TestCometAPIBalanceRequiresAPIKey(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ _, err := m.Balance(&APIConfig{})
+ if err == nil || !strings.Contains(err.Error(), "api key is required") {
+ t.Errorf("Balance: expected api-key error, got %v", err)
+ }
+}
+
+func TestCometAPIBalanceRequiresConfiguredURL(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ m.URLSuffix.Balance = ""
+ apiKey := "test-key"
+ _, err := m.Balance(&APIConfig{ApiKey: &apiKey})
+ if err == nil || !strings.Contains(err.Error(), "balance URL is required") {
+ t.Errorf("Balance: expected balance URL error, got %v", err)
+ }
+}
+
+func TestCometAPIRerankReturnsNoSuchMethod(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ q := "gpt-5"
+ _, err := m.Rerank(&q, "what is rag?", []string{"a", "b"}, &APIConfig{}, &RerankConfig{TopN: 2})
+ if err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Rerank: expected 'no such method', got %v", err)
+ }
+}
+
+func TestCometAPIEmbedHappyPath(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/embeddings", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "text-embedding-3-small" {
+ t.Errorf("model=%v want text-embedding-3-small", body["model"])
+ }
+ if body["dimensions"] != float64(256) {
+ t.Errorf("dimensions=%v want 256", body["dimensions"])
+ }
+ inputs, ok := body["input"].([]interface{})
+ if !ok || len(inputs) != 3 {
+ t.Errorf("input=%v want 3-element array", body["input"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{0.1, 0.2}, "index": 0},
+ {"embedding": []float64{0.3, 0.4}, "index": 1},
+ {"embedding": []float64{0.5, 0.6}, "index": 2},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ vecs, err := m.Embed(&model, []string{"a", "b", "c"}, &APIConfig{ApiKey: &apiKey}, &EmbeddingConfig{Dimension: 256})
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+ if len(vecs) != 3 {
+ t.Fatalf("len(vecs)=%d want 3", len(vecs))
+ }
+ if vecs[1].Embedding[0] != 0.3 || vecs[1].Index != 1 {
+ t.Errorf("vecs[1]=%+v want {Embedding:[0.3 0.4] Index:1}", vecs[1])
+ }
+}
+
+func TestCometAPIEmbedReordersByIndex(t *testing.T) {
+ // Upstream returns the three vectors in shuffled order. The driver
+ // must reorder them so the slot at position i corresponds to input i.
+ srv := newCometAPIServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{2}, "index": 2},
+ {"embedding": []float64{0}, "index": 0},
+ {"embedding": []float64{1}, "index": 1},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ vecs, err := m.Embed(&model, []string{"a", "b", "c"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+ for i, v := range vecs {
+ if v.Index != i || v.Embedding[0] != float64(i) {
+ t.Errorf("slot %d = %+v, want Embedding=[%d] Index=%d", i, v, i, i)
+ }
+ }
+}
+
+func TestCometAPIEmbedEmptyInputShortCircuits(t *testing.T) {
+ // Empty input must NOT make an HTTP call; the test fails the request
+ // rather than the assertion if it does.
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ t.Error("Embed([]) made an unexpected HTTP call")
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ vecs, err := m.Embed(&model, []string{}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("Embed([]): %v", err)
+ }
+ if len(vecs) != 0 {
+ t.Errorf("len(vecs)=%d want 0", len(vecs))
+ }
+}
+
+func TestCometAPIEmbedRequiresAPIKey(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ model := "text-embedding-3-small"
+ _, err := m.Embed(&model, []string{"a"}, &APIConfig{}, nil)
+ if err == nil || !strings.Contains(err.Error(), "api key is required") {
+ t.Errorf("expected api-key error, got %v", err)
+ }
+}
+
+func TestCometAPIEmbedRequiresModelName(t *testing.T) {
+ m := newCometAPIForTest("http://unused")
+ apiKey := "test-key"
+ _, err := m.Embed(nil, []string{"a"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("expected model-name error, got %v", err)
+ }
+ empty := ""
+ _, err = m.Embed(&empty, []string{"a"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("empty model: expected model-name error, got %v", err)
+ }
+}
+
+func TestCometAPIEmbedRejectsDuplicateIndex(t *testing.T) {
+ // A malformed upstream that repeats data[*].index would silently
+ // overwrite the earlier vector; the driver must fail loudly instead.
+ srv := newCometAPIServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{1}, "index": 0},
+ {"embedding": []float64{2}, "index": 0},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ _, err := m.Embed(&model, []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "duplicate embedding index 0") {
+ t.Errorf("expected duplicate-index error, got %v", err)
+ }
+}
+
+func TestCometAPIEmbedRejectsOutOfRangeIndex(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{1}, "index": 7}, // out of range for 2-input request
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ _, err := m.Embed(&model, []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "out of range") {
+ t.Errorf("expected out-of-range error, got %v", err)
+ }
+}
+
+func TestCometAPIEmbedRejectsMissingSlot(t *testing.T) {
+ // Upstream returns only one of the two requested embeddings.
+ srv := newCometAPIServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{
+ {"embedding": []float64{1}, "index": 0},
+ },
+ })
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ _, err := m.Embed(&model, []string{"a", "b"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "missing embedding for input index 1") {
+ t.Errorf("expected missing-embedding error for slot 1, got %v", err)
+ }
+}
+
+func TestCometAPIEmbedRejectsHTTPError(t *testing.T) {
+ srv := newCometAPIServer(t, "/v1/embeddings", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error":"unauthorized"}`))
+ })
+ defer srv.Close()
+
+ m := newCometAPIForTest(srv.URL)
+ apiKey := "test-key"
+ model := "text-embedding-3-small"
+ _, err := m.Embed(&model, []string{"a"}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "CometAPI embeddings API error") {
+ t.Errorf("expected CometAPI embeddings API error, got %v", err)
+ }
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index b1ca708051c..3989a85881c 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -73,6 +73,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewBaiduModel(baseURL, urlSuffix), nil
case "cohere":
return NewCoHereModel(baseURL, urlSuffix), nil
+ case "cometapi":
+ return NewCometAPIModel(baseURL, urlSuffix), nil
case "fishaudio":
return NewFishAudioModel(baseURL, urlSuffix), nil
case "mistral":
From fe82a961932fdb656d4d65517e3705746b361e0c Mon Sep 17 00:00:00 2001
From: dale053
Date: Sun, 17 May 2026 23:32:44 -0700
Subject: [PATCH 179/666] Fix: add SSRF guard for agent test_db_connection
endpoint (#14860)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Closes #14858
The `test_db_connection` endpoint in the agent API accepts a
user-supplied `host` and connects to it directly via database drivers
(MySQL/PostgreSQL) without any validation. This allows an attacker to
probe internal network addresses (e.g. `127.0.0.1`, `10.x.x.x`,
link-local, etc.) through the server — a classic Server-Side Request
Forgery (SSRF) vulnerability.
This PR adds an SSRF guard that resolves the host and rejects any
address that is not globally routable before the database connection is
attempted.
**Changes:**
- **`common/ssrf_guard.py`** — Added `assert_host_is_safe()`, a
host-level counterpart of the existing `assert_url_is_safe()`, designed
for non-HTTP protocols (database drivers) where there is no URL to
parse.
- **`api/apps/restful_apis/agent_api.py`** — Call
`assert_host_is_safe(req["host"])` at the top of `test_db_connection` so
that non-public hosts are rejected early with a clear error message.
Fixes #14858
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---------
Co-authored-by: Jin Hai
---
api/apps/restful_apis/agent_api.py | 30 +++++++++++++++++------
common/ssrf_guard.py | 39 ++++++++++++++++++++++++++++++
2 files changed, 62 insertions(+), 7 deletions(-)
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index 4d078425922..f88ce90b3f5 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -60,6 +60,7 @@
validate_request,
)
from common import settings
+from common.ssrf_guard import assert_host_is_safe
from common.constants import RetCode
from common.misc_utils import get_uuid, thread_pool_exec
from peewee import MySQLDatabase, PostgresqlDatabase
@@ -782,12 +783,27 @@ async def rerun_agent(tenant_id):
@login_required
async def test_db_connection():
req = await get_request_json()
+ try:
+ safe_host = assert_host_is_safe(req["host"])
+ except ValueError as exc:
+ logging.warning(
+ "Rejected test_db_connection: unsafe host %r (db_type=%s, user=%s): %s",
+ req.get("host"), req.get("db_type"), current_user.id, exc,
+ )
+ return get_data_error_result(message=str(exc))
+ except OSError as exc:
+ logging.warning(
+ "Rejected test_db_connection: cannot resolve host %r (db_type=%s, user=%s): %s",
+ req.get("host"), req.get("db_type"), current_user.id, exc,
+ )
+ logging.debug("Full resolver exception for host %r", req.get("host"), exc_info=True)
+ return get_data_error_result(message=f"Could not resolve host {req.get('host')!r}.")
try:
if req["db_type"] in ["mysql", "mariadb"]:
db = MySQLDatabase(
req["database"],
user=req["username"],
- host=req["host"],
+ host=safe_host,
port=req["port"],
password=req["password"],
)
@@ -797,7 +813,7 @@ async def test_db_connection():
db = MySQLDatabase(
req["database"],
user=req["username"],
- host=req["host"],
+ host=safe_host,
port=req["port"],
password=req["password"],
charset="utf8mb4",
@@ -808,7 +824,7 @@ async def test_db_connection():
db = PostgresqlDatabase(
req["database"],
user=req["username"],
- host=req["host"],
+ host=safe_host,
port=req["port"],
password=req["password"],
)
@@ -819,7 +835,7 @@ async def test_db_connection():
connection_string = (
f"DRIVER={{ODBC Driver 17 for SQL Server}};"
- f"SERVER={req['host']},{req['port']};"
+ f"SERVER={safe_host},{req['port']};"
f"DATABASE={req['database']};"
f"UID={req['username']};"
f"PWD={req['password']};"
@@ -838,7 +854,7 @@ async def test_db_connection():
conn_str = (
f"DATABASE={req['database']};"
- f"HOSTNAME={req['host']};"
+ f"HOSTNAME={safe_host};"
f"PORT={req['port']};"
f"PROTOCOL=TCPIP;"
f"UID={req['username']};"
@@ -847,7 +863,7 @@ async def test_db_connection():
logging.info(
"DATABASE=%s;HOSTNAME=%s;PORT=%s;PROTOCOL=TCPIP;UID=%s;PWD=****;",
req["database"],
- req["host"],
+ safe_host,
req["port"],
req["username"],
)
@@ -873,7 +889,7 @@ async def test_db_connection():
auth = trino.BasicAuthentication(req.get("username") or "ragflow", req["password"])
conn = trino.dbapi.connect(
- host=req["host"],
+ host=safe_host,
port=int(req["port"] or 8080),
user=req["username"] or "ragflow",
catalog=catalog,
diff --git a/common/ssrf_guard.py b/common/ssrf_guard.py
index b60bcd4bc99..4f87b94d7b8 100644
--- a/common/ssrf_guard.py
+++ b/common/ssrf_guard.py
@@ -170,3 +170,42 @@ def assert_url_is_safe(
raise ValueError(f"Hostname {hostname!r} resolved to no addresses.")
return hostname, resolved_ip
+
+
+def assert_host_is_safe(host: str) -> str:
+ """Raise ``ValueError`` if *host* resolves to a non-public IP (SSRF guard for raw host/port connections).
+
+ This is the host-level counterpart of :func:`assert_url_is_safe`, intended
+ for callers that connect via database drivers or other non-HTTP protocols
+ where there is no URL to parse.
+
+ Returns the first validated public IP string so the caller can pin it if needed.
+ """
+ if not host:
+ raise ValueError("Host must not be empty.")
+
+ try:
+ addr_infos = socket.getaddrinfo(host, None)
+ except socket.gaierror as exc:
+ logger.warning("SSRF guard could not resolve host=%r reason=%s", host, exc)
+ raise ValueError(f"Could not resolve host {host!r}: {exc}") from exc
+
+ resolved_ip: str | None = None
+ for _family, _type, _proto, _canonname, sockaddr in addr_infos:
+ raw_ip = ipaddress.ip_address(sockaddr[0])
+ eff_ip = _effective_ip(raw_ip)
+ if not eff_ip.is_global:
+ logger.warning(
+ "SSRF guard blocked host: host=%r resolved to non-public address=%s",
+ host,
+ raw_ip,
+ )
+ raise ValueError(f"Host resolves to a non-public address ({raw_ip}), which is not allowed.")
+ if resolved_ip is None:
+ resolved_ip = str(raw_ip)
+
+ if resolved_ip is None:
+ logger.warning("SSRF guard blocked host: host=%r resolved to no addresses", host)
+ raise ValueError(f"Host {host!r} resolved to no addresses.")
+
+ return resolved_ip
From b40b0bf99694bf72ab97c95ac440e91231e8804b Mon Sep 17 00:00:00 2001
From: buua436
Date: Mon, 18 May 2026 15:07:07 +0800
Subject: [PATCH 180/666] Go: fix siliconflow embedding response (#14975)
### What problem does this PR solve?
fix siliconflow embedding response
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
internal/entity/models/siliconflow.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index 11ba909365c..7e9d4f7c576 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -372,7 +372,7 @@ func (z *SiliconflowModel) ChatStreamlyWithSender(modelName string, messages []M
}
type siliconflowEmbeddingResponse struct {
- Object []string `json:"object"`
+ Object string `json:"object"`
Model string `json:"model"`
Data []siliconflowEmbeddingData `json:"data"`
Usage siliconflowUsage `json:"usage"`
From 56d73d0c2c833215e716b3b43523a675dc21f93d Mon Sep 17 00:00:00 2001
From: Wang Qi
Date: Mon, 18 May 2026 15:55:59 +0800
Subject: [PATCH 181/666] Refactor: speed up ragflow server, save startup
memory (#14973)
### What problem does this PR solve?
Refactor: speed up ragflow server, save startup memory, saved 200MiB,
and 5-9 seconds start time.
##### Before
1241292 | | \_ python3 api/ragflow_server.py
RAGFlow server is ready after 25.61845850944519s initialization.
##### After
1019968 | | \_ python3 api/ragflow_server.py
RAGFlow server is ready after 16.205134391784668s initialization.
### Type of change
- [x] Refactoring
---
api/apps/llm_app.py | 8 +++++--
api/apps/restful_apis/agent_api.py | 33 ++++++++++++++++++++++-----
api/apps/restful_apis/chunk_api.py | 15 ++++++++++--
api/db/services/tenant_llm_service.py | 3 ++-
4 files changed, 48 insertions(+), 11 deletions(-)
diff --git a/api/apps/llm_app.py b/api/apps/llm_app.py
index e1d4954df48..58997fb4df4 100644
--- a/api/apps/llm_app.py
+++ b/api/apps/llm_app.py
@@ -25,8 +25,6 @@
from api.utils.api_utils import get_allowed_llm_factories, get_data_error_result, get_json_result, get_request_json, server_error_response, validate_request
from common.constants import StatusEnum, LLMType
from api.db.db_models import TenantLLM
-from rag.utils.base64_image import test_image
-from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel, OcrModel, Seq2txtModel
def _resolve_my_llm_is_tools(o_dict: dict) -> bool:
@@ -78,6 +76,8 @@ def factories():
@validate_request("llm_factory", "api_key")
async def set_api_key():
req = await get_request_json()
+ from rag.llm import ChatModel, EmbeddingModel, RerankModel
+
# test if api key works
chat_passed, embd_passed, rerank_passed = False, False, False
factory = req["llm_factory"]
@@ -178,6 +178,8 @@ async def check_streamly():
@validate_request("llm_factory")
async def add_llm():
req = await get_request_json()
+ from rag.llm import ChatModel, CvModel, EmbeddingModel, OcrModel, RerankModel, Seq2txtModel, TTSModel
+
factory = req["llm_factory"]
api_key = req.get("api_key", "x")
llm_name = req.get("llm_name")
@@ -318,6 +320,8 @@ async def check_streamly():
msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
case LLMType.IMAGE2TEXT.value:
+ from rag.utils.base64_image import test_image
+
assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
mdl = CvModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
try:
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index f88ce90b3f5..8478571ac1d 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -29,9 +29,6 @@
import jwt
from quart import Response, jsonify, request
-from agent.canvas import Canvas
-from agent.component import LLM
-from agent.dsl_migration import normalize_chunker_dsl
from api.apps import current_user, login_required
from api.apps.services.canvas_replica_service import CanvasReplicaService
from api.db import CanvasCategory
@@ -64,9 +61,6 @@
from common.constants import RetCode
from common.misc_utils import get_uuid, thread_pool_exec
from peewee import MySQLDatabase, PostgresqlDatabase
-from rag.flow.pipeline import Pipeline
-from rag.nlp import search
-from rag.utils.redis_conn import REDIS_CONN
def _require_canvas_access_sync(func):
@@ -195,6 +189,8 @@ def list_agent_sessions(agent_id, tenant_id):
@add_tenant_id_to_kwargs
@_require_canvas_access_async
async def create_agent_session(agent_id, tenant_id):
+ from agent.canvas import Canvas
+
req = await get_request_json()
user_id = req.get("user_id") or request.args.get("user_id", tenant_id)
release_mode = bool(req.get("release", request.args.get("release", False)))
@@ -522,6 +518,8 @@ async def upload_agent_file(agent_id, tenant_id):
@_require_canvas_access_sync
def get_agent_component_input_form(agent_id, component_id, tenant_id):
try:
+ from agent.canvas import Canvas
+
exists, user_canvas = UserCanvasService.get_by_id(agent_id)
if not exists:
return get_data_error_result(message="canvas not found.")
@@ -539,6 +537,9 @@ def get_agent_component_input_form(agent_id, component_id, tenant_id):
async def debug_agent_component(agent_id, component_id, tenant_id):
req = await get_request_json()
try:
+ from agent.canvas import Canvas
+ from agent.component import LLM
+
_, user_canvas = UserCanvasService.get_by_id(agent_id)
canvas = Canvas(json.dumps(user_canvas.dsl), tenant_id, canvas_id=user_canvas.id)
canvas.reset()
@@ -597,6 +598,8 @@ def get_agent(agent_id, tenant_id):
released_versions.sort(key=lambda version: version.update_time, reverse=True)
last_publish_time = released_versions[0].update_time
+ from agent.dsl_migration import normalize_chunker_dsl
+
canvas["dsl"] = normalize_chunker_dsl(canvas.get("dsl", {}))
canvas["last_publish_time"] = last_publish_time
@@ -642,6 +645,8 @@ def get_agent_version(agent_id, version_id, tenant_id):
@_require_canvas_access_async
async def get_agent_logs(agent_id, message_id, tenant_id):
try:
+ from rag.utils.redis_conn import REDIS_CONN
+
binary = await thread_pool_exec(REDIS_CONN.get, f"{agent_id}-{message_id}-logs")
if not binary:
return get_json_result(data={})
@@ -719,6 +724,8 @@ async def update_agent(agent_id, tenant_id):
@_require_canvas_access_async
async def reset_agent(agent_id, tenant_id):
try:
+ from agent.canvas import Canvas
+
exists, user_canvas = UserCanvasService.get_by_id(agent_id)
if not exists:
return get_data_error_result(message="canvas not found.")
@@ -747,6 +754,8 @@ async def reset_agent(agent_id, tenant_id):
@login_required
@add_tenant_id_to_kwargs
async def rerun_agent(tenant_id):
+ from rag.nlp import search
+
req = await get_request_json()
doc = PipelineOperationLogService.get_documents_info(req["id"])
if not doc:
@@ -1042,6 +1051,8 @@ async def agent_chat_completion(tenant_id, agent_id=None):
dsl_str = json.dumps(replica_dsl, ensure_ascii=False)
if cvs.canvas_category == CanvasCategory.DataFlow:
+ from rag.flow.pipeline import Pipeline
+
task_id = get_uuid()
Pipeline(
dsl_str,
@@ -1064,6 +1075,8 @@ async def agent_chat_completion(tenant_id, agent_id=None):
return get_json_result(data={"message_id": task_id})
try:
+ from agent.canvas import Canvas
+
canvas = Canvas(dsl_str, str(tenant_id), canvas_id=agent_id, custom_header=custom_header)
except Exception as exc:
return server_error_response(exc)
@@ -1349,6 +1362,8 @@ def _validate_rate_limit(security_cfg):
now = time.time()
try:
+ from rag.utils.redis_conn import REDIS_CONN
+
res = REDIS_CONN.lua_token_bucket(
keys=[key],
args=[capacity, rate, now, cost],
@@ -1456,6 +1471,8 @@ def _validate_jwt_auth(security_cfg):
if not isinstance(cvs.dsl, str):
dsl = json.dumps(cvs.dsl, ensure_ascii=False)
try:
+ from agent.canvas import Canvas
+
canvas = Canvas(dsl, cvs.user_id, agent_id, canvas_id=agent_id)
except Exception as e:
resp=get_data_error_result(code=RetCode.BAD_REQUEST,message=str(e))
@@ -1709,6 +1726,8 @@ def validate_type(value, t):
response_cfg = webhook_cfg.get("response", {})
def append_webhook_trace(agent_id: str, start_ts: float,event: dict, ttl=600):
+ from rag.utils.redis_conn import REDIS_CONN
+
key = f"webhook-trace-{agent_id}-logs"
raw = REDIS_CONN.get(key)
@@ -1908,6 +1927,8 @@ def decode_webhook_id(enc_id: str, webhooks: dict) -> str | None:
webhook_id = request.args.get("webhook_id")
key = f"webhook-trace-{agent_id}-logs"
+ from rag.utils.redis_conn import REDIS_CONN
+
raw = REDIS_CONN.get(key)
if since_ts is None:
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.py
index d3a30710e86..fe45209dd01 100644
--- a/api/apps/restful_apis/chunk_api.py
+++ b/api/apps/restful_apis/chunk_api.py
@@ -43,8 +43,6 @@
from common.misc_utils import thread_pool_exec
from common.string_utils import is_content_empty, remove_redundant_spaces
from common.tag_feature_utils import validate_tag_features
-from rag.app.qa import beAdoc, rmPrefix
-from rag.nlp import rag_tokenizer, search
class Chunk(BaseModel):
@@ -107,6 +105,8 @@ def _get_dataset_tenant_id(dataset_id):
@login_required
@add_tenant_id_to_kwargs
async def list_chunks(tenant_id, dataset_id, document_id):
+ from rag.nlp import search
+
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
@@ -191,6 +191,8 @@ async def list_chunks(tenant_id, dataset_id, document_id):
@login_required
@add_tenant_id_to_kwargs
async def get_chunk(tenant_id, dataset_id, document_id, chunk_id):
+ from rag.nlp import search
+
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
@@ -214,6 +216,8 @@ async def get_chunk(tenant_id, dataset_id, document_id, chunk_id):
@login_required
@add_tenant_id_to_kwargs
async def add_chunk(tenant_id, dataset_id, document_id):
+ from rag.nlp import rag_tokenizer, search
+
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
@@ -303,6 +307,8 @@ async def add_chunk(tenant_id, dataset_id, document_id):
@login_required
@add_tenant_id_to_kwargs
async def rm_chunk(tenant_id, dataset_id, document_id):
+ from rag.nlp import search
+
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
@@ -350,6 +356,9 @@ async def rm_chunk(tenant_id, dataset_id, document_id):
@login_required
@add_tenant_id_to_kwargs
async def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
+ from rag.app.qa import beAdoc, rmPrefix
+ from rag.nlp import rag_tokenizer, search
+
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
@@ -436,6 +445,8 @@ async def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
@login_required
@add_tenant_id_to_kwargs
async def switch_chunks(tenant_id, dataset_id, document_id):
+ from rag.nlp import search
+
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
diff --git a/api/db/services/tenant_llm_service.py b/api/db/services/tenant_llm_service.py
index ee2eab6648a..f14f97fcef6 100644
--- a/api/db/services/tenant_llm_service.py
+++ b/api/db/services/tenant_llm_service.py
@@ -24,7 +24,6 @@
from api.db.services.common_service import CommonService
from api.db.services.langfuse_service import TenantLangfuseService
from api.db.services.user_service import TenantService
-from rag.llm import ChatModel, CvModel, EmbeddingModel, OcrModel, RerankModel, Seq2txtModel, TTSModel
class LLMFactoriesService(CommonService):
@@ -183,6 +182,8 @@ def get_model_config(cls, tenant_id, llm_type, llm_name=None):
def model_instance(cls, model_config: dict, lang="Chinese", **kwargs):
if not model_config:
raise LookupError("Model config is required")
+ from rag.llm import ChatModel, CvModel, EmbeddingModel, OcrModel, RerankModel, Seq2txtModel, TTSModel
+
kwargs.update({"provider": model_config["llm_factory"]})
api_key = model_config.get("api_key_payload", model_config["api_key"])
if model_config["model_type"] == LLMType.EMBEDDING.value:
From b12eaee38b77681dc27abadaf5a54957c917b453 Mon Sep 17 00:00:00 2001
From: dev <280872861+dev111-actor@users.noreply.github.com>
Date: Sun, 17 May 2026 22:09:26 -1000
Subject: [PATCH 182/666] fix(api): enforce tenant access for connector routes
(#14747)
### What problem does this PR solve?
Fixes #14746.
Adds tenant access checks for connector-by-id REST routes before reading
connector details, mutating connector config/status, deleting
connectors, rebuilding, or listing sync logs. Unauthorized callers now
receive `RetCode.AUTHENTICATION_ERROR` with `No authorization.` without
reaching the connector/log mutation paths.
Validation:
- `python3 -m pytest
--confcutdir=test/testcases/test_web_api/test_connector_app
test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py`
- `uvx ruff check api/apps/restful_apis/connector_api.py
api/db/services/connector_service.py
test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py`
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
Co-authored-by: dev111-actor
---
api/apps/restful_apis/connector_api.py | 43 ++++++++++++++++++-
api/db/services/connector_service.py | 31 ++++++++++++-
.../restful_api/test_connector_routes_unit.py | 41 ++++++++++++++++++
.../test_connector_routes_unit.py | 41 ++++++++++++++++++
4 files changed, 153 insertions(+), 3 deletions(-)
diff --git a/api/apps/restful_apis/connector_api.py b/api/apps/restful_apis/connector_api.py
index 5e428d1a223..21ab7fd4d00 100644
--- a/api/apps/restful_apis/connector_api.py
+++ b/api/apps/restful_apis/connector_api.py
@@ -35,9 +35,23 @@
from api.apps import login_required, current_user
from box_sdk_gen import BoxOAuth, OAuthConfig, GetAuthorizeUrlOptions
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _connector_auth_error(connector_id: str, user_id: str):
+ """Return the connector authorization failure response and log the denial."""
+ LOGGER.warning("connector access denied: connector_id=%s user_id=%s", connector_id, user_id)
+ return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
+
+
@manager.route("/connectors/", methods=["PATCH"]) # noqa: F821
@login_required
async def update_connector(connector_id):
+ """Update an accessible connector's polling configuration."""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
req = await get_request_json()
e, conn = ConnectorService.get_by_id(connector_id)
if not e:
@@ -57,6 +71,7 @@ async def update_connector(connector_id):
@manager.route("/connectors", methods=["POST"]) # noqa: F821
@login_required
async def create_connector():
+ """Create a connector owned by the current tenant."""
req = await get_request_json()
if req:
req["id"] = get_uuid()
@@ -83,12 +98,17 @@ async def create_connector():
@manager.route("/connectors", methods=["GET"]) # noqa: F821
@login_required
def list_connector():
+ """List connectors owned by the current tenant."""
return get_json_result(data=ConnectorService.list(current_user.id))
@manager.route("/connectors/", methods=["GET"]) # noqa: F821
@login_required
def get_connector(connector_id):
+ """Return connector details when the current user can access it."""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
e, conn = ConnectorService.get_by_id(connector_id)
if not e:
return get_data_error_result(message="Can't find this Connector!")
@@ -98,6 +118,10 @@ def get_connector(connector_id):
@manager.route("/connectors//logs", methods=["GET"]) # noqa: F821
@login_required
def list_logs(connector_id):
+ """List sync logs for a connector the current user can access."""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
req = request.args.to_dict(flat=True)
arr, total = SyncLogsService.list_sync_tasks(connector_id, int(req.get("page", 1)), int(req.get("page_size", 15)))
return get_json_result(data={"total": total, "logs": arr})
@@ -106,6 +130,10 @@ def list_logs(connector_id):
@manager.route("/connectors//resume", methods=["POST"]) # noqa: F821
@login_required
async def resume(connector_id):
+ """Resume or cancel sync for an accessible connector."""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
req = await get_request_json()
if req.get("resume"):
ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
@@ -116,9 +144,15 @@ async def resume(connector_id):
@manager.route("/connectors//rebuild", methods=["POST"]) # noqa: F821
@login_required
-@validate_request("kb_id")
async def rebuild(connector_id):
+ """Schedule a rebuild for an accessible connector and knowledge base."""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
req = await get_request_json()
+ if "kb_id" not in req:
+ return get_json_result(code=RetCode.ARGUMENT_ERROR, message="required argument is missing: kb_id")
+
err = ConnectorService.rebuild(req["kb_id"], connector_id, current_user.id)
if err:
return get_json_result(data=False, message=err, code=RetCode.SERVER_ERROR)
@@ -128,6 +162,10 @@ async def rebuild(connector_id):
@manager.route("/connectors/", methods=["DELETE"]) # noqa: F821
@login_required
def rm_connector(connector_id):
+ """Delete an accessible connector after canceling its sync tasks."""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
ConnectorService.resume(connector_id, TaskStatus.CANCEL)
ConnectorService.delete_by_id(connector_id)
return get_json_result(data=True)
@@ -141,6 +179,9 @@ async def test_connector(connector_id):
For the REST API connector, this uses `RestAPIConnector.validate_config`
against the existing saved configuration.
"""
+ if not ConnectorService.accessible(connector_id, current_user.id):
+ return _connector_auth_error(connector_id, current_user.id)
+
from common.data_source.rest_api_connector import RestAPIConnector
from common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationError
diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py
index ab754101e1f..4d9fd2258ab 100644
--- a/api/db/services/connector_service.py
+++ b/api/db/services/connector_service.py
@@ -22,7 +22,7 @@
from peewee import SQL, fn
from api.db import InputType
-from api.db.db_models import Connector, SyncLogs, Connector2Kb, Knowledgebase
+from api.db.db_models import DB, Connector, SyncLogs, Connector2Kb, Knowledgebase
from api.db.services.common_service import CommonService
from api.db.services.document_service import DocumentService
from api.db.services.document_service import DocMetadataService
@@ -32,9 +32,37 @@
from common.settings import TIMEZONE
from common.time_utils import current_timestamp, timestamp_to_date
+LOGGER = logging.getLogger(__name__)
+
+
class ConnectorService(CommonService):
model = Connector
+ @classmethod
+ @DB.connection_context()
+ def accessible(cls, connector_id: str, user_id: str) -> bool:
+ """Return whether the user can access the connector's tenant."""
+ e, connector = cls.get_by_id(connector_id)
+ if not e:
+ LOGGER.warning("connector access denied: connector not found connector_id=%s user_id=%s", connector_id, user_id)
+ return False
+
+ if connector.tenant_id == user_id:
+ return True
+
+ from api.db.services.user_service import TenantService
+
+ joined_tenants = TenantService.get_joined_tenants_by_user_id(user_id)
+ has_access = any(tenant["tenant_id"] == connector.tenant_id for tenant in joined_tenants)
+ if not has_access:
+ LOGGER.warning(
+ "connector access denied: tenant mismatch connector_id=%s user_id=%s tenant_id=%s",
+ connector_id,
+ user_id,
+ connector.tenant_id,
+ )
+ return has_access
+
@classmethod
def resume(cls, connector_id, status):
for c2k in Connector2KbService.query(connector_id=connector_id):
@@ -370,4 +398,3 @@ def list_connectors(cls, kb_id):
cls.model.kb_id==kb_id
).dicts()
)
-
diff --git a/test/testcases/restful_api/test_connector_routes_unit.py b/test/testcases/restful_api/test_connector_routes_unit.py
index ad47aef3795..33c4d7a8f12 100644
--- a/test/testcases/restful_api/test_connector_routes_unit.py
+++ b/test/testcases/restful_api/test_connector_routes_unit.py
@@ -200,6 +200,10 @@ def get_by_id(_connector_id):
def list(_tenant_id):
return []
+ @staticmethod
+ def accessible(*_args, **_kwargs):
+ return True
+
@staticmethod
def resume(*_args, **_kwargs):
return True
@@ -246,6 +250,7 @@ async def _get_request_json():
SERVER_ERROR=500,
RUNNING=102,
PERMISSION_ERROR=403,
+ AUTHENTICATION_ERROR=109,
)
constants_mod.TaskStatus = SimpleNamespace(SCHEDULE="schedule", CANCEL="cancel")
monkeypatch.setitem(sys.modules, "common.constants", constants_mod)
@@ -420,6 +425,42 @@ def _save(**payload):
assert delete_calls == ["conn-rm"]
+@pytest.mark.p2
+def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
+ """Verify per-id connector routes stop before body parsing or service access."""
+ module = _load_connector_app(monkeypatch)
+
+ touched = []
+ monkeypatch.setattr(module.ConnectorService, "accessible", lambda cid, uid: False)
+ monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda *_args: touched.append("get_by_id"))
+ monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda *_args: touched.append("list_sync_tasks"))
+ monkeypatch.setattr(module.ConnectorService, "resume", lambda *_args: touched.append("resume"))
+ monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda *_args: touched.append("delete_by_id"))
+ monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda *_args: touched.append("update_by_id"))
+ monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: touched.append("rebuild"))
+
+ def _get_request_json():
+ touched.append("get_request_json")
+ return _AwaitableValue({"resume": True, "config": {"x": 1}})
+
+ monkeypatch.setattr(module, "get_request_json", _get_request_json)
+
+ responses = [
+ _run(module.update_connector("conn-victim")),
+ module.get_connector("conn-victim"),
+ module.list_logs("conn-victim"),
+ _run(module.resume("conn-victim")),
+ _run(module.rebuild("conn-victim")),
+ module.rm_connector("conn-victim"),
+ _run(module.test_connector("conn-victim")),
+ ]
+
+ assert all(res["code"] == module.RetCode.AUTHENTICATION_ERROR for res in responses)
+ assert all(res["message"] == "No authorization." for res in responses)
+ assert all(res["data"] is False for res in responses)
+ assert touched == []
+
+
@pytest.mark.p2
def test_connector_oauth_helper_functions(monkeypatch):
module = _load_connector_app(monkeypatch)
diff --git a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
index 9d9e1c9c14a..3807fb8e15c 100644
--- a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
+++ b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
@@ -200,6 +200,10 @@ def get_by_id(_connector_id):
def list(_tenant_id):
return []
+ @staticmethod
+ def accessible(*_args, **_kwargs):
+ return True
+
@staticmethod
def resume(*_args, **_kwargs):
return True
@@ -246,6 +250,7 @@ async def _get_request_json():
SERVER_ERROR=500,
RUNNING=102,
PERMISSION_ERROR=403,
+ AUTHENTICATION_ERROR=109,
)
constants_mod.TaskStatus = SimpleNamespace(SCHEDULE="schedule", CANCEL="cancel")
monkeypatch.setitem(sys.modules, "common.constants", constants_mod)
@@ -420,6 +425,42 @@ def _save(**payload):
assert delete_calls == ["conn-rm"]
+@pytest.mark.p2
+def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
+ """Verify per-id connector routes stop before body parsing or service access."""
+ module = _load_connector_app(monkeypatch)
+
+ touched = []
+ monkeypatch.setattr(module.ConnectorService, "accessible", lambda cid, uid: False)
+ monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda *_args: touched.append("get_by_id"))
+ monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda *_args: touched.append("list_sync_tasks"))
+ monkeypatch.setattr(module.ConnectorService, "resume", lambda *_args: touched.append("resume"))
+ monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda *_args: touched.append("delete_by_id"))
+ monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda *_args: touched.append("update_by_id"))
+ monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: touched.append("rebuild"))
+
+ def _get_request_json():
+ touched.append("get_request_json")
+ return _AwaitableValue({"resume": True, "config": {"x": 1}})
+
+ monkeypatch.setattr(module, "get_request_json", _get_request_json)
+
+ responses = [
+ _run(module.update_connector("conn-victim")),
+ module.get_connector("conn-victim"),
+ module.list_logs("conn-victim"),
+ _run(module.resume("conn-victim")),
+ _run(module.rebuild("conn-victim")),
+ module.rm_connector("conn-victim"),
+ _run(module.test_connector("conn-victim")),
+ ]
+
+ assert all(res["code"] == module.RetCode.AUTHENTICATION_ERROR for res in responses)
+ assert all(res["message"] == "No authorization." for res in responses)
+ assert all(res["data"] is False for res in responses)
+ assert touched == []
+
+
@pytest.mark.p2
def test_connector_oauth_helper_functions(monkeypatch):
module = _load_connector_app(monkeypatch)
From 13b422037fce1847bcf8def78cbcc533c2a6aeee Mon Sep 17 00:00:00 2001
From: Wang Qi
Date: Mon, 18 May 2026 16:10:21 +0800
Subject: [PATCH 183/666] Refactor: enhance graphrag - part 2 (#14972)
### What problem does this PR solve?
1. expose batch_chunk_token_size for configuration
2. retrieve chunks when build subgraph for the doc, not retreive all
docs chunks at the begining
3. get all chunks for a document, used to be hard coded 10000
4. delete not used method run_graphrag
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- [x] Refactoring
Follow on: #14617
---
api/utils/api_utils.py | 1 +
api/utils/validation_utils.py | 1 +
rag/graphrag/general/index.py | 150 ++++--------------
rag/nlp/search.py | 14 +-
rag/svr/task_executor.py | 1 +
test/testcases/configs.py | 1 +
.../test_update_document.py | 1 +
.../test_update_document.py | 1 +
.../graph-rag-form-fields.tsx | 15 ++
web/src/interfaces/database/dataset.ts | 1 +
web/src/interfaces/database/document.ts | 1 +
web/src/locales/en.ts | 3 +
web/src/locales/zh.ts | 3 +
.../dataset/dataset-setting/form-schema.ts | 6 +
.../pages/dataset/dataset-setting/index.tsx | 1 +
15 files changed, 82 insertions(+), 118 deletions(-)
diff --git a/api/utils/api_utils.py b/api/utils/api_utils.py
index a041ee0819f..5e034f0c509 100644
--- a/api/utils/api_utils.py
+++ b/api/utils/api_utils.py
@@ -439,6 +439,7 @@ def get_parser_config(chunk_method, parser_config):
"category",
],
"method": "light",
+ "batch_chunk_token_size": 4096,
},
"parent_child": {
"use_parent_child": False,
diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py
index 1e6c0056b73..861f94ee228 100644
--- a/api/utils/validation_utils.py
+++ b/api/utils/validation_utils.py
@@ -362,6 +362,7 @@ class GraphragConfig(Base):
method: Annotated[Literal["light", "general", "ner"], Field(default="light")]
community: Annotated[bool, Field(default=False)]
resolution: Annotated[bool, Field(default=False)]
+ batch_chunk_token_size: Annotated[int, Field(default=4096, ge=512, le=8196)]
class ParentChildConfig(Base):
diff --git a/rag/graphrag/general/index.py b/rag/graphrag/general/index.py
index 9898b19a32e..396f3aae0b1 100644
--- a/rag/graphrag/general/index.py
+++ b/rag/graphrag/general/index.py
@@ -54,6 +54,22 @@
from common.doc_store.doc_store_base import OrderByExpr
+DEFAULT_GRAPHRAG_BATCH_CHUNK_TOKEN_SIZE = 4096
+
+
+def _positive_int_config(config: dict, key: str, default: int) -> int:
+ value = config.get(key, default)
+ try:
+ value = int(value)
+ except (TypeError, ValueError):
+ logging.warning("Invalid GraphRAG config %s=%r, using default %s", key, value, default)
+ return default
+ if value < 512 or value > 8196:
+ logging.warning("Invalid GraphRAG config %s=%r, using default %s", key, value, default)
+ return default
+ return value
+
+
def _select_extractor(graphrag_config: dict):
"""Return the extractor class matching ``graphrag_config["method"]``.
@@ -121,100 +137,6 @@ async def load_subgraph_from_store(tenant_id: str, kb_id: str, doc_id: str):
return None
-async def run_graphrag(
- row: dict,
- language,
- with_resolution: bool,
- with_community: bool,
- chat_model,
- embedding_model,
- callback,
-):
- enable_timeout_assertion = os.environ.get("ENABLE_TIMEOUT_ASSERTION")
- start = asyncio.get_running_loop().time()
- tenant_id, kb_id, doc_id = row["tenant_id"], str(row["kb_id"]), row["doc_id"]
- chunks = []
- for d in settings.retriever.chunk_list(doc_id, tenant_id, [kb_id], max_count=10000, fields=["content_with_weight", "doc_id"], sort_by_position=True):
- chunks.append(d["content_with_weight"])
-
- timeout_sec = max(120, len(chunks) * 60 * 10) if enable_timeout_assertion else 10000000000
-
- try:
- subgraph = await asyncio.wait_for(
- generate_subgraph(
- _select_extractor(row["kb_parser_config"].get("graphrag", {})),
- tenant_id,
- kb_id,
- doc_id,
- chunks,
- language,
- row["kb_parser_config"]["graphrag"].get("entity_types", []),
- chat_model,
- embedding_model,
- callback,
- ),
- timeout=timeout_sec,
- )
- except asyncio.TimeoutError:
- logging.error("generate_subgraph timeout")
- raise
-
- if not subgraph:
- return
-
- graphrag_task_lock = RedisDistributedLock(f"graphrag_task_{kb_id}", lock_value=doc_id, timeout=1200)
- await graphrag_task_lock.spin_acquire()
- callback(msg=f"run_graphrag {doc_id} graphrag_task_lock acquired")
-
- try:
- subgraph_nodes = set(subgraph.nodes())
- new_graph = await merge_subgraph(
- tenant_id,
- kb_id,
- doc_id,
- subgraph,
- embedding_model,
- callback,
- )
- assert new_graph is not None
-
- if not with_resolution and not with_community:
- return
-
- if with_resolution:
- await graphrag_task_lock.spin_acquire()
- callback(msg=f"run_graphrag {doc_id} graphrag_task_lock acquired")
- await resolve_entities(
- new_graph,
- subgraph_nodes,
- tenant_id,
- kb_id,
- doc_id,
- chat_model,
- embedding_model,
- callback,
- task_id=row["id"],
- )
- if with_community:
- await graphrag_task_lock.spin_acquire()
- callback(msg=f"run_graphrag {doc_id} graphrag_task_lock acquired")
- await extract_community(
- new_graph,
- tenant_id,
- kb_id,
- doc_id,
- chat_model,
- embedding_model,
- callback,
- task_id=row["id"],
- )
- finally:
- graphrag_task_lock.release()
- now = asyncio.get_running_loop().time()
- callback(msg=f"GraphRAG for doc {doc_id} done in {now - start:.2f} seconds.")
- return
-
-
async def run_graphrag_for_kb(
row: dict,
doc_ids: list[str],
@@ -232,6 +154,8 @@ async def run_graphrag_for_kb(
enable_timeout_assertion = os.environ.get("ENABLE_TIMEOUT_ASSERTION")
start = asyncio.get_running_loop().time()
fields_for_chunks = ["content_with_weight", "doc_id"]
+ graphrag_config = kb_parser_config.get("graphrag", {})
+ batch_chunk_token_size = _positive_int_config(graphrag_config, "batch_chunk_token_size", DEFAULT_GRAPHRAG_BATCH_CHUNK_TOKEN_SIZE)
if not doc_ids:
logging.info(f"Fetching all docs for {kb_id}")
@@ -259,21 +183,20 @@ def load_doc_chunks(doc_id: str) -> list[str]:
chunks = []
current_chunk = ""
- # DEBUG: Obtener todos los chunks primero
raw_chunks = list(settings.retriever.chunk_list(
doc_id,
tenant_id,
[kb_id],
- max_count=10000, # FIX: Aumentar límite para procesar todos los chunks
fields=fields_for_chunks,
sort_by_position=True,
+ retrieve_all=True
))
- callback(msg=f"[DEBUG] chunk_list() returned {len(raw_chunks)} raw chunks for doc {doc_id}")
+ callback(msg=f"[GraphRAG] chunk_list returned {len(raw_chunks)} raw chunks for doc:{doc_id}")
for d in raw_chunks:
content = d["content_with_weight"]
- if num_tokens_from_string(current_chunk + content) < 4096:
+ if num_tokens_from_string(current_chunk + content) < batch_chunk_token_size:
current_chunk += content
else:
if current_chunk:
@@ -285,16 +208,7 @@ def load_doc_chunks(doc_id: str) -> list[str]:
return chunks
- all_doc_chunks: dict[str, list[str]] = {}
total_chunks = 0
- for doc_id in doc_ids:
- chunks = load_doc_chunks(doc_id)
- all_doc_chunks[doc_id] = chunks
- total_chunks += len(chunks)
-
- if total_chunks == 0:
- callback(msg=f"[GraphRAG] kb:{kb_id} has no available chunks in all documents, skip.")
- return {"ok_docs": [], "failed_docs": doc_ids, "total_docs": len(doc_ids), "total_chunks": 0, "seconds": 0.0}
semaphore = asyncio.Semaphore(max_parallel_docs)
@@ -302,18 +216,13 @@ def load_doc_chunks(doc_id: str) -> list[str]:
failed_docs: list[tuple[str, str]] = [] # (doc_id, error)
async def build_one(doc_id: str):
+ nonlocal total_chunks
+
if has_canceled(row["id"]):
callback(msg=f"Task {row['id']} cancelled, stopping execution.")
raise TaskCanceledException(f"Task {row['id']} was cancelled")
- chunks = all_doc_chunks.get(doc_id, [])
- if not chunks:
- callback(msg=f"[GraphRAG] doc:{doc_id} has no available chunks, skip generation.")
- return
-
- kg_extractor = _select_extractor(kb_parser_config.get("graphrag", {}))
-
- deadline = max(120, len(chunks) * 60 * 10) if enable_timeout_assertion else 10000000000
+ kg_extractor = _select_extractor(graphrag_config)
async with semaphore:
# CHECKPOINT: bounded by semaphore so doc-store lookups respect max_parallel_docs
@@ -323,6 +232,13 @@ async def build_one(doc_id: str):
callback(msg=f"[GraphRAG] doc:{doc_id} subgraph found in store, skipping LLM extraction.")
return
try:
+ chunks = load_doc_chunks(doc_id)
+ total_chunks += len(chunks)
+ if not chunks:
+ callback(msg=f"[GraphRAG] doc:{doc_id} has no available chunks, skip generation.")
+ return
+
+ deadline = max(120, len(chunks) * 60 * 10) if enable_timeout_assertion else 10000000000
msg = f"[GraphRAG] build_subgraph doc:{doc_id}"
callback(msg=f"{msg} start (chunks={len(chunks)}, timeout={deadline}s)")
@@ -373,6 +289,10 @@ async def build_one(doc_id: str):
await asyncio.gather(*tasks, return_exceptions=True)
raise
+ if total_chunks == 0 and not subgraphs:
+ callback(msg=f"[GraphRAG] kb:{kb_id} has no available chunks in all documents, skip.")
+ return {"ok_docs": [], "failed_docs": [(doc_id, "no available chunks") for doc_id in doc_ids], "total_docs": len(doc_ids), "total_chunks": 0, "seconds": 0.0}
+
if has_canceled(row["id"]):
callback(msg=f"Task {row['id']} cancelled after document processing.")
raise TaskCanceledException(f"Task {row['id']} was cancelled")
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 980dba04d93..e79671f04eb 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -753,7 +753,13 @@ def chunk_list(self, doc_id: str, tenant_id: str,
kb_ids: list[str], max_count=1024,
offset=0,
fields=["docnm_kwd", "content_with_weight", "img_id"],
- sort_by_position: bool = False):
+ sort_by_position: bool = False,
+ retrieve_all: bool = False):
+ """Return chunks for a document.
+
+ By default, preserve the historical max_count cap. When retrieve_all is
+ True, keep paging until the doc store returns fewer rows than requested.
+ """
condition = {"doc_id": doc_id}
fields_set = set(fields or [])
@@ -771,8 +777,9 @@ def chunk_list(self, doc_id: str, tenant_id: str,
res = []
bs = 128
- for p in range(offset, max_count, bs):
- limit = min(bs, max_count - p)
+ p = offset
+ while retrieve_all or p < max_count:
+ limit = bs if retrieve_all else min(bs, max_count - p)
if limit <= 0:
break
es_res = self.dataStore.search(fields, [], condition, [], orderBy, p, limit, index_name(tenant_id),
@@ -785,6 +792,7 @@ def chunk_list(self, doc_id: str, tenant_id: str,
chunk_count = len(dict_chunks)
if chunk_count == 0 or chunk_count < limit:
break
+ p += limit
return res
def all_tags(self, tenant_id: str, kb_ids: list[str], S=1000):
diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py
index 548d88ab1ba..e639ba6e46d 100644
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@@ -1390,6 +1390,7 @@ async def do_handle_task(task):
"category",
],
"method": "light",
+ "batch_chunk_token_size": 4096,
}
}
)
diff --git a/test/testcases/configs.py b/test/testcases/configs.py
index 546cd378c9d..a4711bf1583 100644
--- a/test/testcases/configs.py
+++ b/test/testcases/configs.py
@@ -65,6 +65,7 @@
"category",
],
"method": "light",
+ "batch_chunk_token_size": 4096,
},
"parent_child": {
"use_parent_child": False,
diff --git a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
index b24d9deeacf..de0b4189b96 100644
--- a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
+++ b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
@@ -387,6 +387,7 @@ def test_update_doc_guards_and_error_paths(self, HttpApiAuth, add_documents, pay
"category",
],
"method": "light",
+ "batch_chunk_token_size": 4096,
},
}
diff --git a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
index f174f0e5462..2b02c0b19cc 100644
--- a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
+++ b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
@@ -313,6 +313,7 @@ def test_immutable_fields_progress(self, add_documents, payload, expected_messag
"category",
],
"method": "light",
+ "batch_chunk_token_size": 4096,
},
}
diff --git a/web/src/components/parse-configuration/graph-rag-form-fields.tsx b/web/src/components/parse-configuration/graph-rag-form-fields.tsx
index d85c8836485..f3791c57548 100644
--- a/web/src/components/parse-configuration/graph-rag-form-fields.tsx
+++ b/web/src/components/parse-configuration/graph-rag-form-fields.tsx
@@ -1,3 +1,4 @@
+import { FormLayout } from '@/constants/form';
import { DocumentParserType } from '@/constants/knowledge';
import { useTranslate } from '@/hooks/common-hooks';
import { cn } from '@/lib/utils';
@@ -12,6 +13,7 @@ import { useCallback, useMemo } from 'react';
import { useFormContext, useWatch } from 'react-hook-form';
import { EntityTypesFormField } from '../entity-types-form-field';
import { FormContainer } from '../form-container';
+import { SliderInputFormField } from '../slider-input-form-field';
import {
FormControl,
FormField,
@@ -191,6 +193,19 @@ const GraphRagItems = ({
)}
/>
+
+
General: Use prompts provided by github.com/microsoft/graphrag to extract entities and relationships.
NER: Use spaCy NER and rule-based keyword extraction to extract entities and relationships. No LLM is required for extraction itself, making it fast and resource-efficient.`,
+ graphRagBatchChunkTokenSize: 'Batch chunk token size',
+ graphRagBatchChunkTokenSizeTip:
+ 'The token limit for each batch of chunks sent to the LLM for knowledge graph entity and relation extraction. Not applied to NER.',
resolution: 'Entity resolution',
resolutionTip: `An entity deduplication switch. When enabled, the LLM will combine similar entities - e.g., '2025' and 'the year of 2025', or 'IT' and 'Information Technology' - to construct a more accurate graph`,
community: 'Community reports',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index dcd8f5871de..81b95ee887d 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -818,6 +818,9 @@ export default {
graphRagMethodTip: `Light:实体和关系提取提示来自 GitHub - HKUDS/LightRAG:“LightRAG:简单快速的检索增强生成”
General:实体和关系提取提示来自 GitHub - microsoft/graphrag:基于图的模块化检索增强生成 (RAG) 系统
NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系,无需 LLM 参与提取过程,速度快且资源消耗低`,
+ graphRagBatchChunkTokenSize: '批量chunk token 大小',
+ graphRagBatchChunkTokenSizeTip:
+ '发送给 LLM 进行知识图谱实体和关系抽取时,每批文本块的 token 上限。NER 不适用。',
resolution: '实体归一化',
resolutionTip: `解析过程会将具有相同含义的实体合并在一起,从而使知识图谱更简洁、更准确。应合并以下实体:特朗普总统、唐纳德·特朗普、唐纳德·J·特朗普、唐纳德·约翰·特朗普`,
community: '社区报告生成',
diff --git a/web/src/pages/dataset/dataset-setting/form-schema.ts b/web/src/pages/dataset/dataset-setting/form-schema.ts
index 03424921c17..acb0eaf1094 100644
--- a/web/src/pages/dataset/dataset-setting/form-schema.ts
+++ b/web/src/pages/dataset/dataset-setting/form-schema.ts
@@ -70,6 +70,12 @@ export const formSchema = z
method: z.string().optional(),
resolution: z.boolean().optional(),
community: z.boolean().optional(),
+ batch_chunk_token_size: z
+ .number()
+ .int()
+ .min(512)
+ .max(8196)
+ .optional(),
})
.refine(
(data) => {
diff --git a/web/src/pages/dataset/dataset-setting/index.tsx b/web/src/pages/dataset/dataset-setting/index.tsx
index 930ec8f51cf..072e84f8780 100644
--- a/web/src/pages/dataset/dataset-setting/index.tsx
+++ b/web/src/pages/dataset/dataset-setting/index.tsx
@@ -103,6 +103,7 @@ export default function DatasetSettings() {
use_graphrag: true,
entity_types: initialEntityTypes,
method: MethodValue.Light,
+ batch_chunk_token_size: 4096,
},
metadata: {
type: 'object',
From b8ac9976061aff826cad3a89424fe8af3206e17c Mon Sep 17 00:00:00 2001
From: buua436
Date: Mon, 18 May 2026 16:57:14 +0800
Subject: [PATCH 184/666] Go: add restful api route aliases (#14977)
### What problem does this PR solve?
add restful api route aliases
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---
internal/router/router.go | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/internal/router/router.go b/internal/router/router.go
index 6b858a89f7a..ddb13e0ac95 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -142,6 +142,10 @@ func (r *Router) Setup(engine *gin.Engine) {
users.GET("/me", r.userHandler.Info)
// User settings endpoint
users.PATCH("/me", r.userHandler.Setting)
+ // User tenant info endpoint
+ users.GET("/me/models", r.tenantHandler.TenantInfo)
+ // User set tenant info endpoint
+ users.PATCH("/me/models", r.userHandler.SetTenantInfo)
}
tenants := v1.Group("/tenants")
@@ -149,6 +153,8 @@ func (r *Router) Setup(engine *gin.Engine) {
tenants.GET("", r.tenantHandler.TenantList)
}
+ v1.GET("/tenant/list", r.tenantHandler.TenantList)
+
// Document routes
documents := v1.Group("/documents")
{
@@ -198,6 +204,7 @@ func (r *Router) Setup(engine *gin.Engine) {
file.DELETE("", r.fileHandler.DeleteFiles)
file.POST("/move", r.fileHandler.MoveFiles)
file.GET("/:id/ancestors", r.fileHandler.GetFileAncestors)
+ file.GET("/:id/parent", r.fileHandler.GetParentFolder)
file.GET("/:id", r.fileHandler.Download)
}
@@ -288,6 +295,11 @@ func (r *Router) Setup(engine *gin.Engine) {
model.PATCH("/", r.tenantHandler.SetModels)
}
+ connector := v1.Group("/connectors")
+ {
+ connector.GET("/", r.connectorHandler.ListConnectors)
+ }
+
system := v1.Group("/system")
{
system.GET("/configs", r.systemHandler.GetConfigs)
From 92145dc7642a63820966b0f4e3b06ab21ec7b48a Mon Sep 17 00:00:00 2001
From: Haruko386
Date: Mon, 18 May 2026 16:57:42 +0800
Subject: [PATCH 185/666] Go: implement provider: DeepInfra, XunFei (#14978)
### What problem does this PR solve?
This PR implement implement provider and Mistral, DeepInfra, XunFei
**The following functionalities are now supported:**
**DeepInfra**
- [x] chat / think chat / stream chat / stream think chat
- [x] Embedding
- [x] ASR
- [x] TTS
- [x] ListModels
- [x] Provider connection checking
- [x] Balance
- [ ] ~~Rerank~~
**XunFei**
- [x] chat / think chat / stream chat / stream think chat
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- [x] Refactoring
---
conf/models/deepinfra.json | 49 ++
conf/models/xunfei.json | 23 +
internal/entity/models/deepinfra.go | 840 ++++++++++++++++++++++++++
internal/entity/models/factory.go | 4 +
internal/entity/models/minimax.go | 2 +-
internal/entity/models/openrouter.go | 6 +-
internal/entity/models/siliconflow.go | 2 +-
internal/entity/models/xunfei.go | 452 ++++++++++++++
8 files changed, 1374 insertions(+), 4 deletions(-)
create mode 100644 conf/models/deepinfra.json
create mode 100644 conf/models/xunfei.json
create mode 100644 internal/entity/models/deepinfra.go
create mode 100644 internal/entity/models/xunfei.go
diff --git a/conf/models/deepinfra.json b/conf/models/deepinfra.json
new file mode 100644
index 00000000000..a9277fc6e72
--- /dev/null
+++ b/conf/models/deepinfra.json
@@ -0,0 +1,49 @@
+{
+ "name": "DeepInfra",
+ "url": {
+ "default": "https://api.deepinfra.com"
+ },
+ "url_suffix": {
+ "chat": "v1/chat/completions",
+ "models": "models/list",
+ "balance": "payment/checklist",
+ "embedding": "v1/embeddings",
+ "tts": "v1/text-to-speech",
+ "asr": "v1/audio/transcriptions"
+ },
+ "class": "deepinfra",
+ "models": [
+ {
+ "name": "deepseek-ai/DeepSeek-V3.2",
+ "max_tokens": 32768,
+ "model_types": [
+ "chat"
+ ],
+ "thinking": {
+ "default_value": true,
+ "clear_thinking": true
+ }
+ },
+ {
+ "name": "Qwen/Qwen3-Embedding-4B",
+ "max_tokens": 8192,
+ "model_types": [
+ "embedding"
+ ]
+ },
+ {
+ "name": "hexgrad/Kokoro-82M",
+ "max_tokens": 16384,
+ "model_types": [
+ "tts"
+ ]
+ },
+ {
+ "name": "bosonai/HiggsAudioV2.5",
+ "max_tokens": 8192,
+ "model_types": [
+ "asr"
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/conf/models/xunfei.json b/conf/models/xunfei.json
new file mode 100644
index 00000000000..3d963d96e95
--- /dev/null
+++ b/conf/models/xunfei.json
@@ -0,0 +1,23 @@
+{
+ "name": "XunFei",
+ "url": {
+ "default": "https://"
+ },
+ "url_suffix": {
+ "chat": "spark-api-open.xf-yun.com/v2/chat/completions"
+ },
+ "class": "xunfei",
+ "models": [
+ {
+ "name": "spark-x",
+ "max_tokens": 134144,
+ "model_types": [
+ "chat"
+ ],
+ "thinking": {
+ "default_value": true,
+ "clear_thinking": true
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/internal/entity/models/deepinfra.go b/internal/entity/models/deepinfra.go
new file mode 100644
index 00000000000..dd70981fe6f
--- /dev/null
+++ b/internal/entity/models/deepinfra.go
@@ -0,0 +1,840 @@
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "mime/multipart"
+ "net/http"
+ "os"
+ "path/filepath"
+ "ragflow/internal/common"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type DeepInfraModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewDeepInfraModel(baseURL map[string]string, urlSuffix URLSuffix) *DeepInfraModel {
+ return &DeepInfraModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Timeout: time.Second * 120,
+ Transport: &http.Transport{
+ MaxIdleConns: 10,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: time.Second * 90,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (d *DeepInfraModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return &DeepInfraModel{
+ BaseURL: baseURL,
+ URLSuffix: d.URLSuffix,
+ httpClient: &http.Client{
+ Timeout: time.Second * 120,
+ Transport: &http.Transport{
+ MaxIdleConns: 10,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: time.Second * 90,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (d *DeepInfraModel) Name() string {
+ return "deepinfra"
+}
+
+func (d *DeepInfraModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", d.BaseURL[region], d.URLSuffix.Chat)
+
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": false,
+ }
+
+ if chatModelConfig != nil {
+ if chatModelConfig.Stream != nil {
+ reqBody["stream"] = *chatModelConfig.Stream
+ }
+
+ if chatModelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *chatModelConfig.MaxTokens
+ }
+
+ if chatModelConfig.Temperature != nil {
+ reqBody["temperature"] = *chatModelConfig.Temperature
+ }
+
+ if chatModelConfig.TopP != nil {
+ reqBody["top_p"] = *chatModelConfig.TopP
+ }
+
+ if chatModelConfig.Stop != nil {
+ reqBody["stop"] = *chatModelConfig.Stop
+ }
+
+ if chatModelConfig.Effort != nil {
+ reqBody["reasoning_effort"] = *chatModelConfig.Effort
+ }
+
+ if chatModelConfig.Thinking != nil && *chatModelConfig.Thinking {
+ reasoningMap := map[string]interface{}{
+ "enabled": true,
+ }
+ if chatModelConfig.Effort != nil {
+ reasoningMap["effort"] = *chatModelConfig.Effort
+ }
+ reqBody["reasoning"] = reasoningMap
+ }
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ // Parse result
+ var result map[string]interface{}
+ if err := json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response body: %w", err)
+ }
+
+ choices, ok := result["choices"].([]interface{})
+ if !ok || len(choices) == 0 {
+ return nil, fmt.Errorf("no choices in response")
+ }
+
+ firstChoice, ok := choices[0].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("invalid choice format")
+ }
+
+ messageMap, ok := firstChoice["message"].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("invalid message format")
+ }
+
+ content, ok := messageMap["content"].(string)
+ if !ok {
+ return nil, fmt.Errorf("invalid content format")
+ }
+
+ var reasonContent string
+ if rc, ok := messageMap["reasoning_content"].(string); ok {
+ reasonContent = rc
+ }
+
+ chatResponse := &ChatResponse{
+ Answer: &content,
+ }
+ if reasonContent != "" {
+ chatResponse.ReasonContent = &reasonContent
+ }
+
+ return chatResponse, nil
+}
+
+func (d *DeepInfraModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
+ if len(messages) == 0 {
+ return fmt.Errorf("messages is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", d.BaseURL[region], d.URLSuffix.Chat)
+
+ // Convert messages to API format
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ // Build request body with streaming enabled
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": true,
+ "temperature": 1,
+ }
+
+ if modelConfig != nil {
+ if modelConfig.Stream != nil {
+ reqBody["stream"] = *modelConfig.Stream
+ }
+
+ if modelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *modelConfig.MaxTokens
+ }
+
+ if modelConfig.Temperature != nil {
+ reqBody["temperature"] = *modelConfig.Temperature
+ }
+
+ if modelConfig.DoSample != nil {
+ reqBody["do_sample"] = *modelConfig.DoSample
+ }
+
+ if modelConfig.TopP != nil {
+ reqBody["top_p"] = *modelConfig.TopP
+ }
+
+ if modelConfig.Stop != nil {
+ reqBody["stop"] = *modelConfig.Stop
+ }
+
+ if modelConfig.Effort != nil {
+ reqBody["reasoning_effort"] = *modelConfig.Effort
+ }
+
+ if modelConfig.Thinking != nil && *modelConfig.Thinking {
+ reasoningMap := map[string]interface{}{
+ "enabled": true,
+ }
+ if modelConfig.Effort != nil {
+ reasoningMap["effort"] = *modelConfig.Effort
+ }
+ reqBody["reasoning"] = reasoningMap
+ }
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ // SSE parsing: read line by line
+ scanner := bufio.NewScanner(resp.Body)
+ for scanner.Scan() {
+ line := scanner.Text()
+ common.Info(line)
+
+ // SSE data line starts with "data:"
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+
+ // Extract JSON after "data:"
+ data := strings.TrimSpace(line[5:])
+
+ // [DONE] marks the end of stream
+ if data == "[DONE]" {
+ break
+ }
+
+ // Parse the JSON event
+ var event map[string]interface{}
+ if err = json.Unmarshal([]byte(data), &event); err != nil {
+ continue
+ }
+
+ choices, ok := event["choices"].([]interface{})
+ if !ok || len(choices) == 0 {
+ continue
+ }
+
+ firstChoice, ok := choices[0].(map[string]interface{})
+ if !ok {
+ continue
+ }
+
+ delta, ok := firstChoice["delta"].(map[string]interface{})
+ if !ok {
+ continue
+ }
+
+ reasoningContent, ok := delta["reasoning_content"].(string)
+ if ok && reasoningContent != "" {
+ if err := sender(nil, &reasoningContent); err != nil {
+ return err
+ }
+ }
+
+ content, ok := delta["content"].(string)
+ if ok && content != "" {
+ if err := sender(&content, nil); err != nil {
+ return err
+ }
+ }
+
+ finishReason, ok := firstChoice["finish_reason"].(string)
+ if ok && finishReason != "" {
+ break
+ }
+ }
+
+ // Send [DONE] marker for OpenAI compatibility
+ endOfStream := "[DONE]"
+ if err = sender(&endOfStream, nil); err != nil {
+ return err
+ }
+
+ return scanner.Err()
+}
+
+func (d *DeepInfraModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ if len(texts) == 0 {
+ return []EmbeddingData{}, fmt.Errorf("texts is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", d.BaseURL[region], d.URLSuffix.Embedding)
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": texts,
+ }
+
+ if embeddingConfig != nil && embeddingConfig.Dimension >= 32 {
+ reqBody["dimensions"] = embeddingConfig.Dimension
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("DeepInfra embedding API error: status %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ var parsed struct {
+ Data []struct {
+ Embedding []float64 `json:"embedding"`
+ Index int `json:"index"`
+ } `json:"data"`
+ }
+
+ if err = json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+ }
+
+ // 组装 RAGFlow 需要的返回格式
+ var embeddings []EmbeddingData
+ for _, data := range parsed.Data {
+ embeddings = append(embeddings, EmbeddingData{
+ Embedding: data.Embedding,
+ Index: data.Index,
+ })
+ }
+
+ return embeddings, nil
+
+ return embeddings, nil
+}
+
+func (d *DeepInfraModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("%s no such method", d.Name())
+}
+
+func (d *DeepInfraModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("DeepInfra API key is missing")
+ }
+
+ if file == nil || *file == "" {
+ return nil, fmt.Errorf("file is missing")
+ }
+
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is missing")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", d.BaseURL[region], d.URLSuffix.ASR)
+
+ var body bytes.Buffer
+ writer := multipart.NewWriter(&body)
+
+ if err := writer.WriteField("model", *modelName); err != nil {
+ return nil, fmt.Errorf("failed to write model field: %w", err)
+ }
+
+ // Open File
+ audioFile, err := os.Open(*file)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open audio file: %w", err)
+ }
+ defer audioFile.Close()
+
+ part, err := writer.CreateFormFile("file", filepath.Base(*file))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create multipart file: %w", err)
+ }
+
+ if _, err = io.Copy(part, audioFile); err != nil {
+ return nil, fmt.Errorf("failed to copy audio data: %w", err)
+ }
+
+ // get config
+ if asrConfig != nil && asrConfig.Params != nil {
+ for key, value := range asrConfig.Params {
+ var val string
+
+ switch v := value.(type) {
+ case string:
+ val = v
+ case bool:
+ val = strconv.FormatBool(v)
+ case int:
+ val = strconv.Itoa(v)
+ case int64:
+ val = strconv.FormatInt(v, 10)
+ case float64:
+ val = strconv.FormatFloat(v, 'f', -1, 64)
+ case float32:
+ val = strconv.FormatFloat(float64(v), 'f', -1, 32)
+ default:
+ val = fmt.Sprintf("%v", v)
+ }
+
+ if err := writer.WriteField(key, val); err != nil {
+ return nil, fmt.Errorf("failed to write field %s: %w", key, err)
+ }
+ }
+ }
+
+ if err = writer.Close(); err != nil {
+ return nil, fmt.Errorf("failed to close multipart writer: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, &body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("Content-Type", writer.FormDataContentType())
+ req.Header.Set("Accept", "application/json")
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ respBody, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("DeepInfra ASR error: %s - %s", resp.Status, string(respBody))
+ }
+
+ // Parse result
+ var result struct {
+ Text string `json:"text"`
+ }
+
+ if err := json.Unmarshal(respBody, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+ }
+
+ return &ASRResponse{
+ Text: result.Text,
+ }, nil
+}
+
+func (d *DeepInfraModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s no such method", d.Name())
+}
+
+func (d *DeepInfraModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("DeepInfra API key is missing")
+ }
+
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("text content is missing")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ reqBody := map[string]interface{}{
+ "text": *audioContent,
+ }
+ voiceID := ""
+
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ if v, ok := ttsConfig.Params["voice_id"].(string); ok && v != "" {
+ voiceID = v
+ } else if v, ok := ttsConfig.Params["voice"].(string); ok && v != "" {
+ voiceID = v
+ }
+
+ for key, value := range ttsConfig.Params {
+ if key != "voice_id" && key != "voice" {
+ reqBody[key] = value
+ }
+ }
+ }
+
+ if voiceID == "" {
+ return nil, fmt.Errorf("voice_id is missing (must be provided in params or model name)")
+ }
+
+ // URL: https://api.deepinfra.com/v1/text-to-speech/{voice_id}
+ url := fmt.Sprintf("%s/%s/%s", d.BaseURL[region], d.URLSuffix.TTS, voiceID)
+
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["output_format"] = ttsConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("DeepInfra TTS error: status %d - %s", resp.StatusCode, string(body))
+ }
+
+ return &TTSResponse{Audio: body}, nil
+}
+
+func (d *DeepInfraModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("DeepInfra API key is missing")
+ }
+
+ if audioContent == nil || *audioContent == "" {
+ return fmt.Errorf("text content is missing")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ voiceID := ""
+
+ reqBody := map[string]interface{}{
+ "text": *audioContent,
+ }
+
+ if ttsConfig != nil && ttsConfig.Params != nil {
+ if v, ok := ttsConfig.Params["voice_id"].(string); ok && v != "" {
+ voiceID = v
+ } else if v, ok := ttsConfig.Params["voice"].(string); ok && v != "" {
+ voiceID = v
+ }
+
+ for key, value := range ttsConfig.Params {
+ if key != "voice_id" && key != "voice" {
+ reqBody[key] = value
+ }
+ }
+ }
+
+ if voiceID == "" {
+ return fmt.Errorf("voice_id is missing")
+ }
+
+ // URL: https://api.deepinfra.com/v1/text-to-speech/{voice_id}/stream
+ url := fmt.Sprintf("%s/%s/%s/stream", d.BaseURL[region], d.URLSuffix.TTS, voiceID)
+
+ if ttsConfig != nil && ttsConfig.Format != "" {
+ reqBody["output_format"] = ttsConfig.Format
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("DeepInfra TTS Stream error: status %d - %s", resp.StatusCode, string(body))
+ }
+
+ buffer := make([]byte, 4096)
+ for {
+ n, err := resp.Body.Read(buffer)
+
+ if n > 0 {
+ chunkStr := string(buffer[:n])
+ if sendErr := sender(&chunkStr, nil); sendErr != nil {
+ return sendErr
+ }
+ }
+
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return fmt.Errorf("error reading stream: %w", err)
+ }
+ }
+
+ endOfStream := "[DONE]"
+ if err = sender(&endOfStream, nil); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (d *DeepInfraModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s no such method", d.Name())
+}
+
+func (d *DeepInfraModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s no such method", d.Name())
+}
+
+func (d *DeepInfraModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", d.BaseURL[region], d.URLSuffix.Models)
+
+ reqBody := map[string]interface{}{}
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("GET", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to read response: %s", string(body))
+ }
+
+ // Parse response
+ var result []struct {
+ ModelName string `json:"model_name"`
+ }
+ if err := json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+ }
+
+ models := make([]string, 0)
+ for _, model := range result {
+ if model.ModelName != "" {
+ models = append(models, model.ModelName)
+ }
+ }
+
+ return models, nil
+}
+
+func (d *DeepInfraModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", d.BaseURL[region], d.URLSuffix.Balance)
+
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := d.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to read response: %s", string(body))
+ }
+
+ var result struct {
+ Balance interface{} `json:"stripe_balance"`
+ }
+
+ if err := json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+ }
+
+ return map[string]interface{}{
+ "balance": result.Balance,
+ "currence": "USD",
+ }, nil
+}
+
+func (d *DeepInfraModel) CheckConnection(apiConfig *APIConfig) error {
+ _, err := d.ListModels(apiConfig)
+ return err
+}
+
+func (d *DeepInfraModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s no such method", d.Name())
+}
+
+func (d *DeepInfraModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s no such method", d.Name())
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index 3989a85881c..dcef9dd9981 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -97,6 +97,10 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewVoyageModel(baseURL, urlSuffix), nil
case "paddleocr":
return NewPaddleOCRModel(baseURL, urlSuffix), nil
+ case "xunfei":
+ return NewXunFeiModel(baseURL, urlSuffix), nil
+ case "deepinfra":
+ return NewDeepInfraModel(baseURL, urlSuffix), nil
default:
return NewDummyModel(baseURL, urlSuffix), nil
}
diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go
index 9ec953b9068..61e4110e7cf 100644
--- a/internal/entity/models/minimax.go
+++ b/internal/entity/models/minimax.go
@@ -201,7 +201,7 @@ func (z *MinimaxModel) ChatStreamlyWithSender(modelName string, messages []Messa
var region = "default"
- if apiConfig.Region != nil {
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
diff --git a/internal/entity/models/openrouter.go b/internal/entity/models/openrouter.go
index 4e99ef5bea4..c42493d0109 100644
--- a/internal/entity/models/openrouter.go
+++ b/internal/entity/models/openrouter.go
@@ -109,8 +109,10 @@ func (o *OpenRouterModel) ChatWithMessages(modelName string, messages []Message,
reqBody["do_sample"] = *chatModelConfig.DoSample
}
- reqBody["reasoning"] = map[string]interface{}{
- "effort": "low",
+ if chatModelConfig.Effort != nil {
+ reqBody["reasoning"] = map[string]interface{}{
+ "effort": chatModelConfig.Effort,
+ }
}
}
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index 7e9d4f7c576..f726e3db998 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -221,7 +221,7 @@ func (z *SiliconflowModel) ChatStreamlyWithSender(modelName string, messages []M
region = *apiConfig.Region
}
- url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region])
+ url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat)
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
diff --git a/internal/entity/models/xunfei.go b/internal/entity/models/xunfei.go
new file mode 100644
index 00000000000..e1b89b535d6
--- /dev/null
+++ b/internal/entity/models/xunfei.go
@@ -0,0 +1,452 @@
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "ragflow/internal/common"
+ "strings"
+ "time"
+)
+
+type XunFeiModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewXunFeiModel(baseURL map[string]string, urlSuffix URLSuffix) *XunFeiModel {
+ return &XunFeiModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Timeout: time.Second * 120,
+ Transport: &http.Transport{
+ MaxIdleConns: 10,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: time.Second * 90,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (x *XunFeiModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return &XunFeiModel{
+ BaseURL: baseURL,
+ URLSuffix: x.URLSuffix,
+ httpClient: &http.Client{
+ Timeout: time.Second * 120,
+ Transport: &http.Transport{
+ MaxIdleConns: 10,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: time.Second * 90,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (x *XunFeiModel) Name() string {
+ return "xunfei"
+}
+
+func (x *XunFeiModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", x.BaseURL[region], x.URLSuffix.Chat)
+
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": false,
+ "temperature": 1,
+ }
+
+ if chatModelConfig != nil {
+ if chatModelConfig.Temperature != nil {
+ reqBody["temperature"] = *chatModelConfig.Temperature
+ }
+
+ if chatModelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *chatModelConfig.MaxTokens
+ }
+
+ if chatModelConfig.Stream != nil {
+ reqBody["stream"] = *chatModelConfig.Stream
+ }
+
+ if chatModelConfig.TopP != nil {
+ reqBody["top_p"] = *chatModelConfig.TopP
+ }
+
+ if chatModelConfig.Thinking != nil {
+ if *chatModelConfig.Thinking {
+ reqBody["thinking"] = map[string]interface{}{
+ "type": "enabled",
+ }
+ } else {
+ reqBody["thinking"] = map[string]interface{}{
+ "type": "disabled",
+ }
+ }
+ }
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request body: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := x.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ // Parse Response
+ var result map[string]interface{}
+ if err := json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response body: %w", err)
+ }
+
+ choices, ok := result["choices"].([]interface{})
+ if !ok {
+ return nil, fmt.Errorf("no choices in response")
+ }
+
+ firstChoice, ok := choices[0].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("no choices in response")
+ }
+
+ messageMap, ok := firstChoice["message"].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("no message in response")
+ }
+
+ content, ok := messageMap["content"].(string)
+ if !ok {
+ return nil, fmt.Errorf("no message in response")
+ }
+
+ var reasonContent string
+ if chatModelConfig != nil && chatModelConfig.Thinking != nil && *chatModelConfig.Thinking {
+ reasonContent, ok = messageMap["reasoning_content"].(string)
+ if !ok {
+ return nil, fmt.Errorf("invalid content format")
+ }
+ if reasonContent != "" && reasonContent[0] == '\n' {
+ reasonContent = reasonContent[1:]
+ }
+ }
+
+ chatResponse := &ChatResponse{
+ Answer: &content,
+ ReasonContent: &reasonContent,
+ }
+
+ return chatResponse, nil
+}
+
+func (x *XunFeiModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
+ if len(messages) == 0 {
+ return fmt.Errorf("messages is empty")
+ }
+
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", x.BaseURL[region], x.URLSuffix.Chat)
+
+ // Convert messages to API format
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": true,
+ "temperature": 1,
+ }
+
+ if modelConfig != nil {
+ if modelConfig.Stream != nil {
+ reqBody["stream"] = *modelConfig.Stream
+ }
+
+ if modelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *modelConfig.MaxTokens
+ }
+
+ if modelConfig.Temperature != nil {
+ reqBody["temperature"] = *modelConfig.Temperature
+ }
+
+ if modelConfig.TopP != nil {
+ reqBody["top_p"] = *modelConfig.TopP
+ }
+
+ if modelConfig.Stop != nil {
+ reqBody["stop"] = *modelConfig.Stop
+ }
+
+ if modelConfig.Thinking != nil {
+ if *modelConfig.Thinking {
+ reqBody["thinking"] = map[string]interface{}{
+ "type": "enabled",
+ }
+ } else {
+ reqBody["thinking"] = map[string]interface{}{
+ "type": "disabled",
+ }
+ }
+ }
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := x.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("invalid status code: %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ // SSE parsing: read line by line
+ scanner := bufio.NewScanner(resp.Body)
+ for scanner.Scan() {
+ line := scanner.Text()
+ common.Info(line)
+
+ // SSE data line starts with "data:"
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+
+ // Extract JSON after "data:"
+ data := strings.TrimSpace(line[5:])
+
+ // [DONE] marks the end of stream
+ if data == "[DONE]" {
+ break
+ }
+
+ // Parse the JSON event
+ var event map[string]interface{}
+ if err = json.Unmarshal([]byte(data), &event); err != nil {
+ continue
+ }
+
+ choices, ok := event["choices"].([]interface{})
+ if !ok || len(choices) == 0 {
+ continue
+ }
+
+ firstChoice, ok := choices[0].(map[string]interface{})
+ if !ok {
+ continue
+ }
+
+ delta, ok := firstChoice["delta"].(map[string]interface{})
+ if !ok {
+ continue
+ }
+
+ reasoningContent, ok := delta["reasoning_content"].(string)
+ if ok && reasoningContent != "" {
+ if err := sender(nil, &reasoningContent); err != nil {
+ return err
+ }
+ }
+
+ content, ok := delta["content"].(string)
+ if ok && content != "" {
+ if err := sender(&content, nil); err != nil {
+ return err
+ }
+ }
+
+ finishReason, ok := firstChoice["finish_reason"].(string)
+ if ok && finishReason != "" {
+ break
+ }
+ }
+
+ // Send [DONE] marker for OpenAI compatibility
+ endOfStream := "[DONE]"
+ if err = sender(&endOfStream, nil); err != nil {
+ return err
+ }
+
+ return scanner.Err()
+}
+
+func (x *XunFeiModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ var region = "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ url := fmt.Sprintf("%s/%s", x.BaseURL[region], x.URLSuffix.Models)
+
+ // Build request body
+ reqBody := map[string]interface{}{}
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("GET", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := x.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s : %s", resp.StatusCode, string(body))
+ }
+
+ // Parse response
+ var result map[string]interface{}
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ // convert result["data"] to []map[string]interface{}
+ models := make([]string, 0)
+ for _, model := range result["data"].([]interface{}) {
+ modelMap := model.(map[string]interface{})
+ modelName := modelMap["id"].(string)
+ models = append(models, modelName)
+ }
+
+ return models, nil
+}
+
+func (x *XunFeiModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) CheckConnection(apiConfig *APIConfig) error {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ //TODO implement me
+ panic("implement me")
+}
+
+func (x *XunFeiModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ //TODO implement me
+ panic("implement me")
+}
From 2dbe3b8a6227c769df50a837f8799fecbbb1ba6a Mon Sep 17 00:00:00 2001
From: Hamza Amin Khokhar <137163270+hak2979@users.noreply.github.com>
Date: Mon, 18 May 2026 15:54:30 +0500
Subject: [PATCH 186/666] fix: metadata_condition returning all docs when
filter matches nothing (#14967)
### What problem does this PR solve?
When _parse_doc_id_filter_with_metadata returns [], the empty list is
falsy so the WHERE id IN (...) clause was silently skipped, causing the
full dataset to be returned instead of an empty result.
Change `if doc_ids:` to `if doc_ids is not None:` in both get_list() and
get_by_kb_id() to distinguish between no filter (None) and a filter that
matched zero documents ([]).
Fixes #14962
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/restful_apis/document_api.py | 2 +-
api/db/services/document_service.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.py
index 7547d2f20b5..57215080d35 100644
--- a/api/apps/restful_apis/document_api.py
+++ b/api/apps/restful_apis/document_api.py
@@ -995,7 +995,7 @@ def _parse_doc_id_filter_with_metadata(req, kb_id):
if not doc_ids_filter:
return RetCode.SUCCESS, "", [], return_empty_metadata
- return RetCode.SUCCESS, "", list(doc_ids_filter) if doc_ids_filter is not None else [], return_empty_metadata
+ return RetCode.SUCCESS, "", list(doc_ids_filter) if doc_ids_filter is not None else None, return_empty_metadata
@manager.route("/datasets//documents", methods=["DELETE"]) # noqa: F821
diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py
index 3d9bc09dbbb..7b0bce6e62b 100644
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -88,7 +88,7 @@ def get_list(cls, kb_id, page_number, items_per_page, orderby, desc, keywords, i
docs = docs.where(cls.model.name == name)
if keywords:
docs = docs.where(fn.LOWER(cls.model.name).contains(keywords.lower()))
- if doc_ids:
+ if doc_ids is not None:
docs = docs.where(cls.model.id.in_(doc_ids))
if suffix:
docs = docs.where(cls.model.suffix.in_(suffix))
@@ -143,7 +143,7 @@ def get_by_kb_id(cls, kb_id, page_number, items_per_page, orderby, desc, keyword
.join(User, on=(cls.model.created_by == User.id), join_type=JOIN.LEFT_OUTER)
.where(cls.model.kb_id == kb_id)
)
- if doc_ids:
+ if doc_ids is not None:
docs = docs.where(cls.model.id.in_(doc_ids))
if run_status:
docs = docs.where(cls.model.run.in_(run_status))
From 732e4741c4cde4286a759f30f80bf7b303b120a3 Mon Sep 17 00:00:00 2001
From: Wang Qi
Date: Mon, 18 May 2026 18:55:01 +0800
Subject: [PATCH 187/666] Bugfix: fix tag show (#14980)
### What problem does this PR solve?
Bugfix: fix tag show
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/services/dataset_api_service.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 5927b780ec6..8d5f512a358 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -598,9 +598,8 @@ def aggregate_tags(dataset_ids: list[str], tenant_id: str):
merged = {}
for kb_tenant_id, kb_ids in dataset_ids_by_tenant.items():
- for bucket in settings.retriever.all_tags(kb_tenant_id, kb_ids):
- tag = bucket["value"]
- merged[tag] = merged.get(tag, 0) + bucket["count"]
+ for tag, count in settings.retriever.all_tags(kb_tenant_id, kb_ids):
+ merged[tag] = merged.get(tag, 0) + count
return True, [{"value": tag, "count": count} for tag, count in merged.items()]
From 93d3deb5e425966197575449a88ccb077312251a Mon Sep 17 00:00:00 2001
From: Jake Armstrong <65635253+jakearmstrong59@users.noreply.github.com>
Date: Mon, 18 May 2026 01:08:45 -1000
Subject: [PATCH 188/666] Fix admin CLI system variable commands (#14956)
## What
Fixes #12409.
Implements admin CLI support for:
- `list vars;`
- `show var ;`
- `set var ;`
## Changes
- Wire Go CLI variable commands to the admin API.
- Support integer and quoted string values in `SET VAR`.
- Return variable rows as `data_type`, `name`, `setting_type`, and
`value`.
- Add exact-name lookup with prefix fallback for `SHOW VAR`.
- Validate values by stored data type: `string`, `integer`, `bool`, and
`json`.
- Keep the legacy Python admin CLI/server behavior aligned.
- Update admin CLI docs and add focused tests.
## Verification
- `go test -count=1 ./internal/cli`
- `python3.12 -m py_compile admin/server/services.py
admin/server/routes.py api/db/services/system_settings_service.py
admin/client/parser.py admin/client/ragflow_client.py`
- Python admin CLI parser smoke test for `SET VAR`, quoted values, `SHOW
VAR`, and `LIST VARS`.
- Attempted `./run_go_tests.sh`; local environment is missing native
tokenizer/linker artifacts:
- `internal/cpp/cmake-build-release/librag_tokenizer_c_api.a`
- `-lstdc++`
Co-authored-by: Jin Hai
---
admin/client/parser.py | 3 +-
admin/client/ragflow_client.py | 15 +-
admin/server/routes.py | 4 +-
admin/server/services.py | 74 ++++---
api/db/services/system_settings_service.py | 8 +-
docs/administrator/admin/ragflow_cli.md | 26 +--
internal/admin/service.go | 102 ++++++---
internal/admin/service_variables_test.go | 65 ++++++
internal/cli/admin_command.go | 136 ++++++++++++
internal/cli/admin_parser.go | 2 +-
internal/cli/admin_variables_test.go | 235 +++++++++++++++++++++
internal/cli/client.go | 6 +
internal/cli/parser.go | 9 +
internal/cli/response.go | 28 +++
internal/cli/user_parser.go | 2 +-
internal/dao/system_settings.go | 15 +-
16 files changed, 646 insertions(+), 84 deletions(-)
create mode 100644 internal/admin/service_variables_test.go
create mode 100644 internal/cli/admin_variables_test.go
diff --git a/admin/client/parser.py b/admin/client/parser.py
index cdb20b491dd..7e668c4e299 100644
--- a/admin/client/parser.py
+++ b/admin/client/parser.py
@@ -264,7 +264,7 @@
list_keys: LIST KEYS OF quoted_string ";"
drop_key: DROP KEY quoted_string OF quoted_string ";"
-set_variable: SET VAR identifier identifier ";"
+set_variable: SET VAR identifier variable_value ";"
show_variable: SHOW VAR identifier ";"
list_variables: LIST VARS ";"
list_configs: LIST CONFIGS ";"
@@ -378,6 +378,7 @@
identifier_list: identifier (COMMA identifier)*
identifier: WORD
+variable_value: WORD | NUMBER | QUOTED_STRING
quoted_string: QUOTED_STRING
status: ON | WORD
diff --git a/admin/client/ragflow_client.py b/admin/client/ragflow_client.py
index 148af4b45fe..71a5541bbae 100644
--- a/admin/client/ragflow_client.py
+++ b/admin/client/ragflow_client.py
@@ -43,6 +43,12 @@ def encrypt(input_string):
return base64.b64encode(cipher_text).decode("utf-8")
+def _strip_tree_value(value):
+ if isinstance(value, Tree):
+ value = value.children[0]
+ return str(value).strip("'\"")
+
+
class RAGFlowClient:
def __init__(self, http_client: HttpClient, server_type: str):
self.http_client = http_client
@@ -526,10 +532,8 @@ def set_variable(self, command):
if self.server_type != "admin":
print("This command is only allowed in ADMIN mode")
- var_name_tree: Tree = command["var_name"]
- var_name = var_name_tree.children[0].strip("'\"")
- var_value_tree: Tree = command["var_value"]
- var_value = var_value_tree.children[0].strip("'\"")
+ var_name = _strip_tree_value(command["var_name"])
+ var_value = _strip_tree_value(command["var_value"])
response = self.http_client.request("PUT", "/admin/variables",
json_body={"var_name": var_name, "var_value": var_value}, use_api_base=True,
auth_kind="admin")
@@ -544,8 +548,7 @@ def show_variable(self, command):
if self.server_type != "admin":
print("This command is only allowed in ADMIN mode")
- var_name_tree: Tree = command["var_name"]
- var_name = var_name_tree.children[0].strip("'\"")
+ var_name = _strip_tree_value(command["var_name"])
response = self.http_client.request(method="GET", path="/admin/variables", json_body={"var_name": var_name},
use_api_base=True, auth_kind="admin")
res_json = response.json()
diff --git a/admin/server/routes.py b/admin/server/routes.py
index 658cec48c09..0313d8230be 100644
--- a/admin/server/routes.py
+++ b/admin/server/routes.py
@@ -421,7 +421,7 @@ def get_user_permission(user_name: str):
def set_variable():
try:
data = request.get_json()
- if not data and "var_name" not in data:
+ if not data or "var_name" not in data:
return error_response("Var name is required", 400)
if "var_value" not in data:
@@ -449,7 +449,7 @@ def get_variable():
# get var
data = request.get_json()
- if not data and "var_name" not in data:
+ if not data or "var_name" not in data:
return error_response("Var name is required", 400)
var_name: str = data["var_name"]
res = SettingsMgr.get_by_name(var_name)
diff --git a/admin/server/services.py b/admin/server/services.py
index 43646d7918a..cc68f98dc3c 100644
--- a/admin/server/services.py
+++ b/admin/server/services.py
@@ -330,36 +330,65 @@ def restart_service(service_id: int):
class SettingsMgr:
+ @staticmethod
+ def _format_setting(setting):
+ return {
+ "data_type": setting.data_type,
+ "name": setting.name,
+ "setting_type": "config",
+ "value": setting.value,
+ }
+
+ @staticmethod
+ def _validate_value(name: str, data_type: str, value: str):
+ data_type = data_type.lower()
+ value = str(value)
+ if data_type == "string":
+ return
+ if data_type == "integer":
+ try:
+ int(value)
+ except ValueError:
+ raise AdminException(f"Invalid integer value for {name}: {value}")
+ return
+ if data_type in {"bool", "boolean"}:
+ if value not in {"true", "false"}:
+ raise AdminException(f"Invalid bool value for {name}: expected true or false")
+ return
+ if data_type == "json":
+ try:
+ json.loads(value)
+ except json.JSONDecodeError:
+ raise AdminException(f"Invalid JSON value for {name}")
+ return
+ raise AdminException(f"Unsupported data type for {name}: {data_type}")
+
+ @staticmethod
+ def _infer_data_type(name: str):
+ if name.startswith("sandbox."):
+ return "json"
+ if name.endswith(".enabled"):
+ return "bool"
+ return "string"
+
@staticmethod
def get_all():
- settings = SystemSettingsService.get_all()
+ settings = SystemSettingsService.get_all(reverse=False, order_by="name")
result = []
for setting in settings:
- result.append(
- {
- "name": setting.name,
- "source": setting.source,
- "data_type": setting.data_type,
- "value": setting.value,
- }
- )
+ result.append(SettingsMgr._format_setting(setting))
return result
@staticmethod
def get_by_name(name: str):
settings = SystemSettingsService.get_by_name(name)
if len(settings) == 0:
- raise AdminException(f"Can't get setting: {name}")
+ settings = SystemSettingsService.get_by_name_prefix(name)
+ if len(settings) == 0:
+ raise AdminException(f"Can't get setting: {name}")
result = []
for setting in settings:
- result.append(
- {
- "name": setting.name,
- "source": setting.source,
- "data_type": setting.data_type,
- "value": setting.value,
- }
- )
+ result.append(SettingsMgr._format_setting(setting))
return result
@staticmethod
@@ -367,6 +396,7 @@ def update_by_name(name: str, value: str):
settings = SystemSettingsService.get_by_name(name)
if len(settings) == 1:
setting = settings[0]
+ SettingsMgr._validate_value(name, setting.data_type, value)
setting.value = value
setting_dict = setting.to_dict()
SystemSettingsService.update_by_name(name, setting_dict)
@@ -376,12 +406,8 @@ def update_by_name(name: str, value: str):
# Create new setting if it doesn't exist
# Determine data_type based on name and value
- if name.startswith("sandbox."):
- data_type = "json"
- elif name.endswith(".enabled"):
- data_type = "boolean"
- else:
- data_type = "string"
+ data_type = SettingsMgr._infer_data_type(name)
+ SettingsMgr._validate_value(name, data_type, value)
new_setting = {
"name": name,
diff --git a/api/db/services/system_settings_service.py b/api/db/services/system_settings_service.py
index eac7019e6a1..0b0bde80242 100644
--- a/api/db/services/system_settings_service.py
+++ b/api/db/services/system_settings_service.py
@@ -26,7 +26,13 @@ class SystemSettingsService(CommonService):
@classmethod
@DB.connection_context()
def get_by_name(cls, name):
- objs = cls.model.select().where(cls.model.name == name)
+ objs = cls.model.select().where(cls.model.name == name).order_by(cls.model.name.asc())
+ return objs
+
+ @classmethod
+ @DB.connection_context()
+ def get_by_name_prefix(cls, name_prefix):
+ objs = cls.model.select().where(cls.model.name.startswith(name_prefix)).order_by(cls.model.name.asc())
return objs
@classmethod
diff --git a/docs/administrator/admin/ragflow_cli.md b/docs/administrator/admin/ragflow_cli.md
index 6c7bc5943c9..682105116c8 100644
--- a/docs/administrator/admin/ragflow_cli.md
+++ b/docs/administrator/admin/ragflow_cli.md
@@ -468,18 +468,18 @@ Revoke successfully!
```
ragflow> list vars;
+-----------+---------------------+--------------+-----------+
-| data_type | name | source | value |
+| data_type | name | setting_type | value |
+-----------+---------------------+--------------+-----------+
-| string | default_role | variable | user |
-| bool | enable_whitelist | variable | true |
-| string | mail.default_sender | variable | |
-| string | mail.password | variable | |
-| integer | mail.port | variable | 15 |
-| string | mail.server | variable | localhost |
-| integer | mail.timeout | variable | 10 |
-| bool | mail.use_ssl | variable | true |
-| bool | mail.use_tls | variable | false |
-| string | mail.username | variable | |
+| string | default_role | config | user |
+| bool | enable_whitelist | config | true |
+| string | mail.default_sender | config | |
+| string | mail.password | config | |
+| integer | mail.port | config | 15 |
+| string | mail.server | config | localhost |
+| integer | mail.timeout | config | 10 |
+| bool | mail.use_ssl | config | true |
+| bool | mail.use_tls | config | false |
+| string | mail.username | config | |
+-----------+---------------------+--------------+-----------+
```
@@ -490,9 +490,9 @@ ragflow> list vars;
```
ragflow> show var mail.server;
+-----------+-------------+--------------+-----------+
-| data_type | name | source | value |
+| data_type | name | setting_type | value |
+-----------+-------------+--------------+-----------+
-| string | mail.server | variable | localhost |
+| string | mail.server | config | localhost |
+-----------+-------------+--------------+-----------+
```
diff --git a/internal/admin/service.go b/internal/admin/service.go
index b857ac59aae..4b30b4f26cd 100644
--- a/internal/admin/service.go
+++ b/internal/admin/service.go
@@ -21,6 +21,7 @@ import (
"crypto/tls"
"encoding/base64"
"encoding/hex"
+ "encoding/json"
"errors"
"fmt"
"net/http"
@@ -1451,9 +1452,59 @@ func NewAdminException(message string) *AdminException {
}
}
+func formatSystemSetting(setting entity.SystemSettings) map[string]interface{} {
+ return map[string]interface{}{
+ "data_type": setting.DataType,
+ "name": setting.Name,
+ "setting_type": "config",
+ "value": setting.Value,
+ }
+}
+
+func formatSystemSettings(settings []entity.SystemSettings) []map[string]interface{} {
+ result := make([]map[string]interface{}, 0, len(settings))
+ for _, setting := range settings {
+ result = append(result, formatSystemSetting(setting))
+ }
+ return result
+}
+
+func validateSystemSettingValue(setting entity.SystemSettings, value string) error {
+ dataType := strings.ToLower(setting.DataType)
+ switch dataType {
+ case "string":
+ return nil
+ case "integer", "int":
+ if _, err := strconv.Atoi(value); err != nil {
+ return NewAdminException(fmt.Sprintf("Invalid integer value for %s: %s", setting.Name, value))
+ }
+ case "bool", "boolean":
+ if value != "true" && value != "false" {
+ return NewAdminException(fmt.Sprintf("Invalid bool value for %s: expected true or false", setting.Name))
+ }
+ case "json":
+ if !json.Valid([]byte(value)) {
+ return NewAdminException(fmt.Sprintf("Invalid JSON value for %s", setting.Name))
+ }
+ default:
+ return NewAdminException(fmt.Sprintf("Unsupported data type for %s: %s", setting.Name, setting.DataType))
+ }
+ return nil
+}
+
+func inferSystemSettingDataType(name string) string {
+ if strings.HasPrefix(name, "sandbox.") {
+ return "json"
+ }
+ if strings.HasSuffix(name, ".enabled") {
+ return "bool"
+ }
+ return "string"
+}
+
// GetVariable get variable by name
-// Returns the system setting with the given name
-// Returns AdminException if the setting is not found
+// Returns the exact system setting with the given name, or settings matching the
+// given name prefix when an exact setting does not exist.
func (s *Service) GetVariable(varName string) ([]map[string]interface{}, error) {
settings, err := s.systemSettingsDAO.GetByName(varName)
if err != nil {
@@ -1461,19 +1512,15 @@ func (s *Service) GetVariable(varName string) ([]map[string]interface{}, error)
}
if len(settings) == 0 {
- return nil, NewAdminException("Can't get setting: " + varName)
- }
-
- result := make([]map[string]interface{}, 0, len(settings))
- for _, setting := range settings {
- result = append(result, map[string]interface{}{
- "name": setting.Name,
- "source": setting.Source,
- "data_type": setting.DataType,
- "value": setting.Value,
- })
+ settings, err = s.systemSettingsDAO.GetByNamePrefix(varName)
+ if err != nil {
+ return nil, err
+ }
+ if len(settings) == 0 {
+ return nil, NewAdminException("Can't get setting: " + varName)
+ }
}
- return result, nil
+ return formatSystemSettings(settings), nil
}
// GetAllVariables get all variables
@@ -1484,16 +1531,7 @@ func (s *Service) GetAllVariables() ([]map[string]interface{}, error) {
return nil, err
}
- result := make([]map[string]interface{}, 0, len(settings))
- for _, setting := range settings {
- result = append(result, map[string]interface{}{
- "name": setting.Name,
- "source": setting.Source,
- "data_type": setting.DataType,
- "value": setting.Value,
- })
- }
- return result, nil
+ return formatSystemSettings(settings), nil
}
// SetVariable set variable
@@ -1507,27 +1545,25 @@ func (s *Service) SetVariable(varName, varValue string) error {
if len(settings) == 1 {
setting := &settings[0]
+ if err := validateSystemSettingValue(*setting, varValue); err != nil {
+ return err
+ }
setting.Value = varValue
return s.systemSettingsDAO.UpdateByName(varName, setting)
} else if len(settings) > 1 {
return NewAdminException("Can't update more than 1 setting: " + varName)
}
- // Create new setting if it doesn't exist
- // Determine data_type based on name and value
- dataType := "string"
- if len(varName) >= 7 && varName[:7] == "sandbox" {
- dataType = "json"
- } else if len(varName) >= 9 && varName[len(varName)-9:] == ".enabled" {
- dataType = "boolean"
- }
-
+ dataType := inferSystemSettingDataType(varName)
newSetting := &entity.SystemSettings{
Name: varName,
Value: varValue,
Source: "admin",
DataType: dataType,
}
+ if err := validateSystemSettingValue(*newSetting, varValue); err != nil {
+ return err
+ }
return s.systemSettingsDAO.Create(newSetting)
}
diff --git a/internal/admin/service_variables_test.go b/internal/admin/service_variables_test.go
new file mode 100644
index 00000000000..2b94a09088e
--- /dev/null
+++ b/internal/admin/service_variables_test.go
@@ -0,0 +1,65 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package admin
+
+import (
+ "ragflow/internal/entity"
+ "testing"
+)
+
+func TestValidateSystemSettingValue(t *testing.T) {
+ tests := []struct {
+ name string
+ dataType string
+ value string
+ wantError bool
+ }{
+ {name: "string accepts arbitrary text", dataType: "string", value: "local host"},
+ {name: "integer accepts digits", dataType: "integer", value: "15"},
+ {name: "integer rejects text", dataType: "integer", value: "localhost", wantError: true},
+ {name: "bool accepts true", dataType: "bool", value: "true"},
+ {name: "bool accepts false", dataType: "bool", value: "false"},
+ {name: "bool rejects non bool", dataType: "bool", value: "yes", wantError: true},
+ {name: "json accepts object", dataType: "json", value: `{"endpoint":"http://localhost:9385"}`},
+ {name: "json rejects invalid", dataType: "json", value: "{", wantError: true},
+ {name: "unknown type rejects", dataType: "float", value: "1.2", wantError: true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ setting := entity.SystemSettings{Name: "test.setting", DataType: tt.dataType}
+ err := validateSystemSettingValue(setting, tt.value)
+ if (err != nil) != tt.wantError {
+ t.Fatalf("validateSystemSettingValue() error = %v, wantError %v", err, tt.wantError)
+ }
+ })
+ }
+}
+
+func TestInferSystemSettingDataType(t *testing.T) {
+ tests := map[string]string{
+ "sandbox.self_managed": "json",
+ "mail.enabled": "bool",
+ "mail.server": "string",
+ }
+
+ for name, want := range tests {
+ if got := inferSystemSettingDataType(name); got != want {
+ t.Fatalf("inferSystemSettingDataType(%q) = %q, want %q", name, got, want)
+ }
+ }
+}
diff --git a/internal/cli/admin_command.go b/internal/cli/admin_command.go
index f6ab603af5c..d1c37636016 100644
--- a/internal/cli/admin_command.go
+++ b/internal/cli/admin_command.go
@@ -596,6 +596,142 @@ func (c *RAGFlowClient) ShowService(cmd *Command) (ResponseIf, error) {
return &result, nil
}
+func normalizeVariableRows(rows []map[string]interface{}) {
+ for _, row := range rows {
+ if _, ok := row["setting_type"]; ok {
+ delete(row, "source")
+ continue
+ }
+ if _, ok := row["source"]; ok {
+ row["setting_type"] = "config"
+ delete(row, "source")
+ }
+ }
+}
+
+// ListVariables lists all system variables (admin mode only).
+func (c *RAGFlowClient) ListVariables(cmd *Command) (ResponseIf, error) {
+ if c.ServerType != "admin" {
+ return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
+ }
+
+ iterations := 1
+ if val, ok := cmd.Params["iterations"].(int); ok && val > 1 {
+ iterations = val
+ }
+
+ if iterations > 1 {
+ return c.HTTPClient.RequestWithIterations("GET", "/admin/variables", "admin", nil, nil, iterations)
+ }
+
+ resp, err := c.HTTPClient.Request("GET", "/admin/variables", "admin", nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list variables: %w", err)
+ }
+
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to list variables: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+
+ var result CommonResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("list variables failed: invalid JSON (%w)", err)
+ }
+
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+
+ normalizeVariableRows(result.Data)
+ result.Duration = resp.Duration
+ return &result, nil
+}
+
+// ShowVariable shows system variables by exact name or name prefix (admin mode only).
+func (c *RAGFlowClient) ShowVariable(cmd *Command) (ResponseIf, error) {
+ if c.ServerType != "admin" {
+ return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
+ }
+
+ varName, ok := cmd.Params["var_name"].(string)
+ if !ok {
+ return nil, fmt.Errorf("var_name not provided")
+ }
+
+ iterations := 1
+ if val, ok := cmd.Params["iterations"].(int); ok && val > 1 {
+ iterations = val
+ }
+
+ payload := map[string]interface{}{"var_name": varName}
+ if iterations > 1 {
+ return c.HTTPClient.RequestWithIterations("GET", "/admin/variables", "admin", nil, payload, iterations)
+ }
+
+ resp, err := c.HTTPClient.Request("GET", "/admin/variables", "admin", nil, payload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to show variable: %w", err)
+ }
+
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to show variable: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+
+ var result CommonResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("show variable failed: invalid JSON (%w)", err)
+ }
+
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+
+ normalizeVariableRows(result.Data)
+ result.Duration = resp.Duration
+ return &result, nil
+}
+
+// SetVariable updates a system variable (admin mode only).
+func (c *RAGFlowClient) SetVariable(cmd *Command) (ResponseIf, error) {
+ if c.ServerType != "admin" {
+ return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
+ }
+
+ varName, ok := cmd.Params["var_name"].(string)
+ if !ok {
+ return nil, fmt.Errorf("var_name not provided")
+ }
+ varValue, ok := cmd.Params["var_value"].(string)
+ if !ok {
+ return nil, fmt.Errorf("var_value not provided")
+ }
+
+ payload := map[string]interface{}{
+ "var_name": varName,
+ "var_value": varValue,
+ }
+ resp, err := c.HTTPClient.Request("PUT", "/admin/variables", "admin", nil, payload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to set variable: %w", err)
+ }
+
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to set variable: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+ }
+
+ var result MessageResponse
+ if err = json.Unmarshal(resp.Body, &result); err != nil {
+ return nil, fmt.Errorf("set variable failed: invalid JSON (%w)", err)
+ }
+
+ if result.Code != 0 {
+ return nil, fmt.Errorf("%s", result.Message)
+ }
+
+ result.Duration = resp.Duration
+ return &result, nil
+}
+
// ListUsers lists all users (admin mode only)
// Returns (result_map, error) - result_map is non-nil for benchmark mode
func (c *RAGFlowClient) ListUsers(cmd *Command) (ResponseIf, error) {
diff --git a/internal/cli/admin_parser.go b/internal/cli/admin_parser.go
index c1b2edab5a7..9f4c6228e88 100644
--- a/internal/cli/admin_parser.go
+++ b/internal/cli/admin_parser.go
@@ -1173,7 +1173,7 @@ func (p *Parser) parseAdminSetVariable() (*Command, error) {
}
p.nextToken()
- varValue, err := p.parseIdentifier()
+ varValue, err := p.parseVariableValue()
if err != nil {
return nil, err
}
diff --git a/internal/cli/admin_variables_test.go b/internal/cli/admin_variables_test.go
new file mode 100644
index 00000000000..c871bca18e8
--- /dev/null
+++ b/internal/cli/admin_variables_test.go
@@ -0,0 +1,235 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package cli
+
+import (
+ "encoding/json"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strconv"
+ "testing"
+)
+
+func TestParseAdminVariableCommands(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ command string
+ varName string
+ varValue string
+ hasValue bool
+ adminMode bool
+ }{
+ {
+ name: "list variables",
+ input: "list vars;",
+ command: "list_variables",
+ adminMode: true,
+ },
+ {
+ name: "show variables by prefix",
+ input: "show var mail;",
+ command: "show_variable",
+ varName: "mail",
+ adminMode: true,
+ },
+ {
+ name: "set integer variable",
+ input: "set var mail.port 15;",
+ command: "set_variable",
+ varName: "mail.port",
+ varValue: "15",
+ hasValue: true,
+ adminMode: true,
+ },
+ {
+ name: "set quoted string variable",
+ input: `set var mail.server "local host";`,
+ command: "set_variable",
+ varName: "mail.server",
+ varValue: "local host",
+ hasValue: true,
+ adminMode: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cmd, err := NewParser(tt.input).Parse(tt.adminMode)
+ if err != nil {
+ t.Fatalf("Parse() error = %v", err)
+ }
+ if cmd.Type != tt.command {
+ t.Fatalf("command type = %q, want %q", cmd.Type, tt.command)
+ }
+ if tt.varName != "" && cmd.Params["var_name"] != tt.varName {
+ t.Fatalf("var_name = %v, want %q", cmd.Params["var_name"], tt.varName)
+ }
+ if tt.hasValue && cmd.Params["var_value"] != tt.varValue {
+ t.Fatalf("var_value = %v, want %q", cmd.Params["var_value"], tt.varValue)
+ }
+ })
+ }
+}
+
+func newAdminTestClient(t *testing.T, handler http.HandlerFunc) (*RAGFlowClient, func()) {
+ t.Helper()
+
+ server := httptest.NewServer(handler)
+ serverURL, err := url.Parse(server.URL)
+ if err != nil {
+ t.Fatalf("parse test server URL: %v", err)
+ }
+ host, portText, err := net.SplitHostPort(serverURL.Host)
+ if err != nil {
+ t.Fatalf("split host port: %v", err)
+ }
+ port, err := strconv.Atoi(portText)
+ if err != nil {
+ t.Fatalf("parse port: %v", err)
+ }
+
+ client := NewRAGFlowClient("admin")
+ client.HTTPClient.Host = host
+ client.HTTPClient.Port = port
+ client.HTTPClient.client = server.Client()
+ client.HTTPClient.LoginToken = "test-token"
+
+ return client, server.Close
+}
+
+func TestListVariablesUsesAdminVariablesEndpoint(t *testing.T) {
+ client, closeServer := newAdminTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ t.Errorf("method = %s, want GET", r.Method)
+ return
+ }
+ if r.URL.Path != "/api/v1/admin/variables" {
+ t.Errorf("path = %s, want /api/v1/admin/variables", r.URL.Path)
+ return
+ }
+ if r.Header.Get("Authorization") != "test-token" {
+ t.Errorf("Authorization header = %q", r.Header.Get("Authorization"))
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "code": 0,
+ "message": "",
+ "data": []map[string]interface{}{
+ {"data_type": "string", "name": "mail.server", "source": "variable", "value": "localhost"},
+ },
+ })
+ })
+ defer closeServer()
+
+ resp, err := client.ListVariables(NewCommand("list_variables"))
+ if err != nil {
+ t.Fatalf("ListVariables() error = %v", err)
+ }
+ result := resp.(*CommonResponse)
+ if got := result.Data[0]["setting_type"]; got != "config" {
+ t.Fatalf("setting_type = %v, want config", got)
+ }
+ if _, ok := result.Data[0]["source"]; ok {
+ t.Fatalf("source column should be normalized away: %#v", result.Data[0])
+ }
+}
+
+func TestShowVariableSendsRequestedName(t *testing.T) {
+ client, closeServer := newAdminTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ t.Errorf("method = %s, want GET", r.Method)
+ return
+ }
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read body: %v", err)
+ return
+ }
+ var request map[string]string
+ if err := json.Unmarshal(body, &request); err != nil {
+ t.Errorf("request body is not JSON: %v", err)
+ return
+ }
+ if request["var_name"] != "mail" {
+ t.Errorf("var_name = %q, want mail", request["var_name"])
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "code": 0,
+ "message": "",
+ "data": []map[string]interface{}{
+ {"data_type": "string", "name": "mail.server", "setting_type": "config", "value": "localhost"},
+ },
+ })
+ })
+ defer closeServer()
+
+ cmd := NewCommand("show_variable")
+ cmd.Params["var_name"] = "mail"
+ resp, err := client.ShowVariable(cmd)
+ if err != nil {
+ t.Fatalf("ShowVariable() error = %v", err)
+ }
+ result := resp.(*CommonResponse)
+ if got := result.Data[0]["name"]; got != "mail.server" {
+ t.Fatalf("name = %v, want mail.server", got)
+ }
+}
+
+func TestSetVariableReturnsServerConfirmation(t *testing.T) {
+ client, closeServer := newAdminTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPut {
+ t.Errorf("method = %s, want PUT", r.Method)
+ return
+ }
+ var request map[string]string
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Errorf("request body is not JSON: %v", err)
+ return
+ }
+ if request["var_name"] != "mail.server" {
+ t.Errorf("var_name = %q, want mail.server", request["var_name"])
+ return
+ }
+ if request["var_value"] != "localhost" {
+ t.Errorf("var_value = %q, want localhost", request["var_value"])
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "code": 0,
+ "message": "Set variable successfully",
+ "data": nil,
+ })
+ })
+ defer closeServer()
+
+ cmd := NewCommand("set_variable")
+ cmd.Params["var_name"] = "mail.server"
+ cmd.Params["var_value"] = "localhost"
+ resp, err := client.SetVariable(cmd)
+ if err != nil {
+ t.Fatalf("SetVariable() error = %v", err)
+ }
+ result := resp.(*MessageResponse)
+ if result.Message != "Set variable successfully" {
+ t.Fatalf("message = %q, want Set variable successfully", result.Message)
+ }
+}
diff --git a/internal/cli/client.go b/internal/cli/client.go
index 679e1d19b55..eebfb33d0d9 100644
--- a/internal/cli/client.go
+++ b/internal/cli/client.go
@@ -155,6 +155,12 @@ func (c *RAGFlowClient) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) {
return c.ShowAdminVersion(cmd)
case "show_user":
return c.ShowUser(cmd)
+ case "list_variables":
+ return c.ListVariables(cmd)
+ case "show_variable":
+ return c.ShowVariable(cmd)
+ case "set_variable":
+ return c.SetVariable(cmd)
case "list_user_datasets":
return c.ListUserDatasets(cmd)
case "list_agents":
diff --git a/internal/cli/parser.go b/internal/cli/parser.go
index 035d6b12e57..02723f5467b 100644
--- a/internal/cli/parser.go
+++ b/internal/cli/parser.go
@@ -285,6 +285,15 @@ func (p *Parser) parseIdentifier() (string, error) {
return p.curToken.Value, nil
}
+func (p *Parser) parseVariableValue() (string, error) {
+ switch p.curToken.Type {
+ case TokenIdentifier, TokenQuotedString, TokenInteger, TokenFloat:
+ return p.curToken.Value, nil
+ default:
+ return "", fmt.Errorf("expected variable value, got %s", p.curToken.Value)
+ }
+}
+
func (p *Parser) parseNumber() (int, error) {
if p.curToken.Type != TokenInteger {
return 0, fmt.Errorf("expected number, got %s", p.curToken.Value)
diff --git a/internal/cli/response.go b/internal/cli/response.go
index 440345be9d9..19daffc9e66 100644
--- a/internal/cli/response.go
+++ b/internal/cli/response.go
@@ -149,6 +149,34 @@ func (r *SimpleResponse) PrintOut() {
}
}
+type MessageResponse struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ Duration float64
+ OutputFormat OutputFormat
+}
+
+func (r *MessageResponse) Type() string {
+ return "message"
+}
+
+func (r *MessageResponse) TimeCost() float64 {
+ return r.Duration
+}
+
+func (r *MessageResponse) SetOutputFormat(format OutputFormat) {
+ r.OutputFormat = format
+}
+
+func (r *MessageResponse) PrintOut() {
+ if r.Code == 0 {
+ fmt.Println(r.Message)
+ } else {
+ fmt.Println("ERROR")
+ fmt.Printf("%d, %s\n", r.Code, r.Message)
+ }
+}
+
type NonStreamResponse struct {
Code int `json:"code"`
ReasoningContent string `json:"reasoning_content"`
diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go
index 38b2d2d29d9..36b7a352973 100644
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -1875,7 +1875,7 @@ func (p *Parser) parseSetVariable() (*Command, error) {
}
p.nextToken()
- varValue, err := p.parseIdentifier()
+ varValue, err := p.parseVariableValue()
if err != nil {
return nil, err
}
diff --git a/internal/dao/system_settings.go b/internal/dao/system_settings.go
index c224ad0fcde..dd7762e42e0 100644
--- a/internal/dao/system_settings.go
+++ b/internal/dao/system_settings.go
@@ -35,7 +35,7 @@ func NewSystemSettingsDAO() *SystemSettingsDAO {
// Returns all system settings records from database
func (d *SystemSettingsDAO) GetAll() ([]entity.SystemSettings, error) {
var settings []entity.SystemSettings
- err := DB.Find(&settings).Error
+ err := DB.Order("name ASC").Find(&settings).Error
if err != nil {
return nil, err
}
@@ -46,7 +46,18 @@ func (d *SystemSettingsDAO) GetAll() ([]entity.SystemSettings, error) {
// Returns settings records that match the given name
func (d *SystemSettingsDAO) GetByName(name string) ([]entity.SystemSettings, error) {
var settings []entity.SystemSettings
- err := DB.Where("name = ?", name).Find(&settings).Error
+ err := DB.Where("name = ?", name).Order("name ASC").Find(&settings).Error
+ if err != nil {
+ return nil, err
+ }
+ return settings, nil
+}
+
+// GetByNamePrefix get system settings by name prefix
+// Returns settings records whose names start with the given prefix.
+func (d *SystemSettingsDAO) GetByNamePrefix(namePrefix string) ([]entity.SystemSettings, error) {
+ var settings []entity.SystemSettings
+ err := DB.Where("name LIKE ?", namePrefix+"%").Order("name ASC").Find(&settings).Error
if err != nil {
return nil, err
}
From 32902570141c8da9976b3adbbeb5f0781ccdf887 Mon Sep 17 00:00:00 2001
From: buua436
Date: Mon, 18 May 2026 19:21:47 +0800
Subject: [PATCH 189/666] Go: fix forgetting policy validation and fix memory
update diff checks (#14976)
### What problem does this PR solve?
fix forgetting policy validation and fix memory update diff checks
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
internal/service/memory.go | 143 ++++++++++++++++++++++++++++++++++++-
1 file changed, 142 insertions(+), 1 deletion(-)
diff --git a/internal/service/memory.go b/internal/service/memory.go
index 882282f4ccb..93face56a93 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -499,7 +499,7 @@ func (s *MemoryService) UpdateMemory(tenantID string, memoryID string, req *Upda
}
if req.ForgettingPolicy != nil {
- fp := ForgettingPolicy(strings.ToLower(*req.ForgettingPolicy))
+ fp := ForgettingPolicy(strings.ToUpper(strings.TrimSpace(*req.ForgettingPolicy)))
if !validForgettingPolicies[fp] {
return nil, fmt.Errorf("forgetting policy '%s' is not supported", *req.ForgettingPolicy)
}
@@ -544,6 +544,108 @@ func (s *MemoryService) UpdateMemory(tenantID string, memoryID string, req *Upda
return formatRetDataFromMemory(currentMemory), nil
}
+ currentMemoryTypes := dao.GetMemoryTypeHuman(currentMemory.MemoryType)
+ normalizedCurrentMemoryTypes := normalizeMemoryTypes(currentMemoryTypes)
+
+ filteredUpdateDict := make(map[string]interface{}, len(updateDict))
+ for field, value := range updateDict {
+ switch field {
+ case "name":
+ currentName := strings.TrimSpace(currentMemory.Name)
+ requestName := strings.TrimSpace(fmt.Sprint(value))
+ if currentName != requestName {
+ filteredUpdateDict[field] = value
+ }
+ case "permissions":
+ currentPermissions := strings.ToLower(strings.TrimSpace(currentMemory.Permissions))
+ requestPermissions := strings.ToLower(strings.TrimSpace(fmt.Sprint(value)))
+ if currentPermissions != requestPermissions {
+ filteredUpdateDict[field] = value
+ }
+ case "llm_id":
+ currentLLMID := strings.TrimSpace(currentMemory.LLMID)
+ requestLLMID := strings.TrimSpace(fmt.Sprint(value))
+ if currentLLMID != requestLLMID {
+ filteredUpdateDict[field] = value
+ }
+ case "embd_id":
+ currentEmbdID := strings.TrimSpace(currentMemory.EmbdID)
+ requestEmbdID := strings.TrimSpace(fmt.Sprint(value))
+ if currentEmbdID != requestEmbdID {
+ filteredUpdateDict[field] = value
+ }
+ case "tenant_llm_id":
+ if currentMemory.TenantLLMID == nil || *currentMemory.TenantLLMID != value.(int64) {
+ filteredUpdateDict[field] = value
+ }
+ case "tenant_embd_id":
+ if currentMemory.TenantEmbdID == nil || *currentMemory.TenantEmbdID != value.(int64) {
+ filteredUpdateDict[field] = value
+ }
+ case "memory_type":
+ if types, ok := value.([]string); ok {
+ if !sameStringSet(normalizedCurrentMemoryTypes, normalizeMemoryTypes(types)) {
+ filteredUpdateDict[field] = value
+ }
+ } else {
+ filteredUpdateDict[field] = value
+ }
+ case "memory_size":
+ if currentMemory.MemorySize != value.(int64) {
+ filteredUpdateDict[field] = value
+ }
+ case "forgetting_policy":
+ currentForgettingPolicy := strings.ToUpper(strings.TrimSpace(currentMemory.ForgettingPolicy))
+ requestForgettingPolicy := strings.ToUpper(strings.TrimSpace(fmt.Sprint(value)))
+ if currentForgettingPolicy != requestForgettingPolicy {
+ filteredUpdateDict[field] = value
+ }
+ case "temperature":
+ if currentMemory.Temperature != value.(float64) {
+ filteredUpdateDict[field] = value
+ }
+ case "avatar":
+ currentAvatar := ""
+ if currentMemory.Avatar != nil {
+ currentAvatar = *currentMemory.Avatar
+ }
+ if currentAvatar != fmt.Sprint(value) {
+ filteredUpdateDict[field] = value
+ }
+ case "description":
+ currentDescription := ""
+ if currentMemory.Description != nil {
+ currentDescription = *currentMemory.Description
+ }
+ if currentDescription != fmt.Sprint(value) {
+ filteredUpdateDict[field] = value
+ }
+ case "system_prompt":
+ currentSystemPrompt := ""
+ if currentMemory.SystemPrompt != nil {
+ currentSystemPrompt = *currentMemory.SystemPrompt
+ }
+ if currentSystemPrompt != fmt.Sprint(value) {
+ filteredUpdateDict[field] = value
+ }
+ case "user_prompt":
+ currentUserPrompt := ""
+ if currentMemory.UserPrompt != nil {
+ currentUserPrompt = *currentMemory.UserPrompt
+ }
+ if currentUserPrompt != fmt.Sprint(value) {
+ filteredUpdateDict[field] = value
+ }
+ default:
+ filteredUpdateDict[field] = value
+ }
+ }
+ updateDict = filteredUpdateDict
+
+ if len(updateDict) == 0 {
+ return formatRetDataFromMemory(currentMemory), nil
+ }
+
memorySize := currentMemory.MemorySize
notAllowedUpdate := []string{}
for _, f := range []string{"tenant_embd_id", "embd_id", "memory_type"} {
@@ -581,6 +683,45 @@ func (s *MemoryService) UpdateMemory(tenantID string, memoryID string, req *Upda
return formatRetDataFromMemory(updatedMemory), nil
}
+func normalizeMemoryTypes(memoryTypes []string) []string {
+ normalized := make([]string, 0, len(memoryTypes))
+ seen := make(map[string]struct{}, len(memoryTypes))
+ for _, mt := range memoryTypes {
+ mt = strings.ToLower(strings.TrimSpace(mt))
+ if mt == "" {
+ continue
+ }
+ if _, exists := seen[mt]; exists {
+ continue
+ }
+ seen[mt] = struct{}{}
+ normalized = append(normalized, mt)
+ }
+ return normalized
+}
+
+func sameStringSet(a, b []string) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ counts := make(map[string]int, len(a))
+ for _, item := range a {
+ counts[item]++
+ }
+ for _, item := range b {
+ counts[item]--
+ if counts[item] < 0 {
+ return false
+ }
+ }
+ for _, count := range counts {
+ if count != 0 {
+ return false
+ }
+ }
+ return true
+}
+
// DeleteMemory deletes a memory by ID
// It also deletes associated message indexes before removing the memory record
//
From d7fb4bdb4ec50db877afda7dbed8e3bf7ba526c0 Mon Sep 17 00:00:00 2001
From: buua436
Date: Mon, 18 May 2026 20:00:11 +0800
Subject: [PATCH 190/666] Go: align document list response (#14982)
### What problem does this PR solve?
align document list response
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
internal/dao/document.go | 16 ++++-
internal/entity/document.go | 33 +++++++++++
internal/handler/document.go | 110 ++++++++++++++++++++++++++++++++---
internal/service/document.go | 6 +-
web/src/constants/file.ts | 2 +-
5 files changed, 151 insertions(+), 16 deletions(-)
diff --git a/internal/dao/document.go b/internal/dao/document.go
index 49ef0e88dc7..2fa2fa5b0a2 100644
--- a/internal/dao/document.go
+++ b/internal/dao/document.go
@@ -86,15 +86,25 @@ func (dao *DocumentDAO) List(offset, limit int) ([]*entity.Document, int64, erro
}
// ListByKBID list documents by knowledge base ID
-func (dao *DocumentDAO) ListByKBID(kbID string, offset, limit int) ([]*entity.Document, int64, error) {
- var documents []*entity.Document
+func (dao *DocumentDAO) ListByKBID(kbID string, offset, limit int) ([]*entity.DocumentListItem, int64, error) {
+ var documents []*entity.DocumentListItem
var total int64
if err := DB.Model(&entity.Document{}).Where("kb_id = ?", kbID).Count(&total).Error; err != nil {
return nil, 0, err
}
- err := DB.Where("kb_id = ?", kbID).Offset(offset).Limit(limit).Find(&documents).Error
+ err := DB.Table("document").
+ Select(`document.*, user_canvas.title as pipeline_name, user.nickname`).
+ Joins("JOIN file2document ON file2document.document_id = document.id").
+ Joins("JOIN file ON file.id = file2document.file_id").
+ Joins("LEFT JOIN user_canvas ON document.pipeline_id = user_canvas.id").
+ Joins("LEFT JOIN user ON document.created_by = user.id").
+ Where("document.kb_id = ?", kbID).
+ Order("document.create_time DESC").
+ Offset(offset).
+ Limit(limit).
+ Scan(&documents).Error
return documents, total, err
}
diff --git a/internal/entity/document.go b/internal/entity/document.go
index 36012196663..9b25ff46634 100644
--- a/internal/entity/document.go
+++ b/internal/entity/document.go
@@ -46,6 +46,39 @@ type Document struct {
BaseModel
}
+// DocumentListItem represents a document list row with joined fields.
+type DocumentListItem struct {
+ ID string `gorm:"column:id" json:"id"`
+ Thumbnail *string `gorm:"column:thumbnail" json:"thumbnail,omitempty"`
+ KbID string `gorm:"column:kb_id" json:"kb_id"`
+ ParserID string `gorm:"column:parser_id" json:"parser_id"`
+ PipelineID *string `gorm:"column:pipeline_id" json:"pipeline_id,omitempty"`
+ PipelineName *string `gorm:"column:pipeline_name" json:"pipeline_name,omitempty"`
+ ParserConfig string `gorm:"column:parser_config" json:"parser_config"`
+ SourceType string `gorm:"column:source_type" json:"source_type"`
+ Type string `gorm:"column:type" json:"type"`
+ CreatedBy string `gorm:"column:created_by" json:"created_by"`
+ Nickname *string `gorm:"column:nickname" json:"nickname,omitempty"`
+ Name *string `gorm:"column:name" json:"name,omitempty"`
+ Location *string `gorm:"column:location" json:"location,omitempty"`
+ Size int64 `gorm:"column:size" json:"size"`
+ TokenNum int64 `gorm:"column:token_num" json:"token_num"`
+ ChunkNum int64 `gorm:"column:chunk_num" json:"chunk_num"`
+ Progress float64 `gorm:"column:progress" json:"progress"`
+ ProgressMsg *string `gorm:"column:progress_msg" json:"progress_msg,omitempty"`
+ ProcessBeginAt *time.Time `gorm:"column:process_begin_at" json:"process_begin_at,omitempty"`
+ ProcessDuration float64 `gorm:"column:process_duration" json:"process_duration"`
+ ContentHash *string `gorm:"column:content_hash" json:"content_hash,omitempty"`
+ MetaFields *string `gorm:"column:meta_fields" json:"meta_fields,omitempty"`
+ Suffix string `gorm:"column:suffix" json:"suffix"`
+ Run *string `gorm:"column:run" json:"run,omitempty"`
+ Status *string `gorm:"column:status" json:"status,omitempty"`
+ CreateTime *int64 `gorm:"column:create_time" json:"create_time,omitempty"`
+ CreateDate *time.Time `gorm:"column:create_date" json:"create_date,omitempty"`
+ UpdateTime *int64 `gorm:"column:update_time" json:"update_time,omitempty"`
+ UpdateDate *time.Time `gorm:"column:update_date" json:"update_date,omitempty"`
+}
+
// TableName specify table name
func (Document) TableName() string {
return "document"
diff --git a/internal/handler/document.go b/internal/handler/document.go
index fd7e775dfa4..9b3307a724a 100644
--- a/internal/handler/document.go
+++ b/internal/handler/document.go
@@ -21,8 +21,10 @@ import (
"fmt"
"net/http"
"ragflow/internal/common"
+ "ragflow/internal/entity"
"strconv"
"strings"
+ "time"
"github.com/gin-gonic/gin"
@@ -241,15 +243,7 @@ func (h *DocumentHandler) ListDocuments(c *gin.Context) {
metaFields = make(map[string]interface{})
}
- docs = append(docs, map[string]interface{}{
- "id": doc.ID,
- "name": doc.Name,
- "size": doc.Size,
- "type": doc.Type,
- "status": doc.Status,
- "created_at": doc.CreatedAt,
- "meta_fields": metaFields,
- })
+ docs = append(docs, mapDocumentListItem(doc, metaFields))
}
c.JSON(http.StatusOK, gin.H{
@@ -262,6 +256,104 @@ func (h *DocumentHandler) ListDocuments(c *gin.Context) {
})
}
+func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]interface{}) map[string]interface{} {
+ item := map[string]interface{}{
+ "id": doc.ID,
+ "dataset_id": doc.KbID,
+ "name": stringValue(doc.Name),
+ "thumbnail": stringValue(doc.Thumbnail),
+ "size": doc.Size,
+ "type": doc.Type,
+ "created_by": doc.CreatedBy,
+ "location": stringValue(doc.Location),
+ "token_count": doc.TokenNum,
+ "chunk_count": doc.ChunkNum,
+ "progress": doc.Progress,
+ "progress_msg": stringValue(doc.ProgressMsg),
+ "process_begin_at": formatTimePtr(doc.ProcessBeginAt),
+ "process_duration": doc.ProcessDuration,
+ "suffix": doc.Suffix,
+ "run": mapRunStatus(doc.Run),
+ "status": stringValue(doc.Status),
+ "chunk_method": doc.ParserID,
+ "parser_id": doc.ParserID,
+ "pipeline_id": stringValue(doc.PipelineID),
+ "pipeline_name": stringValue(doc.PipelineName),
+ "nickname": stringValue(doc.Nickname),
+ "parser_config": decodeJSONMap(string(doc.ParserConfig)),
+ "meta_fields": metaFields,
+ "create_time": int64(0),
+ "create_date": "",
+ "update_time": int64(0),
+ "update_date": "",
+ }
+
+ if doc.CreateTime != nil {
+ item["create_time"] = *doc.CreateTime
+ }
+ if doc.CreateDate != nil {
+ item["create_date"] = doc.CreateDate.Format("2006-01-02 15:04:05")
+ }
+ if doc.UpdateTime != nil {
+ item["update_time"] = *doc.UpdateTime
+ }
+ if doc.UpdateDate != nil {
+ item["update_date"] = doc.UpdateDate.Format("2006-01-02 15:04:05")
+ }
+
+ return item
+}
+
+func decodeJSONMap(raw string) map[string]interface{} {
+ if strings.TrimSpace(raw) == "" {
+ return map[string]interface{}{}
+ }
+
+ var data map[string]interface{}
+ if err := json.Unmarshal([]byte(raw), &data); err != nil {
+ return map[string]interface{}{}
+ }
+
+ return data
+}
+
+func mapRunStatus(run *string) string {
+ if run == nil {
+ return "UNSTART"
+ }
+
+ switch strings.TrimSpace(*run) {
+ case "0":
+ return "UNSTART"
+ case "1":
+ return "RUNNING"
+ case "2":
+ return "CANCEL"
+ case "3":
+ return "DONE"
+ case "4":
+ return "FAIL"
+ default:
+ return strings.TrimSpace(*run)
+ }
+}
+
+func formatTimePtr(value *time.Time) string {
+ if value == nil {
+ return ""
+ }
+
+ return value.Format("2006-01-02 15:04:05")
+}
+
+func stringValue(value *string) string {
+ if value == nil {
+ return ""
+ }
+
+ return *value
+}
+
// GetDocumentsByAuthorID get documents by author ID
// @Summary Get Author Documents
// @Description Get paginated document list by author ID
diff --git a/internal/service/document.go b/internal/service/document.go
index 29ed2d4b694..aeef7fb0048 100644
--- a/internal/service/document.go
+++ b/internal/service/document.go
@@ -176,16 +176,16 @@ func (s *DocumentService) ListDocuments(page, pageSize int) ([]*DocumentResponse
}
// ListDocumentsByDatasetID list documents by knowledge base ID
-func (s *DocumentService) ListDocumentsByDatasetID(kbID string, page, pageSize int) ([]*DocumentResponse, int64, error) {
+func (s *DocumentService) ListDocumentsByDatasetID(kbID string, page, pageSize int) ([]*entity.DocumentListItem, int64, error) {
offset := (page - 1) * pageSize
documents, total, err := s.documentDAO.ListByKBID(kbID, offset, pageSize)
if err != nil {
return nil, 0, err
}
- responses := make([]*DocumentResponse, len(documents))
+ responses := make([]*entity.DocumentListItem, len(documents))
for i, doc := range documents {
- responses[i] = s.toResponse(doc)
+ responses[i] = doc
}
return responses, total, nil
diff --git a/web/src/constants/file.ts b/web/src/constants/file.ts
index 8d488c9713a..9b713e3d70a 100644
--- a/web/src/constants/file.ts
+++ b/web/src/constants/file.ts
@@ -10,7 +10,7 @@ export const FileIconMap = {
jpeg: 'jpg',
png: 'png',
txt: 'text',
- csv: 'pdf',
+ csv: 'excel',
md: 'md',
mdx: 'md',
mp4: 'mp4',
From 41a9fc003066b8f030a426e2a5c0f3e818141332 Mon Sep 17 00:00:00 2001
From: buua436
Date: Mon, 18 May 2026 20:02:53 +0800
Subject: [PATCH 191/666] Go: add dataset graph api (#14984)
### What problem does this PR solve?
add dataset graph api
### Type of change
- [x] Refactoring
---
internal/handler/datasets.go | 244 +++++++++++++++++++++++++++++++++++
internal/router/router.go | 2 +
2 files changed, 246 insertions(+)
diff --git a/internal/handler/datasets.go b/internal/handler/datasets.go
index aa0d896cb07..b856509511a 100644
--- a/internal/handler/datasets.go
+++ b/internal/handler/datasets.go
@@ -18,7 +18,11 @@ package handler
import (
"encoding/json"
+ "fmt"
"net/http"
+ "ragflow/internal/engine"
+ "ragflow/internal/engine/types"
+ "sort"
"strconv"
"strings"
@@ -198,3 +202,243 @@ func (h *DatasetsHandler) DeleteDatasets(c *gin.Context) {
"data": result,
})
}
+
+// GetKnowledgeGraph handles GET /api/v1/datasets/:dataset_id/graph.
+func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) {
+ user, errorCode, errorMessage := GetUser(c)
+ if errorCode != common.CodeSuccess {
+ jsonError(c, errorCode, errorMessage)
+ return
+ }
+
+ datasetID := strings.TrimSpace(c.Param("dataset_id"))
+ if datasetID == "" {
+ jsonError(c, common.CodeDataError, "dataset_id is required")
+ return
+ }
+
+ dataset, code, err := h.datasetsService.GetDataset(datasetID, user.ID)
+ if err != nil {
+ jsonError(c, code, err.Error())
+ return
+ }
+
+ tenantID, _ := dataset["tenant_id"].(string)
+ if tenantID == "" {
+ jsonError(c, common.CodeDataError, "tenant_id is required")
+ return
+ }
+
+ docEngine := engine.Get()
+ if docEngine == nil {
+ jsonError(c, common.CodeServerError, "Document engine is not initialized")
+ return
+ }
+
+ indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ exists, err := docEngine.TableExists(c.Request.Context(), indexName)
+ if err != nil {
+ jsonError(c, common.CodeServerError, err.Error())
+ return
+ }
+
+ result := gin.H{
+ "graph": map[string]interface{}{},
+ "mind_map": map[string]interface{}{},
+ }
+ if !exists {
+ jsonResponse(c, common.CodeSuccess, result, "success")
+ return
+ }
+
+ searchResult, err := docEngine.Search(c.Request.Context(), &types.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Offset: 0,
+ Limit: 1,
+ SelectFields: []string{"content_with_weight", "knowledge_graph_kwd"},
+ Filter: map[string]interface{}{
+ "kb_id": []string{datasetID},
+ "knowledge_graph_kwd": []string{"graph"},
+ },
+ })
+ if err != nil {
+ jsonError(c, common.CodeServerError, err.Error())
+ return
+ }
+ if searchResult == nil || len(searchResult.Chunks) == 0 {
+ jsonResponse(c, common.CodeSuccess, result, "success")
+ return
+ }
+
+ chunk := searchResult.Chunks[0]
+ graphType := firstStringValue(chunk["knowledge_graph_kwd"])
+ contentWithWeight, _ := chunk["content_with_weight"].(string)
+ if strings.TrimSpace(contentWithWeight) == "" {
+ jsonResponse(c, common.CodeSuccess, result, "success")
+ return
+ }
+
+ var graphData map[string]interface{}
+ if err := json.Unmarshal([]byte(contentWithWeight), &graphData); err != nil {
+ jsonResponse(c, common.CodeSuccess, result, "success")
+ return
+ }
+ if len(graphData) == 0 {
+ jsonResponse(c, common.CodeSuccess, result, "success")
+ return
+ }
+
+ if graphType == "" {
+ graphType = "graph"
+ }
+ if graphType == "graph" {
+ sortKnowledgeGraph(graphData)
+ result["graph"] = graphData
+ } else {
+ result[graphType] = graphData
+ }
+
+ jsonResponse(c, common.CodeSuccess, result, "success")
+}
+
+// DeleteKnowledgeGraph handles DELETE /api/v1/datasets/:dataset_id/graph.
+func (h *DatasetsHandler) DeleteKnowledgeGraph(c *gin.Context) {
+ user, errorCode, errorMessage := GetUser(c)
+ if errorCode != common.CodeSuccess {
+ jsonError(c, errorCode, errorMessage)
+ return
+ }
+
+ datasetID := strings.TrimSpace(c.Param("dataset_id"))
+ if datasetID == "" {
+ jsonError(c, common.CodeDataError, "dataset_id is required")
+ return
+ }
+
+ dataset, code, err := h.datasetsService.GetDataset(datasetID, user.ID)
+ if err != nil {
+ jsonError(c, code, err.Error())
+ return
+ }
+
+ tenantID, _ := dataset["tenant_id"].(string)
+ if tenantID == "" {
+ jsonError(c, common.CodeDataError, "tenant_id is required")
+ return
+ }
+
+ docEngine := engine.Get()
+ if docEngine == nil {
+ jsonError(c, common.CodeServerError, "Document engine is not initialized")
+ return
+ }
+
+ indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ if _, err := docEngine.Delete(c.Request.Context(), map[string]interface{}{
+ "knowledge_graph_kwd": []string{"graph", "subgraph", "entity", "relation", "community_report"},
+ }, indexName, datasetID); err != nil {
+ jsonError(c, common.CodeServerError, err.Error())
+ return
+ }
+
+ jsonResponse(c, common.CodeSuccess, true, "success")
+}
+
+func firstStringValue(value interface{}) string {
+ switch v := value.(type) {
+ case string:
+ return strings.TrimSpace(v)
+ case []string:
+ if len(v) > 0 {
+ return strings.TrimSpace(v[0])
+ }
+ case []interface{}:
+ for _, item := range v {
+ if s, ok := item.(string); ok {
+ s = strings.TrimSpace(s)
+ if s != "" {
+ return s
+ }
+ }
+ }
+ }
+ return ""
+}
+
+func sortKnowledgeGraph(graphData map[string]interface{}) {
+ nodes := mapSlice(graphData["nodes"])
+ if len(nodes) > 0 {
+ sort.Slice(nodes, func(i, j int) bool {
+ return numericValue(nodes[i]["pagerank"]) > numericValue(nodes[j]["pagerank"])
+ })
+ if len(nodes) > 256 {
+ nodes = nodes[:256]
+ }
+ graphData["nodes"] = nodes
+ }
+
+ edges := mapSlice(graphData["edges"])
+ if len(edges) > 0 {
+ nodeIDSet := make(map[string]struct{}, len(nodes))
+ for _, node := range nodes {
+ if id, ok := node["id"].(string); ok {
+ nodeIDSet[id] = struct{}{}
+ }
+ }
+ filteredEdges := make([]map[string]interface{}, 0, len(edges))
+ for _, edge := range edges {
+ source, _ := edge["source"].(string)
+ target, _ := edge["target"].(string)
+ if source == "" || target == "" || source == target {
+ continue
+ }
+ if _, ok := nodeIDSet[source]; !ok {
+ continue
+ }
+ if _, ok := nodeIDSet[target]; !ok {
+ continue
+ }
+ filteredEdges = append(filteredEdges, edge)
+ }
+ sort.Slice(filteredEdges, func(i, j int) bool {
+ return numericValue(filteredEdges[i]["weight"]) > numericValue(filteredEdges[j]["weight"])
+ })
+ if len(filteredEdges) > 128 {
+ filteredEdges = filteredEdges[:128]
+ }
+ graphData["edges"] = filteredEdges
+ }
+}
+
+func mapSlice(value interface{}) []map[string]interface{} {
+ raw, ok := value.([]interface{})
+ if !ok {
+ return nil
+ }
+ result := make([]map[string]interface{}, 0, len(raw))
+ for _, item := range raw {
+ if m, ok := item.(map[string]interface{}); ok {
+ result = append(result, m)
+ }
+ }
+ return result
+}
+
+func numericValue(value interface{}) float64 {
+ switch v := value.(type) {
+ case float64:
+ return v
+ case float32:
+ return float64(v)
+ case int:
+ return float64(v)
+ case int64:
+ return float64(v)
+ case json.Number:
+ f, _ := v.Float64()
+ return f
+ default:
+ return 0
+ }
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index ddb13e0ac95..5b8a840e7a1 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -179,6 +179,8 @@ func (r *Router) Setup(engine *gin.Engine) {
{
datasets.GET("", r.datasetsHandler.ListDatasets)
datasets.GET("/:dataset_id", r.datasetsHandler.GetDataset)
+ datasets.GET("/:dataset_id/graph", r.datasetsHandler.GetKnowledgeGraph)
+ datasets.DELETE("/:dataset_id/graph", r.datasetsHandler.DeleteKnowledgeGraph)
datasets.POST("", r.datasetsHandler.CreateDataset)
datasets.DELETE("", r.datasetsHandler.DeleteDatasets)
datasets.POST("/search", r.chunkHandler.RetrievalTest)
From b69a6a5d803432c32a7176a0ae42443140e64604 Mon Sep 17 00:00:00 2001
From: Magicbook1108
Date: Tue, 19 May 2026 10:07:11 +0800
Subject: [PATCH 192/666] Feat: full optimization on connector dashboard
(#14979)
### What problem does this PR solve?
This PR improves the connector dashboard task management experience and
adds better visibility into connector execution logs.
### Overview:
#### Before
#### Now:
### 1. Add a new logging page to the connector dashboard
A new logging page has been added so users can view connector task
execution logs directly from the connector dashboard.
### 2. Merge the Resume button into Confirm
The separate **Resume** button has been removed. The **Confirm** button
now represents different actions depending on the current task state:
- **Save**: Save form changes and reschedule tasks.
- **Stop**: Cancel currently scheduled or running tasks.
- **Resume**: Create new scheduled tasks after the previous tasks have
been stopped.
- **Start**: Start tasks when no task has been started yet.
### 3. Separate syncing and pruning tasks
Connector tasks are now separated into **syncing** and **pruning**.
Pruning is controlled by the **Sync deleted files** option:
- When **Sync deleted files** is disabled, only syncing tasks are shown.
- When **Sync deleted files** is enabled, both syncing and pruning tasks
are shown.
**Now: Sync deleted files disabled**
**Now: Sync deleted files enabled**
### 4. Update logs in backend
### 5. Remove connector resume API
- Removed: `POST /v1/connectors//resume`
- Replaced by: `PATCH /v1/connectors/`
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
---
api/apps/restful_apis/connector_api.py | 48 +-
api/db/db_models.py | 2 +
api/db/services/connector_service.py | 218 ++++++++--
common/constants.py | 5 +
rag/svr/sync_data_source.py | 411 +++++-------------
.../restful_api/test_connector_routes_unit.py | 28 +-
.../test_connector_routes_unit.py | 21 +-
test/unit_test/rag/test_sync_data_source.py | 83 +++-
web/src/components/dynamic-form.tsx | 9 +
.../data-source-detail-page/index.tsx | 174 ++++----
.../data-source-detail-page/log-table.tsx | 142 ++++--
.../pages/user-setting/data-source/hooks.ts | 102 +++--
.../user-setting/data-source/interface.ts | 16 +-
web/src/services/data-source-service.ts | 5 +-
web/src/utils/api.ts | 1 -
15 files changed, 707 insertions(+), 558 deletions(-)
diff --git a/api/apps/restful_apis/connector_api.py b/api/apps/restful_apis/connector_api.py
index 21ab7fd4d00..89287a706d0 100644
--- a/api/apps/restful_apis/connector_api.py
+++ b/api/apps/restful_apis/connector_api.py
@@ -53,17 +53,34 @@ async def update_connector(connector_id):
return _connector_auth_error(connector_id, current_user.id)
req = await get_request_json()
+ if isinstance(req, dict) and isinstance(req.get("data"), dict):
+ req = req["data"]
+
e, conn = ConnectorService.get_by_id(connector_id)
if not e:
return get_data_error_result(message="Can't find this Connector!")
+ should_sleep = False
if req:
- conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
- conn["id"] = connector_id
- ConnectorService.update_by_id(connector_id, conn)
-
- await asyncio.sleep(1)
+ update_fields = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
+ if update_fields:
+ update_fields["id"] = connector_id
+ ConnectorService.update_by_id(connector_id, update_fields)
+ should_sleep = True
+
+ if req.get("reschedule"):
+ ConnectorService.cancel_tasks(connector_id)
+ ConnectorService.schedule_tasks(connector_id)
+ elif req.get("status") in [TaskStatus.CANCEL, "CANCEL"]:
+ ConnectorService.cancel_tasks(connector_id)
+ elif req.get("status") in [TaskStatus.SCHEDULE, "SCHEDULE"]:
+ ConnectorService.schedule_tasks(connector_id)
+
+ if should_sleep:
+ await asyncio.sleep(1)
e, conn = ConnectorService.get_by_id(connector_id)
+ if not e:
+ return get_data_error_result(message="Can't find this Connector!")
return get_json_result(data=conn.to_dict())
@@ -83,9 +100,9 @@ async def create_connector():
"input_type": InputType.POLL,
"config": req["config"],
"refresh_freq": int(req.get("refresh_freq", 5)),
- "prune_freq": int(req.get("prune_freq", 720)),
+ "prune_freq": int(req.get("prune_freq", 5)),
"timeout_secs": int(req.get("timeout_secs", 60 * 29)),
- "status": TaskStatus.SCHEDULE,
+ "status": TaskStatus.UNSTART,
}
ConnectorService.save(**conn)
@@ -127,21 +144,6 @@ def list_logs(connector_id):
return get_json_result(data={"total": total, "logs": arr})
-@manager.route("/connectors//resume", methods=["POST"]) # noqa: F821
-@login_required
-async def resume(connector_id):
- """Resume or cancel sync for an accessible connector."""
- if not ConnectorService.accessible(connector_id, current_user.id):
- return _connector_auth_error(connector_id, current_user.id)
-
- req = await get_request_json()
- if req.get("resume"):
- ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
- else:
- ConnectorService.resume(connector_id, TaskStatus.CANCEL)
- return get_json_result(data=True)
-
-
@manager.route("/connectors//rebuild", methods=["POST"]) # noqa: F821
@login_required
async def rebuild(connector_id):
@@ -166,7 +168,7 @@ def rm_connector(connector_id):
if not ConnectorService.accessible(connector_id, current_user.id):
return _connector_auth_error(connector_id, current_user.id)
- ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+ ConnectorService.cancel_tasks(connector_id)
ConnectorService.delete_by_id(connector_id)
return get_json_result(data=True)
diff --git a/api/db/db_models.py b/api/db/db_models.py
index 3ed32ed3f20..a207b00788f 100644
--- a/api/db/db_models.py
+++ b/api/db/db_models.py
@@ -1224,6 +1224,7 @@ def python_value(self, value: str|None) -> datetime|None:
class SyncLogs(DataBaseModel):
id = CharField(max_length=32, primary_key=True)
connector_id = CharField(max_length=32, index=True)
+ task_type = CharField(max_length=32, null=False, default="sync", index=True)
status = CharField(max_length=128, null=False, help_text="Processing status", index=True)
from_beginning = CharField(max_length=1, null=True, help_text="", default="0", index=False)
new_docs_indexed = IntegerField(default=0, index=False)
@@ -1632,6 +1633,7 @@ def migrate_db():
alter_db_add_column(migrator, "llm_factories", "rank", IntegerField(default=0, index=False))
alter_db_add_column(migrator, "api_4_conversation", "name", CharField(max_length=255, null=True, help_text="conversation name", index=False))
alter_db_add_column(migrator, "api_4_conversation", "exp_user_id", CharField(max_length=255, null=True, help_text="exp_user_id", index=True))
+ alter_db_add_column(migrator, "sync_logs", "task_type", CharField(max_length=32, null=False, default="sync", index=True))
# Migrate system_settings.value from CharField to TextField for longer sandbox configs
alter_db_column_type(migrator, "system_settings", "value", TextField(null=False, help_text="Configuration value (JSON, string, etc.)"))
alter_db_add_column(migrator, "document", "content_hash", CharField(max_length=32, null=True, help_text="xxhash128 of document content for change detection", default="", index=True))
diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py
index 4d9fd2258ab..9fa868c6038 100644
--- a/api/db/services/connector_service.py
+++ b/api/db/services/connector_service.py
@@ -28,7 +28,7 @@
from api.db.services.document_service import DocMetadataService
from api.utils.common import hash128
from common.misc_utils import get_uuid
-from common.constants import TaskStatus
+from common.constants import ConnectorTaskType, TaskStatus
from common.settings import TIMEZONE
from common.time_utils import current_timestamp, timestamp_to_date
@@ -38,6 +38,33 @@
class ConnectorService(CommonService):
model = Connector
+ @classmethod
+ def cancel_tasks(cls, connector_id):
+ e, conn = cls.get_by_id(connector_id)
+ if not e:
+ return
+
+ logging.info(
+ "[Connector] stop connector=%s(%s)",
+ conn.name,
+ connector_id,
+ )
+ for c2k in Connector2KbService.query(connector_id=connector_id):
+ SyncLogsService.filter_update(
+ [
+ SyncLogs.connector_id == connector_id,
+ SyncLogs.kb_id == c2k.kb_id,
+ SyncLogs.status.in_([TaskStatus.SCHEDULE, TaskStatus.RUNNING]),
+ ],
+ {"status": TaskStatus.CANCEL},
+ )
+ ConnectorService.update_by_id(connector_id, {"status": TaskStatus.CANCEL})
+ logging.info(
+ "[Connector] connector=%s status updated to %s",
+ connector_id,
+ TaskStatus.CANCEL,
+ )
+
@classmethod
@DB.connection_context()
def accessible(cls, connector_id: str, user_id: str) -> bool:
@@ -64,25 +91,39 @@ def accessible(cls, connector_id: str, user_id: str) -> bool:
return has_access
@classmethod
- def resume(cls, connector_id, status):
+ def schedule_tasks(cls, connector_id):
+ e, conn = cls.get_by_id(connector_id)
+ if not e:
+ return
+
+ logging.info("[Connector] schedule connector=%s(%s)", conn.name, connector_id)
+ prune_enabled = bool((conn.config or {}).get("sync_deleted_files"))
for c2k in Connector2KbService.query(connector_id=connector_id):
- task = SyncLogsService.get_latest_task(connector_id, c2k.kb_id)
- if not task:
- if status == TaskStatus.SCHEDULE:
- SyncLogsService.schedule(connector_id, c2k.kb_id)
- ConnectorService.update_by_id(connector_id, {"status": status})
- return
-
- if task.status == TaskStatus.DONE:
- if status == TaskStatus.SCHEDULE:
- SyncLogsService.schedule(connector_id, c2k.kb_id, task.poll_range_end, total_docs_indexed=task.total_docs_indexed)
- ConnectorService.update_by_id(connector_id, {"status": status})
- return
-
- task = task.to_dict()
- task["status"] = status
- SyncLogsService.update_by_id(task["id"], task)
- ConnectorService.update_by_id(connector_id, {"status": status})
+ sync_task = SyncLogsService.get_latest_task(
+ connector_id,
+ c2k.kb_id,
+ ConnectorTaskType.SYNC,
+ )
+ poll_range_start = None
+ total_docs_indexed = 0
+ if sync_task and sync_task.status == TaskStatus.DONE:
+ poll_range_start = sync_task.poll_range_end
+ total_docs_indexed = sync_task.total_docs_indexed
+
+ SyncLogsService.schedule(
+ connector_id,
+ c2k.kb_id,
+ poll_range_start,
+ total_docs_indexed=total_docs_indexed,
+ task_type=ConnectorTaskType.SYNC,
+ )
+
+ if prune_enabled:
+ SyncLogsService.schedule(
+ connector_id,
+ c2k.kb_id,
+ task_type=ConnectorTaskType.PRUNE,
+ )
@classmethod
def list(cls, tenant_id):
@@ -105,7 +146,9 @@ def rebuild(cls, kb_id:str, connector_id: str, tenant_id:str):
SyncLogsService.filter_delete([SyncLogs.connector_id==connector_id, SyncLogs.kb_id==kb_id])
docs = DocumentService.query(source_type=f"{conn.source}/{conn.id}", kb_id=kb_id)
err = FileService.delete_docs([d.id for d in docs], tenant_id)
- SyncLogsService.schedule(connector_id, kb_id, reindex=True)
+ SyncLogsService.schedule(connector_id, kb_id, reindex=True, task_type=ConnectorTaskType.SYNC)
+ if (conn.config or {}).get("sync_deleted_files"):
+ SyncLogsService.schedule(connector_id, kb_id, task_type=ConnectorTaskType.PRUNE)
return err
@classmethod
@@ -170,30 +213,25 @@ def cleanup_stale_documents_for_task(
class SyncLogsService(CommonService):
model = SyncLogs
+
@classmethod
def list_sync_tasks(cls, connector_id=None, page_number=None, items_per_page=15) -> Tuple[List[dict], int]:
fields = [
cls.model.id,
cls.model.connector_id,
+ cls.model.task_type,
cls.model.kb_id,
cls.model.update_date,
- cls.model.poll_range_start,
- cls.model.poll_range_end,
cls.model.new_docs_indexed,
cls.model.total_docs_indexed,
+ cls.model.docs_removed_from_index,
cls.model.error_msg,
- cls.model.full_exception_trace,
cls.model.error_count,
- Connector.name,
- Connector.source,
- Connector.tenant_id,
- Connector.timeout_secs,
+ cls.model.time_started.alias("time_started"),
+ Connector.refresh_freq.alias("refresh_freq"),
+ Connector.prune_freq.alias("prune_freq"),
Knowledgebase.name.alias("kb_name"),
- Knowledgebase.avatar.alias("kb_avatar"),
- Connector2Kb.auto_parse,
- cls.model.from_beginning.alias("reindex"),
cls.model.status,
- cls.model.update_time
]
if not connector_id:
fields.append(Connector.config)
@@ -225,6 +263,80 @@ def list_sync_tasks(cls, connector_id=None, page_number=None, items_per_page=15)
return list(query.dicts()), total
+ @classmethod
+ def list_due_sync_tasks(cls) -> List[dict]:
+ return cls._list_due_tasks_for_freq(
+ ConnectorTaskType.SYNC,
+ "refresh_freq",
+ )
+
+ @classmethod
+ def list_due_prune_tasks(cls) -> List[dict]:
+ tasks = cls._list_due_tasks_for_freq(
+ ConnectorTaskType.PRUNE,
+ "prune_freq",
+ )
+ return [
+ task for task in tasks
+ # Prune is opt-in at the connector config level; keep the scheduler
+ # blind to prune_freq until the flag is enabled.
+ if bool((task.get("config") or {}).get("sync_deleted_files"))
+ and int(task.get("prune_freq") or 0) > 0
+ ]
+
+ @classmethod
+ def _list_due_tasks_for_freq(cls, task_type: str, freq_field: str) -> List[dict]:
+ fields = [
+ cls.model.id,
+ cls.model.connector_id,
+ cls.model.task_type,
+ cls.model.kb_id,
+ cls.model.update_date,
+ cls.model.poll_range_start,
+ cls.model.poll_range_end,
+ cls.model.new_docs_indexed,
+ cls.model.total_docs_indexed,
+ cls.model.error_msg,
+ cls.model.full_exception_trace,
+ cls.model.error_count,
+ Connector.name,
+ Connector.source,
+ Connector.tenant_id,
+ Connector.timeout_secs,
+ Connector.config,
+ Connector.refresh_freq,
+ Connector.prune_freq,
+ Knowledgebase.name.alias("kb_name"),
+ Knowledgebase.avatar.alias("kb_avatar"),
+ Connector2Kb.auto_parse,
+ cls.model.from_beginning.alias("reindex"),
+ cls.model.status,
+ cls.model.update_time,
+ ]
+
+ query = cls.model.select(*fields)\
+ .join(Connector, on=(cls.model.connector_id==Connector.id))\
+ .join(Connector2Kb, on=(cls.model.kb_id==Connector2Kb.kb_id))\
+ .join(Knowledgebase, on=(cls.model.kb_id==Knowledgebase.id))
+
+ query = query.where(
+ Connector.input_type == InputType.POLL,
+ Connector.status == TaskStatus.SCHEDULE,
+ cls.model.status == TaskStatus.SCHEDULE,
+ cls.model.task_type == task_type,
+ )
+
+ database_type = os.getenv("DB_TYPE", "mysql")
+ if "postgres" in database_type.lower():
+ expr = SQL(
+ f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.{freq_field})"
+ )
+ else:
+ expr = SQL(f"NOW() - INTERVAL `t2`.`{freq_field}` MINUTE")
+ query = query.where(cls.model.update_date < expr)
+
+ return list(query.distinct().order_by(cls.model.update_time.desc()).dicts())
+
@classmethod
def start(cls, id, connector_id):
cls.update_by_id(id, {"status": TaskStatus.RUNNING, "time_started": datetime.now().strftime('%Y-%m-%d %H:%M:%S') })
@@ -236,7 +348,15 @@ def done(cls, id, connector_id):
ConnectorService.update_by_id(connector_id, {"status": TaskStatus.DONE})
@classmethod
- def schedule(cls, connector_id, kb_id, poll_range_start=None, reindex=False, total_docs_indexed=0):
+ def schedule(
+ cls,
+ connector_id,
+ kb_id,
+ poll_range_start=None,
+ reindex=False,
+ total_docs_indexed=0,
+ task_type=ConnectorTaskType.SYNC,
+ ):
try:
if cls.model.select().where(cls.model.kb_id == kb_id, cls.model.connector_id == connector_id).count() > 100:
rm_ids = [m.id for m in cls.model.select(cls.model.id).where(cls.model.kb_id == kb_id, cls.model.connector_id == connector_id).order_by(cls.model.update_time.asc()).limit(70)]
@@ -246,21 +366,33 @@ def schedule(cls, connector_id, kb_id, poll_range_start=None, reindex=False, tot
logging.exception(e)
try:
- e = cls.query(kb_id=kb_id, connector_id=connector_id, status=TaskStatus.SCHEDULE)
+ e = cls.query(
+ kb_id=kb_id,
+ connector_id=connector_id,
+ status=TaskStatus.SCHEDULE,
+ task_type=task_type,
+ )
if e:
- logging.warning(f"{kb_id}--{connector_id} has already had a scheduling sync task which is abnormal.")
+ logging.warning(
+ "%s--%s already has a scheduled %s task.",
+ kb_id,
+ connector_id,
+ task_type,
+ )
return None
reindex = "1" if reindex else "0"
ConnectorService.update_by_id(connector_id, {"status": TaskStatus.SCHEDULE})
return cls.save(**{
"id": get_uuid(),
"kb_id": kb_id, "status": TaskStatus.SCHEDULE, "connector_id": connector_id,
+ "task_type": task_type,
"poll_range_start": poll_range_start, "from_beginning": reindex,
- "total_docs_indexed": total_docs_indexed
+ "total_docs_indexed": total_docs_indexed,
+ "time_started": datetime.now().strftime('%Y-%m-%d %H:%M:%S')
})
except Exception as e:
logging.exception(e)
- task = cls.get_latest_task(connector_id, kb_id)
+ task = cls.get_latest_task(connector_id, kb_id, task_type)
if task:
cls.model.update(status=TaskStatus.SCHEDULE,
poll_range_start=poll_range_start,
@@ -337,11 +469,14 @@ def read(self) -> bytes:
return errs, doc_ids
@classmethod
- def get_latest_task(cls, connector_id, kb_id):
- return cls.model.select().where(
+ def get_latest_task(cls, connector_id, kb_id, task_type=None):
+ query = cls.model.select().where(
cls.model.connector_id==connector_id,
cls.model.kb_id == kb_id
- ).order_by(cls.model.update_time.desc()).first()
+ )
+ if task_type is not None:
+ query = query.where(cls.model.task_type == task_type)
+ return query.order_by(cls.model.update_time.desc()).first()
class Connector2KbService(CommonService):
@@ -364,7 +499,10 @@ def link_connectors(cls, kb_id:str, connectors: list[dict], tenant_id:str):
"kb_id": kb_id,
"auto_parse": conn.get("auto_parse", "1")
})
- SyncLogsService.schedule(conn_id, kb_id, reindex=True)
+ SyncLogsService.schedule(conn_id, kb_id, reindex=True, task_type=ConnectorTaskType.SYNC)
+ e, full_conn = ConnectorService.get_by_id(conn_id)
+ if e and (full_conn.config or {}).get("sync_deleted_files"):
+ SyncLogsService.schedule(conn_id, kb_id, task_type=ConnectorTaskType.PRUNE)
errs = []
for conn_id in old_conn_ids:
diff --git a/common/constants.py b/common/constants.py
index c80735255a0..c76dcdbb099 100644
--- a/common/constants.py
+++ b/common/constants.py
@@ -93,6 +93,11 @@ class TaskStatus(StrEnum):
VALID_TASK_STATUS = {TaskStatus.UNSTART, TaskStatus.RUNNING, TaskStatus.CANCEL, TaskStatus.DONE, TaskStatus.FAIL, TaskStatus.SCHEDULE}
+class ConnectorTaskType(StrEnum):
+ SYNC = "sync"
+ PRUNE = "prune"
+
+
class ParserType(StrEnum):
PRESENTATION = "presentation"
LAWS = "laws"
diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py
index 8d397fc2d67..a5ba3958204 100644
--- a/rag/svr/sync_data_source.py
+++ b/rag/svr/sync_data_source.py
@@ -41,7 +41,7 @@
from api.db.services.document_service import DocumentService
from api.db.services.knowledgebase_service import KnowledgebaseService
from common import settings
-from common.constants import FileSource, TaskStatus
+from common.constants import ConnectorTaskType, FileSource, TaskStatus
from common.config_utils import show_configs
from common.data_source.config import INDEX_BATCH_SIZE
from common.data_source import (
@@ -76,8 +76,6 @@
from common.signal_utils import start_tracemalloc_and_snapshot, stop_tracemalloc
from common.versions import get_ragflow_version
from box_sdk_gen import BoxOAuth, OAuthConfig, AccessToken
-from collections import namedtuple
-
MAX_CONCURRENT_TASKS = int(os.environ.get("MAX_CONCURRENT_TASKS", "5"))
task_limiter = asyncio.Semaphore(MAX_CONCURRENT_TASKS)
@@ -157,30 +155,37 @@ async def __call__(self, task: dict):
})
return
- SyncLogsService.schedule(task["connector_id"], task["kb_id"], task["poll_range_start"])
+ task_type = task.get("task_type", ConnectorTaskType.SYNC)
+ if task_type == ConnectorTaskType.SYNC:
+ SyncLogsService.schedule(
+ task["connector_id"],
+ task["kb_id"],
+ task.get("poll_range_start"),
+ task_type=ConnectorTaskType.SYNC,
+ )
+ elif task_type == ConnectorTaskType.PRUNE and self.conf.get("sync_deleted_files"):
+ SyncLogsService.schedule(
+ task["connector_id"],
+ task["kb_id"],
+ task_type=ConnectorTaskType.PRUNE,
+ )
async def _run_task_logic(self, task: dict):
+ task_type = task.get("task_type", ConnectorTaskType.SYNC)
+ if task_type == ConnectorTaskType.PRUNE:
+ await self._run_prune_task_logic(task)
+ return
+ await self._run_sync_task_logic(task)
+
+ async def _run_sync_task_logic(self, task: dict):
"""
Executes the core synchronization pipeline for a data source task.
-
- This method retrieves documents from the external source via the `_generate` method,
- parses and upserts them into the Knowledge Base (KB), and handles stale document
- reconciliation (sync deletion) if a remote snapshot (`file_list`) is provided.
"""
- generate_output = await self._generate(task)
- # `_generate()` currently supports two outputs:
- # 1. `document_batch_generator`
- # 2. `(document_batch_generator, file_list)`
- if isinstance(generate_output, tuple):
- document_batch_generator, file_list = generate_output
- else:
- document_batch_generator = generate_output
- file_list = None
+ document_batch_generator = await self._generate(task)
failed_docs = 0
added_docs = 0
updated_docs = 0
- removed_docs = 0
next_update = datetime(1970, 1, 1, tzinfo=timezone.utc)
source_type = f"{self.SOURCE_NAME}/{task['connector_id']}"
existing_doc_ids = {
@@ -252,34 +257,12 @@ async def _run_task_logic(self, task: dict):
prefix = self._get_source_prefix()
prefix = f"{prefix} " if prefix else ""
next_update_info = self._format_window_boundary(next_update)
- expects_deleted_file_snapshot = (
- task.get("reindex") != "1"
- and task.get("poll_range_start")
- and self.conf.get("sync_deleted_files")
- )
- cleanup_errors = []
- if expects_deleted_file_snapshot and file_list is None:
- logging.warning(
- "%s deleted-file snapshot retrieval failed "
- "(connector_id=%s, kb_id=%s)",
- self.SOURCE_NAME,
- task["connector_id"],
- task["kb_id"],
- )
- elif file_list is not None:
- removed_docs, cleanup_errors = ConnectorService.cleanup_stale_documents_for_task(
- task["id"],
- task["connector_id"],
- task["kb_id"],
- task["tenant_id"],
- file_list,
- )
- total_changed_docs = added_docs + updated_docs + removed_docs
+ total_changed_docs = added_docs + updated_docs
summary = (
f"{prefix}sync summary till {next_update_info}: "
f"total={total_changed_docs}, added={added_docs}, "
- f"updated={updated_docs}, deleted={removed_docs}"
+ f"updated={updated_docs}"
)
if failed_docs > 0:
summary = f"{summary}, skipped={failed_docs}"
@@ -288,19 +271,80 @@ async def _run_task_logic(self, task: dict):
if (
isinstance(self, _RDBMSBase)
and failed_docs == 0
- and (not expects_deleted_file_snapshot or file_list is not None)
- and not cleanup_errors
):
self.connector.persist_sync_state()
SyncLogsService.done(task["id"], task["connector_id"])
task["poll_range_start"] = next_update
+ async def _run_prune_task_logic(self, task: dict):
+ if not self.conf.get("sync_deleted_files"):
+ SyncLogsService.done(task["id"], task["connector_id"])
+ return
+
+ await self._initialize_for_prune(task)
+
+ file_list = self._collect_prune_snapshot(task)
+ if file_list is None:
+ logging.warning(
+ "%s prune snapshot retrieval failed (connector_id=%s, kb_id=%s)",
+ self.SOURCE_NAME,
+ task["connector_id"],
+ task["kb_id"],
+ )
+ SyncLogsService.done(task["id"], task["connector_id"])
+ return
+
+ removed_docs, cleanup_errors = ConnectorService.cleanup_stale_documents_for_task(
+ task["id"],
+ task["connector_id"],
+ task["kb_id"],
+ task["tenant_id"],
+ file_list,
+ )
+ logging.info(
+ "%s prune summary: deleted=%s, errors=%s",
+ self.SOURCE_NAME,
+ removed_docs,
+ len(cleanup_errors),
+ )
+ SyncLogsService.done(task["id"], task["connector_id"])
+
async def _generate(self, task: dict):
raise NotImplementedError
def _get_source_prefix(self):
return ""
+ async def _initialize_for_prune(self, task: dict):
+ await self._generate(task)
+
+ def _get_prune_snapshot_kwargs(self, task: dict) -> dict[str, Any]:
+ return {}
+
+ def _collect_prune_snapshot(self, task: dict):
+ if not getattr(self, "connector", None):
+ return None
+ if not hasattr(self.connector, "retrieve_all_slim_docs_perm_sync"):
+ return None
+
+ file_list = []
+ snapshot_kwargs = self._get_prune_snapshot_kwargs(task)
+ try:
+ for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync(**snapshot_kwargs):
+ file_list.extend(slim_batch)
+ except TypeError:
+ for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
+ file_list.extend(slim_batch)
+ except Exception:
+ logging.exception(
+ "%s prune snapshot failed (connector_id=%s, kb_id=%s)",
+ self.SOURCE_NAME,
+ task["connector_id"],
+ task["kb_id"],
+ )
+ return None
+ return file_list
+
class _BlobLikeBase(SyncBase):
DEFAULT_BUCKET_TYPE: str = "s3"
@@ -391,7 +435,6 @@ async def _generate(self, task: dict):
self.connector.set_allow_images(self.conf.get("allow_images", False))
self.connector.load_credentials(self.conf["credentials"])
- file_list = None
# Fingerprint-bypass path: skip GetObject for unchanged ETags. Disabled
# on full reindex (we want to re-fetch everything in that case).
use_fingerprint_path = task["reindex"] != "1"
@@ -400,15 +443,6 @@ async def _generate(self, task: dict):
else:
document_batch_generator = self.connector.load_from_state()
- if (
- task["reindex"] != "1"
- and task["poll_range_start"]
- and self.conf.get("sync_deleted_files")
- ):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
-
_begin_info = (
"fingerprint-bypass"
if use_fingerprint_path
@@ -423,7 +457,7 @@ async def _generate(self, task: dict):
_begin_info,
)
)
- return document_batch_generator, file_list
+ return document_batch_generator
class S3(_BlobLikeBase):
@@ -461,28 +495,11 @@ async def _generate(self, task: dict):
return self.connector.load_from_state()
end_time = datetime.now(timezone.utc).timestamp()
- file_list = None
- if self.conf.get("sync_deleted_files"):
- logging.info(
- "[RSS] Syncing deleted files via slim snapshot (connector_id=%s)",
- task["connector_id"],
- )
- snapshot_start = time.perf_counter()
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
- logging.info(
- "[RSS] Slim snapshot fetched %d docs in %.2f seconds",
- len(file_list),
- time.perf_counter() - snapshot_start,
- )
document_generator = self.connector.poll_source(
task["poll_range_start"].timestamp(),
end_time,
)
- if file_list is not None:
- return document_generator, file_list
return document_generator
@@ -525,16 +542,11 @@ async def _generate(self, task: dict):
credential_json=self.conf["credentials"])
self.connector.set_credentials_provider(credentials_provider)
- file_list = None
# Determine the time range for synchronization based on reindex or poll_range_start
if task["reindex"] == "1" or not task["poll_range_start"]:
start_time = 0.0
else:
start_time = task["poll_range_start"].timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
end_time = datetime.now(timezone.utc).timestamp()
@@ -580,7 +592,7 @@ def wrapper():
yield batch
self.log_connection("Confluence", self.conf["wiki_base"], task)
- return wrapper(), file_list
+ return wrapper()
class Notion(SyncBase):
@@ -589,7 +601,6 @@ class Notion(SyncBase):
async def _generate(self, task: dict):
self.connector = NotionConnector(root_page_id=self.conf["root_page_id"])
self.connector.load_credentials(self.conf["credentials"])
- file_list = None
document_generator = (
self.connector.load_from_state()
if task["reindex"] == "1" or not task["poll_range_start"]
@@ -597,19 +608,10 @@ async def _generate(self, task: dict):
datetime.now(timezone.utc).timestamp())
)
- if (
- task["reindex"] != "1"
- and task["poll_range_start"]
- and self.conf.get("sync_deleted_files")
- ):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
-
_begin_info = "totally" if task["reindex"] == "1" or not task["poll_range_start"] else "from {}".format(
task["poll_range_start"])
self.log_connection("Notion", f"root({self.conf['root_page_id']})", task)
- return document_generator, file_list
+ return document_generator
class Discord(SyncBase):
@@ -627,26 +629,17 @@ async def _generate(self, task: dict):
batch_size=self.conf.get("batch_size", 1024),
)
self.connector.load_credentials(self.conf["credentials"])
- file_list = None
document_generator = (
self.connector.load_from_state()
if task["reindex"] == "1" or not task["poll_range_start"]
else self.connector.poll_source(task["poll_range_start"].timestamp(),
datetime.now(timezone.utc).timestamp())
)
- if (
- task["reindex"] != "1"
- and task["poll_range_start"]
- and self.conf.get("sync_deleted_files")
- ):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
_begin_info = "totally" if task["reindex"] == "1" or not task["poll_range_start"] else "from {}".format(
task["poll_range_start"])
self.log_connection("Discord", f"servers({server_ids}), channel({channel_names})", task)
- return document_generator, file_list
+ return document_generator
class Gmail(SyncBase):
@@ -685,8 +678,6 @@ async def _generate(self, task: dict):
task["connector_id"],
)
- file_list = None
-
# Decide between full reindex and incremental polling by time range.
if task["reindex"] == "1" or not task.get("poll_range_start"):
start_time = None
@@ -706,17 +697,13 @@ async def _generate(self, task: dict):
end_time = datetime.now(timezone.utc).timestamp()
_begin_info = f"from {poll_start}"
document_generator = self.connector.poll_source(start_time, end_time)
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
try:
admin_email = self.connector.primary_admin_email
except RuntimeError:
admin_email = "unknown"
self.log_connection("Gmail", f"as {admin_email}", task)
- return document_generator, file_list
+ return document_generator
class Dropbox(SyncBase):
@@ -726,22 +713,16 @@ async def _generate(self, task: dict):
self.connector = DropboxConnector(batch_size=self.conf.get("batch_size", INDEX_BATCH_SIZE))
self.connector.load_credentials(self.conf["credentials"])
poll_start = task["poll_range_start"]
- file_list = None
-
if task["reindex"] == "1" or not poll_start:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
end_time = datetime.now(timezone.utc).timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
document_generator = self.connector.poll_source(poll_start.timestamp(), end_time)
_begin_info = f"from {poll_start}"
self.log_connection("Dropbox", "workspace", task)
- return document_generator, file_list
+ return document_generator
class GoogleDrive(SyncBase):
@@ -775,8 +756,6 @@ async def _generate(self, task: dict):
if new_credentials:
self._persist_rotated_credentials(task["connector_id"], new_credentials)
- file_list = None
-
# Capture end_time BEFORE the snapshot to prevent the ingestion race condition
end_time = datetime.now(timezone.utc).timestamp()
@@ -786,18 +765,6 @@ async def _generate(self, task: dict):
else:
start_time = task["poll_range_start"].timestamp()
_begin_info = f"from {task['poll_range_start']}"
-
- if self.conf.get("sync_deleted_files"):
- file_list = []
- SlimDoc = namedtuple('SlimDoc', ['id'])
-
- # Add observability timing so operators can track the O(N) cost
- snapshot_start = time.perf_counter()
-
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(SlimDoc(doc.id) for doc in slim_batch)
-
- logging.info("Slim snapshot fetched %d files in %.2f seconds", len(file_list), time.perf_counter() - snapshot_start)
raw_batch_size = self.conf.get("sync_batch_size") or self.conf.get("batch_size") or INDEX_BATCH_SIZE
try:
@@ -843,7 +810,7 @@ def document_batches():
admin_email = "unknown"
self.log_connection("Google Drive", f"as {admin_email}", task)
- return document_batches(), file_list
+ return document_batches()
def _persist_rotated_credentials(self, connector_id: str, credentials: dict[str, Any]) -> None:
"""Saves refreshed OAuth credentials back to the database configuration."""
@@ -886,17 +853,12 @@ async def _generate(self, task: dict):
self.connector.load_credentials(credentials)
self.connector.validate_connector_settings()
- file_list = None
if task["reindex"] == "1" or not task["poll_range_start"]:
start_time = 0.0
_begin_info = "totally"
else:
start_time = task["poll_range_start"].timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
_begin_info = f"from {task['poll_range_start']}"
end_time = datetime.now(timezone.utc).timestamp()
@@ -955,7 +917,7 @@ def document_batches():
f"overlap_buffer_s={getattr(self.connector, 'time_buffer_seconds', connector_kwargs.get('time_buffer_seconds'))}"
),
)
- return document_batches(), file_list
+ return document_batches()
@staticmethod
def _normalize_list(values: Any) -> list[str] | None:
@@ -1007,25 +969,11 @@ async def _generate(self, task: dict):
self.connector.set_allow_images(self.conf.get("allow_images", False))
self.connector.load_credentials(self.conf["credentials"])
- file_list = None
if task["reindex"] == "1" or not task["poll_range_start"]:
document_batch_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
end_ts = datetime.now(timezone.utc).timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- try:
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
- except Exception:
- logging.exception(
- "WebDAV slim snapshot failed; continuing without stale-document cleanup "
- "(connector_id=%s, kb_id=%s)",
- task["connector_id"],
- task["kb_id"],
- )
- file_list = None
document_batch_generator = self.connector.poll_source(
task["poll_range_start"].timestamp(),
end_ts,
@@ -1038,7 +986,7 @@ def wrapper():
for document_batch in document_batch_generator:
yield document_batch
- return wrapper(), file_list
+ return wrapper()
class Moodle(SyncBase):
@@ -1054,7 +1002,6 @@ async def _generate(self, task: dict):
# Determine the time range for synchronization based on reindex or poll_range_start
poll_start = task.get("poll_range_start")
- file_list = None
if task["reindex"] == "1" or poll_start is None:
document_generator = self.connector.load_from_state()
@@ -1066,20 +1013,6 @@ async def _generate(self, task: dict):
# could be polled as new and at the same time be missing from
# the slim list, which would mark it as stale and delete it.
end_ts = datetime.now(timezone.utc).timestamp()
-
- if self.conf.get("sync_deleted_files"):
- file_list = []
- try:
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
- except Exception:
- logging.exception(
- "Moodle slim snapshot failed; skipping stale-document cleanup "
- "(connector_id=%s, kb_id=%s)",
- task.get("connector_id"),
- task.get("kb_id"),
- )
- file_list = None
document_generator = self.connector.poll_source(
poll_start.timestamp(),
end_ts,
@@ -1087,7 +1020,7 @@ async def _generate(self, task: dict):
_begin_info = f"from {poll_start}"
self.log_connection("Moodle", self.conf["moodle_url"], task)
- return document_generator, file_list
+ return document_generator
class BOX(SyncBase):
@@ -1115,23 +1048,18 @@ async def _generate(self, task: dict):
self.connector.load_credentials(auth)
poll_start = task["poll_range_start"]
- file_list = None
if task["reindex"] == "1" or poll_start is None:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
document_generator = self.connector.poll_source(
poll_start.timestamp(),
datetime.now(timezone.utc).timestamp(),
)
_begin_info = f"from {poll_start}"
self.log_connection("Box", f"folder_id({self.conf['folder_id']})", task)
- return document_generator, file_list
+ return document_generator
class Airtable(SyncBase):
@@ -1156,16 +1084,11 @@ async def _generate(self, task: dict):
)
poll_start = task.get("poll_range_start")
- file_list = None
if task.get("reindex") == "1" or poll_start is None:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
document_generator = self.connector.poll_source(
poll_start.timestamp(),
datetime.now(timezone.utc).timestamp(),
@@ -1178,7 +1101,7 @@ async def _generate(self, task: dict):
task,
)
- return document_generator, file_list
+ return document_generator
class Asana(SyncBase):
SOURCE_NAME: str = FileSource.ASANA
@@ -1198,17 +1121,12 @@ async def _generate(self, task: dict):
)
poll_start = task.get("poll_range_start")
- file_list = None
if task.get("reindex") == "1" or not poll_start:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
end_time = datetime.now(timezone.utc).timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
document_generator = self.connector.poll_source(
poll_start.timestamp(),
end_time,
@@ -1221,7 +1139,7 @@ async def _generate(self, task: dict):
task,
)
- return document_generator, file_list
+ return document_generator
class Github(SyncBase):
SOURCE_NAME: str = FileSource.GITHUB
@@ -1247,15 +1165,10 @@ async def _generate(self, task: dict):
{"github_access_token": credentials["github_access_token"]}
)
- file_list = None
if task.get("reindex") == "1" or not task.get("poll_range_start"):
start_time = datetime.fromtimestamp(0, tz=timezone.utc)
else:
start_time = task.get("poll_range_start")
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
end_time = datetime.now(timezone.utc)
@@ -1292,7 +1205,7 @@ def wrapper():
task,
)
- return wrapper(), file_list
+ return wrapper()
class IMAP(SyncBase):
SOURCE_NAME: str = FileSource.IMAP
@@ -1348,27 +1261,10 @@ async def _generate(self, task):
task["connector_id"],
)
- file_list = None
- if (
- task["reindex"] != "1"
- and task["poll_range_start"]
- and self.conf.get("sync_deleted_files")
- ):
- file_list = []
- try:
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync(
- start=initial_sync_start,
- end=end_time,
- ):
- file_list.extend(slim_batch)
- except Exception:
- logging.exception(
- "IMAP slim snapshot failed; continuing without stale-document cleanup "
- "(connector_id=%s, kb_id=%s)",
- task["connector_id"],
- task["kb_id"],
- )
- file_list = None
+ self._prune_snapshot_kwargs = {
+ "start": initial_sync_start,
+ "end": end_time,
+ }
raw_batch_size = self.conf.get("sync_batch_size") or self.conf.get("batch_size") or INDEX_BATCH_SIZE
try:
@@ -1414,7 +1310,10 @@ def wrapper():
f"host({self.conf['imap_host']}) port({self.conf['imap_port']}) user({self.conf['credentials']['imap_username']}) folder({self.conf['imap_mailbox']})",
task,
)
- return wrapper(), file_list
+ return wrapper()
+
+ def _get_prune_snapshot_kwargs(self, task: dict) -> dict[str, Any]:
+ return getattr(self, "_prune_snapshot_kwargs", {})
class Zendesk(SyncBase):
@@ -1424,26 +1323,11 @@ async def _generate(self, task: dict):
self.connector.load_credentials(self.conf["credentials"])
end_time = datetime.now(timezone.utc).timestamp()
- file_list = None
if task["reindex"] == "1" or not task.get("poll_range_start"):
start_time = 0
_begin_info = "totally"
else:
start_time = task["poll_range_start"].timestamp()
- if self.conf.get("sync_deleted_files"):
- logging.info(
- "[Zendesk] Syncing deleted files via slim snapshot (connector_id=%s)",
- task.get("connector_id"),
- )
- snapshot_start = time.perf_counter()
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
- logging.info(
- "[Zendesk] Slim snapshot fetched %d docs in %.2f seconds",
- len(file_list),
- time.perf_counter() - snapshot_start,
- )
_begin_info = f"from {task['poll_range_start']}"
raw_batch_size = (
@@ -1504,9 +1388,6 @@ def wrapper():
yield batch
self.log_connection("Zendesk", f"subdomain({self.conf['credentials'].get('zendesk_subdomain')})", task)
-
- if file_list is not None:
- return wrapper(), file_list
return wrapper()
@@ -1533,7 +1414,6 @@ async def _generate(self, task: dict):
}
)
- file_list = None
if task["reindex"] == "1" or not task["poll_range_start"]:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
@@ -1547,13 +1427,9 @@ async def _generate(self, task: dict):
poll_start.timestamp(),
datetime.now(timezone.utc).timestamp()
)
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
_begin_info = "from {}".format(poll_start)
self.log_connection("Gitlab", f"({self.conf['project_name']})", task)
- return document_generator, file_list
+ return document_generator
class Bitbucket(SyncBase):
@@ -1572,17 +1448,12 @@ async def _generate(self, task: dict):
"bitbucket_api_token": self.conf["credentials"].get("bitbucket_api_token"),
}
)
- file_list = None
if task["reindex"] == "1" or not task["poll_range_start"]:
start_time = datetime.fromtimestamp(0, tz=timezone.utc)
_begin_info = "totally"
else:
start_time = task.get("poll_range_start")
- if self.conf.get("sync_deleted_files"):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
_begin_info = f"from {start_time}"
end_time = datetime.now(timezone.utc)
@@ -1614,8 +1485,6 @@ def wrapper():
yield batch
self.log_connection("Bitbucket", f"workspace({self.conf.get('workspace')})", task)
- if file_list is not None:
- return wrapper(), file_list
return wrapper()
@@ -1642,26 +1511,12 @@ async def _generate(self, task: dict):
)
self.connector.load_credentials(conf["credentials"])
- file_list = None
poll_start = task.get("poll_range_start")
if task["reindex"] == "1" or poll_start is None:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
end_ts = datetime.now(timezone.utc).timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- try:
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
- except Exception:
- logging.exception(
- "SeaFile slim snapshot failed; continuing without stale-document cleanup "
- "(connector_id=%s, kb_id=%s)",
- task["connector_id"],
- task["kb_id"],
- )
- file_list = None
document_generator = self.connector.poll_source(
poll_start.timestamp(),
end_ts,
@@ -1676,7 +1531,7 @@ async def _generate(self, task: dict):
extra += f" path={conf.get('sync_path')}"
self.log_connection("SeaFile", f"{conf['seafile_url']} (scope={scope}{extra})", task)
- return document_generator, file_list
+ return document_generator
class DingTalkAITable(SyncBase):
@@ -1709,33 +1564,12 @@ async def _generate(self, task: dict):
)
poll_start = task.get("poll_range_start")
- file_list = None
if task.get("reindex") == "1" or poll_start is None:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
else:
end_ts = datetime.now(timezone.utc).timestamp()
- if self.conf.get("sync_deleted_files"):
- file_list = []
- logging.info(
- "DingTalk AI Table: fetching slim snapshot for stale-document reconciliation "
- "(connector_id=%s, kb_id=%s, table_id=%s)",
- task["connector_id"],
- task["kb_id"],
- self.conf.get("table_id"),
- )
- try:
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
- except Exception:
- logging.exception(
- "DingTalk AI Table slim snapshot failed; continuing without stale-document cleanup "
- "(connector_id=%s, kb_id=%s)",
- task["connector_id"],
- task["kb_id"],
- )
- file_list = None
document_generator = self.connector.poll_source(
poll_start.timestamp(),
end_ts,
@@ -1748,7 +1582,7 @@ async def _generate(self, task: dict):
task,
)
- return document_generator, file_list
+ return document_generator
class _RDBMSBase(SyncBase):
@@ -1778,16 +1612,6 @@ async def _generate(self, task: dict):
self.connector.validate_connector_settings()
self.connector.prepare_sync_state(task["connector_id"], self.conf)
- file_list = None
- if (
- task["reindex"] != "1"
- and task["poll_range_start"]
- and self.conf.get("sync_deleted_files")
- ):
- file_list = []
- for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync():
- file_list.extend(slim_batch)
-
if task["reindex"] == "1" or not task["poll_range_start"]:
document_generator = self.connector.load_from_state()
_begin_info = "totally"
@@ -1804,7 +1628,7 @@ async def _generate(self, task: dict):
_begin_info = f"from {poll_start}"
self.log_connection(self.LOG_NAME, f"{self.conf.get('host')}:{self.conf.get('database')}", task)
- return document_generator, file_list
+ return document_generator
class MySQL(_RDBMSBase):
@@ -1886,14 +1710,17 @@ async def dispatch_tasks():
"""Polls the database for pending synchronization tasks and dispatches them concurrently."""
while True:
try:
- list(SyncLogsService.list_sync_tasks()[0])
+ SyncLogsService.list_due_sync_tasks()
+ SyncLogsService.list_due_prune_tasks()
break
except Exception as e:
logging.warning(f"DB is not ready yet: {e}")
await asyncio.sleep(3)
+ due_sync_tasks = SyncLogsService.list_due_sync_tasks()
+ due_prune_tasks = SyncLogsService.list_due_prune_tasks()
tasks = []
- for task in SyncLogsService.list_sync_tasks()[0]:
+ for task in [*due_sync_tasks, *due_prune_tasks]:
if task["poll_range_start"]:
task["poll_range_start"] = task["poll_range_start"].astimezone(timezone.utc)
if task["poll_range_end"]:
diff --git a/test/testcases/restful_api/test_connector_routes_unit.py b/test/testcases/restful_api/test_connector_routes_unit.py
index 33c4d7a8f12..80cd5662a6c 100644
--- a/test/testcases/restful_api/test_connector_routes_unit.py
+++ b/test/testcases/restful_api/test_connector_routes_unit.py
@@ -205,7 +205,7 @@ def accessible(*_args, **_kwargs):
return True
@staticmethod
- def resume(*_args, **_kwargs):
+ def cancel_tasks(*_args, **_kwargs):
return True
@staticmethod
@@ -252,7 +252,11 @@ async def _get_request_json():
PERMISSION_ERROR=403,
AUTHENTICATION_ERROR=109,
)
- constants_mod.TaskStatus = SimpleNamespace(SCHEDULE="schedule", CANCEL="cancel")
+ constants_mod.TaskStatus = SimpleNamespace(
+ UNSTART="unstart",
+ SCHEDULE="schedule",
+ CANCEL="cancel",
+ )
monkeypatch.setitem(sys.modules, "common.constants", constants_mod)
config_mod = ModuleType("common.data_source.config")
@@ -349,7 +353,7 @@ async def _no_sleep(_secs):
records = {"conn-1": _FakeConnectorRecord({"id": "conn-1", "source": "drive"})}
update_calls = []
save_calls = []
- resume_calls = []
+ cancel_calls = []
delete_calls = []
monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda cid, payload: update_calls.append((cid, payload)))
@@ -362,7 +366,7 @@ def _save(**payload):
monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda cid: (True, records[cid]))
monkeypatch.setattr(module.ConnectorService, "list", lambda tenant_id: [{"id": "listed", "tenant": tenant_id}])
monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda cid, page, page_size: ([{"id": "log-1"}], 9))
- monkeypatch.setattr(module.ConnectorService, "resume", lambda cid, status: resume_calls.append((cid, status)))
+ monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda cid: cancel_calls.append(cid))
monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda cid: delete_calls.append(cid))
monkeypatch.setattr(module, "get_uuid", lambda: "generated-id")
@@ -384,6 +388,7 @@ def _save(**payload):
assert save_calls[-1]["id"] == "generated-id"
assert save_calls[-1]["tenant_id"] == "tenant-1"
assert save_calls[-1]["input_type"] == module.InputType.POLL
+ assert save_calls[-1]["status"] == module.TaskStatus.UNSTART
assert res["data"]["id"] == "generated-id"
list_res = module.list_connector()
@@ -401,14 +406,6 @@ def _save(**payload):
logs_res = module.list_logs("conn-log")
assert logs_res["data"] == {"total": 9, "logs": [{"id": "log-1"}]}
- monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": True}))
- assert _run(module.resume("conn-r1"))["data"] is True
-
- monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": False}))
- assert _run(module.resume("conn-r2"))["data"] is True
- assert ("conn-r1", module.TaskStatus.SCHEDULE) in resume_calls
- assert ("conn-r2", module.TaskStatus.CANCEL) in resume_calls
-
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"kb_id": "kb-1"}))
monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: "rebuild-failed")
failed_rebuild = _run(module.rebuild("conn-rb"))
@@ -421,7 +418,7 @@ def _save(**payload):
rm_res = module.rm_connector("conn-rm")
assert rm_res["data"] is True
- assert ("conn-rm", module.TaskStatus.CANCEL) in resume_calls
+ assert cancel_calls == ["conn-rm"]
assert delete_calls == ["conn-rm"]
@@ -434,14 +431,14 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
monkeypatch.setattr(module.ConnectorService, "accessible", lambda cid, uid: False)
monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda *_args: touched.append("get_by_id"))
monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda *_args: touched.append("list_sync_tasks"))
- monkeypatch.setattr(module.ConnectorService, "resume", lambda *_args: touched.append("resume"))
+ monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda *_args: touched.append("cancel_tasks"))
monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda *_args: touched.append("delete_by_id"))
monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda *_args: touched.append("update_by_id"))
monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: touched.append("rebuild"))
def _get_request_json():
touched.append("get_request_json")
- return _AwaitableValue({"resume": True, "config": {"x": 1}})
+ return _AwaitableValue({"config": {"x": 1}})
monkeypatch.setattr(module, "get_request_json", _get_request_json)
@@ -449,7 +446,6 @@ def _get_request_json():
_run(module.update_connector("conn-victim")),
module.get_connector("conn-victim"),
module.list_logs("conn-victim"),
- _run(module.resume("conn-victim")),
_run(module.rebuild("conn-victim")),
module.rm_connector("conn-victim"),
_run(module.test_connector("conn-victim")),
diff --git a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
index 3807fb8e15c..605ec415f15 100644
--- a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
+++ b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
@@ -205,7 +205,7 @@ def accessible(*_args, **_kwargs):
return True
@staticmethod
- def resume(*_args, **_kwargs):
+ def cancel_tasks(*_args, **_kwargs):
return True
@staticmethod
@@ -349,7 +349,7 @@ async def _no_sleep(_secs):
records = {"conn-1": _FakeConnectorRecord({"id": "conn-1", "source": "drive"})}
update_calls = []
save_calls = []
- resume_calls = []
+ cancel_calls = []
delete_calls = []
monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda cid, payload: update_calls.append((cid, payload)))
@@ -362,7 +362,7 @@ def _save(**payload):
monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda cid: (True, records[cid]))
monkeypatch.setattr(module.ConnectorService, "list", lambda tenant_id: [{"id": "listed", "tenant": tenant_id}])
monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda cid, page, page_size: ([{"id": "log-1"}], 9))
- monkeypatch.setattr(module.ConnectorService, "resume", lambda cid, status: resume_calls.append((cid, status)))
+ monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda cid: cancel_calls.append(cid))
monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda cid: delete_calls.append(cid))
monkeypatch.setattr(module, "get_uuid", lambda: "generated-id")
@@ -401,14 +401,6 @@ def _save(**payload):
logs_res = module.list_logs("conn-log")
assert logs_res["data"] == {"total": 9, "logs": [{"id": "log-1"}]}
- monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": True}))
- assert _run(module.resume("conn-r1"))["data"] is True
-
- monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": False}))
- assert _run(module.resume("conn-r2"))["data"] is True
- assert ("conn-r1", module.TaskStatus.SCHEDULE) in resume_calls
- assert ("conn-r2", module.TaskStatus.CANCEL) in resume_calls
-
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"kb_id": "kb-1"}))
monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: "rebuild-failed")
failed_rebuild = _run(module.rebuild("conn-rb"))
@@ -421,7 +413,7 @@ def _save(**payload):
rm_res = module.rm_connector("conn-rm")
assert rm_res["data"] is True
- assert ("conn-rm", module.TaskStatus.CANCEL) in resume_calls
+ assert cancel_calls == ["conn-rm"]
assert delete_calls == ["conn-rm"]
@@ -434,14 +426,14 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
monkeypatch.setattr(module.ConnectorService, "accessible", lambda cid, uid: False)
monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda *_args: touched.append("get_by_id"))
monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda *_args: touched.append("list_sync_tasks"))
- monkeypatch.setattr(module.ConnectorService, "resume", lambda *_args: touched.append("resume"))
+ monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda *_args: touched.append("cancel_tasks"))
monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda *_args: touched.append("delete_by_id"))
monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda *_args: touched.append("update_by_id"))
monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: touched.append("rebuild"))
def _get_request_json():
touched.append("get_request_json")
- return _AwaitableValue({"resume": True, "config": {"x": 1}})
+ return _AwaitableValue({"config": {"x": 1}})
monkeypatch.setattr(module, "get_request_json", _get_request_json)
@@ -449,7 +441,6 @@ def _get_request_json():
_run(module.update_connector("conn-victim")),
module.get_connector("conn-victim"),
module.list_logs("conn-victim"),
- _run(module.resume("conn-victim")),
_run(module.rebuild("conn-victim")),
module.rm_connector("conn-victim"),
_run(module.test_connector("conn-victim")),
diff --git a/test/unit_test/rag/test_sync_data_source.py b/test/unit_test/rag/test_sync_data_source.py
index be9d89372a3..8bb5e4cd437 100644
--- a/test/unit_test/rag/test_sync_data_source.py
+++ b/test/unit_test/rag/test_sync_data_source.py
@@ -133,7 +133,53 @@ def _patch_common_dependencies(monkeypatch):
@pytest.mark.anyio
@pytest.mark.p2
-async def test_run_task_logic_cleans_up_for_empty_snapshot(monkeypatch):
+async def test_run_task_logic_skips_empty_sync_batches(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "increase_docs",
+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not be called for empty batches"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: pytest.fail("get_by_id should not be called for empty batches"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "duplicate_and_parse",
+ lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not be called for empty batches"),
+ )
+
+ await _FakeSync(iter(([],)))._run_task_logic(_make_task())
+
+
+@pytest.mark.anyio
+@pytest.mark.p2
+async def test_run_task_logic_skips_multiple_empty_sync_batches(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "increase_docs",
+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not be called for empty batches"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: pytest.fail("get_by_id should not be called for empty batches"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "duplicate_and_parse",
+ lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not be called for empty batches"),
+ )
+
+ await _FakeSync(iter(([], [],)))._run_task_logic(_make_task())
+
+
+@pytest.mark.anyio
+@pytest.mark.p2
+async def test_run_prune_task_logic_cleans_up_for_empty_snapshot(monkeypatch):
cleanup_calls = []
_patch_common_dependencies(monkeypatch)
@@ -148,7 +194,14 @@ def _fake_cleanup(*args, **kwargs):
_fake_cleanup,
)
- await _FakeSync((iter(()), []))._run_task_logic(_make_task())
+ task = {**_make_task(), "task_type": sync_data_source.ConnectorTaskType.PRUNE}
+ sync = _FakeSync(iter(()))
+ sync.conf["sync_deleted_files"] = True
+ sync.connector = types.SimpleNamespace(
+ retrieve_all_slim_docs_perm_sync=lambda: iter(([],))
+ )
+
+ await sync._run_task_logic(task)
assert cleanup_calls == [
(
@@ -166,7 +219,7 @@ def _fake_cleanup(*args, **kwargs):
@pytest.mark.anyio
@pytest.mark.p2
-async def test_run_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
+async def test_run_prune_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
cleanup_calls = []
_patch_common_dependencies(monkeypatch)
@@ -182,7 +235,14 @@ def _fake_cleanup(*args, **kwargs):
)
file_list = [types.SimpleNamespace(id="doc-1")]
- await _FakeSync((iter(()), file_list))._run_task_logic(_make_task())
+ task = {**_make_task(), "task_type": sync_data_source.ConnectorTaskType.PRUNE}
+ sync = _FakeSync(iter(()))
+ sync.conf["sync_deleted_files"] = True
+ sync.connector = types.SimpleNamespace(
+ retrieve_all_slim_docs_perm_sync=lambda: iter((file_list,))
+ )
+
+ await sync._run_task_logic(task)
assert cleanup_calls == [
(
@@ -285,12 +345,13 @@ async def test_rdbms_generate_keeps_deleted_file_snapshot_without_timestamp_colu
}
)
- document_generator, file_list = await sync._generate(task)
+ document_generator = await sync._generate(task)
connector = _FakeRDBMSConnector.instance
assert connector is not None
assert connector.load_from_state_called is True
assert connector.load_from_cursor_range_called is False
+ file_list = sync._collect_prune_snapshot(task)
assert connector.retrieve_all_slim_docs_perm_sync_called is True
assert file_list is not None
assert [doc.id for doc in file_list] == ["row-1"]
@@ -447,14 +508,15 @@ async def test_dropbox_generate_returns_snapshot_when_sync_deleted_enabled(monke
}
)
- document_generator, file_list = await sync._generate(task)
+ document_generator = await sync._generate(task)
connector = _FakeDropboxConnector.instance
assert list(document_generator) == [["poll-sync"]]
+ file_list = sync._collect_prune_snapshot(task)
assert [doc.id for doc in file_list] == ["dropbox:id-1", "dropbox:id-2"]
assert connector.credentials == {"dropbox_access_token": "token-1"}
assert connector.retrieve_all_slim_docs_perm_sync_called is True
- assert connector.snapshot_called_before_poll is True
+ assert connector.snapshot_called_before_poll is False
assert connector.poll_source_call[0] == poll_start.timestamp()
assert connector.poll_source_call[1] >= poll_start.timestamp()
@@ -477,11 +539,12 @@ async def test_dropbox_generate_skips_snapshot_for_full_reindex(monkeypatch):
}
)
- document_generator, file_list = await sync._generate(task)
+ document_generator = await sync._generate(task)
connector = _FakeDropboxConnector.instance
assert list(document_generator) == [["full-sync"]]
- assert file_list is None
assert connector.load_from_state_called is True
- assert connector.retrieve_all_slim_docs_perm_sync_called is False
+ file_list = sync._collect_prune_snapshot(task)
+ assert [doc.id for doc in file_list] == ["dropbox:id-1", "dropbox:id-2"]
+ assert connector.retrieve_all_slim_docs_perm_sync_called is True
assert connector.poll_source_called is False
diff --git a/web/src/components/dynamic-form.tsx b/web/src/components/dynamic-form.tsx
index 0ef13df1c13..0920e2422ef 100644
--- a/web/src/components/dynamic-form.tsx
+++ b/web/src/components/dynamic-form.tsx
@@ -111,10 +111,12 @@ interface DynamicFormProps {
// Form ref interface
export interface DynamicFormRef {
submit: () => void;
+ isDirty: () => boolean;
getValues: (name?: string) => any;
reset: (values?: any) => void;
trigger: UseFormTrigger;
watch: (field: string, callback: (value: any) => void) => () => void;
+ watchDirty: (callback: (isDirty: boolean, values: any) => void) => () => void;
updateFieldType: (fieldName: string, newType: FormFieldType) => void;
onFieldUpdate: (
fieldName: string,
@@ -809,6 +811,7 @@ const DynamicForm = {
onSubmit(filteredValues);
})();
},
+ isDirty: () => form.formState.isDirty,
getValues: form.getValues,
reset: (values?: T) => {
if (values) {
@@ -828,6 +831,12 @@ const DynamicForm = {
});
return unsubscribe;
},
+ watchDirty: (callback: (isDirty: boolean, values: any) => void) => {
+ const { unsubscribe } = form.watch((values: any) => {
+ callback(form.formState.isDirty, values);
+ });
+ return unsubscribe;
+ },
onFieldUpdate: (
fieldName: string,
diff --git a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
index dfeb7e0830e..a55c2af8eec 100644
--- a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
+++ b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
@@ -9,9 +9,9 @@ import { Button } from '@/components/ui/button';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { Input } from '@/components/ui/input';
import { Separator } from '@/components/ui/separator';
-import { RunningStatus } from '@/constants/knowledge';
+import { RunningStatus, RunningStatusOld } from '@/constants/knowledge';
import { t } from 'i18next';
-import { CirclePause, Repeat } from 'lucide-react';
+import { isEqual } from 'lodash';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { FieldValues } from 'react-hook-form';
import {
@@ -25,9 +25,9 @@ import {
} from '../constant';
import {
useAddDataSource,
- useDataSourceResume,
useFetchDataSourceDetail,
useTestDataSource,
+ useUpdateDataSourceStatus,
} from '../hooks';
import { DataSourceLogsTable } from './log-table';
@@ -35,7 +35,8 @@ const SourceDetailPage = () => {
const formRef = useRef(null);
const { data: detail } = useFetchDataSourceDetail();
- const { handleResume } = useDataSourceResume();
+ const { updateStatus, loading: statusUpdateLoading } =
+ useUpdateDataSourceStatus();
const { dataSourceInfo } = useDataSourceInfo();
const detailInfo = useMemo(() => {
if (detail) {
@@ -44,83 +45,52 @@ const SourceDetailPage = () => {
}, [detail, dataSourceInfo]);
const [fields, setFields] = useState([]);
+ const [isDirty, setIsDirty] = useState(false);
const [defaultValues, setDefaultValues] = useState(
DataSourceFormDefaultValues[
detail?.source as keyof typeof DataSourceFormDefaultValues
] as FieldValues,
);
- const runSchedule = useCallback(() => {
- handleResume({
- resume:
- detail?.status === RunningStatus.RUNNING ||
- detail?.status === RunningStatus.SCHEDULE
- ? false
- : true,
- });
- }, [detail, handleResume]);
-
const customFields = useMemo(() => {
return [
- {
- label: 'Refresh Freq',
- name: 'refresh_freq',
- type: FormFieldType.Number,
- required: false,
- render: (fieldProps: FormFieldConfig) => (
-
-
-
- {t('setting.minutes')}
-
- }
- />
-
-
{
- runSchedule();
- }}
- >
- {detail?.status === RunningStatus.RUNNING ||
- detail?.status === RunningStatus.SCHEDULE ? (
-
- ) : (
-
- )}
-
-
- ),
- },
{
label: 'Prune Freq',
name: 'prune_freq',
type: FormFieldType.Number,
required: false,
- hidden: true,
+ shouldRender: (values: any) => !!values?.config?.sync_deleted_files,
render: (fieldProps: FormFieldConfig) => {
return (
-
-
-
- hours
-
- }
- />
-
-
+
+ {t('setting.minutes')}
+
+ }
+ />
);
},
},
+ {
+ label: 'Refresh Freq',
+ name: 'refresh_freq',
+ type: FormFieldType.Number,
+ required: false,
+ render: (fieldProps: FormFieldConfig) => (
+
+ {t('setting.minutes')}
+
+ }
+ />
+ ),
+ },
{
label: 'Timeout Secs',
name: 'timeout_secs',
@@ -143,7 +113,7 @@ const SourceDetailPage = () => {
),
},
];
- }, [detail, runSchedule]);
+ }, []);
const { addLoading, handleAddOk } = useAddDataSource({ isEdit: true });
const { loading: testLoading, handleTest } = useTestDataSource();
@@ -152,6 +122,54 @@ const SourceDetailPage = () => {
formRef?.current?.submit();
}, []);
+ const isUnstarted = useMemo(
+ () =>
+ detail?.status === RunningStatus.UNSTART ||
+ detail?.status === RunningStatusOld.UNSTART,
+ [detail?.status],
+ );
+
+ const isConnectorActive = useMemo(
+ () =>
+ detail?.status === RunningStatus.RUNNING ||
+ detail?.status === RunningStatus.SCHEDULE ||
+ detail?.status === RunningStatusOld.RUNNING ||
+ detail?.status === RunningStatusOld.SCHEDULE,
+ [detail?.status],
+ );
+
+ const actionMode = useMemo(() => {
+ if (isDirty) {
+ return 'save' as const;
+ }
+
+ if (isUnstarted) {
+ return 'save' as const;
+ }
+
+ if (isConnectorActive) {
+ return 'stop' as const;
+ }
+
+ return 'resume' as const;
+ }, [isConnectorActive, isDirty, isUnstarted]);
+
+ const handlePrimaryAction = useCallback(() => {
+ if (actionMode === 'save') {
+ onSubmit();
+ return;
+ }
+ updateStatus(
+ actionMode === 'resume' ? RunningStatus.SCHEDULE : RunningStatus.CANCEL,
+ );
+ }, [actionMode, onSubmit, updateStatus]);
+
+ const primaryActionLabel = useMemo(() => {
+ if (actionMode === 'stop') return 'Stop';
+ if (actionMode === 'resume') return 'Resume';
+ return 'Save';
+ }, [actionMode]);
+
useEffect(() => {
const baseFields = DataSourceFormBaseFields.map((field) => {
if (field.name === 'name') {
@@ -191,9 +209,20 @@ const SourceDetailPage = () => {
),
};
setDefaultValues(defaultValueTemp);
+ setIsDirty(false);
}
}, [detail, customFields, onSubmit]);
+ useEffect(() => {
+ const instance = formRef.current;
+ if (!instance) return;
+
+ setIsDirty(!isEqual(instance.getValues(), defaultValues));
+ return instance.watchDirty((_nextIsDirty, values) => {
+ setIsDirty(!isEqual(values, defaultValues));
+ });
+ }, [defaultValues, fields]);
+
return (
@@ -229,22 +258,21 @@ const SourceDetailPage = () => {
)}
- {t('common.confirm')}
- {/* {addLoading && }
- {addLoading
- ? t('modal.loadingText', { defaultValue: 'Submitting...' })
- : t('modal.okText', { defaultValue: 'Submit' })} */}
+ {primaryActionLabel}
diff --git a/web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx b/web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx
index 33bef3d9023..c00301b7939 100644
--- a/web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx
+++ b/web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx
@@ -1,6 +1,5 @@
import FileStatusBadge from '@/components/file-status-badge';
import { RAGFlowAvatar } from '@/components/ragflow-avatar';
-import { Button } from '@/components/ui/button';
import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
import {
Table,
@@ -14,11 +13,6 @@ import { RunningStatusMap } from '@/constants/knowledge';
import { RunningStatus } from '@/pages/dataset/dataset/constant';
import { Routes } from '@/routes';
import { formatDate } from '@/utils/date';
-import {
- HoverCard,
- HoverCardContent,
- HoverCardTrigger,
-} from '@radix-ui/react-hover-card';
import {
ColumnDef,
flexRender,
@@ -30,15 +24,86 @@ import {
} from '@tanstack/react-table';
import { t } from 'i18next';
import { pick } from 'lodash';
-import { Eye } from 'lucide-react';
-import { useCallback, useMemo } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
import { useNavigate } from 'react-router';
import { useLogListDataSource } from '../hooks';
+import { IDataSourceLog } from '../interface';
+
+const formatDuration = (seconds: number) => {
+ const safeSeconds = Math.max(0, seconds);
+ const hours = Math.floor(safeSeconds / 3600);
+ const minutes = Math.floor((safeSeconds % 3600) / 60);
+ const remainingSeconds = safeSeconds % 60;
+
+ if (hours > 0) {
+ return `${hours}h ${minutes}m ${remainingSeconds}s`;
+ }
+ if (minutes > 0) {
+ return `${minutes}m ${remainingSeconds}s`;
+ }
+ return `${remainingSeconds}s`;
+};
+
+const getTaskCountdownSeconds = (row: IDataSourceLog, now: number) => {
+ const freqMinutes =
+ row.task_type === 'prune'
+ ? Number(row.prune_freq || 0)
+ : Number(row.refresh_freq || 0);
+ const scheduledAt = row.time_started
+ ? new Date(row.time_started).getTime()
+ : 0;
+
+ if (!freqMinutes || !scheduledAt) {
+ return null;
+ }
+
+ const nextRunAt = scheduledAt + freqMinutes * 60 * 1000;
+ return Math.ceil((nextRunAt - now) / 1000);
+};
+
+const TaskCountdown = ({ row, now }: { row: IDataSourceLog; now: number }) => {
+ const remainingSeconds = getTaskCountdownSeconds(row, now);
+
+ if (remainingSeconds === null) {
+ return '';
+ }
+
+ return Task starts in {formatDuration(remainingSeconds)} ;
+};
+
+const getSummary = (row: IDataSourceLog, now: number) => {
+ if (row.status === RunningStatus.SCHEDULE || row.status === '5') {
+ return ;
+ }
+
+ if (row.status === RunningStatus.RUNNING || row.status === '1') {
+ return row.task_type === 'prune' ? 'Prune in progress' : 'Sync in progress';
+ }
+
+ if (row.status === RunningStatus.FAIL || row.status === '4') {
+ return row.error_msg || 'Task failed';
+ }
+
+ if (row.status === RunningStatus.CANCEL || row.status === '2') {
+ return '';
+ }
+
+ if (row.task_type === 'prune') {
+ return `deleted=${row.docs_removed_from_index || 0}, error=${row.error_count || 0}`;
+ }
+
+ return `total=${row.total_docs_indexed || 0}, added=${row.new_docs_indexed || 0}, updated=${Math.max(
+ 0,
+ (row.total_docs_indexed || 0) - (row.new_docs_indexed || 0),
+ )}, error=${row.error_count || 0}`;
+};
const columns = ({
handleToDataSetDetail,
+ now,
}: {
handleToDataSetDetail: (id: string) => void;
+ now: number;
}) => {
return [
{
@@ -71,7 +136,6 @@ const columns = ({
{
- console.log('handleToDataSetDetail', row.original.kb_id);
handleToDataSetDetail(row.original.kb_id);
}}
>
@@ -86,39 +150,16 @@ const columns = ({
},
},
{
- accessorKey: 'new_docs_indexed',
- header: t('setting.newDocs'),
+ accessorKey: 'task_type',
+ header: 'Task Type',
+ cell: ({ row }) => row.original.task_type || 'sync',
},
-
{
- id: 'operations',
- header: t('setting.errorMsg'),
+ id: 'summary',
+ header: 'Summary',
cell: ({ row }) => (
-
- {row.original.error_msg}
- {row.original.error_msg && (
-
-
-
- {
- // showLog(row, LogTabs.FILE_LOGS);
- // }}
- >
-
-
-
-
-
- {row.original.full_exception_trace}
-
-
-
-
- )}
+
+ {getSummary(row.original as IDataSourceLog, now)}
),
},
@@ -131,14 +172,22 @@ const columns = ({
// total: 0,
// };
export const DataSourceLogsTable = ({
- refresh_freq,
+ autoRefresh,
}: {
- refresh_freq: number | false;
+ autoRefresh: boolean;
}) => {
- // const [pagination, setPagination] = useState(paginationInit);
- const { data, pagination, setPagination } =
- useLogListDataSource(refresh_freq);
+ const { data, pagination, setPagination } = useLogListDataSource(autoRefresh);
const navigate = useNavigate();
+ const [now, setNow] = useState(() => Date.now());
+
+ useEffect(() => {
+ const timer = window.setInterval(() => {
+ setNow(Date.now());
+ }, 1000);
+
+ return () => window.clearInterval(timer);
+ }, []);
+
const currentPagination = useMemo(
() => ({
pageIndex: (pagination.current || 1) - 1,
@@ -149,15 +198,14 @@ export const DataSourceLogsTable = ({
const handleToDataSetDetail = useCallback(
(id: string) => {
- console.log('handleToDataSetDetail', id);
- navigate(`${Routes.DatasetBase}${Routes.DatasetBase}/${id}`);
+ navigate(`${Routes.Dataset}/${id}`);
},
[navigate],
);
const table = useReactTable
({
data: data || [],
- columns: columns({ handleToDataSetDetail }),
+ columns: columns({ handleToDataSetDetail, now }),
manualPagination: true,
getCoreRowModel: getCoreRowModel(),
getPaginationRowModel: getPaginationRowModel(),
diff --git a/web/src/pages/user-setting/data-source/hooks.ts b/web/src/pages/user-setting/data-source/hooks.ts
index 686da32865e..1fe074e7940 100644
--- a/web/src/pages/user-setting/data-source/hooks.ts
+++ b/web/src/pages/user-setting/data-source/hooks.ts
@@ -1,9 +1,9 @@
import message from '@/components/ui/message';
+import { RunningStatus } from '@/constants/knowledge';
import { useSetModalState } from '@/hooks/common-hooks';
import { useGetPaginationWithRouter } from '@/hooks/logic-hooks';
import dataSourceService, {
dataSourceRebuild,
- dataSourceResume,
dataSourceUpdate,
deleteDataSource,
featchDataSourceDetail,
@@ -15,7 +15,12 @@ import { t } from 'i18next';
import { useCallback, useMemo, useState } from 'react';
import { useParams, useSearchParams } from 'react-router';
import { DataSourceKey, useDataSourceInfo } from './constant';
-import { IDataSorceInfo, IDataSource, IDataSourceBase } from './interface';
+import {
+ IDataSorceInfo,
+ IDataSource,
+ IDataSourceBase,
+ IDataSourceLog,
+} from './interface';
export const useListDataSource = () => {
const { dataSourceInfo } = useDataSourceInfo();
@@ -28,10 +33,8 @@ export const useListDataSource = () => {
});
const categorizeDataBySource = (data: IDataSourceBase[]) => {
- const categorizedData: Record = {} as Record<
- DataSourceKey,
- any[]
- >;
+ const categorizedData: Partial> =
+ {};
data.forEach((item) => {
const source = item.source;
@@ -93,17 +96,29 @@ export const useAddDataSource = ({ isEdit = false }: { isEdit?: boolean }) => {
async (data: any) => {
setAddLoading(true);
const { data: res } = isEdit
- ? await dataSourceUpdate(data.id, data)
+ ? await dataSourceUpdate(data.id, {
+ ...data,
+ reschedule: true,
+ })
: await dataSourceService.dataSourceSet(data);
console.log('🚀 ~ handleAddOk ~ code:', res.code);
if (res.code === 0) {
+ if (isEdit && res.data?.id) {
+ queryClient.setQueryData(
+ ['data-source-detail', res.data.id],
+ res.data,
+ );
+ queryClient.invalidateQueries({
+ queryKey: ['data-source-detail', res.data.id],
+ });
+ }
queryClient.invalidateQueries({ queryKey: ['data-source'] });
message.success(t(`message.operated`));
hideAddingModal();
}
setAddLoading(false);
},
- [hideAddingModal, queryClient],
+ [hideAddingModal, isEdit, queryClient],
);
return {
@@ -117,24 +132,25 @@ export const useAddDataSource = ({ isEdit = false }: { isEdit?: boolean }) => {
};
};
-export const useLogListDataSource = (refresh_freq: number | false) => {
+export const useLogListDataSource = (autoRefresh: boolean) => {
const { pagination, setPagination } = useGetPaginationWithRouter();
const [currentQueryParameters] = useSearchParams();
const id = currentQueryParameters.get('id');
- const { data, isFetching } = useQuery<{ logs: IDataSource[]; total: number }>(
- {
- queryKey: ['data-source-logs', id, pagination, refresh_freq],
- refetchInterval: refresh_freq ? refresh_freq * 60 * 1000 : false,
- queryFn: async () => {
- const { data } = await getDataSourceLogs(id as string, {
- page_size: pagination.pageSize,
- page: pagination.current,
- });
- return data.data;
- },
+ const { data, isFetching } = useQuery<{
+ logs: IDataSourceLog[];
+ total: number;
+ }>({
+ queryKey: ['data-source-logs', id, pagination, autoRefresh],
+ refetchInterval: autoRefresh ? 15 * 1000 : false,
+ queryFn: async () => {
+ const { data } = await getDataSourceLogs(id as string, {
+ page_size: pagination.pageSize,
+ page: pagination.current,
+ });
+ return data.data;
},
- );
+ });
return {
data: data?.logs,
isFetching,
@@ -179,21 +195,49 @@ export const useFetchDataSourceDetail = () => {
return { data };
};
-export const useDataSourceResume = () => {
+export const useUpdateDataSourceStatus = () => {
const [currentQueryParameters] = useSearchParams();
const id = currentQueryParameters.get('id');
const queryClient = useQueryClient();
- const handleResume = useCallback(
- async (param: { resume: boolean }) => {
- const { data } = await dataSourceResume(id as string, param);
- if (data.code === 0) {
- queryClient.invalidateQueries({ queryKey: ['data-source-detail', id] });
- message.success(t(`message.operated`));
+ const [loading, setLoading] = useState(false);
+ const updateStatus = useCallback(
+ async (status: RunningStatus.SCHEDULE | RunningStatus.CANCEL) => {
+ if (!id) return;
+
+ setLoading(true);
+ try {
+ const { data } = await dataSourceUpdate(id, {
+ status,
+ });
+ if (data.code === 0) {
+ queryClient.setQueryData(
+ ['data-source-detail', id],
+ (previous?: IDataSource) => ({
+ ...(previous || {}),
+ ...(data.data || {}),
+ status: data.data?.status ?? status,
+ }),
+ );
+
+ await Promise.all([
+ queryClient.invalidateQueries({
+ queryKey: ['data-source-detail', id],
+ }),
+ queryClient.invalidateQueries({ queryKey: ['data-source'] }),
+ queryClient.invalidateQueries({
+ queryKey: ['data-source-logs', id],
+ }),
+ ]);
+
+ message.success(t(`message.operated`));
+ }
+ } finally {
+ setLoading(false);
}
},
[id, queryClient],
);
- return { handleResume };
+ return { updateStatus, loading };
};
export const useDataSourceRebuild = () => {
diff --git a/web/src/pages/user-setting/data-source/interface.ts b/web/src/pages/user-setting/data-source/interface.ts
index 5cca9974877..812a2f8e0da 100644
--- a/web/src/pages/user-setting/data-source/interface.ts
+++ b/web/src/pages/user-setting/data-source/interface.ts
@@ -1,5 +1,5 @@
import { RunningStatus } from '@/constants/knowledge';
-import { DataSourceKey } from './contant';
+import { DataSourceKey } from './constant';
export interface IDataSorceInfo {
id: DataSourceKey;
@@ -28,20 +28,20 @@ export interface IDataSourceBase {
export interface IDataSourceLog {
connector_id: string;
+ docs_removed_from_index?: number;
error_count: number;
error_msg: string;
id: string;
kb_id: string;
kb_name: string;
- name: string;
new_docs_indexed: number;
- poll_range_end: null | string;
- poll_range_start: null | string;
- reindex: string;
- source: DataSourceKey;
+ prune_freq?: number;
+ refresh_freq?: number;
status: RunningStatus;
- tenant_id: string;
- timeout_secs: number;
+ task_type?: string;
+ time_started?: string | null;
+ total_docs_indexed?: number;
+ update_date: string;
}
interface IDataSourceInfoItem {
diff --git a/web/src/services/data-source-service.ts b/web/src/services/data-source-service.ts
index 7be85dce85a..2118899340a 100644
--- a/web/src/services/data-source-service.ts
+++ b/web/src/services/data-source-service.ts
@@ -20,15 +20,12 @@ const dataSourceService = registerServer(
export const deleteDataSource = (id: string) =>
request.delete(api.dataSourceDel(id));
-export const dataSourceResume = (id: string, data: { resume: boolean }) => {
- return request.post(api.dataSourceResume(id), { data });
-};
export const dataSourceRebuild = (id: string, data: { kb_id: string }) => {
return request.post(api.dataSourceRebuild(id), { data });
};
-export const dataSourceUpdate = (id: string, data: { kb_id: string }) => {
+export const dataSourceUpdate = (id: string, data: Record) => {
return request.patch(api.dataSourceUpdate(id), { data });
};
diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts
index 32a3d5bfd62..03b065cffe7 100644
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -39,7 +39,6 @@ export default {
dataSourceSet: `${restAPIv1}/connectors`,
dataSourceList: `${restAPIv1}/connectors`,
dataSourceDel: (id: string) => `${restAPIv1}/connectors/${id}`,
- dataSourceResume: (id: string) => `${restAPIv1}/connectors/${id}/resume`,
dataSourceRebuild: (id: string) => `${restAPIv1}/connectors/${id}/rebuild`,
dataSourceLogs: (id: string) => `${restAPIv1}/connectors/${id}/logs`,
dataSourceDetail: (id: string) => `${restAPIv1}/connectors/${id}`,
From 198f3c4b9a4750a80a7ecf6abcdd3c421e8beff4 Mon Sep 17 00:00:00 2001
From: jony376
Date: Tue, 19 May 2026 05:11:46 +0300
Subject: [PATCH 193/666] Fix: validate memory tenant model IDs on update and
enforce tenant scope in memory pipeline (#14923)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### Related issues
Closes #14922
### What problem does this PR solve?
`POST /memories` already resolves `tenant_llm_id` and `tenant_embd_id`
through `ensure_tenant_model_id_for_params`, but `PUT
/memories/` accepted client-supplied `tenant_llm_id` /
`tenant_embd_id` without checking that those `tenant_llm` rows belong to
the memory owner’s tenant. A caller could persist another tenant’s row
IDs and later trigger extraction or embedding that loaded foreign model
credentials via `get_model_config_by_id(tenant_model_id)` with no tenant
allow-list.
This change aligns the update path with create: updates that change
models must go through `llm_id` / `embd_id` and
`ensure_tenant_model_id_for_params` scoped to the **memory’s**
`tenant_id` (not only the current user, so team-access cases stay
correct). Direct `tenant_*` fields in the body without `llm_id` /
`embd_id` are rejected. As defense in depth, `memory_message_service`
passes `allowed_tenant_ids` / `requester_tenant_id` into
`get_model_config_by_id` for LLM and embedding resolution so mismatched
IDs cannot be used even if bad data existed. A regression test rejects
payloads that set only `tenant_llm_id` / `tenant_embd_id`.
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---------
Co-authored-by: jony376
---
api/apps/services/memory_api_service.py | 41 ++++++++++++++-----
.../joint_services/memory_message_service.py | 18 ++++++--
.../test_memory_app/test_update_memory.py | 8 ++++
3 files changed, 54 insertions(+), 13 deletions(-)
diff --git a/api/apps/services/memory_api_service.py b/api/apps/services/memory_api_service.py
index 9040f0ce445..53cff623ceb 100644
--- a/api/apps/services/memory_api_service.py
+++ b/api/apps/services/memory_api_service.py
@@ -21,10 +21,11 @@
from api.db.services.task_service import TaskService
from api.db.joint_services.memory_message_service import get_memory_size_cache, judge_system_prompt_is_default, queue_save_to_memory_task, query_message
from api.utils.memory_utils import format_ret_data_from_memory, get_memory_type_human
+from api.utils.tenant_utils import ensure_tenant_model_id_for_params
from api.constants import MEMORY_NAME_LIMIT, MEMORY_SIZE_LIMIT
from memory.services.messages import MessageService
from memory.utils.prompt_util import PromptAssembler
-from common.constants import MemoryType, ForgettingPolicy
+from common.constants import MemoryType, ForgettingPolicy, LLMType
from common.exceptions import ArgumentException, NotFoundException
from common.time_utils import current_timestamp, timestamp_to_date
@@ -131,6 +132,9 @@ async def update_memory(memory_id: str, new_memory_setting: dict):
"user_prompt": str
}
"""
+ current_memory = _require_memory_access(memory_id)
+ owner_tenant_id = current_memory.tenant_id
+
update_dict = {}
# check name length
if "name" in new_memory_setting:
@@ -146,14 +150,32 @@ async def update_memory(memory_id: str, new_memory_setting: dict):
if new_memory_setting["permissions"] not in [e.value for e in TenantPermission]:
raise ArgumentException(f"Unknown permission '{new_memory_setting['permissions']}'.")
update_dict["permissions"] = new_memory_setting["permissions"]
- if new_memory_setting.get("llm_id"):
- update_dict["llm_id"] = new_memory_setting["llm_id"]
- if new_memory_setting.get("embd_id"):
- update_dict["embd_id"] = new_memory_setting["embd_id"]
- if new_memory_setting.get("tenant_llm_id"):
- update_dict["tenant_llm_id"] = new_memory_setting["tenant_llm_id"]
- if new_memory_setting.get("tenant_embd_id"):
- update_dict["tenant_embd_id"] = new_memory_setting["tenant_embd_id"]
+ if ("tenant_llm_id" in new_memory_setting or "tenant_embd_id" in new_memory_setting) and not (
+ new_memory_setting.get("llm_id") or new_memory_setting.get("embd_id")
+ ):
+ raise ArgumentException(
+ "Do not set tenant_llm_id or tenant_embd_id directly; update llm_id and/or embd_id instead."
+ )
+ if new_memory_setting.get("llm_id") or new_memory_setting.get("embd_id"):
+ merged = {
+ "llm_id": new_memory_setting.get("llm_id") or current_memory.llm_id,
+ "embd_id": new_memory_setting.get("embd_id") or current_memory.embd_id,
+ }
+ merged = ensure_tenant_model_id_for_params(owner_tenant_id, merged)
+ if not merged.get("tenant_llm_id"):
+ raise ArgumentException(
+ f"Tenant Model with name {merged['llm_id']} and type {LLMType.CHAT.value} not found"
+ )
+ if new_memory_setting.get("embd_id") and not merged.get("tenant_embd_id"):
+ raise ArgumentException(
+ f"Tenant Model with name {merged['embd_id']} and type {LLMType.EMBEDDING.value} not found"
+ )
+ if new_memory_setting.get("llm_id"):
+ update_dict["llm_id"] = merged["llm_id"]
+ if new_memory_setting.get("embd_id"):
+ update_dict["embd_id"] = merged["embd_id"]
+ update_dict["tenant_llm_id"] = merged["tenant_llm_id"]
+ update_dict["tenant_embd_id"] = merged.get("tenant_embd_id")
if new_memory_setting.get("memory_type"):
memory_type = set(new_memory_setting["memory_type"])
invalid_type = memory_type - {e.name.lower() for e in MemoryType}
@@ -180,7 +202,6 @@ async def update_memory(memory_id: str, new_memory_setting: dict):
for field in ["avatar", "description", "system_prompt", "user_prompt"]:
if field in new_memory_setting:
update_dict[field] = new_memory_setting[field]
- current_memory = _require_memory_access(memory_id)
memory_dict = current_memory.to_dict()
memory_dict.update({"memory_type": get_memory_type_human(current_memory.memory_type)})
diff --git a/api/db/joint_services/memory_message_service.py b/api/db/joint_services/memory_message_service.py
index 4765b2bdbb6..1a6da3a8d6b 100644
--- a/api/db/joint_services/memory_message_service.py
+++ b/api/db/joint_services/memory_message_service.py
@@ -154,7 +154,11 @@ async def extract_by_llm(tenant_id: str, tenant_llm_id: int, extract_conf: dict,
else:
user_prompts.append({"role": "user", "content": PromptAssembler.assemble_user_prompt(conversation_content, conversation_time, conversation_time)})
if tenant_llm_id:
- llm_config = get_model_config_by_id(tenant_llm_id)
+ llm_config = get_model_config_by_id(
+ tenant_llm_id,
+ allowed_tenant_ids=tenant_id,
+ requester_tenant_id=tenant_id,
+ )
else:
llm_config = get_model_config_by_type_and_name(tenant_id, LLMType.CHAT, llm_id)
llm = LLMBundle(tenant_id, llm_config)
@@ -174,7 +178,11 @@ async def extract_by_llm(tenant_id: str, tenant_llm_id: int, extract_conf: dict,
async def embed_and_save(memory, message_list: list[dict], task_id: str=None):
if memory.tenant_embd_id:
- embd_model_config = get_model_config_by_id(memory.tenant_embd_id)
+ embd_model_config = get_model_config_by_id(
+ memory.tenant_embd_id,
+ allowed_tenant_ids=memory.tenant_id,
+ requester_tenant_id=memory.tenant_id,
+ )
else:
embd_model_config = get_model_config_by_type_and_name(memory.tenant_id, LLMType.EMBEDDING, memory.embd_id)
embedding_model = LLMBundle(memory.tenant_id, embd_model_config)
@@ -248,7 +256,11 @@ def query_message(filter_dict: dict, params: dict):
question = question.strip()
memory = memory_list[0]
if memory.tenant_embd_id:
- embd_model_config = get_model_config_by_id(memory.tenant_embd_id)
+ embd_model_config = get_model_config_by_id(
+ memory.tenant_embd_id,
+ allowed_tenant_ids=memory.tenant_id,
+ requester_tenant_id=memory.tenant_id,
+ )
else:
embd_model_config = get_model_config_by_type_and_name(memory.tenant_id, LLMType.EMBEDDING, memory.embd_id)
embd_model = LLMBundle(memory.tenant_id, embd_model_config)
diff --git a/test/testcases/test_web_api/test_memory_app/test_update_memory.py b/test/testcases/test_web_api/test_memory_app/test_update_memory.py
index 1fa92b8e448..72ecfaa8ec3 100644
--- a/test/testcases/test_web_api/test_memory_app/test_update_memory.py
+++ b/test/testcases/test_web_api/test_memory_app/test_update_memory.py
@@ -106,6 +106,14 @@ def test_llm(self, WebApiAuth, add_memory_func):
assert res["code"] == 0, res
assert res["data"]["llm_id"] == llm_id, res
+ @pytest.mark.p2
+ def test_reject_direct_tenant_model_ids(self, WebApiAuth, add_memory_func):
+ memory_ids = add_memory_func
+ payload = {"tenant_llm_id": 999999, "tenant_embd_id": 999998}
+ res = update_memory(WebApiAuth, memory_ids[0], payload)
+ assert res["code"] == 101, res
+ assert "Do not set tenant_llm_id or tenant_embd_id directly" in res["message"], res
+
@pytest.mark.p2
@pytest.mark.parametrize(
"permission",
From 525a87be0ff3e3ae75d68c0a0e12d186f0235f20 Mon Sep 17 00:00:00 2001
From: kingloon <1044057492@qq.com>
Date: Tue, 19 May 2026 10:47:06 +0800
Subject: [PATCH 194/666] Misc: fix some typos (#14987)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Fix minor code quality issues:
1. Fix typo in assertion error message: "Can't fine" → "Can't find"
2. Remove duplicate line in common/connection_utils.py
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] Refactoring
---
api/db/services/document_service.py | 4 ++--
common/connection_utils.py | 2 --
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py
index 7b0bce6e62b..348d8a3a604 100644
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -709,7 +709,7 @@ def delete_document_and_update_kb_counts(cls, doc_id) -> bool:
def clear_chunk_num(cls, doc_id):
"""Deprecated: use delete_document_and_update_kb_counts instead."""
doc = cls.model.get_by_id(doc_id)
- assert doc, "Can't fine document in database."
+ assert doc, "Can't find document in database."
num = (
Knowledgebase.update(token_num=Knowledgebase.token_num - doc.token_num, chunk_num=Knowledgebase.chunk_num - doc.chunk_num, doc_num=Knowledgebase.doc_num - 1)
@@ -722,7 +722,7 @@ def clear_chunk_num(cls, doc_id):
@DB.connection_context()
def clear_chunk_num_when_rerun(cls, doc_id):
doc = cls.model.get_by_id(doc_id)
- assert doc, "Can't fine document in database."
+ assert doc, "Can't find document in database."
num = (
Knowledgebase.update(
diff --git a/common/connection_utils.py b/common/connection_utils.py
index 86ebc371d8c..0218d99a281 100644
--- a/common/connection_utils.py
+++ b/common/connection_utils.py
@@ -115,7 +115,6 @@ async def construct_response(code=RetCode.SUCCESS, message="success", data=None,
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Method"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
- response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Expose-Headers"] = "Authorization"
return response
@@ -135,6 +134,5 @@ def sync_construct_response(code=RetCode.SUCCESS, message="success", data=None,
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Method"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
- response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Expose-Headers"] = "Authorization"
return response
From db9e78274757fae76458bd32ddb79864ea40c773 Mon Sep 17 00:00:00 2001
From: Haruko386
Date: Tue, 19 May 2026 10:49:33 +0800
Subject: [PATCH 195/666] Go: implement provider: MinerU (#14990)
### What problem does this PR solve?
Implement MinerU Provider
**The following functionalities are now supported:**
**MinerU**
----
- [x] Parse file
- [x] Show task
- [ ] ~~List tasks~~
**Verified examples from the CLI:**
```plaintext
RAGFlow(user)> parse with 'vlm@test@mineru' file 'https://arxiv.org/pdf/2505.09358'
+--------------------------------------+
| task_id |
+--------------------------------------+
| 142ac8ea-d9d0-4a68-a2d1-d3af67635dc9 |
+--------------------------------------+
RAGFlow(user)> show 'test@mineru' task '142ac8ea-d9d0-4a68-a2d1-d3af67635dc9'
+--------------------------------------------+-------+
| content | index |
+--------------------------------------------+-------+
| Task is running... Progress: 17 / 18 pages | 0 |
+--------------------------------------------+-------+
RAGFlow(user)> show 'test@mineru' task '142ac8ea-d9d0-4a68-a2d1-d3af67635dc9'
+--------------------------------------------------------------------------------------------+-------+
| content | index |
+--------------------------------------------------------------------------------------------+-------+
| https://cdn-mineru.openxlab.org.cn/pdf/2026-05-18/142ac8ea-d9d0-4a68-a2d1-d3af67635dc9.zip | 0 |
+--------------------------------------------------------------------------------------------+-------+
```
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
- [x] Refactoring
---
conf/models/baidu.json | 2 +-
conf/models/mineru.json | 25 +++
conf/models/xunfei.json | 4 +-
internal/cli/user_command.go | 15 +-
internal/entity/models/factory.go | 2 +
internal/entity/models/gitee.go | 8 +-
internal/entity/models/mineru.go | 265 ++++++++++++++++++++++++++++++
internal/entity/models/types.go | 1 +
8 files changed, 310 insertions(+), 12 deletions(-)
create mode 100644 conf/models/mineru.json
create mode 100644 internal/entity/models/mineru.go
diff --git a/conf/models/baidu.json b/conf/models/baidu.json
index 1c584d27ad1..b1654652697 100644
--- a/conf/models/baidu.json
+++ b/conf/models/baidu.json
@@ -1,5 +1,5 @@
{
- "Name": "Baidu",
+ "name": "Baidu",
"url": {
"default": "https://qianfan.baidubce.com/v2"
},
diff --git a/conf/models/mineru.json b/conf/models/mineru.json
new file mode 100644
index 00000000000..f47accd954d
--- /dev/null
+++ b/conf/models/mineru.json
@@ -0,0 +1,25 @@
+{
+ "name": "MinerU",
+ "url": {
+ "default": "https://mineru.net"
+ },
+ "url_suffix": {
+ "doc_parse": "v4/extract/task",
+ "tasks": ""
+ },
+ "class": "mineru",
+ "models": [
+ {
+ "name": "vlm",
+ "model_types": [
+ "doc_parse"
+ ]
+ },
+ {
+ "name": "MinerU-HTML",
+ "model_types": [
+ "doc_parse"
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/conf/models/xunfei.json b/conf/models/xunfei.json
index 3d963d96e95..6ab0385b55f 100644
--- a/conf/models/xunfei.json
+++ b/conf/models/xunfei.json
@@ -1,10 +1,10 @@
{
"name": "XunFei",
"url": {
- "default": "https://"
+ "default": "https://spark-api-open.xf-yun.com"
},
"url_suffix": {
- "chat": "spark-api-open.xf-yun.com/v2/chat/completions"
+ "chat": "v2/chat/completions"
},
"class": "xunfei",
"models": [
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index 7ead18285cb..4960dbbb6c9 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -2381,11 +2381,16 @@ func (c *RAGFlowClient) ParseFileUserCommand(cmd *Command) (ResponseIf, error) {
filename, ok = cmd.Params["file"].(string)
if ok {
- // read file and convert to base64
- var err error
- fileContent, err = os.ReadFile(filename)
- if err != nil {
- return nil, fmt.Errorf("failed to read file: %w", err)
+ // For online file
+ if strings.HasPrefix(filename, "http://") || strings.HasPrefix(filename, "https://") {
+ fileURL = filename
+ } else {
+ // read file and convert to base64
+ var err error
+ fileContent, err = os.ReadFile(filename)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read file: %w", err)
+ }
}
} else {
fileURL, ok = cmd.Params["url"].(string)
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index dcef9dd9981..e3719d34747 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -101,6 +101,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewXunFeiModel(baseURL, urlSuffix), nil
case "deepinfra":
return NewDeepInfraModel(baseURL, urlSuffix), nil
+ case "mineru":
+ return NewMinerUModel(baseURL, urlSuffix), nil
default:
return NewDummyModel(baseURL, urlSuffix), nil
}
diff --git a/internal/entity/models/gitee.go b/internal/entity/models/gitee.go
index 6a493906bd4..05e4baa33a7 100644
--- a/internal/entity/models/gitee.go
+++ b/internal/entity/models/gitee.go
@@ -874,7 +874,7 @@ func (g *GiteeModel) getParseFile(baseURL *string, apiKey, taskID *string, timeO
func (g *GiteeModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
- if apiConfig.Region != nil {
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -931,7 +931,7 @@ func (g *GiteeModel) ListModels(apiConfig *APIConfig) ([]string, error) {
func (g *GiteeModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
var region = "default"
- if apiConfig.Region != nil {
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -1058,7 +1058,7 @@ type giteeTaskURLs struct {
func (g *GiteeModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
var region = "default"
- if apiConfig.Region != nil {
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -1113,7 +1113,7 @@ func (g *GiteeModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
func (g *GiteeModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
var region = "default"
- if apiConfig.Region != nil {
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
diff --git a/internal/entity/models/mineru.go b/internal/entity/models/mineru.go
new file mode 100644
index 00000000000..1ff4697db2f
--- /dev/null
+++ b/internal/entity/models/mineru.go
@@ -0,0 +1,265 @@
+package models
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "time"
+)
+
+type MinerUModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewMinerUModel(baseURL map[string]string, urlSuffix URLSuffix) *MinerUModel {
+ return &MinerUModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Timeout: time.Second * 120,
+ Transport: &http.Transport{
+ MaxIdleConns: 10,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: time.Second * 90,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (m *MinerUModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return &MinerUModel{
+ BaseURL: baseURL,
+ URLSuffix: m.URLSuffix,
+ httpClient: &http.Client{
+ Timeout: time.Second * 120,
+ Transport: &http.Transport{
+ MaxIdleConns: 10,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: time.Second * 90,
+ DisableCompression: false,
+ },
+ },
+ }
+}
+
+func (m *MinerUModel) Name() string {
+ return "mineru"
+}
+
+func (m *MinerUModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+func (m *MinerUModel) CheckConnection(apiConfig *APIConfig) error {
+ return fmt.Errorf("%s no such method", m.Name())
+}
+
+type mineruTaskSubmitResponse struct {
+ Code int `json:"code"`
+ Data struct {
+ TaskID string `json:"task_id"`
+ } `json:"data"`
+ Msg string `json:"msg"`
+ TraceID string `json:"trace_id"`
+}
+
+func (m *MinerUModel) ParseFile(modelName *string, content []byte, documentURL *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ if documentURL == nil || *documentURL == "" {
+ return nil, fmt.Errorf("MinerU API requires a valid public document URL; direct file upload is not supported")
+ }
+
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ apiURL := fmt.Sprintf("%s/api/%s", m.BaseURL[region], m.URLSuffix.DocumentParse)
+
+ reqBody := map[string]interface{}{
+ "url": *documentURL,
+ }
+
+ if modelName != nil && *modelName != "" {
+ reqBody["model_version"] = *modelName
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := m.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("MinerU API failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var taskResp mineruTaskSubmitResponse
+ if err := json.Unmarshal(body, &taskResp); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ if taskResp.Code != 0 {
+ return nil, fmt.Errorf("MinerU task creation failed (code %d): %s", taskResp.Code, taskResp.Msg)
+ }
+
+ return &ParseFileResponse{
+ TaskID: taskResp.Data.TaskID,
+ }, nil
+}
+
+func (m *MinerUModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s no such method", m.Name())
+}
+
+type mineruTaskQueryResponse struct {
+ Code int `json:"code"`
+ Data struct {
+ TaskID string `json:"task_id"`
+ State string `json:"state"` // including: pending, running, done, failed, converting
+ FullZipURL string `json:"full_zip_url"`
+ ErrMsg string `json:"err_msg"`
+ ExtractProgress struct {
+ ExtractedPages int `json:"extracted_pages"`
+ TotalPages int `json:"total_pages"`
+ } `json:"extract_progress"`
+ } `json:"data"`
+ Msg string `json:"msg"`
+}
+
+func (m *MinerUModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ var region = "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ // URL: https://mineru.net/api/v4/extract/task/{task_id}
+ apiURL := fmt.Sprintf("%s/api/%s/%s", m.BaseURL[region], m.URLSuffix.DocumentParse, taskID)
+
+ req, err := http.NewRequest("GET", apiURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := m.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("MinerU query API failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var queryResp mineruTaskQueryResponse
+ if err := json.Unmarshal(body, &queryResp); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ if queryResp.Code != 0 {
+ return nil, fmt.Errorf("MinerU task query failed: %s", queryResp.Msg)
+ }
+
+ // failed state
+ if queryResp.Data.State == "failed" {
+ return nil, fmt.Errorf("MinerU task failed: %s", queryResp.Data.ErrMsg)
+ }
+
+ content := ""
+ if queryResp.Data.State == "done" {
+ content = queryResp.Data.FullZipURL
+ } else if queryResp.Data.State == "running" {
+ content = fmt.Sprintf("Task is running... Progress: %d / %d pages",
+ queryResp.Data.ExtractProgress.ExtractedPages,
+ queryResp.Data.ExtractProgress.TotalPages)
+ } else {
+ // queue or formating
+ content = fmt.Sprintf("Task state: %s", queryResp.Data.State)
+ }
+
+ return &TaskResponse{
+ Segments: []TaskSegment{
+ {
+ Index: 0,
+ Content: content,
+ },
+ },
+ }, nil
+}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index d791ac04cb3..12534851046 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -79,6 +79,7 @@ type OCRFileResponse struct {
}
type ParseFileResponse struct {
+ TaskID string `json:"task_id"`
}
type ListTaskStatus struct {
From 4c9529ef3685ba6d81afc6e974dc156cf84fff1c Mon Sep 17 00:00:00 2001
From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com>
Date: Mon, 18 May 2026 17:10:36 -1000
Subject: [PATCH 196/666] Add Replicate chat provider (#14958)
## What
- Add Replicate as a chat provider backed by the documented predictions
API
- Register Replicate in the Go model factory and provider config
- Support non-streaming chat through sync predictions, polling fallback,
streaming through `urls.stream`, model listing, and connection checks
## Notes
- Uses `POST /v1/predictions` with Replicate model identifiers in
`version`, which supports official and community model identifiers
- Maps RAGFlow messages into Replicate prompt-shaped inputs (`prompt`,
optional `system_prompt`) and forwards common documented LLM inputs:
`max_new_tokens`, `temperature`, `top_p`
- Preserves whitespace in SSE output chunks and emits RAGFlow `[DONE]`
at stream completion
## Tests
- `go test -vet=off -run TestReplicate -count=1
./internal/entity/models`
- `go test -vet=off -count=1 ./internal/entity/models`
Refs #14736
---
conf/models/replicate.json | 27 +
internal/entity/models/factory.go | 2 +
internal/entity/models/replicate.go | 611 +++++++++++++++++++++++
internal/entity/models/replicate_test.go | 321 ++++++++++++
4 files changed, 961 insertions(+)
create mode 100644 conf/models/replicate.json
create mode 100644 internal/entity/models/replicate.go
create mode 100644 internal/entity/models/replicate_test.go
diff --git a/conf/models/replicate.json b/conf/models/replicate.json
new file mode 100644
index 00000000000..91111351ad5
--- /dev/null
+++ b/conf/models/replicate.json
@@ -0,0 +1,27 @@
+{
+ "name": "Replicate",
+ "url": {
+ "default": "https://api.replicate.com"
+ },
+ "url_suffix": {
+ "chat": "v1/predictions",
+ "models": "v1/models"
+ },
+ "class": "replicate",
+ "models": [
+ {
+ "name": "meta/meta-llama-3-70b-instruct",
+ "max_tokens": 8192,
+ "model_types": [
+ "chat"
+ ]
+ },
+ {
+ "name": "meta/meta-llama-3-8b-instruct",
+ "max_tokens": 8192,
+ "model_types": [
+ "chat"
+ ]
+ }
+ ]
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index e3719d34747..d8ccc9f7661 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -93,6 +93,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewLongCatModel(baseURL, urlSuffix), nil
case "novita":
return NewNovitaModel(baseURL, urlSuffix), nil
+ case "replicate":
+ return NewReplicateModel(baseURL, urlSuffix), nil
case "voyage":
return NewVoyageModel(baseURL, urlSuffix), nil
case "paddleocr":
diff --git a/internal/entity/models/replicate.go b/internal/entity/models/replicate.go
new file mode 100644
index 00000000000..0757b832507
--- /dev/null
+++ b/internal/entity/models/replicate.go
@@ -0,0 +1,611 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+)
+
+const replicatePollInterval = time.Second
+
+type ReplicateModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewReplicateModel(baseURL map[string]string, urlSuffix URLSuffix) *ReplicateModel {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxIdleConns = 100
+ transport.MaxIdleConnsPerHost = 10
+ transport.IdleConnTimeout = 90 * time.Second
+ transport.DisableCompression = false
+ transport.ResponseHeaderTimeout = 60 * time.Second
+
+ return &ReplicateModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Transport: transport,
+ },
+ }
+}
+
+func (r *ReplicateModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return NewReplicateModel(baseURL, r.URLSuffix)
+}
+
+func (r *ReplicateModel) Name() string {
+ return "replicate"
+}
+
+type replicatePredictionURLs struct {
+ Get string `json:"get"`
+ Stream string `json:"stream"`
+}
+
+type replicatePrediction struct {
+ ID string `json:"id"`
+ Status string `json:"status"`
+ Output interface{} `json:"output"`
+ Error interface{} `json:"error"`
+ URLs replicatePredictionURLs `json:"urls"`
+}
+
+type replicateModelsResponse struct {
+ Results []struct {
+ Owner string `json:"owner"`
+ Name string `json:"name"`
+ } `json:"results"`
+}
+
+type replicateSSEEvent struct {
+ event string
+ data string
+}
+
+func (r *ReplicateModel) baseURLForRegion(region string) (string, error) {
+ base, ok := r.BaseURL[region]
+ if !ok || base == "" {
+ return "", fmt.Errorf("replicate: no base URL configured for region %q", region)
+ }
+ return strings.TrimSuffix(base, "/"), nil
+}
+
+func (r *ReplicateModel) endpoint(apiConfig *APIConfig, suffix string) (string, error) {
+ region := "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := r.baseURLForRegion(region)
+ if err != nil {
+ return "", err
+ }
+ return fmt.Sprintf("%s/%s", baseURL, suffix), nil
+}
+
+func replicateUsesVersionEndpoint(modelName string) bool {
+ name := strings.TrimSpace(modelName)
+ return !strings.Contains(name, "/") || strings.Contains(name, ":")
+}
+
+func (r *ReplicateModel) predictionEndpoint(apiConfig *APIConfig, modelName string) (string, string, error) {
+ if replicateUsesVersionEndpoint(modelName) {
+ endpoint, err := r.endpoint(apiConfig, r.URLSuffix.Chat)
+ return endpoint, modelName, err
+ }
+
+ parts := strings.Split(modelName, "/")
+ if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+ return "", "", fmt.Errorf("replicate: official model name must be owner/name")
+ }
+
+ modelsPrefix := strings.TrimSuffix(r.URLSuffix.Models, "models")
+ if modelsPrefix == "" {
+ modelsPrefix = "v1/"
+ }
+ officialSuffix := fmt.Sprintf("%smodels/%s/%s/predictions",
+ modelsPrefix,
+ url.PathEscape(parts[0]),
+ url.PathEscape(parts[1]),
+ )
+ endpoint, err := r.endpoint(apiConfig, officialSuffix)
+ return endpoint, "", err
+}
+
+func replicateMessageContent(content interface{}) string {
+ switch v := content.(type) {
+ case string:
+ return v
+ default:
+ b, err := json.Marshal(v)
+ if err != nil {
+ return fmt.Sprint(v)
+ }
+ return string(b)
+ }
+}
+
+func replicatePromptFromMessages(messages []Message) (string, string) {
+ var systemParts []string
+ var promptParts []string
+ nonSystemCount := 0
+ for _, msg := range messages {
+ content := replicateMessageContent(msg.Content)
+ if msg.Role == "system" {
+ systemParts = append(systemParts, content)
+ continue
+ }
+ nonSystemCount++
+ if nonSystemCount == 1 && msg.Role == "user" && len(messages) == len(systemParts)+1 {
+ promptParts = append(promptParts, content)
+ continue
+ }
+ promptParts = append(promptParts, fmt.Sprintf("%s: %s", msg.Role, content))
+ }
+ return strings.Join(promptParts, "\n"), strings.Join(systemParts, "\n\n")
+}
+
+func replicateInputFromMessages(messages []Message, chatModelConfig *ChatConfig) map[string]interface{} {
+ prompt, systemPrompt := replicatePromptFromMessages(messages)
+ input := map[string]interface{}{
+ "prompt": prompt,
+ }
+ if systemPrompt != "" {
+ input["system_prompt"] = systemPrompt
+ }
+ if chatModelConfig != nil {
+ if chatModelConfig.MaxTokens != nil {
+ input["max_new_tokens"] = *chatModelConfig.MaxTokens
+ }
+ if chatModelConfig.Temperature != nil {
+ input["temperature"] = *chatModelConfig.Temperature
+ }
+ if chatModelConfig.TopP != nil {
+ input["top_p"] = *chatModelConfig.TopP
+ }
+ // Replicate model inputs are model-specific. Forward only the
+ // common prompt-model fields above; Stop is intentionally
+ // omitted because upstream behavior is undefined for many
+ // hosted models.
+ }
+ return input
+}
+
+func replicateOutputToString(output interface{}) (string, error) {
+ switch v := output.(type) {
+ case nil:
+ return "", nil
+ case string:
+ return v, nil
+ case []interface{}:
+ var b strings.Builder
+ for _, item := range v {
+ text, err := replicateOutputToString(item)
+ if err != nil {
+ return "", err
+ }
+ b.WriteString(text)
+ }
+ return b.String(), nil
+ case map[string]interface{}:
+ raw, err := json.Marshal(v)
+ if err != nil {
+ return "", err
+ }
+ return string(raw), nil
+ default:
+ return fmt.Sprint(v), nil
+ }
+}
+
+func (r *ReplicateModel) createPrediction(ctx context.Context, url string, version string, input map[string]interface{}, stream bool, apiKey string, preferWait bool) (*replicatePrediction, error) {
+ body := map[string]interface{}{
+ "input": input,
+ "stream": stream,
+ }
+ if version != "" {
+ body["version"] = version
+ }
+
+ jsonData, err := json.Marshal(body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+ if preferWait {
+ req.Header.Set("Prefer", "wait=60")
+ }
+
+ resp, err := r.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ bodyBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(bodyBytes))
+ }
+
+ var prediction replicatePrediction
+ if err = json.Unmarshal(bodyBytes, &prediction); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+ if prediction.Error != nil {
+ return nil, fmt.Errorf("replicate: upstream error: %v", prediction.Error)
+ }
+ return &prediction, nil
+}
+
+func replicatePredictionDone(status string) bool {
+ return replicatePredictionSucceeded(status) || status == "failed" || status == "canceled"
+}
+
+func replicatePredictionSucceeded(status string) bool {
+ return status == "successful"
+}
+
+func (r *ReplicateModel) getPrediction(ctx context.Context, url string, apiKey string) (*replicatePrediction, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+
+ resp, err := r.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var prediction replicatePrediction
+ if err = json.Unmarshal(body, &prediction); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+ if prediction.Error != nil {
+ return nil, fmt.Errorf("replicate: upstream error: %v", prediction.Error)
+ }
+ return &prediction, nil
+}
+
+func (r *ReplicateModel) waitForPrediction(ctx context.Context, prediction *replicatePrediction, apiKey string) (*replicatePrediction, error) {
+ if prediction == nil {
+ return nil, fmt.Errorf("replicate: empty prediction response")
+ }
+ if replicatePredictionDone(prediction.Status) {
+ return prediction, nil
+ }
+ if prediction.URLs.Get == "" {
+ return nil, fmt.Errorf("replicate: prediction is %q and no polling URL was returned", prediction.Status)
+ }
+
+ ticker := time.NewTicker(replicatePollInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return nil, fmt.Errorf("replicate: prediction did not finish before timeout: %w", ctx.Err())
+ case <-ticker.C:
+ next, err := r.getPrediction(ctx, prediction.URLs.Get, apiKey)
+ if err != nil {
+ return nil, err
+ }
+ if replicatePredictionDone(next.Status) {
+ return next, nil
+ }
+ }
+ }
+}
+
+func (r *ReplicateModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+ if strings.TrimSpace(modelName) == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ url, version, err := r.predictionEndpoint(apiConfig, modelName)
+ if err != nil {
+ return nil, err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ prediction, err := r.createPrediction(ctx, url, version, replicateInputFromMessages(messages, chatModelConfig), false, *apiConfig.ApiKey, true)
+ if err != nil {
+ return nil, err
+ }
+ prediction, err = r.waitForPrediction(ctx, prediction, *apiConfig.ApiKey)
+ if err != nil {
+ return nil, err
+ }
+ if !replicatePredictionSucceeded(prediction.Status) {
+ return nil, fmt.Errorf("replicate: prediction ended with status %q", prediction.Status)
+ }
+
+ answer, err := replicateOutputToString(prediction.Output)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse prediction output: %w", err)
+ }
+ reasonContent := ""
+ return &ChatResponse{Answer: &answer, ReasonContent: &reasonContent}, nil
+}
+
+func (r *ReplicateModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
+ if sender == nil {
+ return fmt.Errorf("sender is required")
+ }
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("api key is required")
+ }
+ if strings.TrimSpace(modelName) == "" {
+ return fmt.Errorf("model name is required")
+ }
+ if len(messages) == 0 {
+ return fmt.Errorf("messages is empty")
+ }
+ if chatModelConfig != nil && chatModelConfig.Stream != nil && !*chatModelConfig.Stream {
+ return fmt.Errorf("stream must be true in ChatStreamlyWithSender")
+ }
+
+ url, version, err := r.predictionEndpoint(apiConfig, modelName)
+ if err != nil {
+ return err
+ }
+
+ prediction, err := r.createPrediction(context.Background(), url, version, replicateInputFromMessages(messages, chatModelConfig), true, *apiConfig.ApiKey, false)
+ if err != nil {
+ return err
+ }
+ if prediction.URLs.Stream == "" {
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+ prediction, err = r.waitForPrediction(ctx, prediction, *apiConfig.ApiKey)
+ if err != nil {
+ return err
+ }
+ answer, err := replicateOutputToString(prediction.Output)
+ if err != nil {
+ return fmt.Errorf("failed to parse prediction output: %w", err)
+ }
+ if answer != "" {
+ if err := sender(&answer, nil); err != nil {
+ return err
+ }
+ }
+ endOfStream := "[DONE]"
+ return sender(&endOfStream, nil)
+ }
+
+ return r.readPredictionStream(prediction.URLs.Stream, *apiConfig.ApiKey, sender)
+}
+
+func (r *ReplicateModel) readPredictionStream(url string, apiKey string, sender func(*string, *string) error) error {
+ req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+ req.Header.Set("Accept", "text/event-stream")
+
+ resp, err := r.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+ current := replicateSSEEvent{}
+ sawDone := false
+ for scanner.Scan() {
+ line := scanner.Text()
+ if line == "" {
+ done, err := dispatchReplicateSSEEvent(current, sender)
+ if err != nil {
+ return err
+ }
+ if done {
+ sawDone = true
+ break
+ }
+ current = replicateSSEEvent{}
+ continue
+ }
+ if strings.HasPrefix(line, "event:") {
+ current.event = strings.TrimSpace(line[6:])
+ }
+ if strings.HasPrefix(line, "data:") {
+ if current.data != "" {
+ current.data += "\n"
+ }
+ data := line[5:]
+ if strings.HasPrefix(data, " ") {
+ data = data[1:]
+ }
+ current.data += data
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("failed to scan response body: %w", err)
+ }
+ if !sawDone && (current.event != "" || current.data != "") {
+ done, err := dispatchReplicateSSEEvent(current, sender)
+ if err != nil {
+ return err
+ }
+ sawDone = done
+ }
+ if !sawDone {
+ return fmt.Errorf("replicate: stream ended before done event")
+ }
+
+ endOfStream := "[DONE]"
+ return sender(&endOfStream, nil)
+}
+
+func dispatchReplicateSSEEvent(event replicateSSEEvent, sender func(*string, *string) error) (bool, error) {
+ switch event.event {
+ case "output", "":
+ if event.data == "" {
+ return false, nil
+ }
+ return false, sender(&event.data, nil)
+ case "error":
+ return false, fmt.Errorf("replicate: upstream stream error: %s", event.data)
+ case "done":
+ return true, nil
+ default:
+ return false, nil
+ }
+}
+
+func (r *ReplicateModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ url, err := r.endpoint(apiConfig, r.URLSuffix.Models)
+ if err != nil {
+ return nil, err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := r.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var result replicateModelsResponse
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ models := make([]string, 0, len(result.Results))
+ for _, model := range result.Results {
+ if model.Owner != "" && model.Name != "" {
+ models = append(models, fmt.Sprintf("%s/%s", model.Owner, model.Name))
+ }
+ }
+ return models, nil
+}
+
+func (r *ReplicateModel) CheckConnection(apiConfig *APIConfig) error {
+ _, err := r.ListModels(apiConfig)
+ return err
+}
+
+func (r *ReplicateModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
+
+func (r *ReplicateModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", r.Name())
+}
diff --git a/internal/entity/models/replicate_test.go b/internal/entity/models/replicate_test.go
new file mode 100644
index 00000000000..d9eb1efd6ba
--- /dev/null
+++ b/internal/entity/models/replicate_test.go
@@ -0,0 +1,321 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func newReplicateForTest(baseURL string) *ReplicateModel {
+ return NewReplicateModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{Chat: "v1/predictions", Models: "v1/models"},
+ )
+}
+
+func TestReplicateName(t *testing.T) {
+ if got := newReplicateForTest("http://unused").Name(); got != "replicate" {
+ t.Errorf("Name()=%q", got)
+ }
+}
+
+func TestReplicateFactory(t *testing.T) {
+ driver, err := NewModelFactory().CreateModelDriver("Replicate", map[string]string{"default": "http://unused"}, URLSuffix{})
+ if err != nil {
+ t.Fatalf("CreateModelDriver: %v", err)
+ }
+ if _, ok := driver.(*ReplicateModel); !ok {
+ t.Fatalf("driver type=%T, want *ReplicateModel", driver)
+ }
+}
+
+func TestReplicatePromptFromMessages(t *testing.T) {
+ prompt, system := replicatePromptFromMessages([]Message{
+ {Role: "system", Content: "be terse"},
+ {Role: "user", Content: "hello"},
+ {Role: "assistant", Content: "hi"},
+ {Role: "user", Content: map[string]interface{}{"text": "again"}},
+ })
+ if system != "be terse" {
+ t.Errorf("system=%q", system)
+ }
+ want := "user: hello\nassistant: hi\nuser: {\"text\":\"again\"}"
+ if prompt != want {
+ t.Errorf("prompt=%q want %q", prompt, want)
+ }
+}
+
+func TestReplicatePredictionEndpoint(t *testing.T) {
+ m := newReplicateForTest("https://api.example.test")
+
+ endpoint, version, err := m.predictionEndpoint(&APIConfig{}, "meta/meta-llama-3-70b-instruct")
+ if err != nil {
+ t.Fatalf("official endpoint: %v", err)
+ }
+ if endpoint != "https://api.example.test/v1/models/meta/meta-llama-3-70b-instruct/predictions" {
+ t.Errorf("official endpoint=%q", endpoint)
+ }
+ if version != "" {
+ t.Errorf("official version=%q want empty", version)
+ }
+
+ endpoint, version, err = m.predictionEndpoint(&APIConfig{}, "replicate/hello-world:5c7d5dc6dd8bf75c1acaa8565735e7986bc5b66206b55cca93cb72c9bf15ccaa")
+ if err != nil {
+ t.Fatalf("version endpoint: %v", err)
+ }
+ if endpoint != "https://api.example.test/v1/predictions" {
+ t.Errorf("version endpoint=%q", endpoint)
+ }
+ if version != "replicate/hello-world:5c7d5dc6dd8bf75c1acaa8565735e7986bc5b66206b55cca93cb72c9bf15ccaa" {
+ t.Errorf("version=%q", version)
+ }
+}
+
+func TestReplicateOfficialChatHappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/models/meta/meta-llama-3-70b-instruct/predictions" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+ t.Errorf("Authorization=%q", got)
+ }
+ if got := r.Header.Get("Prefer"); got != "wait=60" {
+ t.Errorf("Prefer=%q", got)
+ }
+ raw, _ := io.ReadAll(r.Body)
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("body: %v", err)
+ return
+ }
+ if body["version"] != nil {
+ t.Errorf("official model requests must not send version=%v", body["version"])
+ }
+ if body["stream"] != false {
+ t.Errorf("stream=%v", body["stream"])
+ }
+ input := body["input"].(map[string]interface{})
+ if input["prompt"] != "hello" {
+ t.Errorf("prompt=%v", input["prompt"])
+ }
+ if input["system_prompt"] != "be helpful" {
+ t.Errorf("system_prompt=%v", input["system_prompt"])
+ }
+ if input["max_new_tokens"] != float64(128) {
+ t.Errorf("max_new_tokens=%v", input["max_new_tokens"])
+ }
+ // Stop is deliberately filtered out because Replicate model
+ // inputs are model-specific and upstream support is undefined.
+ if input["stop"] != nil {
+ t.Errorf("unexpected stop=%v", input["stop"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "status": "successful",
+ "output": []string{"hel", "lo"},
+ })
+ }))
+ defer srv.Close()
+
+ apiKey := "test-key"
+ maxTokens := 128
+ stop := []string{"END"}
+ resp, err := newReplicateForTest(srv.URL).ChatWithMessages(
+ "meta/meta-llama-3-70b-instruct",
+ []Message{{Role: "system", Content: "be helpful"}, {Role: "user", Content: "hello"}},
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{MaxTokens: &maxTokens, Stop: &stop},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if *resp.Answer != "hello" {
+ t.Errorf("Answer=%q", *resp.Answer)
+ }
+ if *resp.ReasonContent != "" {
+ t.Errorf("ReasonContent=%q", *resp.ReasonContent)
+ }
+}
+
+func TestReplicateCommunityChatUsesVersionEndpoint(t *testing.T) {
+ const version = "replicate/hello-world:5c7d5dc6dd8bf75c1acaa8565735e7986bc5b66206b55cca93cb72c9bf15ccaa"
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/predictions" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ raw, _ := io.ReadAll(r.Body)
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("body: %v", err)
+ return
+ }
+ if body["version"] != version {
+ t.Errorf("version=%v", body["version"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "status": "successful",
+ "output": "ok",
+ })
+ }))
+ defer srv.Close()
+
+ apiKey := "test-key"
+ resp, err := newReplicateForTest(srv.URL).ChatWithMessages(
+ version,
+ []Message{{Role: "user", Content: "hello"}},
+ &APIConfig{ApiKey: &apiKey}, nil,
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if *resp.Answer != "ok" {
+ t.Errorf("Answer=%q", *resp.Answer)
+ }
+}
+
+func TestReplicateChatPollsUntilSucceeded(t *testing.T) {
+ var getCount int
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+ t.Errorf("Authorization=%q", got)
+ }
+ switch r.URL.Path {
+ case "/v1/models/meta/meta-llama-3-70b-instruct/predictions":
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "status": "processing",
+ "urls": map[string]string{
+ "get": "http://" + r.Host + "/v1/predictions/p1",
+ },
+ })
+ case "/v1/predictions/p1":
+ getCount++
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "status": "successful",
+ "output": "done",
+ })
+ default:
+ t.Errorf("unexpected path=%s", r.URL.Path)
+ }
+ }))
+ defer srv.Close()
+
+ apiKey := "test-key"
+ resp, err := newReplicateForTest(srv.URL).ChatWithMessages(
+ "meta/meta-llama-3-70b-instruct",
+ []Message{{Role: "user", Content: "hello"}},
+ &APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if getCount != 1 {
+ t.Errorf("getCount=%d", getCount)
+ }
+ if *resp.Answer != "done" {
+ t.Errorf("Answer=%q", *resp.Answer)
+ }
+}
+
+func TestReplicateStreamHappyPath(t *testing.T) {
+ var streamURL string
+ streamSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if got := r.Header.Get("Accept"); got != "text/event-stream" {
+ t.Errorf("Accept=%q", got)
+ }
+ w.Header().Set("Content-Type", "text/event-stream")
+ _, _ = io.WriteString(w, "event: output\n")
+ _, _ = io.WriteString(w, "data: Hello\n\n")
+ _, _ = io.WriteString(w, "event: output\n")
+ _, _ = io.WriteString(w, "data: world\n\n")
+ _, _ = io.WriteString(w, "event: done\n")
+ _, _ = io.WriteString(w, "data: {}\n\n")
+ }))
+ defer streamSrv.Close()
+ streamURL = streamSrv.URL
+
+ apiSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/models/meta/meta-llama-3-70b-instruct/predictions" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ raw, _ := io.ReadAll(r.Body)
+ var body map[string]interface{}
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("body: %v", err)
+ return
+ }
+ if body["stream"] != true {
+ t.Errorf("stream=%v", body["stream"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "status": "starting",
+ "urls": map[string]string{
+ "stream": streamURL,
+ },
+ })
+ }))
+ defer apiSrv.Close()
+
+ apiKey := "test-key"
+ var chunks []string
+ err := newReplicateForTest(apiSrv.URL).ChatStreamlyWithSender(
+ "meta/meta-llama-3-70b-instruct",
+ []Message{{Role: "user", Content: "hello"}},
+ &APIConfig{ApiKey: &apiKey}, nil,
+ func(c *string, _ *string) error {
+ if c != nil {
+ chunks = append(chunks, *c)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("ChatStreamlyWithSender: %v", err)
+ }
+ if strings.Join(chunks, "") != "Hello world[DONE]" {
+ t.Errorf("chunks=%q", strings.Join(chunks, ""))
+ }
+}
+
+func TestReplicateListModelsAndCheckConnection(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/models" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+ t.Errorf("Authorization=%q", got)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "results": []map[string]string{
+ {"owner": "meta", "name": "meta-llama-3-70b-instruct"},
+ {"owner": "replicate", "name": "hello-world"},
+ },
+ })
+ }))
+ defer srv.Close()
+
+ apiKey := "test-key"
+ model := newReplicateForTest(srv.URL)
+ models, err := model.ListModels(&APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ if strings.Join(models, ",") != "meta/meta-llama-3-70b-instruct,replicate/hello-world" {
+ t.Errorf("models=%v", models)
+ }
+ if err := model.CheckConnection(&APIConfig{ApiKey: &apiKey}); err != nil {
+ t.Fatalf("CheckConnection: %v", err)
+ }
+}
+
+func TestReplicateUnsupportedMethods(t *testing.T) {
+ m := newReplicateForTest("http://unused")
+ if _, err := m.Embed(nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Embed error=%v", err)
+ }
+ if _, err := m.Rerank(nil, "", nil, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Rerank error=%v", err)
+ }
+ if _, err := m.Balance(nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Balance error=%v", err)
+ }
+}
From 87d22a44153fb4deea692520de8876b9e90cf714 Mon Sep 17 00:00:00 2001
From: buua436
Date: Tue, 19 May 2026 12:00:02 +0800
Subject: [PATCH 197/666] Fix: agent session log message (#14991)
### What problem does this PR solve?
agent session log message
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/restful_apis/agent_api.py | 419 +++++++++++++++++++++++------
1 file changed, 330 insertions(+), 89 deletions(-)
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index 8478571ac1d..3e6226158d9 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -108,9 +108,23 @@ def _build_sse_response(body):
return resp
+def _normalize_agent_reference_entry(reference):
+ if not isinstance(reference, dict):
+ return {"chunks": [], "doc_aggs": []}
+ if "chunks" in reference or "doc_aggs" in reference:
+ return {
+ "chunks": reference.get("chunks", []),
+ "doc_aggs": reference.get("doc_aggs", []),
+ }
+ return {
+ "chunks": reference.get("reference", reference.get("chunks", [])) or [],
+ "doc_aggs": reference.get("doc_aggs", []) or [],
+ }
+
+
def _normalize_agent_session(conv):
- conv["messages"] = conv.pop("message")
- for info in conv["messages"]:
+ conv["message"] = conv.get("message", [])
+ for info in conv["message"]:
if "prompt" in info:
info.pop("prompt")
conv["agent_id"] = conv.pop("dialog_id")
@@ -119,11 +133,15 @@ def _normalize_agent_session(conv):
conv["reference"] = [conv["reference"]]
else:
conv["reference"] = [value for _, value in sorted(conv["reference"].items(), key=lambda item: int(item[0]))]
+ elif isinstance(conv["reference"], list):
+ conv["reference"] = [_normalize_agent_reference_entry(reference) for reference in conv["reference"]]
+ else:
+ conv["reference"] = []
if conv["reference"]:
- messages = [message for i, message in enumerate(conv["messages"]) if i != 0 and message["role"] != "user"]
+ messages = [message for i, message in enumerate(conv["message"]) if i != 0 and message["role"] != "user"]
for message, reference in zip(messages, conv["reference"]):
- chunks = reference["chunks"]
+ chunks = reference.get("chunks", [])
message["reference"] = [
{
"id": chunk.get("chunk_id", chunk.get("id")),
@@ -144,6 +162,171 @@ def _agent_session_list_result(data, total):
return jsonify({"code": RetCode.SUCCESS, "message": "success", "data": data, "total": total})
+async def _run_workflow_session(
+ tenant_id,
+ agent_id,
+ workflow_conv,
+ canvas,
+ query,
+ files,
+ inputs,
+ user_id,
+ session_id,
+ custom_header,
+ canvas_title,
+ canvas_category,
+ return_trace,
+ stream,
+):
+ async def commit_runtime_replica():
+ commit_ok = CanvasReplicaService.commit_after_run(
+ canvas_id=agent_id,
+ tenant_id=str(tenant_id),
+ runtime_user_id=user_id,
+ dsl=json.loads(str(canvas)),
+ canvas_category=canvas_category,
+ title=canvas_title,
+ )
+ if not commit_ok:
+ logging.error(
+ "Canvas runtime replica commit failed: canvas_id=%s tenant_id=%s runtime_user_id=%s",
+ agent_id,
+ tenant_id,
+ user_id,
+ )
+
+ workflow_conv.setdefault("message", [])
+ if isinstance(workflow_conv.get("reference"), dict):
+ if "chunks" in workflow_conv["reference"]:
+ workflow_conv["reference"] = [workflow_conv["reference"]]
+ else:
+ workflow_conv["reference"] = [
+ value for _, value in sorted(workflow_conv["reference"].items(), key=lambda item: int(item[0]))
+ ]
+ elif not isinstance(workflow_conv.get("reference"), list):
+ workflow_conv["reference"] = []
+ workflow_conv["reference"] = [_normalize_agent_reference_entry(reference) for reference in workflow_conv["reference"]]
+
+ turn_id = workflow_conv["message"][-1].get("id") if workflow_conv["message"] else get_uuid()
+ full_content = ""
+ reference = {}
+ final_ans = {}
+ trace_items = []
+ structured_output = {}
+
+ async def persist_workflow_session():
+ if not final_ans:
+ return
+ workflow_conv["message"].append(
+ {
+ "role": "assistant",
+ "content": full_content,
+ "created_at": time.time(),
+ "id": turn_id,
+ }
+ )
+ workflow_conv["reference"].append(_normalize_agent_reference_entry(reference))
+ workflow_conv["dsl"] = json.loads(str(canvas))
+ workflow_conv["source"] = workflow_conv.get("source") or "workflow"
+ await thread_pool_exec(API4ConversationService.append_message, session_id, workflow_conv)
+ await commit_runtime_replica()
+
+ if stream:
+
+ async def sse():
+ nonlocal full_content, reference, final_ans, trace_items, structured_output
+ done_sent = False
+ try:
+ async for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs):
+ ans["session_id"] = session_id
+ if ans.get("event") == "message":
+ full_content += ans.get("data", {}).get("content", "")
+ if ans.get("data", {}).get("reference", None):
+ reference.update(ans["data"]["reference"])
+ if ans.get("event") == "node_finished":
+ data = ans.get("data", {})
+ node_out = data.get("outputs", {})
+ component_id = data.get("component_id")
+ if component_id is not None and "structured" in node_out:
+ structured_output[component_id] = copy.deepcopy(node_out["structured"])
+ if return_trace:
+ trace_items.append(
+ {
+ "component_id": data.get("component_id"),
+ "trace": [copy.deepcopy(data)],
+ }
+ )
+ final_ans = ans
+ yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
+
+ if final_ans:
+ if "data" not in final_ans or not isinstance(final_ans["data"], dict):
+ final_ans["data"] = {}
+ final_ans["data"]["content"] = full_content
+ final_ans["data"]["reference"] = reference
+ if structured_output:
+ final_ans["data"]["structured"] = structured_output
+ if trace_items:
+ final_ans["data"]["trace"] = trace_items
+ await persist_workflow_session()
+ except Exception as exc:
+ logging.exception(exc)
+ canvas.cancel_task()
+ yield (
+ "data:"
+ + json.dumps({"code": 500, "message": str(exc), "data": False}, ensure_ascii=False)
+ + "\n\n"
+ )
+ finally:
+ if not done_sent:
+ done_sent = True
+ yield "data:[DONE]\n\n"
+
+ return _build_sse_response(sse())
+
+ try:
+ async for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs):
+ ans["session_id"] = session_id
+ if ans.get("event") == "message":
+ full_content += ans.get("data", {}).get("content", "")
+ if ans.get("data", {}).get("reference", None):
+ reference.update(ans["data"]["reference"])
+ if ans.get("event") == "node_finished":
+ data = ans.get("data", {})
+ node_out = data.get("outputs", {})
+ component_id = data.get("component_id")
+ if component_id is not None and "structured" in node_out:
+ structured_output[component_id] = copy.deepcopy(node_out["structured"])
+ if return_trace:
+ trace_items.append(
+ {
+ "component_id": data.get("component_id"),
+ "trace": [copy.deepcopy(data)],
+ }
+ )
+ final_ans = ans
+ except Exception as exc:
+ logging.exception(exc)
+ canvas.cancel_task()
+ return get_result(data=f"**ERROR**: {str(exc)}")
+
+ if not final_ans:
+ await commit_runtime_replica()
+ return get_result(data={})
+
+ if "data" not in final_ans or not isinstance(final_ans["data"], dict):
+ final_ans["data"] = {}
+ final_ans["data"]["content"] = full_content
+ final_ans["data"]["reference"] = reference
+ if structured_output:
+ final_ans["data"]["structured"] = structured_output
+ if trace_items:
+ final_ans["data"]["trace"] = trace_items
+
+ await persist_workflow_session()
+ return get_result(data=final_ans)
+
+
@manager.route("/agents//sessions", methods=["GET"]) # noqa: F821
@login_required
@add_tenant_id_to_kwargs
@@ -957,6 +1140,8 @@ async def agent_chat_completion(tenant_id, agent_id=None):
req.pop("agent_id", None)
req.pop("openai-compatible", None)
session_id = req.get("session_id")
+ workflow_session = False
+ workflow_conv = None
if session_id:
exists, conv = API4ConversationService.get_by_id(session_id)
if not exists:
@@ -973,6 +1158,9 @@ async def agent_chat_completion(tenant_id, agent_id=None):
message="Only authorized users can access this agent session.",
code=RetCode.OPERATING_ERROR,
)
+ workflow_session = getattr(conv, "source", "") == "workflow"
+ if workflow_session:
+ workflow_conv = conv.to_dict()
if openai_compatible:
# OpenAI-compatible mode uses a different wire format, keep it separate from regular agent events.
@@ -1005,8 +1193,7 @@ async def agent_chat_completion(tenant_id, agent_id=None):
return jsonify(response)
return None
- if not session_id:
- # Without session state, run against the runtime replica that tracks draft edits.
+ if workflow_session:
query = req.get("query", "") or req.get("question", "")
files = req.get("files", [])
inputs = req.get("inputs", {})
@@ -1014,6 +1201,64 @@ async def agent_chat_completion(tenant_id, agent_id=None):
user_id = str(runtime_user_id)
custom_header = req.get("custom_header", "")
+ _, cvs = await thread_pool_exec(UserCanvasService.get_by_id, agent_id)
+ if not cvs:
+ return get_data_error_result(message="canvas not found.")
+
+ if not isinstance(workflow_conv.get("message"), list):
+ workflow_conv["message"] = []
+ if isinstance(workflow_conv.get("reference"), dict):
+ if "chunks" in workflow_conv["reference"]:
+ workflow_conv["reference"] = [workflow_conv["reference"]]
+ else:
+ workflow_conv["reference"] = [
+ value for _, value in sorted(workflow_conv["reference"].items(), key=lambda item: int(item[0]))
+ ]
+ elif not isinstance(workflow_conv.get("reference"), list):
+ workflow_conv["reference"] = []
+ workflow_conv["reference"] = [_normalize_agent_reference_entry(reference) for reference in workflow_conv["reference"]]
+ turn_id = get_uuid()
+ workflow_conv["message"].append(
+ {
+ "role": "user",
+ "content": query,
+ "id": turn_id,
+ "files": files,
+ "created_at": time.time(),
+ }
+ )
+ await thread_pool_exec(API4ConversationService.update_by_id, session_id, workflow_conv)
+
+ try:
+ from agent.canvas import Canvas
+
+ workflow_dsl = workflow_conv.get("dsl", {})
+ if isinstance(workflow_dsl, str):
+ dsl_str = workflow_dsl
+ else:
+ dsl_str = json.dumps(workflow_dsl, ensure_ascii=False)
+ canvas = Canvas(dsl_str, str(tenant_id), canvas_id=agent_id, custom_header=custom_header)
+ except Exception as exc:
+ return server_error_response(exc)
+
+ return await _run_workflow_session(
+ tenant_id=tenant_id,
+ agent_id=agent_id,
+ workflow_conv=workflow_conv,
+ canvas=canvas,
+ query=query,
+ files=files,
+ inputs=inputs,
+ user_id=user_id,
+ session_id=session_id,
+ custom_header=custom_header,
+ canvas_title=getattr(cvs, "title", ""),
+ canvas_category=getattr(cvs, "canvas_category", CanvasCategory.Agent),
+ return_trace=bool(req.get("return_trace", False)),
+ stream=req.get("stream", True),
+ )
+
+ if not session_id:
if not UserCanvasService.accessible(agent_id, tenant_id):
return get_json_result(
data=False,
@@ -1021,6 +1266,16 @@ async def agent_chat_completion(tenant_id, agent_id=None):
code=RetCode.OPERATING_ERROR,
)
+ # Keep the original workflow execution path, but assign a session_id so the
+ # response shape stays closer to the older agent completion contract.
+ query = req.get("query", "") or req.get("question", "")
+ files = req.get("files", [])
+ inputs = req.get("inputs", {})
+ runtime_user_id = req.get("user_id") or tenant_id
+ user_id = str(runtime_user_id)
+ custom_header = req.get("custom_header", "")
+ session_id = get_uuid()
+
_, cvs = await thread_pool_exec(UserCanvasService.get_by_id, agent_id)
if not cvs:
return get_data_error_result(message="canvas not found.")
@@ -1054,6 +1309,31 @@ async def agent_chat_completion(tenant_id, agent_id=None):
from rag.flow.pipeline import Pipeline
task_id = get_uuid()
+ workflow_conv = {
+ "id": session_id,
+ "dialog_id": cvs.id,
+ "user_id": user_id,
+ "exp_user_id": user_id,
+ "name": req.get("name", ""),
+ "message": [
+ {
+ "role": "user",
+ "content": query,
+ "id": task_id,
+ "files": files,
+ "created_at": time.time(),
+ }
+ ],
+ "reference": [],
+ "source": "workflow",
+ "dsl": replica_dsl,
+ "version_title": await thread_pool_exec(
+ UserCanvasVersionService.get_latest_version_title,
+ cvs.id,
+ release_mode=False,
+ ),
+ }
+ await thread_pool_exec(API4ConversationService.save, **workflow_conv)
Pipeline(
dsl_str,
tenant_id=str(tenant_id),
@@ -1072,7 +1352,7 @@ async def agent_chat_completion(tenant_id, agent_id=None):
)
if not ok:
return get_data_error_result(message=error_message)
- return get_json_result(data={"message_id": task_id})
+ return get_json_result(data={"message_id": task_id, "session_id": session_id})
try:
from agent.canvas import Canvas
@@ -1080,88 +1360,49 @@ async def agent_chat_completion(tenant_id, agent_id=None):
canvas = Canvas(dsl_str, str(tenant_id), canvas_id=agent_id, custom_header=custom_header)
except Exception as exc:
return server_error_response(exc)
-
- async def commit_runtime_replica():
- commit_ok = CanvasReplicaService.commit_after_run(
- canvas_id=agent_id,
- tenant_id=str(tenant_id),
- runtime_user_id=user_id,
- dsl=json.loads(str(canvas)),
- canvas_category=canvas_category,
- title=canvas_title,
- )
- if not commit_ok:
- logging.error(
- "Canvas runtime replica commit failed: canvas_id=%s tenant_id=%s runtime_user_id=%s",
- agent_id,
- tenant_id,
- user_id,
- )
-
- if req.get("stream", True):
- async def sse():
- nonlocal canvas
- try:
- async for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs):
- yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
-
- await commit_runtime_replica()
- except Exception as exc:
- logging.exception(exc)
- canvas.cancel_task()
- yield (
- "data:"
- + json.dumps({"code": 500, "message": str(exc), "data": False}, ensure_ascii=False)
- + "\n\n"
- )
-
- return _build_sse_response(sse())
-
- full_content = ""
- reference = {}
- final_ans = {}
- trace_items = []
- structured_output = {}
- try:
- async for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs):
- if ans.get("event") == "message":
- full_content += ans.get("data", {}).get("content", "")
- if ans.get("data", {}).get("reference", None):
- reference.update(ans["data"]["reference"])
- if ans.get("event") == "node_finished":
- data = ans.get("data", {})
- node_out = data.get("outputs", {})
- component_id = data.get("component_id")
- if component_id is not None and "structured" in node_out:
- structured_output[component_id] = copy.deepcopy(node_out["structured"])
- if req.get("return_trace", False):
- trace_items.append(
- {
- "component_id": data.get("component_id"),
- "trace": [copy.deepcopy(data)],
- }
- )
- final_ans = ans
- except Exception as exc:
- logging.exception(exc)
- canvas.cancel_task()
- return get_result(data=f"**ERROR**: {str(exc)}")
-
- if not final_ans:
- await commit_runtime_replica()
- return get_result(data={})
-
- if "data" not in final_ans or not isinstance(final_ans["data"], dict):
- final_ans["data"] = {}
- final_ans["data"]["content"] = full_content
- final_ans["data"]["reference"] = reference
- if structured_output:
- final_ans["data"]["structured"] = structured_output
- if trace_items:
- final_ans["data"]["trace"] = trace_items
-
- await commit_runtime_replica()
- return get_result(data=final_ans)
+ turn_id = get_uuid()
+ workflow_conv = {
+ "id": session_id,
+ "dialog_id": cvs.id,
+ "user_id": user_id,
+ "exp_user_id": user_id,
+ "name": req.get("name", ""),
+ "message": [
+ {
+ "role": "user",
+ "content": query,
+ "id": turn_id,
+ "files": files,
+ "created_at": time.time(),
+ }
+ ],
+ "reference": [],
+ "source": "workflow",
+ "dsl": replica_dsl,
+ "version_title": await thread_pool_exec(
+ UserCanvasVersionService.get_latest_version_title,
+ cvs.id,
+ release_mode=False,
+ ),
+ }
+ workflow_conv["reference"] = [_normalize_agent_reference_entry(reference) for reference in workflow_conv["reference"]]
+ await thread_pool_exec(API4ConversationService.save, **workflow_conv)
+ return await _run_workflow_session(
+ tenant_id=tenant_id,
+ agent_id=agent_id,
+ workflow_conv=workflow_conv,
+ canvas=canvas,
+ query=query,
+ files=files,
+ inputs=inputs,
+ user_id=user_id,
+ session_id=session_id,
+ custom_header=custom_header,
+ canvas_title=canvas_title,
+ canvas_category=canvas_category,
+ return_trace=bool(req.get("return_trace", False)),
+ stream=req.get("stream", True),
+ )
return_trace = bool(req.get("return_trace", False))
if req.get("stream", True):
From c6e3a2e713c1aae9f4f051e370a6cc396c0e6e61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=88=98=E5=BA=B7=E4=BC=9F?=
<71975350+lksr1201@users.noreply.github.com>
Date: Tue, 19 May 2026 12:28:31 +0800
Subject: [PATCH 198/666] Fix: MinerU vlm-http-client backend output file
detection (#14240)
## Problem
When using MinerU with `vlm-http-client` backend, the parser fails to
find the output files because they are located in a `vlm/` subdirectory,
but the `_read_output`
method doesn't check this location.
## Error Message
[ERROR]MinerU not found.
[MinerU] Missing output file, tried: ...
## Root Cause
The MinerU API with `vlm-http-client` backend returns output files in
the following structure:
output_dir/
vlm/
filename_content_list.json
filename.md
images/
However, the `_read_output` method in `mineru_parser.py` only checks:
1. `output_dir/filename_content_list.json`
2. `output_dir/sanitized_filename_content_list.json`
3. `output_dir/sanitized_filename/sanitized_filename_content_list.json`
It doesn't check the `vlm/` subdirectory.
## Solution
Added two additional fallback paths to check the `vlm/` subdirectory:
- `output_dir/vlm/filename_content_list.json`
- `output_dir/vlm/sanitized_filename_content_list.json`
## Testing
Tested with MinerU API using `vlm-http-client` backend. The parser now
successfully finds and processes the output files.
## Related
This issue occurs specifically when using:
- MinerU backend: `vlm-http-client`
- MinerU server URL configured for remote vLLM inference
---
deepdoc/parser/mineru_parser.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py
index 90c33573cef..b369f9122a0 100644
--- a/deepdoc/parser/mineru_parser.py
+++ b/deepdoc/parser/mineru_parser.py
@@ -539,6 +539,21 @@ def _sanitize_filename(name: str) -> str:
if nested_alt.exists():
subdir = nested_alt.parent
json_file = nested_alt
+ else:
+ # Try vlm subdirectory (for vlm-http-client backend)
+ vlm_path = output_dir / "vlm" / f"{file_stem}_content_list.json"
+ self.logger.info(f"[MinerU] Trying vlm subdirectory: {vlm_path}")
+ attempted.append(vlm_path)
+ if vlm_path.exists():
+ subdir = vlm_path.parent
+ json_file = vlm_path
+ else:
+ vlm_safe = output_dir / "vlm" / f"{safe_stem}_content_list.json"
+ self.logger.info(f"[MinerU] Trying vlm subdirectory with sanitized name: {vlm_safe}")
+ attempted.append(vlm_safe)
+ if vlm_safe.exists():
+ subdir = vlm_safe.parent
+ json_file = vlm_safe
if not json_file:
parse_subdir = None
From f17a66d4f0946f83e9eae8cfa2d3d1414db94f53 Mon Sep 17 00:00:00 2001
From: OrbisAI Security
Date: Tue, 19 May 2026 11:25:33 +0530
Subject: [PATCH 199/666] fix: the opencc c library uses fgets() to read
dicti... in text.c (#13970)
## Summary
Fix critical severity security issue in
`internal/cpp/opencc/dictionary/text.c`.
## Vulnerability
| Field | Value |
|-------|-------|
| **ID** | V-001 |
| **Severity** | CRITICAL |
| **Scanner** | multi_agent_ai |
| **Rule** | `V-001` |
| **File** | `internal/cpp/opencc/dictionary/text.c:107` |
**Description**: The OpenCC C library uses fgets() to read dictionary
and configuration files without proper bounds validation on subsequent
buffer operations. While fgets() itself is bounds-checked, the sprintf()
call at config_reader.c:174 constructs file paths by concatenating
home_path and filename without verifying the result fits in pkg_filename
buffer. An attacker providing malformed OpenCC configuration files with
excessively long path components can overflow the fixed-size buffer,
overwriting adjacent memory including return addresses and function
pointers.
## Changes
- `internal/cpp/opencc/config_reader.c`
- `internal/cpp/opencc/dictionary/text.c`
- `internal/cpp/opencc/utils.c`
## Verification
- [x] Build passes
- [x] Scanner re-scan confirms fix
- [x] LLM code review passed
---
*Automated security fix by [OrbisAI Security](https://orbisappsec.com)*
## Summary by CodeRabbit
* **Bug Fixes**
* Improved error detection and handling for malformed configuration and
dictionary entries during file parsing.
* Enhanced memory cleanup in error recovery paths to prevent potential
issues.
* Strengthened robustness of string operations and buffer handling
throughout the library.
Co-authored-by: Ubuntu
---
internal/cpp/opencc/config_reader.c | 25 ++++++++++++++++++++-----
internal/cpp/opencc/dictionary/text.c | 21 ++++++++++++++++++---
internal/cpp/opencc/utils.c | 6 ++++--
3 files changed, 42 insertions(+), 10 deletions(-)
diff --git a/internal/cpp/opencc/config_reader.c b/internal/cpp/opencc/config_reader.c
index 06f191e75b0..8271ff48c06 100644
--- a/internal/cpp/opencc/config_reader.c
+++ b/internal/cpp/opencc/config_reader.c
@@ -170,8 +170,9 @@ static char *parse_trim(char *str) {
static int parse(config_desc *config, const char *filename, const char *home_path) {
FILE *fp = fopen(filename, "rb");
if (!fp) {
- char *pkg_filename = (char *)malloc(sizeof(char) * (strlen(filename) + strlen(home_path) + 2));
- sprintf(pkg_filename, "%s/%s", home_path, filename);
+ size_t pkg_filename_len = strlen(filename) + strlen(home_path) + 2;
+ char *pkg_filename = (char *)malloc(sizeof(char) * pkg_filename_len);
+ snprintf(pkg_filename, pkg_filename_len, "%s/%s", home_path, filename);
printf("pkg_filename %s\n", pkg_filename);
fp = fopen(pkg_filename, "rb");
if (!fp) {
@@ -182,12 +183,26 @@ static int parse(config_desc *config, const char *filename, const char *home_pat
free(pkg_filename);
}
- config->home_dir = (char *)malloc(sizeof(char) * (strlen(home_path) + 1));
- sprintf(config->home_dir, "%s", home_path);
+ size_t home_dir_len = strlen(home_path) + 1;
+ config->home_dir = (char *)malloc(sizeof(char) * home_dir_len);
+ snprintf(config->home_dir, home_dir_len, "%s", home_path);
- static char buff[BUFFER_SIZE];
+ char buff[BUFFER_SIZE];
while (fgets(buff, BUFFER_SIZE, fp) != NULL) {
+ /* Detect line truncation: if buffer is full and last char is not newline,
+ * the line was longer than BUFFER_SIZE-1 bytes. Drain the remainder and
+ * treat this as a parse error to avoid processing partial config lines. */
+ size_t buff_len = strlen(buff);
+ if (buff_len == BUFFER_SIZE - 1 && buff[buff_len - 1] != '\n') {
+ int c;
+ while ((c = fgetc(fp)) != '\n' && c != EOF)
+ ;
+ fclose(fp);
+ errnum = CONFIG_ERROR_PARSE;
+ return -1;
+ }
+
char *trimed_buff = parse_trim(buff);
if (*trimed_buff == ';' || *trimed_buff == '#' || *trimed_buff == '\0') {
/* Comment Line or empty line */
diff --git a/internal/cpp/opencc/dictionary/text.c b/internal/cpp/opencc/dictionary/text.c
index 41bcdbb45af..84a65167a93 100644
--- a/internal/cpp/opencc/dictionary/text.c
+++ b/internal/cpp/opencc/dictionary/text.c
@@ -20,7 +20,7 @@
#include "../encoding.h"
#define INITIAL_DICTIONARY_SIZE 1024
-#define ENTRY_BUFF_SIZE 128
+#define ENTRY_BUFF_SIZE 4096
#define ENTRY_WBUFF_SIZE ENTRY_BUFF_SIZE / sizeof(size_t)
struct _text_dictionary {
@@ -69,10 +69,14 @@ int parse_entry(const char *buff, entry *entry_i) {
if (ucs4_buff == (ucs4_t *)-1) {
/* 發生錯誤 回退內存申請 */
ssize_t i;
- for (i = value_i - 1; i >= 0; --i)
+ for (i = value_i - 1; i >= 0; --i) {
free(entry_i->value[i]);
+ entry_i->value[i] = NULL;
+ }
free(entry_i->value);
+ entry_i->value = NULL;
free(entry_i->key);
+ entry_i->key = NULL;
return -1;
}
@@ -95,7 +99,7 @@ dictionary_t dictionary_text_open(const char *filename) {
text_dictionary->lexicon = (entry *)malloc(sizeof(entry) * text_dictionary->entry_count);
text_dictionary->word_buff = NULL;
- static char buff[ENTRY_BUFF_SIZE];
+ char buff[ENTRY_BUFF_SIZE];
FILE *fp = fopen(filename, "rb");
if (fp == NULL) {
@@ -105,6 +109,17 @@ dictionary_t dictionary_text_open(const char *filename) {
size_t i = 0;
while (fgets(buff, ENTRY_BUFF_SIZE, fp)) {
+ /* Detect line truncation: if buffer is full and last char is not newline,
+ * the line was longer than ENTRY_BUFF_SIZE-1 bytes. Drain the remainder
+ * and skip this malformed entry to prevent parsing partial data. */
+ size_t buff_len = strlen(buff);
+ if (buff_len == ENTRY_BUFF_SIZE - 1 && buff[buff_len - 1] != '\n') {
+ int c;
+ while ((c = fgetc(fp)) != '\n' && c != EOF)
+ ;
+ continue;
+ }
+
if (i >= text_dictionary->entry_count) {
text_dictionary->entry_count += text_dictionary->entry_count;
text_dictionary->lexicon = (entry *)realloc(text_dictionary->lexicon, sizeof(entry) * text_dictionary->entry_count);
diff --git a/internal/cpp/opencc/utils.c b/internal/cpp/opencc/utils.c
index 9f93aae8f3f..733b1e5d191 100644
--- a/internal/cpp/opencc/utils.c
+++ b/internal/cpp/opencc/utils.c
@@ -23,8 +23,10 @@ void perr(const char *str) { fputs(str, stderr); }
int qsort_int_cmp(const void *a, const void *b) { return *((int *)a) - *((int *)b); }
char *mstrcpy(const char *str) {
- char *strbuf = (char *)malloc(sizeof(char) * (strlen(str) + 1));
- strcpy(strbuf, str);
+ size_t len = strlen(str);
+ char *strbuf = (char *)malloc(sizeof(char) * (len + 1));
+ strncpy(strbuf, str, len);
+ strbuf[len] = '\0';
return strbuf;
}
From f169ab4b3980c78a26e8da8b8c6ffda258d35b04 Mon Sep 17 00:00:00 2001
From: plind <59729252+plind-junior@users.noreply.github.com>
Date: Mon, 18 May 2026 23:20:40 -0700
Subject: [PATCH 200/666] feat(tts): cache synthesized speech in Redis to avoid
redundant calls (#14851)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## What problem does this PR solve?
Closes #12017.
TTS output is deterministic for a given `(model, text)` pair, so
re-running the same text through the same TTS model produces the same
bytes — yet `Canvas.tts` and `dialog_service.tts` re-synthesized on
every request. That's slow and wastes provider quota whenever the same
assistant response is replayed, shared across users, or repeated within
a session.
### Change
New helper `rag/utils/tts_cache.py` with `synthesize_with_cache(tts_mdl,
cleaned_text)`:
- **Key:** `tts:cache:{model_id}:{sha256(text)}` — separate namespace
per model, identical cleaned text reuses a single entry across both call
sites.
- **Value:** the hex-encoded audio blob both call sites already
returned. No format change for downstream consumers.
- **TTL:** 7 days by default, configurable via
`RAGFLOW_TTS_CACHE_TTL_SECONDS`.
- **Failure modes:** a Redis hiccup falls back to direct synthesis; a
failed synthesis still returns `None` (existing contract preserved).
[`Canvas.tts`](https://github.com/infiniflow/ragflow/blob/main/agent/canvas.py#L683-L724)
and
[`dialog_service.tts`](https://github.com/infiniflow/ragflow/blob/main/api/db/services/dialog_service.py#L1367-L1380)
now route through the helper; the per-file bytes-accumulation/hex-encode
loop has been removed in favor of one shared implementation.
## Type of change
- [x] New Feature (non-breaking change which adds functionality)
## Test plan
- [ ] **Cache hit, chat path:** Configure a dialog with TTS enabled, ask
the same question twice with `stream=false`. Verify the second response
returns the same `audio_binary` and that the second invocation doesn't
hit the TTS provider (e.g., observe provider-side logs / usage counters;
check no `LLMBundle.tts can't update token usage` log line on the second
run).
- [ ] **Cache hit, agent path:** Same exercise via a Conversational
Agent that includes a Message component playing back the answer.
- [ ] **Cache isolation per model:** Switch tenant's `tts_id` between
two models, run the same text against each — confirm the second model's
first synthesis still happens (no cross-model hits).
- [ ] **TTL override:** Set `RAGFLOW_TTS_CACHE_TTL_SECONDS=120`, confirm
the entry expires after 2 minutes.
- [ ] **Redis unavailable:** Stop Redis (or break the connection).
Verify the TTS endpoint still works — synthesis falls back to direct
calls, with a `TTS cache lookup failed` / `TTS cache store failed`
warning logged.
- [ ] **Failure path:** Configure a TTS model with an invalid API key,
ensure the response still returns successfully with `audio_binary=None`
(no regression vs. current behavior).
---
agent/canvas.py | 11 +--
api/db/services/dialog_service.py | 11 +--
rag/utils/tts_cache.py | 120 ++++++++++++++++++++++++++++++
3 files changed, 124 insertions(+), 18 deletions(-)
create mode 100644 rag/utils/tts_cache.py
diff --git a/agent/canvas.py b/agent/canvas.py
index bbd06facbb9..3421d207ed2 100644
--- a/agent/canvas.py
+++ b/agent/canvas.py
@@ -17,7 +17,6 @@
import base64
import datetime
import inspect
-import binascii
import json
import logging
import re
@@ -39,6 +38,7 @@
from common.exceptions import TaskCanceledException
from rag.prompts.generator import chunks_format
from rag.utils.redis_conn import REDIS_CONN
+from rag.utils.tts_cache import synthesize_with_cache
class Graph:
"""
@@ -714,14 +714,7 @@ def clean_tts_text(text: str) -> str:
text = clean_tts_text(text)
if not text:
return None
- bin = b""
- try:
- for chunk in tts_mdl.tts(text):
- bin += chunk
- except Exception as e:
- logging.error(f"TTS failed: {e}, text={text!r}")
- return None
- return binascii.hexlify(bin).decode("utf-8")
+ return synthesize_with_cache(tts_mdl, text)
def get_history(self, window_size):
convs = []
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py
index f7a5befc3f3..aa6d2550978 100644
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -14,7 +14,6 @@
# limitations under the License.
#
import asyncio
-import binascii
import logging
import re
import time
@@ -51,6 +50,7 @@
from rag.prompts.generator import chunks_format, citation_prompt, cross_languages, full_question, kb_prompt, keyword_extraction, message_fit_in, PROMPT_JINJA_ENV, ASK_SUMMARY
from common.token_utils import num_tokens_from_string
from rag.utils.tavily_conn import Tavily
+from rag.utils.tts_cache import synthesize_with_cache
from common.string_utils import remove_redundant_spaces
from common import settings
@@ -1427,14 +1427,7 @@ def tts(tts_mdl, text):
text = clean_tts_text(text)
if not text:
return None
- bin = b""
- try:
- for chunk in tts_mdl.tts(text):
- bin += chunk
- except Exception as e:
- logging.error(f"TTS failed: {e}, text={text!r}")
- return None
- return binascii.hexlify(bin).decode("utf-8")
+ return synthesize_with_cache(tts_mdl, text)
class _ThinkStreamState:
diff --git a/rag/utils/tts_cache.py b/rag/utils/tts_cache.py
new file mode 100644
index 00000000000..a96f1925288
--- /dev/null
+++ b/rag/utils/tts_cache.py
@@ -0,0 +1,120 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import binascii
+import hashlib
+import logging
+import os
+from typing import Any, Optional
+
+from rag.utils.redis_conn import REDIS_CONN
+
+_DEFAULT_TTL_SECONDS = 7 * 24 * 60 * 60
+_KEY_PREFIX = "tts:cache:"
+
+
+def _ttl_seconds() -> int:
+ raw = os.environ.get("RAGFLOW_TTS_CACHE_TTL_SECONDS")
+ if not raw:
+ return _DEFAULT_TTL_SECONDS
+ try:
+ v = int(raw)
+ return v if v > 0 else 0
+ except ValueError:
+ logging.warning("Invalid RAGFLOW_TTS_CACHE_TTL_SECONDS=%r, using default", raw)
+ return _DEFAULT_TTL_SECONDS
+
+
+def _model_id(tts_mdl: Any) -> Optional[str]:
+ cfg = getattr(tts_mdl, "model_config", None)
+ if isinstance(cfg, dict):
+ mid = cfg.get("id")
+ if mid is not None:
+ return str(mid)
+ name = cfg.get("llm_name") or cfg.get("model_name")
+ if name:
+ return str(name)
+ return None
+
+
+def _build_key(tts_mdl: Any, text: str) -> Optional[str]:
+ mid = _model_id(tts_mdl)
+ if not mid:
+ return None
+ digest = hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()
+ return f"{_KEY_PREFIX}{mid}:{digest}"
+
+
+def _to_hex_string(value: Any) -> Optional[str]:
+ if value is None:
+ return None
+ if isinstance(value, bytes):
+ try:
+ return value.decode("utf-8")
+ except Exception:
+ return None
+ if isinstance(value, str):
+ return value
+ return None
+
+
+def synthesize_with_cache(tts_mdl: Any, cleaned_text: str) -> Optional[str]:
+ """
+ Synthesize ``cleaned_text`` through ``tts_mdl`` and return a hex-encoded
+ audio blob, reusing a Redis-cached result when available.
+
+ The cache key is derived from the TTS model identifier and a SHA-256 of the
+ text, so different models keep separate caches and the same text on the
+ same model resolves to the same key regardless of call site. Returns
+ ``None`` on synthesis failure; callers should treat that as a no-op the
+ same way they do today.
+ """
+ if not tts_mdl or not cleaned_text:
+ return None
+
+ key = _build_key(tts_mdl, cleaned_text)
+
+ if key:
+ try:
+ cached = REDIS_CONN.get(key)
+ except Exception as e:
+ logging.warning("TTS cache lookup failed: %s", e)
+ cached = None
+ hex_cached = _to_hex_string(cached)
+ if hex_cached:
+ return hex_cached
+
+ buf = b""
+ try:
+ for chunk in tts_mdl.tts(cleaned_text):
+ if isinstance(chunk, (bytes, bytearray)):
+ buf += bytes(chunk)
+ except Exception as e:
+ logging.error("TTS failed: %s (text length=%d)", e, len(cleaned_text))
+ return None
+
+ if not buf:
+ return None
+
+ hex_value = binascii.hexlify(buf).decode("utf-8")
+
+ ttl = _ttl_seconds()
+ if key and ttl > 0:
+ try:
+ REDIS_CONN.set(key, hex_value, exp=ttl)
+ except Exception as e:
+ logging.warning("TTS cache store failed: %s", e)
+
+ return hex_value
From 7edabdf7c35ad3689846ec9d89e70d0de06ffe90 Mon Sep 17 00:00:00 2001
From: plind <59729252+plind-junior@users.noreply.github.com>
Date: Tue, 19 May 2026 00:08:31 -0700
Subject: [PATCH 201/666] fix(retrieval): keep manual metadata filter reusable
inside Iteration (#14849)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## What problem does this PR solve?
Closes #12582.
When a Retrieval component sits inside an Iteration with a **manual**
metadata filter that references the iteration variable (e.g.
`{IterationItem:abc@item}`), every iteration reuses the value resolved
on the **first** pass.
Root cause: [`_resolve_manual_filter` in
`agent/tools/retrieval.py`](https://github.com/infiniflow/ragflow/blob/main/agent/tools/retrieval.py#L144-L171)
mutated `flt["value"]` in place. The `filters` list passed in is the
live `self._param.meta_data_filter["manual"]` (see
[`apply_meta_data_filter` in
`common/metadata_utils.py:257-261`](https://github.com/infiniflow/ragflow/blob/main/common/metadata_utils.py#L257-L261)),
so after the first iteration the param dict permanently held the
resolved string instead of the original variable reference.
```text
iter #1: flt["value"] = "{IterationItem:abc@item}" → resolved to "AI"
after mutation: flt["value"] = "AI" ← written back into _param
iter #2: flt["value"] = "AI" ← no {…} matches
retrieval keeps filtering by "AI" forever
```
This PR returns a shallow copy with the resolved value instead, leaving
the original filter (and its variable reference) intact for the next
iteration.
## Type of change
- [x] Bug fix (non-breaking change which fixes an issue)
## Test plan
- [ ] Build an agent: `Agent (structured output → list of areas) →
Iteration → Retrieval (manual filter: Area = {IterationItem/Item}) →
Message`. Run with a multi-area query and confirm each iteration's
Retrieval result matches its own item, not the first item.
- [ ] Regression: Retrieval with a manual metadata filter outside an
Iteration still resolves the variable correctly on each request.
- [ ] Regression: Retrieval with no metadata filter and with `auto` /
`semi_auto` filters behave unchanged.
---
agent/tools/retrieval.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/agent/tools/retrieval.py b/agent/tools/retrieval.py
index 0ccf056a292..02cb3e2ce6d 100644
--- a/agent/tools/retrieval.py
+++ b/agent/tools/retrieval.py
@@ -142,6 +142,11 @@ def _load_metas() -> dict:
return DocMetadataService.get_flatted_meta_by_kbs(kb_ids)
def _resolve_manual_filter(flt: dict) -> dict:
+ # Return a new dict instead of mutating `flt` in place. The
+ # caller passes filters straight out of self._param.meta_data_filter,
+ # so mutating them would replace the variable reference with its
+ # resolved value and every subsequent invocation (e.g. inside an
+ # Iteration component) would reuse that stale value.
pat = re.compile(self.variable_ref_patt)
s = flt.get("value", "")
out_parts = []
@@ -167,8 +172,9 @@ def _resolve_manual_filter(flt: dict) -> dict:
last = m.end()
out_parts.append(s[last:])
- flt["value"] = "".join(out_parts)
- return flt
+ resolved = dict(flt)
+ resolved["value"] = "".join(out_parts)
+ return resolved
chat_mdl = None
if self._param.meta_data_filter.get("method") in ["auto", "semi_auto"]:
From 09a06f1b00fa24116f1e90af1ac40a90fd4414b0 Mon Sep 17 00:00:00 2001
From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com>
Date: Mon, 18 May 2026 21:10:13 -1000
Subject: [PATCH 202/666] Go: implement provider: Xinference (#14938)
### What problem does this PR solve?
Closes #14808.
Adds a Go model driver for Xinference so self-hosted Xinference chat
models can be used through the Go provider layer instead of falling
through to the dummy driver. Xinference exposes an OpenAI-compatible API
under `/v1`; the driver accepts either a root endpoint such as
`http://127.0.0.1:9997` or an OpenAI-compatible endpoint such as
`http://127.0.0.1:9997/v1` and normalizes it before calling chat or
model-listing routes.
### What is changed?
- Add `internal/entity/models/xinference.go` implementing `ModelDriver`
for Xinference chat.
- Route provider name `xinference` in
`internal/entity/models/factory.go`.
- Add `conf/models/xinference.json` as a local provider config.
- Add focused unit tests in `internal/entity/models/xinference_test.go`.
Initial method coverage:
- `ChatWithMessages`: POST `/v1/chat/completions`.
- `ChatStreamlyWithSender`: SSE streaming from `/v1/chat/completions`.
- `ListModels`: GET `/v1/models`.
- `CheckConnection`: lightweight `ListModels` probe.
- Optional auth: send `Authorization: Bearer ` only when a
non-empty key is configured, matching Xinference no-auth and
auth-enabled deployments.
- `Balance`, `Embed`, `Rerank`, ASR, TTS, and OCR return `no such
method` for this initial chat-provider PR.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- [x] Bug Fix (non-breaking change which fixes an issue)
### Tests
- `go test -vet=off -run TestXinference -count=1
./internal/entity/models/...`
- `go test -vet=off -count=1 ./internal/entity/models/...`
### References
- Xinference docs:
https://inference.readthedocs.io/zh-cn/latest/index.html
- OpenAI-compatible chat usage:
https://inference.readthedocs.io/zh-cn/latest/getting_started/using_xinference.html
- API key auth:
https://inference.readthedocs.io/zh-cn/latest/user_guide/auth_system.html
---------
Co-authored-by: Jin Hai
---
conf/models/xinference.json | 8 +
internal/entity/models/factory.go | 2 +
internal/entity/models/xinference.go | 476 ++++++++++++++++++++++
internal/entity/models/xinference_test.go | 313 ++++++++++++++
4 files changed, 799 insertions(+)
create mode 100644 conf/models/xinference.json
create mode 100644 internal/entity/models/xinference.go
create mode 100644 internal/entity/models/xinference_test.go
diff --git a/conf/models/xinference.json b/conf/models/xinference.json
new file mode 100644
index 00000000000..cf50dbc7313
--- /dev/null
+++ b/conf/models/xinference.json
@@ -0,0 +1,8 @@
+{
+ "name": "xinference",
+ "url_suffix": {
+ "chat": "v1/chat/completions",
+ "models": "v1/models"
+ },
+ "class": "local"
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index d8ccc9f7661..88cb3338b86 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -89,6 +89,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewJinaModel(baseURL, urlSuffix), nil
case "localai":
return NewLocalAIModel(baseURL, urlSuffix), nil
+ case "xinference":
+ return NewXinferenceModel(baseURL, urlSuffix), nil
case "longcat":
return NewLongCatModel(baseURL, urlSuffix), nil
case "novita":
diff --git a/internal/entity/models/xinference.go b/internal/entity/models/xinference.go
new file mode 100644
index 00000000000..d8f8fa39f54
--- /dev/null
+++ b/internal/entity/models/xinference.go
@@ -0,0 +1,476 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+)
+
+// xinferenceStreamIdleTimeout bounds how long a stream can go without
+// receiving any SSE line. Self-hosted models can be slow, but a stream
+// that stays silent for a full minute is more useful as a surfaced error
+// than as a stuck goroutine.
+var xinferenceStreamIdleTimeout = 60 * time.Second
+
+// XinferenceModel implements ModelDriver for Xinference chat models.
+//
+// Xinference exposes an OpenAI-compatible API under /v1. The
+// tenant may configure either the root endpoint (http://127.0.0.1:9997)
+// or the OpenAI-compatible endpoint (http://127.0.0.1:9997/v1); the
+// driver normalizes both to the root endpoint before adding URLSuffix
+// values that match Xinference docs, such as v1/chat/completions.
+// Authentication is optional: no-auth deployments ignore API keys, while
+// auth-enabled deployments require Authorization: Bearer .
+type XinferenceModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+type xinferenceChatChoice struct {
+ Message struct {
+ Content string `json:"content"`
+ ReasoningContent string `json:"reasoning_content"`
+ Reasoning string `json:"reasoning"`
+ Thinking string `json:"thinking"`
+ } `json:"message"`
+}
+
+type xinferenceChatResponse struct {
+ Choices []xinferenceChatChoice `json:"choices"`
+}
+
+type xinferenceModelListResponse struct {
+ Data []struct {
+ ID string `json:"id"`
+ } `json:"data"`
+}
+
+// NewXinferenceModel creates a new Xinference model instance.
+func NewXinferenceModel(baseURL map[string]string, urlSuffix URLSuffix) *XinferenceModel {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxIdleConns = 100
+ transport.MaxIdleConnsPerHost = 10
+ transport.IdleConnTimeout = 90 * time.Second
+ transport.DisableCompression = false
+ transport.ResponseHeaderTimeout = 60 * time.Second
+
+ return &XinferenceModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Transport: transport,
+ },
+ }
+}
+
+func (x *XinferenceModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return NewXinferenceModel(baseURL, x.URLSuffix)
+}
+
+func (x *XinferenceModel) Name() string {
+ return "xinference"
+}
+
+func (x *XinferenceModel) baseURLForRegion(region string) (string, error) {
+ if base, ok := x.BaseURL[region]; ok && strings.TrimSpace(base) != "" {
+ return normalizeXinferenceBaseURL(base), nil
+ }
+ if base, ok := x.BaseURL["default"]; ok && strings.TrimSpace(base) != "" {
+ return normalizeXinferenceBaseURL(base), nil
+ }
+ return "", fmt.Errorf("xinference: missing base URL, configure the Xinference endpoint (e.g., http://127.0.0.1:9997 or http://127.0.0.1:9997/v1)")
+}
+
+func normalizeXinferenceBaseURL(base string) string {
+ trimmed := strings.TrimRight(strings.TrimSpace(base), "/")
+ if trimmed == "" {
+ return trimmed
+ }
+ if strings.HasSuffix(trimmed, "/v1") {
+ return strings.TrimSuffix(trimmed, "/v1")
+ }
+ return trimmed
+}
+
+func setXinferenceAuth(req *http.Request, apiConfig *APIConfig) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return
+ }
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+}
+
+func xinferenceRegion(apiConfig *APIConfig) string {
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ return *apiConfig.Region
+ }
+ return "default"
+}
+
+func xinferenceReasoningFromStrings(reasoningContent string, reasoning string, thinking string) string {
+ switch {
+ case reasoningContent != "":
+ return reasoningContent
+ case reasoning != "":
+ return reasoning
+ case thinking != "":
+ return thinking
+ default:
+ return ""
+ }
+}
+
+func xinferenceReasoningFromMap(value map[string]interface{}) string {
+ for _, field := range []string{"reasoning_content", "reasoning", "thinking"} {
+ if text, ok := value[field].(string); ok && text != "" {
+ return text
+ }
+ }
+ return ""
+}
+
+func buildXinferenceChatBody(modelName string, messages []Message, stream bool, chatModelConfig *ChatConfig) map[string]interface{} {
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": stream,
+ }
+
+ if chatModelConfig != nil {
+ if chatModelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *chatModelConfig.MaxTokens
+ }
+ if chatModelConfig.Temperature != nil {
+ reqBody["temperature"] = *chatModelConfig.Temperature
+ }
+ if chatModelConfig.TopP != nil {
+ reqBody["top_p"] = *chatModelConfig.TopP
+ }
+ if chatModelConfig.Stop != nil {
+ reqBody["stop"] = *chatModelConfig.Stop
+ }
+ }
+
+ return reqBody
+}
+
+// ChatWithMessages sends multiple messages with roles and returns the response.
+func (x *XinferenceModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ baseURL, err := x.baseURLForRegion(xinferenceRegion(apiConfig))
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, x.URLSuffix.Chat)
+
+ reqBody := buildXinferenceChatBody(modelName, messages, false, chatModelConfig)
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ setXinferenceAuth(req, apiConfig)
+
+ resp, err := x.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var result xinferenceChatResponse
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+ if len(result.Choices) == 0 {
+ return nil, fmt.Errorf("no choices in response")
+ }
+
+ content := result.Choices[0].Message.Content
+ reasonContent := xinferenceReasoningFromStrings(
+ result.Choices[0].Message.ReasoningContent,
+ result.Choices[0].Message.Reasoning,
+ result.Choices[0].Message.Thinking,
+ )
+
+ return &ChatResponse{
+ Answer: &content,
+ ReasonContent: &reasonContent,
+ }, nil
+}
+
+// ChatStreamlyWithSender sends messages and streams response via sender.
+func (x *XinferenceModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
+ if sender == nil {
+ return fmt.Errorf("sender is required")
+ }
+ if len(messages) == 0 {
+ return fmt.Errorf("messages is empty")
+ }
+ if chatModelConfig != nil && chatModelConfig.Stream != nil && !*chatModelConfig.Stream {
+ return fmt.Errorf("stream must be true in ChatStreamlyWithSender")
+ }
+
+ baseURL, err := x.baseURLForRegion(xinferenceRegion(apiConfig))
+ if err != nil {
+ return err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, x.URLSuffix.Chat)
+
+ reqBody := buildXinferenceChatBody(modelName, messages, true, chatModelConfig)
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ setXinferenceAuth(req, apiConfig)
+
+ resp, err := x.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ lastActive := time.Now()
+ var lastActiveMu sync.Mutex
+ done := make(chan struct{})
+ defer close(done)
+ go func() {
+ ticker := time.NewTicker(xinferenceStreamIdleTimeout / 4)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-done:
+ return
+ case now := <-ticker.C:
+ lastActiveMu.Lock()
+ idle := now.Sub(lastActive)
+ lastActiveMu.Unlock()
+ if idle >= xinferenceStreamIdleTimeout {
+ cancel()
+ return
+ }
+ }
+ }
+ }()
+
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+ sawTerminal := false
+ for scanner.Scan() {
+ lastActiveMu.Lock()
+ lastActive = time.Now()
+ lastActiveMu.Unlock()
+
+ line := scanner.Text()
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+ data := strings.TrimSpace(line[5:])
+ if data == "[DONE]" {
+ sawTerminal = true
+ break
+ }
+
+ var event map[string]interface{}
+ if err = json.Unmarshal([]byte(data), &event); err != nil {
+ continue
+ }
+
+ choices, ok := event["choices"].([]interface{})
+ if !ok || len(choices) == 0 {
+ continue
+ }
+ firstChoice, ok := choices[0].(map[string]interface{})
+ if !ok {
+ continue
+ }
+
+ if delta, ok := firstChoice["delta"].(map[string]interface{}); ok {
+ if reasoning := xinferenceReasoningFromMap(delta); reasoning != "" {
+ if err := sender(nil, &reasoning); err != nil {
+ return err
+ }
+ }
+ if content, ok := delta["content"].(string); ok && content != "" {
+ if err := sender(&content, nil); err != nil {
+ return err
+ }
+ }
+ }
+
+ if finishReason, ok := firstChoice["finish_reason"].(string); ok && finishReason != "" {
+ sawTerminal = true
+ break
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ if ctx.Err() != nil {
+ return fmt.Errorf("xinference: stream idle for more than %s, aborted", xinferenceStreamIdleTimeout)
+ }
+ return fmt.Errorf("failed to scan response body: %w", err)
+ }
+ if !sawTerminal {
+ return fmt.Errorf("xinference: stream ended before [DONE] or finish_reason")
+ }
+
+ endOfStream := "[DONE]"
+ return sender(&endOfStream, nil)
+}
+
+func (x *XinferenceModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+// ListModels returns the model IDs exposed by Xinference's OpenAI-compatible
+// /v1/models endpoint.
+func (x *XinferenceModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ baseURL, err := x.baseURLForRegion(xinferenceRegion(apiConfig))
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, x.URLSuffix.Models)
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ setXinferenceAuth(req, apiConfig)
+
+ resp, err := x.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var result xinferenceModelListResponse
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ models := make([]string, 0, len(result.Data))
+ for _, model := range result.Data {
+ if model.ID != "" {
+ models = append(models, model.ID)
+ }
+ }
+ return models, nil
+}
+
+func (x *XinferenceModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) CheckConnection(apiConfig *APIConfig) error {
+ _, err := x.ListModels(apiConfig)
+ return err
+}
+
+func (x *XinferenceModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
+
+func (x *XinferenceModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", x.Name())
+}
diff --git a/internal/entity/models/xinference_test.go b/internal/entity/models/xinference_test.go
new file mode 100644
index 00000000000..af3179ea0ed
--- /dev/null
+++ b/internal/entity/models/xinference_test.go
@@ -0,0 +1,313 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+func newXinferenceForTest(baseURL string) *XinferenceModel {
+ return NewXinferenceModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{
+ Chat: "v1/chat/completions",
+ Models: "v1/models",
+ },
+ )
+}
+
+func withXinferenceIdleTimeout(t *testing.T, d time.Duration) {
+ t.Helper()
+ original := xinferenceStreamIdleTimeout
+ xinferenceStreamIdleTimeout = d
+ t.Cleanup(func() {
+ xinferenceStreamIdleTimeout = original
+ })
+}
+
+func TestXinferenceName(t *testing.T) {
+ x := newXinferenceForTest("http://unused")
+ if got := x.Name(); got != "xinference" {
+ t.Errorf("Name()=%q, want %q", got, "xinference")
+ }
+}
+
+func TestNormalizeXinferenceBaseURL(t *testing.T) {
+ cases := []struct {
+ in string
+ want string
+ }{
+ {"http://127.0.0.1:9997", "http://127.0.0.1:9997"},
+ {"http://127.0.0.1:9997/", "http://127.0.0.1:9997"},
+ {"http://127.0.0.1:9997/v1", "http://127.0.0.1:9997"},
+ {" http://127.0.0.1:9997/v1/ ", "http://127.0.0.1:9997"},
+ }
+ for _, tc := range cases {
+ if got := normalizeXinferenceBaseURL(tc.in); got != tc.want {
+ t.Errorf("normalizeXinferenceBaseURL(%q)=%q, want %q", tc.in, got, tc.want)
+ }
+ }
+}
+
+func TestXinferenceFactoryRoute(t *testing.T) {
+ driver, err := NewModelFactory().CreateModelDriver("xinference", map[string]string{"default": "http://unused"}, URLSuffix{})
+ if err != nil {
+ t.Fatalf("CreateModelDriver: %v", err)
+ }
+ if driver.Name() != "xinference" {
+ t.Errorf("driver.Name()=%q, want xinference", driver.Name())
+ }
+}
+
+func TestXinferenceChatHappyPathNormalizesBaseURLAndOmitsEmptyAuth(t *testing.T) {
+ var seen map[string]interface{}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/chat/completions" {
+ t.Errorf("path=%s, want /v1/chat/completions", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "" {
+ t.Errorf("expected no Authorization header, got %q", got)
+ }
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read body: %v", err)
+ return
+ }
+ if err := json.Unmarshal(raw, &seen); err != nil {
+ t.Errorf("unmarshal request: %v", err)
+ return
+ }
+ _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"pong"}}]}`)
+ }))
+ defer srv.Close()
+
+ x := newXinferenceForTest(srv.URL)
+ maxTokens := 32
+ temp := 0.2
+ resp, err := x.ChatWithMessages("qwen2.5-instruct",
+ []Message{{Role: "user", Content: "ping"}},
+ &APIConfig{},
+ &ChatConfig{MaxTokens: &maxTokens, Temperature: &temp})
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if resp.Answer == nil || *resp.Answer != "pong" {
+ t.Fatalf("Answer=%v, want pong", resp.Answer)
+ }
+ if seen["stream"] != false {
+ t.Errorf("stream=%v, want false", seen["stream"])
+ }
+ if seen["max_tokens"] != float64(32) {
+ t.Errorf("max_tokens=%v, want 32", seen["max_tokens"])
+ }
+ if seen["temperature"] != 0.2 {
+ t.Errorf("temperature=%v, want 0.2", seen["temperature"])
+ }
+}
+
+func TestXinferenceChatSendsAuthHeaderWhenKeyProvided(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if got := r.Header.Get("Authorization"); got != "Bearer sk-test" {
+ t.Errorf("Authorization=%q, want Bearer sk-test", got)
+ }
+ _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"ok"}}]}`)
+ }))
+ defer srv.Close()
+
+ x := newXinferenceForTest(srv.URL + "/v1")
+ key := "sk-test"
+ _, err := x.ChatWithMessages("qwen2.5-instruct",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{ApiKey: &key}, nil)
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+}
+
+func TestXinferenceChatExtractsReasoningFields(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = io.WriteString(w, `{"choices":[{"message":{
+ "content":"12",
+ "reasoning_content":"0.15 * 80 = 12"
+ }}]}`)
+ }))
+ defer srv.Close()
+
+ x := newXinferenceForTest(srv.URL)
+ resp, err := x.ChatWithMessages("qwen3",
+ []Message{{Role: "user", Content: "15% of 80?"}},
+ &APIConfig{}, nil)
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if resp.ReasonContent == nil || *resp.ReasonContent != "0.15 * 80 = 12" {
+ t.Errorf("ReasonContent=%v", resp.ReasonContent)
+ }
+}
+
+func TestXinferenceStreamHappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/chat/completions" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ var seen map[string]interface{}
+ raw, _ := io.ReadAll(r.Body)
+ _ = json.Unmarshal(raw, &seen)
+ if seen["stream"] != true {
+ t.Errorf("stream=%v, want true", seen["stream"])
+ }
+ w.Header().Set("Content-Type", "text/event-stream")
+ _, _ = io.WriteString(w,
+ `data: {"choices":[{"delta":{"reasoning_content":"step. "}}]}`+"\n"+
+ `data: {"choices":[{"delta":{"content":"Hello"}}]}`+"\n"+
+ `data: {"choices":[{"delta":{"content":" world"},"finish_reason":"stop"}]}`+"\n"+
+ `data: [DONE]`+"\n",
+ )
+ }))
+ defer srv.Close()
+
+ x := newXinferenceForTest(srv.URL)
+ var content []string
+ var reasoning []string
+ var sawDone bool
+ err := x.ChatStreamlyWithSender("qwen2.5-instruct",
+ []Message{{Role: "user", Content: "hi"}},
+ &APIConfig{}, nil,
+ func(c *string, r *string) error {
+ if r != nil && *r != "" {
+ reasoning = append(reasoning, *r)
+ }
+ if c != nil && *c == "[DONE]" {
+ sawDone = true
+ }
+ if c != nil && *c != "" && *c != "[DONE]" {
+ content = append(content, *c)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("ChatStreamlyWithSender: %v", err)
+ }
+ if strings.Join(reasoning, "") != "step. " {
+ t.Errorf("reasoning=%q", strings.Join(reasoning, ""))
+ }
+ if strings.Join(content, "") != "Hello world" {
+ t.Errorf("content=%q", strings.Join(content, ""))
+ }
+ if !sawDone {
+ t.Error("expected [DONE] callback")
+ }
+}
+
+func TestXinferenceStreamRejectsFalseStreamConfig(t *testing.T) {
+ x := newXinferenceForTest("http://unused")
+ stream := false
+ err := x.ChatStreamlyWithSender("qwen2.5-instruct",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{},
+ &ChatConfig{Stream: &stream},
+ func(*string, *string) error { return nil })
+ if err == nil || !strings.Contains(err.Error(), "stream must be true") {
+ t.Errorf("expected stream-must-be-true error, got %v", err)
+ }
+}
+
+func TestXinferenceStreamCancelsOnIdle(t *testing.T) {
+ withXinferenceIdleTimeout(t, 200*time.Millisecond)
+
+ hold := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ w.WriteHeader(http.StatusOK)
+ if f, ok := w.(http.Flusher); ok {
+ _, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"hi"}}]}`+"\n")
+ f.Flush()
+ }
+ select {
+ case <-hold:
+ case <-r.Context().Done():
+ }
+ }))
+ t.Cleanup(srv.Close)
+ t.Cleanup(func() { close(hold) })
+
+ x := newXinferenceForTest(srv.URL)
+ err := x.ChatStreamlyWithSender("qwen2.5-instruct",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{}, nil,
+ func(*string, *string) error { return nil })
+ if err == nil || !strings.Contains(err.Error(), "stream idle") {
+ t.Errorf("expected stream-idle error, got %v", err)
+ }
+}
+
+func TestXinferenceListModelsAndCheckConnection(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/models" {
+ t.Errorf("path=%s, want /v1/models", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer sk-test" {
+ t.Errorf("Authorization=%q, want Bearer sk-test", got)
+ }
+ _, _ = io.WriteString(w, `{"object":"list","data":[{"id":"qwen2.5-instruct"},{"id":"custom-chat"}]}`)
+ }))
+ defer srv.Close()
+
+ x := newXinferenceForTest(srv.URL)
+ key := "sk-test"
+ apiConfig := &APIConfig{ApiKey: &key}
+ models, err := x.ListModels(apiConfig)
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ if strings.Join(models, ",") != "qwen2.5-instruct,custom-chat" {
+ t.Errorf("models=%v", models)
+ }
+ if err := x.CheckConnection(apiConfig); err != nil {
+ t.Fatalf("CheckConnection: %v", err)
+ }
+}
+
+func TestXinferenceMissingBaseURLFailsClearly(t *testing.T) {
+ x := NewXinferenceModel(map[string]string{}, URLSuffix{Chat: "v1/chat/completions"})
+ _, err := x.ChatWithMessages("qwen2.5-instruct",
+ []Message{{Role: "user", Content: "x"}},
+ &APIConfig{}, nil)
+ if err == nil || !strings.Contains(err.Error(), "missing base URL") {
+ t.Errorf("expected missing-base-URL error, got %v", err)
+ }
+}
+
+func TestXinferenceUnsupportedMethodsReturnNoSuchMethod(t *testing.T) {
+ x := newXinferenceForTest("http://unused")
+ model := "qwen2.5-instruct"
+
+ if _, err := x.Embed(&model, []string{"x"}, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Embed: expected no such method, got %v", err)
+ }
+ if _, err := x.Rerank(&model, "q", []string{"d"}, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Rerank: expected no such method, got %v", err)
+ }
+ if _, err := x.Balance(&APIConfig{}); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Balance: expected no such method, got %v", err)
+ }
+ if _, err := x.TranscribeAudio(&model, nil, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("TranscribeAudio: expected no such method, got %v", err)
+ }
+ if err := x.TranscribeAudioWithSender(&model, nil, &APIConfig{}, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("TranscribeAudioWithSender: expected no such method, got %v", err)
+ }
+ if _, err := x.AudioSpeech(&model, nil, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("AudioSpeech: expected no such method, got %v", err)
+ }
+ if err := x.AudioSpeechWithSender(&model, nil, &APIConfig{}, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("AudioSpeechWithSender: expected no such method, got %v", err)
+ }
+ if _, err := x.OCRFile(&model, nil, nil, &APIConfig{}, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("OCRFile: expected no such method, got %v", err)
+ }
+}
From 243d9ed2819befd79bf5767256bd5857c4e42809 Mon Sep 17 00:00:00 2001
From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com>
Date: Mon, 18 May 2026 21:10:42 -1000
Subject: [PATCH 203/666] Add TogetherAI chat provider (#14957)
## What
- Add TogetherAI as a chat provider backed by its OpenAI-compatible
`/v1/chat/completions` API
- Register TogetherAI in the Go model factory and provider config
- Support non-streaming chat, SSE streaming chat, model listing, and
connection checks
## Notes
- Uses the current TogetherAI OpenAI-compatible base URL
`https://api.together.ai/v1`
- Forwards documented chat parameters from `ChatConfig`: `max_tokens`,
`temperature`, `top_p`, `stop`, and GPT-OSS `reasoning_effort`
- Routes Together reasoning traces from `reasoning` /
`reasoning_content` into `ReasonContent`
## Tests
- `go test -vet=off -run TestTogetherAI -count=1
./internal/entity/models`
- `go test -vet=off -count=1 ./internal/entity/models`
Refs #14736
---
conf/models/togetherai.json | 34 ++
internal/entity/models/factory.go | 2 +
internal/entity/models/togetherai.go | 430 ++++++++++++++++++++++
internal/entity/models/togetherai_test.go | 277 ++++++++++++++
4 files changed, 743 insertions(+)
create mode 100644 conf/models/togetherai.json
create mode 100644 internal/entity/models/togetherai.go
create mode 100644 internal/entity/models/togetherai_test.go
diff --git a/conf/models/togetherai.json b/conf/models/togetherai.json
new file mode 100644
index 00000000000..f8660e059bf
--- /dev/null
+++ b/conf/models/togetherai.json
@@ -0,0 +1,34 @@
+{
+ "name": "TogetherAI",
+ "url": {
+ "default": "https://api.together.ai/v1"
+ },
+ "url_suffix": {
+ "chat": "chat/completions",
+ "models": "models"
+ },
+ "class": "together",
+ "models": [
+ {
+ "name": "openai/gpt-oss-20b",
+ "max_tokens": 131072,
+ "model_types": [
+ "chat"
+ ]
+ },
+ {
+ "name": "meta-llama/Llama-3.3-70B-Instruct-Turbo",
+ "max_tokens": 131072,
+ "model_types": [
+ "chat"
+ ]
+ },
+ {
+ "name": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8",
+ "max_tokens": 262144,
+ "model_types": [
+ "chat"
+ ]
+ }
+ ]
+}
diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go
index 88cb3338b86..b4bcbbe1ab3 100644
--- a/internal/entity/models/factory.go
+++ b/internal/entity/models/factory.go
@@ -97,6 +97,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewNovitaModel(baseURL, urlSuffix), nil
case "replicate":
return NewReplicateModel(baseURL, urlSuffix), nil
+ case "togetherai":
+ return NewTogetherAIModel(baseURL, urlSuffix), nil
case "voyage":
return NewVoyageModel(baseURL, urlSuffix), nil
case "paddleocr":
diff --git a/internal/entity/models/togetherai.go b/internal/entity/models/togetherai.go
new file mode 100644
index 00000000000..7e9219180e9
--- /dev/null
+++ b/internal/entity/models/togetherai.go
@@ -0,0 +1,430 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+)
+
+type TogetherAIModel struct {
+ BaseURL map[string]string
+ URLSuffix URLSuffix
+ httpClient *http.Client
+}
+
+func NewTogetherAIModel(baseURL map[string]string, urlSuffix URLSuffix) *TogetherAIModel {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.MaxIdleConns = 100
+ transport.MaxIdleConnsPerHost = 10
+ transport.IdleConnTimeout = 90 * time.Second
+ transport.DisableCompression = false
+ transport.ResponseHeaderTimeout = 60 * time.Second
+
+ return &TogetherAIModel{
+ BaseURL: baseURL,
+ URLSuffix: urlSuffix,
+ httpClient: &http.Client{
+ Transport: transport,
+ },
+ }
+}
+
+func (t *TogetherAIModel) NewInstance(baseURL map[string]string) ModelDriver {
+ return NewTogetherAIModel(baseURL, t.URLSuffix)
+}
+
+func (t *TogetherAIModel) Name() string {
+ return "togetherai"
+}
+
+func (t *TogetherAIModel) baseURLForRegion(region string) (string, error) {
+ base, ok := t.BaseURL[region]
+ if !ok || base == "" {
+ return "", fmt.Errorf("togetherai: no base URL configured for region %q", region)
+ }
+ return strings.TrimSuffix(base, "/"), nil
+}
+
+type togetherAIReasoningOptions struct {
+ Enabled bool `json:"enabled"`
+}
+
+func (t *TogetherAIModel) chatPayload(modelName string, messages []Message, stream bool, chatModelConfig *ChatConfig) map[string]interface{} {
+ apiMessages := make([]map[string]interface{}, len(messages))
+ for i, msg := range messages {
+ apiMessages[i] = map[string]interface{}{
+ "role": msg.Role,
+ "content": msg.Content,
+ }
+ }
+
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "messages": apiMessages,
+ "stream": stream,
+ }
+
+ if chatModelConfig != nil {
+ if chatModelConfig.MaxTokens != nil {
+ reqBody["max_tokens"] = *chatModelConfig.MaxTokens
+ }
+ if chatModelConfig.Temperature != nil {
+ reqBody["temperature"] = *chatModelConfig.Temperature
+ }
+ if chatModelConfig.TopP != nil {
+ reqBody["top_p"] = *chatModelConfig.TopP
+ }
+ if chatModelConfig.Stop != nil {
+ reqBody["stop"] = *chatModelConfig.Stop
+ }
+ if chatModelConfig.Thinking != nil {
+ reqBody["reasoning"] = togetherAIReasoningOptions{
+ Enabled: *chatModelConfig.Thinking,
+ }
+ }
+ if chatModelConfig.Effort != nil && strings.Contains(strings.ToLower(modelName), "gpt-oss") {
+ reqBody["reasoning_effort"] = *chatModelConfig.Effort
+ }
+ }
+
+ return reqBody
+}
+
+func (t *TogetherAIModel) chatURL(apiConfig *APIConfig) (string, error) {
+ region := "default"
+ if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := t.baseURLForRegion(region)
+ if err != nil {
+ return "", err
+ }
+ return fmt.Sprintf("%s/%s", baseURL, t.URLSuffix.Chat), nil
+}
+
+type togetherAIChatMessage struct {
+ Content string `json:"content"`
+ ReasoningContent string `json:"reasoning_content"`
+ Reasoning string `json:"reasoning"`
+}
+
+type togetherAIChatChoice struct {
+ Message togetherAIChatMessage `json:"message"`
+ Delta togetherAIChatMessage `json:"delta"`
+ FinishReason string `json:"finish_reason"`
+}
+
+type togetherAIChatResponse struct {
+ Choices []togetherAIChatChoice `json:"choices"`
+ Error interface{} `json:"error"`
+ FinishReason string `json:"finish_reason"`
+}
+
+func (t *TogetherAIModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+ if strings.TrimSpace(modelName) == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+ if len(messages) == 0 {
+ return nil, fmt.Errorf("messages is empty")
+ }
+
+ url, err := t.chatURL(apiConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ jsonData, err := json.Marshal(t.chatPayload(modelName, messages, false, chatModelConfig))
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := t.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var result togetherAIChatResponse
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+ if result.Error != nil {
+ return nil, fmt.Errorf("togetherai: upstream error: %v", result.Error)
+ }
+ if len(result.Choices) == 0 {
+ return nil, fmt.Errorf("no choices in response")
+ }
+
+ content := result.Choices[0].Message.Content
+ reasonContent := result.Choices[0].Message.ReasoningContent
+ if reasonContent == "" {
+ reasonContent = result.Choices[0].Message.Reasoning
+ }
+ return &ChatResponse{
+ Answer: &content,
+ ReasonContent: &reasonContent,
+ }, nil
+}
+
+const togetherAIStreamTimeout = 10 * time.Minute
+
+func (t *TogetherAIModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
+ if sender == nil {
+ return fmt.Errorf("sender is required")
+ }
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return fmt.Errorf("api key is required")
+ }
+ if strings.TrimSpace(modelName) == "" {
+ return fmt.Errorf("model name is required")
+ }
+ if len(messages) == 0 {
+ return fmt.Errorf("messages is empty")
+ }
+ if chatModelConfig != nil && chatModelConfig.Stream != nil && !*chatModelConfig.Stream {
+ return fmt.Errorf("stream must be true in ChatStreamlyWithSender")
+ }
+
+ url, err := t.chatURL(apiConfig)
+ if err != nil {
+ return err
+ }
+
+ jsonData, err := json.Marshal(t.chatPayload(modelName, messages, true, chatModelConfig))
+ if err != nil {
+ return fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ // ResponseHeaderTimeout caps the initial header wait. This context
+ // also caps the body-read phase so a stalled SSE stream cannot hold
+ // the caller's goroutine and connection indefinitely.
+ ctx, cancel := context.WithTimeout(context.Background(), togetherAIStreamTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("Accept", "text/event-stream")
+
+ resp, err := t.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ scanner := bufio.NewScanner(resp.Body)
+ scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+ sawTerminal := false
+ for scanner.Scan() {
+ line := scanner.Text()
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+
+ data := strings.TrimSpace(line[5:])
+ if data == "[DONE]" {
+ sawTerminal = true
+ break
+ }
+
+ var event togetherAIChatResponse
+ if err = json.Unmarshal([]byte(data), &event); err != nil {
+ return fmt.Errorf("togetherai: invalid SSE event: %w", err)
+ }
+ if event.Error != nil {
+ return fmt.Errorf("togetherai: upstream stream error: %v", event.Error)
+ }
+ if len(event.Choices) == 0 {
+ continue
+ }
+
+ choice := event.Choices[0]
+ if choice.Delta.ReasoningContent != "" {
+ if err := sender(nil, &choice.Delta.ReasoningContent); err != nil {
+ return err
+ }
+ }
+ if choice.Delta.Reasoning != "" {
+ if err := sender(nil, &choice.Delta.Reasoning); err != nil {
+ return err
+ }
+ }
+ if choice.Delta.Content != "" {
+ if err := sender(&choice.Delta.Content, nil); err != nil {
+ return err
+ }
+ }
+ if choice.FinishReason != "" || event.FinishReason != "" {
+ sawTerminal = true
+ break
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("failed to scan response body: %w", err)
+ }
+ if !sawTerminal {
+ return fmt.Errorf("togetherai: stream ended before [DONE] or finish_reason")
+ }
+
+ endOfStream := "[DONE]"
+ return sender(&endOfStream, nil)
+}
+
+type togetherAIModelInfo struct {
+ ID string `json:"id"`
+}
+
+func (t *TogetherAIModel) ListModels(apiConfig *APIConfig) ([]string, error) {
+ if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
+ return nil, fmt.Errorf("api key is required")
+ }
+
+ region := "default"
+ if apiConfig.Region != nil && *apiConfig.Region != "" {
+ region = *apiConfig.Region
+ }
+
+ baseURL, err := t.baseURLForRegion(region)
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", baseURL, t.URLSuffix.Models)
+
+ ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+
+ resp, err := t.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+ }
+
+ var result []togetherAIModelInfo
+ if err = json.Unmarshal(body, &result); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ models := make([]string, 0, len(result))
+ for _, model := range result {
+ if model.ID != "" {
+ models = append(models, model.ID)
+ }
+ }
+ return models, nil
+}
+
+func (t *TogetherAIModel) CheckConnection(apiConfig *APIConfig) error {
+ _, err := t.ListModels(apiConfig)
+ return err
+}
+
+func (t *TogetherAIModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
+ return fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
+
+func (t *TogetherAIModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
+ return nil, fmt.Errorf("%s, no such method", t.Name())
+}
diff --git a/internal/entity/models/togetherai_test.go b/internal/entity/models/togetherai_test.go
new file mode 100644
index 00000000000..aecdf20c95b
--- /dev/null
+++ b/internal/entity/models/togetherai_test.go
@@ -0,0 +1,277 @@
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func newTogetherAIServer(t *testing.T, handler func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+ t.Errorf("expected Authorization=Bearer test-key, got %q", got)
+ return
+ }
+ if got := r.Header.Get("Content-Type"); !strings.HasPrefix(got, "application/json") {
+ t.Errorf("expected Content-Type to start with application/json, got %q", got)
+ return
+ }
+ var body map[string]interface{}
+ if r.Method == http.MethodPost {
+ raw, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read body: %v", err)
+ return
+ }
+ if err := json.Unmarshal(raw, &body); err != nil {
+ t.Errorf("unmarshal: %v\nraw=%s", err, string(raw))
+ return
+ }
+ }
+ handler(t, r, body, w)
+ }))
+}
+
+func newTogetherAIForTest(baseURL string) *TogetherAIModel {
+ return NewTogetherAIModel(
+ map[string]string{"default": baseURL},
+ URLSuffix{Chat: "chat/completions", Models: "models"},
+ )
+}
+
+func TestTogetherAIName(t *testing.T) {
+ if got := newTogetherAIForTest("http://unused").Name(); got != "togetherai" {
+ t.Errorf("Name()=%q", got)
+ }
+}
+
+func TestTogetherAIFactory(t *testing.T) {
+ driver, err := NewModelFactory().CreateModelDriver("TogetherAI", map[string]string{"default": "http://unused"}, URLSuffix{})
+ if err != nil {
+ t.Fatalf("CreateModelDriver: %v", err)
+ }
+ if _, ok := driver.(*TogetherAIModel); !ok {
+ t.Fatalf("driver type=%T, want *TogetherAIModel", driver)
+ }
+}
+
+func TestTogetherAIChatHappyPath(t *testing.T) {
+ srv := newTogetherAIServer(t, func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) {
+ if r.URL.Path != "/chat/completions" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ if body["model"] != "openai/gpt-oss-20b" {
+ t.Errorf("model=%v", body["model"])
+ }
+ if body["stream"] != false {
+ t.Errorf("stream=%v want false", body["stream"])
+ }
+ if body["reasoning_effort"] != "high" {
+ t.Errorf("reasoning_effort=%v", body["reasoning_effort"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{{
+ "message": map[string]interface{}{
+ "content": "pong",
+ "reasoning": "thinking",
+ },
+ }},
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ mt := 32
+ temp := 0.3
+ topP := 0.9
+ stop := []string{"END"}
+ effort := "high"
+ resp, err := newTogetherAIForTest(srv.URL).ChatWithMessages(
+ "openai/gpt-oss-20b",
+ []Message{{Role: "user", Content: "ping"}},
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{MaxTokens: &mt, Temperature: &temp, TopP: &topP, Stop: &stop, Effort: &effort},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if *resp.Answer != "pong" {
+ t.Errorf("Answer=%q", *resp.Answer)
+ }
+ if *resp.ReasonContent != "thinking" {
+ t.Errorf("ReasonContent=%q", *resp.ReasonContent)
+ }
+}
+
+func TestTogetherAIChatForwardsReasoningEnabled(t *testing.T) {
+ srv := newTogetherAIServer(t, func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) {
+ if body["model"] != "Qwen/Qwen3.5-9B" {
+ t.Errorf("model=%v", body["model"])
+ }
+ reasoning, ok := body["reasoning"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("reasoning=%T, want object", body["reasoning"])
+ }
+ if reasoning["enabled"] != false {
+ t.Errorf("reasoning.enabled=%v, want false", reasoning["enabled"])
+ }
+ if _, ok := body["reasoning_effort"]; ok {
+ t.Errorf("reasoning_effort should not be sent for non-GPT-OSS model: %v", body["reasoning_effort"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "choices": []map[string]interface{}{{
+ "message": map[string]interface{}{
+ "content": "pong",
+ },
+ }},
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ thinking := false
+ resp, err := newTogetherAIForTest(srv.URL).ChatWithMessages(
+ "Qwen/Qwen3.5-9B",
+ []Message{{Role: "user", Content: "ping"}},
+ &APIConfig{ApiKey: &apiKey},
+ &ChatConfig{Thinking: &thinking},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if *resp.Answer != "pong" {
+ t.Errorf("Answer=%q", *resp.Answer)
+ }
+}
+
+func TestTogetherAIChatRequiresModelName(t *testing.T) {
+ apiKey := "test-key"
+ _, err := newTogetherAIForTest("http://unused").ChatWithMessages("", []Message{{Role: "user", Content: "x"}}, &APIConfig{ApiKey: &apiKey}, nil)
+ if err == nil || !strings.Contains(err.Error(), "model name is required") {
+ t.Errorf("expected model-name error, got %v", err)
+ }
+}
+
+func TestTogetherAIStreamHappyPath(t *testing.T) {
+ srv := newTogetherAIServer(t, func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) {
+ if r.URL.Path != "/chat/completions" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ if body["stream"] != true {
+ t.Errorf("stream=%v want true", body["stream"])
+ }
+ if got := r.Header.Get("Accept"); got != "text/event-stream" {
+ t.Errorf("Accept=%q", got)
+ }
+ w.Header().Set("Content-Type", "text/event-stream")
+ _, _ = io.WriteString(w,
+ `data: {"choices":[{"delta":{"reasoning":"think "}}]}`+"\n"+
+ `data: {"choices":[{"delta":{"content":"Hello"}}]}`+"\n"+
+ `data: {"choices":[{"delta":{"content":" world"},"finish_reason":"stop"}]}`+"\n",
+ )
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ var content []string
+ var reasoning []string
+ err := newTogetherAIForTest(srv.URL).ChatStreamlyWithSender(
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo",
+ []Message{{Role: "user", Content: "hi"}},
+ &APIConfig{ApiKey: &apiKey}, nil,
+ func(c *string, r *string) error {
+ if c != nil {
+ content = append(content, *c)
+ }
+ if r != nil {
+ reasoning = append(reasoning, *r)
+ }
+ return nil
+ },
+ )
+ if err != nil {
+ t.Fatalf("ChatStreamlyWithSender: %v", err)
+ }
+ if strings.Join(content, "") != "Hello world[DONE]" {
+ t.Errorf("content=%q", strings.Join(content, ""))
+ }
+ if strings.Join(reasoning, "") != "think " {
+ t.Errorf("reasoning=%q", strings.Join(reasoning, ""))
+ }
+}
+
+func TestTogetherAIStreamStopsOnRootFinishReason(t *testing.T) {
+ srv := newTogetherAIServer(t, func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ _, _ = io.WriteString(w,
+ `data: {"choices":[{"delta":{"content":"Done"}}],"finish_reason":"stop"}`+"\n",
+ )
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ var chunks []string
+ err := newTogetherAIForTest(srv.URL).ChatStreamlyWithSender(
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo",
+ []Message{{Role: "user", Content: "hi"}},
+ &APIConfig{ApiKey: &apiKey}, nil,
+ func(c *string, _ *string) error {
+ if c != nil {
+ chunks = append(chunks, *c)
+ }
+ return nil
+ },
+ )
+ if err != nil {
+ t.Fatalf("ChatStreamlyWithSender: %v", err)
+ }
+ if strings.Join(chunks, "") != "Done[DONE]" {
+ t.Errorf("chunks=%q", strings.Join(chunks, ""))
+ }
+}
+
+func TestTogetherAIListModelsAndCheckConnection(t *testing.T) {
+ srv := newTogetherAIServer(t, func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) {
+ if r.Method != http.MethodGet {
+ t.Errorf("method=%s", r.Method)
+ }
+ if r.URL.Path != "/models" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ _ = json.NewEncoder(w).Encode([]map[string]interface{}{
+ {"id": "openai/gpt-oss-20b"},
+ {"id": "meta-llama/Llama-3.3-70B-Instruct-Turbo"},
+ })
+ })
+ defer srv.Close()
+
+ apiKey := "test-key"
+ model := newTogetherAIForTest(srv.URL)
+ models, err := model.ListModels(&APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ if strings.Join(models, ",") != "openai/gpt-oss-20b,meta-llama/Llama-3.3-70B-Instruct-Turbo" {
+ t.Errorf("models=%v", models)
+ }
+ if err := model.CheckConnection(&APIConfig{ApiKey: &apiKey}); err != nil {
+ t.Fatalf("CheckConnection: %v", err)
+ }
+}
+
+func TestTogetherAIUnsupportedMethods(t *testing.T) {
+ m := newTogetherAIForTest("http://unused")
+ if _, err := m.Embed(nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Embed error=%v", err)
+ }
+ if _, err := m.Rerank(nil, "", nil, nil, nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Rerank error=%v", err)
+ }
+ if _, err := m.Balance(nil); err == nil || !strings.Contains(err.Error(), "no such method") {
+ t.Errorf("Balance error=%v", err)
+ }
+}
From ce3402cbb99a61541539b8bcf750b2164335b929 Mon Sep 17 00:00:00 2001
From: Rene Arredondo <120709323+Rene0422@users.noreply.github.com>
Date: Tue, 19 May 2026 00:32:09 -0700
Subject: [PATCH 204/666] Fix: restore saved api_key fallback in add_llm
(#14921) (#14941)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
Closes #14921.
Reconfiguring an existing LLM provider to enable **tool call** or
**vision** fails with `Your API key is invalid. Fail to access model.`
even when the saved API key is correct. The most visible report is
VLLM ("Cannot add vllm model" once `--enable-auto-tool-choice` /
vision is toggled on), but the bug applies to every provider whose
api_key field stays blank in edit mode.
## Root cause
PR #14885 ("Fix: llm add api key overridden") removed the existing-key
lookup in `api/apps/llm_app.py::add_llm`. The intent was correct —
stop the saved key from clobbering a user-provided new one — but the
removal was unconditional, so the edit path now has no fallback at all:
1. `web/src/pages/user-setting/setting-model/hooks.tsx:230` sets the
initial `api_key` form value to `''` in edit mode (the real key is
never returned to the browser).
2. The user toggles `is_tools` / `vision` without retyping the key.
3. `hooks.tsx:183-185` strips the empty `api_key` from the payload.
4. `add_llm` defaults to the placeholder `"x"`
(`api/apps/llm_app.py:182`).
5. The upstream provider rejects `"x"` with `Your API key is invalid`.
## Fix
Restore the fallback **narrowly**, before any factory-specific handler
runs:
- If `req.get("api_key") is None`, look up the tenant's existing record
(using the correctly suffixed `llm_name` for VLLM /
OpenAI-API-Compatible / LocalAI / HuggingFace).
- Decode the saved blob with `_decode_api_key_config` and write **only
the decoded `api_key` string** back into `req["api_key"]`. Never use
the raw JSON payload — that was the exact thing PR #14885 was trying
to avoid.
- When the user **does** type a new key, `req.get("api_key")` is not
`None` and the fallback is skipped, so PR #14885's fix is preserved.
| Scenario | Before this PR | After this PR |
|---|---|---|
| Plain factory (VLLM, Ollama, …), retype key | OK | OK |
| Plain factory, blank key in edit (the bug) | Fails with "API key is
invalid" | Recovers saved key, validates against the real one |
| OpenRouter / Bedrock, change `provider_order` only | Fails |
`apikey_json([...])` rebuilds the JSON with saved `api_key` + new field
|
| User clears the form and types a brand-new key | OK (key replaced) |
OK (key replaced — fallback skipped) |
## Files changed
- `api/apps/llm_app.py` — restored fallback in `add_llm` (no other call
sites touched).
## Test plan
- [ ] Add a VLLM chat model with a valid api_key, no toggles → save
succeeds.
- [ ] Edit the same model, toggle **tool call** on, leave api_key blank
→ save succeeds, validation runs against the saved key.
- [ ] Edit again, toggle **vision** on (model_type → `image2text`),
leave api_key blank → save succeeds.
- [ ] Edit again and **type a new api_key** → the new key replaces the
saved one (`is None` check skips the fallback). Verify via the DB
row or by deliberately typing a wrong key and observing the
validation failure.
- [ ] Repeat the blank-key edit with **OpenRouter**, changing only
`provider_order` → resulting api_key JSON contains the saved
`api_key` and the new `provider_order`.
- [ ] First-time add of a new model name → no existing record, fallback
no-ops, behaves as before.
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
- [ ] New Feature (non-breaking change which adds functionality)
- [ ] Documentation Update
- [ ] Refactoring
- [ ] Performance Improvement
- [ ] Other (please describe):
---
api/apps/llm_app.py | 47 ++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 46 insertions(+), 1 deletion(-)
diff --git a/api/apps/llm_app.py b/api/apps/llm_app.py
index 58997fb4df4..430b7d8dc36 100644
--- a/api/apps/llm_app.py
+++ b/api/apps/llm_app.py
@@ -181,13 +181,58 @@ async def add_llm():
from rag.llm import ChatModel, CvModel, EmbeddingModel, OcrModel, RerankModel, Seq2txtModel, TTSModel
factory = req["llm_factory"]
- api_key = req.get("api_key", "x")
llm_name = req.get("llm_name")
timeout_seconds = int(os.environ.get("LLM_TIMEOUT_SECONDS", 10))
if factory not in [f.name for f in get_allowed_llm_factories()]:
return get_data_error_result(message=f"LLM factory {factory} is not allowed")
+ # When editing an existing model the frontend leaves the api_key input blank
+ # and strips it from the payload, so req["api_key"] is missing. Without a
+ # fallback the validation below would run with the "x" placeholder and the
+ # upstream provider would return "Your API key is invalid" — recover the
+ # saved key from DB. Use only the *decoded* api_key (never the raw JSON
+ # payload) so factories that pack extra fields into api_key
+ # (OpenRouter, Bedrock, …) can rebuild their JSON correctly with whatever
+ # new fields the user did provide via apikey_json.
+ if req.get("api_key") is None and llm_name:
+ _LLM_NAME_SUFFIX = {
+ "LocalAI": "___LocalAI",
+ "HuggingFace": "___HuggingFace",
+ "OpenAI-API-Compatible": "___OpenAI-API",
+ "VLLM": "___VLLM",
+ }
+ saved_llm_name = llm_name + _LLM_NAME_SUFFIX.get(factory, "")
+ logging.debug(
+ "add_llm: attempting api_key recovery factory=%s llm_name=%s saved_llm_name=%s tenant_id=%s",
+ factory, llm_name, saved_llm_name, current_user.id,
+ )
+ existing_llms = TenantLLMService.query(
+ tenant_id=current_user.id,
+ llm_factory=factory,
+ llm_name=saved_llm_name,
+ )
+ logging.debug(
+ "add_llm: api_key recovery query matched=%d factory=%s saved_llm_name=%s",
+ len(existing_llms) if existing_llms else 0, factory, saved_llm_name,
+ )
+ if existing_llms:
+ existing_api_key, _, _ = TenantLLMService._decode_api_key_config(
+ existing_llms[0].api_key
+ )
+ logging.debug(
+ "add_llm: api_key recovery decoded=%s factory=%s saved_llm_name=%s",
+ "present" if existing_api_key else "absent", factory, saved_llm_name,
+ )
+ if existing_api_key:
+ req["api_key"] = existing_api_key
+ logging.info(
+ "add_llm: recovered saved api_key from existing record factory=%s saved_llm_name=%s tenant_id=%s",
+ factory, saved_llm_name, current_user.id,
+ )
+
+ api_key = req.get("api_key", "x")
+
def apikey_json(keys):
nonlocal req
return json.dumps({k: req.get(k, "") for k in keys})
From 95b56e73f2a676d792512b86e367715a6a0568e9 Mon Sep 17 00:00:00 2001
From: Idriss Sbaaoui <112825897+6ba3i@users.noreply.github.com>
Date: Tue, 19 May 2026 15:43:15 +0800
Subject: [PATCH 205/666] Feat: add new tests and tescases for restful api
suite (#14993)
### What problem does this PR solve?
extend restful api suite
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- [x] Other (please describe): test
---
test/testcases/restful_api/test_agents.py | 25 +-
test/testcases/restful_api/test_chats.py | 222 ++++++++++++++++++
.../restful_api/test_openai_compatible.py | 2 +
test/testcases/restful_api/test_sessions.py | 53 ++++-
4 files changed, 300 insertions(+), 2 deletions(-)
diff --git a/test/testcases/restful_api/test_agents.py b/test/testcases/restful_api/test_agents.py
index 9748b9fb96d..d7f01e9a44b 100644
--- a/test/testcases/restful_api/test_agents.py
+++ b/test/testcases/restful_api/test_agents.py
@@ -85,6 +85,14 @@ def test_agents_crud_validation_contract(rest_client, create_agent_resource):
assert "canvas" in list_empty_payload["data"], list_empty_payload
assert "total" in list_empty_payload["data"], list_empty_payload
+ paged_list = rest_client.get(
+ "/agents",
+ params={"title": "missing_restful_agent", "desc": "true", "page_size": 1},
+ )
+ assert paged_list.status_code == 200
+ paged_list_payload = paged_list.json()
+ assert paged_list_payload["code"] == 0, paged_list_payload
+
missing_dsl = rest_client.post("/agents", json={"title": "missing_dsl_agent"})
assert missing_dsl.status_code == 200
missing_dsl_payload = missing_dsl.json()
@@ -105,6 +113,12 @@ def test_agents_crud_validation_contract(rest_client, create_agent_resource):
assert duplicate_payload["code"] == 102, duplicate_payload
assert "already exists" in duplicate_payload["message"], duplicate_payload
+ invalid_update = rest_client.put("/agents/invalid-agent-id", json={"title": "updated", "dsl": MINIMAL_DSL})
+ assert invalid_update.status_code == 200
+ invalid_update_payload = invalid_update.json()
+ assert invalid_update_payload["code"] == 103, invalid_update_payload
+ assert "Make sure you have permission to access the agent." in invalid_update_payload["message"], invalid_update_payload
+
get_res = rest_client.get(f"/agents/{agent_id}")
assert get_res.status_code == 200
get_payload = get_res.json()
@@ -122,6 +136,12 @@ def test_agents_crud_validation_contract(rest_client, create_agent_resource):
assert list_after_update_payload["code"] == 0, list_after_update_payload
assert list_after_update_payload["data"]["total"] >= 1, list_after_update_payload
+ invalid_delete = rest_client.delete("/agents/invalid-agent-id")
+ assert invalid_delete.status_code == 200
+ invalid_delete_payload = invalid_delete.json()
+ assert invalid_delete_payload["code"] == 103, invalid_delete_payload
+ assert "Only the owner of the agent is authorized for this operation." in invalid_delete_payload["message"], invalid_delete_payload
+
delete_res = rest_client.delete(f"/agents/{agent_id}")
assert delete_res.status_code == 200
delete_payload = delete_res.json()
@@ -188,8 +208,11 @@ def test_agent_chat_completion_nonstream(rest_client, create_agent_resource):
payload = res.json()
assert payload["code"] == 0, payload
assert isinstance(payload["data"], dict), payload
+ assert payload["data"].get("session_id") == session_id, payload
assert isinstance(payload["data"].get("data"), dict), payload
- assert "content" in payload["data"]["data"], payload
+ content = payload["data"]["data"].get("content", "")
+ assert content, payload
+ assert "hello" in content, payload
@pytest.mark.p2
diff --git a/test/testcases/restful_api/test_chats.py b/test/testcases/restful_api/test_chats.py
index 45fe5a8d2af..5f5bfa25591 100644
--- a/test/testcases/restful_api/test_chats.py
+++ b/test/testcases/restful_api/test_chats.py
@@ -16,6 +16,29 @@
import pytest
+from test.testcases.utils import encode_avatar
+from test.testcases.utils.file_utils import create_image_file
+
+
+DEFAULT_CHAT_EMPTY_RESPONSE = "Sorry! No relevant content was found in the knowledge base!"
+DEFAULT_CHAT_PROLOGUE = "Hi! I'm your assistant. What can I do for you?"
+DEFAULT_CHAT_SYSTEM_PROMPT = (
+ 'You are an intelligent assistant. Please summarize the content of the dataset to answer the question. '
+ 'Please list the data in the dataset and answer in detail. When all dataset content is irrelevant to the '
+ 'question, your answer must include the sentence "The answer you are looking for is not found in the dataset!" '
+ "Answers need to consider chat history.\n"
+ " Here is the knowledge base:\n"
+ " {knowledge}\n"
+ " The above is the knowledge base."
+)
+
+
+def _get_nested(data, path):
+ current = data
+ for key in path:
+ current = current[key]
+ return current
+
@pytest.mark.p1
class TestChatsAuthorization:
@@ -121,3 +144,202 @@ def test_chat_list_pagination(rest_client, clear_chats):
assert page_payload["code"] == 0, page_payload
assert len(page_payload["data"]["chats"]) == 2, page_payload
assert page_payload["data"]["total"] >= 3, page_payload
+
+
+@pytest.mark.p1
+def test_chat_create_dataset_ids_contract(rest_client, clear_chats, ensure_parsed_document):
+ dataset_id, _ = ensure_parsed_document()
+ cases = [
+ ("empty dataset_ids", [], 0, "", []),
+ ("owned parsed dataset", [dataset_id], 0, "", [dataset_id]),
+ ("invalid dataset id", ["invalid_dataset_id"], 102, "You don't own the dataset invalid_dataset_id", None),
+ ("dataset_ids wrong type", "invalid_dataset_id", 102, "`dataset_ids` should be a list.", None),
+ ]
+
+ for index, (scenario_name, dataset_ids, expected_code, expected_message, expected_dataset_ids) in enumerate(cases, start=1):
+ res = rest_client.post(
+ "/chats",
+ json={"name": f"restful_chat_dataset_ids_{index}", "dataset_ids": dataset_ids},
+ )
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == expected_code, (scenario_name, payload)
+ if expected_code == 0:
+ assert payload["data"]["dataset_ids"] == expected_dataset_ids, (scenario_name, payload)
+ else:
+ assert payload["message"] == expected_message, (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_create_avatar_contract(rest_client, clear_chats, tmp_path):
+ image_path = create_image_file(tmp_path / "restful_chat_avatar.png")
+ encoded_avatar = encode_avatar(image_path)
+
+ res = rest_client.post(
+ "/chats",
+ json={"name": "restful_chat_avatar", "dataset_ids": [], "icon": encoded_avatar},
+ )
+ assert res.status_code == 200
+ payload = res.json()
+ assert payload["code"] == 0, payload
+ assert payload["data"]["icon"] == encoded_avatar, payload
+
+
+@pytest.mark.p2
+def test_chat_create_llm_contract(rest_client, clear_chats, ensure_parsed_document):
+ dataset_id, _ = ensure_parsed_document()
+ cases = [
+ ("default llm", {}, 0, "", "glm-4-flash@ZHIPU-AI", {}),
+ ("explicit llm_id", {"llm_id": "glm-4"}, 0, "", "glm-4", {}),
+ ("unknown llm_id", {"llm_id": "unknown"}, 102, "`llm_id` unknown doesn't exist", None, None),
+ ("temperature zero", {"llm_setting": {"temperature": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": 0}),
+ ("temperature one", {"llm_setting": {"temperature": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": 1}),
+ ("temperature negative one", {"llm_setting": {"temperature": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": -1}),
+ ("temperature ten", {"llm_setting": {"temperature": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": 10}),
+ ("temperature string", {"llm_setting": {"temperature": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": "a"}),
+ ("top_p zero", {"llm_setting": {"top_p": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": 0}),
+ ("top_p one", {"llm_setting": {"top_p": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": 1}),
+ ("top_p negative one", {"llm_setting": {"top_p": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": -1}),
+ ("top_p ten", {"llm_setting": {"top_p": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": 10}),
+ ("top_p string", {"llm_setting": {"top_p": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": "a"}),
+ ("presence_penalty zero", {"llm_setting": {"presence_penalty": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": 0}),
+ ("presence_penalty one", {"llm_setting": {"presence_penalty": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": 1}),
+ ("presence_penalty negative one", {"llm_setting": {"presence_penalty": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": -1}),
+ ("presence_penalty ten", {"llm_setting": {"presence_penalty": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": 10}),
+ ("presence_penalty string", {"llm_setting": {"presence_penalty": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": "a"}),
+ ("frequency_penalty zero", {"llm_setting": {"frequency_penalty": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": 0}),
+ ("frequency_penalty one", {"llm_setting": {"frequency_penalty": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": 1}),
+ ("frequency_penalty negative one", {"llm_setting": {"frequency_penalty": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": -1}),
+ ("frequency_penalty ten", {"llm_setting": {"frequency_penalty": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": 10}),
+ ("frequency_penalty string", {"llm_setting": {"frequency_penalty": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": "a"}),
+ ("max_token zero", {"llm_setting": {"max_token": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": 0}),
+ ("max_token 1024", {"llm_setting": {"max_token": 1024}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": 1024}),
+ ("max_token negative one", {"llm_setting": {"max_token": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": -1}),
+ ("max_token ten", {"llm_setting": {"max_token": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": 10}),
+ ("max_token string", {"llm_setting": {"max_token": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": "a"}),
+ ("unknown llm setting key", {"llm_setting": {"unknown": "unknown"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"unknown": "unknown"}),
+ ]
+
+ for index, (scenario_name, extra_payload, expected_code, expected_message, expected_llm_id, expected_llm_setting) in enumerate(cases, start=1):
+ payload = {
+ "name": f"restful_chat_llm_{index}",
+ "dataset_ids": [dataset_id],
+ }
+ payload.update(extra_payload)
+ res = rest_client.post("/chats", json=payload)
+ assert res.status_code == 200, (scenario_name, res.text)
+ body = res.json()
+ assert body["code"] == expected_code, (scenario_name, body)
+ if expected_code == 0:
+ assert body["data"]["llm_id"] == expected_llm_id, (scenario_name, body)
+ assert body["data"]["llm_setting"] == expected_llm_setting, (scenario_name, body)
+ else:
+ assert body["message"] == expected_message, (scenario_name, body)
+
+
+@pytest.mark.p2
+def test_chat_create_prompt_contract(rest_client, clear_chats):
+ cases = [
+ (
+ "default prompt config",
+ {},
+ {
+ ("similarity_threshold",): 0.1,
+ ("vector_similarity_weight",): 0.3,
+ ("top_n",): 6,
+ ("rerank_id",): "",
+ ("prompt_config", "parameters"): [{"key": "knowledge", "optional": False}],
+ ("prompt_config", "empty_response"): DEFAULT_CHAT_EMPTY_RESPONSE,
+ ("prompt_config", "prologue"): DEFAULT_CHAT_PROLOGUE,
+ ("prompt_config", "quote"): True,
+ ("prompt_config", "system"): DEFAULT_CHAT_SYSTEM_PROMPT,
+ },
+ ),
+ ("similarity_threshold zero", {"similarity_threshold": 0}, {("similarity_threshold",): 0}),
+ ("similarity_threshold one", {"similarity_threshold": 1}, {("similarity_threshold",): 1}),
+ ("similarity_threshold negative one", {"similarity_threshold": -1}, {("similarity_threshold",): -1.0}),
+ ("similarity_threshold ten", {"similarity_threshold": 10}, {("similarity_threshold",): 10.0}),
+ ("similarity_threshold string", {"similarity_threshold": "a"}, {("similarity_threshold",): 0.0}),
+ ("vector_similarity_weight one", {"vector_similarity_weight": 1}, {("vector_similarity_weight",): 1}),
+ ("vector_similarity_weight zero", {"vector_similarity_weight": 0}, {("vector_similarity_weight",): 0}),
+ ("vector_similarity_weight two", {"vector_similarity_weight": 2}, {("vector_similarity_weight",): 2.0}),
+ ("vector_similarity_weight negative nine", {"vector_similarity_weight": -9}, {("vector_similarity_weight",): -9.0}),
+ ("vector_similarity_weight string", {"vector_similarity_weight": "a"}, {("vector_similarity_weight",): 0.0}),
+ ("empty prompt parameters", {"prompt_config": {"parameters": []}}, {("prompt_config", "parameters"): []}),
+ ("top_n zero", {"top_n": 0}, {("top_n",): 0}),
+ ("top_n one", {"top_n": 1}, {("top_n",): 1}),
+ ("top_n negative one", {"top_n": -1}, {("top_n",): -1}),
+ ("top_n ten", {"top_n": 10}, {("top_n",): 10}),
+ ("top_n string", {"top_n": "a"}, {("top_n",): 0}),
+ ("empty_response plain text", {"prompt_config": {"empty_response": "Hello World"}}, {("prompt_config", "empty_response"): "Hello World"}),
+ ("empty_response empty string", {"prompt_config": {"empty_response": ""}}, {("prompt_config", "empty_response"): ""}),
+ ("empty_response punctuation", {"prompt_config": {"empty_response": "!@#$%^&*()"}}, {("prompt_config", "empty_response"): "!@#$%^&*()"}),
+ ("empty_response chinese text", {"prompt_config": {"empty_response": "中文测试"}}, {("prompt_config", "empty_response"): "中文测试"}),
+ ("empty_response integer", {"prompt_config": {"empty_response": 123}}, {("prompt_config", "empty_response"): 123}),
+ ("empty_response boolean", {"prompt_config": {"empty_response": True}}, {("prompt_config", "empty_response"): True}),
+ ("empty_response space", {"prompt_config": {"empty_response": " "}}, {("prompt_config", "empty_response"): " "}),
+ ("prologue plain text", {"prompt_config": {"prologue": "Hello World"}}, {("prompt_config", "prologue"): "Hello World"}),
+ ("prologue empty string", {"prompt_config": {"prologue": ""}}, {("prompt_config", "prologue"): ""}),
+ ("prologue punctuation", {"prompt_config": {"prologue": "!@#$%^&*()"}}, {("prompt_config", "prologue"): "!@#$%^&*()"}),
+ ("prologue chinese text", {"prompt_config": {"prologue": "中文测试"}}, {("prompt_config", "prologue"): "中文测试"}),
+ ("prologue integer", {"prompt_config": {"prologue": 123}}, {("prompt_config", "prologue"): 123}),
+ ("prologue boolean", {"prompt_config": {"prologue": True}}, {("prompt_config", "prologue"): True}),
+ ("prologue space", {"prompt_config": {"prologue": " "}}, {("prompt_config", "prologue"): " "}),
+ ("quote true", {"prompt_config": {"quote": True}}, {("prompt_config", "quote"): True}),
+ ("quote false", {"prompt_config": {"quote": False}}, {("prompt_config", "quote"): False}),
+ ("system prompt with knowledge prefix", {"prompt_config": {"system": "Hello World {knowledge}"}}, {("prompt_config", "system"): "Hello World {knowledge}"}),
+ ("system prompt only knowledge", {"prompt_config": {"system": "{knowledge}"}}, {("prompt_config", "system"): "{knowledge}"}),
+ ("system prompt punctuation", {"prompt_config": {"system": "!@#$%^&*() {knowledge}"}}, {("prompt_config", "system"): "!@#$%^&*() {knowledge}"}),
+ ("system prompt chinese text", {"prompt_config": {"system": "中文测试 {knowledge}"}}, {("prompt_config", "system"): "中文测试 {knowledge}"}),
+ ("system prompt plain text", {"prompt_config": {"system": "Hello World"}}, {("prompt_config", "system"): "Hello World"}),
+ (
+ "system prompt with explicit empty parameters",
+ {"prompt_config": {"system": "Hello World", "parameters": []}},
+ {("prompt_config", "system"): "Hello World", ("prompt_config", "parameters"): []},
+ ),
+ ("system prompt integer", {"prompt_config": {"system": 123}}, {("prompt_config", "system"): 123}),
+ ("system prompt boolean", {"prompt_config": {"system": True}}, {("prompt_config", "system"): True}),
+ ("unknown prompt_config key", {"prompt_config": {"unknown": "unknown"}}, {("prompt_config", "unknown"): "unknown"}),
+ ]
+
+ for index, (scenario_name, extra_payload, expected_values) in enumerate(cases, start=1):
+ res = rest_client.post(
+ "/chats",
+ json={"name": f"restful_chat_prompt_{index}", "dataset_ids": [], **extra_payload},
+ )
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == 0, (scenario_name, payload)
+ for path, expected_value in expected_values.items():
+ assert _get_nested(payload["data"], path) == expected_value, (scenario_name, path, payload)
+
+
+@pytest.mark.p2
+def test_chat_create_additional_guards_contract(rest_client, clear_chats):
+ cases = [
+ ("reject tenant_id override", {"tenant_id": "tenant-should-not-pass"}, "`tenant_id` must not be provided."),
+ ("reject unknown rerank_id", {"rerank_id": "unknown-rerank-model"}, "`rerank_id` unknown-rerank-model doesn't exist"),
+ ]
+
+ for index, (scenario_name, extra_payload, expected_message) in enumerate(cases, start=1):
+ res = rest_client.post(
+ "/chats",
+ json={"name": f"restful_chat_guard_{index}", "dataset_ids": [], **extra_payload},
+ )
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == 102, (scenario_name, payload)
+ assert expected_message in payload["message"], (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_create_rejects_unparsed_document(rest_client, clear_chats, create_document):
+ dataset_id, _ = create_document()
+ res = rest_client.post(
+ "/chats",
+ json={"name": "restful_chat_unparsed_document", "dataset_ids": [dataset_id]},
+ )
+ assert res.status_code == 200
+ payload = res.json()
+ assert payload["code"] == 102, payload
+ assert "doesn't own parsed file" in payload["message"], payload
diff --git a/test/testcases/restful_api/test_openai_compatible.py b/test/testcases/restful_api/test_openai_compatible.py
index d8585767256..49e2c55ca59 100644
--- a/test/testcases/restful_api/test_openai_compatible.py
+++ b/test/testcases/restful_api/test_openai_compatible.py
@@ -135,6 +135,8 @@ def test_openai_compatible_nonstream_shape(rest_client, create_chat):
assert "prompt_tokens" in usage, usage
assert "completion_tokens" in usage, usage
assert "total_tokens" in usage, usage
+ assert usage["prompt_tokens"] > 0, usage
+ assert usage["completion_tokens"] > 0, usage
assert usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"], usage
diff --git a/test/testcases/restful_api/test_sessions.py b/test/testcases/restful_api/test_sessions.py
index 6a7a9de82c6..ca1c8ea5c6e 100644
--- a/test/testcases/restful_api/test_sessions.py
+++ b/test/testcases/restful_api/test_sessions.py
@@ -146,9 +146,46 @@ def test_chat_completion_nonstream_with_session(rest_client, create_chat):
completion_payload = completion_res.json()
assert completion_payload["code"] == 0, completion_payload
assert isinstance(completion_payload["data"], dict), completion_payload
+ for key in ["answer", "reference", "audio_binary", "id", "session_id"]:
+ assert key in completion_payload["data"], completion_payload
assert completion_payload["data"]["session_id"] == session_id, completion_payload
+
+
+@pytest.mark.p2
+def test_chat_completion_nonstream_with_chat_without_session(rest_client, create_chat):
+ chat_id = create_chat("restful_completion_nonstream_without_session_chat")
+
+ completion_res = rest_client.post(
+ "/chat/completions",
+ json={
+ "chat_id": chat_id,
+ "messages": [{"role": "user", "content": "hello"}],
+ "stream": False,
+ },
+ timeout=60,
+ )
+ assert completion_res.status_code == 200
+ completion_payload = completion_res.json()
+ assert completion_payload["code"] == 0, completion_payload
+ assert isinstance(completion_payload["data"], dict), completion_payload
+ assert completion_payload["data"]["session_id"], completion_payload
+
+
+@pytest.mark.p2
+def test_chat_completion_nonstream_without_chat(rest_client):
+ completion_res = rest_client.post(
+ "/chat/completions",
+ json={
+ "messages": [{"role": "user", "content": "hello"}],
+ "stream": False,
+ },
+ timeout=60,
+ )
+ assert completion_res.status_code == 200
+ completion_payload = completion_res.json()
+ assert completion_payload["code"] == 0, completion_payload
+ assert isinstance(completion_payload["data"], dict), completion_payload
assert "answer" in completion_payload["data"], completion_payload
- assert "reference" in completion_payload["data"], completion_payload
@pytest.mark.p2
@@ -204,6 +241,20 @@ def test_chat_completion_validation_errors(rest_client, create_chat):
assert missing_chat_for_session_payload["code"] == 102, missing_chat_for_session_payload
assert "`chat_id` is required when `session_id` is provided." in missing_chat_for_session_payload["message"], missing_chat_for_session_payload
+ invalid_session = rest_client.post(
+ "/chat/completions",
+ json={
+ "chat_id": chat_id,
+ "messages": [{"role": "user", "content": "hello"}],
+ "stream": False,
+ "session_id": "invalid_session",
+ },
+ )
+ assert invalid_session.status_code == 200
+ invalid_session_payload = invalid_session.json()
+ assert invalid_session_payload["code"] == 102, invalid_session_payload
+ assert "Session not found!" in invalid_session_payload["message"], invalid_session_payload
+
invalid_chat = rest_client.post(
"/chat/completions",
json={
From f58e0b3ecac52dcf0c12992b168bb296456beebf Mon Sep 17 00:00:00 2001
From: Rene Arredondo <120709323+Rene0422@users.noreply.github.com>
Date: Tue, 19 May 2026 01:08:10 -0700
Subject: [PATCH 206/666] Feat: VLM image descriptions in MinerU parser
(#14869) (#14946)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
Closes #14869.
Adds VLM-based semantic descriptions to **image chunks produced by the
MinerU parser**, closing a long-standing parity gap with the deepdoc
parser's `VisionFigureParser`. A maintainer flagged this in #13342
("We may add the VLM enhancement to MinerU parser as well") and an
earlier proposal exists in #13824; this PR lands the change end-to-end
inside the existing parser plumbing.
## Why
Today the MinerU parser returns image chunks containing only the
native `image_caption` and `image_footnote` strings from MinerU's
JSON. When neither is present (or when both are sparse), the chunk
carries effectively no searchable content for the figure and
retrieval misses it entirely. Users who configured a local VLM
(reporter's case: Gemma-4-31B) had to post-process MinerU's
`tmp/*.json` themselves.
The deepdoc parser already solves this via
[`VisionFigureParser`](deepdoc/parser/figure_parser.py): when the
tenant has an `IMAGE2TEXT` model configured, each figure gets a
semantic description merged into its chunk. This PR brings the same
behavior to MinerU.
## What changed
### `deepdoc/parser/mineru_parser.py`
- **New method `_enhance_images_with_vlm(outputs, vision_model,
callback=None)`** —
collects every `IMAGE` block with a readable `img_path`, runs
`rag.app.picture.vision_llm_chunk` in a 10-worker
`ThreadPoolExecutor` using the existing
`vision_llm_figure_describe_prompt`, and writes the result back as
`vlm_description`. Per-image failures are logged and skipped — they
never abort the run.
- **`_transfer_to_sections` (IMAGE branch)** — folds
`vlm_description` into the section text alongside caption +
footnote, so the description becomes part of the chunk and is
searchable / retrievable.
- **`parse_pdf`** — after `_read_output`, calls
`_enhance_images_with_vlm(outputs, vision_model, callback=callback)`
when a `vision_model` kwarg is supplied. Wrapped in `try / except`
so a VLM outage cannot break parsing.
### `rag/app/naive.py` (`by_mineru`)
After successfully resolving the MinerU OCR parser, also resolves the
tenant's default `LLMType.IMAGE2TEXT` model via
`get_tenant_default_model_by_type`, wraps it in an `LLMBundle`, and
injects it as `kwargs["vision_model"]` before delegating to
`parse_pdf`.
## Behavior
| Tenant config | Behavior |
|---|---|
| `IMAGE2TEXT` model configured | MinerU image chunks contain `caption +
footnote + VLM description`. Retrieval against figures now actually
works. |
| No `IMAGE2TEXT` model configured | Exact same output as today (caption
+ footnote only). Lookup fails silently with an info log; no error, no
regression. |
| VLM call fails for a single image | That image silently falls back to
caption + footnote; other images proceed. |
| Caller already passes `vision_model` in kwargs | We don't override it
— `if "vision_model" not in kwargs` guards the lookup. |
## Files
- `deepdoc/parser/mineru_parser.py` (+56)
- `rag/app/naive.py` (+13)
---
deepdoc/parser/mineru_parser.py | 56 +++++++++++++++++++++++++++++++++
rag/app/naive.py | 13 ++++++++
2 files changed, 69 insertions(+)
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py
index b369f9122a0..2c35ead98c6 100644
--- a/deepdoc/parser/mineru_parser.py
+++ b/deepdoc/parser/mineru_parser.py
@@ -644,6 +644,12 @@ def _transfer_to_sections(self, outputs: list[dict[str, Any]], parse_method: str
case MinerUContentType.IMAGE:
section = "".join(output.get("image_caption", [])) + "\n" + "".join(
output.get("image_footnote", []))
+ # If a vision model enriched this image with a semantic
+ # description (see _enhance_images_with_vlm), embed it in
+ # the chunk so it becomes searchable / retrievable.
+ vlm_description = (output.get("vlm_description") or "").strip()
+ if vlm_description:
+ section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description
case MinerUContentType.EQUATION:
section = output.get("text", "")
case MinerUContentType.CODE:
@@ -664,6 +670,49 @@ def _transfer_to_sections(self, outputs: list[dict[str, Any]], parse_method: str
def _transfer_to_tables(self, outputs: list[dict[str, Any]]):
return []
+ def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None):
+ """Generate semantic descriptions for image blocks via the tenant's
+ IMAGE2TEXT model, mirroring deepdoc's VisionFigureParser. Each
+ IMAGE block with a readable img_path gets a ``vlm_description``
+ field that ``_transfer_to_sections`` then folds into the chunk
+ text — closing issue #14869.
+ """
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+ from rag.app.picture import vision_llm_chunk
+ from rag.prompts.generator import vision_llm_figure_describe_prompt
+
+ image_jobs = [
+ (idx, item)
+ for idx, item in enumerate(outputs)
+ if item.get("type") == MinerUContentType.IMAGE
+ and item.get("img_path")
+ and os.path.exists(item["img_path"])
+ ]
+ if not image_jobs:
+ return
+
+ if callback:
+ callback(0.78, f"[MinerU] Generating VLM descriptions for {len(image_jobs)} images...")
+
+ prompt = vision_llm_figure_describe_prompt()
+
+ def worker(idx, item):
+ try:
+ with Image.open(item["img_path"]) as img:
+ img.load()
+ desc = vision_llm_chunk(binary=img, vision_model=vision_model, prompt=prompt)
+ return idx, (desc or "").strip()
+ except Exception as e:
+ logging.warning(f"[MinerU] VLM description failed for image #{idx}: {e}")
+ return idx, ""
+
+ with ThreadPoolExecutor(max_workers=10) as executor:
+ futures = [executor.submit(worker, idx, item) for idx, item in image_jobs]
+ for fut in as_completed(futures):
+ idx, desc = fut.result()
+ if desc:
+ outputs[idx]["vlm_description"] = desc
+
def parse_pdf(
self,
filepath: str | PathLike[str],
@@ -744,6 +793,13 @@ def parse_pdf(
if callback:
callback(0.75, f"[MinerU] Parsed {len(outputs)} blocks from PDF.")
+ vision_model = kwargs.get("vision_model")
+ if vision_model is not None:
+ try:
+ self._enhance_images_with_vlm(outputs, vision_model, callback=callback)
+ except Exception as e:
+ self.logger.warning(f"[MinerU] VLM image enhancement failed: {e}. Continuing without descriptions.")
+
return self._transfer_to_sections(outputs, parse_method), self._transfer_to_tables(outputs)
finally:
if temp_pdf and temp_pdf.exists():
diff --git a/rag/app/naive.py b/rag/app/naive.py
index f91e2a8f946..7bf4743e7db 100644
--- a/rag/app/naive.py
+++ b/rag/app/naive.py
@@ -131,6 +131,19 @@ def by_mineru(
ocr_model_config = get_model_config_by_type_and_name(tenant_id, LLMType.OCR, mineru_llm_name)
ocr_model = LLMBundle(tenant_id=tenant_id, model_config=ocr_model_config, lang=lang)
pdf_parser = ocr_model.mdl
+
+ # Closes #14869: when the tenant has an IMAGE2TEXT model
+ # configured, let the MinerU parser enrich image chunks with
+ # VLM-generated semantic descriptions (parity with deepdoc's
+ # VisionFigureParser). Best-effort — fall back silently if
+ # no vision model is available.
+ if "vision_model" not in kwargs:
+ try:
+ vision_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.IMAGE2TEXT)
+ kwargs["vision_model"] = LLMBundle(tenant_id=tenant_id, model_config=vision_model_config, lang=lang)
+ except Exception as vlm_err:
+ logging.info(f"[MinerU] no IMAGE2TEXT model for tenant; skipping image VLM enhancement: {vlm_err}")
+
sections, tables = pdf_parser.parse_pdf(
filepath=filename,
binary=binary,
From 6796a47b8d37ff4464e06ffc1a4ddb5c8ea830d1 Mon Sep 17 00:00:00 2001
From: plind <59729252+plind-junior@users.noreply.github.com>
Date: Tue, 19 May 2026 01:14:57 -0700
Subject: [PATCH 207/666] feat(sdk): make Begin inputs discoverable on
Session.ask (#14842)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What problem does this PR solve?
Closes #14751.
The user reported that after adding a variable (e.g. `key1`) to an
agent's **Begin** component, the Python SDK gave them no way to pass it:
their call `session.ask(question=user_question, stream=False)` had no
parameter for `key1`, and the `ask()` signature was just `(question,
stream, **kwargs)` with a docstring that only described streaming
behavior.
The functionality already works — `_ask_agent` does
`json_data.update(kwargs)` and the server reads `inputs` from the
request body at `agent_api.py:902`. The canonical shape is also in the
public API docs (`docs/references/python_api_reference.md:1817-1840`):
```python
session.ask(
"",
stream=False,
inputs={"line_var": {"type": "line", "value": "I am line_var"}},
return_trace=True,
)
```
But because `inputs`, `release`, and `return_trace` were hidden behind
`**kwargs`, they did not appear in IDE signature help, and the docstring
did not mention them. Users had no path from "I added a key in the UI"
to "I need to pass `inputs=...` with this exact shape."
This PR promotes the three most relevant Begin-related arguments to
named parameters and rewrites the docstring with a worked example.
### What this PR changes
- `sdk/python/ragflow_sdk/modules/session.py`:
- `Session.ask()` signature becomes `ask(question="", stream=False,
inputs=None, release=None, return_trace=None, **kwargs)`.
- These three new named params are forwarded into the existing `kwargs`
dict before dispatch, so the wire format and downstream behavior are
unchanged.
- Docstring rewritten in numpy style, including the structured `{"type":
..., "value": ...}` shape that the Begin component requires (see
`agent/component/begin.py:45-60`).
No backend changes. `**kwargs` is preserved for forward compatibility
with other body fields (`session_id`, `files`, `user_id`,
`custom_header`, …).
### Test plan
- [ ] `session.ask(question="hi", stream=False)` — existing call still
works
- [ ] `session.ask("", stream=False, inputs={"key1": {"type": "line",
"value": "v"}})` — Begin component receives `key1 = "v"`
- [ ] `session.ask("", stream=True, return_trace=True)` — streaming
response includes trace events
- [ ] IDE / `help(Session.ask)` now shows `inputs`, `release`,
`return_trace` with descriptions
### Type of change
- [x] Refactoring
- [x] Documentation Update
---
sdk/python/ragflow_sdk/modules/session.py | 66 +++++++++++++++++++++--
1 file changed, 63 insertions(+), 3 deletions(-)
diff --git a/sdk/python/ragflow_sdk/modules/session.py b/sdk/python/ragflow_sdk/modules/session.py
index f9c4799fd7a..5152160f6a4 100644
--- a/sdk/python/ragflow_sdk/modules/session.py
+++ b/sdk/python/ragflow_sdk/modules/session.py
@@ -15,8 +15,12 @@
#
import json
+import logging
+
from .base import Base
+logger = logging.getLogger(__name__)
+
class Session(Base):
def __init__(self, rag, res_dict):
@@ -33,11 +37,67 @@ def __init__(self, rag, res_dict):
super().__init__(rag, res_dict)
- def ask(self, question="", stream=False, **kwargs):
+ def ask(
+ self,
+ question="",
+ stream=False,
+ inputs=None,
+ release=None,
+ return_trace=None,
+ **kwargs,
+ ):
"""
- Ask a question to the session. If stream=True, yields Message objects as they arrive (SSE streaming).
- If stream=False, returns a single Message object for the final answer.
+ Ask a question to the session.
+
+ Parameters
+ ----------
+ question : str
+ The user's question. May be empty when the agent is driven solely by
+ Begin component inputs.
+ stream : bool
+ If ``True``, yields ``Message`` objects as they arrive (SSE streaming).
+ If ``False``, yields a single ``Message`` with the final answer.
+ inputs : dict, optional
+ Values for variables declared on the agent's **Begin** component. Each
+ value must be a dict containing at least a ``"value"`` key, and may
+ include ``"type"``. Example::
+
+ session.ask(
+ "",
+ stream=False,
+ inputs={"key1": {"type": "line", "value": "hello"}},
+ )
+
+ Only meaningful for agent sessions; ignored for chat sessions.
+ release : bool, optional
+ If ``True``, run against the latest published agent version instead of
+ the editable draft. Only meaningful for agent sessions.
+ return_trace : bool, optional
+ If ``True``, include execution trace information in the response.
+ Only meaningful for agent sessions.
+ **kwargs
+ Additional fields forwarded verbatim to the completion endpoint
+ (e.g. ``session_id``, ``files``, ``user_id``, ``custom_header``).
+ See the HTTP API reference for the full list.
"""
+ if inputs is not None:
+ kwargs["inputs"] = inputs
+ if release is not None:
+ kwargs["release"] = release
+ if return_trace is not None:
+ kwargs["return_trace"] = return_trace
+
+ if inputs is not None or release is not None or return_trace is not None:
+ logger.debug(
+ "Session.ask explicit-params session_type=%s session_id=%s "
+ "input_keys=%s release=%s return_trace=%s",
+ self.__session_type,
+ getattr(self, "id", None),
+ list(inputs.keys()) if isinstance(inputs, dict) else None,
+ release,
+ return_trace,
+ )
+
if self.__session_type == "agent":
res = self._ask_agent(question, stream, **kwargs)
elif self.__session_type == "chat":
From 6b2fcb41160b470ca9187b7d6c0b191e76136e25 Mon Sep 17 00:00:00 2001
From: Idriss Sbaaoui <112825897+6ba3i@users.noreply.github.com>
Date: Tue, 19 May 2026 17:17:31 +0800
Subject: [PATCH 208/666] Feat: add new tests and tescases for restful api
suite (#14996)
### What problem does this PR solve?
extend restful api suite
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- [x] Other (please describe): test
---
test/testcases/restful_api/test_chats.py | 1792 ++++++++++++++++-
.../test_user_tenant_routes_unit.py | 246 +++
2 files changed, 1992 insertions(+), 46 deletions(-)
diff --git a/test/testcases/restful_api/test_chats.py b/test/testcases/restful_api/test_chats.py
index 5f5bfa25591..eaf94a13c9d 100644
--- a/test/testcases/restful_api/test_chats.py
+++ b/test/testcases/restful_api/test_chats.py
@@ -14,8 +14,20 @@
# limitations under the License.
#
+import asyncio
+import importlib.util
+import sys
+from copy import deepcopy
+from concurrent.futures import ThreadPoolExecutor
+from enum import Enum
+from functools import wraps
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+
import pytest
+from test.testcases.configs import CHAT_ASSISTANT_NAME_LIMIT, INVALID_API_TOKEN
+from test.testcases.restful_api.helpers.client import RestClient
from test.testcases.utils import encode_avatar
from test.testcases.utils.file_utils import create_image_file
@@ -40,6 +52,27 @@ def _get_nested(data, path):
return current
+def _chat_names(payload):
+ return [chat["name"] for chat in payload["data"]["chats"]]
+
+
+def _reset_chat_batch(rest_client, prefix, count=5):
+ cleanup_res = rest_client.delete("/chats", json={"ids": None, "delete_all": True})
+ assert cleanup_res.status_code == 200, cleanup_res.text
+ cleanup_payload = cleanup_res.json()
+ assert cleanup_payload["code"] in (0, 102), cleanup_payload
+
+ ids = []
+ for index in range(count):
+ res = rest_client.post("/chats", json={"name": f"{prefix}_{index}", "dataset_ids": []})
+ assert res.status_code == 200, (prefix, index, res.text)
+ payload = res.json()
+ assert payload["code"] == 0, (prefix, index, payload)
+ ids.append(payload["data"]["id"])
+ return ids
+
+
+
@pytest.mark.p1
class TestChatsAuthorization:
def test_create_requires_auth(self, rest_client_noauth):
@@ -49,10 +82,7 @@ def test_create_requires_auth(self, rest_client_noauth):
@pytest.mark.p1
def test_chat_crud_cycle(rest_client, clear_chats):
- create_res = rest_client.post(
- "/chats",
- json={"name": "restful_chat_crud", "dataset_ids": []},
- )
+ create_res = rest_client.post("/chats", json={"name": "restful_chat_crud", "dataset_ids": []})
assert create_res.status_code == 200
create_payload = create_res.json()
assert create_payload["code"] == 0, create_payload
@@ -62,9 +92,8 @@ def test_chat_crud_cycle(rest_client, clear_chats):
assert list_res.status_code == 200
list_payload = list_res.json()
assert list_payload["code"] == 0, list_payload
- chats = list_payload["data"]["chats"]
- assert len(chats) == 1, list_payload
- assert chats[0]["id"] == chat_id, list_payload
+ assert len(list_payload["data"]["chats"]) == 1, list_payload
+ assert list_payload["data"]["chats"][0]["id"] == chat_id, list_payload
get_res = rest_client.get(f"/chats/{chat_id}")
assert get_res.status_code == 200
@@ -72,10 +101,7 @@ def test_chat_crud_cycle(rest_client, clear_chats):
assert get_payload["code"] == 0, get_payload
assert get_payload["data"]["id"] == chat_id, get_payload
- update_res = rest_client.put(
- f"/chats/{chat_id}",
- json={"name": "restful_chat_crud_updated", "dataset_ids": []},
- )
+ update_res = rest_client.put(f"/chats/{chat_id}", json={"name": "restful_chat_crud_updated", "dataset_ids": []})
assert update_res.status_code == 200
update_payload = update_res.json()
assert update_payload["code"] == 0, update_payload
@@ -101,49 +127,1368 @@ def test_chat_crud_cycle(rest_client, clear_chats):
@pytest.mark.p2
-@pytest.mark.parametrize(
- "name, expected_fragment",
- [
- ("", "`name` is required."),
- (" ", "`name` is required."),
- ],
-)
-def test_chat_create_name_validation(rest_client, clear_chats, name, expected_fragment):
- res = rest_client.post("/chats", json={"name": name, "dataset_ids": []})
- assert res.status_code == 200
- payload = res.json()
- assert payload["code"] == 102, payload
- assert expected_fragment in payload["message"], payload
+@pytest.mark.parametrize(
+ "name, expected_fragment",
+ [
+ ("", "`name` is required."),
+ (" ", "`name` is required."),
+ ],
+)
+def test_chat_create_name_validation(rest_client, clear_chats, name, expected_fragment):
+ res = rest_client.post("/chats", json={"name": name, "dataset_ids": []})
+ assert res.status_code == 200
+ payload = res.json()
+ assert payload["code"] == 102, payload
+ assert expected_fragment in payload["message"], payload
+
+
+@pytest.mark.p2
+def test_chat_duplicate_name_validation(rest_client, clear_chats):
+ first = rest_client.post("/chats", json={"name": "duplicate_chat_name", "dataset_ids": []})
+ assert first.status_code == 200
+ first_payload = first.json()
+ assert first_payload["code"] == 0, first_payload
+
+ second = rest_client.post("/chats", json={"name": "duplicate_chat_name", "dataset_ids": []})
+ assert second.status_code == 200
+ second_payload = second.json()
+ assert second_payload["code"] == 102, second_payload
+ assert "Duplicated chat name" in second_payload["message"], second_payload
+
+
+@pytest.mark.p2
+def test_chat_list_pagination(rest_client, clear_chats):
+ for i in range(3):
+ res = rest_client.post("/chats", json={"name": f"chat_page_{i}", "dataset_ids": []})
+ assert res.status_code == 200
+ payload = res.json()
+ assert payload["code"] == 0, payload
+
+ page_res = rest_client.get("/chats", params={"page": 1, "page_size": 2, "orderby": "create_time", "desc": "true"})
+ assert page_res.status_code == 200
+ page_payload = page_res.json()
+ assert page_payload["code"] == 0, page_payload
+ assert len(page_payload["data"]["chats"]) == 2, page_payload
+ assert page_payload["data"]["total"] >= 3, page_payload
+
+
+@pytest.mark.p1
+def test_chat_delete_requires_auth():
+ for scenario_name, client in (("missing token", RestClient(token=None)), ("invalid token", RestClient(token=INVALID_API_TOKEN))):
+ res = client.delete("/chats", json={"ids": []})
+ assert res.status_code == 401, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == 401, (scenario_name, payload)
+ assert payload["message"] == "", (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_delete_basic_scenarios(rest_client, clear_chats):
+ existing_ids = _reset_chat_batch(rest_client, "delete_basic")
+ existing_res = rest_client.delete("/chats", json={"ids": existing_ids})
+ assert existing_res.status_code == 200
+ existing_payload = existing_res.json()
+ assert existing_payload["code"] == 0, existing_payload
+ assert existing_payload["data"]["success_count"] == len(existing_ids), existing_payload
+
+ list_after_existing = rest_client.get("/chats").json()
+ assert list_after_existing["code"] == 0, list_after_existing
+ assert list_after_existing["data"]["chats"] == [], list_after_existing
+
+ empty_res = rest_client.delete("/chats", json={"ids": []})
+ assert empty_res.status_code == 200
+ empty_payload = empty_res.json()
+ assert empty_payload["code"] == 0, empty_payload
+ assert empty_payload["message"] == "success", empty_payload
+
+ delete_all_ids = _reset_chat_batch(rest_client, "delete_all")
+ delete_all_res = rest_client.delete("/chats", json={"ids": None, "delete_all": True})
+ assert delete_all_res.status_code == 200
+ delete_all_payload = delete_all_res.json()
+ assert delete_all_payload["code"] == 0, delete_all_payload
+ assert delete_all_payload["data"]["success_count"] == len(delete_all_ids), delete_all_payload
+
+ list_after_delete_all = rest_client.get("/chats").json()
+ assert list_after_delete_all["code"] == 0, list_after_delete_all
+ assert list_after_delete_all["data"]["chats"] == [], list_after_delete_all
+
+
+@pytest.mark.p2
+def test_chat_delete_error_and_repeat_contract(rest_client, clear_chats):
+ partial_cases = [
+ ("partial invalid id", lambda ids: {"ids": ids + ["invalid_id"]}),
+ ("partial invalid punctuation id", lambda ids: {"ids": ids + ["!@#$%^&*()"]}),
+ ]
+ for scenario_name, payload in partial_cases:
+ ids = _reset_chat_batch(rest_client, f"delete_partial_{scenario_name.replace(' ', '_')}")
+ res = rest_client.delete("/chats", json=payload(ids))
+ assert res.status_code == 200, (scenario_name, res.text)
+ body = res.json()
+ assert body["code"] == 0, (scenario_name, body)
+ assert len(body["data"]["errors"]) == 1, (scenario_name, body)
+ assert body["data"]["success_count"] == 5, (scenario_name, body)
+
+ list_payload = rest_client.get("/chats").json()
+ assert list_payload["code"] == 0, (scenario_name, list_payload)
+ assert list_payload["data"]["chats"] == [], (scenario_name, list_payload)
+
+ duplicate_ids = _reset_chat_batch(rest_client, "delete_duplicate_all")
+ duplicate_all_res = rest_client.delete("/chats", json={"ids": duplicate_ids + duplicate_ids})
+ assert duplicate_all_res.status_code == 200
+ duplicate_all_payload = duplicate_all_res.json()
+ assert duplicate_all_payload["code"] == 0, duplicate_all_payload
+ assert duplicate_all_payload["data"]["success_count"] == 5, duplicate_all_payload
+ assert len(duplicate_all_payload["data"]["errors"]) == 5, duplicate_all_payload
+ assert all(error.startswith("Duplicate chat ids: ") for error in duplicate_all_payload["data"]["errors"]), duplicate_all_payload
+
+ duplicate_one_ids = _reset_chat_batch(rest_client, "delete_duplicate_one")
+ duplicate_one_res = rest_client.delete("/chats", json={"ids": [duplicate_one_ids[0], duplicate_one_ids[0]]})
+ assert duplicate_one_res.status_code == 200
+ duplicate_one_payload = duplicate_one_res.json()
+ assert duplicate_one_payload["code"] == 0, duplicate_one_payload
+ assert duplicate_one_payload["data"]["success_count"] == 1, duplicate_one_payload
+ assert duplicate_one_payload["data"]["errors"] == [f"Duplicate chat ids: {duplicate_one_ids[0]}"], duplicate_one_payload
+
+ all_missing_res = rest_client.delete("/chats", json={"ids": ["missing-1", "missing-2"]})
+ assert all_missing_res.status_code == 200
+ all_missing_payload = all_missing_res.json()
+ assert all_missing_payload["code"] == 102, all_missing_payload
+ assert "Chat(missing-1) not found." in all_missing_payload["message"], all_missing_payload
+ assert "Chat(missing-2) not found." in all_missing_payload["message"], all_missing_payload
+
+ repeated_ids = _reset_chat_batch(rest_client, "delete_repeated")
+ first_res = rest_client.delete("/chats", json={"ids": repeated_ids})
+ assert first_res.status_code == 200
+ first_payload = first_res.json()
+ assert first_payload["code"] == 0, first_payload
+ assert first_payload["data"]["success_count"] == 5, first_payload
+
+ second_res = rest_client.delete("/chats", json={"ids": repeated_ids})
+ assert second_res.status_code == 200
+ second_payload = second_res.json()
+ assert second_payload["code"] == 102, second_payload
+ for chat_id in repeated_ids:
+ assert f"Chat({chat_id}) not found." in second_payload["message"], second_payload
+
+
+@pytest.mark.p2
+def test_chat_delete_concurrent_and_bulk_contract(rest_client, clear_chats):
+ concurrent_ids = _reset_chat_batch(rest_client, "delete_concurrent", count=20)
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ results = list(executor.map(lambda chat_id: rest_client.delete("/chats", json={"ids": [chat_id]}).json(), concurrent_ids))
+ assert len(results) == 20, results
+ assert all(result["code"] == 0 for result in results), results
+ assert all(result["data"]["success_count"] == 1 for result in results), results
+
+ list_after_concurrent = rest_client.get("/chats").json()
+ assert list_after_concurrent["code"] == 0, list_after_concurrent
+ assert list_after_concurrent["data"]["chats"] == [], list_after_concurrent
+
+ bulk_ids = _reset_chat_batch(rest_client, "delete_bulk", count=100)
+ bulk_res = rest_client.delete("/chats", json={"ids": bulk_ids})
+ assert bulk_res.status_code == 200
+ bulk_payload = bulk_res.json()
+ assert bulk_payload["code"] == 0, bulk_payload
+ assert bulk_payload["data"]["success_count"] == len(bulk_ids), bulk_payload
+
+
+@pytest.mark.p1
+def test_chat_list_requires_auth():
+ for scenario_name, client in (("missing token", RestClient(token=None)), ("invalid token", RestClient(token=INVALID_API_TOKEN))):
+ res = client.get("/chats")
+ assert res.status_code == 401, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == 401, (scenario_name, payload)
+ assert payload["message"] == "", (scenario_name, payload)
+
+
+@pytest.mark.p1
+def test_chat_list_default_get_and_separate_lookup_contract(rest_client, clear_chats):
+ ids = _reset_chat_batch(rest_client, "list_default")
+
+ default_res = rest_client.get("/chats")
+ assert default_res.status_code == 200
+ default_payload = default_res.json()
+ assert default_payload["code"] == 0, default_payload
+ assert len(default_payload["data"]["chats"]) == 5, default_payload
+ assert default_payload["data"]["total"] == 5, default_payload
+
+ valid_get_res = rest_client.get(f"/chats/{ids[0]}")
+ assert valid_get_res.status_code == 200
+ valid_get_payload = valid_get_res.json()
+ assert valid_get_payload["code"] == 0, valid_get_payload
+ assert valid_get_payload["data"]["id"] == ids[0], valid_get_payload
+
+ invalid_get_res = rest_client.get("/chats/unknown")
+ assert invalid_get_res.status_code == 200
+ invalid_get_payload = invalid_get_res.json()
+ assert invalid_get_payload["code"] == 109, invalid_get_payload
+ assert invalid_get_payload["message"] == "No authorization.", invalid_get_payload
+
+ for chat_id, keywords, expected_count in ((ids[0], "list_default_0", 1), (ids[0], "list_default_1", 1), (ids[0], "unknown", 0)):
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ list_res = rest_client.get("/chats", params={"keywords": keywords})
+ assert get_res.status_code == 200, (keywords, get_res.text)
+ assert list_res.status_code == 200, (keywords, list_res.text)
+ get_payload = get_res.json()
+ list_payload = list_res.json()
+ assert get_payload["code"] == 0, (keywords, get_payload)
+ assert list_payload["code"] == 0, (keywords, list_payload)
+ assert len(list_payload["data"]["chats"]) == expected_count, (keywords, list_payload)
+
+
+@pytest.mark.p2
+def test_chat_list_keyword_and_invalid_param_contract(rest_client, clear_chats):
+ _reset_chat_batch(rest_client, "list_keyword")
+ cases = [
+ ("keywords none", {"keywords": None}, 5, None),
+ ("keywords empty", {"keywords": ""}, 5, None),
+ ("keywords exact", {"keywords": "list_keyword_1"}, 1, "list_keyword_1"),
+ ("keywords unknown", {"keywords": "unknown"}, 0, None),
+ ("invalid params ignored", {"a": "b"}, 5, None),
+ ]
+
+ for scenario_name, params, expected_count, expected_name in cases:
+ res = rest_client.get("/chats", params=params)
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == 0, (scenario_name, payload)
+ assert len(payload["data"]["chats"]) == expected_count, (scenario_name, payload)
+ if expected_name is not None:
+ assert payload["data"]["chats"][0]["name"] == expected_name, (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_list_page_and_page_size_contract(rest_client, clear_chats):
+ cases = [
+ ("page none", {"page": None, "page_size": 2}, 0, lambda total: total, ""),
+ ("page zero", {"page": 0, "page_size": 2}, 0, lambda total: total, ""),
+ ("page two", {"page": 2, "page_size": 2}, 0, lambda total: min(max(total - 2, 0), 2), ""),
+ ("page three", {"page": 3, "page_size": 2}, 0, lambda total: min(max(total - 4, 0), 2), ""),
+ ("page string", {"page": "3", "page_size": 2}, 0, lambda total: min(max(total - 4, 0), 2), ""),
+ ("page negative", {"page": -1, "page_size": 2}, 100, None, "ProgrammingError(1064"),
+ ("page alpha", {"page": "a", "page_size": 2}, 100, None, "ValueError(\"invalid literal for int() with base 10: 'a'\")"),
+ ("page_size none", {"page_size": None}, 0, lambda total: total, ""),
+ ("page_size zero", {"page_size": 0}, 0, lambda total: total, ""),
+ ("page_size one", {"page_size": 1}, 0, lambda total: total, ""),
+ ("page_size six", {"page_size": 6}, 0, lambda total: total, ""),
+ ("page_size string", {"page_size": "1"}, 0, lambda total: total, ""),
+ ("page_size negative", {"page_size": -1}, 0, lambda total: total, ""),
+ ("page_size alpha", {"page_size": "a"}, 100, None, "ValueError(\"invalid literal for int() with base 10: 'a'\")"),
+ ]
+
+ for scenario_name, params, expected_code, expected_count_fn, expected_message in cases:
+ _reset_chat_batch(rest_client, f"list_page_{scenario_name.replace(' ', '_')}")
+ baseline_payload = rest_client.get("/chats").json()
+ assert baseline_payload["code"] == 0, (scenario_name, baseline_payload)
+ baseline_total = baseline_payload["data"]["total"]
+
+ res = rest_client.get("/chats", params=params)
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == expected_code, (scenario_name, payload)
+ if expected_code == 0:
+ assert len(payload["data"]["chats"]) == expected_count_fn(baseline_total), (scenario_name, payload)
+ assert payload["data"]["total"] == baseline_total, (scenario_name, payload)
+ else:
+ assert expected_message in payload["message"], (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_list_sorting_contract(rest_client, clear_chats):
+ _reset_chat_batch(rest_client, "list_sort")
+ ascending_names = [f"list_sort_{i}" for i in range(5)]
+ descending_names = list(reversed(ascending_names))
+ cases = [
+ ("orderby none", {"orderby": None}, 0, descending_names, ""),
+ ("orderby create", {"orderby": "create_time"}, 0, descending_names, ""),
+ ("orderby update", {"orderby": "update_time"}, 0, descending_names, ""),
+ ("orderby name ascending", {"orderby": "name", "desc": "False"}, 0, ascending_names, ""),
+ ("orderby unknown", {"orderby": "unknown"}, 100, None, "AttributeError(\"type object 'Dialog' has no attribute 'unknown'\")"),
+ ("desc none", {"desc": None}, 0, descending_names, ""),
+ ("desc true", {"desc": "true"}, 0, descending_names, ""),
+ ("desc True", {"desc": "True"}, 0, descending_names, ""),
+ ("desc bool true", {"desc": True}, 0, descending_names, ""),
+ ("desc false", {"desc": "false"}, 0, ascending_names, ""),
+ ("desc False", {"desc": "False"}, 0, ascending_names, ""),
+ ("desc bool false", {"desc": False}, 0, ascending_names, ""),
+ ("desc False update_time", {"desc": "False", "orderby": "update_time"}, 0, ascending_names, ""),
+ ("desc unknown", {"desc": "unknown"}, 0, descending_names, ""),
+ ]
+
+ for scenario_name, params, expected_code, expected_names, expected_message in cases:
+ res = rest_client.get("/chats", params=params)
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == expected_code, (scenario_name, payload)
+ if expected_code == 0:
+ assert _chat_names(payload) == expected_names, (scenario_name, payload)
+ else:
+ assert expected_message in payload["message"], (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_list_concurrent_and_dataset_delete_contract(rest_client, clear_chats, ensure_parsed_document):
+ _reset_chat_batch(rest_client, "list_concurrent")
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ results = list(executor.map(lambda _idx: rest_client.get("/chats").json(), range(10)))
+ assert len(results) == 10, results
+ assert all(result["code"] == 0 for result in results), results
+ assert all(len(result["data"]["chats"]) == 5 for result in results), results
+
+ dataset_id, _ = ensure_parsed_document()
+ create_res = rest_client.post("/chats", json={"name": "list_after_dataset_delete", "dataset_ids": [dataset_id]})
+ assert create_res.status_code == 200
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, create_payload
+
+ delete_dataset_res = rest_client.delete("/datasets", json={"ids": [dataset_id]})
+ assert delete_dataset_res.status_code == 200
+ delete_dataset_payload = delete_dataset_res.json()
+ assert delete_dataset_payload["code"] == 0, delete_dataset_payload
+
+ list_res = rest_client.get("/chats", params={"keywords": "list_after_dataset_delete"})
+ assert list_res.status_code == 200
+ list_payload = list_res.json()
+ assert list_payload["code"] == 0, list_payload
+ assert len(list_payload["data"]["chats"]) == 1, list_payload
+
+
+class _DummyManager:
+ def route(self, *_args, **_kwargs):
+ def decorator(func):
+ return func
+
+ return decorator
+
+
+class _AwaitableValue:
+ def __init__(self, value):
+ self._value = value
+
+ def __await__(self):
+ async def _co():
+ return self._value
+
+ return _co().__await__()
+
+
+class _DummyArgs(dict):
+ def get(self, key, default=None):
+ return super().get(key, default)
+
+ def getlist(self, key):
+ value = self.get(key, [])
+ if value is None:
+ return []
+ if isinstance(value, list):
+ return value
+ return [value]
+
+
+class _StubHeaders:
+ def __init__(self):
+ self._items = []
+
+ def add_header(self, key, value):
+ self._items.append((key, value))
+
+ def get(self, key, default=None):
+ for existing_key, value in reversed(self._items):
+ if existing_key == key:
+ return value
+ return default
+
+
+class _StubResponse:
+ def __init__(self, body=None, mimetype=None, content_type=None):
+ self.body = body
+ self.mimetype = mimetype
+ self.content_type = content_type
+ self.headers = _StubHeaders()
+
+
+class _DummyUploadFile:
+ def __init__(self, filename):
+ self.filename = filename
+ self.saved_path = None
+
+ async def save(self, path):
+ self.saved_path = path
+
+
+def _passthrough_login_required(func):
+ @wraps(func)
+ async def _wrapper(*args, **kwargs):
+ return await func(*args, **kwargs)
+
+ return _wrapper
+
+
+class _DummyKB:
+ def __init__(self, kid="kb-1", embd_id="embd@factory", chunk_num=1, name="Dataset A", status="1"):
+ self.id = kid
+ self.embd_id = embd_id
+ self.chunk_num = chunk_num
+ self.name = name
+ self.status = status
+
+
+class _DummyDialogRecord:
+ def __init__(self, data=None):
+ self._data = data or {
+ "id": "chat-1",
+ "name": "chat-name",
+ "description": "desc",
+ "icon": "icon.png",
+ "kb_ids": ["kb-1"],
+ "llm_id": "glm-4",
+ "llm_setting": {"temperature": 0.1},
+ "prompt_config": {
+ "system": "Answer with {knowledge}",
+ "parameters": [{"key": "knowledge", "optional": False}],
+ "prologue": "hello",
+ "quote": True,
+ },
+ "similarity_threshold": 0.2,
+ "vector_similarity_weight": 0.3,
+ "top_n": 6,
+ "top_k": 1024,
+ "rerank_id": "",
+ "meta_data_filter": {},
+ "tenant_id": "tenant-1",
+ }
+
+ def to_dict(self):
+ return deepcopy(self._data)
+
+
+def _run(coro):
+ return asyncio.run(coro)
+
+
+async def _collect_stream(body):
+ items = []
+ if hasattr(body, "__aiter__"):
+ async for item in body:
+ if isinstance(item, bytes):
+ item = item.decode("utf-8")
+ items.append(item)
+ else:
+ for item in body:
+ if isinstance(item, bytes):
+ item = item.decode("utf-8")
+ items.append(item)
+ return items
+
+
+def _load_chat_routes_unit_module(monkeypatch):
+ repo_root = Path(__file__).resolve().parents[3]
+ module_name = "test_chat_restful_routes_unit_module"
+ module_path = repo_root / "api" / "apps" / "restful_apis" / "chat_api.py"
+
+ quart_mod = ModuleType("quart")
+ quart_mod.request = SimpleNamespace(args=_DummyArgs())
+ quart_mod.Response = _StubResponse
+ monkeypatch.setitem(sys.modules, "quart", quart_mod)
+
+ api_pkg = ModuleType("api")
+ api_pkg.__path__ = [str(repo_root / "api")]
+ monkeypatch.setitem(sys.modules, "api", api_pkg)
+
+ apps_pkg = ModuleType("api.apps")
+ apps_pkg.__path__ = [str(repo_root / "api" / "apps")]
+ apps_pkg.current_user = SimpleNamespace(id="tenant-1")
+ apps_pkg.login_required = _passthrough_login_required
+ monkeypatch.setitem(sys.modules, "api.apps", apps_pkg)
+ api_pkg.apps = apps_pkg
+
+ common_pkg = ModuleType("common")
+ common_pkg.__path__ = [str(repo_root / "common")]
+ monkeypatch.setitem(sys.modules, "common", common_pkg)
+
+ common_constants_mod = ModuleType("common.constants")
+
+ class _StubLLMType(str, Enum):
+ CHAT = "chat"
+ IMAGE2TEXT = "image2text"
+ RERANK = "rerank"
+ SPEECH2TEXT = "speech2text"
+ TTS = "tts"
+
+ class _StubRetCode(int, Enum):
+ SUCCESS = 0
+ DATA_ERROR = 102
+ OPERATING_ERROR = 103
+ AUTHENTICATION_ERROR = 109
+
+ class _StubStatusEnum(str, Enum):
+ VALID = "1"
+ INVALID = "0"
+
+ common_constants_mod.LLMType = _StubLLMType
+ common_constants_mod.RetCode = _StubRetCode
+ common_constants_mod.StatusEnum = _StubStatusEnum
+ from common.constants import MAXIMUM_PAGE_NUMBER as _MPN, MAXIMUM_TASK_PAGE_NUMBER as _MTPN
+ common_constants_mod.MAXIMUM_PAGE_NUMBER = _MPN
+ common_constants_mod.MAXIMUM_TASK_PAGE_NUMBER = _MTPN
+ monkeypatch.setitem(sys.modules, "common.constants", common_constants_mod)
+
+ misc_utils_mod = ModuleType("common.misc_utils")
+ misc_utils_mod.get_uuid = lambda: "generated-chat-id"
+
+ async def _thread_pool_exec(func, *args, **kwargs):
+ return func(*args, **kwargs)
+
+ misc_utils_mod.thread_pool_exec = _thread_pool_exec
+ monkeypatch.setitem(sys.modules, "common.misc_utils", misc_utils_mod)
+
+ settings_mod = ModuleType("common.settings")
+ settings_mod.STORAGE_IMPL = type("_StorageImpl", (), {"rm": staticmethod(lambda *_args, **_kwargs: None)})()
+ monkeypatch.setitem(sys.modules, "common.settings", settings_mod)
+
+ dialog_service_mod = ModuleType("api.db.services.dialog_service")
+
+ class _StubDialogService:
+ model = SimpleNamespace(
+ _meta=SimpleNamespace(
+ fields={
+ "id": None,
+ "tenant_id": None,
+ "name": None,
+ "description": None,
+ "icon": None,
+ "kb_ids": None,
+ "llm_id": None,
+ "llm_setting": None,
+ "prompt_config": None,
+ "similarity_threshold": None,
+ "vector_similarity_weight": None,
+ "top_n": None,
+ "top_k": None,
+ "rerank_id": None,
+ "meta_data_filter": None,
+ "created_by": None,
+ "create_time": None,
+ "create_date": None,
+ "update_time": None,
+ "update_date": None,
+ "status": None,
+ }
+ )
+ )
+
+ @staticmethod
+ def query(**_kwargs):
+ return []
+
+ @staticmethod
+ def save(**_kwargs):
+ return True
+
+ @staticmethod
+ def get_by_id(_chat_id):
+ return False, None
+
+ @staticmethod
+ def update_by_id(_chat_id, _payload):
+ return True
+
+ @staticmethod
+ def get_by_tenant_ids(*_args, **_kwargs):
+ return [], 0
+
+ dialog_service_mod.DialogService = _StubDialogService
+ dialog_service_mod.async_ask = lambda *_args, **_kwargs: None
+ dialog_service_mod.async_chat = lambda *_args, **_kwargs: None
+ dialog_service_mod.gen_mindmap = lambda *_args, **_kwargs: None
+ monkeypatch.setitem(sys.modules, "api.db.services.dialog_service", dialog_service_mod)
+
+ conversation_service_mod = ModuleType("api.db.services.conversation_service")
+
+ class _StubConversationService:
+ @staticmethod
+ def query(**_kwargs):
+ return []
+
+ @staticmethod
+ def get_list(*_args, **_kwargs):
+ return []
+
+ @staticmethod
+ def get_by_id(_session_id):
+ return False, None
+
+ @staticmethod
+ def update_by_id(_session_id, _payload):
+ return True
+
+ @staticmethod
+ def delete_by_id(_session_id):
+ return True
+
+ @staticmethod
+ def save(**_kwargs):
+ return True
+
+ conversation_service_mod.ConversationService = _StubConversationService
+ conversation_service_mod.structure_answer = lambda *_args, **_kwargs: {}
+ monkeypatch.setitem(sys.modules, "api.db.services.conversation_service", conversation_service_mod)
+
+ kb_service_mod = ModuleType("api.db.services.knowledgebase_service")
+
+ class _StubKnowledgebaseService:
+ @staticmethod
+ def accessible(**_kwargs):
+ return []
+
+ @staticmethod
+ def query(**_kwargs):
+ return []
+
+ @staticmethod
+ def get_by_id(_kb_id):
+ return False, None
+
+ kb_service_mod.KnowledgebaseService = _StubKnowledgebaseService
+ monkeypatch.setitem(sys.modules, "api.db.services.knowledgebase_service", kb_service_mod)
+
+ tenant_llm_service_mod = ModuleType("api.db.services.tenant_llm_service")
+
+ class _StubTenantLLMService:
+ @staticmethod
+ def split_model_name_and_factory(model_name):
+ if model_name and "@" in model_name:
+ return tuple(model_name.split("@", 1))
+ return model_name, None
+
+ @staticmethod
+ def query(**_kwargs):
+ return []
+
+ @staticmethod
+ def get_api_key(*_args, **_kwargs):
+ return SimpleNamespace(id=1)
+
+ tenant_llm_service_mod.TenantLLMService = _StubTenantLLMService
+ monkeypatch.setitem(sys.modules, "api.db.services.tenant_llm_service", tenant_llm_service_mod)
+
+ llm_service_mod = ModuleType("api.db.services.llm_service")
+ llm_service_mod.LLMBundle = lambda *_args, **_kwargs: None
+ monkeypatch.setitem(sys.modules, "api.db.services.llm_service", llm_service_mod)
+
+ search_service_mod = ModuleType("api.db.services.search_service")
+ search_service_mod.SearchService = SimpleNamespace()
+ monkeypatch.setitem(sys.modules, "api.db.services.search_service", search_service_mod)
+
+ tenant_model_service_mod = ModuleType("api.db.joint_services.tenant_model_service")
+ tenant_model_service_mod.get_model_config_by_type_and_name = lambda *_args, **_kwargs: {}
+ tenant_model_service_mod.get_tenant_default_model_by_type = lambda *_args, **_kwargs: {}
+ monkeypatch.setitem(sys.modules, "api.db.joint_services.tenant_model_service", tenant_model_service_mod)
+
+ user_service_mod = ModuleType("api.db.services.user_service")
+
+ class _StubTenantService:
+ @staticmethod
+ def get_by_id(_tenant_id):
+ return True, SimpleNamespace(llm_id="glm-4")
+
+ @staticmethod
+ def get_joined_tenants_by_user_id(_user_id):
+ return [{"tenant_id": "tenant-1"}, {"tenant_id": "team-tenant-2"}]
+
+ class _StubUserTenantService:
+ @staticmethod
+ def query(**_kwargs):
+ return []
+
+ user_service_mod.UserService = type("UserService", (), {})
+ user_service_mod.TenantService = _StubTenantService
+ user_service_mod.UserTenantService = _StubUserTenantService
+ monkeypatch.setitem(sys.modules, "api.db.services.user_service", user_service_mod)
+
+ chunk_feedback_service_mod = ModuleType("api.db.services.chunk_feedback_service")
+ chunk_feedback_service_mod.ChunkFeedbackService = type(
+ "ChunkFeedbackService",
+ (),
+ {"apply_feedback": staticmethod(lambda **_kwargs: {"success_count": 0, "fail_count": 0, "chunk_ids": []})},
+ )
+ monkeypatch.setitem(sys.modules, "api.db.services.chunk_feedback_service", chunk_feedback_service_mod)
+
+ api_utils_mod = ModuleType("api.utils.api_utils")
+
+ def _check_duplicate_ids(ids, label):
+ counts = {}
+ for item in ids or []:
+ counts[item] = counts.get(item, 0) + 1
+ duplicate_messages = [f"Duplicate {label} ids: {item}" for item, count in counts.items() if count > 1]
+ return list(dict.fromkeys(ids or [])), duplicate_messages
+
+ api_utils_mod.check_duplicate_ids = _check_duplicate_ids
+ api_utils_mod.get_data_error_result = lambda message="": {"code": 102, "data": None, "message": message}
+ api_utils_mod.get_json_result = lambda data=None, message="", code=0: {"code": code, "data": data, "message": message}
+ api_utils_mod.get_request_json = lambda: _AwaitableValue({})
+ api_utils_mod.server_error_response = lambda ex: {"code": 500, "data": None, "message": str(ex)}
+ api_utils_mod.validate_request = lambda *_args, **_kwargs: (lambda func: func)
+ monkeypatch.setitem(sys.modules, "api.utils.api_utils", api_utils_mod)
+
+ tenant_utils_mod = ModuleType("api.utils.tenant_utils")
+ tenant_utils_mod.ensure_tenant_model_id_for_params = lambda _tenant_id, req: req
+ monkeypatch.setitem(sys.modules, "api.utils.tenant_utils", tenant_utils_mod)
+
+ rag_pkg = ModuleType("rag")
+ rag_pkg.__path__ = [str(repo_root / "rag")]
+ monkeypatch.setitem(sys.modules, "rag", rag_pkg)
+
+ rag_prompts_pkg = ModuleType("rag.prompts")
+ rag_prompts_pkg.__path__ = [str(repo_root / "rag" / "prompts")]
+ monkeypatch.setitem(sys.modules, "rag.prompts", rag_prompts_pkg)
+
+ rag_prompts_generator_mod = ModuleType("rag.prompts.generator")
+ rag_prompts_generator_mod.chunks_format = lambda reference: reference.get("chunks", []) if isinstance(reference, dict) else []
+ monkeypatch.setitem(sys.modules, "rag.prompts.generator", rag_prompts_generator_mod)
+
+ rag_prompts_template_mod = ModuleType("rag.prompts.template")
+ rag_prompts_template_mod.load_prompt = lambda *_args, **_kwargs: ""
+ monkeypatch.setitem(sys.modules, "rag.prompts.template", rag_prompts_template_mod)
+
+ spec = importlib.util.spec_from_file_location(module_name, module_path)
+ module = importlib.util.module_from_spec(spec)
+ module.manager = _DummyManager()
+ monkeypatch.setitem(sys.modules, module_name, module)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _set_route_unit_request_json(monkeypatch, module, payload):
+ monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue(deepcopy(payload)))
+
+
+@pytest.mark.p2
+def test_chat_session_create_and_update_guard_matrix_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+
+ _set_route_unit_request_json(monkeypatch, module, {"name": "session"})
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [])
+ res = _run(module.create_session.__wrapped__("chat-1"))
+ assert res["message"] == "No authorization."
+
+ dia = SimpleNamespace(prompt_config={"prologue": "hello"})
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [dia])
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, dia))
+ monkeypatch.setattr(module.ConversationService, "save", lambda **_kwargs: None)
+ monkeypatch.setattr(module.ConversationService, "get_by_id", lambda _id: (False, None))
+ res = _run(module.create_session.__wrapped__("chat-1"))
+ assert "Fail to create a session" in res["message"]
+
+ _set_route_unit_request_json(monkeypatch, module, {})
+ monkeypatch.setattr(module.ConversationService, "query", lambda **_kwargs: [])
+ res = _run(module.update_session.__wrapped__("chat-1", "session-1"))
+ assert res["message"] == "Session not found!"
+
+ monkeypatch.setattr(module.ConversationService, "query", lambda **_kwargs: [SimpleNamespace(id="session-1")])
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [])
+ res = _run(module.update_session.__wrapped__("chat-1", "session-1"))
+ assert res["message"] == "No authorization."
+
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [SimpleNamespace(id="chat-1")])
+ _set_route_unit_request_json(monkeypatch, module, {"message": []})
+ res = _run(module.update_session.__wrapped__("chat-1", "session-1"))
+ assert "`messages` cannot be changed." in res["message"]
+
+ _set_route_unit_request_json(monkeypatch, module, {"reference": []})
+ res = _run(module.update_session.__wrapped__("chat-1", "session-1"))
+ assert "`reference` cannot be changed." in res["message"]
+
+ _set_route_unit_request_json(monkeypatch, module, {"name": ""})
+ res = _run(module.update_session.__wrapped__("chat-1", "session-1"))
+ assert "`name` can not be empty." in res["message"]
+
+ _set_route_unit_request_json(monkeypatch, module, {"name": "renamed"})
+ monkeypatch.setattr(module.ConversationService, "update_by_id", lambda *_args, **_kwargs: False)
+ res = _run(module.update_session.__wrapped__("chat-1", "session-1"))
+ assert res["message"] == "Session not found!"
+
+
+@pytest.mark.p2
+def test_chat_session_list_projection_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=SimpleNamespace(
+ get=lambda key, default=None: {
+ "page": 1,
+ "page_size": 30,
+ "orderby": "create_time",
+ "desc": "true",
+ "id": None,
+ "name": None,
+ "user_id": None,
+ }.get(key, default)
+ )
+ ),
+ )
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [SimpleNamespace(id="chat-1")])
+ monkeypatch.setattr(
+ module.ConversationService,
+ "get_list",
+ lambda *_args, **_kwargs: [
+ {
+ "id": "session-1",
+ "dialog_id": "chat-1",
+ "message": [{"role": "assistant", "content": "hello"}],
+ "reference": [],
+ }
+ ],
+ )
+
+ res = _run(module.list_sessions.__wrapped__("chat-1"))
+ assert res["data"][0]["chat_id"] == "chat-1"
+ assert res["data"][0]["messages"][0]["content"] == "hello"
+
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=SimpleNamespace(
+ get=lambda key, default=None: {
+ "page": 1,
+ "page_size": 0,
+ "orderby": "create_time",
+ "desc": "true",
+ "id": None,
+ "name": None,
+ "user_id": None,
+ }.get(key, default)
+ )
+ ),
+ )
+ res = _run(module.list_sessions.__wrapped__("chat-1"))
+ assert res["data"] == []
+
+
+@pytest.mark.p2
+def test_chat_session_delete_routes_partial_duplicate_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [SimpleNamespace(id="chat-1")])
+ _set_route_unit_request_json(monkeypatch, module, {})
+ res = _run(module.delete_sessions.__wrapped__("chat-1"))
+ assert res["code"] == 0
+
+ monkeypatch.setattr(module.ConversationService, "delete_by_id", lambda *_args, **_kwargs: True)
+
+ def _conversation_query(**kwargs):
+ if "dialog_id" in kwargs and "id" not in kwargs:
+ return [SimpleNamespace(id="seed")]
+ if kwargs.get("id") == "ok":
+ return [SimpleNamespace(id="ok")]
+ return []
+
+ monkeypatch.setattr(module.ConversationService, "query", _conversation_query)
+ _set_route_unit_request_json(monkeypatch, module, {"ids": ["ok", "bad"]})
+ monkeypatch.setattr(module, "check_duplicate_ids", lambda ids, _kind: (ids, []))
+ res = _run(module.delete_sessions.__wrapped__("chat-1"))
+ assert res["code"] == 0
+ assert res["data"]["success_count"] == 1
+ assert res["data"]["errors"] == ["The chat doesn't own the session bad"]
+
+ _set_route_unit_request_json(monkeypatch, module, {"ids": ["bad"]})
+ monkeypatch.setattr(module, "check_duplicate_ids", lambda ids, _kind: (ids, []))
+ res = _run(module.delete_sessions.__wrapped__("chat-1"))
+ assert res["message"] == "The chat doesn't own the session bad"
+
+ _set_route_unit_request_json(monkeypatch, module, {"ids": ["ok", "ok"]})
+ monkeypatch.setattr(module, "check_duplicate_ids", lambda ids, _kind: (["ok"], ["Duplicate session ids: ok"]))
+ res = _run(module.delete_sessions.__wrapped__("chat-1"))
+ assert res["code"] == 0
+ assert res["data"]["success_count"] == 1
+ assert res["data"]["errors"] == ["Duplicate session ids: ok"]
+
+
+@pytest.mark.p2
+def test_chat_audio_transcription_routes_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ monkeypatch.setattr(module, "Response", _StubResponse)
+ monkeypatch.setattr(module.tempfile, "mkstemp", lambda suffix: (11, f"/tmp/audio{suffix}"))
+ monkeypatch.setattr(module.os, "close", lambda _fd: None)
+
+ def _set_request(form, files):
+ monkeypatch.setattr(module, "request", SimpleNamespace(form=_AwaitableValue(form), files=_AwaitableValue(files)))
+
+ _set_request({"stream": "false"}, {})
+ res = _run(module.transcription.__wrapped__())
+ assert "Missing 'file' in multipart form-data" in res["message"]
+
+ _set_request({"stream": "false"}, {"file": _DummyUploadFile("bad.txt")})
+ res = _run(module.transcription.__wrapped__())
+ assert "Unsupported audio format: .txt" in res["message"]
+
+ _set_request({"stream": "false"}, {"file": _DummyUploadFile("audio.wav")})
+ monkeypatch.setattr(
+ module,
+ "get_tenant_default_model_by_type",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(LookupError("Tenant not found!")),
+ )
+ res = _run(module.transcription.__wrapped__())
+ assert res["message"] == "Tenant not found!"
+
+ _set_request({"stream": "false"}, {"file": _DummyUploadFile("audio.wav")})
+ monkeypatch.setattr(
+ module,
+ "get_tenant_default_model_by_type",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(Exception("No default ASR model is set")),
+ )
+ res = _run(module.transcription.__wrapped__())
+ assert res["message"] == "No default ASR model is set"
+
+ class _SyncASR:
+ def transcription(self, _path):
+ return "transcribed text"
+
+ def stream_transcription(self, _path):
+ return []
+
+ _set_request({"stream": "false"}, {"file": _DummyUploadFile("audio.wav")})
+ monkeypatch.setattr(module, "get_tenant_default_model_by_type", lambda *_args, **_kwargs: {"llm_name": "asr-x"})
+ monkeypatch.setattr(module, "LLMBundle", lambda *_args, **_kwargs: _SyncASR())
+ monkeypatch.setattr(module.os, "remove", lambda _path: (_ for _ in ()).throw(RuntimeError("cleanup fail")))
+ res = _run(module.transcription.__wrapped__())
+ assert res["code"] == 0
+ assert res["data"]["text"] == "transcribed text"
+
+ class _StreamASR:
+ def transcription(self, _path):
+ return ""
+
+ def stream_transcription(self, _path):
+ yield {"event": "partial", "text": "hello"}
+
+ _set_request({"stream": "true"}, {"file": _DummyUploadFile("audio.wav")})
+ monkeypatch.setattr(module, "LLMBundle", lambda *_args, **_kwargs: _StreamASR())
+ monkeypatch.setattr(module.os, "remove", lambda _path: None)
+ resp = _run(module.transcription.__wrapped__())
+ assert isinstance(resp, _StubResponse)
+ assert resp.content_type == "text/event-stream"
+ chunks = _run(_collect_stream(resp.body))
+ assert any('"event": "partial"' in chunk for chunk in chunks)
+
+ class _ErrorASR:
+ def transcription(self, _path):
+ return ""
+
+ def stream_transcription(self, _path):
+ raise RuntimeError("stream asr boom")
+
+ _set_request({"stream": "true"}, {"file": _DummyUploadFile("audio.wav")})
+ monkeypatch.setattr(module, "LLMBundle", lambda *_args, **_kwargs: _ErrorASR())
+ monkeypatch.setattr(module.os, "remove", lambda _path: (_ for _ in ()).throw(RuntimeError("cleanup boom")))
+ resp = _run(module.transcription.__wrapped__())
+ chunks = _run(_collect_stream(resp.body))
+ assert any("stream asr boom" in chunk for chunk in chunks)
+
+
+@pytest.mark.p2
+def test_chat_audio_speech_routes_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ monkeypatch.setattr(module, "Response", _StubResponse)
+ _set_route_unit_request_json(monkeypatch, module, {"text": "A。B"})
+
+ monkeypatch.setattr(
+ module,
+ "get_tenant_default_model_by_type",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(LookupError("Tenant not found!")),
+ )
+ res = _run(module.tts.__wrapped__())
+ assert res["message"] == "Tenant not found!"
+
+ monkeypatch.setattr(
+ module,
+ "get_tenant_default_model_by_type",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(Exception("No default TTS model is set")),
+ )
+ res = _run(module.tts.__wrapped__())
+ assert res["message"] == "No default TTS model is set"
+
+ class _TTSOk:
+ def tts(self, txt):
+ if not txt:
+ return []
+ yield f"chunk-{txt}".encode("utf-8")
+
+ monkeypatch.setattr(module, "get_tenant_default_model_by_type", lambda *_args, **_kwargs: {"llm_name": "tts-x"})
+ monkeypatch.setattr(module, "LLMBundle", lambda *_args, **_kwargs: _TTSOk())
+ resp = _run(module.tts.__wrapped__())
+ assert resp.mimetype == "audio/mpeg"
+ assert resp.headers.get("Cache-Control") == "no-cache"
+ assert resp.headers.get("Connection") == "keep-alive"
+ assert resp.headers.get("X-Accel-Buffering") == "no"
+ chunks = _run(_collect_stream(resp.body))
+ assert any("chunk-A" in chunk for chunk in chunks)
+ assert any("chunk-B" in chunk for chunk in chunks)
+
+ class _TTSErr:
+ def tts(self, _txt):
+ raise RuntimeError("tts boom")
+
+ monkeypatch.setattr(module, "LLMBundle", lambda *_args, **_kwargs: _TTSErr())
+ resp = _run(module.tts.__wrapped__())
+ chunks = _run(_collect_stream(resp.body))
+ assert any('"code": 500' in chunk and "**ERROR**: tts boom" in chunk for chunk in chunks)
+
+
+@pytest.mark.p1
+def test_chat_create_accepts_provider_scoped_rerank_id_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ saved = {}
+ query_calls = []
+
+ _set_route_unit_request_json(
+ monkeypatch,
+ module,
+ {
+ "name": "chat-a",
+ "icon": "icon.png",
+ "dataset_ids": ["kb-1"],
+ "llm_id": "glm-4@ZHIPU-AI",
+ "llm_setting": {"temperature": 0.8},
+ "prompt_config": {
+ "system": "Answer with {knowledge}",
+ "parameters": [{"key": "knowledge", "optional": False}],
+ "prologue": "Hi",
+ },
+ "rerank_id": "custom-reranker@OpenAI",
+ "vector_similarity_weight": 0.25,
+ },
+ )
+ monkeypatch.setattr(module.TenantService, "get_by_id", lambda _tid: (True, SimpleNamespace(llm_id="glm-4@ZHIPU-AI")))
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [])
+ monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: [SimpleNamespace(id="kb-1")])
+ monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [_DummyKB()])
+ monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (True, _DummyKB()))
+
+ def _split_model_name_and_factory(model_name):
+ return {
+ "glm-4@ZHIPU-AI": ("glm-4", "ZHIPU-AI"),
+ "custom-reranker@OpenAI": ("custom-reranker", "OpenAI"),
+ }.get(model_name, (model_name, None))
+
+ def _query(**kwargs):
+ query_calls.append(kwargs)
+ if kwargs == {
+ "tenant_id": "tenant-1",
+ "llm_name": "glm-4",
+ "llm_factory": "ZHIPU-AI",
+ "model_type": "chat",
+ }:
+ return [SimpleNamespace(id="llm-1")]
+ if kwargs == {
+ "tenant_id": "tenant-1",
+ "llm_name": "custom-reranker",
+ "llm_factory": "OpenAI",
+ "model_type": "rerank",
+ }:
+ return [SimpleNamespace(id="rerank-1")]
+ return []
+
+ monkeypatch.setattr(module.TenantLLMService, "split_model_name_and_factory", _split_model_name_and_factory)
+ monkeypatch.setattr(module.TenantLLMService, "query", _query)
+
+ def _save(**kwargs):
+ saved.update(kwargs)
+ return True
+
+ monkeypatch.setattr(module.DialogService, "save", _save)
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, _DummyDialogRecord(saved)))
+
+ res = _run(module.create.__wrapped__())
+ assert res["code"] == 0
+ assert saved["rerank_id"] == "custom-reranker@OpenAI"
+ assert {
+ "tenant_id": "tenant-1",
+ "llm_name": "custom-reranker",
+ "llm_factory": "OpenAI",
+ "model_type": "rerank",
+ } in query_calls
+
+
+@pytest.mark.p1
+def test_chat_create_allows_default_knowledge_placeholder_without_sources_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ saved = {}
+ _set_route_unit_request_json(monkeypatch, module, {"name": "chat-a"})
+ monkeypatch.setattr(module.TenantService, "get_by_id", lambda _tid: (True, SimpleNamespace(llm_id="glm-4")))
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [])
+ monkeypatch.setattr(module.TenantLLMService, "get_api_key", lambda *_args, **_kwargs: SimpleNamespace(id=1))
+
+ def _save(**kwargs):
+ saved.update(kwargs)
+ return True
+
+ monkeypatch.setattr(module.DialogService, "save", _save)
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, _DummyDialogRecord(saved)))
+
+ res = _run(module.create.__wrapped__())
+ assert res["code"] == 0
+ assert saved["kb_ids"] == []
+ assert saved["prompt_config"]["system"].find("{knowledge}") >= 0
+ assert saved["prompt_config"]["parameters"] == [{"key": "knowledge", "optional": False}]
+
+
+@pytest.mark.p2
+def test_chat_create_uses_direct_chat_fields_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ saved = {}
+ _set_route_unit_request_json(
+ monkeypatch,
+ module,
+ {
+ "name": "chat-a",
+ "icon": "icon.png",
+ "dataset_ids": ["kb-1"],
+ "llm_id": "glm-4",
+ "llm_setting": {"temperature": 0.8},
+ "prompt_config": {
+ "system": "Answer with {knowledge}",
+ "parameters": [{"key": "knowledge", "optional": False}],
+ "prologue": "Hi",
+ },
+ "vector_similarity_weight": 0.25,
+ },
+ )
+ monkeypatch.setattr(module.TenantService, "get_by_id", lambda _tid: (True, SimpleNamespace(llm_id="glm-4")))
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [])
+ monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: [SimpleNamespace(id="kb-1")])
+ monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [_DummyKB()])
+ monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (True, _DummyKB()))
+ monkeypatch.setattr(module.TenantLLMService, "split_model_name_and_factory", lambda model: (model.split("@")[0], "factory"))
+ monkeypatch.setattr(module.TenantLLMService, "query", lambda **_kwargs: [SimpleNamespace(id="llm-1")])
+
+ def _save(**kwargs):
+ saved.update(kwargs)
+ return True
+
+ monkeypatch.setattr(module.DialogService, "save", _save)
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, _DummyDialogRecord(saved)))
+
+ res = _run(module.create.__wrapped__())
+ assert res["code"] == 0
+ assert saved["kb_ids"] == ["kb-1"]
+ assert saved["prompt_config"]["prologue"] == "Hi"
+ assert saved["llm_id"] == "glm-4"
+ assert saved["llm_setting"]["temperature"] == 0.8
+ assert res["data"]["dataset_ids"] == ["kb-1"]
+ assert res["data"]["kb_names"] == ["Dataset A"]
+ assert "kb_ids" not in res["data"]
+ assert "prompt" not in res["data"]
+ assert "llm" not in res["data"]
+ assert "avatar" not in res["data"]
+
+
+@pytest.mark.p2
+def test_list_chats_defaults_to_authorized_owner_ids_when_omitted_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ captured = {}
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=SimpleNamespace(
+ get=lambda key, default=None: {
+ "keywords": "",
+ "page": "1",
+ "page_size": "10",
+ "orderby": "create_time",
+ "desc": "true",
+ "id": None,
+ "name": None,
+ }.get(key, default),
+ getlist=lambda _key: [],
+ )
+ ),
+ )
+
+ def _get_by_tenant_ids(owner_ids, *_args, **_kwargs):
+ captured["owner_ids"] = owner_ids
+ return ([], 0)
+
+ monkeypatch.setattr(module.DialogService, "get_by_tenant_ids", _get_by_tenant_ids)
+ res = _run(module.list_chats.__wrapped__())
+ assert res["code"] == 0
+ assert set(captured["owner_ids"]) == {"tenant-1", "team-tenant-2"}
+
+
+@pytest.mark.p2
+def test_list_chats_rejects_unauthorized_owner_ids_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=SimpleNamespace(
+ get=lambda key, default=None: {
+ "keywords": "",
+ "page": "0",
+ "page_size": "0",
+ "orderby": "create_time",
+ "desc": "true",
+ "id": None,
+ "name": None,
+ }.get(key, default),
+ getlist=lambda key: ["foreign-tenant-id"] if key == "owner_ids" else [],
+ )
+ ),
+ )
+ res = _run(module.list_chats.__wrapped__())
+ assert res["code"] == module.RetCode.OPERATING_ERROR
+ assert "authorized owner_ids" in res["message"]
+
+
+@pytest.mark.p2
+def test_list_chats_returns_old_business_fields_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ monkeypatch.setattr(
+ module,
+ "request",
+ SimpleNamespace(
+ args=SimpleNamespace(
+ get=lambda key, default=None: {
+ "keywords": "",
+ "page": 1,
+ "page_size": 20,
+ "orderby": "create_time",
+ "desc": "true",
+ }.get(key, default),
+ getlist=lambda _key: [],
+ )
+ ),
+ )
+ monkeypatch.setattr(module.DialogService, "get_by_tenant_ids", lambda *_args, **_kwargs: ([_DummyDialogRecord().to_dict()], 1))
+ monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (True, _DummyKB()))
+
+ res = _run(module.list_chats.__wrapped__())
+ assert res["code"] == 0
+ chat = res["data"]["chats"][0]
+ assert chat["icon"] == "icon.png"
+ assert chat["dataset_ids"] == ["kb-1"]
+ assert chat["kb_names"] == ["Dataset A"]
+ assert "kb_ids" not in chat
+ assert chat["prompt_config"]["prologue"] == "hello"
+ assert "dataset_names" not in chat
+ assert "prompt" not in chat
+ assert "llm" not in chat
+
+
+@pytest.mark.p2
+def test_patch_chat_drops_response_only_fields_before_update_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ updated = {}
+ existing = _DummyDialogRecord().to_dict()
+ payload = {
+ "name": "renamed-chat",
+ "description": existing["description"],
+ "icon": existing["icon"],
+ "dataset_ids": existing["kb_ids"],
+ "kb_names": ["Dataset A"],
+ "llm_id": existing["llm_id"],
+ "llm_setting": existing["llm_setting"],
+ "prompt_config": existing["prompt_config"],
+ "similarity_threshold": existing["similarity_threshold"],
+ "vector_similarity_weight": existing["vector_similarity_weight"],
+ "top_n": existing["top_n"],
+ "top_k": existing["top_k"],
+ "rerank_id": existing["rerank_id"],
+ }
+
+ _set_route_unit_request_json(monkeypatch, module, payload)
+ monkeypatch.setattr(module.DialogService, "query", lambda **kwargs: [] if "name" in kwargs else [SimpleNamespace(id="chat-1")])
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, _DummyDialogRecord(existing)))
+ monkeypatch.setattr(module.TenantService, "get_by_id", lambda _tid: (True, SimpleNamespace(llm_id="glm-4")))
+ monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: [SimpleNamespace(id="kb-1")])
+ monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [_DummyKB()])
+ monkeypatch.setattr(module.TenantLLMService, "split_model_name_and_factory", lambda model: (model.split("@")[0], "factory"))
+ monkeypatch.setattr(module.TenantLLMService, "query", lambda **_kwargs: [SimpleNamespace(id="llm-1")])
+
+ def _update(_chat_id, req):
+ updated.update(req)
+ return True
+
+ monkeypatch.setattr(module.DialogService, "update_by_id", _update)
+ res = _run(module.patch_chat.__wrapped__("chat-1"))
+ assert res["code"] == 0
+ assert updated["name"] == "renamed-chat"
+ assert "kb_names" not in updated
@pytest.mark.p2
-def test_chat_duplicate_name_validation(rest_client, clear_chats):
- first = rest_client.post("/chats", json={"name": "duplicate_chat_name", "dataset_ids": []})
- assert first.status_code == 200
- first_payload = first.json()
- assert first_payload["code"] == 0, first_payload
+def test_patch_chat_merges_prompt_and_llm_settings_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ updated = {}
+ existing = _DummyDialogRecord().to_dict()
+ _set_route_unit_request_json(
+ monkeypatch,
+ module,
+ {"prompt_config": {"prologue": "updated opener"}, "llm_setting": {"temperature": 0.9}},
+ )
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [SimpleNamespace(id="chat-1")])
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, _DummyDialogRecord(existing)))
+ monkeypatch.setattr(module.TenantService, "get_by_id", lambda _tid: (True, SimpleNamespace(llm_id="glm-4")))
- second = rest_client.post("/chats", json={"name": "duplicate_chat_name", "dataset_ids": []})
- assert second.status_code == 200
- second_payload = second.json()
- assert second_payload["code"] == 102, second_payload
- assert "Duplicated chat name" in second_payload["message"], second_payload
+ def _update(_chat_id, payload):
+ updated.update(payload)
+ return True
+
+ monkeypatch.setattr(module.DialogService, "update_by_id", _update)
+ res = _run(module.patch_chat.__wrapped__("chat-1"))
+ assert res["code"] == 0
+ assert updated["prompt_config"]["system"] == "Answer with {knowledge}"
+ assert updated["prompt_config"]["prologue"] == "updated opener"
+ assert updated["llm_setting"]["temperature"] == 0.9
@pytest.mark.p2
-def test_chat_list_pagination(rest_client, clear_chats):
- for i in range(3):
- res = rest_client.post("/chats", json={"name": f"chat_page_{i}", "dataset_ids": []})
- assert res.status_code == 200
- payload = res.json()
- assert payload["code"] == 0, payload
+def test_update_chat_allows_knowledge_placeholder_without_sources_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ existing = _DummyDialogRecord().to_dict()
+ _set_route_unit_request_json(
+ monkeypatch,
+ module,
+ {
+ "name": "chat-name",
+ "description": "desc",
+ "icon": "icon.png",
+ "dataset_ids": [],
+ "llm_id": "glm-4",
+ "llm_setting": {"temperature": 0.1},
+ "prompt_config": {
+ "system": "Answer with {knowledge}",
+ "parameters": [{"key": "knowledge", "optional": False}],
+ "prologue": "hello",
+ "quote": True,
+ },
+ "similarity_threshold": 0.2,
+ "vector_similarity_weight": 0.3,
+ "top_n": 6,
+ "top_k": 1024,
+ "rerank_id": "",
+ },
+ )
+ monkeypatch.setattr(module.DialogService, "query", lambda **_kwargs: [SimpleNamespace(id="chat-1")])
+ monkeypatch.setattr(module.DialogService, "get_by_id", lambda _id: (True, _DummyDialogRecord(existing)))
+ monkeypatch.setattr(module.TenantService, "get_by_id", lambda _tid: (True, SimpleNamespace(llm_id="glm-4")))
+ monkeypatch.setattr(module.TenantLLMService, "split_model_name_and_factory", lambda model: (model.split("@")[0], "factory"))
+ monkeypatch.setattr(module.TenantLLMService, "query", lambda **_kwargs: [SimpleNamespace(id="llm-1")])
+ updated = {}
- page_res = rest_client.get("/chats", params={"page": 1, "page_size": 2, "orderby": "create_time", "desc": "true"})
- assert page_res.status_code == 200
- page_payload = page_res.json()
- assert page_payload["code"] == 0, page_payload
- assert len(page_payload["data"]["chats"]) == 2, page_payload
- assert page_payload["data"]["total"] >= 3, page_payload
+ def _update(_chat_id, payload):
+ updated.update(payload)
+ return True
+
+ monkeypatch.setattr(module.DialogService, "update_by_id", _update)
+ res = _run(module.update_chat.__wrapped__("chat-1"))
+ assert res["code"] == 0
+ assert updated["prompt_config"]["system"] == "Answer with {knowledge}"
@pytest.mark.p1
@@ -343,3 +1688,358 @@ def test_chat_create_rejects_unparsed_document(rest_client, clear_chats, create_
payload = res.json()
assert payload["code"] == 102, payload
assert "doesn't own parsed file" in payload["message"], payload
+
+
+@pytest.mark.p2
+def test_chat_update_name_contract(rest_client, clear_chats):
+ duplicate_res = rest_client.post("/chats", json={"name": "restful_chat_update_duplicate", "dataset_ids": []})
+ assert duplicate_res.status_code == 200
+ duplicate_payload = duplicate_res.json()
+ assert duplicate_payload["code"] == 0, duplicate_payload
+
+ target_res = rest_client.post("/chats", json={"name": "restful_chat_update_name_target", "dataset_ids": []})
+ assert target_res.status_code == 200
+ target_payload = target_res.json()
+ assert target_payload["code"] == 0, target_payload
+ chat_id = target_payload["data"]["id"]
+
+ cases = [
+ ("valid name", {"name": "valid_name"}, 0, "", "valid_name"),
+ (
+ "name too long",
+ {"name": "a" * (CHAT_ASSISTANT_NAME_LIMIT + 1)},
+ 102,
+ f"Chat name length is {CHAT_ASSISTANT_NAME_LIMIT + 1} which is larger than {CHAT_ASSISTANT_NAME_LIMIT}.",
+ None,
+ ),
+ ("name wrong type", {"name": 1}, 102, "Chat name must be a string.", None),
+ ("name empty", {"name": ""}, 102, "`name` cannot be empty.", None),
+ ("duplicate lowercase", {"name": "restful_chat_update_duplicate"}, 102, "Duplicated chat name.", None),
+ ("duplicate uppercase", {"name": "RESTFUL_CHAT_UPDATE_DUPLICATE"}, 102, "Duplicated chat name.", None),
+ ]
+
+ for scenario_name, patch_payload, expected_code, expected_message, expected_name in cases:
+ res = rest_client.patch(f"/chats/{chat_id}", json=patch_payload)
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == expected_code, (scenario_name, payload)
+ if expected_code == 0:
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ assert get_res.status_code == 200, (scenario_name, get_res.text)
+ get_payload = get_res.json()
+ assert get_payload["code"] == 0, (scenario_name, get_payload)
+ assert get_payload["data"]["name"] == expected_name, (scenario_name, get_payload)
+ else:
+ assert payload["message"] == expected_message, (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_update_dataset_ids_contract(rest_client, clear_chats, ensure_parsed_document):
+ dataset_id, _ = ensure_parsed_document()
+ target_res = rest_client.post("/chats", json={"name": "restful_chat_update_dataset_target", "dataset_ids": []})
+ assert target_res.status_code == 200
+ target_payload = target_res.json()
+ assert target_payload["code"] == 0, target_payload
+ chat_id = target_payload["data"]["id"]
+
+ cases = [
+ ("empty dataset_ids", [], 0, "", []),
+ ("owned parsed dataset", [dataset_id], 0, "", [dataset_id]),
+ ("invalid dataset id", ["invalid_dataset_id"], 102, "You don't own the dataset invalid_dataset_id", None),
+ ("dataset_ids wrong type", "invalid_dataset_id", 102, "`dataset_ids` should be a list.", None),
+ ]
+
+ for scenario_name, dataset_ids, expected_code, expected_message, expected_dataset_ids in cases:
+ res = rest_client.put(
+ f"/chats/{chat_id}",
+ json={"name": "ragflow test", "dataset_ids": dataset_ids},
+ )
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == expected_code, (scenario_name, payload)
+ if expected_code == 0:
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ assert get_res.status_code == 200, (scenario_name, get_res.text)
+ get_payload = get_res.json()
+ assert get_payload["code"] == 0, (scenario_name, get_payload)
+ assert get_payload["data"]["name"] == "ragflow test", (scenario_name, get_payload)
+ assert get_payload["data"]["dataset_ids"] == expected_dataset_ids, (scenario_name, get_payload)
+ else:
+ assert payload["message"] == expected_message, (scenario_name, payload)
+
+
+@pytest.mark.p2
+def test_chat_update_avatar_contract(rest_client, clear_chats, ensure_parsed_document, tmp_path):
+ dataset_id, _ = ensure_parsed_document()
+ create_res = rest_client.post("/chats", json={"name": "restful_chat_update_avatar_target", "dataset_ids": []})
+ assert create_res.status_code == 200
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, create_payload
+ chat_id = create_payload["data"]["id"]
+
+ image_path = create_image_file(tmp_path / "restful_chat_update_avatar.png")
+ encoded_avatar = encode_avatar(image_path)
+
+ res = rest_client.put(
+ f"/chats/{chat_id}",
+ json={"name": "avatar_test", "icon": encoded_avatar, "dataset_ids": [dataset_id]},
+ )
+ assert res.status_code == 200
+ payload = res.json()
+ assert payload["code"] == 0, payload
+
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ assert get_res.status_code == 200
+ get_payload = get_res.json()
+ assert get_payload["code"] == 0, get_payload
+ assert get_payload["data"]["name"] == "avatar_test", get_payload
+ assert get_payload["data"]["icon"] == encoded_avatar, get_payload
+ assert get_payload["data"]["dataset_ids"] == [dataset_id], get_payload
+
+
+@pytest.mark.p2
+def test_chat_update_llm_contract(rest_client, clear_chats, ensure_parsed_document):
+ dataset_id, _ = ensure_parsed_document()
+ cases = [
+ ("default llm", {}, 0, "", "glm-4-flash@ZHIPU-AI", {}),
+ ("explicit llm_id", {"llm_id": "glm-4"}, 0, "", "glm-4", {}),
+ ("unknown llm_id", {"llm_id": "unknown"}, 102, "`llm_id` unknown doesn't exist", None, None),
+ ("temperature zero", {"llm_setting": {"temperature": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": 0}),
+ ("temperature one", {"llm_setting": {"temperature": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": 1}),
+ ("temperature negative one", {"llm_setting": {"temperature": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": -1}),
+ ("temperature ten", {"llm_setting": {"temperature": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": 10}),
+ ("temperature string", {"llm_setting": {"temperature": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"temperature": "a"}),
+ ("top_p zero", {"llm_setting": {"top_p": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": 0}),
+ ("top_p one", {"llm_setting": {"top_p": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": 1}),
+ ("top_p negative one", {"llm_setting": {"top_p": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": -1}),
+ ("top_p ten", {"llm_setting": {"top_p": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": 10}),
+ ("top_p string", {"llm_setting": {"top_p": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"top_p": "a"}),
+ ("presence_penalty zero", {"llm_setting": {"presence_penalty": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": 0}),
+ ("presence_penalty one", {"llm_setting": {"presence_penalty": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": 1}),
+ ("presence_penalty negative one", {"llm_setting": {"presence_penalty": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": -1}),
+ ("presence_penalty ten", {"llm_setting": {"presence_penalty": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": 10}),
+ ("presence_penalty string", {"llm_setting": {"presence_penalty": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"presence_penalty": "a"}),
+ ("frequency_penalty zero", {"llm_setting": {"frequency_penalty": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": 0}),
+ ("frequency_penalty one", {"llm_setting": {"frequency_penalty": 1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": 1}),
+ ("frequency_penalty negative one", {"llm_setting": {"frequency_penalty": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": -1}),
+ ("frequency_penalty ten", {"llm_setting": {"frequency_penalty": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": 10}),
+ ("frequency_penalty string", {"llm_setting": {"frequency_penalty": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"frequency_penalty": "a"}),
+ ("max_token zero", {"llm_setting": {"max_token": 0}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": 0}),
+ ("max_token 1024", {"llm_setting": {"max_token": 1024}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": 1024}),
+ ("max_token negative one", {"llm_setting": {"max_token": -1}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": -1}),
+ ("max_token ten", {"llm_setting": {"max_token": 10}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": 10}),
+ ("max_token string", {"llm_setting": {"max_token": "a"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"max_token": "a"}),
+ ("unknown llm setting key", {"llm_setting": {"unknown": "unknown"}}, 0, "", "glm-4-flash@ZHIPU-AI", {"unknown": "unknown"}),
+ ]
+
+ for index, (scenario_name, extra_payload, expected_code, expected_message, expected_llm_id, expected_llm_setting) in enumerate(cases, start=1):
+ create_res = rest_client.post(
+ "/chats",
+ json={"name": f"restful_chat_update_llm_target_{index}", "dataset_ids": [dataset_id]},
+ )
+ assert create_res.status_code == 200, (scenario_name, create_res.text)
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, (scenario_name, create_payload)
+ chat_id = create_payload["data"]["id"]
+
+ updated_name = f"llm_test_{index}"
+ payload = {"name": updated_name, "dataset_ids": [dataset_id]}
+ payload.update(extra_payload)
+ res = rest_client.put(f"/chats/{chat_id}", json=payload)
+ assert res.status_code == 200, (scenario_name, res.text)
+ body = res.json()
+ assert body["code"] == expected_code, (scenario_name, body)
+ if expected_code == 0:
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ assert get_res.status_code == 200, (scenario_name, get_res.text)
+ get_payload = get_res.json()
+ assert get_payload["code"] == 0, (scenario_name, get_payload)
+ assert get_payload["data"]["name"] == updated_name, (scenario_name, get_payload)
+ assert get_payload["data"]["llm_id"] == expected_llm_id, (scenario_name, get_payload)
+ assert get_payload["data"]["llm_setting"] == expected_llm_setting, (scenario_name, get_payload)
+ else:
+ assert body["message"] == expected_message, (scenario_name, body)
+
+
+@pytest.mark.p2
+def test_chat_update_prompt_contract(rest_client, clear_chats, ensure_parsed_document):
+ dataset_id, _ = ensure_parsed_document()
+ cases = [
+ (
+ "default prompt config",
+ {},
+ {
+ ("similarity_threshold",): 0.1,
+ ("vector_similarity_weight",): 0.3,
+ ("top_n",): 6,
+ ("prompt_config", "parameters"): [{"key": "knowledge", "optional": False}],
+ ("prompt_config", "empty_response"): DEFAULT_CHAT_EMPTY_RESPONSE,
+ ("prompt_config", "prologue"): DEFAULT_CHAT_PROLOGUE,
+ ("prompt_config", "quote"): True,
+ ("prompt_config", "system"): DEFAULT_CHAT_SYSTEM_PROMPT,
+ },
+ ),
+ ("similarity_threshold zero", {"similarity_threshold": 0}, {("similarity_threshold",): 0}),
+ ("similarity_threshold one", {"similarity_threshold": 1}, {("similarity_threshold",): 1}),
+ ("similarity_threshold negative one", {"similarity_threshold": -1}, {("similarity_threshold",): -1.0}),
+ ("similarity_threshold ten", {"similarity_threshold": 10}, {("similarity_threshold",): 10.0}),
+ ("similarity_threshold string", {"similarity_threshold": "a"}, {("similarity_threshold",): 0.0}),
+ ("vector_similarity_weight zero", {"vector_similarity_weight": 0}, {("vector_similarity_weight",): 0}),
+ ("vector_similarity_weight one", {"vector_similarity_weight": 1}, {("vector_similarity_weight",): 1}),
+ ("vector_similarity_weight negative one", {"vector_similarity_weight": -1}, {("vector_similarity_weight",): -1.0}),
+ ("vector_similarity_weight ten", {"vector_similarity_weight": 10}, {("vector_similarity_weight",): 10.0}),
+ ("vector_similarity_weight string", {"vector_similarity_weight": "a"}, {("vector_similarity_weight",): 0.0}),
+ ("empty prompt parameters", {"prompt_config": {"parameters": []}}, {("prompt_config", "parameters"): []}),
+ ("top_n zero", {"top_n": 0}, {("top_n",): 0}),
+ ("top_n one", {"top_n": 1}, {("top_n",): 1}),
+ ("top_n negative one", {"top_n": -1}, {("top_n",): -1}),
+ ("top_n ten", {"top_n": 10}, {("top_n",): 10}),
+ ("top_n string", {"top_n": "a"}, {("top_n",): 0}),
+ ("empty_response plain text", {"prompt_config": {"empty_response": "Hello World"}}, {("prompt_config", "empty_response"): "Hello World"}),
+ ("empty_response empty string", {"prompt_config": {"empty_response": ""}}, {("prompt_config", "empty_response"): ""}),
+ ("empty_response punctuation", {"prompt_config": {"empty_response": "!@#$%^&*()"}}, {("prompt_config", "empty_response"): "!@#$%^&*()"}),
+ ("empty_response chinese text", {"prompt_config": {"empty_response": "中文测试"}}, {("prompt_config", "empty_response"): "中文测试"}),
+ ("empty_response integer", {"prompt_config": {"empty_response": 123}}, {("prompt_config", "empty_response"): 123}),
+ ("empty_response boolean", {"prompt_config": {"empty_response": True}}, {("prompt_config", "empty_response"): True}),
+ ("empty_response space", {"prompt_config": {"empty_response": " "}}, {("prompt_config", "empty_response"): " "}),
+ ("prologue plain text", {"prompt_config": {"prologue": "Hello World"}}, {("prompt_config", "prologue"): "Hello World"}),
+ ("prologue empty string", {"prompt_config": {"prologue": ""}}, {("prompt_config", "prologue"): ""}),
+ ("prologue punctuation", {"prompt_config": {"prologue": "!@#$%^&*()"}}, {("prompt_config", "prologue"): "!@#$%^&*()"}),
+ ("prologue chinese text", {"prompt_config": {"prologue": "中文测试"}}, {("prompt_config", "prologue"): "中文测试"}),
+ ("prologue integer", {"prompt_config": {"prologue": 123}}, {("prompt_config", "prologue"): 123}),
+ ("prologue boolean", {"prompt_config": {"prologue": True}}, {("prompt_config", "prologue"): True}),
+ ("prologue space", {"prompt_config": {"prologue": " "}}, {("prompt_config", "prologue"): " "}),
+ ("quote true", {"prompt_config": {"quote": True}}, {("prompt_config", "quote"): True}),
+ ("quote false", {"prompt_config": {"quote": False}}, {("prompt_config", "quote"): False}),
+ ("system prompt with knowledge prefix", {"prompt_config": {"system": "Hello World {knowledge}"}}, {("prompt_config", "system"): "Hello World {knowledge}"}),
+ ("system prompt only knowledge", {"prompt_config": {"system": "{knowledge}"}}, {("prompt_config", "system"): "{knowledge}"}),
+ ("system prompt punctuation", {"prompt_config": {"system": "!@#$%^&*() {knowledge}"}}, {("prompt_config", "system"): "!@#$%^&*() {knowledge}"}),
+ ("system prompt chinese text", {"prompt_config": {"system": "中文测试 {knowledge}"}}, {("prompt_config", "system"): "中文测试 {knowledge}"}),
+ ("system prompt plain text", {"prompt_config": {"system": "Hello World"}}, {("prompt_config", "system"): "Hello World"}),
+ (
+ "system prompt with explicit empty parameters",
+ {"prompt_config": {"system": "Hello World", "parameters": []}},
+ {("prompt_config", "system"): "Hello World", ("prompt_config", "parameters"): []},
+ ),
+ ("system prompt integer", {"prompt_config": {"system": 123}}, {("prompt_config", "system"): 123}),
+ ("system prompt boolean", {"prompt_config": {"system": True}}, {("prompt_config", "system"): True}),
+ ("unknown prompt key", {"unknown": "unknown"}, {}),
+ ]
+
+ for index, (scenario_name, extra_payload, expected_values) in enumerate(cases, start=1):
+ create_res = rest_client.post(
+ "/chats",
+ json={"name": f"restful_chat_update_prompt_target_{index}", "dataset_ids": [dataset_id]},
+ )
+ assert create_res.status_code == 200, (scenario_name, create_res.text)
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, (scenario_name, create_payload)
+ chat_id = create_payload["data"]["id"]
+
+ updated_name = f"prompt_test_{index}"
+ res = rest_client.put(
+ f"/chats/{chat_id}",
+ json={"name": updated_name, "dataset_ids": [dataset_id], **extra_payload},
+ )
+ assert res.status_code == 200, (scenario_name, res.text)
+ payload = res.json()
+ assert payload["code"] == 0, (scenario_name, payload)
+
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ assert get_res.status_code == 200, (scenario_name, get_res.text)
+ get_payload = get_res.json()
+ assert get_payload["code"] == 0, (scenario_name, get_payload)
+ assert get_payload["data"]["name"] == updated_name, (scenario_name, get_payload)
+ assert get_payload["data"]["dataset_ids"] == [dataset_id], (scenario_name, get_payload)
+ for path, expected_value in expected_values.items():
+ assert _get_nested(get_payload["data"], path) == expected_value, (scenario_name, path, get_payload)
+
+
+@pytest.mark.p2
+def test_chat_update_mapping_and_validation_branches_p2(rest_client, clear_chats):
+ duplicate_res = rest_client.post("/chats", json={"name": "restful_chat_update_mapping_duplicate", "dataset_ids": []})
+ assert duplicate_res.status_code == 200
+ duplicate_payload = duplicate_res.json()
+ assert duplicate_payload["code"] == 0, duplicate_payload
+
+ target_res = rest_client.post("/chats", json={"name": "restful_chat_update_mapping_target", "dataset_ids": []})
+ assert target_res.status_code == 200
+ target_payload = target_res.json()
+ assert target_payload["code"] == 0, target_payload
+ chat_id = target_payload["data"]["id"]
+
+ unauthorized = rest_client.patch("/chats/invalid-chat-id", json={"name": "anything"})
+ assert unauthorized.status_code == 200
+ unauthorized_payload = unauthorized.json()
+ assert unauthorized_payload["code"] == 109, unauthorized_payload
+ assert unauthorized_payload["message"] == "No authorization.", unauthorized_payload
+
+ quote_res = rest_client.patch(f"/chats/{chat_id}", json={"prompt_config": {"quote": False}})
+ assert quote_res.status_code == 200
+ quote_payload = quote_res.json()
+ assert quote_payload["code"] == 0, quote_payload
+ assert quote_payload["data"]["prompt_config"]["quote"] is False, quote_payload
+
+ invalid_llm_res = rest_client.patch(
+ f"/chats/{chat_id}",
+ json={"llm_id": "unknown-llm-model", "llm_setting": {"model_type": "chat"}},
+ )
+ assert invalid_llm_res.status_code == 200
+ invalid_llm_payload = invalid_llm_res.json()
+ assert invalid_llm_payload["code"] == 102, invalid_llm_payload
+ assert "`llm_id` unknown-llm-model doesn't exist" in invalid_llm_payload["message"], invalid_llm_payload
+
+ invalid_rerank_res = rest_client.patch(f"/chats/{chat_id}", json={"rerank_id": "unknown-rerank-model"})
+ assert invalid_rerank_res.status_code == 200
+ invalid_rerank_payload = invalid_rerank_res.json()
+ assert invalid_rerank_payload["code"] == 102, invalid_rerank_payload
+ assert "`rerank_id` unknown-rerank-model doesn't exist" in invalid_rerank_payload["message"], invalid_rerank_payload
+
+ empty_name_res = rest_client.patch(f"/chats/{chat_id}", json={"name": ""})
+ assert empty_name_res.status_code == 200
+ empty_name_payload = empty_name_res.json()
+ assert empty_name_payload["code"] == 102, empty_name_payload
+ assert empty_name_payload["message"] == "`name` cannot be empty.", empty_name_payload
+
+ duplicate_name_res = rest_client.patch(f"/chats/{chat_id}", json={"name": "restful_chat_update_mapping_duplicate"})
+ assert duplicate_name_res.status_code == 200
+ duplicate_name_payload = duplicate_name_res.json()
+ assert duplicate_name_payload["code"] == 102, duplicate_name_payload
+ assert duplicate_name_payload["message"] == "Duplicated chat name.", duplicate_name_payload
+
+ prompt_without_placeholder_res = rest_client.patch(
+ f"/chats/{chat_id}",
+ json={"prompt_config": {"system": "No required placeholder", "parameters": [{"key": "knowledge", "optional": False}]}},
+ )
+ assert prompt_without_placeholder_res.status_code == 200
+ prompt_without_placeholder_payload = prompt_without_placeholder_res.json()
+ assert prompt_without_placeholder_payload["code"] == 0, prompt_without_placeholder_payload
+
+ icon_res = rest_client.patch(f"/chats/{chat_id}", json={"icon": "raw-avatar-value"})
+ assert icon_res.status_code == 200
+ icon_payload = icon_res.json()
+ assert icon_payload["code"] == 0, icon_payload
+
+ get_res = rest_client.get(f"/chats/{chat_id}")
+ assert get_res.status_code == 200
+ get_payload = get_res.json()
+ assert get_payload["code"] == 0, get_payload
+ assert get_payload["data"]["prompt_config"]["system"] == "No required placeholder", get_payload
+ assert get_payload["data"]["prompt_config"]["parameters"] == [{"key": "knowledge", "optional": False}], get_payload
+ assert get_payload["data"]["icon"] == "raw-avatar-value", get_payload
+
+
+@pytest.mark.p2
+def test_chat_update_rejects_unparsed_document(rest_client, clear_chats, create_document):
+ dataset_id, _ = create_document()
+ create_res = rest_client.post("/chats", json={"name": "restful_chat_update_unparsed_target", "dataset_ids": []})
+ assert create_res.status_code == 200
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, create_payload
+ chat_id = create_payload["data"]["id"]
+
+ res = rest_client.patch(f"/chats/{chat_id}", json={"dataset_ids": [dataset_id]})
+ assert res.status_code == 200
+ payload = res.json()
+ assert payload["code"] == 102, payload
+ assert "doesn't own parsed file" in payload["message"], payload
diff --git a/test/testcases/restful_api/test_user_tenant_routes_unit.py b/test/testcases/restful_api/test_user_tenant_routes_unit.py
index 811a40654cc..4d006f66821 100644
--- a/test/testcases/restful_api/test_user_tenant_routes_unit.py
+++ b/test/testcases/restful_api/test_user_tenant_routes_unit.py
@@ -70,6 +70,14 @@ def _run(coro):
return asyncio.run(coro)
+def _passthrough_login_required(func):
+ async def _wrapper(*args, **kwargs):
+ return await func(*args, **kwargs)
+
+ _wrapper.__wrapped__ = func
+ return _wrapper
+
+
def _set_request_json(monkeypatch, module, payload):
async def _request_json():
return payload
@@ -1380,3 +1388,241 @@ def _raise_update_password(_user_id, _new_pwd):
assert res["code"] == module.RetCode.SUCCESS, res
assert res["auth"] == user.get_id(), res
assert module.REDIS_CONN.get(v_key) is None, module.REDIS_CONN.store
+
+
+def _load_chat_routes_unit_module(monkeypatch):
+ repo_root = Path(__file__).resolve().parents[3]
+ module_name = "test_chat_restful_routes_unit_module_for_tenant"
+ module_path = repo_root / "api" / "apps" / "restful_apis" / "chat_api.py"
+
+ quart_mod = ModuleType("quart")
+ quart_mod.request = SimpleNamespace(args=SimpleNamespace(get=lambda _key, default=None: default, getlist=lambda _key: []))
+ quart_mod.Response = type("_StubResponse", (), {})
+ monkeypatch.setitem(sys.modules, "quart", quart_mod)
+
+ api_pkg = ModuleType("api")
+ api_pkg.__path__ = [str(repo_root / "api")]
+ monkeypatch.setitem(sys.modules, "api", api_pkg)
+
+ apps_pkg = ModuleType("api.apps")
+ apps_pkg.__path__ = [str(repo_root / "api" / "apps")]
+ apps_pkg.current_user = SimpleNamespace(id="tenant-1")
+ apps_pkg.login_required = _passthrough_login_required
+ monkeypatch.setitem(sys.modules, "api.apps", apps_pkg)
+ api_pkg.apps = apps_pkg
+
+ common_pkg = ModuleType("common")
+ common_pkg.__path__ = [str(repo_root / "common")]
+ monkeypatch.setitem(sys.modules, "common", common_pkg)
+
+ settings_mod = ModuleType("common.settings")
+ settings_mod.STORAGE_IMPL = type("_StorageImpl", (), {"rm": staticmethod(lambda *_args, **_kwargs: None)})()
+ monkeypatch.setitem(sys.modules, "common.settings", settings_mod)
+
+ constants_mod = ModuleType("common.constants")
+ constants_mod.LLMType = SimpleNamespace(CHAT="chat", IMAGE2TEXT="image2text", RERANK="rerank", SPEECH2TEXT="speech2text", TTS="tts")
+ constants_mod.RetCode = SimpleNamespace(SUCCESS=0, DATA_ERROR=102, OPERATING_ERROR=103, AUTHENTICATION_ERROR=109)
+ constants_mod.StatusEnum = SimpleNamespace(VALID=SimpleNamespace(value="1"), INVALID=SimpleNamespace(value="0"))
+ from common.constants import MAXIMUM_PAGE_NUMBER as _MPN, MAXIMUM_TASK_PAGE_NUMBER as _MTPN
+ constants_mod.MAXIMUM_PAGE_NUMBER = _MPN
+ constants_mod.MAXIMUM_TASK_PAGE_NUMBER = _MTPN
+ monkeypatch.setitem(sys.modules, "common.constants", constants_mod)
+
+ misc_utils_mod = ModuleType("common.misc_utils")
+ misc_utils_mod.get_uuid = lambda: "generated-chat-id"
+ async def _thread_pool_exec(func, *args, **kwargs):
+ return func(*args, **kwargs)
+ misc_utils_mod.thread_pool_exec = _thread_pool_exec
+ monkeypatch.setitem(sys.modules, "common.misc_utils", misc_utils_mod)
+
+ dialog_service_mod = ModuleType("api.db.services.dialog_service")
+ class _DialogService:
+ model = SimpleNamespace(_meta=SimpleNamespace(fields={
+ "id": None,
+ "tenant_id": None,
+ "name": None,
+ "description": None,
+ "icon": None,
+ "kb_ids": None,
+ "llm_id": None,
+ "llm_setting": None,
+ "prompt_config": None,
+ "similarity_threshold": None,
+ "vector_similarity_weight": None,
+ "top_n": None,
+ "top_k": None,
+ "rerank_id": None,
+ "meta_data_filter": None,
+ "created_by": None,
+ "create_time": None,
+ "create_date": None,
+ "update_time": None,
+ "update_date": None,
+ "status": None,
+ }))
+ @staticmethod
+ def query(**_kwargs):
+ return []
+ @staticmethod
+ def save(**_kwargs):
+ return True
+ @staticmethod
+ def get_by_id(_chat_id):
+ return False, None
+ @staticmethod
+ def get_by_tenant_ids(*_args, **_kwargs):
+ return [], 0
+ dialog_service_mod.DialogService = _DialogService
+ dialog_service_mod.async_ask = lambda *_args, **_kwargs: None
+ dialog_service_mod.async_chat = lambda *_args, **_kwargs: None
+ dialog_service_mod.gen_mindmap = lambda *_args, **_kwargs: None
+ monkeypatch.setitem(sys.modules, "api.db.services.dialog_service", dialog_service_mod)
+
+ conversation_service_mod = ModuleType("api.db.services.conversation_service")
+ conversation_service_mod.ConversationService = type("ConversationService", (), {})
+ conversation_service_mod.structure_answer = lambda *_args, **_kwargs: {}
+ monkeypatch.setitem(sys.modules, "api.db.services.conversation_service", conversation_service_mod)
+
+ kb_service_mod = ModuleType("api.db.services.knowledgebase_service")
+ class _KB:
+ def __init__(self):
+ self.id = "kb-1"
+ self.embd_id = "embd@factory"
+ self.chunk_num = 1
+ self.name = "Dataset A"
+ self.status = "1"
+ kb_service_mod.KnowledgebaseService = type('KnowledgebaseService', (), {
+ 'accessible': staticmethod(lambda **_kwargs: [SimpleNamespace(id='kb-1')]),
+ 'query': staticmethod(lambda **_kwargs: [_KB()]),
+ 'get_by_id': staticmethod(lambda _id: (True, _KB())),
+ })
+ monkeypatch.setitem(sys.modules, "api.db.services.knowledgebase_service", kb_service_mod)
+
+ tenant_llm_service_mod = ModuleType("api.db.services.tenant_llm_service")
+ tenant_llm_service_mod.TenantLLMService = type('TenantLLMService', (), {
+ 'split_model_name_and_factory': staticmethod(lambda model: (model.split('@', 1)[0], model.split('@', 1)[1] if '@' in model else None)),
+ 'query': staticmethod(lambda **_kwargs: [SimpleNamespace(id='llm-1')]),
+ 'get_api_key': staticmethod(lambda *_args, **_kwargs: SimpleNamespace(id=1)),
+ })
+ monkeypatch.setitem(sys.modules, "api.db.services.tenant_llm_service", tenant_llm_service_mod)
+
+ llm_service_mod = ModuleType("api.db.services.llm_service")
+ llm_service_mod.LLMBundle = lambda *_args, **_kwargs: None
+ monkeypatch.setitem(sys.modules, "api.db.services.llm_service", llm_service_mod)
+
+ search_service_mod = ModuleType("api.db.services.search_service")
+ search_service_mod.SearchService = SimpleNamespace()
+ monkeypatch.setitem(sys.modules, "api.db.services.search_service", search_service_mod)
+
+ tenant_model_service_mod = ModuleType("api.db.joint_services.tenant_model_service")
+ tenant_model_service_mod.get_model_config_by_type_and_name = lambda *_args, **_kwargs: {}
+ tenant_model_service_mod.get_tenant_default_model_by_type = lambda *_args, **_kwargs: {}
+ monkeypatch.setitem(sys.modules, "api.db.joint_services.tenant_model_service", tenant_model_service_mod)
+
+ user_service_mod = ModuleType("api.db.services.user_service")
+ user_service_mod.UserService = type('UserService', (), {})
+ user_service_mod.TenantService = type('TenantService', (), {
+ 'get_by_id': staticmethod(lambda _tenant_id: (True, SimpleNamespace(llm_id='glm-4'))),
+ 'get_joined_tenants_by_user_id': staticmethod(lambda _user_id: [{'tenant_id': 'tenant-1'}, {'tenant_id': 'team-tenant-2'}]),
+ })
+ user_service_mod.UserTenantService = type('UserTenantService', (), {'query': staticmethod(lambda **_kwargs: [])})
+ monkeypatch.setitem(sys.modules, "api.db.services.user_service", user_service_mod)
+
+ chunk_feedback_service_mod = ModuleType("api.db.services.chunk_feedback_service")
+ chunk_feedback_service_mod.ChunkFeedbackService = type('ChunkFeedbackService', (), {'apply_feedback': staticmethod(lambda **_kwargs: {'success_count': 0, 'fail_count': 0, 'chunk_ids': []})})
+ monkeypatch.setitem(sys.modules, "api.db.services.chunk_feedback_service", chunk_feedback_service_mod)
+
+ api_utils_mod = ModuleType("api.utils.api_utils")
+ api_utils_mod.check_duplicate_ids = lambda ids, _label: (list(dict.fromkeys(ids or [])), [])
+ api_utils_mod.get_data_error_result = lambda message='': {'code': 102, 'data': None, 'message': message}
+ api_utils_mod.get_json_result = lambda data=None, message='', code=0: {'code': code, 'data': data, 'message': message}
+ api_utils_mod.server_error_response = lambda ex: {'code': 500, 'data': None, 'message': str(ex)}
+ api_utils_mod.validate_request = lambda *_args, **_kwargs: (lambda func: func)
+ api_utils_mod.get_request_json = lambda: _AwaitableValue({})
+ monkeypatch.setitem(sys.modules, "api.utils.api_utils", api_utils_mod)
+
+ tenant_utils_mod = ModuleType("api.utils.tenant_utils")
+ tenant_utils_mod.ensure_tenant_model_id_for_params = lambda _tenant_id, req: req
+ monkeypatch.setitem(sys.modules, "api.utils.tenant_utils", tenant_utils_mod)
+
+ rag_pkg = ModuleType("rag")
+ rag_pkg.__path__ = [str(repo_root / 'rag')]
+ monkeypatch.setitem(sys.modules, 'rag', rag_pkg)
+ rag_prompts_pkg = ModuleType('rag.prompts')
+ rag_prompts_pkg.__path__ = [str(repo_root / 'rag' / 'prompts')]
+ monkeypatch.setitem(sys.modules, 'rag.prompts', rag_prompts_pkg)
+ rag_prompts_generator_mod = ModuleType('rag.prompts.generator')
+ rag_prompts_generator_mod.chunks_format = lambda reference: reference.get('chunks', []) if isinstance(reference, dict) else []
+ monkeypatch.setitem(sys.modules, 'rag.prompts.generator', rag_prompts_generator_mod)
+ rag_prompts_template_mod = ModuleType('rag.prompts.template')
+ rag_prompts_template_mod.load_prompt = lambda *_args, **_kwargs: ''
+ monkeypatch.setitem(sys.modules, 'rag.prompts.template', rag_prompts_template_mod)
+
+ spec = importlib.util.spec_from_file_location(module_name, module_path)
+ module = importlib.util.module_from_spec(spec)
+ module.manager = _DummyManager()
+ monkeypatch.setitem(sys.modules, module_name, module)
+ spec.loader.exec_module(module)
+ return module
+
+
+@pytest.mark.p1
+def test_create_chat_uses_tenant_default_llm_when_llm_id_is_null_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ saved = {}
+
+ async def _request_json():
+ return {
+ 'name': 'chat-a',
+ 'dataset_ids': ['kb-1'],
+ 'llm_id': None,
+ 'llm_setting': {'temperature': 0.8},
+ 'prompt_config': {'system': 'Answer with {knowledge}', 'parameters': [{'key': 'knowledge', 'optional': False}]},
+ }
+
+ monkeypatch.setattr(module, 'get_request_json', _request_json)
+ monkeypatch.setattr(module.DialogService, 'query', lambda **_kwargs: [])
+
+ def _save(**kwargs):
+ saved.update(kwargs)
+ return True
+
+ monkeypatch.setattr(module.DialogService, 'save', _save)
+ monkeypatch.setattr(module.DialogService, 'get_by_id', lambda _id: (True, SimpleNamespace(to_dict=lambda: saved)))
+
+ res = _run(module.create.__wrapped__())
+ assert res['code'] == 0
+ assert saved['llm_id'] == 'glm-4'
+ assert saved['llm_setting']['temperature'] == 0.8
+
+
+@pytest.mark.p2
+def test_list_chats_authorized_multi_tenant_unit(monkeypatch):
+ module = _load_chat_routes_unit_module(monkeypatch)
+ captured = {}
+ monkeypatch.setattr(
+ module,
+ 'request',
+ SimpleNamespace(
+ args=SimpleNamespace(
+ get=lambda key, default=None: {
+ 'keywords': '', 'page': '1', 'page_size': '10', 'orderby': 'create_time', 'desc': 'true', 'id': None, 'name': None,
+ }.get(key, default),
+ getlist=lambda key: ['tenant-1', 'team-tenant-2'] if key == 'owner_ids' else [],
+ )
+ ),
+ )
+
+ def _get_by_tenant_ids(owner_ids, user_id, *args, **kwargs):
+ captured['owner_ids'] = owner_ids
+ captured['user_id'] = user_id
+ return ([{'id': 'c1', 'tenant_id': 'tenant-1'}, {'id': 'c2', 'tenant_id': 'team-tenant-2'}], 2)
+
+ monkeypatch.setattr(module.DialogService, 'get_by_tenant_ids', _get_by_tenant_ids)
+ monkeypatch.setattr(module.KnowledgebaseService, 'get_by_id', lambda _id: (False, None))
+ res = _run(module.list_chats.__wrapped__())
+ assert res['code'] == 0
+ assert res['data']['total'] == 2
+ assert {c['id'] for c in res['data']['chats']} == {'c1', 'c2'}
+ assert set(captured['owner_ids']) == {'tenant-1', 'team-tenant-2'}
+ assert captured['user_id'] == 'tenant-1'
From 77834870fcaad410823dbd583e1ea81028e46689 Mon Sep 17 00:00:00 2001
From: qinling0210 <88864212+qinling0210@users.noreply.github.com>
Date: Tue, 19 May 2026 17:34:59 +0800
Subject: [PATCH 209/666] Refact functions in engine in GO (#14981)
### What problem does this PR solve?
Refact functions in engine in GO
### Type of change
- [x] Refactoring
---
internal/engine/elasticsearch/chunk.go | 1216 ++++++++++++
internal/engine/elasticsearch/common.go | 98 +
internal/engine/elasticsearch/get.go | 49 -
internal/engine/elasticsearch/index.go | 362 ----
internal/engine/elasticsearch/metadata.go | 275 +++
internal/engine/elasticsearch/search.go | 583 ------
internal/engine/engine.go | 31 +-
internal/engine/infinity/chunk.go | 2038 +++++++++++++++++++++
internal/engine/infinity/common.go | 112 +-
internal/engine/infinity/dataset.go | 655 -------
internal/engine/infinity/get.go | 303 ---
internal/engine/infinity/metadata.go | 118 +-
internal/engine/infinity/search.go | 1100 -----------
internal/handler/datasets.go | 4 +-
internal/handler/kb.go | 2 +-
internal/service/chunk.go | 4 +-
internal/service/file.go | 2 +-
internal/service/kb.go | 10 +-
internal/service/skill_indexer.go | 18 +-
internal/service/skill_search.go | 2 +-
internal/service/skill_space.go | 2 +-
internal/service/tenant.go | 10 +-
22 files changed, 3799 insertions(+), 3195 deletions(-)
create mode 100644 internal/engine/elasticsearch/chunk.go
create mode 100644 internal/engine/elasticsearch/common.go
delete mode 100644 internal/engine/elasticsearch/get.go
delete mode 100644 internal/engine/elasticsearch/index.go
create mode 100644 internal/engine/elasticsearch/metadata.go
delete mode 100644 internal/engine/elasticsearch/search.go
create mode 100644 internal/engine/infinity/chunk.go
delete mode 100644 internal/engine/infinity/dataset.go
delete mode 100644 internal/engine/infinity/get.go
delete mode 100644 internal/engine/infinity/search.go
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
new file mode 100644
index 00000000000..26414d7e6a9
--- /dev/null
+++ b/internal/engine/elasticsearch/chunk.go
@@ -0,0 +1,1216 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package elasticsearch
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+
+ "github.com/elastic/go-elasticsearch/v8/esapi"
+ "ragflow/internal/common"
+ "ragflow/internal/engine/types"
+
+ "go.uber.org/zap"
+)
+
+// CreateChunkStore creates an index
+func (e *elasticsearchEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+ if baseName == "" {
+ return fmt.Errorf("index name cannot be empty")
+ }
+
+ // Check if index already exists
+ exists, err := e.indexExists(ctx, baseName)
+ if err != nil {
+ return fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if exists {
+ return fmt.Errorf("index '%s' already exists", baseName)
+ }
+
+ // Load mapping based on index type
+ var mapping map[string]interface{}
+ if datasetID == "skill" {
+ // Load skill-specific mapping
+ skillMapping, err := loadSkillMapping()
+ if err != nil {
+ return fmt.Errorf("failed to load skill mapping: %w", err)
+ }
+ mapping = skillMapping
+ } else {
+ // Default mapping for dataset
+ mapping = map[string]interface{}{
+ "settings": map[string]interface{}{
+ "number_of_shards": 1,
+ "number_of_replicas": 0,
+ },
+ }
+ }
+
+ // Prepare request body
+ var body io.Reader
+ if mapping != nil {
+ data, err := json.Marshal(mapping)
+ if err != nil {
+ return fmt.Errorf("failed to marshal mapping: %w", err)
+ }
+ body = bytes.NewReader(data)
+ }
+
+ // Create index
+ req := esapi.IndicesCreateRequest{
+ Index: baseName,
+ Body: body,
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ return fmt.Errorf("failed to create index: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ bodyBytes, _ := io.ReadAll(res.Body)
+ reason := extractErrorReason(bodyBytes)
+ if reason != "" {
+ return fmt.Errorf("elasticsearch error: %s", reason)
+ }
+ return fmt.Errorf("elasticsearch returned error: %s, body: %s", res.Status(), string(bodyBytes))
+ }
+
+ // Parse response
+ var result map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+ return fmt.Errorf("failed to parse response: %w", err)
+ }
+
+ acknowledged, ok := result["acknowledged"].(bool)
+ if !ok || !acknowledged {
+ return fmt.Errorf("index creation not acknowledged")
+ }
+
+ return nil
+}
+
+// InsertChunks inserts documents into a dataset index
+func (e *elasticsearchEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
+ fullIndexName := fmt.Sprintf("%s_%s", baseName, datasetID)
+ common.Info("Inserting chunks into Elasticsearch index", zap.String("index_name", fullIndexName), zap.String("dataset_id", datasetID), zap.Int("doc_count", len(chunks)))
+
+ if len(chunks) == 0 {
+ return []string{}, nil
+ }
+
+ if fullIndexName == "" {
+ return nil, fmt.Errorf("index name cannot be empty")
+ }
+
+ // Build bulk request body
+ var buf bytes.Buffer
+ for _, doc := range chunks {
+ // Action line - index operation
+ action := map[string]interface{}{
+ "index": map[string]interface{}{
+ "_index": fullIndexName,
+ },
+ }
+ actionBytes, err := json.Marshal(action)
+ if err != nil {
+ common.Error("Failed to marshal bulk action", err)
+ return nil, fmt.Errorf("failed to marshal bulk action: %w", err)
+ }
+ buf.Write(actionBytes)
+ buf.WriteByte('\n')
+
+ // Document line
+ docBytes, err := json.Marshal(doc)
+ if err != nil {
+ common.Error("Failed to marshal document", err)
+ return nil, fmt.Errorf("failed to marshal document: %w", err)
+ }
+ buf.Write(docBytes)
+ buf.WriteByte('\n')
+ }
+
+ // Execute bulk request
+ req := esapi.BulkRequest{
+ Body: bytes.NewReader(buf.Bytes()),
+ Refresh: "false",
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ common.Error("Failed to execute bulk request", err)
+ return nil, fmt.Errorf("failed to execute bulk request: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ common.Sugar.Errorw("Elasticsearch bulk request returned error", "status", res.Status())
+ return nil, fmt.Errorf("elasticsearch bulk request returned error: %s", res.Status())
+ }
+
+ // Parse bulk response to check for errors
+ var bulkResponse map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&bulkResponse); err != nil {
+ common.Error("Failed to parse bulk response", err)
+ return nil, fmt.Errorf("failed to parse bulk response: %w", err)
+ }
+
+ // Check for errors in bulk response
+ if errors, ok := bulkResponse["errors"].(bool); ok && errors {
+ common.Warn("Bulk request had some errors")
+ // Could iterate through items to find specific errors if needed
+ }
+
+ common.Info("Successfully inserted chunks into Elasticsearch index", zap.String("index_name", fullIndexName), zap.Int("doc_count", len(chunks)))
+ return []string{}, nil
+}
+
+// UpdateChunks updates chunks by condition
+func (e *elasticsearchEngine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error {
+ fullIndexName := fmt.Sprintf("%s_%s", baseName, datasetID)
+ common.Info("Updating chunks in Elasticsearch index", zap.String("index_name", fullIndexName), zap.Any("condition", condition), zap.Any("new_value", newValue))
+
+ if fullIndexName == "" {
+ return fmt.Errorf("index name cannot be empty")
+ }
+
+ // Check if index exists
+ exists, err := e.indexExists(ctx, fullIndexName)
+ if err != nil {
+ common.Error("Failed to check index existence", err)
+ return fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if !exists {
+ return fmt.Errorf("index '%s' does not exist", fullIndexName)
+ }
+
+ // Build query from condition
+ query := e.buildQueryFromCondition(condition)
+ if query == nil {
+ query = map[string]interface{}{"match_all": map[string]interface{}{}}
+ }
+
+ // Process remove operation if present
+ var removeOperations []map[string]interface{}
+ if removeData, ok := newValue["remove"].(map[string]interface{}); ok {
+ removeOperations = e.buildRemoveOperations(removeData, query, fullIndexName)
+ }
+ delete(newValue, "remove")
+
+ // Build update body
+ updateBody := map[string]interface{}{
+ "query": query,
+ }
+
+ // Handle script-based update if needed (for remove operations or transformations)
+ if len(removeOperations) > 0 || e.needsScriptUpdate(newValue) {
+ script := e.buildUpdateScript(newValue, removeOperations)
+ updateBody["script"] = script
+ } else {
+ updateBody["doc"] = newValue
+ }
+
+ bodyBytes, err := json.Marshal(updateBody)
+ if err != nil {
+ common.Error("Failed to marshal update body", err)
+ return fmt.Errorf("failed to marshal update body: %w", err)
+ }
+
+ // Execute update by query
+ req := esapi.UpdateByQueryRequest{
+ Index: []string{fullIndexName},
+ Body: bytes.NewReader(bodyBytes),
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ common.Error("Failed to execute update by query", err)
+ return fmt.Errorf("failed to execute update by query: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ common.Sugar.Errorw("Elasticsearch update by query returned error", "status", res.Status())
+ return fmt.Errorf("elasticsearch update by query returned error: %s", res.Status())
+ }
+
+ // Parse response
+ var result map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+ common.Error("Failed to parse update response", err)
+ return fmt.Errorf("failed to parse update response: %w", err)
+ }
+
+ if updated, ok := result["updated"].(float64); ok {
+ common.Info("Successfully updated chunks", zap.String("index_name", fullIndexName), zap.Float64("updated_count", updated))
+ }
+
+ return nil
+}
+
+// DeleteChunks deletes chunks from a dataset index by condition
+func (e *elasticsearchEngine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
+ fullIndexName := fmt.Sprintf("%s_%s", indexName, datasetID)
+ common.Info("Deleting chunks from Elasticsearch index", zap.String("index_name", fullIndexName), zap.Any("condition", condition))
+
+ // Check if index exists
+ exists, err := e.indexExists(ctx, fullIndexName)
+ if err != nil {
+ return 0, fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if !exists {
+ common.Warn(fmt.Sprintf("Index %s does not exist, skipping delete", fullIndexName))
+ return 0, nil
+ }
+
+ // Build query from condition
+ query := e.buildQueryFromCondition(condition)
+ if query == nil {
+ query = map[string]interface{}{"match_all": map[string]interface{}{}}
+ }
+
+ // Build delete by query body
+ deleteBody := map[string]interface{}{
+ "query": query,
+ }
+
+ bodyBytes, err := json.Marshal(deleteBody)
+ if err != nil {
+ return 0, fmt.Errorf("failed to marshal delete body: %w", err)
+ }
+
+ // Execute delete by query
+ req := esapi.DeleteByQueryRequest{
+ Index: []string{fullIndexName},
+ Body: bytes.NewReader(bodyBytes),
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ common.Error("Failed to execute delete by query", err)
+ return 0, fmt.Errorf("failed to execute delete by query: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ common.Sugar.Errorw("Elasticsearch delete by query returned error", "status", res.Status())
+ return 0, fmt.Errorf("elasticsearch delete by query returned error: %s", res.Status())
+ }
+
+ // Parse response
+ var result map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+ common.Error("Failed to parse delete response", err)
+ return 0, fmt.Errorf("failed to parse delete response: %w", err)
+ }
+
+ deleted := int64(0)
+ if d, ok := result["deleted"].(float64); ok {
+ deleted = int64(d)
+ }
+
+ common.Info("Successfully deleted chunks", zap.String("index_name", fullIndexName), zap.Int64("deleted_count", deleted))
+ return deleted, nil
+}
+
+// SearchResponse Elasticsearch search response
+type SearchResponse struct {
+ Hits struct {
+ Total struct {
+ Value int64 `json:"value"`
+ } `json:"total"`
+ Hits []struct {
+ ID string `json:"_id"`
+ Score float64 `json:"_score"`
+ Source map[string]interface{} `json:"_source"`
+ } `json:"hits"`
+ } `json:"hits"`
+ Aggregations map[string]interface{} `json:"aggregations"`
+}
+
+// Search executes search with unified types.SearchRequest
+func (e *elasticsearchEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ return e.searchUnified(ctx, req)
+}
+
+// searchUnified handles the unified types.SearchRequest
+func (e *elasticsearchEngine) searchUnified(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ if len(req.IndexNames) == 0 {
+ return nil, fmt.Errorf("index names cannot be empty")
+ }
+
+ // Build pagination parameters
+ offset := req.Offset
+ limit := req.Limit
+ if limit <= 0 {
+ limit = 30 // default ES size
+ }
+
+ // Check if this is a skill index
+ isSkillIndex := len(req.IndexNames) > 0 && strings.HasPrefix(req.IndexNames[0], "skill_")
+
+ // Build filter clauses
+ var filterClauses []map[string]interface{}
+ if isSkillIndex {
+ filterClauses = buildSkillFilterClauses()
+ } else {
+ filterClauses = buildFilterClauses(req.KbIDs, 1)
+ }
+
+ // Add filters from req.Filter
+ if req.Filter != nil && len(req.Filter) > 0 {
+ filterClauses = append(filterClauses, buildFilterFromMap(req.Filter)...)
+ }
+
+ // Build search query body
+ queryBody := make(map[string]interface{})
+
+ // Determine search type from MatchExprs
+ var matchText string
+ var matchDense *types.MatchDenseExpr
+ var hasVectorMatch bool
+
+ for _, expr := range req.MatchExprs {
+ if expr == nil {
+ continue
+ }
+ switch e := expr.(type) {
+ case string:
+ matchText = e
+ case *types.MatchTextExpr:
+ matchText = e.MatchingText
+ case *types.MatchDenseExpr:
+ hasVectorMatch = true
+ matchDense = e
+ }
+ }
+
+ var vectorFieldName string
+ if !hasVectorMatch || matchDense == nil {
+ // Keyword-only search
+ if isSkillIndex {
+ queryBody["query"] = buildSkillKeywordQuery(matchText, filterClauses, 1.0)
+ } else {
+ queryBody["query"] = buildESKeywordQuery(matchText, filterClauses, 1.0)
+ }
+ } else {
+ // Hybrid search: keyword + vector
+ textWeight := 0.7 // default: vector weight = 0.3
+ vectorWeight := 0.3
+ if matchDense.ExtraOptions != nil {
+ if vw, ok := matchDense.ExtraOptions["text_weight"].(float64); ok {
+ textWeight = vw
+ }
+ if vw, ok := matchDense.ExtraOptions["vector_weight"].(float64); ok {
+ vectorWeight = vw
+ }
+ }
+
+ // Build boolean query for text match and filters
+ var boolQuery map[string]interface{}
+ if isSkillIndex {
+ boolQuery = buildSkillKeywordQuery(matchText, filterClauses, 1.0)
+ } else {
+ boolQuery = buildESKeywordQuery(matchText, filterClauses, 1.0)
+ }
+ // Add boost to the bool query (as in Python code)
+ if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
+ boolMap["boost"] = textWeight
+ }
+
+ // Build kNN query
+ vectorData := matchDense.EmbeddingData
+ vectorFieldName = matchDense.VectorColumnName
+ k := matchDense.TopN
+ if k <= 0 {
+ k = req.Limit
+ }
+ if k <= 0 {
+ k = 1024
+ }
+ numCandidates := k * 2
+
+ similarity := 0.0
+ if matchDense.ExtraOptions != nil {
+ if sim, ok := matchDense.ExtraOptions["similarity"].(float64); ok {
+ similarity = sim
+ }
+ }
+
+ knnQuery := map[string]interface{}{
+ "field": vectorFieldName,
+ "query_vector": vectorData,
+ "k": k,
+ "num_candidates": numCandidates,
+ "similarity": similarity,
+ "boost": vectorWeight,
+ }
+
+ queryBody["knn"] = knnQuery
+ queryBody["query"] = boolQuery
+
+ // Add vector column to Source fields (matching Python ES: src.append(f"q_{len(q_vec)}_vec"))
+ // Only modify Source if it was explicitly set by the caller
+ if vectorFieldName != "" && len(req.SelectFields) > 0 {
+ sourceFields := req.SelectFields
+ found := false
+ for _, f := range sourceFields {
+ if f == vectorFieldName {
+ found = true
+ break
+ }
+ }
+ if !found {
+ sourceFields = append(sourceFields, vectorFieldName)
+ }
+ req.SelectFields = sourceFields
+ }
+ }
+
+ queryBody["size"] = limit
+ queryBody["from"] = offset
+
+ // Add sorting if specified
+ if req.OrderBy != nil {
+ sort := parseOrderByExpr(req.OrderBy)
+ if len(sort) > 0 {
+ queryBody["sort"] = sort
+ }
+ }
+
+ // Serialize query
+ var buf bytes.Buffer
+ if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
+ return nil, fmt.Errorf("error encoding query: %w", err)
+ }
+
+ // Log search details
+ common.Debug("Elasticsearch searching indices", zap.Strings("indices", req.IndexNames))
+ common.Debug("Elasticsearch DSL", zap.Any("dsl", queryBody))
+
+ // Build search request
+ reqES := esapi.SearchRequest{
+ Index: req.IndexNames,
+ Body: &buf,
+ }
+
+ // Execute search
+ res, err := reqES.Do(ctx, e.client)
+ if err != nil {
+ return nil, fmt.Errorf("search failed: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ bodyBytes, err := io.ReadAll(res.Body)
+ if err != nil {
+ common.Error("Elasticsearch failed to read error response body", err)
+ } else {
+ common.Warn("Elasticsearch error response", zap.String("body", string(bodyBytes)))
+ }
+ return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status())
+ }
+
+ // Parse response
+ var esResp SearchResponse
+ if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
+ return nil, fmt.Errorf("error parsing response: %w", err)
+ }
+
+ // Convert to unified response
+ chunks := convertESResponse(&esResp, vectorFieldName)
+ return &types.SearchResult{
+ Chunks: chunks,
+ Total: esResp.Hits.Total.Value,
+ }, nil
+}
+
+// GetChunk gets a chunk by ID
+func (e *elasticsearchEngine) GetChunk(ctx context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error) {
+ // Build unified search request to get the chunk by ID
+ searchReq := &types.SearchRequest{
+ IndexNames: []string{baseName},
+ Limit: 1,
+ Offset: 0,
+ Filter: map[string]interface{}{
+ "id": chunkID,
+ },
+ }
+
+ // Execute search
+ searchResp, err := e.Search(ctx, searchReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to search: %w", err)
+ }
+
+ if len(searchResp.Chunks) == 0 {
+ return nil, nil
+ }
+
+ return searchResp.Chunks[0], nil
+}
+
+// GetFields is not implemented for Elasticsearch
+func (e *elasticsearchEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
+ common.Warn("GetFields not implemented for Elasticsearch")
+ return nil
+}
+
+// GetAggregation is not implemented for Elasticsearch
+func (e *elasticsearchEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {
+ common.Warn("GetAggregation not implemented for Elasticsearch")
+ return nil
+}
+
+// GetHighlight is not implemented for Elasticsearch
+func (e *elasticsearchEngine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {
+ common.Warn("GetHighlight not implemented for Elasticsearch")
+ return nil
+}
+
+// DropChunkStore deletes a chunk index
+func (e *elasticsearchEngine) DropChunkStore(ctx context.Context, baseName, datasetID string) error {
+ return e.dropIndex(ctx, baseName)
+}
+
+// ChunkStoreExists checks if a chunk index exists
+func (e *elasticsearchEngine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) {
+ return e.indexExists(ctx, baseName)
+}
+
+// buildQueryFromCondition builds an ES query from condition map
+func (e *elasticsearchEngine) buildQueryFromCondition(condition map[string]interface{}) map[string]interface{} {
+ if len(condition) == 0 {
+ return nil
+ }
+
+ var clauses []map[string]interface{}
+
+ for k, v := range condition {
+ if v == nil {
+ continue
+ }
+
+ switch k {
+ case "kb_id":
+ // Handle kb_id as terms query
+ if listVal, ok := v.([]interface{}); ok {
+ clauses = append(clauses, map[string]interface{}{
+ "terms": map[string]interface{}{k: listVal},
+ })
+ } else {
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ }
+ case "id":
+ // Handle id as terms or term query
+ if listVal, ok := v.([]interface{}); ok {
+ clauses = append(clauses, map[string]interface{}{
+ "terms": map[string]interface{}{k: listVal},
+ })
+ } else {
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ }
+ case "available_int":
+ // Handle available_int as term query
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ default:
+ // Default: treat as term query
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ }
+ }
+
+ if len(clauses) == 0 {
+ return nil
+ }
+
+ if len(clauses) == 1 {
+ return clauses[0]
+ }
+
+ return map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must": clauses,
+ },
+ }
+}
+
+// buildRemoveOperations builds ES script operations for remove
+func (e *elasticsearchEngine) buildRemoveOperations(removeData map[string]interface{}, query map[string]interface{}, indexName string) []map[string]interface{} {
+ // For ES, we handle removals differently - they are typically done via separate update operations
+ // This is a simplified implementation
+ return nil
+}
+
+// needsScriptUpdate checks if the update requires a script (more complex operations)
+func (e *elasticsearchEngine) needsScriptUpdate(newValue map[string]interface{}) bool {
+ // Check if any values contain operations that need scripts
+ return false
+}
+
+// buildUpdateScript builds an ES script for updates
+func (e *elasticsearchEngine) buildUpdateScript(newValue map[string]interface{}, removeOperations []map[string]interface{}) map[string]interface{} {
+ script := map[string]interface{}{
+ "source": "ctx._source.putAll(params.doc)",
+ "params": map[string]interface{}{
+ "doc": newValue,
+ },
+ }
+ return script
+}
+
+// buildMetadataQueryFromCondition builds an ES query for metadata index
+func (e *elasticsearchEngine) buildMetadataQueryFromCondition(condition map[string]interface{}) map[string]interface{} {
+ if len(condition) == 0 {
+ return nil
+ }
+
+ var clauses []map[string]interface{}
+
+ for k, v := range condition {
+ if v == nil {
+ continue
+ }
+
+ switch k {
+ case "kb_id":
+ if listVal, ok := v.([]interface{}); ok {
+ clauses = append(clauses, map[string]interface{}{
+ "terms": map[string]interface{}{k: listVal},
+ })
+ } else {
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ }
+ case "id":
+ if listVal, ok := v.([]interface{}); ok {
+ clauses = append(clauses, map[string]interface{}{
+ "terms": map[string]interface{}{k: listVal},
+ })
+ } else {
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ }
+ default:
+ clauses = append(clauses, map[string]interface{}{
+ "term": map[string]interface{}{k: v},
+ })
+ }
+ }
+
+ if len(clauses) == 0 {
+ return nil
+ }
+
+ if len(clauses) == 1 {
+ return clauses[0]
+ }
+
+ return map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must": clauses,
+ },
+ }
+}
+
+// loadSkillMapping loads the skill index mapping from config file
+func loadSkillMapping() (map[string]interface{}, error) {
+ // Try multiple possible locations for the mapping file
+ possiblePaths := []string{
+ "conf/skill_es_mapping.json",
+ "../conf/skill_es_mapping.json",
+ "/app/conf/skill_es_mapping.json",
+ }
+
+ var data []byte
+ var err error
+ for _, path := range possiblePaths {
+ data, err = os.ReadFile(path)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ // Fallback to default skill mapping if file not found
+ return getDefaultSkillMapping(), nil
+ }
+
+ var mapping map[string]interface{}
+ if err := json.Unmarshal(data, &mapping); err != nil {
+ return nil, fmt.Errorf("failed to parse skill mapping: %w", err)
+ }
+
+ return mapping, nil
+}
+
+// getDefaultSkillMapping returns the default skill index mapping
+func getDefaultSkillMapping() map[string]interface{} {
+ return map[string]interface{}{
+ "settings": map[string]interface{}{
+ "index": map[string]interface{}{
+ "number_of_shards": 1,
+ "number_of_replicas": 0,
+ "refresh_interval": "1000ms",
+ },
+ },
+ "mappings": map[string]interface{}{
+ "dynamic": false,
+ "properties": map[string]interface{}{
+ "skill_id": map[string]interface{}{
+ "type": "keyword",
+ "store": true,
+ },
+ "name": map[string]interface{}{
+ "type": "text",
+ "index": false,
+ "store": true,
+ },
+ "name_tks": map[string]interface{}{
+ "type": "text",
+ "analyzer": "whitespace",
+ "store": true,
+ },
+ "tags": map[string]interface{}{
+ "type": "text",
+ "index": false,
+ "store": true,
+ },
+ "tags_tks": map[string]interface{}{
+ "type": "text",
+ "analyzer": "whitespace",
+ "store": true,
+ },
+ "description": map[string]interface{}{
+ "type": "text",
+ "index": false,
+ "store": true,
+ },
+ "description_tks": map[string]interface{}{
+ "type": "text",
+ "analyzer": "whitespace",
+ "store": true,
+ },
+ "content": map[string]interface{}{
+ "type": "text",
+ "index": false,
+ "store": true,
+ },
+ "content_tks": map[string]interface{}{
+ "type": "text",
+ "analyzer": "whitespace",
+ "store": true,
+ },
+ "q_3072_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 3072,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "q_2560_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 2560,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "q_1536_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 1536,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "q_1024_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 1024,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "q_768_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 768,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "q_512_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 512,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "q_256_vec": map[string]interface{}{
+ "type": "dense_vector",
+ "dims": 256,
+ "index": true,
+ "similarity": "cosine",
+ },
+ "version": map[string]interface{}{
+ "type": "keyword",
+ "store": true,
+ },
+ "status": map[string]interface{}{
+ "type": "keyword",
+ "store": true,
+ },
+ "create_time": map[string]interface{}{
+ "type": "long",
+ "store": true,
+ },
+ "update_time": map[string]interface{}{
+ "type": "long",
+ "store": true,
+ },
+ },
+ },
+ }
+}
+
+// calculatePagination calculates offset and limit based on page, size and topK
+func calculatePagination(page, size, topK int) (int, int) {
+ if page < 1 {
+ page = 1
+ }
+ if size <= 0 {
+ size = 30
+ }
+ if topK <= 0 {
+ topK = 1024
+ }
+
+ RERANK_LIMIT := max(30, (64/size)*size)
+ if RERANK_LIMIT < size {
+ RERANK_LIMIT = size
+ }
+ if RERANK_LIMIT > topK {
+ RERANK_LIMIT = topK
+ }
+
+ offset := (page - 1) * RERANK_LIMIT
+ if offset < 0 {
+ offset = 0
+ }
+
+ return offset, RERANK_LIMIT
+}
+
+// buildFilterClauses builds ES filter clauses from kb_ids and available_int
+// Reference: rag/utils/es_conn.py L60-L78
+// When available=0: available_int < 1
+// When available!=0: NOT (available_int < 1)
+func buildFilterClauses(datasetIDs []string, available int) []map[string]interface{} {
+ var filters []map[string]interface{}
+
+ if len(datasetIDs) > 0 {
+ filters = append(filters, map[string]interface{}{
+ "terms": map[string]interface{}{"kb_id": datasetIDs},
+ })
+ }
+
+ // Add available_int filter
+ // Reference: rag/utils/es_conn.py L63-L68
+ if available == 0 {
+ // available_int < 1
+ filters = append(filters, map[string]interface{}{
+ "range": map[string]interface{}{
+ "available_int": map[string]interface{}{
+ "lt": 1,
+ },
+ },
+ })
+ } else {
+ // must_not: available_int < 1 (i.e., available_int >= 1)
+ filters = append(filters, map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must_not": []map[string]interface{}{
+ {
+ "range": map[string]interface{}{
+ "available_int": map[string]interface{}{
+ "lt": 1,
+ },
+ },
+ },
+ },
+ },
+ })
+ }
+
+ return filters
+}
+
+// buildSkillFilterClauses builds ES filter clauses for skill index
+// Skill index uses 'status' field instead of 'available_int'
+func buildSkillFilterClauses() []map[string]interface{} {
+ // Filter for active skills (status = "1")
+ return []map[string]interface{}{
+ {
+ "term": map[string]interface{}{
+ "status": "1",
+ },
+ },
+ }
+}
+
+// buildFilterFromMap converts a generic filter map to ES filter clauses
+func buildFilterFromMap(filter map[string]interface{}) []map[string]interface{} {
+ var filters []map[string]interface{}
+ for field, value := range filter {
+ switch v := value.(type) {
+ case []string:
+ filters = append(filters, map[string]interface{}{
+ "terms": map[string]interface{}{field: v},
+ })
+ case []interface{}:
+ filters = append(filters, map[string]interface{}{
+ "terms": map[string]interface{}{field: v},
+ })
+ default:
+ filters = append(filters, map[string]interface{}{
+ "term": map[string]interface{}{field: v},
+ })
+ }
+ }
+ return filters
+}
+
+// buildESKeywordQuery builds keyword-only search query for ES
+// Uses query_string if matchText is in query_string format, otherwise uses multi_match
+// boost is applied to the text match clause (query_string or multi_match)
+func buildESKeywordQuery(matchText string, filterClauses []map[string]interface{}, boost float64) map[string]interface{} {
+ var mustClause map[string]interface{}
+
+ // Handle wildcard query (match all)
+ if matchText == "*" || matchText == "" {
+ mustClause = map[string]interface{}{
+ "match_all": map[string]interface{}{},
+ }
+ } else {
+ // Use query_string for complex queries
+ queryString := map[string]interface{}{
+ "query": matchText,
+ "fields": []string{"title_tks^10", "title_sm_tks^5", "important_kwd^30", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"},
+ "type": "best_fields",
+ "minimum_should_match": "30%",
+ "boost": boost,
+ }
+ mustClause = map[string]interface{}{
+ "query_string": queryString,
+ }
+ }
+
+ return map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must": mustClause,
+ "filter": filterClauses,
+ },
+ }
+}
+
+// buildSkillKeywordQuery builds keyword-only search query for skill index
+// Skill index uses different field names: name_tks, tags_tks, description_tks, content_tks
+func buildSkillKeywordQuery(matchText string, filterClauses []map[string]interface{}, boost float64) map[string]interface{} {
+ var mustClause map[string]interface{}
+
+ // Handle wildcard query (match all)
+ if matchText == "*" || matchText == "" {
+ mustClause = map[string]interface{}{
+ "match_all": map[string]interface{}{},
+ }
+ } else {
+ // Use query_string for complex queries with skill-specific fields
+ queryString := map[string]interface{}{
+ "query": matchText,
+ "fields": []string{"name_tks^10", "tags_tks^5", "description_tks^3", "content_tks^1"},
+ "type": "best_fields",
+ "minimum_should_match": "30%",
+ "boost": boost,
+ }
+ mustClause = map[string]interface{}{
+ "query_string": queryString,
+ }
+ }
+
+ return map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must": mustClause,
+ "filter": filterClauses,
+ },
+ }
+}
+
+// convertESResponse converts ES SearchResponse to unified chunks format
+func convertESResponse(esResp *SearchResponse, vectorFieldName string) []map[string]interface{} {
+ if esResp == nil || esResp.Hits.Hits == nil {
+ return []map[string]interface{}{}
+ }
+
+ chunks := make([]map[string]interface{}, len(esResp.Hits.Hits))
+ for i, hit := range esResp.Hits.Hits {
+ chunks[i] = hit.Source
+ chunks[i]["_score"] = hit.Score
+ chunks[i]["_id"] = hit.ID
+ }
+ return chunks
+}
+
+// parseOrderByExpr parses the OrderBy expression into ES sort format
+func parseOrderByExpr(orderBy *types.OrderByExpr) []map[string]interface{} {
+ if orderBy == nil || len(orderBy.Fields) == 0 {
+ return nil
+ }
+
+ var result []map[string]interface{}
+ for _, field := range orderBy.Fields {
+ direction := "asc"
+ if field.Type == types.SortDesc {
+ direction = "desc"
+ }
+
+ if field.Field == "_score" || field.Field == "score" {
+ result = append(result, map[string]interface{}{
+ "_score": direction,
+ })
+ } else {
+ result = append(result, map[string]interface{}{
+ field.Field: direction,
+ })
+ }
+ }
+
+ return result
+}
+
+// Helper query builder functions (legacy)
+
+// BuildMatchTextQuery builds a text match query
+func BuildMatchTextQuery(fields []string, text string, fuzziness string) map[string]interface{} {
+ query := map[string]interface{}{
+ "multi_match": map[string]interface{}{
+ "query": text,
+ "fields": fields,
+ },
+ }
+
+ if fuzziness != "" {
+ if multiMatch, ok := query["multi_match"].(map[string]interface{}); ok {
+ multiMatch["fuzziness"] = fuzziness
+ }
+ }
+
+ return query
+}
+
+// BuildTermQuery builds a term query
+func BuildTermQuery(field string, value interface{}) map[string]interface{} {
+ return map[string]interface{}{
+ "term": map[string]interface{}{
+ field: value,
+ },
+ }
+}
+
+// BuildRangeQuery builds a range query
+func BuildRangeQuery(field string, from, to interface{}) map[string]interface{} {
+ rangeQuery := make(map[string]interface{})
+ if from != nil {
+ rangeQuery["gte"] = from
+ }
+ if to != nil {
+ rangeQuery["lte"] = to
+ }
+
+ return map[string]interface{}{
+ "range": map[string]interface{}{
+ field: rangeQuery,
+ },
+ }
+}
+
+// BuildBoolQuery builds a bool query
+func BuildBoolQuery() map[string]interface{} {
+ return map[string]interface{}{
+ "bool": make(map[string]interface{}),
+ }
+}
+
+// AddMust adds must clause to bool query
+func AddMust(query map[string]interface{}, clauses ...map[string]interface{}) {
+ if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
+ if _, exists := boolQuery["must"]; !exists {
+ boolQuery["must"] = []map[string]interface{}{}
+ }
+ if must, ok := boolQuery["must"].([]map[string]interface{}); ok {
+ boolQuery["must"] = append(must, clauses...)
+ }
+ }
+}
+
+// AddShould adds should clause to bool query
+func AddShould(query map[string]interface{}, clauses ...map[string]interface{}) {
+ if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
+ if _, exists := boolQuery["should"]; !exists {
+ boolQuery["should"] = []map[string]interface{}{}
+ }
+ if should, ok := boolQuery["should"].([]map[string]interface{}); ok {
+ boolQuery["should"] = append(should, clauses...)
+ }
+ }
+}
+
+// AddFilter adds filter clause to bool query
+func AddFilter(query map[string]interface{}, clauses ...map[string]interface{}) {
+ if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
+ if _, exists := boolQuery["filter"]; !exists {
+ boolQuery["filter"] = []map[string]interface{}{}
+ }
+ if filter, ok := boolQuery["filter"].([]map[string]interface{}); ok {
+ boolQuery["filter"] = append(filter, clauses...)
+ }
+ }
+}
+
+// AddMustNot adds must_not clause to bool query
+func AddMustNot(query map[string]interface{}, clauses ...map[string]interface{}) {
+ if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
+ if _, exists := boolQuery["must_not"]; !exists {
+ boolQuery["must_not"] = []map[string]interface{}{}
+ }
+ if mustNot, ok := boolQuery["must_not"].([]map[string]interface{}); ok {
+ boolQuery["must_not"] = append(mustNot, clauses...)
+ }
+ }
+}
+
+// GetDocIDs is not implemented for Elasticsearch
+func (e *elasticsearchEngine) GetDocIDs(chunks []map[string]interface{}) []string {
+ common.Warn("GetDocIDs not implemented for Elasticsearch")
+ return nil
+}
\ No newline at end of file
diff --git a/internal/engine/elasticsearch/common.go b/internal/engine/elasticsearch/common.go
new file mode 100644
index 00000000000..e4bf5e1bed5
--- /dev/null
+++ b/internal/engine/elasticsearch/common.go
@@ -0,0 +1,98 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package elasticsearch
+
+import (
+ "context"
+ "fmt"
+ "io"
+
+ "github.com/elastic/go-elasticsearch/v8/esapi"
+)
+
+// dropIndex deletes an index
+func (e *elasticsearchEngine) dropIndex(ctx context.Context, indexName string) error {
+ if indexName == "" {
+ return fmt.Errorf("index name cannot be empty")
+ }
+
+ // Check if index exists
+ exists, err := e.indexExists(ctx, indexName)
+ if err != nil {
+ return fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if !exists {
+ return fmt.Errorf("index '%s' does not exist", indexName)
+ }
+
+ // Delete index
+ req := esapi.IndicesDeleteRequest{
+ Index: []string{indexName},
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ return fmt.Errorf("failed to delete index: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ bodyBytes, _ := io.ReadAll(res.Body)
+ reason := extractErrorReason(bodyBytes)
+ if reason != "" {
+ return fmt.Errorf("elasticsearch error: %s", reason)
+ }
+ return fmt.Errorf("elasticsearch returned error: %s", res.Status())
+ }
+
+ return nil
+}
+
+// indexExists checks if index exists
+func (e *elasticsearchEngine) indexExists(ctx context.Context, indexName string) (bool, error) {
+ if indexName == "" {
+ return false, fmt.Errorf("index name cannot be empty")
+ }
+
+ req := esapi.IndicesExistsRequest{
+ Index: []string{indexName},
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ return false, fmt.Errorf("failed to check index existence: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode == 200 {
+ return true, nil
+ } else if res.StatusCode == 404 {
+ return false, nil
+ }
+
+ bodyBytes, _ := io.ReadAll(res.Body)
+ reason := extractErrorReason(bodyBytes)
+ if reason != "" {
+ return false, fmt.Errorf("elasticsearch error: %s", reason)
+ }
+ return false, fmt.Errorf("elasticsearch returned error: %s", res.Status())
+}
+
+// buildMetadataIndexName returns the metadata index name for a tenant
+func buildMetadataIndexName(tenantID string) string {
+ return fmt.Sprintf("ragflow_doc_meta_%s", tenantID)
+}
\ No newline at end of file
diff --git a/internal/engine/elasticsearch/get.go b/internal/engine/elasticsearch/get.go
deleted file mode 100644
index 625bacdda70..00000000000
--- a/internal/engine/elasticsearch/get.go
+++ /dev/null
@@ -1,49 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package elasticsearch
-
-import (
- "context"
- "fmt"
-
- "ragflow/internal/engine/types"
-)
-
-// GetChunk gets a chunk by ID
-func (e *elasticsearchEngine) GetChunk(ctx context.Context, indexName, chunkID string, kbIDs []string) (interface{}, error) {
- // Build unified search request to get the chunk by ID
- searchReq := &types.SearchRequest{
- IndexNames: []string{indexName},
- Limit: 1,
- Offset: 0,
- Filter: map[string]interface{}{
- "id": chunkID,
- },
- }
-
- // Execute search
- searchResp, err := e.Search(ctx, searchReq)
- if err != nil {
- return nil, fmt.Errorf("failed to search: %w", err)
- }
-
- if len(searchResp.Chunks) == 0 {
- return nil, nil
- }
-
- return searchResp.Chunks[0], nil
-}
\ No newline at end of file
diff --git a/internal/engine/elasticsearch/index.go b/internal/engine/elasticsearch/index.go
deleted file mode 100644
index b2039691073..00000000000
--- a/internal/engine/elasticsearch/index.go
+++ /dev/null
@@ -1,362 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package elasticsearch
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "os"
-
- "github.com/elastic/go-elasticsearch/v8/esapi"
-)
-
-// CreateDataset creates an index
-func (e *elasticsearchEngine) CreateDataset(ctx context.Context, indexName, datasetID string, vectorSize int, parserID string) error {
- if indexName == "" {
- return fmt.Errorf("index name cannot be empty")
- }
-
- // Check if index already exists
- exists, err := e.TableExists(ctx, indexName)
- if err != nil {
- return fmt.Errorf("failed to check index existence: %w", err)
- }
- if exists {
- return fmt.Errorf("index '%s' already exists", indexName)
- }
-
- // Load mapping based on index type
- var mapping map[string]interface{}
- if datasetID == "skill" {
- // Load skill-specific mapping
- skillMapping, err := loadSkillMapping()
- if err != nil {
- return fmt.Errorf("failed to load skill mapping: %w", err)
- }
- mapping = skillMapping
- } else {
- // Default mapping for dataset
- mapping = map[string]interface{}{
- "settings": map[string]interface{}{
- "number_of_shards": 1,
- "number_of_replicas": 0,
- },
- }
- }
-
- // Prepare request body
- var body io.Reader
- if mapping != nil {
- data, err := json.Marshal(mapping)
- if err != nil {
- return fmt.Errorf("failed to marshal mapping: %w", err)
- }
- body = bytes.NewReader(data)
- }
-
- // Create index
- req := esapi.IndicesCreateRequest{
- Index: indexName,
- Body: body,
- }
-
- res, err := req.Do(ctx, e.client)
- if err != nil {
- return fmt.Errorf("failed to create index: %w", err)
- }
- defer res.Body.Close()
-
- if res.IsError() {
- bodyBytes, _ := io.ReadAll(res.Body)
- reason := extractErrorReason(bodyBytes)
- if reason != "" {
- return fmt.Errorf("elasticsearch error: %s", reason)
- }
- return fmt.Errorf("elasticsearch returned error: %s, body: %s", res.Status(), string(bodyBytes))
- }
-
- // Parse response
- var result map[string]interface{}
- if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
- return fmt.Errorf("failed to parse response: %w", err)
- }
-
- acknowledged, ok := result["acknowledged"].(bool)
- if !ok || !acknowledged {
- return fmt.Errorf("index creation not acknowledged")
- }
-
- return nil
-}
-
-// loadSkillMapping loads the skill index mapping from config file
-func loadSkillMapping() (map[string]interface{}, error) {
- // Try multiple possible locations for the mapping file
- possiblePaths := []string{
- "conf/skill_es_mapping.json",
- "../conf/skill_es_mapping.json",
- "/app/conf/skill_es_mapping.json",
- }
-
- var data []byte
- var err error
- for _, path := range possiblePaths {
- data, err = os.ReadFile(path)
- if err == nil {
- break
- }
- }
-
- if err != nil {
- // Fallback to default skill mapping if file not found
- return getDefaultSkillMapping(), nil
- }
-
- var mapping map[string]interface{}
- if err := json.Unmarshal(data, &mapping); err != nil {
- return nil, fmt.Errorf("failed to parse skill mapping: %w", err)
- }
-
- return mapping, nil
-}
-
-// getDefaultSkillMapping returns the default skill index mapping
-func getDefaultSkillMapping() map[string]interface{} {
- return map[string]interface{}{
- "settings": map[string]interface{}{
- "index": map[string]interface{}{
- "number_of_shards": 1,
- "number_of_replicas": 0,
- "refresh_interval": "1000ms",
- },
- },
- "mappings": map[string]interface{}{
- "dynamic": false,
- "properties": map[string]interface{}{
- "skill_id": map[string]interface{}{
- "type": "keyword",
- "store": true,
- },
- "name": map[string]interface{}{
- "type": "text",
- "index": false,
- "store": true,
- },
- "name_tks": map[string]interface{}{
- "type": "text",
- "analyzer": "whitespace",
- "store": true,
- },
- "tags": map[string]interface{}{
- "type": "text",
- "index": false,
- "store": true,
- },
- "tags_tks": map[string]interface{}{
- "type": "text",
- "analyzer": "whitespace",
- "store": true,
- },
- "description": map[string]interface{}{
- "type": "text",
- "index": false,
- "store": true,
- },
- "description_tks": map[string]interface{}{
- "type": "text",
- "analyzer": "whitespace",
- "store": true,
- },
- "content": map[string]interface{}{
- "type": "text",
- "index": false,
- "store": true,
- },
- "content_tks": map[string]interface{}{
- "type": "text",
- "analyzer": "whitespace",
- "store": true,
- },
- "q_3072_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 3072,
- "index": true,
- "similarity": "cosine",
- },
- "q_2560_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 2560,
- "index": true,
- "similarity": "cosine",
- },
- "q_1536_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 1536,
- "index": true,
- "similarity": "cosine",
- },
- "q_1024_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 1024,
- "index": true,
- "similarity": "cosine",
- },
- "q_768_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 768,
- "index": true,
- "similarity": "cosine",
- },
- "q_512_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 512,
- "index": true,
- "similarity": "cosine",
- },
- "q_256_vec": map[string]interface{}{
- "type": "dense_vector",
- "dims": 256,
- "index": true,
- "similarity": "cosine",
- },
- "version": map[string]interface{}{
- "type": "keyword",
- "store": true,
- },
- "status": map[string]interface{}{
- "type": "keyword",
- "store": true,
- },
- "create_time": map[string]interface{}{
- "type": "long",
- "store": true,
- },
- "update_time": map[string]interface{}{
- "type": "long",
- "store": true,
- },
- },
- },
- }
-}
-
-// DropTable deletes an index
-func (e *elasticsearchEngine) DropTable(ctx context.Context, indexName string) error {
- if indexName == "" {
- return fmt.Errorf("index name cannot be empty")
- }
-
- // Check if index exists
- exists, err := e.TableExists(ctx, indexName)
- if err != nil {
- return fmt.Errorf("failed to check index existence: %w", err)
- }
- if !exists {
- return fmt.Errorf("index '%s' does not exist", indexName)
- }
-
- // Delete index
- req := esapi.IndicesDeleteRequest{
- Index: []string{indexName},
- }
-
- res, err := req.Do(ctx, e.client)
- if err != nil {
- return fmt.Errorf("failed to delete index: %w", err)
- }
- defer res.Body.Close()
-
- if res.IsError() {
- bodyBytes, _ := io.ReadAll(res.Body)
- reason := extractErrorReason(bodyBytes)
- if reason != "" {
- return fmt.Errorf("elasticsearch error: %s", reason)
- }
- return fmt.Errorf("elasticsearch returned error: %s", res.Status())
- }
-
- return nil
-}
-
-// TableExists checks if index exists
-func (e *elasticsearchEngine) TableExists(ctx context.Context, indexName string) (bool, error) {
- if indexName == "" {
- return false, fmt.Errorf("index name cannot be empty")
- }
-
- req := esapi.IndicesExistsRequest{
- Index: []string{indexName},
- }
-
- res, err := req.Do(ctx, e.client)
- if err != nil {
- return false, fmt.Errorf("failed to check index existence: %w", err)
- }
- defer res.Body.Close()
-
- if res.StatusCode == 200 {
- return true, nil
- } else if res.StatusCode == 404 {
- return false, nil
- }
-
- bodyBytes, _ := io.ReadAll(res.Body)
- reason := extractErrorReason(bodyBytes)
- if reason != "" {
- return false, fmt.Errorf("elasticsearch error: %s", reason)
- }
- return false, fmt.Errorf("elasticsearch returned error: %s", res.Status())
-}
-
-// CreateMetadata creates the document metadata index
-func (e *elasticsearchEngine) CreateMetadata(ctx context.Context, indexName string) error {
- // TODO
- return nil
-}
-
-// InsertDataset inserts documents into a dataset index
-func (e *elasticsearchEngine) InsertDataset(ctx context.Context, documents []map[string]interface{}, indexName string, knowledgebaseID string) ([]string, error) {
- // TODO
- return []string{}, nil
-}
-
-// InsertMetadata inserts documents into tenant's metadata index
-func (e *elasticsearchEngine) InsertMetadata(ctx context.Context, documents []map[string]interface{}, tenantID string) ([]string, error) {
- // TODO
- return []string{}, nil
-}
-
-// UpdateDataset updates a chunk by condition
-func (e *elasticsearchEngine) UpdateDataset(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, tableNamePrefix string, knowledgebaseID string) error {
- // TODO
- return nil
-}
-
-// UpdateMetadata updates document metadata in tenant's metadata index
-func (e *elasticsearchEngine) UpdateMetadata(ctx context.Context, docID string, kbID string, metaFields map[string]interface{}, tenantID string) error {
- // TODO
- return nil
-}
-
-// Delete deletes rows from either a dataset index or metadata index
-func (e *elasticsearchEngine) Delete(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
- // TODO
- return 0, nil
-}
diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.go
new file mode 100644
index 00000000000..27270868082
--- /dev/null
+++ b/internal/engine/elasticsearch/metadata.go
@@ -0,0 +1,275 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package elasticsearch
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+
+ "github.com/elastic/go-elasticsearch/v8/esapi"
+ "ragflow/internal/common"
+
+ "go.uber.org/zap"
+)
+
+// CreateMetadataStore creates the document metadata index
+func (e *elasticsearchEngine) CreateMetadataStore(ctx context.Context, tenantID string) error {
+ indexName := buildMetadataIndexName(tenantID)
+ req := esapi.IndicesCreateRequest{
+ Index: indexName,
+ }
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ return fmt.Errorf("failed to create metadata index: %w", err)
+ }
+ defer res.Body.Close()
+ if res.IsError() {
+ return fmt.Errorf("elasticsearch returned error: %s", res.Status())
+ }
+ return nil
+}
+
+// InsertMetadata inserts documents into tenant's metadata index
+func (e *elasticsearchEngine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) {
+ indexName := buildMetadataIndexName(tenantID)
+ common.Info("Inserting metadata into Elasticsearch index", zap.String("index_name", indexName), zap.String("tenant_id", tenantID), zap.Int("doc_count", len(metadata)))
+
+ if len(metadata) == 0 {
+ return []string{}, nil
+ }
+
+ if indexName == "" {
+ return nil, fmt.Errorf("index name cannot be empty")
+ }
+
+ // Check if index exists, create if not
+ exists, err := e.indexExists(ctx, indexName)
+ if err != nil {
+ common.Error("Failed to check index existence", err)
+ return nil, fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if !exists {
+ // Create metadata index
+ if createErr := e.CreateMetadataStore(ctx, tenantID); createErr != nil {
+ return nil, fmt.Errorf("failed to create metadata index: %w", createErr)
+ }
+ }
+
+ // Build bulk request body
+ var buf bytes.Buffer
+ for _, doc := range metadata {
+ // Action line - index operation
+ action := map[string]interface{}{
+ "index": map[string]interface{}{
+ "_index": indexName,
+ },
+ }
+ actionBytes, err := json.Marshal(action)
+ if err != nil {
+ common.Error("Failed to marshal bulk action", err)
+ return nil, fmt.Errorf("failed to marshal bulk action: %w", err)
+ }
+ buf.Write(actionBytes)
+ buf.WriteByte('\n')
+
+ // Document line - meta_fields is stored as-is (ES can handle nested objects)
+ docBytes, err := json.Marshal(doc)
+ if err != nil {
+ common.Error("Failed to marshal document", err)
+ return nil, fmt.Errorf("failed to marshal document: %w", err)
+ }
+ buf.Write(docBytes)
+ buf.WriteByte('\n')
+ }
+
+ // Execute bulk request
+ req := esapi.BulkRequest{
+ Body: bytes.NewReader(buf.Bytes()),
+ Refresh: "false",
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ common.Error("Failed to execute bulk request", err)
+ return nil, fmt.Errorf("failed to execute bulk request: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ common.Sugar.Errorw("Elasticsearch bulk request returned error", "status", res.Status())
+ return nil, fmt.Errorf("elasticsearch bulk request returned error: %s", res.Status())
+ }
+
+ // Parse bulk response to check for errors
+ var bulkResponse map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&bulkResponse); err != nil {
+ common.Error("Failed to parse bulk response", err)
+ return nil, fmt.Errorf("failed to parse bulk response: %w", err)
+ }
+
+ // Check for errors in bulk response
+ if errors, ok := bulkResponse["errors"].(bool); ok && errors {
+ common.Warn("Bulk request had some errors")
+ }
+
+ common.Info("Successfully inserted metadata into Elasticsearch index", zap.String("index_name", indexName), zap.Int("doc_count", len(metadata)))
+ return []string{}, nil
+}
+
+// UpdateMetadata updates document metadata in tenant's metadata index
+func (e *elasticsearchEngine) UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error {
+ indexName := buildMetadataIndexName(tenantID)
+ common.Info("Updating metadata in Elasticsearch index", zap.String("index_name", indexName), zap.String("docID", docID), zap.String("datasetID", datasetID))
+
+ // Check if index exists
+ exists, err := e.indexExists(ctx, indexName)
+ if err != nil {
+ return fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if !exists {
+ return fmt.Errorf("index '%s' does not exist", indexName)
+ }
+
+ // Build the document ID for update
+ docID = strings.ReplaceAll(docID, "'", "''")
+ datasetIDStr := strings.ReplaceAll(datasetID, "'", "''")
+
+ // Build update body - merge meta_fields with existing
+ query := map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must": []map[string]interface{}{
+ {"term": map[string]interface{}{"id": docID}},
+ {"term": map[string]interface{}{"kb_id": datasetIDStr}},
+ },
+ },
+ }
+
+ updateReq := map[string]interface{}{
+ "query": query,
+ "script": map[string]interface{}{
+ "source": "ctx._source.meta_fields = params.meta_fields",
+ "params": map[string]interface{}{
+ "meta_fields": metaFields,
+ },
+ },
+ }
+
+ updateBytes, err := json.Marshal(updateReq)
+ if err != nil {
+ return fmt.Errorf("failed to marshal update request: %w", err)
+ }
+
+ req := esapi.UpdateByQueryRequest{
+ Index: []string{indexName},
+ Body: bytes.NewReader(updateBytes),
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ common.Error("Failed to execute update by query", err)
+ return fmt.Errorf("failed to execute update by query: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ common.Sugar.Errorw("Elasticsearch update by query returned error", "status", res.Status())
+ return fmt.Errorf("elasticsearch update by query returned error: %s", res.Status())
+ }
+
+ common.Info("Successfully updated metadata in Elasticsearch index", zap.String("index_name", indexName), zap.String("docID", docID))
+ return nil
+}
+
+// DeleteMetadata deletes metadata from tenant's metadata index by condition
+func (e *elasticsearchEngine) DeleteMetadata(ctx context.Context, condition map[string]interface{}, tenantID string) (int64, error) {
+ indexName := buildMetadataIndexName(tenantID)
+ common.Info("Deleting metadata from Elasticsearch index", zap.String("index_name", indexName), zap.Any("condition", condition))
+
+ // Check if index exists
+ exists, err := e.indexExists(ctx, indexName)
+ if err != nil {
+ return 0, fmt.Errorf("failed to check index existence: %w", err)
+ }
+ if !exists {
+ common.Warn(fmt.Sprintf("Index %s does not exist, skipping delete", indexName))
+ return 0, nil
+ }
+
+ // Build query from condition
+ query := e.buildMetadataQueryFromCondition(condition)
+ if query == nil {
+ query = map[string]interface{}{"match_all": map[string]interface{}{}}
+ }
+
+ // Build delete by query body
+ deleteBody := map[string]interface{}{
+ "query": query,
+ }
+
+ bodyBytes, err := json.Marshal(deleteBody)
+ if err != nil {
+ return 0, fmt.Errorf("failed to marshal delete body: %w", err)
+ }
+
+ // Execute delete by query
+ req := esapi.DeleteByQueryRequest{
+ Index: []string{indexName},
+ Body: bytes.NewReader(bodyBytes),
+ }
+
+ res, err := req.Do(ctx, e.client)
+ if err != nil {
+ common.Error("Failed to execute delete by query", err)
+ return 0, fmt.Errorf("failed to execute delete by query: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ common.Sugar.Errorw("Elasticsearch delete by query returned error", "status", res.Status())
+ return 0, fmt.Errorf("elasticsearch delete by query returned error: %s", res.Status())
+ }
+
+ // Parse response
+ var result map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+ common.Error("Failed to parse delete response", err)
+ return 0, fmt.Errorf("failed to parse delete response: %w", err)
+ }
+
+ deleted := int64(0)
+ if d, ok := result["deleted"].(float64); ok {
+ deleted = int64(d)
+ }
+
+ common.Info("Successfully deleted metadata", zap.String("index_name", indexName), zap.Int64("deleted_count", deleted))
+ return deleted, nil
+}
+
+// DropMetadataStore drops a metadata index from Elasticsearch
+func (e *elasticsearchEngine) DropMetadataStore(ctx context.Context, tenantID string) error {
+ indexName := buildMetadataIndexName(tenantID)
+ return e.dropIndex(ctx, indexName)
+}
+
+// MetadataStoreExists checks if a metadata index exists in Elasticsearch
+func (e *elasticsearchEngine) MetadataStoreExists(ctx context.Context, tenantID string) (bool, error) {
+ indexName := buildMetadataIndexName(tenantID)
+ return e.indexExists(ctx, indexName)
+}
\ No newline at end of file
diff --git a/internal/engine/elasticsearch/search.go b/internal/engine/elasticsearch/search.go
deleted file mode 100644
index b3c68fbc11b..00000000000
--- a/internal/engine/elasticsearch/search.go
+++ /dev/null
@@ -1,583 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package elasticsearch
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "ragflow/internal/common"
- "strings"
-
- "github.com/elastic/go-elasticsearch/v8/esapi"
- "go.uber.org/zap"
-
- "ragflow/internal/engine/types"
-)
-
-// SearchResponse Elasticsearch search response
-type SearchResponse struct {
- Hits struct {
- Total struct {
- Value int64 `json:"value"`
- } `json:"total"`
- Hits []struct {
- ID string `json:"_id"`
- Score float64 `json:"_score"`
- Source map[string]interface{} `json:"_source"`
- } `json:"hits"`
- } `json:"hits"`
- Aggregations map[string]interface{} `json:"aggregations"`
-}
-
-// Search executes search with unified types.SearchRequest
-func (e *elasticsearchEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- return e.searchUnified(ctx, req)
-}
-
-// searchUnified handles the unified types.SearchRequest
-func (e *elasticsearchEngine) searchUnified(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- if len(req.IndexNames) == 0 {
- return nil, fmt.Errorf("index names cannot be empty")
- }
-
- // Build pagination parameters
- offset := req.Offset
- limit := req.Limit
- if limit <= 0 {
- limit = 30 // default ES size
- }
-
- // Check if this is a skill index
- isSkillIndex := len(req.IndexNames) > 0 && strings.HasPrefix(req.IndexNames[0], "skill_")
-
- // Build filter clauses
- var filterClauses []map[string]interface{}
- if isSkillIndex {
- filterClauses = buildSkillFilterClauses()
- } else {
- filterClauses = buildFilterClauses(req.KbIDs, 1)
- }
-
- // Add filters from req.Filter
- if req.Filter != nil && len(req.Filter) > 0 {
- filterClauses = append(filterClauses, buildFilterFromMap(req.Filter)...)
- }
-
- // Build search query body
- queryBody := make(map[string]interface{})
-
- // Determine search type from MatchExprs
- var matchText string
- var matchDense *types.MatchDenseExpr
- var hasVectorMatch bool
-
- for _, expr := range req.MatchExprs {
- if expr == nil {
- continue
- }
- switch e := expr.(type) {
- case string:
- matchText = e
- case *types.MatchTextExpr:
- matchText = e.MatchingText
- case *types.MatchDenseExpr:
- hasVectorMatch = true
- matchDense = e
- }
- }
-
- var vectorFieldName string
- if !hasVectorMatch || matchDense == nil {
- // Keyword-only search
- if isSkillIndex {
- queryBody["query"] = buildSkillKeywordQuery(matchText, filterClauses, 1.0)
- } else {
- queryBody["query"] = buildESKeywordQuery(matchText, filterClauses, 1.0)
- }
- } else {
- // Hybrid search: keyword + vector
- textWeight := 0.7 // default: vector weight = 0.3
- vectorWeight := 0.3
- if matchDense.ExtraOptions != nil {
- if vw, ok := matchDense.ExtraOptions["text_weight"].(float64); ok {
- textWeight = vw
- }
- if vw, ok := matchDense.ExtraOptions["vector_weight"].(float64); ok {
- vectorWeight = vw
- }
- }
-
- // Build boolean query for text match and filters
- var boolQuery map[string]interface{}
- if isSkillIndex {
- boolQuery = buildSkillKeywordQuery(matchText, filterClauses, 1.0)
- } else {
- boolQuery = buildESKeywordQuery(matchText, filterClauses, 1.0)
- }
- // Add boost to the bool query (as in Python code)
- if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
- boolMap["boost"] = textWeight
- }
-
- // Build kNN query
- vectorData := matchDense.EmbeddingData
- vectorFieldName = matchDense.VectorColumnName
- k := matchDense.TopN
- if k <= 0 {
- k = req.Limit
- }
- if k <= 0 {
- k = 1024
- }
- numCandidates := k * 2
-
- similarity := 0.0
- if matchDense.ExtraOptions != nil {
- if sim, ok := matchDense.ExtraOptions["similarity"].(float64); ok {
- similarity = sim
- }
- }
-
- knnQuery := map[string]interface{}{
- "field": vectorFieldName,
- "query_vector": vectorData,
- "k": k,
- "num_candidates": numCandidates,
- "similarity": similarity,
- "boost": vectorWeight,
- }
-
- queryBody["knn"] = knnQuery
- queryBody["query"] = boolQuery
-
- // Add vector column to Source fields (matching Python ES: src.append(f"q_{len(q_vec)}_vec"))
- // Only modify Source if it was explicitly set by the caller
- if vectorFieldName != "" && len(req.SelectFields) > 0 {
- sourceFields := req.SelectFields
- found := false
- for _, f := range sourceFields {
- if f == vectorFieldName {
- found = true
- break
- }
- }
- if !found {
- sourceFields = append(sourceFields, vectorFieldName)
- }
- req.SelectFields = sourceFields
- }
- }
-
- queryBody["size"] = limit
- queryBody["from"] = offset
-
- // Add sorting if specified
- if req.OrderBy != nil {
- sort := parseOrderByExpr(req.OrderBy)
- if len(sort) > 0 {
- queryBody["sort"] = sort
- }
- }
-
- // Serialize query
- var buf bytes.Buffer
- if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
- return nil, fmt.Errorf("error encoding query: %w", err)
- }
-
- // Log search details
- common.Debug("Elasticsearch searching indices", zap.Strings("indices", req.IndexNames))
- common.Debug("Elasticsearch DSL", zap.Any("dsl", queryBody))
-
- // Build search request
- reqES := esapi.SearchRequest{
- Index: req.IndexNames,
- Body: &buf,
- }
-
- // Execute search
- res, err := reqES.Do(ctx, e.client)
- if err != nil {
- return nil, fmt.Errorf("search failed: %w", err)
- }
- defer res.Body.Close()
-
- if res.IsError() {
- bodyBytes, err := io.ReadAll(res.Body)
- if err != nil {
- common.Error("Elasticsearch failed to read error response body", err)
- } else {
- common.Warn("Elasticsearch error response", zap.String("body", string(bodyBytes)))
- }
- return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status())
- }
-
- // Parse response
- var esResp SearchResponse
- if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
- return nil, fmt.Errorf("error parsing response: %w", err)
- }
-
- // Convert to unified response
- chunks := convertESResponse(&esResp, vectorFieldName)
- return &types.SearchResult{
- Chunks: chunks,
- Total: esResp.Hits.Total.Value,
- }, nil
-}
-
-// calculatePagination calculates offset and limit based on page, size and topK
-func calculatePagination(page, size, topK int) (int, int) {
- if page < 1 {
- page = 1
- }
- if size <= 0 {
- size = 30
- }
- if topK <= 0 {
- topK = 1024
- }
-
- RERANK_LIMIT := max(30, (64/size)*size)
- if RERANK_LIMIT < size {
- RERANK_LIMIT = size
- }
- if RERANK_LIMIT > topK {
- RERANK_LIMIT = topK
- }
-
- offset := (page - 1) * RERANK_LIMIT
- if offset < 0 {
- offset = 0
- }
-
- return offset, RERANK_LIMIT
-}
-
-// buildFilterClauses builds ES filter clauses from kb_ids and available_int
-// Reference: rag/utils/es_conn.py L60-L78
-// When available=0: available_int < 1
-// When available!=0: NOT (available_int < 1)
-func buildFilterClauses(kbIDs []string, available int) []map[string]interface{} {
- var filters []map[string]interface{}
-
- if len(kbIDs) > 0 {
- filters = append(filters, map[string]interface{}{
- "terms": map[string]interface{}{"kb_id": kbIDs},
- })
- }
-
- // Add available_int filter
- // Reference: rag/utils/es_conn.py L63-L68
- if available == 0 {
- // available_int < 1
- filters = append(filters, map[string]interface{}{
- "range": map[string]interface{}{
- "available_int": map[string]interface{}{
- "lt": 1,
- },
- },
- })
- } else {
- // must_not: available_int < 1 (i.e., available_int >= 1)
- filters = append(filters, map[string]interface{}{
- "bool": map[string]interface{}{
- "must_not": []map[string]interface{}{
- {
- "range": map[string]interface{}{
- "available_int": map[string]interface{}{
- "lt": 1,
- },
- },
- },
- },
- },
- })
- }
-
- return filters
-}
-
-// buildSkillFilterClauses builds ES filter clauses for skill index
-// Skill index uses 'status' field instead of 'available_int'
-func buildSkillFilterClauses() []map[string]interface{} {
- // Filter for active skills (status = "1")
- return []map[string]interface{}{
- {
- "term": map[string]interface{}{
- "status": "1",
- },
- },
- }
-}
-
-// buildFilterFromMap converts a generic filter map to ES filter clauses
-func buildFilterFromMap(filter map[string]interface{}) []map[string]interface{} {
- var filters []map[string]interface{}
- for field, value := range filter {
- switch v := value.(type) {
- case []string:
- filters = append(filters, map[string]interface{}{
- "terms": map[string]interface{}{field: v},
- })
- case []interface{}:
- filters = append(filters, map[string]interface{}{
- "terms": map[string]interface{}{field: v},
- })
- default:
- filters = append(filters, map[string]interface{}{
- "term": map[string]interface{}{field: v},
- })
- }
- }
- return filters
-}
-
-// buildESKeywordQuery builds keyword-only search query for ES
-// Uses query_string if matchText is in query_string format, otherwise uses multi_match
-// boost is applied to the text match clause (query_string or multi_match)
-func buildESKeywordQuery(matchText string, filterClauses []map[string]interface{}, boost float64) map[string]interface{} {
- var mustClause map[string]interface{}
-
- // Handle wildcard query (match all)
- if matchText == "*" || matchText == "" {
- mustClause = map[string]interface{}{
- "match_all": map[string]interface{}{},
- }
- } else {
- // Use query_string for complex queries
- queryString := map[string]interface{}{
- "query": matchText,
- "fields": []string{"title_tks^10", "title_sm_tks^5", "important_kwd^30", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"},
- "type": "best_fields",
- "minimum_should_match": "30%",
- "boost": boost,
- }
- mustClause = map[string]interface{}{
- "query_string": queryString,
- }
- }
-
- return map[string]interface{}{
- "bool": map[string]interface{}{
- "must": mustClause,
- "filter": filterClauses,
- },
- }
-}
-
-// buildSkillKeywordQuery builds keyword-only search query for skill index
-// Skill index uses different field names: name_tks, tags_tks, description_tks, content_tks
-func buildSkillKeywordQuery(matchText string, filterClauses []map[string]interface{}, boost float64) map[string]interface{} {
- var mustClause map[string]interface{}
-
- // Handle wildcard query (match all)
- if matchText == "*" || matchText == "" {
- mustClause = map[string]interface{}{
- "match_all": map[string]interface{}{},
- }
- } else {
- // Use query_string for complex queries with skill-specific fields
- queryString := map[string]interface{}{
- "query": matchText,
- "fields": []string{"name_tks^10", "tags_tks^5", "description_tks^3", "content_tks^1"},
- "type": "best_fields",
- "minimum_should_match": "30%",
- "boost": boost,
- }
- mustClause = map[string]interface{}{
- "query_string": queryString,
- }
- }
-
- return map[string]interface{}{
- "bool": map[string]interface{}{
- "must": mustClause,
- "filter": filterClauses,
- },
- }
-}
-
-// convertESResponse converts ES SearchResponse to unified chunks format
-func convertESResponse(esResp *SearchResponse, vectorFieldName string) []map[string]interface{} {
- if esResp == nil || esResp.Hits.Hits == nil {
- return []map[string]interface{}{}
- }
-
- chunks := make([]map[string]interface{}, len(esResp.Hits.Hits))
- for i, hit := range esResp.Hits.Hits {
- chunks[i] = hit.Source
- chunks[i]["_score"] = hit.Score
- chunks[i]["_id"] = hit.ID
- }
- return chunks
-}
-
-// parseOrderByExpr parses the OrderBy expression into ES sort format
-func parseOrderByExpr(orderBy *types.OrderByExpr) []map[string]interface{} {
- if orderBy == nil || len(orderBy.Fields) == 0 {
- return nil
- }
-
- var result []map[string]interface{}
- for _, field := range orderBy.Fields {
- direction := "asc"
- if field.Type == types.SortDesc {
- direction = "desc"
- }
-
- if field.Field == "_score" || field.Field == "score" {
- result = append(result, map[string]interface{}{
- "_score": direction,
- })
- } else {
- result = append(result, map[string]interface{}{
- field.Field: direction,
- })
- }
- }
-
- return result
-}
-
-// Helper query builder functions (legacy)
-
-// BuildMatchTextQuery builds a text match query
-func BuildMatchTextQuery(fields []string, text string, fuzziness string) map[string]interface{} {
- query := map[string]interface{}{
- "multi_match": map[string]interface{}{
- "query": text,
- "fields": fields,
- },
- }
-
- if fuzziness != "" {
- if multiMatch, ok := query["multi_match"].(map[string]interface{}); ok {
- multiMatch["fuzziness"] = fuzziness
- }
- }
-
- return query
-}
-
-// BuildTermQuery builds a term query
-func BuildTermQuery(field string, value interface{}) map[string]interface{} {
- return map[string]interface{}{
- "term": map[string]interface{}{
- field: value,
- },
- }
-}
-
-// BuildRangeQuery builds a range query
-func BuildRangeQuery(field string, from, to interface{}) map[string]interface{} {
- rangeQuery := make(map[string]interface{})
- if from != nil {
- rangeQuery["gte"] = from
- }
- if to != nil {
- rangeQuery["lte"] = to
- }
-
- return map[string]interface{}{
- "range": map[string]interface{}{
- field: rangeQuery,
- },
- }
-}
-
-// BuildBoolQuery builds a bool query
-func BuildBoolQuery() map[string]interface{} {
- return map[string]interface{}{
- "bool": make(map[string]interface{}),
- }
-}
-
-// AddMust adds must clause to bool query
-func AddMust(query map[string]interface{}, clauses ...map[string]interface{}) {
- if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
- if _, exists := boolQuery["must"]; !exists {
- boolQuery["must"] = []map[string]interface{}{}
- }
- if must, ok := boolQuery["must"].([]map[string]interface{}); ok {
- boolQuery["must"] = append(must, clauses...)
- }
- }
-}
-
-// AddShould adds should clause to bool query
-func AddShould(query map[string]interface{}, clauses ...map[string]interface{}) {
- if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
- if _, exists := boolQuery["should"]; !exists {
- boolQuery["should"] = []map[string]interface{}{}
- }
- if should, ok := boolQuery["should"].([]map[string]interface{}); ok {
- boolQuery["should"] = append(should, clauses...)
- }
- }
-}
-
-// AddFilter adds filter clause to bool query
-func AddFilter(query map[string]interface{}, clauses ...map[string]interface{}) {
- if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
- if _, exists := boolQuery["filter"]; !exists {
- boolQuery["filter"] = []map[string]interface{}{}
- }
- if filter, ok := boolQuery["filter"].([]map[string]interface{}); ok {
- boolQuery["filter"] = append(filter, clauses...)
- }
- }
-}
-
-// AddMustNot adds must_not clause to bool query
-func AddMustNot(query map[string]interface{}, clauses ...map[string]interface{}) {
- if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
- if _, exists := boolQuery["must_not"]; !exists {
- boolQuery["must_not"] = []map[string]interface{}{}
- }
- if mustNot, ok := boolQuery["must_not"].([]map[string]interface{}); ok {
- boolQuery["must_not"] = append(mustNot, clauses...)
- }
- }
-}
-
-// GetFields is not implemented for Elasticsearch
-func (e *elasticsearchEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
- common.Warn("GetFields not implemented for Elasticsearch")
- return nil
-}
-
-// GetAggregation is not implemented for Elasticsearch
-func (e *elasticsearchEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {
- common.Warn("GetAggregation not implemented for Elasticsearch")
- return nil
-}
-
-// GetHighlight is not implemented for Elasticsearch
-func (e *elasticsearchEngine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {
- common.Warn("GetHighlight not implemented for Elasticsearch")
- return nil
-}
-
-// GetDocIDs is not implemented for Elasticsearch
-func (e *elasticsearchEngine) GetDocIDs(chunks []map[string]interface{}) []string {
- common.Warn("GetDocIDs not implemented for Elasticsearch")
- return nil
-}
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index 19112d0dd46..a37b5beaf14 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -32,26 +32,23 @@ const (
// DocEngine document storage engine interface
type DocEngine interface {
- // Search
- Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error)
-
- // Dataset operations
- CreateDataset(ctx context.Context, indexName, datasetID string, vectorSize int, parserID string) error
- InsertDataset(ctx context.Context, documents []map[string]interface{}, indexName string, knowledgebaseID string) ([]string, error)
- UpdateDataset(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, tableNamePrefix string, knowledgebaseID string) error
-
// Chunk operations
- GetChunk(ctx context.Context, indexName, chunkID string, kbIDs []string) (interface{}, error)
+ CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error
+ InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)
+ UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error
+ DeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error)
+ Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error)
+ GetChunk(ctx context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error)
+ DropChunkStore(ctx context.Context, baseName, datasetID string) error
+ ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error)
// Document metadata operations
- CreateMetadata(ctx context.Context, indexName string) error
- InsertMetadata(ctx context.Context, documents []map[string]interface{}, tenantID string) ([]string, error)
- UpdateMetadata(ctx context.Context, docID string, kbID string, metaFields map[string]interface{}, tenantID string) error
-
- // Operations for both dataset and metadata tables
- Delete(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error)
- DropTable(ctx context.Context, indexName string) error
- TableExists(ctx context.Context, indexName string) (bool, error)
+ CreateMetadataStore(ctx context.Context, tenantID string) error
+ InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error)
+ UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error
+ DeleteMetadata(ctx context.Context, condition map[string]interface{}, tenantID string) (int64, error)
+ DropMetadataStore(ctx context.Context, tenantID string) error
+ MetadataStoreExists(ctx context.Context, tenantID string) (bool, error)
// Document operations (used by skill indexing)
IndexDocument(ctx context.Context, indexName, docID string, doc interface{}) error
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
new file mode 100644
index 00000000000..2532fef5c5c
--- /dev/null
+++ b/internal/engine/infinity/chunk.go
@@ -0,0 +1,2038 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package infinity
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "ragflow/internal/common"
+ "ragflow/internal/engine/types"
+ "ragflow/internal/utility"
+ "regexp"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "unicode"
+
+ infinity "github.com/infiniflow/infinity-go-sdk"
+ "go.uber.org/zap"
+)
+
+// CreateChunkStore creates a chunk table in Infinity
+// baseName is the table name prefix (e.g., "ragflow_")
+// The full table name is built as "{baseName}_{datasetID}"
+// For skill index (datasetID="skill"), tableName is just baseName and uses skill_infinity_mapping.json
+func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+ vecSize := vectorSize
+
+ // Determine table name and mapping file based on index type
+ var tableName string
+ var mappingFile string
+
+ tableName = buildChunkTableName(baseName, datasetID)
+ if datasetID == "skill" {
+ mappingFile = "skill_infinity_mapping.json"
+ common.Info("Creating skill index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
+ } else {
+ mappingFile = e.mappingFileName
+ common.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
+ }
+
+ // Use configured schema
+ fpMapping := filepath.Join(utility.GetProjectRoot(), "conf", mappingFile)
+
+ schemaData, err := os.ReadFile(fpMapping)
+ if err != nil {
+ return fmt.Errorf("Failed to read mapping file: %w", err)
+ }
+
+ var schema orderedFields
+ if err := json.Unmarshal(schemaData, &schema); err != nil {
+ return fmt.Errorf("Failed to parse mapping file: %w", err)
+ }
+
+ // Get database
+ db, err := e.client.conn.GetDatabase(e.client.dbName)
+ if err != nil {
+ return fmt.Errorf("Failed to get database: %w", err)
+ }
+
+ // Determine vector column name
+ vectorColName := fmt.Sprintf("q_%d_vec", vecSize)
+
+ // Check if table already exists
+ exists, err := e.tableExists(ctx, tableName)
+ if err != nil {
+ return fmt.Errorf("Failed to check if table exists: %w", err)
+ }
+
+ var table *infinity.Table
+ if exists {
+ // Table exists, open it and check if vector column needs to be added
+ common.Info("Table already exists, checking for vector column", zap.String("tableName", tableName))
+ table, err = db.GetTable(tableName)
+ if err != nil {
+ return fmt.Errorf("Failed to open existing table %s: %w", tableName, err)
+ }
+
+ // Check if vector column exists (for embedding model changes)
+ colExists, err := e.columnExists(table, vectorColName)
+ if err != nil {
+ common.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err))
+ }
+
+ // Add new vector column if it doesn't exist (handles embedding model change)
+ if !colExists {
+ common.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize))
+ addColSchema := infinity.TableSchema{
+ &infinity.ColumnDefinition{
+ Name: vectorColName,
+ DataType: fmt.Sprintf("vector,%d,float", vecSize),
+ },
+ }
+ if _, err := table.AddColumns(addColSchema); err != nil {
+ common.Error("Failed to add vector column "+vectorColName, err)
+ return fmt.Errorf("Failed to add vector column %s: %w", vectorColName, err)
+ }
+ common.Info("Successfully added vector column", zap.String("column", vectorColName))
+ }
+ } else {
+ // Table doesn't exist, create it with vector column in the initial schema
+ common.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize))
+
+ // Build column definitions (preserving JSON order)
+ var columns infinity.TableSchema
+ for _, fieldName := range schema.Keys {
+ fieldInfo := schema.Fields[fieldName]
+ col := infinity.ColumnDefinition{
+ Name: fieldName,
+ DataType: fieldInfo.Type,
+ Default: fieldInfo.Default,
+ }
+ columns = append(columns, &col)
+ }
+
+ // Add vector column
+ columns = append(columns, &infinity.ColumnDefinition{
+ Name: vectorColName,
+ DataType: fmt.Sprintf("vector,%d,float", vecSize),
+ })
+
+ // Add chunk_data column for table parser
+ if parserID == "table" {
+ columns = append(columns, &infinity.ColumnDefinition{
+ Name: "chunk_data",
+ DataType: "json",
+ Default: "{}",
+ })
+ }
+
+ // Create table
+ table, err = db.CreateTable(tableName, columns, infinity.ConflictTypeIgnore)
+ if err != nil {
+ return fmt.Errorf("Failed to create table: %w", err)
+ }
+ common.Debug("Infinity created table", zap.String("tableName", tableName))
+ }
+
+ // Create HNSW index on vector column with unique name based on vector size
+ // Use unique index name to avoid conflict when embedding model changes
+ vectorIndexName := fmt.Sprintf("q_%d_vec_idx", vecSize)
+ _, err = table.CreateIndex(
+ vectorIndexName,
+ infinity.NewIndexInfo(vectorColName, infinity.IndexTypeHnsw, map[string]string{
+ "M": "16",
+ "ef_construction": "50",
+ "metric": "cosine",
+ "encode": "lvq",
+ }),
+ infinity.ConflictTypeIgnore,
+ "",
+ )
+ if err != nil {
+ return fmt.Errorf("Failed to create HNSW index %s: %w", vectorIndexName, err)
+ }
+ common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))
+
+ // Create full-text indexes for varchar fields with analyzers
+ for _, fieldName := range schema.Keys {
+ fieldInfo := schema.Fields[fieldName]
+ if fieldInfo.Type != "varchar" || fieldInfo.Analyzer == nil {
+ continue
+ }
+
+ analyzers := []string{}
+ switch a := fieldInfo.Analyzer.(type) {
+ case string:
+ analyzers = []string{a}
+ case []interface{}:
+ for _, v := range a {
+ if s, ok := v.(string); ok {
+ analyzers = append(analyzers, s)
+ }
+ }
+ }
+
+ for _, analyzer := range analyzers {
+ indexNameFt := fmt.Sprintf("ft_%s_%s",
+ regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(fieldName, "_"),
+ regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(analyzer, "_"),
+ )
+ _, err = table.CreateIndex(
+ indexNameFt,
+ infinity.NewIndexInfo(fieldName, infinity.IndexTypeFullText, map[string]string{"ANALYZER": analyzer}),
+ infinity.ConflictTypeIgnore,
+ "",
+ )
+ if err != nil {
+ return fmt.Errorf("Failed to create fulltext index %s: %w", indexNameFt, err)
+ }
+ }
+ }
+
+ // Create secondary indexes for fields with index_type
+ for _, fieldName := range schema.Keys {
+ fieldInfo := schema.Fields[fieldName]
+ if fieldInfo.IndexType == nil {
+ continue
+ }
+
+ indexTypeStr := ""
+ params := map[string]string{}
+
+ switch it := fieldInfo.IndexType.(type) {
+ case string:
+ indexTypeStr = it
+ case map[string]interface{}:
+ if t, ok := it["type"].(string); ok {
+ indexTypeStr = t
+ }
+ if card, ok := it["cardinality"].(string); ok {
+ params["cardinality"] = card
+ }
+ }
+
+ if indexTypeStr == "secondary" {
+ indexNameSec := fmt.Sprintf("sec_%s", fieldName)
+ _, err = table.CreateIndex(
+ indexNameSec,
+ infinity.NewIndexInfo(fieldName, infinity.IndexTypeSecondary, params),
+ infinity.ConflictTypeIgnore,
+ "",
+ )
+ if err != nil {
+ return fmt.Errorf("Failed to create secondary index %s: %w", indexNameSec, err)
+ }
+ }
+ }
+
+ return nil
+}
+
+// InsertChunks inserts documents into a dataset table
+// Table name format: {baseName}_{datasetID}
+// Auto-create the table if it doesn't exist
+// Delete existing rows with matching IDs before insert
+func (e *infinityEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
+ tableName := buildChunkTableName(baseName, datasetID)
+ common.Info("InfinityConnection.InsertChunks called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks)))
+
+ db, err := e.client.conn.GetDatabase(e.client.dbName)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to get database: %w", err)
+ }
+
+ table, err := db.GetTable(tableName)
+ if err != nil {
+ // Table doesn't exist, try to create it
+ errMsg := strings.ToLower(err.Error())
+ if !strings.Contains(errMsg, "not found") && !strings.Contains(errMsg, "doesn't exist") {
+ return nil, fmt.Errorf("Failed to get table %s: %w", tableName, err)
+ }
+
+ // Infer vector size from chunks
+ vectorSize := 0
+ vectorPattern := regexp.MustCompile(`q_(\d+)_vec`)
+ for _, chunk := range chunks {
+ for key := range chunk {
+ matches := vectorPattern.FindStringSubmatch(key)
+ if len(matches) >= 2 {
+ vectorSize, _ = strconv.Atoi(matches[1])
+ break
+ }
+ }
+ if vectorSize > 0 {
+ break
+ }
+ }
+ if vectorSize == 0 {
+ return nil, fmt.Errorf("cannot infer vector size from chunks")
+ }
+
+ // Determine parser_id from chunk structure
+ parserID := ""
+ if chunkData, ok := chunks[0]["chunk_data"].(map[string]interface{}); ok && chunkData != nil {
+ parserID = "table"
+ }
+
+ // Create table
+ if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, parserID); err != nil {
+ return nil, fmt.Errorf("Failed to create table: %w", err)
+ }
+
+ table, err = db.GetTable(tableName)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to get table after creation: %w", err)
+ }
+ }
+
+ // Get embedding columns and their sizes
+ var embeddingCols [][2]interface{}
+ colsResp, err := table.ShowColumns()
+ if err != nil {
+ return nil, fmt.Errorf("Failed to get columns: %w", err)
+ }
+ result, ok := colsResp.(*infinity.QueryResult)
+ if !ok {
+ return nil, fmt.Errorf("unexpected response type: %T", colsResp)
+ }
+
+ // ShowColumns returns a result set where Data contains arrays of column values
+ re := regexp.MustCompile(`Embedding\([a-z]+,(\d+)\)`)
+ if nameArr, ok := result.Data["name"]; ok {
+ if typeArr, ok := result.Data["type"]; ok {
+ for i := 0; i < len(nameArr); i++ {
+ colName, _ := nameArr[i].(string)
+ colType, _ := typeArr[i].(string)
+ matches := re.FindStringSubmatch(colType)
+ if len(matches) >= 2 {
+ size, _ := strconv.Atoi(matches[1])
+ embeddingCols = append(embeddingCols, [2]interface{}{colName, size})
+ }
+ }
+ }
+ }
+
+ // Transform chunks using helper function
+ insertChunks := make([]map[string]interface{}, len(chunks))
+ for i, chunk := range chunks {
+ insertChunks[i] = transformChunkFields(chunk, embeddingCols)
+ }
+
+ // Delete existing rows with matching IDs
+ if len(insertChunks) > 0 {
+ idList := make([]string, len(insertChunks))
+ for i, chunk := range insertChunks {
+ idList[i] = fmt.Sprintf("'%v'", chunk["id"])
+ }
+ filter := fmt.Sprintf("id IN (%s)", strings.Join(idList, ", "))
+ common.Debug(fmt.Sprintf("Deleting existing rows with filter: %s", filter))
+ delResp, delErr := table.Delete(filter)
+ if delErr != nil {
+ common.Warn(fmt.Sprintf("Failed to delete existing rows: %v", delErr))
+ } else {
+ common.Info(fmt.Sprintf("Deleted %d existing rows", delResp.DeletedRows))
+ }
+ }
+
+ // Insert chunks to dataset
+ _, err = table.Insert(insertChunks)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to insert chunks to dataset: %w", err)
+ }
+
+ common.Info("InfinityConnection.InsertChunks result", zap.String("tableName", tableName), zap.Int("count", len(insertChunks)))
+ return []string{}, nil
+}
+
+// UpdateChunks updates chunks in a dataset table
+// Table name format: {baseName}_{datasetID}
+func (e *infinityEngine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error {
+ tableName := buildChunkTableName(baseName, datasetID)
+ common.Info("InfinityConnection.UpdateChunks called", zap.String("tableName", tableName), zap.Any("condition", condition))
+
+ db, err := e.client.conn.GetDatabase(e.client.dbName)
+ if err != nil {
+ return fmt.Errorf("Failed to get database: %w", err)
+ }
+
+ table, err := db.GetTable(tableName)
+ if err != nil {
+ return fmt.Errorf("Failed to get table %s: %w", tableName, err)
+ }
+
+ // Get table columns
+ clmns := make(map[string]struct {
+ Type string
+ Default interface{}
+ })
+ colsResp, err := table.ShowColumns()
+ if err != nil {
+ return fmt.Errorf("Failed to get columns: %w", err)
+ }
+ result, ok := colsResp.(*infinity.QueryResult)
+ if ok {
+ if nameArr, ok := result.Data["name"]; ok {
+ if typeArr, ok := result.Data["type"]; ok {
+ if defArr, ok := result.Data["default"]; ok {
+ for i := 0; i < len(nameArr); i++ {
+ colName, _ := nameArr[i].(string)
+ colType, _ := typeArr[i].(string)
+ var colDefault interface{}
+ if i < len(defArr) {
+ colDefault = defArr[i]
+ }
+ clmns[colName] = struct {
+ Type string
+ Default interface{}
+ }{colType, colDefault}
+ }
+ }
+ }
+ }
+ }
+
+ // Build filter string from condition
+ filter := buildFilterFromCondition(condition, clmns)
+
+ // Process remove operation first
+ removeValue := make(map[string]interface{})
+ if removeData, ok := newValue["remove"].(map[string]interface{}); ok {
+ removeValue = removeData
+ }
+ delete(newValue, "remove")
+
+ // Transform new_value fields using helper function (no embeddings needed for update)
+ transformed := transformChunkFields(newValue, nil)
+ for k, v := range transformed {
+ newValue[k] = v
+ }
+
+ // Remove original fields that were transformed (they're now in transformed with new names/types)
+ // Also remove intermediate token fields that shouldn't be stored in Infinity
+ // This must match Python's delete list in infinity_conn.py
+ for _, key := range []string{"docnm_kwd", "title_tks", "title_sm_tks", "important_kwd", "important_tks",
+ "content_with_weight", "content_ltks", "content_sm_ltks", "authors_tks", "authors_sm_tks",
+ "question_kwd", "question_tks"} {
+ delete(newValue, key)
+ }
+
+ // Handle remove operations if any
+ if len(removeValue) > 0 {
+ colToRemove := make([]string, 0, len(removeValue))
+ for k := range removeValue {
+ colToRemove = append(colToRemove, k)
+ }
+ colToRemove = append(colToRemove, "id")
+
+ // Query rows to be updated
+ queryResult, err := table.Output(colToRemove).Filter(filter).ToResult()
+ if err != nil {
+ common.Warn(fmt.Sprintf("Failed to query rows for remove operation: %v", err))
+ } else {
+ qr, ok := queryResult.(*infinity.QueryResult)
+ if ok && len(qr.Data) > 0 {
+ // Get the id column and columns to remove
+ idCol := qr.Data["id"]
+ removeOpt := make(map[string]map[string][]string) // column -> value -> [ids]
+
+ for colName, colData := range qr.Data {
+ if colName == "id" {
+ continue
+ }
+ removeVal := removeValue[colName]
+ for i, id := range idCol {
+ if i < len(colData) {
+ existingVal := colData[i]
+ if removeStr, ok := removeVal.(string); ok {
+ // Split existing value by ### and remove the target value
+ if existingStr, ok := existingVal.(string); ok {
+ parts := strings.Split(existingStr, "###")
+ var newParts []string
+ for _, p := range parts {
+ if p != removeStr {
+ newParts = append(newParts, p)
+ }
+ }
+ if len(newParts) != len(parts) {
+ idStr := fmt.Sprintf("'%s'", escapeFilterValue(fmt.Sprintf("%v", id)))
+ if removeOpt[colName] == nil {
+ removeOpt[colName] = make(map[string][]string)
+ }
+ removeOpt[colName][strings.Join(newParts, "###")] = append(removeOpt[colName][strings.Join(newParts, "###")], idStr)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Execute remove updates
+ for colName, valueToIDs := range removeOpt {
+ for newVal, ids := range valueToIDs {
+ idFilter := filter + " AND id IN (" + strings.Join(ids, ", ") + ")"
+ common.Info(fmt.Sprintf("INFINITY remove update: table=%s, idFilter=%s, column=%s, newValue=%v", tableName, idFilter, colName, newVal))
+ _, err := table.Update(idFilter, map[string]interface{}{colName: newVal})
+ if err != nil {
+ common.Warn(fmt.Sprintf("Failed to remove value from column %s: %v", colName, err))
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Execute the main update
+ common.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue))
+ _, err = table.Update(filter, newValue)
+ if err != nil {
+ return fmt.Errorf("Failed to update chunks: %w", err)
+ }
+
+ common.Info("InfinityConnection.UpdateChunks completes", zap.String("tableName", tableName))
+ return nil
+}
+
+// DeleteChunks deletes chunks from a dataset table
+// Table name format: {baseName}_{datasetID}
+// condition specifies which chunks to delete
+func (e *infinityEngine) DeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error) {
+ tableName := buildChunkTableName(baseName, datasetID)
+
+ db, err := e.client.conn.GetDatabase(e.client.dbName)
+ if err != nil {
+ return 0, fmt.Errorf("failed to get database: %w", err)
+ }
+
+ table, err := db.GetTable(tableName)
+ if err != nil {
+ common.Warn(fmt.Sprintf("Table %s does not exist, skipping delete", tableName))
+ return 0, nil
+ }
+
+ // Get table columns for building filter
+ clmns := make(map[string]struct {
+ Type string
+ Default interface{}
+ })
+ colsResp, err := table.ShowColumns()
+ if err != nil {
+ return 0, fmt.Errorf("failed to get columns: %w", err)
+ }
+ result, ok := colsResp.(*infinity.QueryResult)
+ if ok {
+ if nameArr, ok := result.Data["name"]; ok {
+ if typeArr, ok := result.Data["type"]; ok {
+ if defArr, ok := result.Data["default"]; ok {
+ for i := 0; i < len(nameArr); i++ {
+ colName, _ := nameArr[i].(string)
+ colType, _ := typeArr[i].(string)
+ var colDefault interface{}
+ if i < len(defArr) {
+ colDefault = defArr[i]
+ }
+ clmns[colName] = struct {
+ Type string
+ Default interface{}
+ }{colType, colDefault}
+ }
+ }
+ }
+ }
+ }
+
+ // Build filter from condition
+ filter := buildFilterFromCondition(condition, clmns)
+
+ delResp, err := table.Delete(filter)
+ if err != nil {
+ return 0, fmt.Errorf("failed to delete: %w", err)
+ }
+
+ return delResp.DeletedRows, nil
+}
+
+// Search searches the Infinity engine for matching chunks.
+// It supports three matching types: MatchTextExpr (full-text), MatchDenseExpr (vector), and FusionExpr (combined).
+// If no match expressions are provided, Search relies solely on filter (e.g., doc_id, available_int) to find results.
+func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ common.Debug("Search in Infinity started", zap.Any("indexNames", req.IndexNames))
+ if common.IsDebugEnabled() {
+ // Format match expressions for logging
+ var matchExprsStr string
+ for i, expr := range req.MatchExprs {
+ switch e := expr.(type) {
+ case *types.MatchTextExpr:
+ matchExprsStr += fmt.Sprintf(" [%d] MatchTextExpr: fields=%v, matchingText=%s, topN=%d, extraOptions=%v\n", i, e.Fields, e.MatchingText, e.TopN, e.ExtraOptions)
+ case *types.MatchDenseExpr:
+ matchExprsStr += fmt.Sprintf(" [%d] MatchDenseExpr: vectorColumn=%s, vectorSize=%d, topN=%d, extraOptions=%v\n", i, e.VectorColumnName, len(e.EmbeddingData), e.TopN, e.ExtraOptions)
+ case *types.FusionExpr:
+ matchExprsStr += fmt.Sprintf(" [%d] FusionExpr: method=%s, topN=%d, fusionParams=%v\n", i, e.Method, e.TopN, e.FusionParams)
+ default:
+ matchExprsStr += fmt.Sprintf(" [%d] unknown type\n", i)
+ }
+ }
+ common.Debug(fmt.Sprintf("Search request:\n"+
+ " indexNames=%v\n"+
+ " KbIDs=%v\n"+
+ " offset=%d, limit=%d\n"+
+ " SelectFields=%v\n"+
+ " Filter=%v\n"+
+ " MatchExprs:\n%s orderBy=%v\n"+
+ " RankFeature=%v",
+ req.IndexNames, req.KbIDs, req.Offset, req.Limit, req.SelectFields, req.Filter, matchExprsStr, req.OrderBy, req.RankFeature))
+ }
+
+ if len(req.IndexNames) == 0 {
+ return nil, fmt.Errorf("index names cannot be empty")
+ }
+
+ // Get retrieval parameters with defaults
+ pageSize := req.Limit
+ if pageSize <= 0 {
+ pageSize = 30
+ }
+
+ offset := req.Offset
+ if offset < 0 {
+ offset = 0
+ }
+
+ db, err := e.client.conn.GetDatabase(e.client.dbName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get database: %w", err)
+ }
+
+ isMetadataTable := false
+ isSkillIndex := false
+ for _, idx := range req.IndexNames {
+ if strings.HasPrefix(idx, "ragflow_doc_meta_") {
+ isMetadataTable = true
+ break
+ }
+ if strings.HasPrefix(idx, "skill_") {
+ isSkillIndex = true
+ break
+ }
+ }
+
+ var outputColumns []string
+ if isMetadataTable {
+ outputColumns = []string{"id", "kb_id", "meta_fields"}
+ } else if isSkillIndex {
+ outputColumns = []string{
+ "skill_id", "space_id", "folder_id", "name", "tags", "description", "content",
+ "version", "status", "create_time", "update_time",
+ }
+ outputColumns = convertSelectFields(outputColumns, true)
+ } else {
+ outputColumns = []string{
+ "id", "doc_id", "kb_id", "content_ltks", "content_with_weight",
+ "title_tks", "docnm_kwd", "img_id", "available_int", "important_kwd",
+ "position_int", "page_num_int", "top_int", "chunk_order_int",
+ "create_timestamp_flt", "knowledge_graph_kwd", "question_kwd", "question_tks",
+ "doc_type_kwd", "mom_id", "tag_kwd", "pagerank_fea", "tag_feas",
+ }
+ outputColumns = convertSelectFields(outputColumns)
+ }
+
+ hasTextMatch := false
+ hasVectorMatch := false
+ var matchText *types.MatchTextExpr
+ var matchDense *types.MatchDenseExpr
+ if req.MatchExprs != nil && len(req.MatchExprs) > 0 {
+ for _, expr := range req.MatchExprs {
+ if expr == nil {
+ continue
+ }
+ switch e := expr.(type) {
+ case string:
+ if e != "" {
+ hasTextMatch = true
+ matchText = &types.MatchTextExpr{
+ MatchingText: e,
+ TopN: pageSize,
+ }
+ }
+ case *types.MatchTextExpr:
+ if e.MatchingText != "" {
+ hasTextMatch = true
+ matchText = e
+ }
+ case *types.MatchDenseExpr:
+ if len(e.EmbeddingData) > 0 {
+ hasVectorMatch = true
+ matchDense = e
+ }
+ }
+ }
+ }
+
+ if hasTextMatch || hasVectorMatch {
+ if hasTextMatch {
+ outputColumns = append(outputColumns, "score()")
+ }
+ // similarity() is only allowed by Infinity when there is ONLY MATCH VECTOR.
+ // When both text and vector matches exist (hybrid search with Fusion),
+ // only score() is valid — Fusion produces a unified SCORE column.
+ if hasVectorMatch && !hasTextMatch {
+ outputColumns = append(outputColumns, "similarity()")
+ }
+ // Skill index does not have pagerank_fea and tag_feas columns
+ if !isSkillIndex {
+ if !slices.Contains(outputColumns, common.PAGERANK_FLD) {
+ outputColumns = append(outputColumns, common.PAGERANK_FLD)
+ }
+ if !slices.Contains(outputColumns, common.TAG_FLD) {
+ outputColumns = append(outputColumns, common.TAG_FLD)
+ }
+ }
+ }
+
+ if !slices.Contains(outputColumns, "row_id") && !slices.Contains(outputColumns, "row_id()") {
+ outputColumns = append(outputColumns, "row_id()")
+ }
+
+ outputColumns = convertSelectFields(outputColumns, isSkillIndex)
+ if hasVectorMatch && matchDense != nil && matchDense.VectorColumnName != "" {
+ outputColumns = append(outputColumns, matchDense.VectorColumnName)
+ }
+
+ var filterParts []string
+ if isMetadataTable && len(req.KbIDs) > 0 && req.KbIDs[0] != "" {
+ kbIDs := req.KbIDs
+ if len(kbIDs) == 1 {
+ filterParts = append(filterParts, fmt.Sprintf("kb_id = '%s'", kbIDs[0]))
+ } else {
+ kbIDStr := strings.Join(kbIDs, "', '")
+ filterParts = append(filterParts, fmt.Sprintf("kb_id IN ('%s')", kbIDStr))
+ }
+ }
+
+ if !isMetadataTable && (hasTextMatch || hasVectorMatch) {
+ if req.Filter != nil {
+ if availInt, ok := req.Filter["available_int"]; ok {
+ filterParts = append(filterParts, fmt.Sprintf("available_int=%v", availInt))
+ } else if status, ok := req.Filter["status"]; ok {
+ filterParts = append(filterParts, fmt.Sprintf("status='%s'", status))
+ } else {
+ if isSkillIndex {
+ filterParts = append(filterParts, "status='1'")
+ } else {
+ filterParts = append(filterParts, "available_int=1")
+ }
+ }
+ } else {
+ if isSkillIndex {
+ filterParts = append(filterParts, "status='1'")
+ } else {
+ filterParts = append(filterParts, "available_int=1")
+ }
+ }
+ }
+
+ // Build filter string from req.Filter
+ if req.Filter != nil {
+ filterCopy := req.Filter
+ if !isMetadataTable {
+ filterCopy = make(map[string]interface{})
+ for k, v := range req.Filter {
+ if k != "kb_id" {
+ filterCopy[k] = v
+ }
+ }
+ }
+
+ condStr := equivalentConditionToStr(filterCopy)
+ if condStr != "" {
+ filterParts = append(filterParts, condStr)
+ }
+ }
+ filterStr := strings.Join(filterParts, " AND ")
+
+ orderBy := req.OrderBy
+ var rankFeature map[string]float64
+ if req.RankFeature != nil {
+ rankFeature = req.RankFeature
+ }
+
+ var fusionExpr *types.FusionExpr
+ if len(req.MatchExprs) > 2 {
+ if fe, ok := req.MatchExprs[2].(*types.FusionExpr); ok {
+ fusionExpr = fe
+ }
+ }
+
+ var allResults []map[string]interface{}
+ totalHits := int64(0)
+
+ for _, indexName := range req.IndexNames {
+ var tableNames []string
+ if strings.HasPrefix(indexName, "ragflow_doc_meta_") {
+ tableNames = []string{indexName}
+ } else {
+ kbIDs := req.KbIDs
+ if len(kbIDs) == 0 {
+ kbIDs = []string{""}
+ }
+ for _, kbID := range kbIDs {
+ if kbID == "" {
+ tableNames = append(tableNames, indexName)
+ } else {
+ tableNames = append(tableNames, fmt.Sprintf("%s_%s", indexName, kbID))
+ }
+ }
+ }
+
+ minMatch := 0.3
+
+ var questionText string
+ var vectorData []float64
+ textTopN := pageSize
+ var originalQuery string
+ if matchText != nil {
+ questionText = matchText.MatchingText
+ textTopN = int(matchText.TopN)
+ if matchText.ExtraOptions != nil {
+ if oq, ok := matchText.ExtraOptions["original_query"].(string); ok {
+ originalQuery = oq
+ }
+ }
+ }
+ if matchDense != nil {
+ vectorData = matchDense.EmbeddingData
+ }
+
+ for _, tableName := range tableNames {
+ tbl, err := db.GetTable(tableName)
+ if err != nil {
+ continue
+ }
+ table := tbl.Output(outputColumns)
+
+ var textFields []string
+ if matchText != nil && len(matchText.Fields) > 0 {
+ textFields = matchText.Fields
+ } else if isSkillIndex {
+ textFields = []string{
+ "name^10",
+ "tags^5",
+ "description^3",
+ "content^1",
+ }
+ } else {
+ textFields = []string{
+ "title_tks^10",
+ "title_sm_tks^5",
+ "important_kwd^30",
+ "important_tks^20",
+ "question_tks^20",
+ "content_ltks^2",
+ "content_sm_ltks",
+ }
+ }
+
+ // Convert field names for Infinity
+ var convertedFields []string
+ for _, f := range textFields {
+ cf := convertMatchingField(f)
+ convertedFields = append(convertedFields, cf)
+ }
+ fields := strings.Join(convertedFields, ",")
+
+ hasTextMatch := questionText != ""
+ hasVectorMatch := len(vectorData) > 0
+ // Add text match if question is provided
+ if hasTextMatch {
+ extraOptions := map[string]string{
+ "minimum_should_match": fmt.Sprintf("%d%%", int(minMatch*100)),
+ }
+
+ if filterStr != "" {
+ extraOptions["filter"] = filterStr
+ }
+
+ if rankFeature != nil {
+ var rankFeaturesList []string
+ for featureName, weight := range rankFeature {
+ rankFeaturesList = append(rankFeaturesList, fmt.Sprintf("%s^%s^%.0f", common.TAG_FLD, featureName, weight))
+ }
+ if len(rankFeaturesList) > 0 {
+ extraOptions["rank_features"] = strings.Join(rankFeaturesList, ",")
+ }
+ }
+
+ if originalQuery != "" {
+ extraOptions["original_query"] = originalQuery
+ }
+
+ table = table.MatchText(fields, questionText, textTopN, extraOptions)
+
+ common.Debug(fmt.Sprintf(
+ "MatchTextExpr:\n"+
+ " fields=%s\n"+
+ " matching_text=%s\n"+
+ " topn=%d\n"+
+ " extra_options=%v",
+ fields, questionText, textTopN, extraOptions,
+ ))
+ }
+
+ // Add vector match if provided
+ if hasVectorMatch {
+ vecFieldName := fmt.Sprintf("q_%d_vec", len(vectorData))
+ dataType := "float"
+ distanceType := "cosine"
+
+ if matchDense != nil {
+ if matchDense.VectorColumnName != "" {
+ vecFieldName = matchDense.VectorColumnName
+ }
+ if matchDense.EmbeddingDataType != "" {
+ dataType = matchDense.EmbeddingDataType
+ }
+ if matchDense.DistanceType != "" {
+ distanceType = matchDense.DistanceType
+ }
+ }
+
+ vectorTopN := pageSize
+ if matchDense != nil && matchDense.TopN > 0 {
+ vectorTopN = int(matchDense.TopN)
+ }
+
+ denseFilterStr := filterStr
+ if denseFilterStr == "" {
+ if isSkillIndex {
+ denseFilterStr = "status='1'"
+ } else {
+ denseFilterStr = "available_int=1"
+ }
+ }
+
+ if hasTextMatch && fusionExpr == nil {
+ fieldsStr := strings.Join(convertedFields, ",")
+ filterFulltext := fmt.Sprintf("filter_fulltext('%s', '%s')", fieldsStr, questionText)
+ denseFilterStr = fmt.Sprintf("(%s) AND %s", denseFilterStr, filterFulltext)
+ }
+ extraOptions := map[string]string{
+ "threshold": utility.FloatToString(0.0),
+ "filter": denseFilterStr,
+ }
+
+ common.Debug("MatchDense for hybrid search",
+ zap.String("fieldName", vecFieldName),
+ zap.String("distanceType", distanceType),
+ zap.Int("topN", vectorTopN),
+ zap.Bool("hasFusion", fusionExpr != nil))
+
+ table = table.MatchDense(vecFieldName, vectorData, dataType, distanceType, vectorTopN, extraOptions)
+ }
+
+ // Add fusion (for text + vector combination)
+ if hasTextMatch && hasVectorMatch && fusionExpr != nil {
+ fusionMethod := fusionExpr.Method
+ fusionTopK := fusionExpr.TopN
+ if fusionTopK == 0 {
+ fusionTopK = pageSize
+ }
+ fusionParams := map[string]interface{}{
+ "normalize": "atan",
+ }
+ if fusionExpr.FusionParams != nil {
+ for k, v := range fusionExpr.FusionParams {
+ fusionParams[k] = v
+ }
+ }
+
+ common.Debug("Applying Fusion for hybrid search",
+ zap.String("method", fusionMethod),
+ zap.Int("topN", fusionTopK),
+ zap.Any("params", fusionParams))
+
+ table = table.Fusion(fusionMethod, fusionTopK, fusionParams)
+ }
+
+ // Add order_by if provided
+ if orderBy != nil && len(orderBy.Fields) > 0 {
+ var sortFields [][2]interface{}
+ for _, orderField := range orderBy.Fields {
+ sortType := infinity.SortTypeAsc
+ if orderField.Type == types.SortDesc {
+ sortType = infinity.SortTypeDesc
+ }
+ sortFields = append(sortFields, [2]interface{}{orderField.Field, sortType})
+ }
+ table = table.Sort(sortFields)
+ }
+
+ // Add filter when there's no text/vector match (like metadata queries)
+ if !hasTextMatch && !hasVectorMatch && filterStr != "" {
+ common.Debug(fmt.Sprintf("Adding filter for no-match query: %s", filterStr))
+ table = table.Filter(filterStr)
+ }
+
+ // Set limit and offset
+ table = table.Limit(pageSize)
+ if offset > 0 {
+ table = table.Offset(offset)
+ }
+
+ // Request total_hits_count from Infinity
+ table = table.Option(map[string]interface{}{"total_hits_count": true})
+
+ // Execute query
+ df, err := table.ToDataFrame()
+ if err != nil {
+ common.Warn("Infinity query failed",
+ zap.String("tableName", tableName),
+ zap.Bool("hasTextMatch", hasTextMatch),
+ zap.Bool("hasVectorMatch", hasVectorMatch),
+ zap.Bool("hasFusion", fusionExpr != nil),
+ zap.Error(err))
+ continue
+ }
+
+ // Convert DataFrame to chunks format (column-oriented to row-oriented)
+ searchChunks := make([]map[string]interface{}, 0)
+ for colName, colData := range df.ColumnData {
+ for i, val := range colData {
+ for len(searchChunks) <= i {
+ searchChunks = append(searchChunks, make(map[string]interface{}))
+ }
+ searchChunks[i][colName] = val
+ }
+ }
+
+ // Apply field name mapping and row_id handling
+ // Skill index uses different schema
+ // so we skip the document-specific field mappings
+ if !isSkillIndex {
+ GetFields(searchChunks, nil)
+ } else {
+ // For skill index, only handle ROW_ID -> row_id() mapping
+ for _, chunk := range searchChunks {
+ if val, ok := chunk["ROW_ID"]; ok {
+ chunk["row_id()"] = val
+ delete(chunk, "ROW_ID")
+ }
+ }
+ }
+
+ // Parse total_hits_count from ExtraInfo
+ var tableTotal int64
+ if df.ExtraInfo != "" {
+ var extraResult map[string]interface{}
+ if err := json.Unmarshal([]byte(df.ExtraInfo), &extraResult); err == nil {
+ if count, ok := extraResult["total_hits_count"].(float64); ok {
+ tableTotal = int64(count)
+ }
+ }
+ }
+
+ searchResult := &types.SearchResult{
+ Chunks: searchChunks,
+ Total: tableTotal,
+ }
+
+ allResults = append(allResults, searchResult.Chunks...)
+ totalHits += searchResult.Total
+ }
+ }
+
+ if hasTextMatch || hasVectorMatch {
+ scoreColumn := ""
+ if hasTextMatch && hasVectorMatch {
+ scoreColumn = "SCORE"
+ } else if hasTextMatch {
+ scoreColumn = "SCORE"
+ } else if hasVectorMatch {
+ scoreColumn = "SIMILARITY"
+ }
+ pagerankField := common.PAGERANK_FLD
+ if isSkillIndex {
+ pagerankField = "" // Skill index has no pagerank field
+ }
+
+ allResults = calculateScores(allResults, scoreColumn, pagerankField)
+ allResults = sortByScore(allResults, len(allResults))
+ }
+
+ if len(allResults) > pageSize {
+ allResults = allResults[:pageSize]
+ }
+
+ common.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))
+
+ return &types.SearchResult{
+ Chunks: allResults,
+ Total: totalHits,
+ }, nil
+}
+
+// GetChunk gets a chunk by ID
+func (e *infinityEngine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) {
+ if e.client == nil || e.client.conn == nil {
+ return nil, fmt.Errorf("Infinity client not initialized")
+ }
+
+ // Build list of table names to search
+ var tableNames []string
+ if strings.HasPrefix(tableName, "ragflow_doc_meta_") {
+ tableNames = []string{tableName}
+ } else {
+ // Search in tables like _ for each datasetID
+ if len(datasetIDs) > 0 {
+ for _, datasetID := range datasetIDs {
+ tableNames = append(tableNames, fmt.Sprintf("%s_%s", tableName, datasetID))
+ }
+ }
+ // Also try the base tableName
+ tableNames = append(tableNames, tableName)
+ }
+
+ // Try each table and collect results from all tables
+ db, err := e.client.conn.GetDatabase(e.client.dbName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get database: %w", err)
+ }
+
+ // Collect chunks from all tables (same as Python's concat_dataframes)
+ allChunks := make(map[string]map[string]interface{})
+
+ for _, tblName := range tableNames {
+ table, err := db.GetTable(tblName)
+ if err != nil {
+ continue
+ }
+
+ // Query with filter for the specific chunk ID
+ filter := fmt.Sprintf("id = '%s'", chunkID)
+ result, err := table.Output([]string{"*"}).Filter(filter).ToResult()
+ if err != nil {
+ continue
+ }
+
+ qr, ok := result.(*infinity.QueryResult)
+ if !ok {
+ continue
+ }
+
+ if len(qr.Data) == 0 {
+ continue
+ }
+
+ // Convert to chunk format
+ chunks := make([]map[string]interface{}, 0)
+ for colName, colData := range qr.Data {
+ for i, val := range colData {
+ for len(chunks) <= i {
+ chunks = append(chunks, make(map[string]interface{}))
+ }
+ chunks[i][colName] = val
+ }
+ }
+
+ // Merge chunks into allChunks (by id), keeping first non-empty value
+ for _, chunk := range chunks {
+ if idVal, ok := chunk["id"].(string); ok {
+ if existing, exists := allChunks[idVal]; exists {
+ // Merge: keep first non-empty value for each field
+ for k, v := range chunk {
+ if _, has := existing[k]; !has || (utility.IsEmpty(existing[k]) && !utility.IsEmpty(v)) {
+ existing[k] = v
+ }
+ }
+ } else {
+ allChunks[idVal] = chunk
+ }
+ }
+ }
+ }
+
+ // Get the chunk by chunkID
+ chunk, found := allChunks[chunkID]
+ if !found {
+ return nil, nil
+ }
+
+ common.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames))
+
+ // Apply field mappings (same as in GetFields)
+ // docnm -> docnm_kwd, title_tks, title_sm_tks
+ if val, ok := chunk["docnm"].(string); ok {
+ chunk["docnm_kwd"] = val
+ chunk["title_tks"] = val
+ chunk["title_sm_tks"] = val
+ }
+
+ // content -> content_with_weight, content_ltks, content_sm_ltks
+ if val, ok := chunk["content"].(string); ok {
+ chunk["content_with_weight"] = val
+ chunk["content_ltks"] = val
+ chunk["content_sm_ltks"] = val
+ }
+
+ // important_keywords -> important_kwd (split by comma), important_tks
+ if val, ok := chunk["important_keywords"].(string); ok {
+ if val == "" {
+ chunk["important_kwd"] = []interface{}{}
+ } else {
+ parts := strings.Split(val, ",")
+ chunk["important_kwd"] = parts
+ }
+ chunk["important_tks"] = val
+ } else {
+ chunk["important_kwd"] = []interface{}{}
+ chunk["important_tks"] = []interface{}{}
+ }
+
+ // questions -> question_kwd (split by newline), question_tks
+ if val, ok := chunk["questions"].(string); ok {
+ if val == "" {
+ chunk["question_kwd"] = []interface{}{}
+ } else {
+ parts := strings.Split(val, "\n")
+ chunk["question_kwd"] = parts
+ }
+ chunk["question_tks"] = val
+ } else {
+ chunk["question_kwd"] = []interface{}{}
+ chunk["question_tks"] = []interface{}{}
+ }
+
+ if posVal, ok := chunk["position_int"].(string); ok {
+ chunk["position_int"] = utility.ConvertHexToPositionIntArray(posVal)
+ } else {
+ chunk["position_int"] = []interface{}{}
+ }
+
+ return chunk, nil
+}
+
+// GetFields applies field mappings to chunks and returns a dict keyed by chunk ID.
+// Equivalent to Python's get_fields() in infinity_conn.py.
+// When fields is nil/empty, returns all fields from chunks.
+func GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
+ result := make(map[string]map[string]interface{})
+ if len(chunks) == 0 {
+ return result
+ }
+
+ // If fields is provided, create a set for lookup
+ fieldSet := make(map[string]bool)
+ for _, f := range fields {
+ fieldSet[f] = true
+ }
+
+ for _, chunk := range chunks {
+ // Apply field mappings
+ // docnm -> docnm_kwd, title_tks, title_sm_tks
+ if val, ok := chunk["docnm"].(string); ok {
+ chunk["docnm_kwd"] = val
+ chunk["title_tks"] = val
+ chunk["title_sm_tks"] = val
+ }
+
+ // important_keywords -> important_kwd (split by comma), important_tks
+ if val, ok := chunk["important_keywords"].(string); ok {
+ if val == "" {
+ chunk["important_kwd"] = []interface{}{}
+ } else {
+ parts := strings.Split(val, ",")
+ chunk["important_kwd"] = parts
+ }
+ chunk["important_tks"] = val
+ } else {
+ chunk["important_kwd"] = []interface{}{}
+ chunk["important_tks"] = []interface{}{}
+ }
+
+ // questions -> question_kwd (split by newline), question_tks
+ if val, ok := chunk["questions"].(string); ok {
+ if val == "" {
+ chunk["question_kwd"] = []interface{}{}
+ } else {
+ parts := strings.Split(val, "\n")
+ chunk["question_kwd"] = parts
+ }
+ chunk["question_tks"] = val
+ } else {
+ chunk["question_kwd"] = []interface{}{}
+ chunk["question_tks"] = []interface{}{}
+ }
+
+ // content -> content_with_weight, content_ltks, content_sm_ltks
+ if val, ok := chunk["content"].(string); ok {
+ chunk["content_with_weight"] = val
+ chunk["content_ltks"] = val
+ chunk["content_sm_ltks"] = val
+ }
+
+ // authors -> authors_tks, authors_sm_tks
+ if val, ok := chunk["authors"].(string); ok {
+ chunk["authors_tks"] = val
+ chunk["authors_sm_tks"] = val
+ }
+
+ // position_int: convert from hex string to array format (grouped by 5)
+ if val, ok := chunk["position_int"].(string); ok {
+ chunk["position_int"] = utility.ConvertHexToPositionIntArray(val)
+ }
+
+ // Convert page_num_int and top_int from hex string to array
+ for _, colName := range []string{"page_num_int", "top_int"} {
+ if val, ok := chunk[colName].(string); ok && val != "" {
+ chunk[colName] = utility.ConvertHexToIntArray(val)
+ }
+ }
+
+ // Post-process: convert nil/empty values to empty slices for array-like fields
+ // and split _kwd fields by "###" (except knowledge_graph_kwd, docnm_kwd, important_kwd, question_kwd)
+ kwdNoSplit := map[string]bool{
+ "knowledge_graph_kwd": true, "docnm_kwd": true,
+ "important_kwd": true, "question_kwd": true,
+ }
+ arrayFields := []string{
+ "doc_type_kwd", "important_kwd", "important_tks", "question_tks",
+ "question_kwd", "authors_tks", "authors_sm_tks", "title_tks",
+ "title_sm_tks", "content_ltks", "content_sm_ltks", "tag_kwd",
+ }
+ for _, colName := range arrayFields {
+ val, ok := chunk[colName]
+ if !ok || val == nil || val == "" {
+ chunk[colName] = []interface{}{}
+ } else if !kwdNoSplit[colName] {
+ // Split by "###" for _kwd fields
+ if strVal, ok := val.(string); ok && strings.Contains(strVal, "###") {
+ parts := strings.Split(strVal, "###")
+ var filtered []interface{}
+ for _, p := range parts {
+ if p != "" {
+ filtered = append(filtered, p)
+ }
+ }
+ chunk[colName] = filtered
+ }
+ }
+ }
+
+ // Handle row_id mapping - Infinity returns "ROW_ID" but we use "row_id()"
+ if val, ok := chunk["ROW_ID"]; ok {
+ chunk["row_id()"] = val
+ delete(chunk, "ROW_ID")
+ }
+
+ // Build result map keyed by id
+ if id, ok := chunk["id"].(string); ok {
+ fieldMap := make(map[string]interface{})
+ for field, value := range chunk {
+ if len(fieldSet) == 0 || fieldSet[field] {
+ fieldMap[field] = value
+ }
+ }
+ result[id] = fieldMap
+ }
+ }
+
+ return result
+}
+
+// GetFields is a method wrapper for infinityEngine to satisfy DocEngine interface
+func (e *infinityEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
+ return GetFields(chunks, fields)
+}
+
+// GetAggregation aggregates chunk values by field name.
+// Input: [{"docnm_kwd": "docA"}, {"docnm_kwd": "docA"}, {"docnm_kwd": "docB"}]
+//
+// GetAggregation(chunks, "docnm_kwd") returns:
+//
+// [{"key": "docA", "count": 2}, {"key": "docB", "count": 1}]
+//
+// For tag_kwd field, splits values by "###" separator.
+// For other fields, uses comma separation.
+func (e *infinityEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {
+ if len(chunks) == 0 {
+ return []map[string]interface{}{}
+ }
+
+ // Check if field exists in first chunk
+ hasField := false
+ for _, chunk := range chunks {
+ if _, ok := chunk[fieldName]; ok {
+ hasField = true
+ break
+ }
+ }
+ if !hasField {
+ return []map[string]interface{}{}
+ }
+
+ // Count occurrences
+ tagCounts := make(map[string]int)
+ for _, chunk := range chunks {
+ value, ok := chunk[fieldName]
+ if !ok || value == nil {
+ continue
+ }
+
+ // Handle string value
+ if valueStr, ok := value.(string); ok {
+ if valueStr == "" {
+ continue
+ }
+
+ var tags []string
+ // Split by "###" for tag_kwd field
+ if fieldName == "tag_kwd" && strings.Contains(valueStr, "###") {
+ for _, tag := range strings.Split(valueStr, "###") {
+ tag = strings.TrimSpace(tag)
+ if tag != "" {
+ tags = append(tags, tag)
+ }
+ }
+ } else {
+ // Fallback to comma separation
+ for _, tag := range strings.Split(valueStr, ",") {
+ tag = strings.TrimSpace(tag)
+ if tag != "" {
+ tags = append(tags, tag)
+ }
+ }
+ }
+
+ for _, tag := range tags {
+ tagCounts[tag]++
+ }
+ continue
+ }
+
+ // Handle list value
+ if valueList, ok := value.([]interface{}); ok {
+ for _, item := range valueList {
+ if itemStr, ok := item.(string); ok {
+ tag := strings.TrimSpace(itemStr)
+ if tag != "" {
+ tagCounts[tag]++
+ }
+ }
+ }
+ }
+ }
+
+ if len(tagCounts) == 0 {
+ return []map[string]interface{}{}
+ }
+
+ // Convert to slice and sort by count descending
+ type tagCountPair struct {
+ tag string
+ count int
+ }
+ pairs := make([]tagCountPair, 0, len(tagCounts))
+ for tag, count := range tagCounts {
+ pairs = append(pairs, tagCountPair{tag, count})
+ }
+ sort.Slice(pairs, func(i, j int) bool {
+ return pairs[i].count > pairs[j].count
+ })
+
+ // Convert to []map[string]interface{} directly
+ result := make([]map[string]interface{}, len(pairs))
+ for i, p := range pairs {
+ result[i] = map[string]interface{}{"key": p.tag, "count": p.count}
+ }
+
+ return result
+}
+
+// GetDocIDs extracts document IDs from search results.
+// Extracts "id" field from each chunk and returns as a list.
+func (e *infinityEngine) GetDocIDs(chunks []map[string]interface{}) []string {
+ if len(chunks) == 0 {
+ return nil
+ }
+ ids := make([]string, 0, len(chunks))
+ for _, chunk := range chunks {
+ if id, ok := chunk["id"].(string); ok {
+ ids = append(ids, id)
+ }
+ }
+ return ids
+}
+
+// GetHighlight generates highlighted text snippets for search results.
+// Matches keywords in text and wraps them with