diff --git a/api/core/workflow/nodes/knowledge_index/entities.py b/api/core/workflow/nodes/knowledge_index/entities.py index bfeb9b5b796a88..72b7a29985547e 100644 --- a/api/core/workflow/nodes/knowledge_index/entities.py +++ b/api/core/workflow/nodes/knowledge_index/entities.py @@ -150,6 +150,15 @@ class ParentChildStructureChunk(BaseModel): data_source_info: Union[FileInfo, OnlineDocumentInfo, WebsiteInfo] +class DocMetadata(BaseModel): + """ + Doc Metadata. + """ + + metadata_id: str + value: str | int | float | list[str] + + class KnowledgeIndexNodeData(BaseNodeData): """ Knowledge index Node Data. @@ -158,5 +167,6 @@ class KnowledgeIndexNodeData(BaseNodeData): type: str = "knowledge-index" chunk_structure: str index_chunk_variable_selector: list[str] + doc_metadata: list[DocMetadata] | None = None indexing_technique: str | None = None summary_index_setting: dict | None = None diff --git a/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py b/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py index 2aff953bc619bc..28d0aaf4e0406c 100644 --- a/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py +++ b/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py @@ -2,11 +2,12 @@ import datetime import logging import time -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any from flask import current_app from sqlalchemy import func, select +from sqlalchemy.orm import attributes from core.app.entities.app_invoke_entities import InvokeFrom from core.rag.index_processor.index_processor_factory import IndexProcessorFactory @@ -18,7 +19,14 @@ from core.workflow.nodes.base.template import Template from core.workflow.runtime import VariablePool from extensions.ext_database import db -from models.dataset import Dataset, Document, DocumentSegment, DocumentSegmentSummary +from models.dataset import ( + Dataset, + DatasetMetadata, + DatasetMetadataBinding, + Document, + DocumentSegment, + DocumentSegmentSummary, +) from services.summary_index_service import SummaryIndexService from tasks.generate_summary_index_task import generate_summary_index_task @@ -29,6 +37,9 @@ logger = logging.getLogger(__name__) +# Constant for built-in metadata identifier +BUILT_IN_METADATA_ID = "built-in" + default_retrieval_model = { "search_method": RetrievalMethod.SEMANTIC_SEARCH, "reranking_enable": False, @@ -192,6 +203,86 @@ def _invoke_knowledge_index( } ) + # Process doc_metadata before commit to ensure it's saved with the same document object + if node_data.doc_metadata: + try: + # Fetch metadata definitions for name mapping + metadata_name_map: dict[str, str] = {} + dataset_metadatas = db.session.scalars( + select(DatasetMetadata).where(DatasetMetadata.dataset_id == dataset.id) + ).all() + for md in dataset_metadatas: + metadata_name_map[md.id] = md.name + + # Collect valid metadata IDs (excluding built-in) + valid_metadata_ids = [ + item.metadata_id + for item in node_data.doc_metadata + if item.metadata_id != BUILT_IN_METADATA_ID and item.metadata_id in metadata_name_map + ] + + # Batch fetch existing bindings to avoid N+1 query + existing_binding_ids: set[str] = set() + if valid_metadata_ids: + existing_bindings = db.session.scalars( + select(DatasetMetadataBinding.metadata_id).where( + DatasetMetadataBinding.dataset_id == dataset.id, + DatasetMetadataBinding.document_id == doc_id_value, + DatasetMetadataBinding.metadata_id.in_(valid_metadata_ids), + ) + ).all() + existing_binding_ids = set(existing_bindings) + + doc_metadata_dict = document.doc_metadata or {} + + for item in node_data.doc_metadata: + # Skip built-in fields + if item.metadata_id == BUILT_IN_METADATA_ID: + continue + + # Resolve Name + md_name = metadata_name_map.get(item.metadata_id) + if not md_name: + logger.warning("[KnowledgeIndexNode] metadata_id %s not found, skipping", item.metadata_id) + continue + + # Resolve Value + value = item.value + if isinstance(value, list): + var_obj = variable_pool.get(value) + if var_obj: + value = var_obj.to_object() + else: + # Variable not found - raise error to notify user of configuration issue + variable_path = ".".join(value) + raise KnowledgeIndexNodeError( + f"Variable '{variable_path}' not found for metadata '{md_name}'. " + f"Please check your variable configuration." + ) + + if value is not None: + doc_metadata_dict[md_name] = value + + # Create DatasetMetadataBinding if not exists + if item.metadata_id not in existing_binding_ids: + binding = DatasetMetadataBinding( + tenant_id=dataset.tenant_id, + dataset_id=dataset.id, + metadata_id=item.metadata_id, + document_id=doc_id_value, + created_by=self.user_id, + ) + db.session.add(binding) + existing_binding_ids.add(item.metadata_id) # Prevent duplicate in same batch + + document.doc_metadata = doc_metadata_dict + # Force SQLAlchemy to recognize the change to the JSON field + attributes.flag_modified(document, "doc_metadata") + + except Exception as e: + logger.exception("[KnowledgeIndexNode] Failed to process doc_metadata") + raise KnowledgeIndexNodeError(f"Failed to process document metadata: {e}") from e + db.session.commit() # Generate summary index if enabled @@ -522,3 +613,28 @@ def get_streaming_template(self) -> Template: Template instance for this knowledge index node """ return Template(segments=[]) + + @classmethod + def _extract_variable_selector_to_variable_mapping( + cls, *, graph_config: Mapping[str, Any], node_id: str, node_data: Mapping[str, Any] + ) -> Mapping[str, Sequence[str]]: + """ + Extract variable selector to variable mapping + :param graph_config: graph config + :param node_id: node id + :param node_data: node data + :return: + """ + variable_mapping = {} + node_data_obj = KnowledgeIndexNodeData(**node_data) + + # index chunk variable + variable_mapping[node_id + ".index_chunk_variable_selector"] = node_data_obj.index_chunk_variable_selector + + # doc_metadata variables + if node_data_obj.doc_metadata: + for item in node_data_obj.doc_metadata: + if isinstance(item.value, list): + variable_mapping[node_id + "." + item.metadata_id] = item.value + + return variable_mapping diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 16945fca6a0cb1..be7ee168b60565 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -41,6 +41,8 @@ Dataset, DatasetAutoDisableLog, DatasetCollectionBinding, + DatasetMetadata, + DatasetMetadataBinding, DatasetPermission, DatasetPermissionEnum, DatasetProcessRule, @@ -1902,6 +1904,36 @@ def save_document_with_dataset_id( else default_retrieval_model ) + # Handle metadata configuration + # 1. Enable built-in metadata if requested + if knowledge_config.enable_built_in_metadata and not dataset.built_in_field_enabled: + dataset.built_in_field_enabled = True + db.session.add(dataset) + + # 2. Process custom metadata - validate and build dict + custom_metadata: dict = {} + metadata_bindings_to_create: list[tuple[str, str]] = [] # (metadata_id, metadata_name) + if knowledge_config.doc_metadata: + # Batch fetch all metadata definitions to avoid N+1 query + metadata_ids = [item.metadata_id for item in knowledge_config.doc_metadata] + metadata_defs = ( + db.session.query(DatasetMetadata) + .filter( + DatasetMetadata.id.in_(metadata_ids), + DatasetMetadata.dataset_id == dataset.id, + ) + .all() + ) + metadata_map = {md.id: md for md in metadata_defs} + + for item in knowledge_config.doc_metadata: + # Validate metadata_id belongs to this dataset + metadata_def = metadata_map.get(item.metadata_id) + if not metadata_def: + raise ValueError(f"Metadata with id '{item.metadata_id}' not found in this dataset") + custom_metadata[metadata_def.name] = item.value + metadata_bindings_to_create.append((item.metadata_id, metadata_def.name)) + documents = [] if knowledge_config.original_document_id: document = DocumentService.update_document_with_dataset_id(dataset, knowledge_config, account) @@ -2024,6 +2056,7 @@ def save_document_with_dataset_id( account, file.name, batch, + custom_metadata=custom_metadata or None, ) db.session.add(document) db.session.flush() @@ -2076,6 +2109,7 @@ def save_document_with_dataset_id( account, truncated_page_name, batch, + custom_metadata=custom_metadata or None, ) db.session.add(document) db.session.flush() @@ -2116,6 +2150,7 @@ def save_document_with_dataset_id( account, document_name, batch, + custom_metadata=custom_metadata or None, ) db.session.add(document) db.session.flush() @@ -2124,6 +2159,20 @@ def save_document_with_dataset_id( position += 1 db.session.commit() + # Create DatasetMetadataBinding records for custom metadata + if metadata_bindings_to_create and document_ids: + for doc_id in document_ids: + for metadata_id, _ in metadata_bindings_to_create: + binding = DatasetMetadataBinding( + tenant_id=dataset.tenant_id, + dataset_id=dataset.id, + document_id=doc_id, + metadata_id=metadata_id, + created_by=account.id, + ) + db.session.add(binding) + db.session.commit() + # trigger async task if document_ids: DocumentIndexingTaskProxy(dataset.tenant_id, dataset.id, document_ids).delay() @@ -2436,6 +2485,7 @@ def build_document( account: Account, name: str, batch: str, + custom_metadata: dict | None = None, ): # Set need_summary based on dataset's summary_index_setting need_summary = False @@ -2466,6 +2516,9 @@ def build_document( BuiltInField.last_update_date: datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"), BuiltInField.source: data_source_type, } + # Merge custom metadata if provided + if custom_metadata: + doc_metadata.update(custom_metadata) if doc_metadata: document.doc_metadata = doc_metadata return document diff --git a/api/services/entities/knowledge_entities/knowledge_entities.py b/api/services/entities/knowledge_entities/knowledge_entities.py index 8dc5b93501e263..3d4f90ee26c788 100644 --- a/api/services/entities/knowledge_entities/knowledge_entities.py +++ b/api/services/entities/knowledge_entities/knowledge_entities.py @@ -112,6 +112,11 @@ class MetaDataConfig(BaseModel): doc_metadata: dict +class DocumentMetadataInput(BaseModel): + metadata_id: str + value: str | int | float | None = None + + class KnowledgeConfig(BaseModel): original_document_id: str | None = None duplicate: bool = True @@ -126,6 +131,8 @@ class KnowledgeConfig(BaseModel): embedding_model_provider: str | None = None name: str | None = None is_multimodal: bool = False + enable_built_in_metadata: bool = False + doc_metadata: list[DocumentMetadataInput] | None = None class SegmentCreateArgs(BaseModel): diff --git a/api/services/metadata_service.py b/api/services/metadata_service.py index 3329ac349c3f5b..beb47f01042a35 100644 --- a/api/services/metadata_service.py +++ b/api/services/metadata_service.py @@ -1,12 +1,15 @@ import copy import logging +from sqlalchemy import or_ + from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource from extensions.ext_database import db from extensions.ext_redis import redis_client from libs.datetime_utils import naive_utc_now from libs.login import current_account_with_tenant -from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding +from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding, Pipeline +from models.workflow import Workflow from services.dataset_service import DocumentService from services.entities.knowledge_entities.knowledge_entities import ( MetadataArgs, @@ -95,6 +98,69 @@ def update_metadata_name(dataset_id: str, metadata_id: str, name: str) -> Datase finally: redis_client.delete(lock_key) + @staticmethod + def check_metadata_used_in_pipeline(dataset_id: str, metadata_id: str) -> tuple[bool, str | None]: + """ + Check if a metadata is used in the associated Pipeline's Knowledge Base node. + + Checks both draft and current published workflows to prevent deletion of metadata + that is actively used in production. + + Returns: + tuple[bool, str | None]: (is_used, pipeline_name) - True if used, with pipeline name + """ + # Get the dataset + dataset = db.session.query(Dataset).filter_by(id=dataset_id).first() + if not dataset or not dataset.pipeline_id: + return False, None + + # Get the pipeline to access workflow_id (current published version) + pipeline = db.session.query(Pipeline).filter_by(id=dataset.pipeline_id).first() + if not pipeline: + return False, None + + # Build conditions for draft and current published workflows only + draft_condition = (Workflow.app_id == pipeline.id) & (Workflow.version == Workflow.VERSION_DRAFT) + + if pipeline.workflow_id: + workflows = ( + db.session.query(Workflow) + .where( + or_( + draft_condition, + Workflow.id == pipeline.workflow_id, + ) + ) + .all() + ) + else: + workflows = db.session.query(Workflow).where(draft_condition).all() + + if not workflows: + return False, None + + # Check each workflow for metadata usage + for workflow in workflows: + try: + graph_dict = workflow.graph_dict + if "nodes" not in graph_dict: + continue + + for node in graph_dict["nodes"]: + node_data = node.get("data", {}) + # Check if this is a knowledge-index node + if node_data.get("type") == "knowledge-index": + doc_metadata = node_data.get("doc_metadata", []) + if doc_metadata: + for item in doc_metadata: + if item.get("metadata_id") == metadata_id: + return True, pipeline.name + except Exception: + logger.exception("Error checking metadata usage in pipeline workflow %s", workflow.id) + continue + + return False, None + @staticmethod def delete_metadata(dataset_id: str, metadata_id: str): lock_key = f"dataset_metadata_lock_{dataset_id}" @@ -103,6 +169,15 @@ def delete_metadata(dataset_id: str, metadata_id: str): metadata = db.session.query(DatasetMetadata).filter_by(id=metadata_id).first() if metadata is None: raise ValueError("Metadata not found.") + + # Check if metadata is used in Pipeline before deletion + is_used, pipeline_name = MetadataService.check_metadata_used_in_pipeline(dataset_id, metadata_id) + if is_used: + raise ValueError( + f"Cannot delete metadata '{metadata.name}' because it is currently used in " + f"Pipeline '{pipeline_name}'." + ) + db.session.delete(metadata) # deal related documents @@ -122,6 +197,8 @@ def delete_metadata(dataset_id: str, metadata_id: str): db.session.add(document) db.session.commit() return metadata + except ValueError: + raise except Exception: logger.exception("Delete metadata failed") finally: diff --git a/api/tests/test_containers_integration_tests/services/test_metadata_service.py b/api/tests/test_containers_integration_tests/services/test_metadata_service.py index c8ced3f3a5a4f1..130cd759e75803 100644 --- a/api/tests/test_containers_integration_tests/services/test_metadata_service.py +++ b/api/tests/test_containers_integration_tests/services/test_metadata_service.py @@ -460,11 +460,9 @@ def test_delete_metadata_not_found(self, db_session_with_containers, mock_extern fake_metadata_id = str(uuid.uuid4()) # Use valid UUID format - # Act: Execute the method under test - result = MetadataService.delete_metadata(dataset.id, fake_metadata_id) - - # Assert: Verify the method returns None when metadata is not found - assert result is None + # Act & Assert: Verify the method raises ValueError when metadata is not found + with pytest.raises(ValueError, match="Metadata not found."): + MetadataService.delete_metadata(dataset.id, fake_metadata_id) def test_delete_metadata_with_document_bindings( self, db_session_with_containers, mock_external_service_dependencies diff --git a/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.py b/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.py new file mode 100644 index 00000000000000..3d6da66a0c978a --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.py @@ -0,0 +1,103 @@ +import unittest +import uuid +from unittest.mock import MagicMock, patch + +from core.app.entities.app_invoke_entities import InvokeFrom +from core.workflow.enums import SystemVariableKey +from core.workflow.nodes.knowledge_index.entities import DocMetadata, KnowledgeIndexNodeData +from core.workflow.nodes.knowledge_index.knowledge_index_node import KnowledgeIndexNode +from core.workflow.runtime import VariablePool +from models.dataset import Dataset, DatasetMetadata, Document +from models.enums import UserFrom + + +class TestKnowledgeIndexNode(unittest.TestCase): + def setUp(self): + self.dataset_id = str(uuid.uuid4()) + self.document_id = str(uuid.uuid4()) + self.mock_dataset = MagicMock(spec=Dataset) + self.mock_dataset.id = self.dataset_id + self.mock_dataset.built_in_field_enabled = False + + self.mock_document = MagicMock(spec=Document) + self.mock_document.id = self.document_id + self.mock_document.doc_metadata = {} + + @patch("core.workflow.nodes.knowledge_index.knowledge_index_node.attributes.flag_modified") + @patch("core.workflow.nodes.knowledge_index.knowledge_index_node.db.session") + @patch("core.workflow.nodes.knowledge_index.knowledge_index_node.IndexProcessorFactory") + def test_run_with_custom_metadata(self, mock_index_processor_factory, mock_db_session, mock_flag_modified): + # Mock DB queries + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + self.mock_dataset, # For dataset query + self.mock_document, # For document query + ] + + # Mock Dataset Metadata + mock_metadata = MagicMock(spec=DatasetMetadata) + mock_metadata.id = "meta_uuid_1" + mock_metadata.name = "Category" + mock_db_session.scalars.return_value.all.return_value = [mock_metadata] + # Simpler mock for the scalar query - switched to bulk fetch + mock_db_session.scalar.return_value = "Category" + + # Mock Variable Pool + pool = MagicMock(spec=VariablePool) + + # Handle the chunk variable + chunk_var_mock = MagicMock() + chunk_var_mock.value = {"chunk": "data"} + + def variable_pool_get(selector): + if selector == ["sys", SystemVariableKey.DATASET_ID]: + return MagicMock(value=self.dataset_id) + if selector == ["sys", SystemVariableKey.DOCUMENT_ID]: + return MagicMock(value=self.document_id) + if selector == ["sys", SystemVariableKey.BATCH]: + return MagicMock(value="test-batch") + if selector == ["sys", SystemVariableKey.ORIGINAL_DOCUMENT_ID]: + return None + if selector == ["Start", "category"]: + var = MagicMock() + var.to_object.return_value = "Financial" + return var + if selector == ["sys", SystemVariableKey.INVOKE_FROM]: + return None + if selector == ["sys", "chunks"]: + return chunk_var_mock + return None + + pool.get.side_effect = variable_pool_get + + # Node Configuration + node_data = KnowledgeIndexNodeData( + id="node1", + title="Knowledge", + chunk_structure="chunk", + index_chunk_variable_selector=["sys", "chunks"], + doc_metadata=[DocMetadata(metadata_id="meta_uuid_1", value=["Start", "category"])], + ) + + # Initialize Node + graph_init_params = MagicMock() + graph_init_params.user_from = UserFrom.ACCOUNT + graph_init_params.invoke_from = InvokeFrom.WEB_APP + + config = {"id": "node1", "data": node_data.model_dump()} + + node = KnowledgeIndexNode( + id="node1", + graph_init_params=graph_init_params, + graph_runtime_state=MagicMock(variable_pool=pool), + config=config, + ) + + # Execute + result = node._run() + + # Verify metadata was set on document + assert self.mock_document.doc_metadata["Category"] == "Financial" + # Verify flag_modified was called for the doc_metadata field + mock_flag_modified.assert_called_with(self.mock_document, "doc_metadata") + # Verify commit was called + mock_db_session.commit.assert_called() diff --git a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py index bd226f7536d6d8..136a0af8ff7b97 100644 --- a/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py +++ b/api/tests/unit_tests/services/test_dataset_service_lock_not_owned.py @@ -86,6 +86,8 @@ def test_save_document_with_dataset_id_ignores_lock_not_owned( process_rule=None, duplicate=False, doc_language="en", + enable_built_in_metadata=False, + doc_metadata=None, ) account = fake_current_user diff --git a/api/tests/unit_tests/services/test_dataset_service_metadata.py b/api/tests/unit_tests/services/test_dataset_service_metadata.py new file mode 100644 index 00000000000000..a3de25ed300c4f --- /dev/null +++ b/api/tests/unit_tests/services/test_dataset_service_metadata.py @@ -0,0 +1,131 @@ +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest + +from models.account import Account +from models.dataset import Dataset, DatasetMetadata, Document +from models.model import UploadFile +from services.dataset_service import DocumentService +from services.entities.knowledge_entities.knowledge_entities import ( + DataSource, + DocumentMetadataInput, + FileInfo, + InfoList, + KnowledgeConfig, +) + + +class TestDocumentServiceMetadata: + @pytest.fixture + def mock_dependencies(self): + with ( + patch("services.dataset_service.db.session") as mock_db, + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch("services.dataset_service.redis_client") as mock_redis, + patch("services.dataset_service.DocumentService.build_document") as mock_build_document, + patch("services.dataset_service.current_user") as mock_current_user, + patch("services.dataset_service.DocumentIndexingTaskProxy") as mock_indexing_task, + # We don't patch DocumentService.save_document_with_dataset_id as that's what we are testing + ): + # Hack to pass isinstance check + mock_current_user.__class__ = Account + mock_current_user.current_tenant_id = "tenant-123" + + yield { + "db": mock_db, + "get_dataset": mock_get_dataset, + "redis": mock_redis, + "build_document": mock_build_document, + "current_user": mock_current_user, + } + + def test_save_document_with_metadata(self, mock_dependencies): + # Arrange + dataset_id = str(uuid4()) + tenant_id = str(uuid4()) + account = Mock(spec=Account) + account.id = "account-1" + account.current_tenant_id = tenant_id + + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.tenant_id = tenant_id + dataset.built_in_field_enabled = False + dataset.doc_form = "text_model" + mock_dependencies["get_dataset"].return_value = dataset + + # Define metadata inputs + metadata_id = str(uuid4()) + doc_metadata_inputs = [DocumentMetadataInput(metadata_id=metadata_id, value="custom_value")] + + # Knowledge config + knowledge_config = KnowledgeConfig( + data_source_type="upload_file", + data_source=DataSource( + info_list=InfoList(data_source_type="upload_file", file_info_list=FileInfo(file_ids=["file-1"])) + ), + doc_form="text_model", + doc_language="en", + indexing_technique="high_quality", + enable_built_in_metadata=True, + doc_metadata=doc_metadata_inputs, + ) + + # Mock local file for upload_file type + with patch("services.dataset_service.db.session.query") as mock_query: + # Mock DatasetMetadata lookup + mock_metadata_def = Mock(spec=DatasetMetadata) + mock_metadata_def.id = metadata_id + mock_metadata_def.name = "custom_field" + mock_metadata_def.field_type = "text" + + # Create a side effect for query(Model) + def query_side_effect(model): + m = Mock() + if model == DatasetMetadata: + m.filter.return_value.filter.return_value.first.return_value = mock_metadata_def + # handle the specific chain in code + m.filter_by.return_value.first.return_value = mock_metadata_def + m.filter.return_value.all.return_value = [mock_metadata_def] + return m + if model == Document: + doc_mock = Mock() + doc_mock.position = 1 + # For get_documents_position + m.filter_by.return_value.order_by.return_value.first.return_value = doc_mock + # For duplicate check + m.where.return_value.all.return_value = [] + return m + if model == UploadFile: + m.where.return_value.all.return_value = [Mock(id="file-1", tenant_id=tenant_id)] + return m + + return m + + mock_query.side_effect = query_side_effect + + # Mock build_document to return a document + mock_document = Mock(spec=Document) + mock_document.id = "doc-123" + mock_document.doc_metadata = {} + mock_dependencies["build_document"].return_value = mock_document + + # Act + DocumentService.save_document_with_dataset_id( + dataset=dataset, knowledge_config=knowledge_config, account=account + ) + + # Assert + # 1. Check built-in metadata enabled + assert dataset.built_in_field_enabled is True + + # 2. Check custom metadata passed to build_document + call_args = mock_dependencies["build_document"].call_args + assert call_args is not None + _, kwargs = call_args + assert "custom_metadata" in kwargs + assert kwargs["custom_metadata"] == {"custom_field": "custom_value"} + + # 3. Check DatasetMetadataBinding creation + assert mock_dependencies["db"].add.call_count >= 1 diff --git a/web/app/components/datasets/metadata/base/date-picker.tsx b/web/app/components/datasets/metadata/base/date-picker.tsx index 2f615498592a12..3c99323599dd57 100644 --- a/web/app/components/datasets/metadata/base/date-picker.tsx +++ b/web/app/components/datasets/metadata/base/date-picker.tsx @@ -38,7 +38,7 @@ const WrappedDatePicker = ({
diff --git a/web/app/components/datasets/metadata/edit-metadata-batch/input-combined.tsx b/web/app/components/datasets/metadata/edit-metadata-batch/input-combined.tsx index aec74bcfefbfb1..4b34f4772a7812 100644 --- a/web/app/components/datasets/metadata/edit-metadata-batch/input-combined.tsx +++ b/web/app/components/datasets/metadata/edit-metadata-batch/input-combined.tsx @@ -22,7 +22,7 @@ const InputCombined: FC = ({ onChange, readOnly, }) => { - const className = cn('h-6 grow p-0.5 text-xs') + const className = cn('h-6 grow p-0.5') if (type === DataType.time) { return ( = ({ return (
= ({ return ( onChange(e.target.value)} readOnly={readOnly} diff --git a/web/app/components/workflow/nodes/knowledge-base/components/metadata-section.tsx b/web/app/components/workflow/nodes/knowledge-base/components/metadata-section.tsx new file mode 100644 index 00000000000000..4867ba48e3d9e1 --- /dev/null +++ b/web/app/components/workflow/nodes/knowledge-base/components/metadata-section.tsx @@ -0,0 +1,346 @@ +'use client' +import type { FC } from 'react' +import type { DocMetadataItem } from '../types' +import type { BuiltInMetadataItem, MetadataItemWithValueLength } from '@/app/components/datasets/metadata/types' +import type { ValueSelector, Var } from '@/app/components/workflow/types' +import { RiAddLine, RiDeleteBinLine, RiDraftLine, RiEditLine } from '@remixicon/react' +import { useCallback, useState } from 'react' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' +import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import { InputNumber } from '@/app/components/base/input-number' +import Toast from '@/app/components/base/toast' +import Tooltip from '@/app/components/base/tooltip' +import Datepicker from '@/app/components/datasets/metadata/base/date-picker' +import DatasetMetadataDrawer from '@/app/components/datasets/metadata/metadata-dataset/dataset-metadata-drawer' +import { DataType } from '@/app/components/datasets/metadata/types' +import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker' +import { VarType } from '@/app/components/workflow/types' +import { + useBuiltInMetaDataFields, + useCreateMetaData, + useDeleteMetaData, + useRenameMeta, + useUpdateBuiltInStatus, +} from '@/service/knowledge/use-metadata' +import { cn } from '@/utils/classnames' + +type MetadataSectionProps = { + nodeId: string + datasetId?: string + enableBuiltInMetadata: boolean + onEnableBuiltInMetadataChange: (enabled: boolean) => void + userMetadata?: MetadataItemWithValueLength[] + docMetadata?: DocMetadataItem[] + onDocMetadataChange?: (metadata: DocMetadataItem[]) => void + onMetadataListChange?: () => void + readonly?: boolean + className?: string +} + +const MetadataSection: FC = ({ + nodeId, + datasetId, + enableBuiltInMetadata, + onEnableBuiltInMetadataChange, + userMetadata = [], + docMetadata = [], + onDocMetadataChange, + onMetadataListChange, + readonly, + className, +}) => { + const { t } = useTranslation() + const [isDrawerOpen, setIsDrawerOpen] = useState(false) + + // Get built-in metadata fields from API + const { data: builtInFieldsData } = useBuiltInMetaDataFields() + const builtInFields = builtInFieldsData?.fields || [] + + // Mutations for drawer + const createMetadataMutation = useCreateMetaData(datasetId || '') + const renameMetadataMutation = useRenameMeta(datasetId || '') + const deleteMetadataMutation = useDeleteMetaData(datasetId || '') + const updateBuiltInStatus = useUpdateBuiltInStatus(datasetId || '') + + // Drawer handlers + const handleAddMetadata = useCallback(async (data: BuiltInMetadataItem) => { + await createMetadataMutation.mutateAsync(data) + Toast.notify({ type: 'success', message: t('api.actionSuccess', { ns: 'common' }) }) + onMetadataListChange?.() + }, [createMetadataMutation, t, onMetadataListChange]) + + const handleRenameMetadata = useCallback(async (data: MetadataItemWithValueLength) => { + await renameMetadataMutation.mutateAsync(data) + Toast.notify({ type: 'success', message: t('api.actionSuccess', { ns: 'common' }) }) + onMetadataListChange?.() + }, [renameMetadataMutation, t, onMetadataListChange]) + + const handleDeleteMetadata = useCallback(async (id: string) => { + await deleteMetadataMutation.mutateAsync(id) + Toast.notify({ type: 'success', message: t('api.actionSuccess', { ns: 'common' }) }) + onMetadataListChange?.() + }, [deleteMetadataMutation, t, onMetadataListChange]) + + const handleBuiltInEnabledChange = useCallback(async (enabled: boolean) => { + onEnableBuiltInMetadataChange(enabled) + if (datasetId) { + await updateBuiltInStatus.mutateAsync(enabled) + } + }, [datasetId, updateBuiltInStatus, onEnableBuiltInMetadataChange]) + + // Document metadata value handlers + const handleAddDocMetadata = useCallback(() => { + if (onDocMetadataChange) { + onDocMetadataChange([...docMetadata, { metadata_id: '', value: '' }]) + } + }, [docMetadata, onDocMetadataChange]) + + const handleRemoveDocMetadata = useCallback((index: number) => { + if (onDocMetadataChange) { + const newMetadata = [...docMetadata] + newMetadata.splice(index, 1) + onDocMetadataChange(newMetadata) + } + }, [docMetadata, onDocMetadataChange]) + + const handleDocMetadataIdChange = useCallback((index: number, metadataId: string) => { + if (onDocMetadataChange) { + const newMetadata = [...docMetadata] + newMetadata[index] = { ...newMetadata[index], metadata_id: metadataId } + onDocMetadataChange(newMetadata) + } + }, [docMetadata, onDocMetadataChange]) + + const handleDocMetadataValueChange = useCallback((index: number, value: string | number | ValueSelector) => { + if (onDocMetadataChange) { + const newMetadata = [...docMetadata] + newMetadata[index] = { ...newMetadata[index], value } + onDocMetadataChange(newMetadata) + } + }, [docMetadata, onDocMetadataChange]) + + const getAvailableMetadataOptions = useCallback((currentId: string) => { + const usedIds = docMetadata.map(m => m.metadata_id).filter(id => id !== currentId) + return userMetadata.filter(m => !usedIds.includes(m.id)) + }, [userMetadata, docMetadata]) + + const getMetadataType = useCallback((metadataId: string): DataType | undefined => { + return userMetadata.find(m => m.id === metadataId)?.type + }, [userMetadata]) + + // Filter variables based on metadata type + const createVarFilter = useCallback((metadataId: string) => { + return (variable: Var): boolean => { + const metadataType = getMetadataType(metadataId) + + if (!metadataType) + return false + + // Type mapping: Metadata DataType -> Workflow VarType + switch (metadataType) { + case DataType.string: + return variable.type === VarType.string + case DataType.number: + return variable.type === VarType.number || variable.type === VarType.integer + case DataType.time: { + // Only allow number variables with time-related names + const varName = variable.variable.toLowerCase() + const isTimeRelated + = varName === 'timestamp' // sys.timestamp + || varName.includes('time') // current_time, expiry_time + || varName.includes('date') // created_date, updated_date + || varName.includes('at') // created_at, updated_at + + return (variable.type === VarType.number || variable.type === VarType.integer) + && isTimeRelated + } + default: + return false + } + } + }, [getMetadataType]) + + return ( +
+
+
+ {t('metadata.metadata', { ns: 'dataset' })} +
+ {datasetId && !readonly && ( + + )} +
+ + {/* Document Metadata Values Section */} + {userMetadata.length > 0 && ( +
+
+ {!readonly && ( + + )} +
+ + {docMetadata.length > 0 + ? ( +
+ {docMetadata.map((item, index) => { + const isVariable = Array.isArray(item.value) + const itemKey = item.metadata_id ? `metadata-${item.metadata_id}` : `new-${index}` + return ( +
+
+
+ +
+
+
+ +
!readonly && handleDocMetadataValueChange(index, [])} + > + +
+ + +
!readonly && handleDocMetadataValueChange(index, '')} + > + +
+ +
+
+
+ {isVariable + ? ( + handleDocMetadataValueChange(index, value)} + isSupportConstantValue={false} + placeholder={t('placeholder.input', { ns: 'common' }) || ''} + className="h-full border-none !bg-transparent p-0" + zIndex={1000} + isShowNodeName + minWidth={360} + filterVar={createVarFilter(item.metadata_id)} + /> + ) + : ( +
+ {(() => { + const metadataType = getMetadataType(item.metadata_id) + + // Time type - use Datepicker + if (metadataType === DataType.time) { + return ( + handleDocMetadataValueChange(index, value || 0)} + /> + ) + } + + // Number type - use InputNumber + if (metadataType === DataType.number) { + return ( + handleDocMetadataValueChange(index, value)} + readOnly={readonly} + size="regular" + /> + ) + } + + // String type (default) - use text input + return ( + handleDocMetadataValueChange(index, e.target.value)} + placeholder={t('placeholder.input', { ns: 'common' }) || ''} + disabled={readonly} + className="h-full w-full bg-transparent text-[13px] text-text-primary outline-none placeholder:text-text-placeholder disabled:opacity-50" + /> + ) + })()} +
+ )} +
+
+
+ {!readonly && ( + + )} +
+ ) + })} +
+ ) + : ( +
+ {t('stepTwo.metadata.noValues', { ns: 'datasetCreation' })} +
+ )} +
+ )} + + {/* Metadata Drawer */} + {isDrawerOpen && datasetId && ( + setIsDrawerOpen(false)} + onAdd={handleAddMetadata} + onRename={handleRenameMetadata} + onRemove={handleDeleteMetadata} + /> + )} +
+ ) +} + +export default MetadataSection diff --git a/web/app/components/workflow/nodes/knowledge-base/hooks/use-config.ts b/web/app/components/workflow/nodes/knowledge-base/hooks/use-config.ts index f26df2c3ee4a9b..0ff5f018063104 100644 --- a/web/app/components/workflow/nodes/knowledge-base/hooks/use-config.ts +++ b/web/app/components/workflow/nodes/knowledge-base/hooks/use-config.ts @@ -1,4 +1,5 @@ import type { + DocMetadataItem, KnowledgeBaseNodeType, RerankingModel, SummaryIndexSetting, @@ -247,6 +248,18 @@ export const useConfig = (id: string) => { }) }, [handleNodeDataUpdate]) + const handleEnableBuiltInMetadataChange = useCallback((enabled: boolean) => { + handleNodeDataUpdate({ + enable_built_in_metadata: enabled, + }) + }, [handleNodeDataUpdate]) + + const handleDocMetadataChange = useCallback((docMetadata: DocMetadataItem[]) => { + handleNodeDataUpdate({ + doc_metadata: docMetadata, + }) + }, [handleNodeDataUpdate]) + const handleSummaryIndexSettingChange = useCallback((summaryIndexSetting: SummaryIndexSetting) => { const nodeData = getNodeData() handleNodeDataUpdate({ @@ -271,6 +284,8 @@ export const useConfig = (id: string) => { handleScoreThresholdChange, handleScoreThresholdEnabledChange, handleInputVariableChange, + handleEnableBuiltInMetadataChange, + handleDocMetadataChange, handleSummaryIndexSettingChange, } } diff --git a/web/app/components/workflow/nodes/knowledge-base/panel.tsx b/web/app/components/workflow/nodes/knowledge-base/panel.tsx index 0a275645a8a449..536ca9639e7bd1 100644 --- a/web/app/components/workflow/nodes/knowledge-base/panel.tsx +++ b/web/app/components/workflow/nodes/knowledge-base/panel.tsx @@ -18,10 +18,13 @@ import { Group, } from '@/app/components/workflow/nodes/_base/components/layout' import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { useDatasetMetaData } from '@/service/knowledge/use-metadata' import Split from '../_base/components/split' import ChunkStructure from './components/chunk-structure' import EmbeddingModel from './components/embedding-model' import IndexMethod from './components/index-method' +import MetadataSection from './components/metadata-section' import RetrievalSetting from './components/retrieval-setting' import { useConfig } from './hooks/use-config' import { @@ -38,6 +41,10 @@ const Panel: FC> = ({ const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding) const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank) + // Get datasetId from context and fetch metadata + const datasetId = useDatasetDetailContextWithSelector(s => s.dataset?.id) + const { data: metadataList, refetch: refetchMetadataList } = useDatasetMetaData(datasetId || '') + const { handleChunkStructureChange, handleIndexMethodChange, @@ -52,6 +59,8 @@ const Panel: FC> = ({ handleScoreThresholdChange, handleScoreThresholdEnabledChange, handleInputVariableChange, + handleEnableBuiltInMetadataChange, + handleDocMetadataChange, handleSummaryIndexSettingChange, } = useConfig(id) @@ -208,6 +217,19 @@ const Panel: FC> = ({ />
+ + + ) } diff --git a/web/app/components/workflow/nodes/knowledge-base/types.ts b/web/app/components/workflow/nodes/knowledge-base/types.ts index aca59d5516c970..5a4d093bd05fb1 100644 --- a/web/app/components/workflow/nodes/knowledge-base/types.ts +++ b/web/app/components/workflow/nodes/knowledge-base/types.ts @@ -42,12 +42,19 @@ export type RetrievalSetting = { score_threshold: number reranking_mode?: RerankingModeEnum } + +export type DocMetadataItem = { + metadata_id: string + value: string | number | string[] // string[] for ValueSelector +} + export type SummaryIndexSetting = { enable?: boolean model_name?: string model_provider_name?: string summary_prompt?: string } + export type KnowledgeBaseNodeType = CommonNodeType & { index_chunk_variable_selector: string[] chunk_structure?: ChunkStructureEnum @@ -56,6 +63,8 @@ export type KnowledgeBaseNodeType = CommonNodeType & { embedding_model_provider?: string keyword_number: number retrieval_model: RetrievalSetting + enable_built_in_metadata?: boolean + doc_metadata?: DocMetadataItem[] _embeddingModelList?: Model[] _rerankModelList?: Model[] summary_index_setting?: SummaryIndexSetting diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-date.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-date.tsx index b34093b7b094e8..2511571d78a553 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-date.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-date.tsx @@ -32,10 +32,10 @@ const ConditionDate = ({ handleClickTrigger, }: TriggerProps) => { return ( -
+
@@ -71,7 +71,7 @@ const ConditionDate = ({ }, [value, handleDateChange, timezone, t]) return ( -
+