From 9818afd5174f0a3e2d8312ad5d17147ac3da9a5e Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:36:36 +0000 Subject: [PATCH 01/30] [SYNPY-1869] Add SearchIndex entity + search-management APIs; rename SchemaOrganization to Organization Build the OOP method surface for search-management resources (TextAnalyzer, ColumnAnalyzerOverride, SynonymSet, SearchConfiguration, SearchConfigBinding) and the SearchIndex entity with autocomplete/query support, following the async-first @async_to_sync pattern with protocol classes and unit tests. Hard-rename the SchemaOrganization model to Organization (class, protocol, module helper list_json_schema_organizations -> list_organizations, files, docs, and mkdocs nav) so it consistently reads as an "Organization" that holds resources. No backwards-compatible alias. --- ...schema_organization.md => organization.md} | 4 +- ...schema_organization.md => organization.md} | 4 +- .../python/tutorial_scripts/json_schema.py | 4 +- mkdocs.yml | 4 +- synapseclient/api/__init__.py | 43 + synapseclient/api/entity_factory.py | 2 + synapseclient/api/search_services.py | 482 +++++++++ .../core/constants/concrete_types.py | 4 + .../extensions/curator/schema_management.py | 4 +- synapseclient/models/__init__.py | 39 +- .../models/mixins/asynchronous_job.py | 2 + .../models/mixins/table_components.py | 2 +- ...schema_organization.py => organization.py} | 118 ++- synapseclient/models/search_index.py | 346 +++++++ synapseclient/models/search_management.py | 939 ++++++++++++++++++ synapseclient/operations/delete_operations.py | 16 +- synapseclient/operations/store_operations.py | 16 +- synapseclient/services/json_schema.py | 4 +- .../curator/test_schema_management.py | 16 +- ...on_async.py => test_organization_async.py} | 46 +- .../models/async/test_search_index_async.py | 167 ++++ .../test_factory_operations_store_async.py | 10 +- .../synapseclient/test_command_line_client.py | 4 +- .../extensions/test_schema_management.py | 2 +- ...ync.py => unit_test_organization_async.py} | 108 +- .../unit_test_search_management_async.py | 574 +++++++++++ .../operations/unit_test_delete_operations.py | 8 +- .../operations/unit_test_store_operations.py | 12 +- 28 files changed, 2784 insertions(+), 196 deletions(-) rename docs/reference/experimental/async/{schema_organization.md => organization.md} (86%) rename docs/reference/experimental/sync/{schema_organization.md => organization.md} (85%) create mode 100644 synapseclient/api/search_services.py rename synapseclient/models/{schema_organization.py => organization.py} (93%) create mode 100644 synapseclient/models/search_index.py create mode 100644 synapseclient/models/search_management.py rename tests/integration/synapseclient/models/async/{test_schema_organization_async.py => test_organization_async.py} (93%) create mode 100644 tests/integration/synapseclient/models/async/test_search_index_async.py rename tests/unit/synapseclient/models/async/{unit_test_schema_organization_async.py => unit_test_organization_async.py} (91%) create mode 100644 tests/unit/synapseclient/models/async/unit_test_search_management_async.py diff --git a/docs/reference/experimental/async/schema_organization.md b/docs/reference/experimental/async/organization.md similarity index 86% rename from docs/reference/experimental/async/schema_organization.md rename to docs/reference/experimental/async/organization.md index e0f88ad69..977e5fdeb 100644 --- a/docs/reference/experimental/async/schema_organization.md +++ b/docs/reference/experimental/async/organization.md @@ -1,4 +1,4 @@ -# SchemaOrganization +# Organization Contained within this file are experimental interfaces for working with the Synapse Python Client. Unless otherwise noted these interfaces are subject to change at any time. Use @@ -6,7 +6,7 @@ at your own risk. ## API Reference -::: synapseclient.models.SchemaOrganization +::: synapseclient.models.Organization options: inherited_members: true members: diff --git a/docs/reference/experimental/sync/schema_organization.md b/docs/reference/experimental/sync/organization.md similarity index 85% rename from docs/reference/experimental/sync/schema_organization.md rename to docs/reference/experimental/sync/organization.md index 88f9f4fb3..5336fee74 100644 --- a/docs/reference/experimental/sync/schema_organization.md +++ b/docs/reference/experimental/sync/organization.md @@ -1,4 +1,4 @@ -# SchemaOrganization +# Organization Contained within this file are experimental interfaces for working with the Synapse Python Client. Unless otherwise noted these interfaces are subject to change at any time. Use @@ -6,7 +6,7 @@ at your own risk. ## API Reference -::: synapseclient.models.SchemaOrganization +::: synapseclient.models.Organization options: inherited_members: true members: diff --git a/docs/tutorials/python/tutorial_scripts/json_schema.py b/docs/tutorials/python/tutorial_scripts/json_schema.py index e5bf16942..98ba322f3 100644 --- a/docs/tutorials/python/tutorial_scripts/json_schema.py +++ b/docs/tutorials/python/tutorial_scripts/json_schema.py @@ -4,7 +4,7 @@ import synapseclient from synapseclient.core.utils import make_bogus_data_file -from synapseclient.models import File, Folder, JSONSchema, Project, SchemaOrganization +from synapseclient.models import File, Folder, JSONSchema, Organization, Project # 1. Set up Synapse Python client syn = synapseclient.Synapse() @@ -48,7 +48,7 @@ # 3. Try create test organization and json schema if they do not exist # --8<-- [start:create_org_and_schema] -organization = SchemaOrganization(name=ORG_NAME) +organization = Organization(name=ORG_NAME) try: organization.store() except Exception as e: diff --git a/mkdocs.yml b/mkdocs.yml index 135436e09..4d4843f01 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -113,7 +113,7 @@ nav: - Curator: reference/experimental/sync/curator.md - Link: reference/experimental/sync/link_entity.md - Functional Interfaces: reference/experimental/functional_interfaces.md - - SchemaOrganization: reference/experimental/sync/schema_organization.md + - Organization: reference/experimental/sync/organization.md - JSONSchema: reference/experimental/sync/json_schema.md - Wiki: reference/experimental/sync/wiki.md - FormGroup and Form: reference/experimental/sync/form.md @@ -143,7 +143,7 @@ nav: - UserProfile: reference/experimental/async/user_profile.md - Curator: reference/experimental/async/curator.md - Link: reference/experimental/async/link_entity.md - - SchemaOrganization: reference/experimental/async/schema_organization.md + - Organization: reference/experimental/async/organization.md - JSONSchema: reference/experimental/async/json_schema.md - Wiki: reference/experimental/async/wiki.md - FormGroup and Form: reference/experimental/async/form.md diff --git a/synapseclient/api/__init__.py b/synapseclient/api/__init__.py index c55e43410..12dd35d3e 100644 --- a/synapseclient/api/__init__.py +++ b/synapseclient/api/__init__.py @@ -145,6 +145,28 @@ get_project_setting, update_project_setting, ) +from .search_services import ( + autocomplete_search, + bind_search_config_to_entity, + clear_search_config_binding, + create_column_analyzer_override, + create_search_configuration, + create_synonym_set, + create_text_analyzer, + get_column_analyzer_override, + get_search_config_binding, + get_search_configuration, + get_synonym_set, + get_text_analyzer, + list_column_analyzer_overrides, + list_search_configurations, + list_synonym_sets, + list_text_analyzers, + update_column_analyzer_override, + update_search_configuration, + update_synonym_set, + update_text_analyzer, +) from .storage_location_services import ( create_storage_location_setting, get_storage_location_setting, @@ -392,4 +414,25 @@ "create_project_setting", "update_project_setting", "delete_project_setting", + # search_services + "create_text_analyzer", + "get_text_analyzer", + "update_text_analyzer", + "list_text_analyzers", + "create_column_analyzer_override", + "get_column_analyzer_override", + "update_column_analyzer_override", + "list_column_analyzer_overrides", + "create_synonym_set", + "get_synonym_set", + "update_synonym_set", + "list_synonym_sets", + "create_search_configuration", + "get_search_configuration", + "update_search_configuration", + "list_search_configurations", + "bind_search_config_to_entity", + "get_search_config_binding", + "clear_search_config_binding", + "autocomplete_search", ] diff --git a/synapseclient/api/entity_factory.py b/synapseclient/api/entity_factory.py index 919af7227..db3472641 100644 --- a/synapseclient/api/entity_factory.py +++ b/synapseclient/api/entity_factory.py @@ -343,6 +343,7 @@ class type. This will also download the file if `download_file` is set to True. MaterializedView, Project, RecordSet, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -379,6 +380,7 @@ class type. This will also download the file if `download_file` is set to True. concrete_types.RECORD_SET_ENTITY: RecordSet, concrete_types.SUBMISSION_VIEW: SubmissionView, concrete_types.VIRTUAL_TABLE: VirtualTable, + concrete_types.SEARCH_INDEX_ENTITY: SearchIndex, concrete_types.LINK_ENTITY: Link, concrete_types.DOCKER_REPOSITORY: DockerRepository, } diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py new file mode 100644 index 000000000..767689d8f --- /dev/null +++ b/synapseclient/api/search_services.py @@ -0,0 +1,482 @@ +"""This module is responsible for exposing the services defined at: + + +It covers TextAnalyzer, ColumnAnalyzerOverride, SynonymSet, SearchConfiguration, +SearchConfigBinding, and the synchronous SearchIndex autocomplete endpoint. +The async SearchIndex query endpoint is exposed via the +`SearchIndexQuery.send_job_and_wait_async()` method on the model class +(`models.search_management.SearchIndexQuery`), which uses the shared +`AsynchronousCommunicator` mixin. +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +if TYPE_CHECKING: + from synapseclient import Synapse + + +# ---------- Text Analyzer ---------- + + +async def create_text_analyzer( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a TextAnalyzer. + + + + Arguments: + request: TextAnalyzer body. Must include organizationName, name, settings. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The created TextAnalyzer. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/text/analyzer", body=json.dumps(request) + ) + + +async def get_text_analyzer( + analyzer_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a TextAnalyzer by ID. + + + + Arguments: + analyzer_id: The numeric ID of the text analyzer to retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The requested TextAnalyzer. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async(uri=f"/search/text/analyzer/{analyzer_id}") + + +async def update_text_analyzer( + analyzer_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a TextAnalyzer. + + + + Arguments: + analyzer_id: The path ID (must match the request body's ID). + request: The updated TextAnalyzer. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated TextAnalyzer. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/text/analyzer/{analyzer_id}", body=json.dumps(request) + ) + + +async def list_text_analyzers( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List TextAnalyzers, optionally filtered by Organization. + + + + Arguments: + organization_name: Optional filter by organization name. + next_page_token: Optional pagination token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Page of TextAnalyzers and a nextPageToken if more results exist. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + body = {k: v for k, v in body.items() if v is not None} + return await client.rest_post_async( + uri="/search/text/analyzer/list", body=json.dumps(body) + ) + + +# ---------- Column Analyzer Override ---------- + + +async def create_column_analyzer_override( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a ColumnAnalyzerOverride. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/column/analyzer/override", body=json.dumps(request) + ) + + +async def get_column_analyzer_override( + column_analyzer_override_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a ColumnAnalyzerOverride by ID. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async( + uri=f"/search/column/analyzer/override/{column_analyzer_override_id}" + ) + + +async def update_column_analyzer_override( + column_analyzer_override_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a ColumnAnalyzerOverride. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/column/analyzer/override/{column_analyzer_override_id}", + body=json.dumps(request), + ) + + +async def list_column_analyzer_overrides( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List ColumnAnalyzerOverrides, optionally filtered by Organization. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + body = {k: v for k, v in body.items() if v is not None} + return await client.rest_post_async( + uri="/search/column/analyzer/override/list", body=json.dumps(body) + ) + + +# ---------- Synonym Set ---------- + + +async def create_synonym_set( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a SynonymSet. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/synonym/set", body=json.dumps(request) + ) + + +async def get_synonym_set( + synonym_set_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a SynonymSet by ID. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async(uri=f"/search/synonym/set/{synonym_set_id}") + + +async def update_synonym_set( + synonym_set_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a SynonymSet. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/synonym/set/{synonym_set_id}", body=json.dumps(request) + ) + + +async def list_synonym_sets( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List SynonymSets, optionally filtered by Organization. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + body = {k: v for k, v in body.items() if v is not None} + return await client.rest_post_async( + uri="/search/synonym/set/list", body=json.dumps(body) + ) + + +# ---------- Search Configuration ---------- + + +async def create_search_configuration( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a SearchConfiguration. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/configuration", body=json.dumps(request) + ) + + +async def get_search_configuration( + search_configuration_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a SearchConfiguration by ID. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async( + uri=f"/search/configuration/{search_configuration_id}" + ) + + +async def update_search_configuration( + search_configuration_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a SearchConfiguration. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/configuration/{search_configuration_id}", + body=json.dumps(request), + ) + + +async def list_search_configurations( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List SearchConfigurations, optionally filtered by Organization. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + body = {k: v for k, v in body.items() if v is not None} + return await client.rest_post_async( + uri="/search/configuration/list", body=json.dumps(body) + ) + + +# ---------- Search Configuration Bindings ---------- + + +async def bind_search_config_to_entity( + entity_id: str, + search_configuration_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Bind a SearchConfiguration to an entity. Replaces any existing binding. + + + + Arguments: + entity_id: The ID of the entity to bind to. + search_configuration_id: The ID of the SearchConfiguration to bind. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The created SearchConfigBinding. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = { + "entityId": entity_id, + "searchConfigurationId": search_configuration_id, + } + return await client.rest_put_async( + uri=f"/entity/{entity_id}/searchconfig/binding", body=json.dumps(body) + ) + + +async def get_search_config_binding( + entity_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get the effective SearchConfigBinding for an entity by walking up the + hierarchy. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async(uri=f"/entity/{entity_id}/searchconfig/binding") + + +async def clear_search_config_binding( + entity_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> None: + """Clear the SearchConfigBinding on a specific entity. + + + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + await client.rest_delete_async(uri=f"/entity/{entity_id}/searchconfig/binding") + + +# ---------- Search Queries ---------- +# +# The async search query endpoint (POST /search/query/async/start + +# GET /search/query/async/get/{token}) is exposed through the shared +# AsynchronousCommunicator mixin. Use: +# +# query = SearchIndexQuery(search_index_id=..., search_query=SearchQuery(...)) +# await query.send_job_and_wait_async() +# +# The synchronous autocomplete endpoint stays here because it is not an +# async job. Build its body with SearchAutocompleteRequest.to_synapse_request(). + + +async def autocomplete_search( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Synchronous autocomplete search. Caps results at 8. + + + + Arguments: + request: SearchAutocompleteRequest body (must include searchIndexId and + a searchQuery with a prefix-style query). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + SearchQueryResults. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/autocomplete", body=json.dumps(request) + ) + + +__all__: List[str] = [ + "create_text_analyzer", + "get_text_analyzer", + "update_text_analyzer", + "list_text_analyzers", + "create_column_analyzer_override", + "get_column_analyzer_override", + "update_column_analyzer_override", + "list_column_analyzer_overrides", + "create_synonym_set", + "get_synonym_set", + "update_synonym_set", + "list_synonym_sets", + "create_search_configuration", + "get_search_configuration", + "update_search_configuration", + "list_search_configurations", + "bind_search_config_to_entity", + "get_search_config_binding", + "clear_search_config_binding", + "autocomplete_search", +] diff --git a/synapseclient/core/constants/concrete_types.py b/synapseclient/core/constants/concrete_types.py index d8cbdbd59..a994665de 100644 --- a/synapseclient/core/constants/concrete_types.py +++ b/synapseclient/core/constants/concrete_types.py @@ -89,8 +89,12 @@ MATERIALIZED_VIEW = "org.sagebionetworks.repo.model.table.MaterializedView" SUBMISSION_VIEW = "org.sagebionetworks.repo.model.table.SubmissionView" VIRTUAL_TABLE = "org.sagebionetworks.repo.model.table.VirtualTable" +SEARCH_INDEX_ENTITY = "org.sagebionetworks.repo.model.search.table.SearchIndex" DOCKER_REPOSITORY = "org.sagebionetworks.repo.model.docker.DockerRepository" +# Search Management +SEARCH_INDEX_QUERY = "org.sagebionetworks.repo.model.search.table.SearchIndexQuery" + # upload requests MULTIPART_UPLOAD_REQUEST = "org.sagebionetworks.repo.model.file.MultipartUploadRequest" MULTIPART_UPLOAD_COPY_REQUEST = ( diff --git a/synapseclient/extensions/curator/schema_management.py b/synapseclient/extensions/curator/schema_management.py index 44b39959a..cf51e71a7 100644 --- a/synapseclient/extensions/curator/schema_management.py +++ b/synapseclient/extensions/curator/schema_management.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from synapseclient import Synapse from synapseclient.models.mixins.json_schema import JSONSchemaBinding - from synapseclient.models.schema_organization import JSONSchema + from synapseclient.models.organization import JSONSchema def register_jsonschema( @@ -129,7 +129,7 @@ async def register_jsonschema_async( ``` """ from synapseclient import Synapse - from synapseclient.models.schema_organization import JSONSchema + from synapseclient.models.organization import JSONSchema syn = Synapse.get_client(synapse_client=synapse_client) diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index ce1a3bf9e..c45c16a00 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -28,10 +28,28 @@ from synapseclient.models.link import Link from synapseclient.models.materializedview import MaterializedView from synapseclient.models.mixins.table_components import QueryMixin +from synapseclient.models.organization import JSONSchema, Organization from synapseclient.models.project import Project from synapseclient.models.project_setting import ProjectSetting from synapseclient.models.recordset import RecordSet -from synapseclient.models.schema_organization import JSONSchema, SchemaOrganization +from synapseclient.models.search_index import SearchIndex +from synapseclient.models.search_management import ( + ColumnAnalyzerOverride, + ColumnAnalyzerOverrideEntry, + SearchAutocompleteRequest, + SearchConfigBinding, + SearchConfiguration, + SearchFieldValue, + SearchHighlight, + SearchHit, + SearchIndexQuery, + SearchIndexState, + SearchIndexStatus, + SearchQuery, + SearchQueryPart, + SynonymSet, + TextAnalyzer, +) from synapseclient.models.services import FailureStrategy from synapseclient.models.storage_location import ( StorageLocation, @@ -166,7 +184,7 @@ "WikiHistorySnapshot", "WikiHeader", # JSON Schema models - "SchemaOrganization", + "Organization", "JSONSchema", # Form models "FormGroup", @@ -177,6 +195,23 @@ "UploadType", # Project Setting models "ProjectSetting", + # SearchIndex / Search Management models + "SearchIndex", + "SearchIndexQuery", + "SearchIndexStatus", + "SearchIndexState", + "SearchQuery", + "SearchQueryPart", + "SearchAutocompleteRequest", + "SearchHit", + "SearchFieldValue", + "SearchHighlight", + "SearchConfiguration", + "SearchConfigBinding", + "TextAnalyzer", + "ColumnAnalyzerOverride", + "ColumnAnalyzerOverrideEntry", + "SynonymSet", ] # Static methods to expose as functions diff --git a/synapseclient/models/mixins/asynchronous_job.py b/synapseclient/models/mixins/asynchronous_job.py index 3110f892b..c51dd5578 100644 --- a/synapseclient/models/mixins/asynchronous_job.py +++ b/synapseclient/models/mixins/asynchronous_job.py @@ -21,6 +21,7 @@ GRID_RECORD_SET_EXPORT_REQUEST, QUERY_BUNDLE_REQUEST, QUERY_TABLE_CSV_REQUEST, + SEARCH_INDEX_QUERY, SYNCHRONIZE_GRID_REQUEST, TABLE_UPDATE_TRANSACTION_REQUEST, UPLOAD_TO_TABLE_PREVIEW_REQUEST, @@ -45,6 +46,7 @@ QUERY_BUNDLE_REQUEST: "/entity/{entityId}/table/query/async", GRID_CSV_IMPORT_REQUEST: "/grid/import/csv/async", UPLOAD_TO_TABLE_PREVIEW_REQUEST: "/table/upload/csv/preview/async", + SEARCH_INDEX_QUERY: "/search/query/async", } diff --git a/synapseclient/models/mixins/table_components.py b/synapseclient/models/mixins/table_components.py index 3d25c0898..e72a7ed26 100644 --- a/synapseclient/models/mixins/table_components.py +++ b/synapseclient/models/mixins/table_components.py @@ -84,7 +84,7 @@ "DatasetCollection", "SubmissionView", ] -CLASSES_WITH_READ_ONLY_SCHEMA = ["MaterializedView", "VirtualTable"] +CLASSES_WITH_READ_ONLY_SCHEMA = ["MaterializedView", "VirtualTable", "SearchIndex"] PANDAS_TABLE_TYPE = { "floating": "DOUBLE", diff --git a/synapseclient/models/schema_organization.py b/synapseclient/models/organization.py similarity index 93% rename from synapseclient/models/schema_organization.py rename to synapseclient/models/organization.py index cfa3f30df..8248e6a0d 100644 --- a/synapseclient/models/schema_organization.py +++ b/synapseclient/models/organization.py @@ -1,5 +1,5 @@ """ -This file contains the SchemaOrganization and JSONSchema classes. +This file contains the Organization and JSONSchema classes. These are used to manage Organization and JSON Schema entities in Synapse. """ @@ -30,15 +30,13 @@ from synapseclient.models.mixins.json_schema import JSONSchemaVersionInfo -class SchemaOrganizationProtocol(Protocol): +class OrganizationProtocol(Protocol): """ The protocol for methods that are asynchronous but also have a synchronous counterpart that may also be called. """ - def get( - self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Organization": """ Gets the metadata from Synapse for this organization @@ -50,17 +48,17 @@ def get( Returns: Itself - Example: Get an existing SchemaOrganization + Example: Get an existing Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") org.get() print(org) ``` @@ -68,9 +66,7 @@ def get( """ return self - def store( - self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Organization": """ Stores this organization in Synapse @@ -82,17 +78,17 @@ def store( Returns: Itself - Example: Store the SchemaOrganization in Synapse + Example: Store the Organization in Synapse   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.store() print(org) ``` @@ -109,17 +105,17 @@ def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Delete the SchemaOrganization from Synapse + Example: Delete the Organization from Synapse   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.delete() ``` """ @@ -138,17 +134,17 @@ def get_json_schemas( Returns: A Generator containing the JSONSchemas that belong to this organization - Example: Get the JSONSchemas that are part of this SchemaOrganization + Example: Get the JSONSchemas that are part of this Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") js_generator = org.get_json_schemas() for item in js_generator: print(item) @@ -172,17 +168,17 @@ def get_acl(self, *, synapse_client: Optional["Synapse"] = None) -> dict[str, An A dictionary in the form of this response: [AccessControlList]https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/AccessControlList.html - Example: Get the ACL for the SchemaOrganization + Example: Get the ACL for the Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.get_acl() ``` """ @@ -207,17 +203,17 @@ def update_acl( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Update the ACL for the SchemaOrganization + Example: Update the ACL for the Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.update_acl( principal_id=1, access_type=["READ"] @@ -229,7 +225,7 @@ def update_acl( @dataclass() @async_to_sync -class SchemaOrganization(SchemaOrganizationProtocol): +class Organization(OrganizationProtocol): """ Represents an [Organization](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/Organization.html). @@ -258,7 +254,7 @@ def __post_init__(self) -> None: async def get_async( self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + ) -> "Organization": """ Gets the metadata from Synapse for this organization @@ -273,12 +269,12 @@ async def get_async( Raises: ValueError: If the name has not been set - Example: Get an existing SchemaOrganization + Example: Get an existing Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -287,7 +283,7 @@ async def get_org(): syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") await org.get_async() return org @@ -297,14 +293,14 @@ async def get_org(): ``` """ if not self.name: - raise ValueError("SchemaOrganization must have a name") + raise ValueError("Organization must have a name") response = await get_organization(self.name, synapse_client=synapse_client) self.fill_from_dict(response) return self async def store_async( self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + ) -> "Organization": """ Stores this organization in Synapse @@ -319,11 +315,11 @@ async def store_async( Raises: ValueError: If the name has not been set - Example: Store a new SchemaOrganization + Example: Store a new Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -332,7 +328,7 @@ async def store_org(): syn = Synapse() syn.login() - org = SchemaOrganization("my.new.org") + org = Organization("my.new.org") await org.store_async() return org @@ -342,7 +338,7 @@ async def store_org(): ``` """ if not self.name: - raise ValueError("SchemaOrganization must have a name") + raise ValueError("Organization must have a name") response = await create_organization(self.name, synapse_client=synapse_client) self.fill_from_dict(response) return self @@ -356,11 +352,11 @@ async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> N `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Delete a SchemaOrganization using a name + Example: Delete a Organization using a name   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -369,17 +365,17 @@ async def delete_org(): syn = Synapse() syn.login() - org = SchemaOrganization("my.org") + org = Organization("my.org") await org.delete_async() asyncio.run(delete_org()) ``` - Example: Delete a SchemaOrganization using an id + Example: Delete a Organization using an id   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -388,7 +384,7 @@ async def delete_org(): syn = Synapse() syn.login() - org = SchemaOrganization(id=1075) + org = Organization(id=1075) await org.delete_async() asyncio.run(delete_org()) @@ -417,11 +413,11 @@ async def get_json_schemas_async( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Get a list of JSONSchemas that belong to the SchemaOrganization + Example: Get a list of JSONSchemas that belong to the Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -430,7 +426,7 @@ async def get_schemas(): syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") js_generator = org.get_json_schemas_async() async for item in js_generator: print(item) @@ -440,7 +436,7 @@ async def get_schemas(): """ if not self.name: - raise ValueError("SchemaOrganization must have a name") + raise ValueError("Organization must have a name") response = list_json_schemas(self.name, synapse_client=synapse_client) async for item in response: yield JSONSchema().fill_from_dict(item) @@ -460,11 +456,11 @@ async def get_acl_async( A dictionary in the form of this response: [AccessControlList]https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/AccessControlList.html - Example: Get the ACL for a SchemaOrganization + Example: Get the ACL for a Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -473,7 +469,7 @@ async def get_acl(): syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") acl = await org.get_acl_async() return acl @@ -512,11 +508,11 @@ async def update_acl_async( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Update the ACL for a SchemaOrganization + Example: Update the ACL for a Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -525,7 +521,7 @@ async def update_acl() -> None: syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") await org.update_acl_async( principal_id=1, access_type=["READ"] @@ -557,7 +553,7 @@ async def update_acl() -> None: synapse_client=synapse_client, ) - def fill_from_dict(self, response: dict[str, Any]) -> "SchemaOrganization": + def fill_from_dict(self, response: dict[str, Any]) -> "Organization": """ Fills in this classes attributes using a Synapse API response @@ -881,7 +877,7 @@ async def get_schema(): # Check that the org exists, # if it doesn't list_json_schemas will unhelpfully return an empty generator. - org = SchemaOrganization(self.organization_name) + org = Organization(self.organization_name) await org.get_async(synapse_client=synapse_client) org_schemas = list_json_schemas( @@ -1396,9 +1392,9 @@ def _create_json_schema_version_from_response( ) -def list_json_schema_organizations( +def list_organizations( synapse_client: Optional["Synapse"] = None, -) -> list[SchemaOrganization]: +) -> list[Organization]: """ Lists all the Schema Organizations currently in Synapse @@ -1408,21 +1404,21 @@ def list_json_schema_organizations( instance from the Synapse class constructor Returns: - A list of SchemaOrganizations + A list of Organizations Example: - from synapseclient.models.schema_organization import list_json_schema_organizations + from synapseclient.models.organization import list_organizations from synapseclient import Synapse syn = Synapse() syn.login() - all_orgs = list_json_schema_organizations() + all_orgs = list_organizations() for item in all_orgs: print(item) """ all_orgs = [ - SchemaOrganization().fill_from_dict(org) + Organization().fill_from_dict(org) for org in list_organizations_sync(synapse_client=synapse_client) ] return all_orgs diff --git a/synapseclient/models/search_index.py b/synapseclient/models/search_index.py new file mode 100644 index 000000000..b8326e978 --- /dev/null +++ b/synapseclient/models/search_index.py @@ -0,0 +1,346 @@ +"""SearchIndex entity model. + +A SearchIndex is a Synapse entity whose content is defined by a Synapse SQL query +(`defining_sql`). An OpenSearch index is built from the query results, supporting +full-text search, faceted search, and autocomplete. +""" + +from collections import OrderedDict +from copy import deepcopy +from dataclasses import dataclass, field, replace +from datetime import date, datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.core.async_utils import async_to_sync +from synapseclient.core.constants import concrete_types +from synapseclient.core.utils import delete_none_keys +from synapseclient.models.activity import Activity +from synapseclient.models.mixins.access_control import AccessControllable +from synapseclient.models.mixins.table_components import ( + DeleteMixin, + GetMixin, + QueryMixin, + TableBase, + TableStoreMixin, +) +from synapseclient.models.table_components import Column + +if TYPE_CHECKING: + from synapseclient.models.search_management import SearchHit + + +class SearchIndexSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SearchIndex operations.""" + + def store( + self, + dry_run: bool = False, + *, + job_timeout: int = 600, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Store metadata about a SearchIndex including the annotations.""" + return self + + def get( + self, + include_columns: bool = True, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Get the metadata about the SearchIndex from Synapse.""" + return self + + def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: + """Delete the SearchIndex from Synapse.""" + return None + + def autocomplete( + self, + query: Dict[str, Any], + source: Optional[Dict[str, Any]] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SearchHit"]: + """Run a synchronous autocomplete search against this index.""" + return [] + + +@dataclass +@async_to_sync +class SearchIndex( + SearchIndexSynchronousProtocol, + AccessControllable, + TableBase, + TableStoreMixin, + DeleteMixin, + GetMixin, + QueryMixin, +): + """ + A SearchIndex is a Synapse entity whose content is defined by a Synapse SQL + query (`defining_sql`). An OpenSearch index is built from the query results, + supporting full-text search, faceted search, and autocomplete. + + The `defining_sql` must reference exactly one table-like entity. Multi-entity + JOIN/UNION queries are not supported. Optionally, a `search_configuration_id` + may be supplied to control the analyzer/synonym settings used when building + the index. If not specified, the configuration is resolved by walking up + the entity hierarchy. + + REST API model: + + Attributes: + id: The unique immutable ID for this entity. + name: The name of this entity. + description: The description of this entity. + etag: Synapse OCC etag. + created_on: Date this entity was created. + modified_on: Date this entity was last modified. + created_by: The ID of the user that created this entity. + modified_by: The ID of the user that last modified this entity. + parent_id: The ID of the parent entity. + version_number: The version number issued to this version on the object. + version_label: The version label for this entity. + version_comment: The version comment for this entity. + is_latest_version: If this is the latest version of the object. + columns: (Read Only) Columns derived from `defining_sql`. + defining_sql: The Synapse SQL statement that defines which columns and + rows are indexed. + search_configuration_id: ID of the SearchConfiguration to apply when + building this index. Optional. + annotations: Additional metadata associated with the entity. + activity: Provenance for this entity. + + Example: Create a new SearchIndex. + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex( + name="My Search Index", + parent_id="syn12345", + defining_sql="SELECT * FROM syn67890", + ) + index = index.store() + print(f"Created SearchIndex: {index.id}") + ``` + """ + + id: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + etag: Optional[str] = field(default=None, compare=False) + created_on: Optional[str] = field(default=None, compare=False) + modified_on: Optional[str] = field(default=None, compare=False) + created_by: Optional[str] = field(default=None, compare=False) + modified_by: Optional[str] = field(default=None, compare=False) + parent_id: Optional[str] = None + version_number: Optional[int] = field(default=None, compare=False) + version_label: Optional[str] = None + version_comment: Optional[str] = None + is_latest_version: Optional[bool] = field(default=None, compare=False) + + columns: Optional[OrderedDict[str, Column]] = field( + default_factory=OrderedDict, compare=False + ) + """(Read Only) Columns of a SearchIndex are derived from the defining SQL.""" + + defining_sql: Optional[str] = None + """The Synapse SQL statement that defines which columns and rows are indexed. + Must reference exactly one entity.""" + + search_configuration_id: Optional[str] = None + """The ID of the SearchConfiguration to apply when building this search + index. If not provided, the system will check for a search configuration + binding on the parent project/folder hierarchy, or use platform defaults.""" + + _last_persistent_instance: Optional["SearchIndex"] = field( + default=None, repr=False, compare=False + ) + + annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + + activity: Optional[Activity] = field(default=None, compare=False) + + @property + def has_changed(self) -> bool: + """Checks if the object has changed since the last persistent instance.""" + return self._last_persistent_instance != self + + def _set_last_persistent_instance(self) -> None: + """Stash the last time this object interacted with Synapse.""" + del self._last_persistent_instance + self._last_persistent_instance = replace(self) + self._last_persistent_instance.activity = ( + replace(self.activity) if self.activity and self.activity.id else None + ) + self._last_persistent_instance.annotations = ( + deepcopy(self.annotations) if self.annotations else {} + ) + + def fill_from_dict( + self, entity: Dict[str, Any], set_annotations: bool = True + ) -> "SearchIndex": + """Populate this dataclass from a Synapse REST API entity dict.""" + self.id = entity.get("id", None) + self.name = entity.get("name", None) + self.description = entity.get("description", None) + self.parent_id = entity.get("parentId", None) + self.etag = entity.get("etag", None) + self.created_on = entity.get("createdOn", None) + self.created_by = entity.get("createdBy", None) + self.modified_on = entity.get("modifiedOn", None) + self.modified_by = entity.get("modifiedBy", None) + self.version_number = entity.get("versionNumber", None) + self.version_label = entity.get("versionLabel", None) + self.version_comment = entity.get("versionComment", None) + self.is_latest_version = entity.get("isLatestVersion", None) + self.defining_sql = entity.get("definingSQL", None) + self.search_configuration_id = entity.get("searchConfigurationId", None) + + if set_annotations: + self.annotations = entity.get("annotations", {}) + + return self + + def to_synapse_request(self) -> Dict[str, Any]: + """Convert the request to the body expected by the Synapse REST API.""" + entity = { + "name": self.name, + "description": self.description, + "id": self.id, + "etag": self.etag, + "createdOn": self.created_on, + "modifiedOn": self.modified_on, + "createdBy": self.created_by, + "modifiedBy": self.modified_by, + "parentId": self.parent_id, + "concreteType": concrete_types.SEARCH_INDEX_ENTITY, + "versionNumber": self.version_number, + "versionLabel": self.version_label, + "versionComment": self.version_comment, + "isLatestVersion": self.is_latest_version, + "definingSQL": self.defining_sql, + "searchConfigurationId": self.search_configuration_id, + } + delete_none_keys(entity) + result = {"entity": entity} + delete_none_keys(result) + return result + + async def store_async( + self, + dry_run: bool = False, + *, + job_timeout: int = 600, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Asynchronously store the SearchIndex entity.""" + if not self.defining_sql: + raise ValueError( + "The defining_sql attribute must be set for a SearchIndex." + ) + return await super().store_async( + dry_run=dry_run, job_timeout=job_timeout, synapse_client=synapse_client + ) + + async def get_async( + self, + include_columns: bool = True, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Asynchronously fetch the SearchIndex metadata.""" + return await super().get_async( + include_columns=include_columns, + include_activity=include_activity, + synapse_client=synapse_client, + ) + + async def delete_async(self, *, synapse_client: Optional[Synapse] = None) -> None: + """Asynchronously delete this SearchIndex from Synapse.""" + await super().delete_async(synapse_client=synapse_client) + + async def autocomplete_async( + self, + query: Dict[str, Any], + source: Optional[Dict[str, Any]] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SearchHit"]: + """Run a synchronous autocomplete search against this index. The + autocomplete endpoint allowlists only prefix-style queries (`prefix`, + `match_phrase_prefix`, or `match_bool_prefix`) and caps results at 8. + + Arguments: + query: The top-level OpenSearch Query DSL clause; restricted + server-side to `prefix`, `match_phrase_prefix`, or + `match_bool_prefix`. + source: Optional source filter selecting which columns are returned + on each hit. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The matching SearchHits, capped at 8. + + Raises: + ValueError: If the ``id`` attribute has not been set. + + Example: Autocomplete titles beginning with "alz". + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex(id="syn12345") + hits = index.autocomplete( + query={"match_phrase_prefix": {"title": {"query": "alz"}}}, + ) + for hit in hits: + print(hit.row_id, hit.fields) + ``` + """ + from synapseclient.api import autocomplete_search + from synapseclient.models.search_management import ( + SearchAutocompleteRequest, + SearchHit, + ) + + if not self.id: + raise ValueError("The id attribute must be set to call autocomplete.") + request = SearchAutocompleteRequest( + search_index_id=self.id, + query=query, + source=source, + ) + response = await autocomplete_search( + request.to_synapse_request(), synapse_client=synapse_client + ) + return [SearchHit().fill_from_dict(h) for h in response.get("hits", []) or []] diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py new file mode 100644 index 000000000..b23dec5a2 --- /dev/null +++ b/synapseclient/models/search_management.py @@ -0,0 +1,939 @@ +"""Search management dataclasses. + +These dataclasses model the org-level search management resources used by +SearchIndex entities (TextAnalyzer, ColumnAnalyzerOverride, SynonymSet, +SearchConfiguration, SearchConfigBinding) and the query/response types for +querying a SearchIndex's OpenSearch index. + +The query model is a thin pass-through over the OpenSearch ``_search`` request +body: ``SearchQuery`` carries the allowlisted top-level keys (``query``, +``post_filter``, ``aggregations``, ``highlight``, ``collapse``, ``rescore``, +``sort``, ``_source``, ``from``, ``size``, ``search_after``) as raw JSON. The +analyzer-resource ``settings`` / ``definition`` / ``analyzer`` slots are likewise +raw OpenSearch JSON objects. + +Each analyzer resource belongs to an Organization and is referenced by qualified +name (``{organizationName}-{name}``). Resources are publicly readable; +create/update operations are restricted to Sage Bionetworks employees. + +REST controller: +""" + +from dataclasses import dataclass, field +from enum import Enum +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Protocol + +from typing_extensions import Self + +from synapseclient.api import ( + bind_search_config_to_entity, + clear_search_config_binding, + create_column_analyzer_override, + create_search_configuration, + create_synonym_set, + create_text_analyzer, + get_column_analyzer_override, + get_search_config_binding, + get_search_configuration, + get_synonym_set, + get_text_analyzer, + list_column_analyzer_overrides, + list_search_configurations, + list_synonym_sets, + list_text_analyzers, + update_column_analyzer_override, + update_search_configuration, + update_synonym_set, + update_text_analyzer, +) +from synapseclient.core.async_utils import async_to_sync +from synapseclient.core.constants import concrete_types +from synapseclient.core.utils import delete_none_keys +from synapseclient.models.mixins.asynchronous_job import AsynchronousCommunicator +from synapseclient.models.table_components import SelectColumn + +if TYPE_CHECKING: + from synapseclient import Synapse + +# ---------- Enums ---------- + + +class SearchIndexState(str, Enum): + """The state of a SearchIndex's OpenSearch index.""" + + CREATING = "CREATING" + ACTIVE = "ACTIVE" + FAILED = "FAILED" + + +class SearchQueryPart(str, Enum): + """Optional response parts for a SearchQuery beyond default HITS. + + These are values for the `responseParts` field on a SearchIndexQuery. + """ + + HITS = "HITS" + TOTAL_HITS = "TOTAL_HITS" + SELECT_COLUMNS = "SELECT_COLUMNS" + + +# ---------- Shared org-scoped resource base ---------- + + +class OrgScopedResourceProtocol(Protocol): + """Synchronous interface shared by the org-scoped search-management resources + (TextAnalyzer, ColumnAnalyzerOverride, SynonymSet, SearchConfiguration).""" + + def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Create this resource, or update it if it already has an ID.""" + return self + + def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Fetch this resource from Synapse by its ID.""" + return self + + def list( + self, + organization_name: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, + ) -> List["Self"]: + """List resources of this type, optionally filtered by organization.""" + return [] + + +@async_to_sync +class OrgScopedResource(OrgScopedResourceProtocol): + """Base implementing the create/get/update/list lifecycle shared by the + org-scoped search-management resources. + + Subclasses set the ``_CREATE_FN`` / ``_GET_FN`` / ``_UPDATE_FN`` / ``_LIST_FN`` + class attributes to the matching ``api.search_services`` functions and implement + ``fill_from_dict`` / ``to_synapse_request``. These resources have no delete + endpoint; ``name`` and ``organizationName`` are immutable after creation. + """ + + _CREATE_FN: Callable = None + _GET_FN: Callable = None + _UPDATE_FN: Callable = None + _LIST_FN: Callable = None + + async def store_async( + self, *, synapse_client: Optional["Synapse"] = None + ) -> "Self": + """Create this resource, or update it if it already has an ID. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated with the server-assigned ID, etag, and timestamps. + """ + cls = type(self) + if self.id: + result = await cls._UPDATE_FN( + self.id, self.to_synapse_request(), synapse_client=synapse_client + ) + else: + result = await cls._CREATE_FN( + self.to_synapse_request(), synapse_client=synapse_client + ) + return self.fill_from_dict(result) + + async def get_async(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Fetch this resource from Synapse by its ID. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the Synapse response. + + Raises: + ValueError: If the ``id`` attribute has not been set. + """ + if not self.id: + raise ValueError(f"{type(self).__name__} must have an id set to call get.") + cls = type(self) + result = await cls._GET_FN(self.id, synapse_client=synapse_client) + return self.fill_from_dict(result) + + @classmethod + async def list_async( + cls, + organization_name: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, + ) -> List["Self"]: + """List resources of this type, paginating over all pages. + + Arguments: + organization_name: If provided, only resources in this organization are + returned; otherwise resources across all organizations are listed. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of every matching resource across all result pages. + """ + results: List["Self"] = [] + next_page_token = None + while True: + page = await cls._LIST_FN( + organization_name=organization_name, + next_page_token=next_page_token, + synapse_client=synapse_client, + ) + for item in page.get("results", []) or []: + results.append(cls().fill_from_dict(item)) + next_page_token = page.get("nextPageToken", None) + if not next_page_token: + break + return results + + +# ---------- Text Analyzer ---------- + + +@dataclass +class TextAnalyzer(OrgScopedResource): + """A shareable, named OpenSearch custom analyzer. Used to configure how text + is tokenized for a search index. + + REST: + """ + + _CREATE_FN = staticmethod(create_text_analyzer) + _GET_FN = staticmethod(get_text_analyzer) + _UPDATE_FN = staticmethod(update_text_analyzer) + _LIST_FN = staticmethod(list_text_analyzers) + + id: Optional[str] = None + organization_name: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + settings: Optional[Dict[str, Any]] = None + """Required. JSON object holding the *contents of* the `settings.analysis` + block of an OpenSearch create-index request body. Allowed root keys are + `char_filter`, `tokenizer`, `filter`, and `analyzer`; the inner `analyzer` + map must declare exactly one `default` entry (and optionally + `default_search`). A `{"$ref": "{org}-{name}"}` entry inside the `filter` + registry resolves to a SynonymSet at index-build time.""" + + etag: Optional[str] = None + created_on: Optional[str] = None + created_by: Optional[str] = None + modified_on: Optional[str] = None + modified_by: Optional[str] = None + + @property + def qualified_name(self) -> Optional[str]: + """The qualified name '{organizationName}-{name}' used to reference + this analyzer from a SearchConfiguration.""" + if self.organization_name and self.name: + return f"{self.organization_name}-{self.name}" + return None + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.id = data.get("id", None) + self.organization_name = data.get("organizationName", None) + self.name = data.get("name", None) + self.description = data.get("description", None) + self.settings = data.get("settings", None) + self.etag = data.get("etag", None) + self.created_on = data.get("createdOn", None) + self.created_by = data.get("createdBy", None) + self.modified_on = data.get("modifiedOn", None) + self.modified_by = data.get("modifiedBy", None) + return self + + def to_synapse_request(self) -> Dict[str, Any]: + body = { + "id": self.id, + "organizationName": self.organization_name, + "name": self.name, + "description": self.description, + "settings": self.settings, + "etag": self.etag, + } + delete_none_keys(body) + return body + + +# ---------- Column Analyzer Override ---------- + + +@dataclass +class ColumnAnalyzerOverrideEntry: + """Assigns one TextAnalyzer to one column. + + REST: + """ + + column_name: Optional[str] = None + """The name of the column to override.""" + + analyzer: Optional[Dict[str, Any]] = None + """The analyzer to use for this column. Either a reference to a saved + TextAnalyzer written as `{"$ref": "{organizationName}-{name}"}`, or an + inline OpenSearch `settings.analysis` block.""" + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.column_name = data.get("columnName", None) + self.analyzer = data.get("analyzer", None) + return self + + def to_synapse_request(self) -> Dict[str, Any]: + body = { + "columnName": self.column_name, + "analyzer": self.analyzer, + } + delete_none_keys(body) + return body + + +@dataclass +class ColumnAnalyzerOverride(OrgScopedResource): + """A shared resource containing per-column analyzer override entries. + + REST: + """ + + _CREATE_FN = staticmethod(create_column_analyzer_override) + _GET_FN = staticmethod(get_column_analyzer_override) + _UPDATE_FN = staticmethod(update_column_analyzer_override) + _LIST_FN = staticmethod(list_column_analyzer_overrides) + + id: Optional[str] = None + organization_name: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + overrides: Optional[List[ColumnAnalyzerOverrideEntry]] = field(default_factory=list) + etag: Optional[str] = None + created_on: Optional[str] = None + created_by: Optional[str] = None + modified_on: Optional[str] = None + modified_by: Optional[str] = None + + @property + def qualified_name(self) -> Optional[str]: + if self.organization_name and self.name: + return f"{self.organization_name}-{self.name}" + return None + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.id = data.get("id", None) + self.organization_name = data.get("organizationName", None) + self.name = data.get("name", None) + self.description = data.get("description", None) + self.overrides = [ + ColumnAnalyzerOverrideEntry().fill_from_dict(o) + for o in data.get("overrides", []) or [] + ] + self.etag = data.get("etag", None) + self.created_on = data.get("createdOn", None) + self.created_by = data.get("createdBy", None) + self.modified_on = data.get("modifiedOn", None) + self.modified_by = data.get("modifiedBy", None) + return self + + def to_synapse_request(self) -> Dict[str, Any]: + body = { + "id": self.id, + "organizationName": self.organization_name, + "name": self.name, + "description": self.description, + "overrides": ( + [o.to_synapse_request() for o in self.overrides] + if self.overrides + else None + ), + "etag": self.etag, + } + delete_none_keys(body) + return body + + +# ---------- Synonym Set ---------- + + +@dataclass +class SynonymSet(OrgScopedResource): + """A shareable OpenSearch synonym_graph (or legacy synonym) token filter. + Referenced by qualified name `{organizationName}-{name}` from a TextAnalyzer's + `settings.filter` registry map via `{"$ref": "{organizationName}-{name}"}`. + + REST: + """ + + _CREATE_FN = staticmethod(create_synonym_set) + _GET_FN = staticmethod(get_synonym_set) + _UPDATE_FN = staticmethod(update_synonym_set) + _LIST_FN = staticmethod(list_synonym_sets) + + id: Optional[str] = None + organization_name: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + definition: Optional[Dict[str, Any]] = None + """Required. The full OpenSearch token filter definition as a JSON object, + exactly as documented for the synonym_graph / synonym token filters, e.g. + `{"type": "synonym_graph", "synonyms": ["tumor, neoplasm, cancer", + "AD => Alzheimer's disease"]}`.""" + etag: Optional[str] = None + created_on: Optional[str] = None + created_by: Optional[str] = None + modified_on: Optional[str] = None + modified_by: Optional[str] = None + + @property + def qualified_name(self) -> Optional[str]: + if self.organization_name and self.name: + return f"{self.organization_name}-{self.name}" + return None + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.id = data.get("id", None) + self.organization_name = data.get("organizationName", None) + self.name = data.get("name", None) + self.description = data.get("description", None) + self.definition = data.get("definition", None) + self.etag = data.get("etag", None) + self.created_on = data.get("createdOn", None) + self.created_by = data.get("createdBy", None) + self.modified_on = data.get("modifiedOn", None) + self.modified_by = data.get("modifiedBy", None) + return self + + def to_synapse_request(self) -> Dict[str, Any]: + body = { + "id": self.id, + "organizationName": self.organization_name, + "name": self.name, + "description": self.description, + "definition": self.definition, + "etag": self.etag, + } + delete_none_keys(body) + return body + + +# ---------- Search Configuration ---------- + + +@dataclass +class SearchConfiguration(OrgScopedResource): + """Bundles the index-wide default analyzer and per-column overrides used to + build a SearchIndex. + + REST: + """ + + _CREATE_FN = staticmethod(create_search_configuration) + _GET_FN = staticmethod(get_search_configuration) + _UPDATE_FN = staticmethod(update_search_configuration) + _LIST_FN = staticmethod(list_search_configurations) + + id: Optional[str] = None + organization_name: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + default_analyzer: Optional[Dict[str, Any]] = None + """Optional. The analyzer that supplies this index's `analysis.analyzer.default` + slot. Either a reference to a saved TextAnalyzer written as + `{"$ref": "{organizationName}-{name}"}`, or an inline OpenSearch + `settings.analysis` block.""" + column_analyzer_overrides: Optional[List[Dict[str, Any]]] = field( + default_factory=list + ) + """Optional ordered list of ColumnAnalyzerOverride entries. Each entry is + either a reference `{"$ref": "{organizationName}-{name}"}` or an inline + ColumnAnalyzerOverride literal.""" + etag: Optional[str] = None + created_on: Optional[str] = None + created_by: Optional[str] = None + modified_on: Optional[str] = None + modified_by: Optional[str] = None + + @property + def qualified_name(self) -> Optional[str]: + if self.organization_name and self.name: + return f"{self.organization_name}-{self.name}" + return None + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.id = data.get("id", None) + self.organization_name = data.get("organizationName", None) + self.name = data.get("name", None) + self.description = data.get("description", None) + self.default_analyzer = data.get("defaultAnalyzer", None) + self.column_analyzer_overrides = data.get("columnAnalyzerOverrides", []) or [] + self.etag = data.get("etag", None) + self.created_on = data.get("createdOn", None) + self.created_by = data.get("createdBy", None) + self.modified_on = data.get("modifiedOn", None) + self.modified_by = data.get("modifiedBy", None) + return self + + def to_synapse_request(self) -> Dict[str, Any]: + body = { + "id": self.id, + "organizationName": self.organization_name, + "name": self.name, + "description": self.description, + "defaultAnalyzer": self.default_analyzer, + "columnAnalyzerOverrides": self.column_analyzer_overrides or None, + "etag": self.etag, + } + delete_none_keys(body) + return body + + +# ---------- Search Config Binding ---------- + + +class SearchConfigBindingProtocol(Protocol): + """Synchronous interface for SearchConfigBinding operations.""" + + def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Bind ``search_configuration_id`` to the entity ``object_id``.""" + return self + + def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Get the effective binding for the entity ``object_id``.""" + return self + + def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: + """Clear the binding on the entity ``object_id``.""" + return None + + +@dataclass +@async_to_sync +class SearchConfigBinding(SearchConfigBindingProtocol): + """A binding between a SearchConfiguration and an entity. + + Effective configuration for an entity is resolved by walking up the + hierarchy (entity -> folder -> project). + """ + + bind_id: Optional[str] = None + search_configuration_id: Optional[str] = None + object_id: Optional[str] = None + """The ID of the entity the SearchConfiguration is bound to.""" + object_type: Optional[str] = None + created_by: Optional[str] = None + created_on: Optional[str] = None + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.bind_id = data.get("bindId", None) + self.search_configuration_id = data.get("searchConfigurationId", None) + self.object_id = data.get("objectId", None) + self.object_type = data.get("objectType", None) + self.created_by = data.get("createdBy", None) + self.created_on = data.get("createdOn", None) + return self + + async def store_async( + self, *, synapse_client: Optional["Synapse"] = None + ) -> "Self": + """Bind ``search_configuration_id`` to the entity ``object_id``. Replaces + any existing binding on that entity. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the created SearchConfigBinding. + + Raises: + ValueError: If ``object_id`` or ``search_configuration_id`` is not set. + """ + if not self.object_id: + raise ValueError("SearchConfigBinding must have an object_id set.") + if not self.search_configuration_id: + raise ValueError( + "SearchConfigBinding must have a search_configuration_id set." + ) + result = await bind_search_config_to_entity( + self.object_id, + self.search_configuration_id, + synapse_client=synapse_client, + ) + return self.fill_from_dict(result) + + async def get_async(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Get the effective binding for the entity ``object_id``, resolved by + walking up the entity hierarchy. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the resolved SearchConfigBinding. + + Raises: + ValueError: If ``object_id`` is not set. + """ + if not self.object_id: + raise ValueError("SearchConfigBinding must have an object_id set.") + result = await get_search_config_binding( + self.object_id, synapse_client=synapse_client + ) + return self.fill_from_dict(result) + + async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> None: + """Clear the binding on the entity ``object_id``. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Raises: + ValueError: If ``object_id`` is not set. + """ + if not self.object_id: + raise ValueError("SearchConfigBinding must have an object_id set.") + await clear_search_config_binding(self.object_id, synapse_client=synapse_client) + + +# ---------- Search Index Status ---------- + + +@dataclass +class SearchIndexStatus: + """The build status of a SearchIndex's OpenSearch index.""" + + search_index_id: Optional[str] = None + state: Optional[SearchIndexState] = None + changed_on: Optional[str] = None + error_message: Optional[str] = None + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.search_index_id = data.get("searchIndexId", None) + st = data.get("state", None) + self.state = SearchIndexState(st) if st else None + self.changed_on = data.get("changedOn", None) + self.error_message = data.get("errorMessage", None) + return self + + +# ---------- Search Query ---------- + + +@dataclass +class SearchQuery: + """The body of an OpenSearch `_search` request, narrowed to the top-level + keys Synapse accepts. Each slot's contents are pass-through OpenSearch query + DSL carried as raw JSON. + + REST: + """ + + query: Optional[Dict[str, Any]] = None + """Required. The OpenSearch query DSL clause. Use `{"match_all": {}}` to + match all documents.""" + + post_filter: Optional[Dict[str, Any]] = None + """Optional. Same DSL shape as `query`, applied after aggregations are + computed.""" + + aggregations: Optional[Dict[str, Any]] = None + """Optional. Map of caller-chosen name to aggregation definition. The raw + aggregation result comes back on `SearchIndexQuery.aggregation_results`.""" + + highlight: Optional[Dict[str, Any]] = None + """Optional. Adds per-field snippet fragments to each hit's highlights.""" + + collapse: Optional[Dict[str, Any]] = None + """Optional. Groups the result list so only one hit is returned per distinct + value of a field.""" + + rescore: Optional[Dict[str, Any]] = None + """Optional. Re-ranks the top hits returned by `query` using a secondary + scoring query.""" + + sort: Optional[List[Any]] = None + """Optional. Result ordering, in native OpenSearch sort shape (a string + column name, `{column: "asc|desc"}`, or `{column: {order: ..., mode: ...}}`). + When omitted, results are sorted by relevance descending.""" + + source: Optional[Dict[str, Any]] = None + """Optional. Source filter selecting which columns are returned on each hit. + Serialized as `_source`.""" + + from_: Optional[int] = None + """Optional. Zero-based pagination offset; default 0. Ignored when + `search_after` is supplied. Serialized as `from`.""" + + size: Optional[int] = None + """Optional. Maximum number of hits to return per page. Default 25. + Maximum 100.""" + + search_after: Optional[List[Any]] = None + """Optional. Opaque cursor emitted as `next_search_after` on the previous + response. Pass back unchanged. When supplied, `from_` is ignored.""" + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.query = data.get("query", None) + self.post_filter = data.get("post_filter", None) + self.aggregations = data.get("aggregations", None) + self.highlight = data.get("highlight", None) + self.collapse = data.get("collapse", None) + self.rescore = data.get("rescore", None) + self.sort = data.get("sort", None) + self.source = data.get("_source", None) + self.from_ = data.get("from", None) + self.size = data.get("size", None) + self.search_after = data.get("search_after", None) + return self + + def to_synapse_request(self) -> Dict[str, Any]: + body = { + "query": self.query, + "post_filter": self.post_filter, + "aggregations": self.aggregations, + "highlight": self.highlight, + "collapse": self.collapse, + "rescore": self.rescore, + "sort": self.sort, + "_source": self.source, + "from": self.from_, + "size": self.size, + "search_after": self.search_after, + } + delete_none_keys(body) + return body + + +# ---------- Autocomplete ---------- + + +@dataclass +class SearchAutocompleteRequest: + """Body of a synchronous autocomplete request against a SearchIndex. The + autocomplete endpoint allowlists only `query` (restricted to `prefix`, + `match_phrase_prefix`, or `match_bool_prefix`) and `_source`. + + REST: + """ + + search_index_id: Optional[str] = None + """The ID of the SearchIndex entity to query.""" + + query: Optional[Dict[str, Any]] = None + """Required. The top-level Query DSL clause; restricted server-side to + `prefix`, `match_phrase_prefix`, or `match_bool_prefix`.""" + + source: Optional[Dict[str, Any]] = None + """Optional. Source filter; same shape as `SearchQuery.source`. Serialized + as `_source`.""" + + def to_synapse_request(self) -> Dict[str, Any]: + search_query = { + "query": self.query, + "_source": self.source, + } + delete_none_keys(search_query) + body = { + "searchIndexId": self.search_index_id, + "searchQuery": search_query or None, + } + delete_none_keys(body) + return body + + +# ---------- Search Results ---------- + + +@dataclass +class SearchFieldValue: + """A name/value pair returned in a SearchHit's `fields`. + + REST: + """ + + name: Optional[str] = None + """The column name.""" + + value: Optional[str] = None + """The column value.""" + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.name = data.get("name", None) + self.value = data.get("value", None) + return self + + +@dataclass +class SearchHighlight: + """A per-field highlight payload on a SearchHit. + + REST: + """ + + name: Optional[str] = None + """The column name.""" + + snippets: Optional[List[str]] = field(default_factory=list) + """Highlighted snippet fragments. Matched terms are wrapped in pre/post tags + (default `` / ``).""" + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.name = data.get("name", None) + self.snippets = data.get("snippets", []) or [] + return self + + +@dataclass +class SearchHit: + """A single matching document in a SearchQueryResults response. + + REST: + """ + + row_id: Optional[int] = None + """The row ID from the source table.""" + + row_version: Optional[int] = None + """The row version from the source table.""" + + score: Optional[float] = None + """The relevance score for this hit.""" + + fields: Optional[List[SearchFieldValue]] = field(default_factory=list) + """Column name/value pairs for the requested return fields.""" + + highlights: Optional[List[SearchHighlight]] = field(default_factory=list) + """Per-field highlight payloads, if highlight was requested.""" + + def fill_from_dict(self, data: Dict[str, Any]) -> "Self": + self.row_id = data.get("rowId", None) + self.row_version = data.get("rowVersion", None) + self.score = data.get("score", None) + self.fields = [ + SearchFieldValue().fill_from_dict(f) for f in data.get("fields", []) or [] + ] + self.highlights = [ + SearchHighlight().fill_from_dict(h) + for h in data.get("highlights", []) or [] + ] + return self + + +@dataclass +class SearchIndexQuery(AsynchronousCommunicator): + """An async request to query a SearchIndex's OpenSearch index. + + Inherits from `AsynchronousCommunicator`: call `send_job_and_wait_async()` to + submit the job, poll the Synapse async job service, and populate response + fields (`hits`, `total_hits`, `select_columns`, `aggregation_results`, + `next_search_after`, `offset`) on this same instance. + + REST: + + Example: Run a search query. + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import SearchIndexQuery, SearchQuery, SearchQueryPart + + async def main(): + Synapse().login() + query = SearchIndexQuery( + search_index_id="syn22806626", + search_query=SearchQuery( + query={"match": {"title": {"query": "alzheimer"}}}, + size=10, + ), + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], + ) + await query.send_job_and_wait_async() + print(query.total_hits, len(query.hits)) + + asyncio.run(main()) + ``` + """ + + concrete_type: str = concrete_types.SEARCH_INDEX_QUERY + """The Synapse concrete type identifying this async request.""" + + search_index_id: Optional[str] = None + """The ID of the SearchIndex entity to query.""" + + search_query: Optional[SearchQuery] = None + """The SearchQuery (OpenSearch `_search` body) to execute against the index.""" + + response_parts: Optional[List[SearchQueryPart]] = field(default_factory=list) + """Optional list of additional response parts beyond default HITS.""" + + hits: Optional[List[SearchHit]] = field(default_factory=list) + """Response: matching documents. Populated after `send_job_and_wait_async()`.""" + + total_hits: Optional[int] = None + """Response: total number of matching documents. Populated when + SearchQueryPart.TOTAL_HITS is requested.""" + + select_columns: Optional[List[SelectColumn]] = field(default_factory=list) + """Response: columns represented in each hit's fields, in SELECT-clause + order. Populated when SearchQueryPart.SELECT_COLUMNS is requested.""" + + aggregation_results: Optional[Dict[str, Any]] = None + """Response: the raw OpenSearch aggregations response, with field references + rewritten back to column names. Populated whenever the request supplied + `search_query.aggregations`. Kept as an opaque JSON object.""" + + next_search_after: Optional[List[Any]] = None + """Response: opaque cursor for the next page. Pass back unchanged on the next + request as `search_query.search_after`. Null when there are no further pages.""" + + offset: Optional[int] = None + """Response: zero-based pagination offset echoed from the request.""" + + def to_synapse_request(self) -> Dict[str, Any]: + """Convert to the SearchIndexQuery body for the async-job /start endpoint.""" + body = { + "concreteType": self.concrete_type, + "searchIndexId": self.search_index_id, + "searchQuery": ( + self.search_query.to_synapse_request() if self.search_query else None + ), + "responseParts": ( + [p.value for p in self.response_parts] if self.response_parts else None + ), + } + delete_none_keys(body) + return body + + def fill_from_dict(self, synapse_response: Dict[str, Any]) -> "Self": + """Populate response fields from a SearchQueryResults body. + + Called by `AsynchronousCommunicator.send_job_and_wait_async()` once the + async job completes. Leaves request fields untouched. + + REST: + """ + self.hits = [ + SearchHit().fill_from_dict(h) + for h in synapse_response.get("hits", []) or [] + ] + self.total_hits = synapse_response.get("totalHits", None) + self.select_columns = [ + SelectColumn.fill_from_dict(c) + for c in synapse_response.get("selectColumns", []) or [] + ] + self.aggregation_results = synapse_response.get("aggregationResults", None) + self.next_search_after = synapse_response.get("nextSearchAfter", None) + self.offset = synapse_response.get("offset", None) + return self diff --git a/synapseclient/operations/delete_operations.py b/synapseclient/operations/delete_operations.py index 304780c8b..139ed8f45 100644 --- a/synapseclient/operations/delete_operations.py +++ b/synapseclient/operations/delete_operations.py @@ -20,9 +20,9 @@ Grid, JSONSchema, MaterializedView, + Organization, Project, RecordSet, - SchemaOrganization, SubmissionView, Table, Team, @@ -45,7 +45,7 @@ def delete( "MaterializedView", "Project", "RecordSet", - "SchemaOrganization", + "Organization", "SubmissionView", "Table", "Team", @@ -89,7 +89,7 @@ def delete( **Not supported for version-specific deletion:** - - Project, Folder, Evaluation, Team, SchemaOrganization, CurationTask, Grid + - Project, Folder, Evaluation, Team, Organization, CurationTask, Grid Arguments: entity: The entity instance to delete, or a Synapse ID string (e.g., "syn123456" @@ -274,7 +274,7 @@ async def delete_async( "MaterializedView", "Project", "RecordSet", - "SchemaOrganization", + "Organization", "SubmissionView", "Table", "Team", @@ -319,7 +319,7 @@ async def delete_async( **Not supported for version-specific deletion:** - - Project, Folder, Evaluation, Team, SchemaOrganization, CurationTask, Grid + - Project, Folder, Evaluation, Team, Organization, CurationTask, Grid Arguments: entity: The entity instance to delete, or a Synapse ID string (e.g., "syn123456" @@ -443,9 +443,9 @@ async def main(): Grid, JSONSchema, MaterializedView, + Organization, Project, RecordSet, - SchemaOrganization, SubmissionView, Table, Team, @@ -570,7 +570,7 @@ async def main(): Project, Evaluation, Team, - SchemaOrganization, + Organization, CurationTask, Grid, DockerRepository, @@ -600,5 +600,5 @@ async def main(): f"Unsupported entity type: {type(entity).__name__}. " "Supported types are: str (Synapse ID), CurationTask, Dataset, DatasetCollection, " "EntityView, Evaluation, File, Folder, Grid, JSONSchema, MaterializedView, " - "Project, RecordSet, SchemaOrganization, SubmissionView, Table, Team, VirtualTable." + "Project, RecordSet, Organization, SubmissionView, Table, Team, VirtualTable." ) diff --git a/synapseclient/operations/store_operations.py b/synapseclient/operations/store_operations.py index dc50b60c0..a61d68053 100644 --- a/synapseclient/operations/store_operations.py +++ b/synapseclient/operations/store_operations.py @@ -23,9 +23,9 @@ JSONSchema, Link, MaterializedView, + Organization, Project, RecordSet, - SchemaOrganization, SubmissionView, Table, Team, @@ -242,7 +242,7 @@ def store( "MaterializedView", "Project", "RecordSet", - "SchemaOrganization", + "Organization", "SubmissionView", "Table", "Team", @@ -273,7 +273,7 @@ def store( "MaterializedView", "Project", "RecordSet", - "SchemaOrganization", + "Organization", "SubmissionView", "Table", "Team", @@ -534,7 +534,7 @@ async def store_async( "MaterializedView", "Project", "RecordSet", - "SchemaOrganization", + "Organization", "SubmissionView", "Table", "Team", @@ -565,7 +565,7 @@ async def store_async( "MaterializedView", "Project", "RecordSet", - "SchemaOrganization", + "Organization", "SubmissionView", "Table", "Team", @@ -773,9 +773,9 @@ async def main(): JSONSchema, Link, MaterializedView, + Organization, Project, RecordSet, - SchemaOrganization, SubmissionView, Table, Team, @@ -833,7 +833,7 @@ async def main(): else: return await entity.create_async(synapse_client=synapse_client) - elif isinstance(entity, SchemaOrganization): + elif isinstance(entity, Organization): return await entity.store_async(synapse_client=synapse_client) elif isinstance(entity, CurationTask): @@ -875,5 +875,5 @@ async def main(): "Supported types are: AgentSession, CurationTask, " "Dataset, DatasetCollection, EntityView, Evaluation, File, Folder, FormData, " "FormGroup, Grid, JSONSchema, Link, MaterializedView, Project, RecordSet, " - "SchemaOrganization, SubmissionView, Table, Team, VirtualTable." + "Organization, SubmissionView, Table, Team, VirtualTable." ) diff --git a/synapseclient/services/json_schema.py b/synapseclient/services/json_schema.py index d001d5353..4e1339778 100644 --- a/synapseclient/services/json_schema.py +++ b/synapseclient/services/json_schema.py @@ -5,7 +5,7 @@ !!! warning Everything in this module has been deprecated. - Use synapseclient.models.SchemaOrganization and synapseclient.models.JSONSchema instead. + Use synapseclient.models.Organization and synapseclient.models.JSONSchema instead. """ from __future__ import annotations @@ -285,7 +285,7 @@ def create( @deprecated( version="4.11.0", - reason="To be removed in 5.0.0. Use synapseclient.models.SchemaOrganization instead.", + reason="To be removed in 5.0.0. Use synapseclient.models.Organization instead.", ) class JsonSchemaOrganization: """Json Schema Organization diff --git a/tests/integration/synapseclient/extensions/curator/test_schema_management.py b/tests/integration/synapseclient/extensions/curator/test_schema_management.py index ccbe1c31b..b3ae69ef5 100644 --- a/tests/integration/synapseclient/extensions/curator/test_schema_management.py +++ b/tests/integration/synapseclient/extensions/curator/test_schema_management.py @@ -12,7 +12,7 @@ from synapseclient.extensions.curator.record_based_metadata_task import ( project_id_from_entity_id, ) -from synapseclient.models import Folder, Project, SchemaOrganization +from synapseclient.models import Folder, Organization, Project def create_test_name(): @@ -22,11 +22,11 @@ def create_test_name(): @pytest.fixture(name="test_organization", scope="module") -def fixture_test_organization(syn: Synapse, request) -> SchemaOrganization: +def fixture_test_organization(syn: Synapse, request) -> Organization: """ Returns a created organization for testing schema registration """ - org = SchemaOrganization(create_test_name()) + org = Organization(create_test_name()) org.store(synapse_client=syn) def delete_org(): @@ -87,7 +87,7 @@ class TestRegisterJsonSchema: """Integration tests for register_jsonschema wrapper function""" def test_register_jsonschema_with_version( - self, syn: Synapse, test_organization: SchemaOrganization, test_schema_file: str + self, syn: Synapse, test_organization: Organization, test_schema_file: str ): """Test registering a JSON schema with a specific version""" schema_name = create_test_name() @@ -109,7 +109,7 @@ def test_register_jsonschema_with_version( assert test_organization.name in json_schema.uri def test_register_jsonschema_without_version( - self, syn: Synapse, test_organization: SchemaOrganization, test_schema_file: str + self, syn: Synapse, test_organization: Organization, test_schema_file: str ): """Test registering a JSON schema without specifying a version""" schema_name = create_test_name() @@ -134,7 +134,7 @@ class TestBindJsonSchema: def test_bind_jsonschema_to_folder( self, syn: Synapse, - test_organization: SchemaOrganization, + test_organization: Organization, test_project: Project, test_schema_file: str, ): @@ -172,7 +172,7 @@ def test_bind_jsonschema_to_folder( def test_bind_jsonschema_with_derived_annotations( self, syn: Synapse, - test_organization: SchemaOrganization, + test_organization: Organization, test_project: Project, test_schema_file: str, ): @@ -214,7 +214,7 @@ class TestRegisterAndBindWorkflow: def test_complete_workflow( self, syn: Synapse, - test_organization: SchemaOrganization, + test_organization: Organization, test_project: Project, test_schema_file: str, ): diff --git a/tests/integration/synapseclient/models/async/test_schema_organization_async.py b/tests/integration/synapseclient/models/async/test_organization_async.py similarity index 93% rename from tests/integration/synapseclient/models/async/test_schema_organization_async.py rename to tests/integration/synapseclient/models/async/test_organization_async.py index f7461c732..97356c9fd 100644 --- a/tests/integration/synapseclient/models/async/test_schema_organization_async.py +++ b/tests/integration/synapseclient/models/async/test_organization_async.py @@ -1,4 +1,4 @@ -"""Integration tests for SchemaOrganization and JSONSchema classes""" +"""Integration tests for Organization and JSONSchema classes""" import asyncio import uuid @@ -10,11 +10,11 @@ from synapseclient import Synapse from synapseclient.core.constants.concrete_types import CREATE_SCHEMA_REQUEST from synapseclient.core.exceptions import SynapseHTTPError -from synapseclient.models import JSONSchema, SchemaOrganization -from synapseclient.models.schema_organization import ( +from synapseclient.models import JSONSchema, Organization +from synapseclient.models.organization import ( CreateSchemaRequest, JSONSchemaVersionInfo, - list_json_schema_organizations, + list_organizations, ) @@ -41,19 +41,19 @@ def org_exists(name: str, synapse_client: Optional[Synapse] = None) -> bool: """ matching_orgs = [ org - for org in list_json_schema_organizations(synapse_client=synapse_client) + for org in list_organizations(synapse_client=synapse_client) if org.name == name ] return len(matching_orgs) == 1 @pytest.fixture(name="module_organization", scope="module") -def fixture_module_organization(syn: Synapse, request) -> SchemaOrganization: +def fixture_module_organization(syn: Synapse, request) -> Organization: """ Returns a created organization at the module scope. Used to hold JSON Schemas created by tests. """ name = create_test_entity_name() - org = SchemaOrganization(name) + org = Organization(name) org.store(synapse_client=syn) def delete_org(): @@ -67,7 +67,7 @@ def delete_org(): @pytest.fixture(name="json_schema", scope="function") -def fixture_json_schema(module_organization: SchemaOrganization) -> JSONSchema: +def fixture_json_schema(module_organization: Organization) -> JSONSchema: """ Returns a JSON Schema """ @@ -77,12 +77,12 @@ def fixture_json_schema(module_organization: SchemaOrganization) -> JSONSchema: @pytest_asyncio.fixture(name="organization", loop_scope="function", scope="function") -async def fixture_organization(syn: Synapse, request) -> SchemaOrganization: +async def fixture_organization(syn: Synapse, request) -> Organization: """ Returns a Synapse organization. """ name = create_test_entity_name() - org = SchemaOrganization(name) + org = Organization(name) def delete_org(): exists = org_exists(name, syn) @@ -97,13 +97,13 @@ def delete_org(): @pytest_asyncio.fixture( name="organization_with_schema", loop_scope="function", scope="function" ) -async def fixture_organization_with_schema(syn: Synapse, request) -> SchemaOrganization: +async def fixture_organization_with_schema(syn: Synapse, request) -> Organization: """ Returns a Synapse organization. As Cleanup it checks for JSON Schemas and deletes them""" name = create_test_entity_name() - org = SchemaOrganization(name) + org = Organization(name) await org.store_async(synapse_client=syn) js1 = JSONSchema("schema1", name) js2 = JSONSchema("schema2", name) @@ -122,14 +122,14 @@ def delete_org(): return org -class TestSchemaOrganization: - """Asynchronous integration tests for SchemaOrganization.""" +class TestOrganization: + """Asynchronous integration tests for Organization.""" @pytest.fixture(autouse=True, scope="function") def init(self, syn: Synapse) -> None: self.syn = syn - async def test_create_and_get(self, organization: SchemaOrganization) -> None: + async def test_create_and_get(self, organization: Organization) -> None: # GIVEN an initialized organization object that hasn't been stored in Synapse # THEN it shouldn't have any metadata besides it's name assert organization.name is not None @@ -150,7 +150,7 @@ async def test_create_and_get(self, organization: SchemaOrganization) -> None: exists = org_exists(organization.name, synapse_client=self.syn) assert exists # AND it should be getable by future instances with the same name - org2 = SchemaOrganization(organization.name) + org2 = Organization(organization.name) await org2.get_async(synapse_client=self.syn) assert organization.name is not None assert organization.id is not None @@ -163,8 +163,8 @@ async def test_create_and_get(self, organization: SchemaOrganization) -> None: async def test_get_json_schemas_async( self, - organization: SchemaOrganization, - organization_with_schema: SchemaOrganization, + organization: Organization, + organization_with_schema: Organization, ) -> None: # GIVEN an organization with no schemas and one with 3 schemas await organization.store_async(synapse_client=self.syn) @@ -181,9 +181,7 @@ async def test_get_json_schemas_async( schema_list2.append(item) assert len(schema_list2) == 3 - async def test_get_acl_and_update_acl( - self, organization: SchemaOrganization - ) -> None: + async def test_get_acl_and_update_acl(self, organization: Organization) -> None: # GIVEN an organization that has been initialized, but not created # THEN get_acl should raise an error with pytest.raises( @@ -243,7 +241,7 @@ async def test_store_and_get(self, json_schema: JSONSchema) -> None: assert js2.created_by assert js2.created_on - async def test_delete(self, organization_with_schema: SchemaOrganization) -> None: + async def test_delete(self, organization_with_schema: Organization) -> None: # GIVEN an organization with 3 schema schemas: list[JSONSchema] = [] async for item in organization_with_schema.get_json_schemas_async( @@ -361,7 +359,7 @@ def init(self, syn: Synapse) -> None: self.syn = syn async def test_create_schema_request_no_version( - self, module_organization: SchemaOrganization + self, module_organization: Organization ) -> None: # GIVEN an organization # WHEN creating a CreateSchemaRequest with no version given @@ -408,7 +406,7 @@ async def test_create_schema_request_no_version( # ] async def test_create_schema_request_with_version( - self, module_organization: SchemaOrganization + self, module_organization: Organization ) -> None: # GIVEN an organization # WHEN creating a CreateSchemaRequest with no version given diff --git a/tests/integration/synapseclient/models/async/test_search_index_async.py b/tests/integration/synapseclient/models/async/test_search_index_async.py new file mode 100644 index 000000000..e8dfec642 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_search_index_async.py @@ -0,0 +1,167 @@ +"""Integration tests for the SearchIndex entity and SearchIndexQuery. + +SearchIndex builds an OpenSearch index from a SQL view of a table-like entity. +Index creation may be restricted to Sage Bionetworks employees on some stacks; +those tests skip gracefully when the server denies the create with a 403. +""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import ( + Project, + SearchIndex, + SearchIndexQuery, + SearchQuery, + SearchQueryPart, + Table, +) +from synapseclient.models.table_components import SchemaStorageStrategy +from tests.integration import ASYNC_JOB_TIMEOUT_SEC, QUERY_TIMEOUT_SEC +from tests.integration.helpers import wait_for_condition + + +class TestSearchIndex: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def _create_source_table(self, project_model: Project) -> Table: + """Create a populated table to use as a SearchIndex source.""" + table = Table(name=str(uuid.uuid4()), parent_id=project_model.id) + table = await table.store_async(synapse_client=self.syn) + self.schedule_for_cleanup(table.id) + await table.store_rows_async( + values={ + "title": ["Alzheimer study", "Cancer cohort", "Diabetes trial"], + "disease_code": ["AD", "CA", "DB"], + }, + schema_storage_strategy=SchemaStorageStrategy.INFER_FROM_DATA, + synapse_client=self.syn, + ) + return table + + async def _store_search_index(self, **kwargs) -> SearchIndex: + """Store a SearchIndex, skipping the test if creation is not permitted.""" + try: + index = await SearchIndex(**kwargs).store_async(synapse_client=self.syn) + except SynapseHTTPError as e: + if e.response.status_code == 403: + pytest.skip(f"SearchIndex creation is restricted on this stack: {e}") + raise + self.schedule_for_cleanup(index.id) + return index + + async def test_empty_defining_sql_validation(self, project_model: Project) -> None: + # GIVEN a SearchIndex with no defining SQL + index = SearchIndex(name=str(uuid.uuid4()), parent_id=project_model.id) + + # WHEN storing it + # THEN a ValueError is raised before any API call + with pytest.raises(ValueError, match="defining_sql"): + await index.store_async(synapse_client=self.syn) + + async def test_create_and_retrieve_search_index( + self, project_model: Project + ) -> None: + # GIVEN a populated source table + table = await self._create_source_table(project_model) + + # WHEN creating a SearchIndex over it + index_name = str(uuid.uuid4()) + index = await self._store_search_index( + name=index_name, + parent_id=project_model.id, + defining_sql=f"SELECT * FROM {table.id}", + ) + + # THEN it is created with an ID + assert index.id is not None + + # AND when retrieving it, the metadata and derived columns are present + retrieved = await SearchIndex(id=index.id).get_async(synapse_client=self.syn) + assert retrieved.name == index_name + assert retrieved.defining_sql == f"SELECT * FROM {table.id}" + # Columns are read-only, derived from the defining SQL. + assert retrieved.columns is not None + assert "title" in retrieved.columns + assert "disease_code" in retrieved.columns + + async def test_query_search_index_match_all(self, project_model: Project) -> None: + # GIVEN a SearchIndex over a populated table + table = await self._create_source_table(project_model) + index = await self._store_search_index( + name=str(uuid.uuid4()), + parent_id=project_model.id, + defining_sql=f"SELECT * FROM {table.id}", + ) + + # WHEN the OpenSearch index has finished building, a match_all query + # returns the indexed rows. The build is asynchronous, so poll until the + # query succeeds and reports the expected total. + async def _query_total_hits() -> int: + query = SearchIndexQuery( + search_index_id=index.id, + search_query=SearchQuery(query={"match_all": {}}, size=10), + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], + ) + await query.send_job_and_wait_async( + timeout=QUERY_TIMEOUT_SEC, synapse_client=self.syn + ) + return query.total_hits + + total_hits = await wait_for_condition( + _query_total_hits, + timeout_seconds=ASYNC_JOB_TIMEOUT_SEC, + description="SearchIndex to build and return all rows", + ) + + # THEN every source row is indexed + assert total_hits == 3 + + async def test_autocomplete_search_index(self, project_model: Project) -> None: + # GIVEN a SearchIndex over a populated table + table = await self._create_source_table(project_model) + index = await self._store_search_index( + name=str(uuid.uuid4()), + parent_id=project_model.id, + defining_sql=f"SELECT * FROM {table.id}", + ) + + # WHEN the index has built, a prefix autocomplete returns the matching row. + # The build is asynchronous, so poll until a hit comes back. + async def _autocomplete_hits(): + return await index.autocomplete_async( + query={"match_phrase_prefix": {"title": {"query": "Alz"}}}, + synapse_client=self.syn, + ) + + hits = await wait_for_condition( + _autocomplete_hits, + timeout_seconds=ASYNC_JOB_TIMEOUT_SEC, + description="SearchIndex to build and return an autocomplete hit", + ) + + # THEN only the Alzheimer row matches the prefix + assert len(hits) == 1 + + async def test_delete_search_index(self, project_model: Project) -> None: + # GIVEN a SearchIndex + table = await self._create_source_table(project_model) + index = await self._store_search_index( + name=str(uuid.uuid4()), + parent_id=project_model.id, + defining_sql=f"SELECT * FROM {table.id}", + ) + + # WHEN deleting it + await SearchIndex(id=index.id).delete_async(synapse_client=self.syn) + + # THEN it can no longer be retrieved + with pytest.raises(SynapseHTTPError): + await SearchIndex(id=index.id).get_async(synapse_client=self.syn) diff --git a/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py b/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py index ee0a909fc..26ddd25d8 100644 --- a/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py +++ b/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py @@ -28,10 +28,10 @@ JSONSchema, Link, MaterializedView, + Organization, Project, RecordBasedMetadataTaskProperties, RecordSet, - SchemaOrganization, SubmissionView, Table, Team, @@ -716,10 +716,10 @@ async def test_store_async_curation_task_basic( assert stored_task.etag is not None async def test_store_async_schema_organization_basic(self) -> None: - """Test storing a SchemaOrganization entity.""" + """Test storing a Organization entity.""" # GIVEN a new schema organization # Name must have each part start with a letter - schema_org = SchemaOrganization( + schema_org = Organization( name=f"test.schema.org.test{str(uuid.uuid4())[:8]}", ) @@ -737,7 +737,7 @@ async def test_store_async_schema_organization_basic(self) -> None: # THEN the schema organization should no longer be retrievable with pytest.raises(Exception): - await SchemaOrganization(organization_name=stored_org.name).get_async( + await Organization(organization_name=stored_org.name).get_async( synapse_client=self.syn ) @@ -864,7 +864,7 @@ async def test_store_async_form_data_basic(self, project_model: Project) -> None async def test_store_async_json_schema_basic(self) -> None: """Test storing a JSONSchema entity.""" # GIVEN a schema organization first - schema_org = SchemaOrganization( + schema_org = Organization( name=f"test.schema.org.test{str(uuid.uuid4())[:8]}", ) stored_org = await store_async(schema_org, synapse_client=self.syn) diff --git a/tests/integration/synapseclient/test_command_line_client.py b/tests/integration/synapseclient/test_command_line_client.py index af66f59ce..af824f8fa 100644 --- a/tests/integration/synapseclient/test_command_line_client.py +++ b/tests/integration/synapseclient/test_command_line_client.py @@ -1278,11 +1278,11 @@ class TestSchemaManagementCommands: @pytest.fixture(scope="class") def schema_organization(self, syn: Synapse, request): """Create a test organization for schema registration.""" - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization # Prefix with 'id' so the name part starts with a letter (required by schema validation) org_name = f"test.org.id{str(uuid.uuid4())[:8]}" - organization = SchemaOrganization(org_name) + organization = Organization(org_name) organization.store(synapse_client=syn) def cleanup(): diff --git a/tests/unit/synapseclient/extensions/test_schema_management.py b/tests/unit/synapseclient/extensions/test_schema_management.py index 1b0d1a252..fcdef1fca 100644 --- a/tests/unit/synapseclient/extensions/test_schema_management.py +++ b/tests/unit/synapseclient/extensions/test_schema_management.py @@ -16,7 +16,7 @@ def mock_synapse_client(): @pytest.fixture def mock_jsonschema(): - with patch("synapseclient.models.schema_organization.JSONSchema") as MockSchema: + with patch("synapseclient.models.organization.JSONSchema") as MockSchema: instance = MockSchema.return_value instance.store_async = AsyncMock() instance.uri = "syn123.456" diff --git a/tests/unit/synapseclient/models/async/unit_test_schema_organization_async.py b/tests/unit/synapseclient/models/async/unit_test_organization_async.py similarity index 91% rename from tests/unit/synapseclient/models/async/unit_test_schema_organization_async.py rename to tests/unit/synapseclient/models/async/unit_test_organization_async.py index 525c21b99..be883caf6 100644 --- a/tests/unit/synapseclient/models/async/unit_test_schema_organization_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_organization_async.py @@ -1,4 +1,4 @@ -"""Unit tests for the SchemaOrganization and JSONSchema models.""" +"""Unit tests for the Organization and JSONSchema models.""" from unittest.mock import AsyncMock, MagicMock, patch @@ -6,10 +6,10 @@ from synapseclient import Synapse from synapseclient.models.mixins.json_schema import JSONSchemaVersionInfo -from synapseclient.models.schema_organization import ( +from synapseclient.models.organization import ( CreateSchemaRequest, JSONSchema, - SchemaOrganization, + Organization, _check_org_name, _check_schema_name, ) @@ -153,8 +153,8 @@ def test_name_part_starts_with_number(self) -> None: _check_schema_name("mytest.1invalid") -class TestSchemaOrganization: - """Unit tests for the SchemaOrganization model.""" +class TestOrganization: + """Unit tests for the Organization model.""" @pytest.fixture(autouse=True, scope="function") def init_syn(self, syn: Synapse) -> None: @@ -164,8 +164,8 @@ def test_fill_from_dict(self) -> None: # GIVEN an organization API response response = _get_organization_response() - # WHEN I fill a SchemaOrganization from the response - org = SchemaOrganization() + # WHEN I fill a Organization from the response + org = Organization() org.fill_from_dict(response) # THEN all fields should be populated @@ -175,12 +175,12 @@ def test_fill_from_dict(self) -> None: assert org.created_by == CREATED_BY async def test_store_async(self) -> None: - # GIVEN a SchemaOrganization with a name - org = SchemaOrganization(name=ORG_NAME) + # GIVEN a Organization with a name + org = Organization(name=ORG_NAME) # WHEN I call store_async with patch( - "synapseclient.models.schema_organization.create_organization", + "synapseclient.models.organization.create_organization", new_callable=AsyncMock, return_value=_get_organization_response(), ) as mock_create: @@ -195,8 +195,8 @@ async def test_store_async(self) -> None: assert result.created_on == CREATED_ON async def test_store_async_without_name_raises(self) -> None: - # GIVEN a SchemaOrganization without a name - org = SchemaOrganization() + # GIVEN a Organization without a name + org = Organization() # WHEN I call store_async # THEN it should raise ValueError @@ -204,12 +204,12 @@ async def test_store_async_without_name_raises(self) -> None: await org.store_async(synapse_client=self.syn) async def test_get_async(self) -> None: - # GIVEN a SchemaOrganization with a name - org = SchemaOrganization(name=ORG_NAME) + # GIVEN a Organization with a name + org = Organization(name=ORG_NAME) # WHEN I call get_async with patch( - "synapseclient.models.schema_organization.get_organization", + "synapseclient.models.organization.get_organization", new_callable=AsyncMock, return_value=_get_organization_response(), ) as mock_get: @@ -223,8 +223,8 @@ async def test_get_async(self) -> None: assert result.id == ORG_ID async def test_get_async_without_name_raises(self) -> None: - # GIVEN a SchemaOrganization without a name - org = SchemaOrganization() + # GIVEN a Organization without a name + org = Organization() # WHEN I call get_async # THEN it should raise ValueError @@ -232,12 +232,12 @@ async def test_get_async_without_name_raises(self) -> None: await org.get_async(synapse_client=self.syn) async def test_delete_async_with_id(self) -> None: - # GIVEN a SchemaOrganization with an id - org = SchemaOrganization(name=ORG_NAME, id=ORG_ID) + # GIVEN a Organization with an id + org = Organization(name=ORG_NAME, id=ORG_ID) # WHEN I call delete_async with patch( - "synapseclient.models.schema_organization.delete_organization", + "synapseclient.models.organization.delete_organization", new_callable=AsyncMock, return_value=None, ) as mock_delete: @@ -249,18 +249,18 @@ async def test_delete_async_with_id(self) -> None: ) async def test_delete_async_without_id_triggers_get(self) -> None: - # GIVEN a SchemaOrganization with only a name (no id) - org = SchemaOrganization(name=ORG_NAME) + # GIVEN a Organization with only a name (no id) + org = Organization(name=ORG_NAME) # WHEN I call delete_async with ( patch( - "synapseclient.models.schema_organization.get_organization", + "synapseclient.models.organization.get_organization", new_callable=AsyncMock, return_value=_get_organization_response(), ) as mock_get, patch( - "synapseclient.models.schema_organization.delete_organization", + "synapseclient.models.organization.delete_organization", new_callable=AsyncMock, return_value=None, ) as mock_delete, @@ -276,8 +276,8 @@ async def test_delete_async_without_id_triggers_get(self) -> None: ) async def test_get_json_schemas_async(self) -> None: - # GIVEN a SchemaOrganization with a name - org = SchemaOrganization(name=ORG_NAME) + # GIVEN a Organization with a name + org = Organization(name=ORG_NAME) schema_response_1 = _get_json_schema_list_response() schema_response_2 = _get_json_schema_list_response( @@ -290,7 +290,7 @@ async def mock_list(*args, **kwargs): # WHEN I call get_json_schemas_async with patch( - "synapseclient.models.schema_organization.list_json_schemas", + "synapseclient.models.organization.list_json_schemas", return_value=mock_list(), ): results = [] @@ -306,8 +306,8 @@ async def mock_list(*args, **kwargs): assert results[1].name == "another.schema" async def test_get_json_schemas_async_without_name_raises(self) -> None: - # GIVEN a SchemaOrganization without a name - org = SchemaOrganization() + # GIVEN a Organization without a name + org = Organization() # WHEN I call get_json_schemas_async # THEN it should raise ValueError @@ -316,14 +316,14 @@ async def test_get_json_schemas_async_without_name_raises(self) -> None: pass # pragma: no cover async def test_get_acl_async(self) -> None: - # GIVEN a SchemaOrganization with an id - org = SchemaOrganization(name=ORG_NAME, id=ORG_ID) + # GIVEN a Organization with an id + org = Organization(name=ORG_NAME, id=ORG_ID) acl_response = _get_acl_response() # WHEN I call get_acl_async with patch( - "synapseclient.models.schema_organization.get_organization_acl", + "synapseclient.models.organization.get_organization_acl", new_callable=AsyncMock, return_value=acl_response, ) as mock_get_acl: @@ -338,20 +338,20 @@ async def test_get_acl_async(self) -> None: assert result["resourceAccess"][0]["principalId"] == PRINCIPAL_ID_1 async def test_get_acl_async_without_id_triggers_get(self) -> None: - # GIVEN a SchemaOrganization with only a name - org = SchemaOrganization(name=ORG_NAME) + # GIVEN a Organization with only a name + org = Organization(name=ORG_NAME) acl_response = _get_acl_response() # WHEN I call get_acl_async (id will be fetched first) with ( patch( - "synapseclient.models.schema_organization.get_organization", + "synapseclient.models.organization.get_organization", new_callable=AsyncMock, return_value=_get_organization_response(), ) as mock_get, patch( - "synapseclient.models.schema_organization.get_organization_acl", + "synapseclient.models.organization.get_organization_acl", new_callable=AsyncMock, return_value=acl_response, ) as mock_get_acl, @@ -365,20 +365,20 @@ async def test_get_acl_async_without_id_triggers_get(self) -> None: mock_get_acl.assert_called_once_with(ORG_ID, synapse_client=self.syn) async def test_update_acl_async_add_new_principal(self) -> None: - # GIVEN a SchemaOrganization with an id - org = SchemaOrganization(name=ORG_NAME, id=ORG_ID) + # GIVEN a Organization with an id + org = Organization(name=ORG_NAME, id=ORG_ID) acl_response = _get_acl_response() # WHEN I call update_acl_async with a new principal with ( patch( - "synapseclient.models.schema_organization.get_organization_acl", + "synapseclient.models.organization.get_organization_acl", new_callable=AsyncMock, return_value=acl_response, ), patch( - "synapseclient.models.schema_organization.update_organization_acl", + "synapseclient.models.organization.update_organization_acl", new_callable=AsyncMock, return_value=None, ) as mock_update, @@ -407,20 +407,20 @@ async def test_update_acl_async_add_new_principal(self) -> None: assert new_entry["accessType"] == ["READ"] async def test_update_acl_async_update_existing_principal(self) -> None: - # GIVEN a SchemaOrganization with an id - org = SchemaOrganization(name=ORG_NAME, id=ORG_ID) + # GIVEN a Organization with an id + org = Organization(name=ORG_NAME, id=ORG_ID) acl_response = _get_acl_response() # WHEN I call update_acl_async for an existing principal with new permissions with ( patch( - "synapseclient.models.schema_organization.get_organization_acl", + "synapseclient.models.organization.get_organization_acl", new_callable=AsyncMock, return_value=acl_response, ), patch( - "synapseclient.models.schema_organization.update_organization_acl", + "synapseclient.models.organization.update_organization_acl", new_callable=AsyncMock, return_value=None, ) as mock_update, @@ -695,12 +695,12 @@ async def mock_list(*args, **kwargs): # WHEN I call get_async (org exists and schema is found) with ( patch( - "synapseclient.models.schema_organization.get_organization", + "synapseclient.models.organization.get_organization", new_callable=AsyncMock, return_value=_get_organization_response(), ), patch( - "synapseclient.models.schema_organization.list_json_schemas", + "synapseclient.models.organization.list_json_schemas", return_value=mock_list(), ), ): @@ -726,12 +726,12 @@ async def mock_list(*args, **kwargs): # WHEN I call get_async with ( patch( - "synapseclient.models.schema_organization.get_organization", + "synapseclient.models.organization.get_organization", new_callable=AsyncMock, return_value=_get_organization_response(), ), patch( - "synapseclient.models.schema_organization.list_json_schemas", + "synapseclient.models.organization.list_json_schemas", return_value=mock_list(), ), ): @@ -763,7 +763,7 @@ async def test_delete_async_without_version(self) -> None: # WHEN I call delete_async without a version with patch( - "synapseclient.models.schema_organization.delete_json_schema", + "synapseclient.models.organization.delete_json_schema", new_callable=AsyncMock, return_value=None, ) as mock_delete: @@ -778,7 +778,7 @@ async def test_delete_async_with_version(self) -> None: # WHEN I call delete_async with a specific version with patch( - "synapseclient.models.schema_organization.delete_json_schema", + "synapseclient.models.organization.delete_json_schema", new_callable=AsyncMock, return_value=None, ) as mock_delete: @@ -819,7 +819,7 @@ async def mock_list(*args, **kwargs): # WHEN I call get_versions_async with patch( - "synapseclient.models.schema_organization.list_json_schema_versions", + "synapseclient.models.organization.list_json_schema_versions", return_value=mock_list(), ): results = [] @@ -859,7 +859,7 @@ async def mock_list(*args, **kwargs): # WHEN I call get_versions_async with patch( - "synapseclient.models.schema_organization.list_json_schema_versions", + "synapseclient.models.organization.list_json_schema_versions", return_value=mock_list(), ): results = [] @@ -880,7 +880,7 @@ async def test_get_body_async_latest(self) -> None: # WHEN I call get_body_async without a version (latest) with patch( - "synapseclient.models.schema_organization.get_json_schema_body", + "synapseclient.models.organization.get_json_schema_body", new_callable=AsyncMock, return_value=expected_body, ) as mock_get_body: @@ -900,7 +900,7 @@ async def test_get_body_async_with_version(self) -> None: # WHEN I call get_body_async with a specific version with patch( - "synapseclient.models.schema_organization.get_json_schema_body", + "synapseclient.models.organization.get_json_schema_body", new_callable=AsyncMock, return_value=expected_body, ) as mock_get_body: diff --git a/tests/unit/synapseclient/models/async/unit_test_search_management_async.py b/tests/unit/synapseclient/models/async/unit_test_search_management_async.py new file mode 100644 index 000000000..6a23eba0a --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_search_management_async.py @@ -0,0 +1,574 @@ +"""Unit tests for the search-management dataclasses. + +Covers fill_from_dict / to_synapse_request round-trips for the raw +OpenSearch-DSL pass-through models and the SearchIndexQuery async flow. +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.constants.concrete_types import SEARCH_INDEX_QUERY +from synapseclient.models.search_index import SearchIndex +from synapseclient.models.search_management import ( + ColumnAnalyzerOverride, + ColumnAnalyzerOverrideEntry, + SearchAutocompleteRequest, + SearchConfigBinding, + SearchConfiguration, + SearchFieldValue, + SearchHighlight, + SearchHit, + SearchIndexQuery, + SearchIndexState, + SearchIndexStatus, + SearchQuery, + SearchQueryPart, + SynonymSet, + TextAnalyzer, +) + + +class TestSearchQuery: + """Round-trip tests for the raw-DSL SearchQuery.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def test_to_synapse_request_maps_special_keys(self): + # GIVEN a SearchQuery exercising the keyword-renamed slots + query = SearchQuery( + query={"match_all": {}}, + post_filter={"term": {"disease": {"value": "AD"}}}, + aggregations={"by_disease": {"terms": {"field": "disease", "size": 10}}}, + highlight={"fields": {"title": {}}}, + collapse={"field": "study"}, + rescore={"window_size": 50}, + sort=[{"title": "asc"}], + source={"includes": ["title", "abstract"]}, + from_=5, + size=25, + search_after=["abc", 123], + ) + # WHEN I serialize it + request = query.to_synapse_request() + # THEN source/from_/search_after map to the OpenSearch key names + assert request["_source"] == {"includes": ["title", "abstract"]} + assert request["from"] == 5 + assert request["search_after"] == ["abc", 123] + assert "source" not in request + assert "from_" not in request + assert request["query"] == {"match_all": {}} + assert request["aggregations"] == { + "by_disease": {"terms": {"field": "disease", "size": 10}} + } + + def test_to_synapse_request_drops_none(self): + # GIVEN a minimal SearchQuery + query = SearchQuery(query={"match_all": {}}) + # WHEN I serialize it + request = query.to_synapse_request() + # THEN only the populated key is present + assert request == {"query": {"match_all": {}}} + + def test_fill_from_dict_round_trip(self): + # GIVEN a SearchQueryResults-style request body + body = { + "query": {"match": {"title": {"query": "alz"}}}, + "_source": {"includes": ["title"]}, + "from": 10, + "size": 50, + "search_after": [1, 2], + "sort": [{"_score": "desc"}], + } + # WHEN I fill a SearchQuery from it + query = SearchQuery().fill_from_dict(body) + # THEN the keyword-renamed slots are populated + assert query.source == {"includes": ["title"]} + assert query.from_ == 10 + assert query.size == 50 + assert query.search_after == [1, 2] + assert query.sort == [{"_score": "desc"}] + # AND re-serializing reproduces the original body + assert query.to_synapse_request() == body + + +class TestSearchHit: + """SearchHit deserialization including SearchHighlight.""" + + synapse_response = { + "rowId": 42, + "rowVersion": 3, + "score": 1.5, + "fields": [ + {"name": "title", "value": "Alzheimer study"}, + {"name": "disease", "value": "AD"}, + ], + "highlights": [ + {"name": "title", "snippets": ["Alzheimer study"]}, + ], + } + + def test_fill_from_dict(self): + # WHEN I fill a SearchHit from a response + hit = SearchHit().fill_from_dict(self.synapse_response) + # THEN scalar fields populate + assert hit.row_id == 42 + assert hit.row_version == 3 + assert hit.score == 1.5 + # AND fields are SearchFieldValue + assert all(isinstance(f, SearchFieldValue) for f in hit.fields) + assert hit.fields[0].name == "title" + assert hit.fields[0].value == "Alzheimer study" + # AND highlights are SearchHighlight with snippets + assert all(isinstance(h, SearchHighlight) for h in hit.highlights) + assert hit.highlights[0].name == "title" + assert hit.highlights[0].snippets == ["Alzheimer study"] + + def test_fill_from_dict_empty(self): + # WHEN I fill from an empty dict + hit = SearchHit().fill_from_dict({}) + # THEN collections default to empty lists + assert hit.fields == [] + assert hit.highlights == [] + + +class TestTextAnalyzer: + """TextAnalyzer carries raw settings.""" + + def test_round_trip(self): + # GIVEN a TextAnalyzer with a raw OpenSearch analysis block + settings = { + "tokenizer": {"std": {"type": "standard"}}, + "filter": {"med_syn": {"$ref": "biomed-medical_terms"}}, + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "std", + "filter": ["lowercase", "med_syn"], + } + }, + } + analyzer = TextAnalyzer( + organization_name="biomed", + name="publications", + settings=settings, + ) + # WHEN I serialize then deserialize + request = analyzer.to_synapse_request() + # THEN settings pass through unchanged + assert request["settings"] == settings + assert request["organizationName"] == "biomed" + # AND qualified_name composes org + name + assert analyzer.qualified_name == "biomed-publications" + # AND fill_from_dict is the inverse + round_tripped = TextAnalyzer().fill_from_dict(request) + assert round_tripped.settings == settings + + +class TestSynonymSet: + """SynonymSet carries a raw definition object.""" + + def test_round_trip(self): + # GIVEN a SynonymSet with a raw synonym_graph definition + definition = { + "type": "synonym_graph", + "synonyms": ["tumor, neoplasm, cancer", "AD => Alzheimer's disease"], + } + synonym_set = SynonymSet( + organization_name="biomed", + name="medical_terms", + definition=definition, + ) + # WHEN I serialize it + request = synonym_set.to_synapse_request() + # THEN definition passes through unchanged + assert request["definition"] == definition + # AND fill_from_dict is the inverse + assert SynonymSet().fill_from_dict(request).definition == definition + + +class TestSearchConfiguration: + """SearchConfiguration with $ref and inline analyzer slots.""" + + def test_round_trip_with_refs(self): + # GIVEN a SearchConfiguration referencing saved resources + config = SearchConfiguration( + organization_name="biomed", + name="publications_v1", + default_analyzer={"$ref": "org.sagebionetworks-SCIENTIFIC"}, + column_analyzer_overrides=[{"$ref": "biomed-publications_overrides"}], + ) + # WHEN I serialize it + request = config.to_synapse_request() + # THEN the analyzer slots pass through as raw objects + assert request["defaultAnalyzer"] == {"$ref": "org.sagebionetworks-SCIENTIFIC"} + assert request["columnAnalyzerOverrides"] == [ + {"$ref": "biomed-publications_overrides"} + ] + # AND fill_from_dict is the inverse + round_tripped = SearchConfiguration().fill_from_dict(request) + assert round_tripped.default_analyzer == { + "$ref": "org.sagebionetworks-SCIENTIFIC" + } + assert round_tripped.column_analyzer_overrides == [ + {"$ref": "biomed-publications_overrides"} + ] + + +class TestColumnAnalyzerOverride: + """ColumnAnalyzerOverride with single analyzer per entry.""" + + def test_round_trip(self): + # GIVEN an override with a single analyzer per column + override = ColumnAnalyzerOverride( + organization_name="biomed", + name="publications_overrides", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="disease_code", + analyzer={"$ref": "biomed-acronym_exact"}, + ), + ], + ) + # WHEN I serialize it + request = override.to_synapse_request() + # THEN each entry carries a single analyzer slot + assert request["overrides"] == [ + {"columnName": "disease_code", "analyzer": {"$ref": "biomed-acronym_exact"}} + ] + # AND fill_from_dict is the inverse + round_tripped = ColumnAnalyzerOverride().fill_from_dict(request) + assert round_tripped.overrides[0].column_name == "disease_code" + assert round_tripped.overrides[0].analyzer == {"$ref": "biomed-acronym_exact"} + + +class TestSearchConfigBinding: + def test_fill_from_dict(self): + # WHEN I fill a binding from a response + binding = SearchConfigBinding().fill_from_dict( + { + "bindId": "1", + "searchConfigurationId": "2", + "objectId": "syn3", + "objectType": "entity", + "createdBy": "9", + "createdOn": "2024-01-01T00:00:00.000Z", + } + ) + # THEN all fields populate + assert binding.bind_id == "1" + assert binding.search_configuration_id == "2" + assert binding.object_id == "syn3" + assert binding.object_type == "entity" + + async def test_store_async_binds_and_fills(self): + # GIVEN a binding with an object and configuration id + binding = SearchConfigBinding(object_id="syn3", search_configuration_id="2") + with patch( + "synapseclient.models.search_management.bind_search_config_to_entity", + new_callable=AsyncMock, + return_value={"bindId": "9", "searchConfigurationId": "2"}, + ) as mock_bind: + # WHEN I store it + result = await binding.store_async(synapse_client=self.syn) + # THEN the entity and configuration ids are forwarded and the response fills + mock_bind.assert_awaited_once_with("syn3", "2", synapse_client=self.syn) + assert result.bind_id == "9" + + @pytest.mark.parametrize( + "binding", + [ + SearchConfigBinding(search_configuration_id="2"), + SearchConfigBinding(object_id="syn3"), + ], + ids=["missing_object_id", "missing_configuration_id"], + ) + async def test_store_async_requires_ids(self, binding): + # WHEN storing a binding missing a required id THEN a ValueError is raised + with pytest.raises(ValueError): + await binding.store_async(synapse_client=self.syn) + + async def test_get_async_resolves_binding(self): + # GIVEN a binding for an entity + binding = SearchConfigBinding(object_id="syn3") + with patch( + "synapseclient.models.search_management.get_search_config_binding", + new_callable=AsyncMock, + return_value={"bindId": "9", "objectId": "syn3"}, + ) as mock_get: + # WHEN I get it + result = await binding.get_async(synapse_client=self.syn) + # THEN the entity id is forwarded and the response fills + mock_get.assert_awaited_once_with("syn3", synapse_client=self.syn) + assert result.bind_id == "9" + + async def test_delete_async_clears_binding(self): + # GIVEN a binding for an entity + binding = SearchConfigBinding(object_id="syn3") + with patch( + "synapseclient.models.search_management.clear_search_config_binding", + new_callable=AsyncMock, + ) as mock_clear: + # WHEN I delete it + await binding.delete_async(synapse_client=self.syn) + # THEN the clear endpoint is called for that entity + mock_clear.assert_awaited_once_with("syn3", synapse_client=self.syn) + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + +class TestOrgScopedResource: + """Dispatch tests for the create/get/update/list lifecycle shared by the + org-scoped search-management resources.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + # Each org-scoped resource wires its own api functions; validate every one. + RESOURCE_CLASSES = [ + TextAnalyzer, + ColumnAnalyzerOverride, + SynonymSet, + SearchConfiguration, + ] + + @pytest.mark.parametrize( + "cls", RESOURCE_CLASSES, ids=[c.__name__ for c in RESOURCE_CLASSES] + ) + async def test_lifecycle_dispatch(self, cls): + # GIVEN a new resource (no id) THEN store dispatches to the create fn + mock_create = AsyncMock( + return_value={"id": "5", "organizationName": "biomed", "name": "n"} + ) + with patch.object(cls, "_CREATE_FN", new=staticmethod(mock_create)): + new_resource = cls(organization_name="biomed", name="n") + expected_create_body = new_resource.to_synapse_request() + stored = await new_resource.store_async(synapse_client=self.syn) + mock_create.assert_awaited_once_with( + expected_create_body, synapse_client=self.syn + ) + assert stored.id == "5" + + # GIVEN a resource with an id THEN store dispatches to update by id + mock_update = AsyncMock(return_value={"id": "5", "name": "n2"}) + with patch.object(cls, "_UPDATE_FN", new=staticmethod(mock_update)): + existing = cls(id="5", organization_name="biomed", name="n2") + expected_update_body = existing.to_synapse_request() + await existing.store_async(synapse_client=self.syn) + assert mock_update.await_args.args[0] == "5" + assert mock_update.await_args.args[1] == expected_update_body + + # WHEN getting by id THEN get dispatches by id + mock_get = AsyncMock(return_value={"id": "5", "name": "n"}) + with patch.object(cls, "_GET_FN", new=staticmethod(mock_get)): + await cls(id="5").get_async(synapse_client=self.syn) + mock_get.assert_awaited_once_with("5", synapse_client=self.syn) + + # WHEN listing THEN all pages are followed via nextPageToken + mock_list = AsyncMock( + side_effect=[ + {"results": [{"id": "1"}], "nextPageToken": "tok"}, + {"results": [{"id": "2"}]}, + ] + ) + with patch.object(cls, "_LIST_FN", new=staticmethod(mock_list)): + listed = await cls.list_async( + organization_name="biomed", synapse_client=self.syn + ) + assert mock_list.await_count == 2 + assert [item.id for item in listed] == ["1", "2"] + # AND the second page request forwards the token from the first page + assert mock_list.await_args_list[1].kwargs["next_page_token"] == "tok" + + async def test_get_async_requires_id(self): + # WHEN getting a resource without an id THEN a ValueError is raised + with pytest.raises(ValueError): + await TextAnalyzer().get_async(synapse_client=self.syn) + + +class TestSearchIndexAutocomplete: + """Dispatch tests for SearchIndex.autocomplete_async.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + async def test_autocomplete_async_dispatch(self): + # GIVEN a SearchIndex with an id + index = SearchIndex(id="syn1") + response = {"hits": [{"rowId": 1, "fields": [{"name": "title", "value": "x"}]}]} + with patch( + "synapseclient.api.autocomplete_search", + new_callable=AsyncMock, + return_value=response, + ) as mock_autocomplete: + # WHEN I run autocomplete + hits = await index.autocomplete_async( + query={"prefix": {"title": {"value": "a"}}}, + source={"includes": ["title"]}, + synapse_client=self.syn, + ) + # THEN the request nests the query/_source under searchQuery for this index + mock_autocomplete.assert_awaited_once_with( + { + "searchIndexId": "syn1", + "searchQuery": { + "query": {"prefix": {"title": {"value": "a"}}}, + "_source": {"includes": ["title"]}, + }, + }, + synapse_client=self.syn, + ) + # AND the response hits deserialize to SearchHit + assert len(hits) == 1 + assert isinstance(hits[0], SearchHit) + assert hits[0].row_id == 1 + + async def test_autocomplete_async_requires_id(self): + # WHEN autocompleting without an id THEN a ValueError is raised + with pytest.raises(ValueError): + await SearchIndex().autocomplete_async( + query={"prefix": {"title": {"value": "a"}}}, + synapse_client=self.syn, + ) + + +class TestSearchIndexStatus: + def test_fill_from_dict(self): + # WHEN I fill a status from a response + status = SearchIndexStatus().fill_from_dict( + { + "searchIndexId": "syn1", + "state": "ACTIVE", + "changedOn": "2024-01-01T00:00:00.000Z", + "errorMessage": None, + } + ) + # THEN the state coerces to the enum + assert status.search_index_id == "syn1" + assert status.state is SearchIndexState.ACTIVE + + +class TestSearchAutocompleteRequest: + def test_to_synapse_request(self): + # GIVEN an autocomplete request with a prefix query + request = SearchAutocompleteRequest( + search_index_id="syn22806626", + query={"match_phrase_prefix": {"title": {"query": "alz"}}}, + source={"includes": ["title"]}, + ) + # WHEN I serialize it + body = request.to_synapse_request() + # THEN the body nests query and _source under searchQuery + assert body == { + "searchIndexId": "syn22806626", + "searchQuery": { + "query": {"match_phrase_prefix": {"title": {"query": "alz"}}}, + "_source": {"includes": ["title"]}, + }, + } + + def test_to_synapse_request_minimal(self): + # GIVEN an autocomplete request with only a query + request = SearchAutocompleteRequest( + search_index_id="syn1", + query={"prefix": {"title": {"value": "a"}}}, + ) + # WHEN I serialize it + body = request.to_synapse_request() + # THEN _source is omitted + assert body == { + "searchIndexId": "syn1", + "searchQuery": {"query": {"prefix": {"title": {"value": "a"}}}}, + } + + +class TestSearchIndexQuery: + """Request serialization, response deserialization, and the async flow.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def _build_query(self) -> SearchIndexQuery: + return SearchIndexQuery( + search_index_id="syn1", + search_query=SearchQuery(query={"match_all": {}}, size=10), + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], + ) + + def test_to_synapse_request(self): + # GIVEN a SearchIndexQuery + query = self._build_query() + # WHEN I serialize it + request = query.to_synapse_request() + # THEN it carries the concrete type, index id, query body, and parts + assert request == { + "concreteType": SEARCH_INDEX_QUERY, + "searchIndexId": "syn1", + "searchQuery": {"query": {"match_all": {}}, "size": 10}, + "responseParts": ["HITS", "TOTAL_HITS"], + } + + def test_fill_from_dict_response(self): + # GIVEN a SearchQueryResults response body + response = { + "hits": [ + { + "rowId": 1, + "fields": [{"name": "title", "value": "x"}], + "highlights": [{"name": "title", "snippets": ["x"]}], + } + ], + "totalHits": 7, + "selectColumns": [{"name": "title", "columnType": "STRING"}], + "aggregationResults": {"by_disease": {"buckets": []}}, + "nextSearchAfter": ["cursor", 99], + "offset": 0, + } + # WHEN I fill the query from it + query = self._build_query().fill_from_dict(response) + # THEN the response fields populate + assert query.total_hits == 7 + assert len(query.hits) == 1 + assert query.hits[0].row_id == 1 + assert query.hits[0].highlights[0].snippets == ["x"] + assert query.select_columns[0].name == "title" + assert query.aggregation_results == {"by_disease": {"buckets": []}} + assert query.next_search_after == ["cursor", 99] + assert query.offset == 0 + + async def test_send_job_and_wait_async(self): + # GIVEN a SearchIndexQuery + query = self._build_query() + response = {"hits": [], "totalHits": 0, "offset": 0} + with ( + patch( + "synapseclient.models.mixins.asynchronous_job.send_job_and_wait_async", + new_callable=AsyncMock, + return_value=response, + ) as mock_send_job, + patch.object( + query, "fill_from_dict", wraps=query.fill_from_dict + ) as mock_fill, + ): + # WHEN I send the job and wait + await query.send_job_and_wait_async(synapse_client=self.syn) + # THEN the async job service is called with the serialized request + mock_send_job.assert_called_once_with( + request=query.to_synapse_request(), + request_type=SEARCH_INDEX_QUERY, + timeout=120, + synapse_client=self.syn, + ) + # AND fill_from_dict is invoked with the response body + mock_fill.assert_called_once_with(synapse_response=response) + # AND the response fields land on the instance + assert query.total_hits == 0 + assert query.hits == [] diff --git a/tests/unit/synapseclient/operations/unit_test_delete_operations.py b/tests/unit/synapseclient/operations/unit_test_delete_operations.py index fc6595536..cb64e4d89 100644 --- a/tests/unit/synapseclient/operations/unit_test_delete_operations.py +++ b/tests/unit/synapseclient/operations/unit_test_delete_operations.py @@ -348,11 +348,11 @@ async def test_delete_team_entity(self): assert result is None async def test_delete_schema_organization_entity(self): - """Test that a SchemaOrganization entity is deleted normally.""" - # GIVEN a mock SchemaOrganization entity - from synapseclient.models import SchemaOrganization + """Test that a Organization entity is deleted normally.""" + # GIVEN a mock Organization entity + from synapseclient.models import Organization - mock_org = SchemaOrganization(name="testorg") + mock_org = Organization(name="testorg") mock_org.delete_async = AsyncMock(return_value=None) # WHEN I call delete_async diff --git a/tests/unit/synapseclient/operations/unit_test_store_operations.py b/tests/unit/synapseclient/operations/unit_test_store_operations.py index 34d439c1f..2a8a4e42d 100644 --- a/tests/unit/synapseclient/operations/unit_test_store_operations.py +++ b/tests/unit/synapseclient/operations/unit_test_store_operations.py @@ -480,15 +480,15 @@ async def test_store_evaluation_entity(self): assert result is mock_eval -class TestStoreSchemaOrganizationRoute: - """Tests for SchemaOrganization entity routing in store_async.""" +class TestStoreOrganizationRoute: + """Tests for Organization entity routing in store_async.""" async def test_store_schema_organization(self): - """Test that SchemaOrganization routes to store_async.""" - # GIVEN a mock SchemaOrganization entity - from synapseclient.models import SchemaOrganization + """Test that Organization routes to store_async.""" + # GIVEN a mock Organization entity + from synapseclient.models import Organization - mock_org = SchemaOrganization(name="testorg") + mock_org = Organization(name="testorg") mock_org.store_async = AsyncMock(return_value=mock_org) # WHEN I call store_async From c721574ac1c9d661d57babe8eb39049c50d516e0 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:49:19 +0000 Subject: [PATCH 02/30] [SYNPY-1869] Wire SearchIndex into container traversal; use delete_none_keys - Return SearchIndex children from sync_from_synapse/walk/get_children by adding searchindex dispatch to StorableContainer and searchindexes fields on Project and Folder - Include searchindex in access_control ENTITY_TYPE_MAPPING so ACL/permission cascades reach SearchIndex children - Replace `{k: v ... if v is not None}` comprehensions with delete_none_keys() in search_services list functions - Fix async docstring examples in search_index to use async form --- synapseclient/api/entity_services.py | 1 + synapseclient/api/search_services.py | 10 +++-- synapseclient/models/folder.py | 5 +++ synapseclient/models/mixins/access_control.py | 3 ++ .../models/mixins/storable_container.py | 28 ++++++++++-- synapseclient/models/project.py | 5 +++ synapseclient/models/search_index.py | 21 +++++---- .../models/async/unit_test_project_async.py | 43 +++++++++++++++++++ .../unit_test_synapseutils_sync.py | 6 +++ 9 files changed, 106 insertions(+), 16 deletions(-) diff --git a/synapseclient/api/entity_services.py b/synapseclient/api/entity_services.py index 27561da16..af81a9ab4 100644 --- a/synapseclient/api/entity_services.py +++ b/synapseclient/api/entity_services.py @@ -1242,6 +1242,7 @@ async def main(): "submissionview", "dataset", "materializedview", + "searchindex", ] request_body = { diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py index 767689d8f..8388cda25 100644 --- a/synapseclient/api/search_services.py +++ b/synapseclient/api/search_services.py @@ -12,6 +12,8 @@ import json from typing import TYPE_CHECKING, Any, Dict, List, Optional +from synapseclient.core.utils import delete_none_keys + if TYPE_CHECKING: from synapseclient import Synapse @@ -121,7 +123,7 @@ async def list_text_analyzers( client = Synapse.get_client(synapse_client=synapse_client) body = {"organizationName": organization_name, "nextPageToken": next_page_token} - body = {k: v for k, v in body.items() if v is not None} + delete_none_keys(body) return await client.rest_post_async( uri="/search/text/analyzer/list", body=json.dumps(body) ) @@ -197,7 +199,7 @@ async def list_column_analyzer_overrides( client = Synapse.get_client(synapse_client=synapse_client) body = {"organizationName": organization_name, "nextPageToken": next_page_token} - body = {k: v for k, v in body.items() if v is not None} + delete_none_keys(body) return await client.rest_post_async( uri="/search/column/analyzer/override/list", body=json.dumps(body) ) @@ -270,7 +272,7 @@ async def list_synonym_sets( client = Synapse.get_client(synapse_client=synapse_client) body = {"organizationName": organization_name, "nextPageToken": next_page_token} - body = {k: v for k, v in body.items() if v is not None} + delete_none_keys(body) return await client.rest_post_async( uri="/search/synonym/set/list", body=json.dumps(body) ) @@ -346,7 +348,7 @@ async def list_search_configurations( client = Synapse.get_client(synapse_client=synapse_client) body = {"organizationName": organization_name, "nextPageToken": next_page_token} - body = {k: v for k, v in body.items() if v is not None} + delete_none_keys(body) return await client.rest_post_async( uri="/search/configuration/list", body=json.dumps(body) ) diff --git a/synapseclient/models/folder.py b/synapseclient/models/folder.py index 3b5a0002e..67a169b7e 100644 --- a/synapseclient/models/folder.py +++ b/synapseclient/models/folder.py @@ -35,6 +35,7 @@ EntityView, MaterializedView, Project, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -78,6 +79,7 @@ class Folder( datasetcollections: Dataset collections that exist within this folder. materializedviews: Materialized views that exist within this folder. virtualtables: Virtual tables that exist within this folder. + searchindexes: Search indexes that exist within this folder. annotations: Additional metadata associated with the folder. The key is the name of your desired annotations. The value is an object containing a list of values (use empty list to represent no values for key) and the value type @@ -160,6 +162,9 @@ class Folder( virtualtables: List["VirtualTable"] = field(default_factory=list, compare=False) """Virtual tables that exist within this folder.""" + searchindexes: List["SearchIndex"] = field(default_factory=list, compare=False) + """Search indexes that exist within this folder.""" + annotations: Optional[ Dict[ str, diff --git a/synapseclient/models/mixins/access_control.py b/synapseclient/models/mixins/access_control.py index ccec02aae..5db5f7eca 100644 --- a/synapseclient/models/mixins/access_control.py +++ b/synapseclient/models/mixins/access_control.py @@ -33,6 +33,7 @@ File, Folder, MaterializedView, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -49,6 +50,7 @@ "datasetcollection": "datasetcollections", "materializedview": "materializedviews", "virtualtable": "virtualtables", + "searchindex": "searchindexes", } @@ -203,6 +205,7 @@ class AccessControllable(AccessControllableSynchronousProtocol): datasetcollections: List["DatasetCollection"] = None materializedviews: List["MaterializedView"] = None virtualtables: List["VirtualTable"] = None + searchindexes: List["SearchIndex"] = None async def get_permissions_async( self, diff --git a/synapseclient/models/mixins/storable_container.py b/synapseclient/models/mixins/storable_container.py index ba375653f..f0b7f68be 100644 --- a/synapseclient/models/mixins/storable_container.py +++ b/synapseclient/models/mixins/storable_container.py @@ -34,6 +34,7 @@ LINK_ENTITY, MATERIALIZED_VIEW, PROJECT_ENTITY, + SEARCH_INDEX_ENTITY, SUBMISSION_VIEW, TABLE_ENTITY, VIRTUAL_TABLE, @@ -69,6 +70,7 @@ File, Folder, MaterializedView, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -92,6 +94,7 @@ class StorableContainer(StorableContainerSynchronousProtocol): - `datasetcollections` - `materializedviews` - `virtualtables` + - `searchindexes` - `_last_persistent_instance` - `_synced_from_synapse` @@ -113,6 +116,7 @@ class StorableContainer(StorableContainerSynchronousProtocol): datasetcollections: List["DatasetCollection"] = None materializedviews: List["MaterializedView"] = None virtualtables: List["VirtualTable"] = None + searchindexes: List["SearchIndex"] = None _last_persistent_instance: None = None _synced_from_synapse: bool = False @@ -220,7 +224,7 @@ async def sync_from_synapse_async( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. + "virtualtable", "searchindex"]`. manifest: Determines whether to generate a manifest CSV file. Options are: - `all` (default): generate `manifest.csv` in every synced directory @@ -527,6 +531,7 @@ async def _sync_from_synapse_async( self.datasetcollections = [] self.materializedviews = [] self.virtualtables = [] + self.searchindexes = [] for child in children: pending_tasks.extend( @@ -886,7 +891,7 @@ async def walk_async( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. The "folder" type is always included so the hierarchy + "virtualtable", "searchindex"]`. The "folder" type is always included so the hierarchy can be traversed. recursive: Whether to recursively traverse subdirectories. Defaults to True. display_ascii_tree: If True, display an ASCII tree representation as the @@ -1018,6 +1023,7 @@ async def my_function(): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ] if follow_link: include_types.append("link") @@ -1170,7 +1176,7 @@ def walk( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. The "folder" type is always included so the hierarchy + "virtualtable", "searchindex"]`. The "folder" type is always included so the hierarchy can be traversed. recursive: Whether to recursively traverse subdirectories. Defaults to True. display_ascii_tree: If True, display an ASCII tree representation as the @@ -1312,7 +1318,7 @@ async def _retrieve_children( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. + "virtualtable", "searchindex"]`. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -1332,6 +1338,7 @@ async def _retrieve_children( "datasetcollection", "materializedview", "virtualtable", + "searchindex", ] if follow_link: include_types.append("link") @@ -1606,6 +1613,17 @@ def _create_task_for_child( ) ) ) + elif synapse_id and child_type == SEARCH_INDEX_ENTITY: + # Lazy import to avoid circular import + from synapseclient.models import SearchIndex + + searchindex = SearchIndex(id=synapse_id, name=name) + self.searchindexes.append(searchindex) + pending_tasks.append( + asyncio.create_task( + wrap_coroutine(searchindex.get_async(synapse_client=synapse_client)) + ) + ) return pending_tasks @@ -1693,6 +1711,7 @@ def _resolve_sync_from_synapse_result( "DatasetCollection", "MaterializedView", "VirtualTable", + "SearchIndex", BaseException, ], failure_strategy: FailureStrategy, @@ -1727,6 +1746,7 @@ def _resolve_sync_from_synapse_result( or result.__class__.__name__ == "DatasetCollection" or result.__class__.__name__ == "MaterializedView" or result.__class__.__name__ == "VirtualTable" + or result.__class__.__name__ == "SearchIndex" ): # Do nothing as the objects are updated in place and the container has # already been updated to append the new objects. diff --git a/synapseclient/models/project.py b/synapseclient/models/project.py index a5cc15074..4266448cf 100644 --- a/synapseclient/models/project.py +++ b/synapseclient/models/project.py @@ -34,6 +34,7 @@ DatasetCollection, EntityView, MaterializedView, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -76,6 +77,7 @@ class Project( datasetcollections: Any dataset collections that are at the root directory of the project. materializedviews: Any materialized views that are at the root directory of the project. virtualtables: Any virtual tables that are at the root directory of the project. + searchindexes: Any search indexes that are at the root directory of the project. annotations: Additional metadata associated with the folder. The key is the name of your desired annotations. The value is an object containing a list of values (use empty list to represent no values for key) and the value type @@ -195,6 +197,9 @@ class Project( virtualtables: List["VirtualTable"] = field(default_factory=list, compare=False) """Any virtual tables that are at the root directory of the project.""" + searchindexes: List["SearchIndex"] = field(default_factory=list, compare=False) + """Any search indexes that are at the root directory of the project.""" + annotations: Optional[ Dict[ str, diff --git a/synapseclient/models/search_index.py b/synapseclient/models/search_index.py index b8326e978..6ce324f85 100644 --- a/synapseclient/models/search_index.py +++ b/synapseclient/models/search_index.py @@ -311,20 +311,25 @@ async def autocomplete_async( ValueError: If the ``id`` attribute has not been set. Example: Autocomplete titles beginning with "alz". +   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import SearchIndex - syn = Synapse() - syn.login() + async def main(): + syn = Synapse() + await syn.login_async() - index = SearchIndex(id="syn12345") - hits = index.autocomplete( - query={"match_phrase_prefix": {"title": {"query": "alz"}}}, - ) - for hit in hits: - print(hit.row_id, hit.fields) + index = SearchIndex(id="syn12345") + hits = await index.autocomplete_async( + query={"match_phrase_prefix": {"title": {"query": "alz"}}}, + ) + for hit in hits: + print(hit.row_id, hit.fields) + + asyncio.run(main()) ``` """ from synapseclient.api import autocomplete_search diff --git a/tests/unit/synapseclient/models/async/unit_test_project_async.py b/tests/unit/synapseclient/models/async/unit_test_project_async.py index ec3fa5e43..98720da47 100644 --- a/tests/unit/synapseclient/models/async/unit_test_project_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_project_async.py @@ -727,6 +727,49 @@ async def mock_get_children(*args, **kwargs): assert result.files[0].id == "syn456" assert result.files[0].name == "example_file_1" + async def test_sync_from_synapse_populates_search_indexes(self) -> None: + # GIVEN a Project object + project = Project(id=PROJECT_ID) + + # AND a SearchIndex child that exists on the project in Synapse + children = [ + { + "id": "syn789", + "type": concrete_types.SEARCH_INDEX_ENTITY, + "name": "example_search_index", + } + ] + + async def mock_get_children(*args, **kwargs): + for child in children: + yield child + + from synapseclient.models import SearchIndex + + # WHEN I call `sync_from_synapse` with the Project object + with ( + patch( + "synapseclient.models.mixins.storable_container.get_children", + side_effect=mock_get_children, + ), + patch( + "synapseclient.api.entity_factory.get_entity_id_bundle2", + new_callable=AsyncMock, + return_value=(self.get_example_rest_api_project_output()), + ), + patch( + "synapseclient.models.search_index.SearchIndex.get_async", + new_callable=AsyncMock, + return_value=SearchIndex(id="syn789", name="example_search_index"), + ) as mocked_search_index_get, + ): + result = await project.sync_from_synapse_async(synapse_client=self.syn) + + # THEN the SearchIndex child should be retrieved and populated + mocked_search_index_get.assert_called_once() + assert result.searchindexes[0].id == "syn789" + assert result.searchindexes[0].name == "example_search_index" + class TestStorageLocationMixin: """Tests for StorageLocationConfigurable mixin methods on Project.""" diff --git a/tests/unit/synapseutils/unit_test_synapseutils_sync.py b/tests/unit/synapseutils/unit_test_synapseutils_sync.py index 8187c6802..5967830a6 100644 --- a/tests/unit/synapseutils/unit_test_synapseutils_sync.py +++ b/tests/unit/synapseutils/unit_test_synapseutils_sync.py @@ -610,6 +610,7 @@ async def mock_get_children(*args, **kwargs): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ], synapse_client=syn, ), @@ -626,6 +627,7 @@ async def mock_get_children(*args, **kwargs): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ], synapse_client=syn, ), @@ -811,6 +813,7 @@ async def mock_get_children(*args, **kwargs): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ], synapse_client=syn, ), @@ -827,6 +830,7 @@ async def mock_get_children(*args, **kwargs): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ], synapse_client=syn, ), @@ -979,6 +983,7 @@ async def mock_get_children(*args, **kwargs): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ], synapse_client=syn, ), @@ -995,6 +1000,7 @@ async def mock_get_children(*args, **kwargs): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ], synapse_client=syn, ), From 872762ee722534524a9d2ef46b578c75e492e3b5 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Tue, 21 Jul 2026 12:35:56 -0400 Subject: [PATCH 03/30] fix docstring --- synapseclient/models/organization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapseclient/models/organization.py b/synapseclient/models/organization.py index 8248e6a0d..4dd5d373e 100644 --- a/synapseclient/models/organization.py +++ b/synapseclient/models/organization.py @@ -508,7 +508,7 @@ async def update_acl_async( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Update the ACL for a Organization + Example: Update the ACL for a Organization   ```python @@ -528,7 +528,7 @@ async def update_acl() -> None: ) asyncio.run(update_acl()) - + ``` """ acl = await self.get_acl_async(synapse_client=synapse_client) From 0d72f00238fea3c773ee770c39f52101e8c0e80b Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Tue, 21 Jul 2026 12:40:38 -0400 Subject: [PATCH 04/30] clean up --- synapseclient/api/search_services.py | 16 ------------ synapseclient/models/search_management.py | 32 ----------------------- 2 files changed, 48 deletions(-) diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py index 8388cda25..cc4024048 100644 --- a/synapseclient/api/search_services.py +++ b/synapseclient/api/search_services.py @@ -18,9 +18,6 @@ from synapseclient import Synapse -# ---------- Text Analyzer ---------- - - async def create_text_analyzer( request: Dict[str, Any], *, @@ -129,9 +126,6 @@ async def list_text_analyzers( ) -# ---------- Column Analyzer Override ---------- - - async def create_column_analyzer_override( request: Dict[str, Any], *, @@ -205,9 +199,6 @@ async def list_column_analyzer_overrides( ) -# ---------- Synonym Set ---------- - - async def create_synonym_set( request: Dict[str, Any], *, @@ -278,9 +269,6 @@ async def list_synonym_sets( ) -# ---------- Search Configuration ---------- - - async def create_search_configuration( request: Dict[str, Any], *, @@ -354,9 +342,6 @@ async def list_search_configurations( ) -# ---------- Search Configuration Bindings ---------- - - async def bind_search_config_to_entity( entity_id: str, search_configuration_id: str, @@ -420,7 +405,6 @@ async def clear_search_config_binding( await client.rest_delete_async(uri=f"/entity/{entity_id}/searchconfig/binding") -# ---------- Search Queries ---------- # # The async search query endpoint (POST /search/query/async/start + # GET /search/query/async/get/{token}) is exposed through the shared diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index b23dec5a2..37f2c4710 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -55,8 +55,6 @@ if TYPE_CHECKING: from synapseclient import Synapse -# ---------- Enums ---------- - class SearchIndexState(str, Enum): """The state of a SearchIndex's OpenSearch index.""" @@ -77,9 +75,6 @@ class SearchQueryPart(str, Enum): SELECT_COLUMNS = "SELECT_COLUMNS" -# ---------- Shared org-scoped resource base ---------- - - class OrgScopedResourceProtocol(Protocol): """Synchronous interface shared by the org-scoped search-management resources (TextAnalyzer, ColumnAnalyzerOverride, SynonymSet, SearchConfiguration).""" @@ -197,9 +192,6 @@ async def list_async( return results -# ---------- Text Analyzer ---------- - - @dataclass class TextAnalyzer(OrgScopedResource): """A shareable, named OpenSearch custom analyzer. Used to configure how text @@ -265,9 +257,6 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -# ---------- Column Analyzer Override ---------- - - @dataclass class ColumnAnalyzerOverrideEntry: """Assigns one TextAnalyzer to one column. @@ -359,9 +348,6 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -# ---------- Synonym Set ---------- - - @dataclass class SynonymSet(OrgScopedResource): """A shareable OpenSearch synonym_graph (or legacy synonym) token filter. @@ -423,9 +409,6 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -# ---------- Search Configuration ---------- - - @dataclass class SearchConfiguration(OrgScopedResource): """Bundles the index-wide default analyzer and per-column overrides used to @@ -494,9 +477,6 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -# ---------- Search Config Binding ---------- - - class SearchConfigBindingProtocol(Protocol): """Synchronous interface for SearchConfigBinding operations.""" @@ -607,9 +587,6 @@ async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> N await clear_search_config_binding(self.object_id, synapse_client=synapse_client) -# ---------- Search Index Status ---------- - - @dataclass class SearchIndexStatus: """The build status of a SearchIndex's OpenSearch index.""" @@ -628,9 +605,6 @@ def fill_from_dict(self, data: Dict[str, Any]) -> "Self": return self -# ---------- Search Query ---------- - - @dataclass class SearchQuery: """The body of an OpenSearch `_search` request, narrowed to the top-level @@ -716,9 +690,6 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -# ---------- Autocomplete ---------- - - @dataclass class SearchAutocompleteRequest: """Body of a synchronous autocomplete request against a SearchIndex. The @@ -753,9 +724,6 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -# ---------- Search Results ---------- - - @dataclass class SearchFieldValue: """A name/value pair returned in a SearchHit's `fields`. From 10fdb9bc5c7450812ce619d8c790420dcc2aed1b Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Tue, 21 Jul 2026 12:49:18 -0400 Subject: [PATCH 05/30] fix docstring --- synapseclient/models/search_management.py | 46 +++++++++++++++++------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 37f2c4710..b9dea2739 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -197,7 +197,7 @@ class TextAnalyzer(OrgScopedResource): """A shareable, named OpenSearch custom analyzer. Used to configure how text is tokenized for a search index. - REST: + Represents a [Synapse TextAnalyzer](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/TextAnalyzer.html). """ _CREATE_FN = staticmethod(create_text_analyzer) @@ -206,9 +206,20 @@ class TextAnalyzer(OrgScopedResource): _LIST_FN = staticmethod(list_text_analyzers) id: Optional[str] = None + """The unique immutable ID for this text analyzer.""" + organization_name: Optional[str] = None + """The name of the organization that owns this analyzer. + Immutable after creation.""" + name: Optional[str] = None + """The name of this analyzer. Must start with a letter and + contain only letters, digits, and underscores; unique within the + organization.""" + description: Optional[str] = None + """Optional description of this analyzer.""" + settings: Optional[Dict[str, Any]] = None """Required. JSON object holding the *contents of* the `settings.analysis` block of an OpenSearch create-index request body. Allowed root keys are @@ -218,10 +229,21 @@ class TextAnalyzer(OrgScopedResource): registry resolves to a SynonymSet at index-build time.""" etag: Optional[str] = None + """Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time this analyzer is updated; + it is used to detect when a client's copy is out-of-date.""" + created_on: Optional[str] = None + """The date on which this analyzer was created.""" + created_by: Optional[str] = None + """The ID of the Synapse user who created this analyzer.""" + modified_on: Optional[str] = None + """The date on which this analyzer was last modified.""" + modified_by: Optional[str] = None + """The ID of the Synapse user who last modified this analyzer.""" @property def qualified_name(self) -> Optional[str]: @@ -261,7 +283,7 @@ def to_synapse_request(self) -> Dict[str, Any]: class ColumnAnalyzerOverrideEntry: """Assigns one TextAnalyzer to one column. - REST: + Represents a [Synapse ColumnAnalyzerOverrideEntry](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/ColumnAnalyzerOverrideEntry.html). """ column_name: Optional[str] = None @@ -290,7 +312,7 @@ def to_synapse_request(self) -> Dict[str, Any]: class ColumnAnalyzerOverride(OrgScopedResource): """A shared resource containing per-column analyzer override entries. - REST: + Represents a [Synapse ColumnAnalyzerOverride](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/ColumnAnalyzerOverride.html). """ _CREATE_FN = staticmethod(create_column_analyzer_override) @@ -354,7 +376,7 @@ class SynonymSet(OrgScopedResource): Referenced by qualified name `{organizationName}-{name}` from a TextAnalyzer's `settings.filter` registry map via `{"$ref": "{organizationName}-{name}"}`. - REST: + Represents a [Synapse SynonymSet](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SynonymSet.html). """ _CREATE_FN = staticmethod(create_synonym_set) @@ -414,7 +436,7 @@ class SearchConfiguration(OrgScopedResource): """Bundles the index-wide default analyzer and per-column overrides used to build a SearchIndex. - REST: + Represents a [Synapse SearchConfiguration](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchConfiguration.html). """ _CREATE_FN = staticmethod(create_search_configuration) @@ -611,7 +633,7 @@ class SearchQuery: keys Synapse accepts. Each slot's contents are pass-through OpenSearch query DSL carried as raw JSON. - REST: + Represents a [Synapse SearchQuery](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/SearchQuery.html). """ query: Optional[Dict[str, Any]] = None @@ -696,7 +718,7 @@ class SearchAutocompleteRequest: autocomplete endpoint allowlists only `query` (restricted to `prefix`, `match_phrase_prefix`, or `match_bool_prefix`) and `_source`. - REST: + Represents a [Synapse SearchAutocompleteRequest](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchAutocompleteRequest.html). """ search_index_id: Optional[str] = None @@ -728,7 +750,7 @@ def to_synapse_request(self) -> Dict[str, Any]: class SearchFieldValue: """A name/value pair returned in a SearchHit's `fields`. - REST: + Represents a [Synapse SearchFieldValue](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/SearchFieldValue.html). """ name: Optional[str] = None @@ -747,7 +769,7 @@ def fill_from_dict(self, data: Dict[str, Any]) -> "Self": class SearchHighlight: """A per-field highlight payload on a SearchHit. - REST: + Represents a [Synapse SearchHighlight](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/SearchHighlight.html). """ name: Optional[str] = None @@ -767,7 +789,7 @@ def fill_from_dict(self, data: Dict[str, Any]) -> "Self": class SearchHit: """A single matching document in a SearchQueryResults response. - REST: + Represents a [Synapse SearchHit](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/SearchHit.html). """ row_id: Optional[int] = None @@ -808,7 +830,7 @@ class SearchIndexQuery(AsynchronousCommunicator): fields (`hits`, `total_hits`, `select_columns`, `aggregation_results`, `next_search_after`, `offset`) on this same instance. - REST: + Represents a [Synapse SearchIndexQuery](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchIndexQuery.html). Example: Run a search query. @@ -890,7 +912,7 @@ def fill_from_dict(self, synapse_response: Dict[str, Any]) -> "Self": Called by `AsynchronousCommunicator.send_job_and_wait_async()` once the async job completes. Leaves request fields untouched. - REST: + Modeled from [Synapse SearchQueryResults](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/SearchQueryResults.html). """ self.hits = [ SearchHit().fill_from_dict(h) From ee0729d419fdb33f1ad578ef1892f44318b632d1 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Tue, 21 Jul 2026 13:14:02 -0400 Subject: [PATCH 06/30] add to docstring --- synapseclient/models/search_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index b9dea2739..2f8fe5877 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -287,7 +287,7 @@ class ColumnAnalyzerOverrideEntry: """ column_name: Optional[str] = None - """The name of the column to override.""" + """The name of the column to override. Silently skipped at index-build time if the column is not present in the target SearchIndex's schema — a single override bundle can therefore be applied across several indexes that share some column names""" analyzer: Optional[Dict[str, Any]] = None """The analyzer to use for this column. Either a reference to a saved From 9620ba3945bb103bf27477f89cca8de5f1215e09 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Tue, 21 Jul 2026 13:34:14 -0400 Subject: [PATCH 07/30] edit docstring --- synapseclient/models/search_management.py | 47 ++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 2f8fe5877..a804e6c5a 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -310,7 +310,7 @@ def to_synapse_request(self) -> Dict[str, Any]: @dataclass class ColumnAnalyzerOverride(OrgScopedResource): - """A shared resource containing per-column analyzer override entries. + """A shareable bundle of per-column analyzer assignments. Each entry binds one column to an analyzer; Represents a [Synapse ColumnAnalyzerOverride](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/ColumnAnalyzerOverride.html). """ @@ -321,15 +321,38 @@ class ColumnAnalyzerOverride(OrgScopedResource): _LIST_FN = staticmethod(list_column_analyzer_overrides) id: Optional[str] = None + """The unique ID of this column analyzer override.""" + organization_name: Optional[str] = None + """The name of the Organization this resource belongs to. Immutable after + creation.""" + name: Optional[str] = None + """The resource name. Must start with a letter and contain only letters, + digits, and underscores. Unique within the organization and immutable + after creation. Used as part of the qualified name + ({organizationName}-{name}) when referenced by other resources.""" + description: Optional[str] = None + """Optional description.""" + overrides: Optional[List[ColumnAnalyzerOverrideEntry]] = field(default_factory=list) + """The per-column analyzer assignments -- see ColumnAnalyzerOverrideEntry.""" + etag: Optional[str] = None + """Synapse employs an Optimistic Concurrency Control (OCC) scheme.""" + created_on: Optional[str] = None + """The date this resource was created.""" + created_by: Optional[str] = None + """The ID of the user that created this resource.""" + modified_on: Optional[str] = None + """The date this resource was last modified.""" + modified_by: Optional[str] = None + """The ID of the user that last modified this resource.""" @property def qualified_name(self) -> Optional[str]: @@ -385,19 +408,41 @@ class SynonymSet(OrgScopedResource): _LIST_FN = staticmethod(list_synonym_sets) id: Optional[str] = None + """The unique ID of this synonym set.""" + organization_name: Optional[str] = None + """The name of the Organization this resource belongs to. Immutable after + creation.""" + name: Optional[str] = None + """The resource name. Must start with a letter and contain only letters, + digits, and underscores. Unique within the organization and immutable + after creation. Used as part of the qualified name + ({organizationName}-{name}) when referenced by other resources.""" + description: Optional[str] = None + """Optional description of the synonym set.""" + definition: Optional[Dict[str, Any]] = None """Required. The full OpenSearch token filter definition as a JSON object, exactly as documented for the synonym_graph / synonym token filters, e.g. `{"type": "synonym_graph", "synonyms": ["tumor, neoplasm, cancer", "AD => Alzheimer's disease"]}`.""" + etag: Optional[str] = None + """Synapse employs an Optimistic Concurrency Control (OCC) scheme.""" + created_on: Optional[str] = None + """The date this resource was created.""" + created_by: Optional[str] = None + """The ID of the user that created this resource.""" + modified_on: Optional[str] = None + """The date this resource was last modified.""" + modified_by: Optional[str] = None + """The ID of the user that last modified this resource.""" @property def qualified_name(self) -> Optional[str]: From ad09eb2dc417c1c7d7d71645d43307cf1e10d30a Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Tue, 21 Jul 2026 16:54:52 -0400 Subject: [PATCH 08/30] update docstring --- synapseclient/models/search_management.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index a804e6c5a..4592776fd 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -490,9 +490,21 @@ class SearchConfiguration(OrgScopedResource): _LIST_FN = staticmethod(list_search_configurations) id: Optional[str] = None + """The unique ID of this search configuration.""" + organization_name: Optional[str] = None + """The name of the Organization this resource belongs to. Immutable after + creation.""" + name: Optional[str] = None + """The resource name. Must start with a letter and contain only letters, + digits, and underscores. Unique within the organization and immutable + after creation. Used as part of the qualified name + ({organizationName}-{name}) when referenced by other resources.""" + description: Optional[str] = None + """Optional description.""" + default_analyzer: Optional[Dict[str, Any]] = None """Optional. The analyzer that supplies this index's `analysis.analyzer.default` slot. Either a reference to a saved TextAnalyzer written as @@ -504,11 +516,21 @@ class SearchConfiguration(OrgScopedResource): """Optional ordered list of ColumnAnalyzerOverride entries. Each entry is either a reference `{"$ref": "{organizationName}-{name}"}` or an inline ColumnAnalyzerOverride literal.""" + etag: Optional[str] = None + """Synapse employs an Optimistic Concurrency Control (OCC) scheme.""" + created_on: Optional[str] = None + """The date this resource was created.""" + created_by: Optional[str] = None + """The ID of the user that created this resource.""" + modified_on: Optional[str] = None + """The date this resource was last modified.""" + modified_by: Optional[str] = None + """The ID of the user that last modified this resource.""" @property def qualified_name(self) -> Optional[str]: From f52f7cf9727b5ff9711211829ad13b1684ad6175 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 11:23:01 -0400 Subject: [PATCH 09/30] add docstring --- synapseclient/models/search_management.py | 37 ++++++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 4592776fd..5d8a9af95 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -585,19 +585,30 @@ def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: @dataclass @async_to_sync class SearchConfigBinding(SearchConfigBindingProtocol): - """A binding between a SearchConfiguration and an entity. + """Attaches a SearchConfiguration to an entity. When a SearchIndex is built, + the effective SearchConfiguration is resolved by walking up the entity + hierarchy (entity -> folder -> project) and using the first binding found. - Effective configuration for an entity is resolved by walking up the - hierarchy (entity -> folder -> project). + Represents a [Synapse SearchConfigBinding](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchConfigBinding.html). """ bind_id: Optional[str] = None + """The unique ID of this binding.""" + search_configuration_id: Optional[str] = None + """The ID of the SearchConfiguration bound to this entity.""" + object_id: Optional[str] = None - """The ID of the entity the SearchConfiguration is bound to.""" + """The ID of the entity this configuration is bound to.""" + object_type: Optional[str] = None + """The type of the object this configuration is bound to.""" + created_by: Optional[str] = None + """The ID of the user that created this binding.""" + created_on: Optional[str] = None + """The date this binding was created.""" def fill_from_dict(self, data: Dict[str, Any]) -> "Self": self.bind_id = data.get("bindId", None) @@ -678,12 +689,28 @@ async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> N @dataclass class SearchIndexStatus: - """The build status of a SearchIndex's OpenSearch index.""" + """Build status of a SearchIndex's OpenSearch index. Read it to find out + whether the index is ready, still being built, or in a failed state -- + and, if FAILED, what went wrong. + + Represents a [Synapse SearchIndexStatus](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchIndexStatus.html). + """ search_index_id: Optional[str] = None + """The ID of the SearchIndex entity.""" + state: Optional[SearchIndexState] = None + """The state of a search index's OpenSearch index.""" + changed_on: Optional[str] = None + """The date-time when the status last changed.""" + error_message: Optional[str] = None + """Set when state is FAILED. Captures the diagnostic from the failed build -- + pre-flight validation errors (e.g. 'TextAnalyzer X does not resolve'), AOSS + rejection messages from index creation, or sample per-document failures + from bulk indexing. Capped at 3000 characters; the most-recent failure + replaces any earlier message.""" def fill_from_dict(self, data: Dict[str, Any]) -> "Self": self.search_index_id = data.get("searchIndexId", None) From b6b613f127e8c7d8e050b627e9718aa404e33ae7 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 11:27:56 -0400 Subject: [PATCH 10/30] move SearchIndexSynchronousProtocol to its own test file --- .../models/protocols/search_index_protocol.py | 49 +++++++++++++++++++ synapseclient/models/search_index.py | 43 ++-------------- 2 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 synapseclient/models/protocols/search_index_protocol.py diff --git a/synapseclient/models/protocols/search_index_protocol.py b/synapseclient/models/protocols/search_index_protocol.py new file mode 100644 index 000000000..4714806ca --- /dev/null +++ b/synapseclient/models/protocols/search_index_protocol.py @@ -0,0 +1,49 @@ +"""Protocol for the specific methods of this class that have synchronous counterparts +generated at runtime.""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol + +from typing_extensions import Self + +from synapseclient import Synapse + +if TYPE_CHECKING: + from synapseclient.models.search_management import SearchHit + + +class SearchIndexSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SearchIndex operations.""" + + def store( + self, + dry_run: bool = False, + *, + job_timeout: int = 600, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Store metadata about a SearchIndex including the annotations.""" + return self + + def get( + self, + include_columns: bool = True, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Get the metadata about the SearchIndex from Synapse.""" + return self + + def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: + """Delete the SearchIndex from Synapse.""" + return None + + def autocomplete( + self, + query: Dict[str, Any], + source: Optional[Dict[str, Any]] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SearchHit"]: + """Run a synchronous autocomplete search against this index.""" + return [] diff --git a/synapseclient/models/search_index.py b/synapseclient/models/search_index.py index 6ce324f85..aad5a5783 100644 --- a/synapseclient/models/search_index.py +++ b/synapseclient/models/search_index.py @@ -9,7 +9,7 @@ from copy import deepcopy from dataclasses import dataclass, field, replace from datetime import date, datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from typing_extensions import Self @@ -26,50 +26,15 @@ TableBase, TableStoreMixin, ) +from synapseclient.models.protocols.search_index_protocol import ( + SearchIndexSynchronousProtocol, +) from synapseclient.models.table_components import Column if TYPE_CHECKING: from synapseclient.models.search_management import SearchHit -class SearchIndexSynchronousProtocol(Protocol): - """Protocol defining the synchronous interface for SearchIndex operations.""" - - def store( - self, - dry_run: bool = False, - *, - job_timeout: int = 600, - synapse_client: Optional[Synapse] = None, - ) -> "Self": - """Store metadata about a SearchIndex including the annotations.""" - return self - - def get( - self, - include_columns: bool = True, - include_activity: bool = False, - *, - synapse_client: Optional[Synapse] = None, - ) -> "Self": - """Get the metadata about the SearchIndex from Synapse.""" - return self - - def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: - """Delete the SearchIndex from Synapse.""" - return None - - def autocomplete( - self, - query: Dict[str, Any], - source: Optional[Dict[str, Any]] = None, - *, - synapse_client: Optional[Synapse] = None, - ) -> List["SearchHit"]: - """Run a synchronous autocomplete search against this index.""" - return [] - - @dataclass @async_to_sync class SearchIndex( From 5b188c0a30314f06de27f8bdcec30a38afba6065 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 11:35:13 -0400 Subject: [PATCH 11/30] remove __all__ because functions already got added to __init__ --- synapseclient/api/search_services.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py index cc4024048..ad4e901c6 100644 --- a/synapseclient/api/search_services.py +++ b/synapseclient/api/search_services.py @@ -10,7 +10,7 @@ """ import json -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional from synapseclient.core.utils import delete_none_keys @@ -442,27 +442,3 @@ async def autocomplete_search( return await client.rest_post_async( uri="/search/autocomplete", body=json.dumps(request) ) - - -__all__: List[str] = [ - "create_text_analyzer", - "get_text_analyzer", - "update_text_analyzer", - "list_text_analyzers", - "create_column_analyzer_override", - "get_column_analyzer_override", - "update_column_analyzer_override", - "list_column_analyzer_overrides", - "create_synonym_set", - "get_synonym_set", - "update_synonym_set", - "list_synonym_sets", - "create_search_configuration", - "get_search_configuration", - "update_search_configuration", - "list_search_configurations", - "bind_search_config_to_entity", - "get_search_config_binding", - "clear_search_config_binding", - "autocomplete_search", -] From 582fbf983182fd114f4b9dea2b0b08b0897730ff Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 11:36:16 -0400 Subject: [PATCH 12/30] clean up comment --- synapseclient/api/search_services.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py index ad4e901c6..126b77467 100644 --- a/synapseclient/api/search_services.py +++ b/synapseclient/api/search_services.py @@ -405,18 +405,6 @@ async def clear_search_config_binding( await client.rest_delete_async(uri=f"/entity/{entity_id}/searchconfig/binding") -# -# The async search query endpoint (POST /search/query/async/start + -# GET /search/query/async/get/{token}) is exposed through the shared -# AsynchronousCommunicator mixin. Use: -# -# query = SearchIndexQuery(search_index_id=..., search_query=SearchQuery(...)) -# await query.send_job_and_wait_async() -# -# The synchronous autocomplete endpoint stays here because it is not an -# async job. Build its body with SearchAutocompleteRequest.to_synapse_request(). - - async def autocomplete_search( request: Dict[str, Any], *, From 3da1f40497efa81485918988b7d98ff2e3caa99a Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 11:49:18 -0400 Subject: [PATCH 13/30] fix and add parameter explanation in docstring --- synapseclient/api/search_services.py | 225 +++++++++++++++++++++++---- 1 file changed, 196 insertions(+), 29 deletions(-) diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py index 126b77467..839a48224 100644 --- a/synapseclient/api/search_services.py +++ b/synapseclient/api/search_services.py @@ -23,18 +23,20 @@ async def create_text_analyzer( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Create a TextAnalyzer. + """Create a new TextAnalyzer within the specified Organization. Arguments: - request: TextAnalyzer body. Must include organizationName, name, settings. + request: The TextAnalyzer object body. The analyzer's `settings` JSON is + parsed and any `$ref` entries inside its `filter` registry are + checked for qualified-name format and existence. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - The created TextAnalyzer. + A dictionary representing the created TextAnalyzer. """ from synapseclient import Synapse @@ -49,7 +51,7 @@ async def get_text_analyzer( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Get a TextAnalyzer by ID. + """Get a TextAnalyzer by its ID. @@ -60,7 +62,7 @@ async def get_text_analyzer( instance from the Synapse class constructor. Returns: - The requested TextAnalyzer. + A dictionary representing the requested TextAnalyzer. """ from synapseclient import Synapse @@ -79,14 +81,17 @@ async def update_text_analyzer( Arguments: - analyzer_id: The path ID (must match the request body's ID). - request: The updated TextAnalyzer. + analyzer_id: The path ID of the text analyzer (must match the request + body's ID). + request: The updated TextAnalyzer object body. The `organizationName` + and `name` fields are immutable and cannot be modified after + creation. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - The updated TextAnalyzer. + A dictionary representing the updated TextAnalyzer. """ from synapseclient import Synapse @@ -102,19 +107,21 @@ async def list_text_analyzers( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """List TextAnalyzers, optionally filtered by Organization. + """List TextAnalyzer objects, optionally filtered by Organization. Arguments: - organization_name: Optional filter by organization name. - next_page_token: Optional pagination token. + organization_name: If `organizationName` is null, all text analyzers + across all Organizations are returned. + next_page_token: Results are paginated using a next page token. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - Page of TextAnalyzers and a nextPageToken if more results exist. + A dictionary representing the ListTextAnalyzersResponse, containing a + page of TextAnalyzers and a nextPageToken if more results exist. """ from synapseclient import Synapse @@ -131,9 +138,18 @@ async def create_column_analyzer_override( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Create a ColumnAnalyzerOverride. + """Create a new ColumnAnalyzerOverride within the specified Organization. + + Arguments: + request: The ColumnAnalyzerOverride object body. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created ColumnAnalyzerOverride. """ from synapseclient import Synapse @@ -148,9 +164,19 @@ async def get_column_analyzer_override( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Get a ColumnAnalyzerOverride by ID. + """Get a ColumnAnalyzerOverride by its ID. + + Arguments: + column_analyzer_override_id: The ID of the column analyzer override to + retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested ColumnAnalyzerOverride. """ from synapseclient import Synapse @@ -169,6 +195,18 @@ async def update_column_analyzer_override( """Update a ColumnAnalyzerOverride. + + Arguments: + column_analyzer_override_id: The path ID (must match the request + body's ID). + request: The updated ColumnAnalyzerOverride object body. Concurrency + is managed via the `etag` field. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated ColumnAnalyzerOverride. """ from synapseclient import Synapse @@ -185,9 +223,22 @@ async def list_column_analyzer_overrides( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """List ColumnAnalyzerOverrides, optionally filtered by Organization. + """List ColumnAnalyzerOverride objects, optionally filtered by Organization. + + Arguments: + organization_name: If organizationName is null, all column analyzer + overrides across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListColumnAnalyzerOverridesResponse, + containing a page of ColumnAnalyzerOverrides and a nextPageToken if + more results exist. """ from synapseclient import Synapse @@ -204,9 +255,21 @@ async def create_synonym_set( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Create a SynonymSet. + """Create a new SynonymSet within the specified Organization. + + Arguments: + request: The SynonymSet object body. The supplied `definition` is + parsed to confirm it is valid JSON before being passed to AOSS; + supply synonym lists inline via the `synonyms` array rather than + using file-based parameters. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created SynonymSet. """ from synapseclient import Synapse @@ -221,9 +284,18 @@ async def get_synonym_set( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Get a SynonymSet by ID. + """Get a SynonymSet by its ID. + + Arguments: + synonym_set_id: The ID of the synonym set to retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested SynonymSet. """ from synapseclient import Synapse @@ -240,6 +312,19 @@ async def update_synonym_set( """Update a SynonymSet. + + Arguments: + synonym_set_id: The path ID (must match the request body's ID). + request: The updated SynonymSet object body. The `definition` field is + re-parsed to validate JSON formatting. The `organizationName` and + `name` fields are immutable after creation. Concurrency is managed + via `etag`; mismatches return a 409 Conflict. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated SynonymSet. """ from synapseclient import Synapse @@ -255,9 +340,21 @@ async def list_synonym_sets( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """List SynonymSets, optionally filtered by Organization. + """List SynonymSet objects, optionally filtered by Organization. + + Arguments: + organization_name: If organizationName is null, all synonym sets + across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListSynonymSetsResponse, containing a + page of SynonymSets and a nextPageToken if more results exist. """ from synapseclient import Synapse @@ -274,9 +371,18 @@ async def create_search_configuration( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Create a SearchConfiguration. + """Create a new SearchConfiguration within the specified Organization. + + Arguments: + request: The SearchConfiguration object body. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created SearchConfiguration. """ from synapseclient import Synapse @@ -291,9 +397,19 @@ async def get_search_configuration( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Get a SearchConfiguration by ID. + """Get a SearchConfiguration by its ID. + + Arguments: + search_configuration_id: The ID of the search configuration to + retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested SearchConfiguration. """ from synapseclient import Synapse @@ -312,6 +428,17 @@ async def update_search_configuration( """Update a SearchConfiguration. + + Arguments: + search_configuration_id: The path ID (must match the request body's + ID). + request: The updated SearchConfiguration object body. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated SearchConfiguration. """ from synapseclient import Synapse @@ -328,9 +455,22 @@ async def list_search_configurations( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """List SearchConfigurations, optionally filtered by Organization. + """List SearchConfiguration objects, optionally filtered by Organization. + + Arguments: + organization_name: If organizationName is null, all search + configurations across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListSearchConfigurationsResponse, + containing a page of SearchConfigurations and a nextPageToken if more + results exist. """ from synapseclient import Synapse @@ -348,7 +488,9 @@ async def bind_search_config_to_entity( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Bind a SearchConfiguration to an entity. Replaces any existing binding. + """Bind a SearchConfiguration to an entity (typically a project). The + caller must have EDIT permission on the entity. Replaces any existing + binding on that entity. @@ -360,7 +502,7 @@ async def bind_search_config_to_entity( instance from the Synapse class constructor. Returns: - The created SearchConfigBinding. + A dictionary representing the created SearchConfigBinding. """ from synapseclient import Synapse @@ -379,10 +521,20 @@ async def get_search_config_binding( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Get the effective SearchConfigBinding for an entity by walking up the - hierarchy. + """Get the effective SearchConfigBinding for an entity. Walks up the + entity hierarchy (entity -> folder -> project) and returns the first + binding found on the entity or any ancestor. + + Arguments: + entity_id: The ID of the entity to look up. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the effective SearchConfigBinding. """ from synapseclient import Synapse @@ -398,6 +550,15 @@ async def clear_search_config_binding( """Clear the SearchConfigBinding on a specific entity. + + Arguments: + entity_id: The ID of the entity whose binding to clear. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + None """ from synapseclient import Synapse @@ -410,19 +571,25 @@ async def autocomplete_search( *, synapse_client: Optional["Synapse"] = None, ) -> Dict[str, Any]: - """Synchronous autocomplete search. Caps results at 8. + """Perform a synchronous autocomplete search query against a SearchIndex. + Purpose-built for type-ahead input: no pagination, sorting customization, + or aggregations are available; the server caps every response at 8 hits, + ordered by relevance. For more complex search needs, use the asynchronous + search endpoint instead. Arguments: - request: SearchAutocompleteRequest body (must include searchIndexId and - a searchQuery with a prefix-style query). + request: The SearchAutocompleteRequest body. Only two top-level keys + are permitted in the `searchQuery`: `query` (required, must use + one of `prefix`, `match_phrase_prefix`, or `match_bool_prefix`) and + `_source` (optional, a source filter to narrow returned fields). synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - SearchQueryResults. + A dictionary representing the SearchQueryResults, capped at 8 hits. """ from synapseclient import Synapse From 4fcb624b24a3977a610b4902ab1223bf28f5da6f Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 12:13:34 -0400 Subject: [PATCH 14/30] fix documentation --- docs/reference/json_schema.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/json_schema.md b/docs/reference/json_schema.md index 13dd5a606..e017be3de 100644 --- a/docs/reference/json_schema.md +++ b/docs/reference/json_schema.md @@ -3,7 +3,7 @@ !!! warning "Deprecated — removed in v5.0.0" The legacy `synapseclient.services.json_schema` classes (`JsonSchema`, `JsonSchemaVersion`, `JsonSchemaOrganization`) are deprecated and will be removed in - **v5.0.0**. Use the object-oriented `JSONSchema` and `SchemaOrganization` models + **v5.0.0**. Use the object-oriented `JSONSchema` and `Organization` models instead — see the [Working with JSON Schema tutorial](../tutorials/python/json_schema.md). ::: synapseclient.services.json_schema From 27b6457ce6d9b55a55fbc074e7ed29477db132be Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 14:47:21 -0400 Subject: [PATCH 15/30] add docstring example --- synapseclient/models/search_management.py | 79 +++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 5d8a9af95..7b6f18ecb 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -197,7 +197,86 @@ class TextAnalyzer(OrgScopedResource): """A shareable, named OpenSearch custom analyzer. Used to configure how text is tokenized for a search index. + A TextAnalyzer belongs to an Organization, referenced by `organization_name`. + Find an Organization you already have access to with + `synapseclient.models.organization.list_organizations()`, or create one with + `synapseclient.models.Organization` before creating a TextAnalyzer. + Represents a [Synapse TextAnalyzer](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/TextAnalyzer.html). + + Example: Create a TextAnalyzer. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import TextAnalyzer + + syn = Synapse() + syn.login() + + analyzer = TextAnalyzer( + organization_name="my.existing.organization", + name="stemmed_english", + description="English analyzer with stemming and lowercase filtering", + settings={ + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "porter_stem"], + } + } + }, + ) + analyzer = analyzer.store() + print(f"Created TextAnalyzer: {analyzer.id} ({analyzer.qualified_name})") + ``` + + Example: Get an existing TextAnalyzer by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import TextAnalyzer + + syn = Synapse() + syn.login() + + analyzer = TextAnalyzer(id="12345").get() + print(analyzer.name, analyzer.settings) + ``` + + Example: Update an existing TextAnalyzer. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import TextAnalyzer + + syn = Synapse() + syn.login() + + analyzer = TextAnalyzer(id="12345").get() + analyzer.description = "Updated description" + analyzer.settings["analyzer"]["default"]["filter"].append("asciifolding") + analyzer = analyzer.store() + print(f"Updated TextAnalyzer etag: {analyzer.etag}") + ``` + + Example: List TextAnalyzers in an Organization. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import TextAnalyzer + + syn = Synapse() + syn.login() + + analyzers = TextAnalyzer.list(organization_name="my.existing.organization") + for analyzer in analyzers: + print(analyzer.id, analyzer.qualified_name) + ``` """ _CREATE_FN = staticmethod(create_text_analyzer) From 6377fd88a2184d829f4c07bd67c7371d45edaacb Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 15:19:49 -0400 Subject: [PATCH 16/30] add examples to all these data classes --- synapseclient/models/search_management.py | 197 ++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 7b6f18ecb..49950c210 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -478,7 +478,85 @@ class SynonymSet(OrgScopedResource): Referenced by qualified name `{organizationName}-{name}` from a TextAnalyzer's `settings.filter` registry map via `{"$ref": "{organizationName}-{name}"}`. + A SynonymSet belongs to an Organization, referenced by `organization_name`. + Find an Organization you already have access to with + `synapseclient.models.organization.list_organizations()`, or create one with + `synapseclient.models.Organization` before creating a SynonymSet. + Represents a [Synapse SynonymSet](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SynonymSet.html). + + Example: Create a SynonymSet. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SynonymSet + + syn = Synapse() + syn.login() + + synonyms = SynonymSet( + organization_name="my.existing.organization", + name="disease_synonyms", + description="Common synonyms for disease terms", + definition={ + "type": "synonym_graph", + "synonyms": [ + # comma-separated = equivalence set, matches either way + "tumor, neoplasm, cancer", + # "=>" = one-way mapping, "AD" expands to the right side only + "AD => Alzheimer's disease", + ], + }, + ) + synonyms = synonyms.store() + print(f"Created SynonymSet: {synonyms.id} ({synonyms.qualified_name})") + ``` + + Example: Get an existing SynonymSet by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SynonymSet + + syn = Synapse() + syn.login() + + synonyms = SynonymSet(id="12345").get() + print(synonyms.name, synonyms.definition) + ``` + + Example: Update an existing SynonymSet. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SynonymSet + + syn = Synapse() + syn.login() + + synonyms = SynonymSet(id="12345").get() + synonyms.definition["synonyms"].append("MI, myocardial infarction, heart attack") + synonyms = synonyms.store() + print(f"Updated SynonymSet etag: {synonyms.etag}") + ``` + + Example: List SynonymSets in an Organization. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SynonymSet + + syn = Synapse() + syn.login() + + synonym_sets = SynonymSet.list(organization_name="my.existing.organization") + for synonyms in synonym_sets: + print(synonyms.id, synonyms.qualified_name) + ``` """ _CREATE_FN = staticmethod(create_synonym_set) @@ -560,7 +638,80 @@ class SearchConfiguration(OrgScopedResource): """Bundles the index-wide default analyzer and per-column overrides used to build a SearchIndex. + A SearchConfiguration belongs to an Organization, referenced by + `organization_name`. Find an Organization you already have access to with + `synapseclient.models.organization.list_organizations()`, or create one with + `synapseclient.models.Organization` before creating a SearchConfiguration. + Represents a [Synapse SearchConfiguration](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchConfiguration.html). + + Example: Create a SearchConfiguration. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfiguration + + syn = Synapse() + syn.login() + + config = SearchConfiguration( + organization_name="my.existing.organization", + name="default_config", + description="Default search configuration for this project's indexes", + # Reference a previously-created TextAnalyzer by its qualified name... + default_analyzer={"$ref": "my.existing.organization-stemmed_english"}, + ) + config = config.store() + print(f"Created SearchConfiguration: {config.id} ({config.qualified_name})") + ``` + + Example: Get an existing SearchConfiguration by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfiguration + + syn = Synapse() + syn.login() + + config = SearchConfiguration(id="12345").get() + print(config.name, config.default_analyzer) + ``` + + Example: Update an existing SearchConfiguration. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfiguration + + syn = Synapse() + syn.login() + + config = SearchConfiguration(id="12345").get() + config.column_analyzer_overrides.append( + {"$ref": "my.existing.organization-abstract_overrides"} + ) + config = config.store() + print(f"Updated SearchConfiguration etag: {config.etag}") + ``` + + Example: List SearchConfigurations in an Organization. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfiguration + + syn = Synapse() + syn.login() + + configs = SearchConfiguration.list(organization_name="my.existing.organization") + for config in configs: + print(config.id, config.qualified_name) + ``` """ _CREATE_FN = staticmethod(create_search_configuration) @@ -714,6 +865,25 @@ async def store_async( Raises: ValueError: If ``object_id`` or ``search_configuration_id`` is not set. + + Example: Bind a SearchConfiguration to a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + binding = SearchConfigBinding( + object_id="syn12345", + search_configuration_id="6789", + ) + binding = binding.store() + print(f"Bound SearchConfiguration {binding.search_configuration_id} " + f"to {binding.object_id}") + ``` """ if not self.object_id: raise ValueError("SearchConfigBinding must have an object_id set.") @@ -742,6 +912,20 @@ async def get_async(self, *, synapse_client: Optional["Synapse"] = None) -> "Sel Raises: ValueError: If ``object_id`` is not set. + + Example: Get the effective binding for a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + binding = SearchConfigBinding(object_id="syn12345").get() + print(binding.search_configuration_id) + ``` """ if not self.object_id: raise ValueError("SearchConfigBinding must have an object_id set.") @@ -760,6 +944,19 @@ async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> N Raises: ValueError: If ``object_id`` is not set. + + Example: Clear the binding on a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + SearchConfigBinding(object_id="syn12345").delete() + ``` """ if not self.object_id: raise ValueError("SearchConfigBinding must have an object_id set.") From 0630b540e453da4b71eac1308bcfd84ea7a487fa Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 15:37:14 -0400 Subject: [PATCH 17/30] add examples and expand docstring for searchIndex and protocol --- .../models/protocols/search_index_protocol.py | 131 ++++++++++++++++- synapseclient/models/search_index.py | 137 +++++++++++++++++- 2 files changed, 261 insertions(+), 7 deletions(-) diff --git a/synapseclient/models/protocols/search_index_protocol.py b/synapseclient/models/protocols/search_index_protocol.py index 4714806ca..5bc7bc94e 100644 --- a/synapseclient/models/protocols/search_index_protocol.py +++ b/synapseclient/models/protocols/search_index_protocol.py @@ -21,7 +21,45 @@ def store( job_timeout: int = 600, synapse_client: Optional[Synapse] = None, ) -> "Self": - """Store metadata about a SearchIndex including the annotations.""" + """Store metadata about a SearchIndex including the annotations. Creates + a new SearchIndex if `id` is not set, or updates the existing one + otherwise. `defining_sql` must be set before calling this. + + Arguments: + dry_run: If True, will not actually store the SearchIndex but will log + to the console what would be created or updated. + job_timeout: The maximum amount of time to wait for the index-build job + to complete before raising an error. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated with the server-assigned ID, etag, and columns. + + Raises: + ValueError: If `defining_sql` is not set. + + Example: Create a new SearchIndex. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex( + name="My Search Index", + parent_id="syn12345", + # syn67890 must be a table or a view; multi-entity JOINs are not supported + defining_sql="SELECT * FROM syn67890", + ) + index = index.store() + print(f"Created SearchIndex: {index.id}") + ``` + """ return self def get( @@ -31,11 +69,59 @@ def get( *, synapse_client: Optional[Synapse] = None, ) -> "Self": - """Get the metadata about the SearchIndex from Synapse.""" + """Get the metadata about the SearchIndex from Synapse. Either `id`, or + `name` and `parent_id`, must be set before calling this. + + Arguments: + include_columns: If True, will include the columns derived from + `defining_sql` on the returned SearchIndex. + include_activity: If True, will include the provenance activity on + the returned SearchIndex. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the Synapse response. + + Example: Get a SearchIndex by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex(id="syn12345").get() + print(index.name, index.defining_sql) + ``` + """ return self def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: - """Delete the SearchIndex from Synapse.""" + """Delete the SearchIndex from Synapse. `id` must be set before calling + this. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Example: Delete a SearchIndex by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + SearchIndex(id="syn12345").delete() + ``` + """ return None def autocomplete( @@ -45,5 +131,42 @@ def autocomplete( *, synapse_client: Optional[Synapse] = None, ) -> List["SearchHit"]: - """Run a synchronous autocomplete search against this index.""" + """Run a synchronous autocomplete search against this index. The + autocomplete endpoint allowlists only prefix-style queries (`prefix`, + `match_phrase_prefix`, or `match_bool_prefix`) and caps results at 8. + + Arguments: + query: The top-level OpenSearch Query DSL clause; restricted + server-side to `prefix`, `match_phrase_prefix`, or + `match_bool_prefix`. + source: Optional source filter selecting which columns are returned + on each hit. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The matching SearchHits, capped at 8. + + Raises: + ValueError: If the ``id`` attribute has not been set. + + Example: Autocomplete titles beginning with "alz". +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex(id="syn12345") + hits = index.autocomplete( + query={"match_phrase_prefix": {"title": {"query": "alz"}}}, + ) + for hit in hits: + print(hit.row_id, hit.fields) + ``` + """ return [] diff --git a/synapseclient/models/search_index.py b/synapseclient/models/search_index.py index aad5a5783..03d9a038e 100644 --- a/synapseclient/models/search_index.py +++ b/synapseclient/models/search_index.py @@ -93,6 +93,7 @@ class SearchIndex( index = SearchIndex( name="My Search Index", parent_id="syn12345", + # syn67890 must be a table or a view; multi-entity JOINs are not supported defining_sql="SELECT * FROM syn67890", ) index = index.store() @@ -101,18 +102,50 @@ class SearchIndex( """ id: Optional[str] = None + """The unique immutable ID for this entity. A new ID will be generated for + new Entities. Once issued, this ID is guaranteed to never change or be + re-issued.""" + name: Optional[str] = None + """The name of this entity. Must be 256 characters or less. Names may only + contain: letters, numbers, spaces, underscores, hyphens, periods, plus + signs, apostrophes, and parentheses.""" + description: Optional[str] = None + """The description of this entity. Must be 1000 characters or less.""" + etag: Optional[str] = field(default=None, compare=False) + """Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. Since the E-Tag changes every time an entity is + updated it is used to detect when a client's current representation of an + entity is out-of-date.""" + created_on: Optional[str] = field(default=None, compare=False) + """The date this entity was created.""" + modified_on: Optional[str] = field(default=None, compare=False) + """The date this entity was last modified.""" + created_by: Optional[str] = field(default=None, compare=False) + """The ID of the user that created this entity.""" + modified_by: Optional[str] = field(default=None, compare=False) + """The ID of the user that last modified this entity.""" + parent_id: Optional[str] = None + """The ID of the Entity that is the parent of this Entity.""" + version_number: Optional[int] = field(default=None, compare=False) + """The version number issued to this version on the object.""" + version_label: Optional[str] = None + """The version label for this entity.""" + version_comment: Optional[str] = None + """The version comment for this entity.""" + is_latest_version: Optional[bool] = field(default=None, compare=False) + """If this is the latest version of the object.""" columns: Optional[OrderedDict[str, Column]] = field( default_factory=OrderedDict, compare=False @@ -221,7 +254,49 @@ async def store_async( job_timeout: int = 600, synapse_client: Optional[Synapse] = None, ) -> "Self": - """Asynchronously store the SearchIndex entity.""" + """Asynchronously store the SearchIndex entity. Creates a new SearchIndex + if `id` is not set, or updates the existing one otherwise. `defining_sql` + must be set before calling this. + + Arguments: + dry_run: If True, will not actually store the SearchIndex but will log + to the console what would be created or updated. + job_timeout: The maximum amount of time to wait for the index-build job + to complete before raising an error. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated with the server-assigned ID, etag, and columns. + + Raises: + ValueError: If `defining_sql` is not set. + + Example: Create a new SearchIndex. +   + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + async def main(): + syn = Synapse() + syn.login() + + index = SearchIndex( + name="My Search Index", + parent_id="syn12345", + # syn67890 must be a table or a view; + defining_sql="SELECT * FROM syn67890", + ) + index = await index.store_async() + print(f"Created SearchIndex: {index.id}") + + asyncio.run(main()) + ``` + """ if not self.defining_sql: raise ValueError( "The defining_sql attribute must be set for a SearchIndex." @@ -237,7 +312,39 @@ async def get_async( *, synapse_client: Optional[Synapse] = None, ) -> "Self": - """Asynchronously fetch the SearchIndex metadata.""" + """Asynchronously fetch the SearchIndex metadata. Either `id`, or `name` + and `parent_id`, must be set before calling this. + + Arguments: + include_columns: If True, will include the columns derived from + `defining_sql` on the returned SearchIndex. + include_activity: If True, will include the provenance activity on + the returned SearchIndex. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the Synapse response. + + Example: Get a SearchIndex by ID. +   + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + async def main(): + syn = Synapse() + await syn.login_async() + + index = await SearchIndex(id="syn12345").get_async() + print(index.name, index.defining_sql) + + asyncio.run(main()) + ``` + """ return await super().get_async( include_columns=include_columns, include_activity=include_activity, @@ -245,7 +352,31 @@ async def get_async( ) async def delete_async(self, *, synapse_client: Optional[Synapse] = None) -> None: - """Asynchronously delete this SearchIndex from Synapse.""" + """Asynchronously delete this SearchIndex from Synapse. `id` must be set + before calling this. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Example: Delete a SearchIndex by ID. +   + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + async def main(): + syn = Synapse() + await syn.login_async() + + await SearchIndex(id="syn12345").delete_async() + + asyncio.run(main()) + ``` + """ await super().delete_async(synapse_client=synapse_client) async def autocomplete_async( From 5a794f3e711ceb4d7b847240aa23f19916daa308 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Wed, 22 Jul 2026 15:41:58 -0400 Subject: [PATCH 18/30] add example to protocol; also separate protocol from search management --- .../protocols/search_management_protocol.py | 106 ++++++++++++++++++ synapseclient/models/search_management.py | 21 +--- 2 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 synapseclient/models/protocols/search_management_protocol.py diff --git a/synapseclient/models/protocols/search_management_protocol.py b/synapseclient/models/protocols/search_management_protocol.py new file mode 100644 index 000000000..11d185b7f --- /dev/null +++ b/synapseclient/models/protocols/search_management_protocol.py @@ -0,0 +1,106 @@ +"""Protocol for the specific methods of this class that have synchronous counterparts +generated at runtime.""" + +from typing import TYPE_CHECKING, Optional, Protocol + +from typing_extensions import Self + +if TYPE_CHECKING: + from synapseclient import Synapse + + +class SearchConfigBindingSynchronousProtocol(Protocol): + """Synchronous interface for SearchConfigBinding operations.""" + + def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Bind ``search_configuration_id`` to the entity ``object_id``. Replaces + any existing binding on that entity. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the created SearchConfigBinding. + + Raises: + ValueError: If ``object_id`` or ``search_configuration_id`` is not set. + + Example: Bind a SearchConfiguration to a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + binding = SearchConfigBinding( + object_id="syn12345", + search_configuration_id="6789", + ) + binding = binding.store() + print(f"Bound SearchConfiguration {binding.search_configuration_id} " + f"to {binding.object_id}") + ``` + """ + return self + + def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Get the effective binding for the entity ``object_id``, resolved by + walking up the entity hierarchy. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the resolved SearchConfigBinding. + + Raises: + ValueError: If ``object_id`` is not set. + + Example: Get the effective binding for a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + binding = SearchConfigBinding(object_id="syn12345").get() + print(binding.search_configuration_id) + ``` + """ + return self + + def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: + """Clear the binding on the entity ``object_id``. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Raises: + ValueError: If ``object_id`` is not set. + + Example: Clear the binding on a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + SearchConfigBinding(object_id="syn12345").delete() + ``` + """ + return None diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 49950c210..2722600f4 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -50,6 +50,9 @@ from synapseclient.core.constants import concrete_types from synapseclient.core.utils import delete_none_keys from synapseclient.models.mixins.asynchronous_job import AsynchronousCommunicator +from synapseclient.models.protocols.search_management_protocol import ( + SearchConfigBindingSynchronousProtocol, +) from synapseclient.models.table_components import SelectColumn if TYPE_CHECKING: @@ -796,25 +799,9 @@ def to_synapse_request(self) -> Dict[str, Any]: return body -class SearchConfigBindingProtocol(Protocol): - """Synchronous interface for SearchConfigBinding operations.""" - - def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": - """Bind ``search_configuration_id`` to the entity ``object_id``.""" - return self - - def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": - """Get the effective binding for the entity ``object_id``.""" - return self - - def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: - """Clear the binding on the entity ``object_id``.""" - return None - - @dataclass @async_to_sync -class SearchConfigBinding(SearchConfigBindingProtocol): +class SearchConfigBinding(SearchConfigBindingSynchronousProtocol): """Attaches a SearchConfiguration to an entity. When a SearchIndex is built, the effective SearchConfiguration is resolved by walking up the entity hierarchy (entity -> folder -> project) and using the first binding found. From 1ebad6f7fbc536b7b9c0053eed9e87461bc701ba Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 10:22:50 -0400 Subject: [PATCH 19/30] added integration test for search_management.py --- synapseclient/models/search_management.py | 80 +++++ .../async/test_search_management_async.py | 299 ++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 tests/integration/synapseclient/models/async/test_search_management_async.py diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 2722600f4..5bc5ba086 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -394,7 +394,87 @@ def to_synapse_request(self) -> Dict[str, Any]: class ColumnAnalyzerOverride(OrgScopedResource): """A shareable bundle of per-column analyzer assignments. Each entry binds one column to an analyzer; + A ColumnAnalyzerOverride belongs to an Organization, referenced by + `organization_name`. Find an Organization you already have access to with + `synapseclient.models.organization.list_organizations()`, or create one with + `synapseclient.models.Organization` before creating a ColumnAnalyzerOverride. + Represents a [Synapse ColumnAnalyzerOverride](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/ColumnAnalyzerOverride.html). + + Example: Create a ColumnAnalyzerOverride. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import ColumnAnalyzerOverride, ColumnAnalyzerOverrideEntry + + syn = Synapse() + syn.login() + + override = ColumnAnalyzerOverride( + organization_name="my.existing.organization", + name="disease_column_overrides", + description="Use a keyword analyzer for the disease_code column", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="disease_code", + analyzer={"analyzer": {"default": {"type": "keyword"}}}, + ), + ], + ) + override = override.store() + print(f"Created ColumnAnalyzerOverride: {override.id} ({override.qualified_name})") + ``` + + Example: Get an existing ColumnAnalyzerOverride by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import ColumnAnalyzerOverride + + syn = Synapse() + syn.login() + + override = ColumnAnalyzerOverride(id="12345").get() + print(override.name, override.overrides) + ``` + + Example: Update an existing ColumnAnalyzerOverride. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import ColumnAnalyzerOverride, ColumnAnalyzerOverrideEntry + + syn = Synapse() + syn.login() + + override = ColumnAnalyzerOverride(id="12345").get() + override.overrides.append( + ColumnAnalyzerOverrideEntry( + column_name="title", + analyzer={"analyzer": {"default": {"type": "standard"}}}, + ) + ) + override = override.store() + print(f"Updated ColumnAnalyzerOverride etag: {override.etag}") + ``` + + Example: List ColumnAnalyzerOverrides in an Organization. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import ColumnAnalyzerOverride + + syn = Synapse() + syn.login() + + overrides = ColumnAnalyzerOverride.list(organization_name="my.existing.organization") + for override in overrides: + print(override.id, override.qualified_name) + ``` """ _CREATE_FN = staticmethod(create_column_analyzer_override) diff --git a/tests/integration/synapseclient/models/async/test_search_management_async.py b/tests/integration/synapseclient/models/async/test_search_management_async.py new file mode 100644 index 000000000..fbe0485d8 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_search_management_async.py @@ -0,0 +1,299 @@ +"""Integration tests for the org-scoped search-management resources: TextAnalyzer, +SynonymSet, ColumnAnalyzerOverride, SearchConfiguration, and SearchConfigBinding. + +TextAnalyzer, SynonymSet, ColumnAnalyzerOverride, and SearchConfiguration have no +delete endpoint on the Synapse REST API, so these tests do not create or update +them -- they only get and list a fixed set of resources pre-seeded under the +`SEARCH_ORG_NAME` organization (identified by the `*_NAME`/`*_ID` constants +below), which keeps the tests idempotent and safe to run concurrently on CI. +Because an Organization cannot be deleted once one of these resources has been +attached to it, `SEARCH_ORG_NAME` is a permanent, shared test Organization +rather than one created fresh per test run. +Each resource's class docstring shows the `store()` call used to seed it. +SearchConfigBinding does support delete/clear; its test creates and tears down +its own binding on a freshly-created Folder. +""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import ( + ColumnAnalyzerOverride, + Folder, + Project, + SearchConfigBinding, + SearchConfiguration, + SynonymSet, + TextAnalyzer, +) + +SEARCH_ORG_NAME = "SYNPY.TEST.SEARCH.MANAGEMENT" +TEXT_ANALYZER_NAME = "test_analyzer" +TEXT_ANALYZER_ID = "1023" +SYNONYM_SET_NAME = "test_synonyms" +SYNONYM_SET_ID = "32" +COLUMN_ANALYZER_OVERRIDE_NAME = "disease_column_overrides" +COLUMN_ANALYZER_OVERRIDE_ID = "15" +TEST_CONFIG_NAME = "test_config" +TEST_CONFIG_ID = "17" + + +@pytest.fixture(scope="function") +async def folder( + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], +) -> Folder: + """A fresh Folder under the shared test Project, used as the bind target for + SearchConfigBinding tests instead of the shared Project itself.""" + folder = await Folder( + name=str(uuid.uuid4()), + parent_id=project_model.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(folder.id) + return folder + + +class TestTextAnalyzer: + async def test_get_and_list(self, syn: Synapse) -> None: + """ + Test that a TextAnalyzer can be retrieved by ID and listed in the + organization. + + The TestTextAnalyzer was stored like below: + + from synapseclient import Synapse + from synapseclient.models import TextAnalyzer + + + syn = Synapse() + syn.login() + analyzer = TextAnalyzer( + organization_name="SYNPY.TEST.SEARCH.MANAGEMENT", + name="test_analyzer", + settings={ + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase"], + } + } + }, + ) + analyzer = analyzer.store(synapse_client=syn) + print(f"Created TextAnalyzer: {analyzer.id} ({analyzer.qualified_name})") + """ + # GIVEN a TextAnalyzer definition + name = TEXT_ANALYZER_NAME + settings = { + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase"], + } + } + } + + # AND it can be retrieved by ID with its settings intact + retrieved = await TextAnalyzer(id=TEXT_ANALYZER_ID).get_async( + synapse_client=syn + ) + assert retrieved.settings == settings + # AND it appears when listing analyzers in the organization + listed = await TextAnalyzer.list_async( + organization_name=SEARCH_ORG_NAME, synapse_client=syn + ) + assert name in [item.name for item in listed] + + +class TestSynonymSet: + async def test_get_and_list(self, syn: Synapse) -> None: + """ + Test that a SynonymSet can be retrieved by ID and listed in the + organization. + + The TestSynonymSet was stored like below: + + from synapseclient import Synapse + from synapseclient.models import SynonymSet + + + syn = Synapse() + syn.login() + synonyms = SynonymSet( + organization_name="SYNPY.TEST.SEARCH.MANAGEMENT", + name="test_synonyms", + definition={ + "type": "synonym_graph", + "synonyms": ["tumor, neoplasm, cancer"], + }, + ) + synonyms = synonyms.store(synapse_client=syn) + print(f"Created SynonymSet: {synonyms.id} ({synonyms.qualified_name})") + """ + # GIVEN a SynonymSet definition + name = SYNONYM_SET_NAME + definition = { + "type": "synonym_graph", + "synonyms": ["tumor, neoplasm, cancer"], + } + + # AND it can be retrieved by ID with its definition intact + retrieved = await SynonymSet(id=SYNONYM_SET_ID).get_async(synapse_client=syn) + assert retrieved.definition == definition + # AND it appears when listing synonym sets in the organization + listed = await SynonymSet.list_async( + organization_name=SEARCH_ORG_NAME, synapse_client=syn + ) + assert name in [item.name for item in listed] + + +class TestColumnAnalyzerOverride: + async def test_get_and_list(self, syn: Synapse) -> None: + """ + Test that a ColumnAnalyzerOverride can be retrieved by ID and listed in the + organization. + + The TestColumnAnalyzerOverride was stored like below: + + from synapseclient import Synapse + from synapseclient.models import ColumnAnalyzerOverride, ColumnAnalyzerOverrideEntry + + + syn = Synapse() + syn.login() + override = ColumnAnalyzerOverride( + organization_name="SYNPY.TEST.SEARCH.MANAGEMENT", + name="disease_column_overrides", + description="Use a keyword analyzer for the disease_code column", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="disease_code", + analyzer={"analyzer": {"default": {"type": "keyword"}}}, + ), + ], + ) + override = override.store(synapse_client=syn) + print(f"Created ColumnAnalyzerOverride: {override.id} ({override.qualified_name})") + """ + # GIVEN a ColumnAnalyzerOverride with a single inline analyzer entry + name = COLUMN_ANALYZER_OVERRIDE_NAME + + # AND it can be retrieved by ID with its entry intact + retrieved = await ColumnAnalyzerOverride( + id=COLUMN_ANALYZER_OVERRIDE_ID + ).get_async(synapse_client=syn) + assert retrieved.overrides[0].column_name == "disease_code" + # AND it appears when listing overrides in the organization + listed = await ColumnAnalyzerOverride.list_async( + organization_name=SEARCH_ORG_NAME, synapse_client=syn + ) + assert name in [item.name for item in listed] + + +class TestSearchConfiguration: + async def test_get_and_list(self, syn: Synapse) -> None: + """ + Test that a SearchConfiguration can be retrieved by ID and listed in the + organization. + + The TestSearchConfiguration was stored like below: + + from synapseclient import Synapse + from synapseclient.models import ColumnAnalyzerOverride, ColumnAnalyzerOverrideEntry, SearchConfiguration, TextAnalyzer + + + syn = Synapse() + syn.login() + analyzer = TextAnalyzer( + organization_name="SYNPY.TEST.SEARCH.MANAGEMENT", + name="test_analyzer", + settings={ + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase"], + } + } + }, + ).store(synapse_client=syn) + override = ColumnAnalyzerOverride( + organization_name="SYNPY.TEST.SEARCH.MANAGEMENT", + name="disease_column_overrides", + description="Use a keyword analyzer for the disease_code column", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="disease_code", + analyzer={"analyzer": {"default": {"type": "keyword"}}}, + ), + ], + ).store(synapse_client=syn) + config = SearchConfiguration( + organization_name="SYNPY.TEST.SEARCH.MANAGEMENT", + name="test_config", + default_analyzer={"$ref": analyzer.qualified_name}, + column_analyzer_overrides=[{"$ref": override.qualified_name}], + ) + config = config.store(synapse_client=syn) + print(f"Created SearchConfiguration: {config.id} ({config.qualified_name})") + """ + # GIVEN a SearchConfiguration referencing a TextAnalyzer and + # ColumnAnalyzerOverride by qualified name + name = TEST_CONFIG_NAME + analyzer_ref = {"$ref": f"{SEARCH_ORG_NAME}-{TEXT_ANALYZER_NAME}"} + override_ref = {"$ref": f"{SEARCH_ORG_NAME}-{COLUMN_ANALYZER_OVERRIDE_NAME}"} + + # AND it can be retrieved by ID with its analyzer references intact + retrieved = await SearchConfiguration(id=TEST_CONFIG_ID).get_async( + synapse_client=syn + ) + assert retrieved.default_analyzer == analyzer_ref + assert retrieved.column_analyzer_overrides == [override_ref] + # AND it appears when listing configurations in the organization + listed = await SearchConfiguration.list_async( + organization_name=SEARCH_ORG_NAME, synapse_client=syn + ) + assert name in [item.name for item in listed] + + +class TestSearchConfigBinding: + async def test_bind_get_and_clear(self, syn: Synapse, folder: Folder) -> None: + """ + Test that a SearchConfiguration can be bound to a Folder, that the + effective binding resolves to it on a fresh get, and that clearing the + binding removes it. + """ + # GIVEN a SearchConfiguration to bind + config = await SearchConfiguration(id=TEST_CONFIG_ID).get_async( + synapse_client=syn + ) + # WHEN binding it to a Folder + binding = await SearchConfigBinding( + object_id=folder.id, + search_configuration_id=config.id, + ).store_async(synapse_client=syn) + + # THEN the binding is created for that entity + assert binding.bind_id is not None + assert binding.object_id == folder.id.removeprefix("syn") + assert binding.search_configuration_id == config.id + + # AND getting the effective binding on the same entity resolves to it + effective = await SearchConfigBinding(object_id=folder.id).get_async( + synapse_client=syn + ) + assert effective.search_configuration_id == config.id + + # WHEN clearing the binding + await SearchConfigBinding(object_id=folder.id).delete_async(synapse_client=syn) + + # THEN there is no longer an effective binding on that entity + with pytest.raises(SynapseHTTPError): + await SearchConfigBinding(object_id=folder.id).get_async(synapse_client=syn) From 2f407433825580e8e92d059a22b6ce8215398e65 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 10:30:46 -0400 Subject: [PATCH 20/30] add to docstring --- synapseclient/models/search_management.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index 5bc5ba086..e0636e276 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -207,6 +207,10 @@ class TextAnalyzer(OrgScopedResource): Represents a [Synapse TextAnalyzer](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/TextAnalyzer.html). + Note: TextAnalyzer has no delete endpoint on the Synapse REST API. Once + created, it cannot be removed, and its owning Organization can no longer be + deleted either. Choose `organization_name` and `name` deliberately. + Example: Create a TextAnalyzer.   @@ -401,6 +405,10 @@ class ColumnAnalyzerOverride(OrgScopedResource): Represents a [Synapse ColumnAnalyzerOverride](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/ColumnAnalyzerOverride.html). + Note: ColumnAnalyzerOverride has no delete endpoint on the Synapse REST API. + Once created, it cannot be removed, and its owning Organization can no + longer be deleted either. Choose `organization_name` and `name` deliberately. + Example: Create a ColumnAnalyzerOverride.   @@ -568,6 +576,10 @@ class SynonymSet(OrgScopedResource): Represents a [Synapse SynonymSet](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SynonymSet.html). + Note: SynonymSet has no delete endpoint on the Synapse REST API. Once + created, it cannot be removed, and its owning Organization can no longer be + deleted either. Choose `organization_name` and `name` deliberately. + Example: Create a SynonymSet.   @@ -728,6 +740,10 @@ class SearchConfiguration(OrgScopedResource): Represents a [Synapse SearchConfiguration](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchConfiguration.html). + Note: SearchConfiguration has no delete endpoint on the Synapse REST API. + Once created, it cannot be removed, and its owning Organization can no + longer be deleted either. Choose `organization_name` and `name` deliberately. + Example: Create a SearchConfiguration.   From e31672c11b68f2d653b13a8afda0e58ff0ed723e Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 11:25:29 -0400 Subject: [PATCH 21/30] move test related to auto complete to unit test file for search index; added unit test for search index --- .../async/unit_test_search_index_async.py | 209 ++++++++++++++++++ .../unit_test_search_management_async.py | 48 ---- 2 files changed, 209 insertions(+), 48 deletions(-) create mode 100644 tests/unit/synapseclient/models/async/unit_test_search_index_async.py diff --git a/tests/unit/synapseclient/models/async/unit_test_search_index_async.py b/tests/unit/synapseclient/models/async/unit_test_search_index_async.py new file mode 100644 index 000000000..ead5d0f7f --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_search_index_async.py @@ -0,0 +1,209 @@ +"""Unit tests for the SearchIndex entity model.""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.constants import concrete_types +from synapseclient.models.mixins.table_components import ( + DeleteMixin, + GetMixin, + TableStoreMixin, +) +from synapseclient.models.search_index import SearchIndex +from synapseclient.models.search_management import SearchHit + + +class TestSearchIndex: + synapse_response = { + "id": "syn1234", + "name": "test_search_index", + "description": "test_description", + "parentId": "syn5678", + "etag": "etag_value", + "createdOn": "createdOn_value", + "createdBy": "createdBy_value", + "modifiedOn": "modifiedOn_value", + "modifiedBy": "modifiedBy_value", + "versionNumber": 1, + "versionLabel": "versionLabel_value", + "versionComment": "versionComment_value", + "isLatestVersion": True, + "definingSQL": "SELECT * FROM syn9999", + "searchConfigurationId": "42", + "annotations": {"key": "value"}, + } + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def test_fill_from_dict(self): + # GIVEN an empty SearchIndex + index = SearchIndex() + # WHEN I fill it from a Synapse response + index.fill_from_dict(self.synapse_response) + # THEN I expect the SearchIndex to be filled with the expected values + assert index.id == self.synapse_response["id"] + assert index.name == self.synapse_response["name"] + assert index.description == self.synapse_response["description"] + assert index.parent_id == self.synapse_response["parentId"] + assert index.etag == self.synapse_response["etag"] + assert index.created_on == self.synapse_response["createdOn"] + assert index.created_by == self.synapse_response["createdBy"] + assert index.modified_on == self.synapse_response["modifiedOn"] + assert index.modified_by == self.synapse_response["modifiedBy"] + assert index.version_number == self.synapse_response["versionNumber"] + assert index.version_label == self.synapse_response["versionLabel"] + assert index.version_comment == self.synapse_response["versionComment"] + assert index.is_latest_version == self.synapse_response["isLatestVersion"] + assert index.defining_sql == self.synapse_response["definingSQL"] + assert ( + index.search_configuration_id + == self.synapse_response["searchConfigurationId"] + ) + assert index.annotations == self.synapse_response["annotations"] + + def test_fill_from_dict_without_annotations(self): + # GIVEN an empty SearchIndex + index = SearchIndex() + # WHEN I fill it from a Synapse response with set_annotations=False + index.fill_from_dict(self.synapse_response, set_annotations=False) + # THEN I expect annotations to remain untouched + assert index.annotations == {} + + def test_to_synapse_request(self): + # GIVEN a SearchIndex + index = SearchIndex( + id="syn1234", + name="test_search_index", + description="test_description", + parent_id="syn5678", + etag="etag_value", + created_on="createdOn_value", + created_by="createdBy_value", + modified_on="modifiedOn_value", + modified_by="modifiedBy_value", + version_number=1, + version_label="versionLabel_value", + version_comment="versionComment_value", + is_latest_version=True, + defining_sql="SELECT * FROM syn9999", + search_configuration_id="42", + ) + # WHEN I convert it to a Synapse request + request = index.to_synapse_request() + # THEN I expect the entity body to carry the expected values + entity = request["entity"] + assert entity["concreteType"] == concrete_types.SEARCH_INDEX_ENTITY + for key, value in self.synapse_response.items(): + if key != "annotations": + assert entity[key] == value + + async def test_store_async_requires_defining_sql(self): + # GIVEN a SearchIndex without defining_sql + index = SearchIndex(name="test_search_index", parent_id="syn5678") + + with patch.object(TableStoreMixin, "store_async") as mock_super_store_async: + # WHEN I store it THEN a ValueError is raised + with pytest.raises( + ValueError, + match="The defining_sql attribute must be set for a SearchIndex.", + ): + await index.store_async(synapse_client=self.syn) + # AND the super().store_async method is never called + mock_super_store_async.assert_not_called() + + async def test_store_async_with_defining_sql_calls_super(self): + # GIVEN a SearchIndex with defining_sql set + index = SearchIndex( + name="test_search_index", + parent_id="syn5678", + defining_sql="SELECT * FROM syn9999", + ) + + with patch.object(TableStoreMixin, "store_async") as mock_super_store_async: + mock_super_store_async.return_value = index + # WHEN I store it + result = await index.store_async( + dry_run=True, job_timeout=100, synapse_client=self.syn + ) + # THEN the super().store_async method is called with the same arguments + mock_super_store_async.assert_called_once_with( + dry_run=True, job_timeout=100, synapse_client=self.syn + ) + assert result == index + + async def test_get_async_calls_super(self): + # GIVEN a SearchIndex with an id + index = SearchIndex(id="syn1234") + + with patch.object(GetMixin, "get_async") as mock_super_get_async: + mock_super_get_async.return_value = index + # WHEN I get it + result = await index.get_async(synapse_client=self.syn) + # THEN the super().get_async method is called with the default arguments + mock_super_get_async.assert_called_once_with( + include_columns=True, + include_activity=False, + synapse_client=self.syn, + ) + assert result == index + + async def test_delete_async_calls_super(self): + # GIVEN a SearchIndex with an id + index = SearchIndex(id="syn1234") + + with patch.object(DeleteMixin, "delete_async") as mock_super_delete_async: + # WHEN I delete it + await index.delete_async(synapse_client=self.syn) + # THEN the super().delete_async method is called + mock_super_delete_async.assert_called_once_with(synapse_client=self.syn) + + +class TestSearchIndexAutocomplete: + """Dispatch tests for SearchIndex.autocomplete_async.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + async def test_autocomplete_async_dispatch(self): + # GIVEN a SearchIndex with an id + index = SearchIndex(id="syn1") + response = {"hits": [{"rowId": 1, "fields": [{"name": "title", "value": "x"}]}]} + with patch( + "synapseclient.api.autocomplete_search", + new_callable=AsyncMock, + return_value=response, + ) as mock_autocomplete: + # WHEN I run autocomplete + hits = await index.autocomplete_async( + query={"prefix": {"title": {"value": "a"}}}, + source={"includes": ["title"]}, + synapse_client=self.syn, + ) + # THEN the request nests the query/_source under searchQuery for this index + mock_autocomplete.assert_awaited_once_with( + { + "searchIndexId": "syn1", + "searchQuery": { + "query": {"prefix": {"title": {"value": "a"}}}, + "_source": {"includes": ["title"]}, + }, + }, + synapse_client=self.syn, + ) + # AND the response hits deserialize to SearchHit + assert len(hits) == 1 + assert isinstance(hits[0], SearchHit) + assert hits[0].row_id == 1 + + async def test_autocomplete_async_requires_id(self): + # WHEN autocompleting without an id THEN a ValueError is raised + with pytest.raises(ValueError): + await SearchIndex().autocomplete_async( + query={"prefix": {"title": {"value": "a"}}}, + synapse_client=self.syn, + ) diff --git a/tests/unit/synapseclient/models/async/unit_test_search_management_async.py b/tests/unit/synapseclient/models/async/unit_test_search_management_async.py index 6a23eba0a..5f95fb207 100644 --- a/tests/unit/synapseclient/models/async/unit_test_search_management_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_search_management_async.py @@ -10,7 +10,6 @@ from synapseclient import Synapse from synapseclient.core.constants.concrete_types import SEARCH_INDEX_QUERY -from synapseclient.models.search_index import SearchIndex from synapseclient.models.search_management import ( ColumnAnalyzerOverride, ColumnAnalyzerOverrideEntry, @@ -392,53 +391,6 @@ async def test_get_async_requires_id(self): await TextAnalyzer().get_async(synapse_client=self.syn) -class TestSearchIndexAutocomplete: - """Dispatch tests for SearchIndex.autocomplete_async.""" - - @pytest.fixture(autouse=True, scope="function") - def init_syn(self, syn: Synapse) -> None: - self.syn = syn - - async def test_autocomplete_async_dispatch(self): - # GIVEN a SearchIndex with an id - index = SearchIndex(id="syn1") - response = {"hits": [{"rowId": 1, "fields": [{"name": "title", "value": "x"}]}]} - with patch( - "synapseclient.api.autocomplete_search", - new_callable=AsyncMock, - return_value=response, - ) as mock_autocomplete: - # WHEN I run autocomplete - hits = await index.autocomplete_async( - query={"prefix": {"title": {"value": "a"}}}, - source={"includes": ["title"]}, - synapse_client=self.syn, - ) - # THEN the request nests the query/_source under searchQuery for this index - mock_autocomplete.assert_awaited_once_with( - { - "searchIndexId": "syn1", - "searchQuery": { - "query": {"prefix": {"title": {"value": "a"}}}, - "_source": {"includes": ["title"]}, - }, - }, - synapse_client=self.syn, - ) - # AND the response hits deserialize to SearchHit - assert len(hits) == 1 - assert isinstance(hits[0], SearchHit) - assert hits[0].row_id == 1 - - async def test_autocomplete_async_requires_id(self): - # WHEN autocompleting without an id THEN a ValueError is raised - with pytest.raises(ValueError): - await SearchIndex().autocomplete_async( - query={"prefix": {"title": {"value": "a"}}}, - synapse_client=self.syn, - ) - - class TestSearchIndexStatus: def test_fill_from_dict(self): # WHEN I fill a status from a response From 7cf8ff0e5a73fd80ee90c6160d59f662d7dc8ec6 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 12:35:56 -0400 Subject: [PATCH 22/30] fix parameter --- .../operations/async/test_factory_operations_store_async.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py b/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py index 26ddd25d8..b30691b69 100644 --- a/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py +++ b/tests/integration/synapseclient/operations/async/test_factory_operations_store_async.py @@ -737,9 +737,7 @@ async def test_store_async_schema_organization_basic(self) -> None: # THEN the schema organization should no longer be retrievable with pytest.raises(Exception): - await Organization(organization_name=stored_org.name).get_async( - synapse_client=self.syn - ) + await Organization(name=stored_org.name).get_async(synapse_client=self.syn) async def test_store_async_unsupported_entity_raises_error(self) -> None: """Test that storing an unsupported entity type raises an error.""" From b548722bc5915da3dc9bb662d86d21705f84b096 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 13:10:26 -0400 Subject: [PATCH 23/30] edit docstring --- synapseclient/models/search_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapseclient/models/search_index.py b/synapseclient/models/search_index.py index 03d9a038e..0dc662b54 100644 --- a/synapseclient/models/search_index.py +++ b/synapseclient/models/search_index.py @@ -268,7 +268,7 @@ async def store_async( instance from the Synapse class constructor. Returns: - Itself, populated with the server-assigned ID, etag, and columns. + Itself. Raises: ValueError: If `defining_sql` is not set. From 44e2e50c303b86c1ff6889de83f7e428efd4ea54 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 15:57:31 -0400 Subject: [PATCH 24/30] remove columns attribute from search index; updated test; also remove include_columns parameter --- synapseclient/models/protocols/search_index_protocol.py | 5 +---- .../synapseclient/models/async/test_search_index_async.py | 6 +----- .../models/async/unit_test_search_index_async.py | 5 +++-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/synapseclient/models/protocols/search_index_protocol.py b/synapseclient/models/protocols/search_index_protocol.py index 5bc7bc94e..6c8e2e68f 100644 --- a/synapseclient/models/protocols/search_index_protocol.py +++ b/synapseclient/models/protocols/search_index_protocol.py @@ -35,7 +35,7 @@ def store( instance from the Synapse class constructor. Returns: - Itself, populated with the server-assigned ID, etag, and columns. + Itself. Raises: ValueError: If `defining_sql` is not set. @@ -64,7 +64,6 @@ def store( def get( self, - include_columns: bool = True, include_activity: bool = False, *, synapse_client: Optional[Synapse] = None, @@ -73,8 +72,6 @@ def get( `name` and `parent_id`, must be set before calling this. Arguments: - include_columns: If True, will include the columns derived from - `defining_sql` on the returned SearchIndex. include_activity: If True, will include the provenance activity on the returned SearchIndex. synapse_client: If not passed in and caching was not disabled by diff --git a/tests/integration/synapseclient/models/async/test_search_index_async.py b/tests/integration/synapseclient/models/async/test_search_index_async.py index e8dfec642..451f83834 100644 --- a/tests/integration/synapseclient/models/async/test_search_index_async.py +++ b/tests/integration/synapseclient/models/async/test_search_index_async.py @@ -83,14 +83,10 @@ async def test_create_and_retrieve_search_index( # THEN it is created with an ID assert index.id is not None - # AND when retrieving it, the metadata and derived columns are present + # AND when retrieving it, the metadata is present retrieved = await SearchIndex(id=index.id).get_async(synapse_client=self.syn) assert retrieved.name == index_name assert retrieved.defining_sql == f"SELECT * FROM {table.id}" - # Columns are read-only, derived from the defining SQL. - assert retrieved.columns is not None - assert "title" in retrieved.columns - assert "disease_code" in retrieved.columns async def test_query_search_index_match_all(self, project_model: Project) -> None: # GIVEN a SearchIndex over a populated table diff --git a/tests/unit/synapseclient/models/async/unit_test_search_index_async.py b/tests/unit/synapseclient/models/async/unit_test_search_index_async.py index ead5d0f7f..e35df3512 100644 --- a/tests/unit/synapseclient/models/async/unit_test_search_index_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_search_index_async.py @@ -143,9 +143,10 @@ async def test_get_async_calls_super(self): mock_super_get_async.return_value = index # WHEN I get it result = await index.get_async(synapse_client=self.syn) - # THEN the super().get_async method is called with the default arguments + # THEN the super().get_async method is called with include_columns + # forced to False, since SearchIndex has no columns field mock_super_get_async.assert_called_once_with( - include_columns=True, + include_columns=False, include_activity=False, synapse_client=self.syn, ) From a9e747e35d6456b2bb63dac5772d9f110a58f5e0 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 16:06:11 -0400 Subject: [PATCH 25/30] hard code include_columns to false --- synapseclient/models/search_index.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/synapseclient/models/search_index.py b/synapseclient/models/search_index.py index 0dc662b54..5e0900c54 100644 --- a/synapseclient/models/search_index.py +++ b/synapseclient/models/search_index.py @@ -5,7 +5,6 @@ full-text search, faceted search, and autocomplete. """ -from collections import OrderedDict from copy import deepcopy from dataclasses import dataclass, field, replace from datetime import date, datetime @@ -29,7 +28,6 @@ from synapseclient.models.protocols.search_index_protocol import ( SearchIndexSynchronousProtocol, ) -from synapseclient.models.table_components import Column if TYPE_CHECKING: from synapseclient.models.search_management import SearchHit @@ -73,7 +71,6 @@ class SearchIndex( version_label: The version label for this entity. version_comment: The version comment for this entity. is_latest_version: If this is the latest version of the object. - columns: (Read Only) Columns derived from `defining_sql`. defining_sql: The Synapse SQL statement that defines which columns and rows are indexed. search_configuration_id: ID of the SearchConfiguration to apply when @@ -147,11 +144,6 @@ class SearchIndex( is_latest_version: Optional[bool] = field(default=None, compare=False) """If this is the latest version of the object.""" - columns: Optional[OrderedDict[str, Column]] = field( - default_factory=OrderedDict, compare=False - ) - """(Read Only) Columns of a SearchIndex are derived from the defining SQL.""" - defining_sql: Optional[str] = None """The Synapse SQL statement that defines which columns and rows are indexed. Must reference exactly one entity.""" @@ -307,7 +299,6 @@ async def main(): async def get_async( self, - include_columns: bool = True, include_activity: bool = False, *, synapse_client: Optional[Synapse] = None, @@ -316,8 +307,6 @@ async def get_async( and `parent_id`, must be set before calling this. Arguments: - include_columns: If True, will include the columns derived from - `defining_sql` on the returned SearchIndex. include_activity: If True, will include the provenance activity on the returned SearchIndex. synapse_client: If not passed in and caching was not disabled by @@ -346,7 +335,7 @@ async def main(): ``` """ return await super().get_async( - include_columns=include_columns, + include_columns=False, include_activity=include_activity, synapse_client=synapse_client, ) From 30a815861c137c0698daf89d188a0d31645af2e2 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Thu, 23 Jul 2026 17:18:21 -0400 Subject: [PATCH 26/30] add a deprecation statement to SchemaOrganization instead of completely removing it --- synapseclient/models/__init__.py | 7 +++++- synapseclient/models/organization.py | 31 +++++++++++++++++++++++++++ synapseclient/services/json_schema.py | 8 +++---- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index c45c16a00..b1f8ed97f 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -28,7 +28,11 @@ from synapseclient.models.link import Link from synapseclient.models.materializedview import MaterializedView from synapseclient.models.mixins.table_components import QueryMixin -from synapseclient.models.organization import JSONSchema, Organization +from synapseclient.models.organization import ( + JSONSchema, + Organization, + SchemaOrganization, +) from synapseclient.models.project import Project from synapseclient.models.project_setting import ProjectSetting from synapseclient.models.recordset import RecordSet @@ -185,6 +189,7 @@ "WikiHeader", # JSON Schema models "Organization", + "SchemaOrganization", "JSONSchema", # Form models "FormGroup", diff --git a/synapseclient/models/organization.py b/synapseclient/models/organization.py index 4dd5d373e..ad1d5f195 100644 --- a/synapseclient/models/organization.py +++ b/synapseclient/models/organization.py @@ -7,6 +7,8 @@ from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Generator, Optional, Protocol +from deprecated import deprecated + from synapseclient import Synapse from synapseclient.api import ( create_organization, @@ -571,6 +573,18 @@ def fill_from_dict(self, response: dict[str, Any]) -> "Organization": return self +@deprecated( + version="5.0.0", + reason="SchemaOrganization has been renamed to Organization. " + "This alias will be removed in a future release; use Organization instead.", +) +class SchemaOrganization(Organization): + """Deprecated alias for [Organization][synapseclient.models.Organization]. + + This class will be removed in a future release. Use `Organization` instead. + """ + + class JSONSchemaProtocol(Protocol): """ The protocol for methods that are asynchronous but also @@ -1424,6 +1438,23 @@ def list_organizations( return all_orgs +@deprecated( + version="5.0.0", + reason="Renamed to list_organizations. " + "This alias will be removed in a future release; use list_organizations instead.", +) +def list_json_schema_organizations( + synapse_client: Optional["Synapse"] = None, +) -> list[Organization]: + """Deprecated alias for + [list_organizations][synapseclient.models.organization.list_organizations]. + + This function will be removed in a future release. Use `list_organizations` + instead. + """ + return list_organizations(synapse_client=synapse_client) + + def _check_org_name(name: str) -> None: """ Checks that the input name is a valid Synapse Organization diff --git a/synapseclient/services/json_schema.py b/synapseclient/services/json_schema.py index 2886e3f1e..09e7c10ed 100644 --- a/synapseclient/services/json_schema.py +++ b/synapseclient/services/json_schema.py @@ -503,14 +503,14 @@ def create_json_schema( version="4.11.0", reason="To be removed in 5.0.0. " "Use the OOP JSON Schema models instead, " - "synapseclient.models.SchemaOrganization and " + "synapseclient.models.Organization and " "synapseclient.models.JSONSchema.", ) class JsonSchemaService: """Json Schema Service Deprecated: To be removed in 5.0.0. Use the OOP JSON Schema models instead: - synapseclient.models.SchemaOrganization for organization management, + synapseclient.models.Organization for organization management, synapseclient.models.JSONSchema for creating and retrieving schemas, and the JSON Schema methods on entity models (e.g. File, Folder, Project) such as bind_schema(), get_schema(), validate_schema(), and unbind_schema() for binding @@ -532,12 +532,12 @@ class JsonSchemaService: # New approach (RECOMMENDED) from synapseclient import Synapse - from synapseclient.models import SchemaOrganization, JSONSchema, Folder + from synapseclient.models import Organization, JSONSchema, Folder syn = Synapse() syn.login() - organization = SchemaOrganization(name="my.organization").store() + organization = Organization(name="my.organization").store() schema = JSONSchema( organization_name="my.organization", name="my.schema" ) From 2b256eb59f3687d3691b2151fa4b7c6fe0999c71 Mon Sep 17 00:00:00 2001 From: Lingling Peng Date: Mon, 27 Jul 2026 17:06:46 -0400 Subject: [PATCH 27/30] remove SearchIndexStatus class --- synapseclient/models/__init__.py | 2 -- synapseclient/models/search_management.py | 34 ------------------- .../unit_test_search_management_async.py | 18 ---------- 3 files changed, 54 deletions(-) diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index b1f8ed97f..d7555c600 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -48,7 +48,6 @@ SearchHit, SearchIndexQuery, SearchIndexState, - SearchIndexStatus, SearchQuery, SearchQueryPart, SynonymSet, @@ -203,7 +202,6 @@ # SearchIndex / Search Management models "SearchIndex", "SearchIndexQuery", - "SearchIndexStatus", "SearchIndexState", "SearchQuery", "SearchQueryPart", diff --git a/synapseclient/models/search_management.py b/synapseclient/models/search_management.py index e0636e276..e5445fc76 100644 --- a/synapseclient/models/search_management.py +++ b/synapseclient/models/search_management.py @@ -1046,40 +1046,6 @@ async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> N await clear_search_config_binding(self.object_id, synapse_client=synapse_client) -@dataclass -class SearchIndexStatus: - """Build status of a SearchIndex's OpenSearch index. Read it to find out - whether the index is ready, still being built, or in a failed state -- - and, if FAILED, what went wrong. - - Represents a [Synapse SearchIndexStatus](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/search/table/SearchIndexStatus.html). - """ - - search_index_id: Optional[str] = None - """The ID of the SearchIndex entity.""" - - state: Optional[SearchIndexState] = None - """The state of a search index's OpenSearch index.""" - - changed_on: Optional[str] = None - """The date-time when the status last changed.""" - - error_message: Optional[str] = None - """Set when state is FAILED. Captures the diagnostic from the failed build -- - pre-flight validation errors (e.g. 'TextAnalyzer X does not resolve'), AOSS - rejection messages from index creation, or sample per-document failures - from bulk indexing. Capped at 3000 characters; the most-recent failure - replaces any earlier message.""" - - def fill_from_dict(self, data: Dict[str, Any]) -> "Self": - self.search_index_id = data.get("searchIndexId", None) - st = data.get("state", None) - self.state = SearchIndexState(st) if st else None - self.changed_on = data.get("changedOn", None) - self.error_message = data.get("errorMessage", None) - return self - - @dataclass class SearchQuery: """The body of an OpenSearch `_search` request, narrowed to the top-level diff --git a/tests/unit/synapseclient/models/async/unit_test_search_management_async.py b/tests/unit/synapseclient/models/async/unit_test_search_management_async.py index 5f95fb207..74a695c24 100644 --- a/tests/unit/synapseclient/models/async/unit_test_search_management_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_search_management_async.py @@ -20,8 +20,6 @@ SearchHighlight, SearchHit, SearchIndexQuery, - SearchIndexState, - SearchIndexStatus, SearchQuery, SearchQueryPart, SynonymSet, @@ -391,22 +389,6 @@ async def test_get_async_requires_id(self): await TextAnalyzer().get_async(synapse_client=self.syn) -class TestSearchIndexStatus: - def test_fill_from_dict(self): - # WHEN I fill a status from a response - status = SearchIndexStatus().fill_from_dict( - { - "searchIndexId": "syn1", - "state": "ACTIVE", - "changedOn": "2024-01-01T00:00:00.000Z", - "errorMessage": None, - } - ) - # THEN the state coerces to the enum - assert status.search_index_id == "syn1" - assert status.state is SearchIndexState.ACTIVE - - class TestSearchAutocompleteRequest: def test_to_synapse_request(self): # GIVEN an autocomplete request with a prefix query From b4f80ba85259c00507828a2d5640a36989ccedfa Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:44:00 -0700 Subject: [PATCH 28/30] [SYNPY-1869] Review fixes for SearchIndex entity and supporting functionality (#1433) * Review fixes for SearchIndex entity and supporting functionality - Fix store_async crash when upserting onto an existing SearchIndex: a freshly-constructed instance has no _last_persistent_instance, which store_entity dereferenced unconditionally for the version_label check. - Guard force_version in store_entity for models that do not define it. - Add SearchIndex to the get_id() type union in services/search.py. - Flatten SearchIndex.to_synapse_request() to return the entity body directly instead of a single-key {"entity": ...} wrapper the only caller immediately unwrapped. - Add reference docs for SearchIndex and the search configuration resources, resolving the dangling mkdocstrings cross-references to SearchQuery, SearchIndexQuery, search_dsl.Query, TextAnalyzer, and SynonymSet. - Rewrite point-in-time comments as durable descriptions of what the code is. - Add unit coverage for the upsert-onto-existing-index path. * Fix class docstring examples not rendering in docs The SearchIndex and SearchIndexQuery class docstrings opened their ```python fence directly under `Example:`, with no ` ` description line. griffe only parses an Examples section when the title line is followed by body text, so both blocks fell through as literal markdown and the fence leaked onto the rendered page instead of becoming a collapsible example admonition. * add docstring * remove searchIndexStatus class * remove SearchIndexState after SearchIndexStatus gets removed * update the link in docstring * added supported class; fix docstring * fix formatting * avoid exporting the mixin because they are only for internal use * fix docstring * fix docstring * no need to export EnumCoercionMixin and ForwardCompatibleStrEnum because they are internal use * remove version and related fields; fix integration tests * add type searchIndex * updated example to use Query class instead of dict from search_dsl; also expose Query class in doc * move searchIndex to non-version block * update constants after creating test resources in dev --------- Co-authored-by: Lingling Peng --- .../experimental/async/search_index.md | 32 + .../experimental/async/search_management.md | 46 + .../experimental/sync/search_index.md | 34 + .../experimental/sync/search_management.md | 51 + mkdocs.yml | 4 + synapseclient/models/__init__.py | 2 - synapseclient/models/mixins/__init__.py | 2 - synapseclient/models/mixins/enum_coercion.py | 33 +- .../models/mixins/table_components.py | 4 +- synapseclient/models/organization.py | 4 +- .../models/protocols/search_index_protocol.py | 98 +- synapseclient/models/search_dsl.py | 1205 +++++++++++++++++ synapseclient/models/search_index.py | 226 +++- synapseclient/models/search_management.py | 224 +-- synapseclient/models/services/search.py | 2 + .../models/services/storable_entity.py | 11 +- .../services/storable_entity_components.py | 29 +- synapseclient/operations/delete_operations.py | 12 +- .../operations/factory_operations.py | 15 + synapseclient/operations/store_operations.py | 14 +- .../models/async/test_search_index_async.py | 13 +- .../async/test_search_management_async.py | 12 +- .../async/unit_test_search_index_async.py | 195 ++- .../operations/unit_test_delete_operations.py | 19 + .../unit_test_factory_operations.py | 30 + .../operations/unit_test_store_operations.py | 24 + 26 files changed, 2115 insertions(+), 226 deletions(-) create mode 100644 docs/reference/experimental/async/search_index.md create mode 100644 docs/reference/experimental/async/search_management.md create mode 100644 docs/reference/experimental/sync/search_index.md create mode 100644 docs/reference/experimental/sync/search_management.md create mode 100644 synapseclient/models/search_dsl.py diff --git a/docs/reference/experimental/async/search_index.md b/docs/reference/experimental/async/search_index.md new file mode 100644 index 000000000..283073081 --- /dev/null +++ b/docs/reference/experimental/async/search_index.md @@ -0,0 +1,32 @@ +# SearchIndex + +## API reference + +::: synapseclient.models.SearchIndex + options: + inherited_members: true + members: + - store_async + - get_async + - delete_async + - query_async + - autocomplete_async + - get_permissions_async + - get_acl_async + - set_permissions_async + - delete_permissions_async + - list_acl_async + +## Supporting types + +::: synapseclient.models.SearchQuery +::: synapseclient.models.SearchQueryPart +::: synapseclient.models.SearchIndexQuery +::: synapseclient.models.SearchAutocompleteRequest +::: synapseclient.models.SearchHit +::: synapseclient.models.SearchFieldValue +::: synapseclient.models.SearchHighlight + +## OpenSearch query DSL + +::: synapseclient.models.search_dsl.Query diff --git a/docs/reference/experimental/async/search_management.md b/docs/reference/experimental/async/search_management.md new file mode 100644 index 000000000..4358942f6 --- /dev/null +++ b/docs/reference/experimental/async/search_management.md @@ -0,0 +1,46 @@ +# Search Configuration + +Analyzer, synonym, and configuration resources that control how a +[SearchIndex](search_index.md) builds its OpenSearch index. + +## API reference + +::: synapseclient.models.SearchConfiguration + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.TextAnalyzer + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.SynonymSet + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.ColumnAnalyzerOverride + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.SearchConfigBinding + options: + inherited_members: true + members: + - store_async + - get_async + - delete_async diff --git a/docs/reference/experimental/sync/search_index.md b/docs/reference/experimental/sync/search_index.md new file mode 100644 index 000000000..d1f8809a9 --- /dev/null +++ b/docs/reference/experimental/sync/search_index.md @@ -0,0 +1,34 @@ +[](){ #searchindex-reference-sync } +# SearchIndex + +## API reference + +::: synapseclient.models.SearchIndex + options: + inherited_members: true + members: + - store + - get + - delete + - query + - autocomplete + - get_permissions + - get_acl + - set_permissions + - delete_permissions + - list_acl + +## Supporting types + +::: synapseclient.models.SearchQuery +::: synapseclient.models.SearchQueryPart +::: synapseclient.models.SearchIndexQuery +::: synapseclient.models.SearchAutocompleteRequest +::: synapseclient.models.SearchHit +::: synapseclient.models.SearchFieldValue +::: synapseclient.models.SearchHighlight +::: synapseclient.models.protocols.search_index_protocol.SearchIndexSynchronousProtocol + +## OpenSearch query DSL + +::: synapseclient.models.search_dsl.Query diff --git a/docs/reference/experimental/sync/search_management.md b/docs/reference/experimental/sync/search_management.md new file mode 100644 index 000000000..43cd677c3 --- /dev/null +++ b/docs/reference/experimental/sync/search_management.md @@ -0,0 +1,51 @@ +[](){ #search-management-reference-sync } +# Search Configuration + +Analyzer, synonym, and configuration resources that control how a +[SearchIndex][searchindex-reference-sync] builds its OpenSearch index. + +## API reference + +::: synapseclient.models.SearchConfiguration + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.TextAnalyzer + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.SynonymSet + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.ColumnAnalyzerOverride + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.SearchConfigBinding + options: + inherited_members: true + members: + - store + - get + - delete + +## Supporting types + +::: synapseclient.models.ColumnAnalyzerOverrideEntry diff --git a/mkdocs.yml b/mkdocs.yml index 4783e355e..60aab2d64 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,6 +86,8 @@ nav: - Dataset Collection: reference/experimental/sync/dataset_collection.md - EntityView: reference/experimental/sync/entityview.md - MaterializedView: reference/experimental/sync/materializedview.md + - SearchIndex: reference/experimental/sync/search_index.md + - Search Configuration: reference/experimental/sync/search_management.md - SubmissionView: reference/experimental/sync/submissionview.md - Activity: reference/experimental/sync/activity.md - Team: reference/experimental/sync/team.md @@ -117,6 +119,8 @@ nav: - Dataset Collection: reference/experimental/async/dataset_collection.md - EntityView: reference/experimental/async/entityview.md - MaterializedView: reference/experimental/async/materializedview.md + - SearchIndex: reference/experimental/async/search_index.md + - Search Configuration: reference/experimental/async/search_management.md - SubmissionView: reference/experimental/async/submissionview.md - Activity: reference/experimental/async/activity.md - Team: reference/experimental/async/team.md diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index d7555c600..90e08c0ad 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -47,7 +47,6 @@ SearchHighlight, SearchHit, SearchIndexQuery, - SearchIndexState, SearchQuery, SearchQueryPart, SynonymSet, @@ -202,7 +201,6 @@ # SearchIndex / Search Management models "SearchIndex", "SearchIndexQuery", - "SearchIndexState", "SearchQuery", "SearchQueryPart", "SearchAutocompleteRequest", diff --git a/synapseclient/models/mixins/__init__.py b/synapseclient/models/mixins/__init__.py index 2e64a1280..f20e4dbb8 100644 --- a/synapseclient/models/mixins/__init__.py +++ b/synapseclient/models/mixins/__init__.py @@ -2,7 +2,6 @@ from synapseclient.models.mixins.access_control import AccessControllable from synapseclient.models.mixins.asynchronous_job import AsynchronousCommunicator -from synapseclient.models.mixins.enum_coercion import EnumCoercionMixin from synapseclient.models.mixins.form import ( FormChangeRequest, FormData, @@ -29,7 +28,6 @@ __all__ = [ "AccessControllable", - "EnumCoercionMixin", "ProjectSettingsMixin", "StorableContainer", "StorageLocationConfigurable", diff --git a/synapseclient/models/mixins/enum_coercion.py b/synapseclient/models/mixins/enum_coercion.py index 77edf6b56..afd227638 100644 --- a/synapseclient/models/mixins/enum_coercion.py +++ b/synapseclient/models/mixins/enum_coercion.py @@ -1,6 +1,7 @@ -"""Mixin for automatic enum coercion in dataclasses.""" +"""Mixins and base classes for enum handling in dataclasses.""" -from typing import Any, ClassVar, Dict +from enum import Enum +from typing import Any, ClassVar, Dict, Optional class EnumCoercionMixin: @@ -31,3 +32,31 @@ def __setattr__(self, name: str, value: Any) -> None: ): value = enum_cls(value) super().__setattr__(name, value) + + +class ForwardCompatibleStrEnum(str, Enum): + """A string enum that accepts values it does not declare. + + Use this for enums whose values are chosen by the Synapse backend. An + undeclared value is kept as a new member rather than raising ``ValueError``, + so a backend that gains a value cannot break deserialization. Comparisons + against the raw string and against declared members both work, and repeated + lookups of the same undeclared value return the same member. + + Example:: + + class MyState(ForwardCompatibleStrEnum): + ACTIVE = "ACTIVE" + + MyState("ACTIVE") is MyState.ACTIVE # True + MyState("BRAND_NEW") == "BRAND_NEW" # True, no exception + """ + + @classmethod + def _missing_(cls, value: object) -> Optional["ForwardCompatibleStrEnum"]: + if not isinstance(value, str): + return None + pseudo_member = str.__new__(cls, value) + pseudo_member._name_ = value + pseudo_member._value_ = value + return cls._value2member_map_.setdefault(value, pseudo_member) diff --git a/synapseclient/models/mixins/table_components.py b/synapseclient/models/mixins/table_components.py index cf2f4dd40..708dce193 100644 --- a/synapseclient/models/mixins/table_components.py +++ b/synapseclient/models/mixins/table_components.py @@ -85,7 +85,7 @@ "DatasetCollection", "SubmissionView", ] -CLASSES_WITH_READ_ONLY_SCHEMA = ["MaterializedView", "VirtualTable", "SearchIndex"] +CLASSES_WITH_READ_ONLY_SCHEMA = ["MaterializedView", "VirtualTable"] PANDAS_TABLE_TYPE = { "floating": "DOUBLE", @@ -1263,7 +1263,7 @@ async def main(): await get_from_entity_factory( entity_to_update=self, - version=self.version_number, + version=self.version_number if hasattr(self, "version_number") else None, synapse_id_or_path=entity_id, synapse_client=synapse_client, ) diff --git a/synapseclient/models/organization.py b/synapseclient/models/organization.py index ad1d5f195..e8ca28eb7 100644 --- a/synapseclient/models/organization.py +++ b/synapseclient/models/organization.py @@ -574,7 +574,7 @@ def fill_from_dict(self, response: dict[str, Any]) -> "Organization": @deprecated( - version="5.0.0", + version="4.14.0", reason="SchemaOrganization has been renamed to Organization. " "This alias will be removed in a future release; use Organization instead.", ) @@ -1439,7 +1439,7 @@ def list_organizations( @deprecated( - version="5.0.0", + version="4.14.0", reason="Renamed to list_organizations. " "This alias will be removed in a future release; use list_organizations instead.", ) diff --git a/synapseclient/models/protocols/search_index_protocol.py b/synapseclient/models/protocols/search_index_protocol.py index 6c8e2e68f..1485c8dc7 100644 --- a/synapseclient/models/protocols/search_index_protocol.py +++ b/synapseclient/models/protocols/search_index_protocol.py @@ -1,14 +1,20 @@ """Protocol for the specific methods of this class that have synchronous counterparts generated at runtime.""" -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol +from typing import TYPE_CHECKING, List, Optional, Protocol from typing_extensions import Self from synapseclient import Synapse +from synapseclient.models.search_dsl import Query, SourceFilter if TYPE_CHECKING: - from synapseclient.models.search_management import SearchHit + from synapseclient.models.search_management import ( + SearchHit, + SearchIndexQuery, + SearchQuery, + SearchQueryPart, + ) class SearchIndexSynchronousProtocol(Protocol): @@ -18,7 +24,6 @@ def store( self, dry_run: bool = False, *, - job_timeout: int = 600, synapse_client: Optional[Synapse] = None, ) -> "Self": """Store metadata about a SearchIndex including the annotations. Creates @@ -28,8 +33,6 @@ def store( Arguments: dry_run: If True, will not actually store the SearchIndex but will log to the console what would be created or updated. - job_timeout: The maximum amount of time to wait for the index-build job - to complete before raising an error. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -53,7 +56,7 @@ def store( index = SearchIndex( name="My Search Index", parent_id="syn12345", - # syn67890 must be a table or a view; multi-entity JOINs are not supported + # syn67890 must be a table or a view; defining_sql="SELECT * FROM syn67890", ) index = index.store() @@ -121,23 +124,87 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: """ return None + def query( + self, + search_query: "SearchQuery", + response_parts: Optional[List["SearchQueryPart"]] = None, + *, + job_timeout: int = 600, + synapse_client: Optional[Synapse] = None, + ) -> "SearchIndexQuery": + """Query this search index. Unlike a SQL-backed Table, a SearchIndex is + queried with the + [OpenSearch Query DSL](https://docs.opensearch.org/latest/query-dsl/) + carried by a [SearchQuery][synapseclient.models.SearchQuery] — not with + Synapse SQL. See [Query][synapseclient.models.search_dsl.Query] for the + supported clause kinds. + + Arguments: + search_query: The OpenSearch + [`_search`](https://docs.opensearch.org/latest/api-reference/search-apis/search/) + body to execute against this index. + response_parts: Additional response parts to request beyond the + default hits, such as the total hit count or the select columns. + job_timeout: The maximum amount of time to wait for the query job to + complete before raising a `SynapseTimeoutError`. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The completed [SearchIndexQuery][synapseclient.models.SearchIndexQuery], carrying the `hits` and any requested response parts. + + Raises: + ValueError: If the `id` attribute has not been set. + + Example: Query an index for documents mentioning "alzheimer". +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex, SearchQuery, SearchQueryPart + from synapseclient.models.search_dsl import Query + + syn = Synapse() + syn.login() + + results = SearchIndex(id="syn12345").query( + search_query=SearchQuery( + query=Query(match={"title": {"query": "alzheimer"}}), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print(results.total_hits) + for hit in results.hits: + print(hit.row_id, hit.fields) + ``` + """ + from synapseclient.models.search_management import SearchIndexQuery + + return SearchIndexQuery() + def autocomplete( self, - query: Dict[str, Any], - source: Optional[Dict[str, Any]] = None, + query: Query, + source: Optional[SourceFilter] = None, *, synapse_client: Optional[Synapse] = None, ) -> List["SearchHit"]: """Run a synchronous autocomplete search against this index. The - autocomplete endpoint allowlists only prefix-style queries (`prefix`, - `match_phrase_prefix`, or `match_bool_prefix`) and caps results at 8. + autocomplete endpoint allowlists only prefix-style queries + ([`prefix`](https://docs.opensearch.org/latest/query-dsl/term/prefix/), + [`match_phrase_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase-prefix/), + or [`match_bool_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-bool-prefix/)) + and caps results at 8. Arguments: - query: The top-level OpenSearch Query DSL clause; restricted - server-side to `prefix`, `match_phrase_prefix`, or + query: The top-level [OpenSearch Query DSL](https://docs.opensearch.org/latest/query-dsl/) + clause -- see [Query][synapseclient.models.search_dsl.Query]; + restricted server-side to `prefix`, `match_phrase_prefix`, or `match_bool_prefix`. - source: Optional source filter selecting which columns are returned - on each hit. + source: Optional [source filter](https://docs.opensearch.org/latest/search-plugins/searching-data/retrieve-specific-fields/) + selecting which columns are returned on each hit. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -154,13 +221,14 @@ def autocomplete( ```python from synapseclient import Synapse from synapseclient.models import SearchIndex + from synapseclient.models.search_dsl import Query syn = Synapse() syn.login() index = SearchIndex(id="syn12345") hits = index.autocomplete( - query={"match_phrase_prefix": {"title": {"query": "alz"}}}, + query=Query(match_phrase_prefix={"title": {"query": "alz"}}), ) for hit in hits: print(hit.row_id, hit.fields) diff --git a/synapseclient/models/search_dsl.py b/synapseclient/models/search_dsl.py new file mode 100644 index 000000000..96e132d8b --- /dev/null +++ b/synapseclient/models/search_dsl.py @@ -0,0 +1,1205 @@ +"""Typed shapes for the OpenSearch query DSL accepted by Synapse. + +These `TypedDict` types mirror the server-side schema for a SearchIndex query +one-for-one, so a dict literal written inline against a +[SearchQuery][synapseclient.models.SearchQuery] slot is checked by your IDE +against the exact shape the Synapse REST API expects. + +Every type is `total=False`: all keys are optional to the type checker, and the +per-field docstrings mark which ones the server requires. A `TypedDict` is a +plain `dict` at runtime, so these annotations add no validation and no overhead, +and a raw dict is always an acceptable value. + +Start at [Query][synapseclient.models.search_dsl.Query] for the query clause +kinds and [Aggregation][synapseclient.models.search_dsl.Aggregation] for the +aggregation kinds. Both link out to the relevant +[OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) reference +for each clause. + +Background reading: + +- [OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) +- [Query vs. filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/) +- [Term-level vs. full-text queries](https://docs.opensearch.org/latest/query-dsl/term-vs-full-text/) +- [Aggregations](https://docs.opensearch.org/latest/aggregations/) +- [The `_search` API](https://docs.opensearch.org/latest/api-reference/search-apis/search/) +""" + +from typing import Any, Dict, List, TypedDict, Union + +ScalarValue = Union[str, int, float, bool] +"""A single column value. Which of these is valid depends on the target +column's type in the SearchIndex schema.""" + + +AnalyzerRef = TypedDict("AnalyzerRef", {"$ref": str}, total=False) +"""A reference to a saved [TextAnalyzer][synapseclient.models.TextAnalyzer] or +[SynonymSet][synapseclient.models.SynonymSet] by its qualified name +`{organizationName}-{name}`, written as `{"$ref": "my.org-stemmed_english"}`.""" + + +class ClauseScoringOptions(TypedDict, total=False): + """Scoring options shared by the OpenSearch query clauses: a relevance + `boost` and a `_name` label.""" + + boost: float + """Optional. Multiplier for the relevance score of this clause. + Default `1.0`.""" + + _name: str + """Optional. Label echoed back in matched-queries metadata.""" + + +class FuzzyMatchOptions(TypedDict, total=False): + """Fuzzy-matching parameters shared by the analyzed full-text match clauses + (`match`, `match_bool_prefix`, `multi_match`).""" + + fuzziness: Union[int, str] + """Optional. Allowed [edit distance](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/): + an integer or `AUTO`.""" + + fuzzy_rewrite: str + """Optional. How the fuzzy query is rewritten internally.""" + + fuzzy_transpositions: bool + """Optional. Whether to count transpositions (ab -> ba) as a single edit. + Default `true`.""" + + prefix_length: int + """Optional. Number of leading characters left unchanged when fuzzy + matching.""" + + +class MinimumShouldMatchOption(TypedDict, total=False): + """The minimum-should-match option shared by the analyzed full-text match + clauses.""" + + minimum_should_match: Union[int, str] + """Optional. [Minimum number of terms](https://docs.opensearch.org/latest/query-dsl/minimum-should-match/) + a document must match. An integer or a percentage / formula string.""" + + +class ZeroTermsQueryOption(TypedDict, total=False): + """The zero-terms-query behavior shared by the analyzed full-text match + clauses.""" + + zero_terms_query: str + """Optional. Behavior when the analyzer removes all tokens: `none` + (default) or `all`.""" + + +class MatchFieldOptions( + ClauseScoringOptions, + FuzzyMatchOptions, + MinimumShouldMatchOption, + ZeroTermsQueryOption, + total=False, +): + """Per-field options for a [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) + full-text clause. Carried as the value of the field-keyed `match` map (the + map key is the column name).""" + + query: ScalarValue + """Required. The text (or scalar value on a non-text column) to match. A + string, number, or boolean depending on the target column type.""" + + operator: str + """Optional. Boolean logic used to combine the analyzed query terms: `or` + (default) or `and`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to the + field's search analyzer.""" + + max_expansions: int + """Optional. Maximum number of terms the fuzzy expansion will generate.""" + + cutoff_frequency: float + """Optional. Term-frequency threshold above which terms are treated as + low-importance.""" + + auto_generate_synonyms_phrase_query: bool + """Optional. Whether to auto-generate phrase queries for multi-term + synonyms. Default `true`.""" + + lenient: bool + """Optional. When `true`, format-based errors (e.g. a text value against a + numeric field) are ignored.""" + + +class MatchPhraseFieldOptions(ClauseScoringOptions, ZeroTermsQueryOption, total=False): + """Per-field options for a [`match_phrase`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase/) + clause. Carried as the value of the field-keyed `match_phrase` map (the map + key is the column name).""" + + query: ScalarValue + """Required. The phrase to match. A string (or scalar value on a non-text + column).""" + + analyzer: str + """Optional. Analyzer used to tokenize the phrase. Defaults to the field's + search analyzer.""" + + slop: int + """Optional. Number of positions allowed between matching terms. Default + `0` (exact phrase).""" + + +class MatchPhrasePrefixFieldOptions( + ClauseScoringOptions, ZeroTermsQueryOption, total=False +): + """Per-field options for a [`match_phrase_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase-prefix/) + clause. Carried as the value of the field-keyed `match_phrase_prefix` map + (the map key is the column name).""" + + query: ScalarValue + """Required. The phrase whose last term is treated as a prefix. A + string.""" + + analyzer: str + """Optional. Analyzer used to tokenize the phrase. Defaults to the field's + search analyzer.""" + + slop: int + """Optional. Number of positions allowed between matching terms. Default + `0`.""" + + max_expansions: int + """Optional. Maximum number of terms the last (prefix) term expands into. + Default `50`.""" + + +class MatchBoolPrefixFieldOptions( + ClauseScoringOptions, FuzzyMatchOptions, MinimumShouldMatchOption, total=False +): + """Per-field options for a [`match_bool_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-bool-prefix/) + clause. Carried as the value of the field-keyed `match_bool_prefix` map + (the map key is the column name).""" + + query: ScalarValue + """Required. The text whose terms are matched, with the final term treated + as a prefix. A string.""" + + operator: str + """Optional. Boolean logic used to combine the analyzed terms: `or` + (default) or `and`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to the + field's search analyzer.""" + + max_expansions: int + """Optional. Maximum number of terms the final (prefix) term expands into. + Default `50`.""" + + +class TermFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`term`](https://docs.opensearch.org/latest/query-dsl/term/term/) + term-level clause (exact, non-analyzed match). Carried as the value of the + field-keyed `term` map (the map key is the column name).""" + + value: ScalarValue + """Required. The exact value to match. A string, number, boolean, or date + depending on the target column type.""" + + case_insensitive: bool + """Optional. When `true`, matches the value regardless of case. Default + `false`.""" + + +class RangeFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`range`](https://docs.opensearch.org/latest/query-dsl/term/range/) + term-level clause. Carried as the value of the field-keyed `range` map (the + map key is the column name).""" + + gte: ScalarValue + """Optional. Greater-than-or-equal-to bound. A number or date string, per + the target column type.""" + + gt: ScalarValue + """Optional. Greater-than bound.""" + + lte: ScalarValue + """Optional. Less-than-or-equal-to bound.""" + + lt: ScalarValue + """Optional. Less-than bound.""" + + format: str + """Optional. Date format used to parse the bound values on a date + column.""" + + relation: str + """Optional. How the range relates to range-typed field values: + `INTERSECTS` (default), `CONTAINS`, or `WITHIN`.""" + + time_zone: str + """Optional. UTC offset or IANA zone used to interpret date bounds.""" + + +class PrefixFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`prefix`](https://docs.opensearch.org/latest/query-dsl/term/prefix/) + term-level clause. Carried as the value of the field-keyed `prefix` map + (the map key is the column name). A leading `*` or `?` in `value` is + rejected (it forces a full index scan).""" + + value: ScalarValue + """Required. The prefix the indexed term must start with. A string (or + scalar value on a non-text column).""" + + case_insensitive: bool + """Optional. When `true`, matches the prefix regardless of case. Default + `false`.""" + + rewrite: str + """Optional. How the multi-term query is rewritten internally.""" + + +class WildcardFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`wildcard`](https://docs.opensearch.org/latest/query-dsl/term/wildcard/) + term-level clause. Carried as the value of the field-keyed `wildcard` map + (the map key is the column name). A leading `*` or `?` in the pattern is + rejected (it forces a full index scan).""" + + value: ScalarValue + """Optional. The wildcard pattern (`*` matches any sequence, `?` matches a + single character). A string. Either `value` or `wildcard` supplies the + pattern.""" + + wildcard: ScalarValue + """Optional. Alias for `value` -- the wildcard pattern. A string.""" + + case_insensitive: bool + """Optional. When `true`, matches the pattern regardless of case. Default + `false`.""" + + rewrite: str + """Optional. How the multi-term query is rewritten internally.""" + + +class FuzzyFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`fuzzy`](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/) + term-level clause. Carried as the value of the field-keyed `fuzzy` map (the + map key is the column name).""" + + value: ScalarValue + """Required. The term to match within the allowed edit distance. A string + (or scalar value on a non-text column).""" + + fuzziness: Union[int, str] + """Optional. Allowed [edit distance](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/): + an integer or `AUTO`.""" + + max_expansions: int + """Optional. Maximum number of terms the fuzzy expansion will generate. + Default `50`.""" + + prefix_length: int + """Optional. Number of leading characters left unchanged when fuzzy + matching.""" + + transpositions: bool + """Optional. Whether to count transpositions (ab -> ba) as a single edit. + Default `true`.""" + + rewrite: str + """Optional. How the multi-term query is rewritten internally.""" + + +class ExistsQuery(ClauseScoringOptions, total=False): + """An [`exists`](https://docs.opensearch.org/latest/query-dsl/term/exists/) + term-level clause. Matches documents that have any non-null value for the + given column.""" + + field: str + """Required. The column that must have a value.""" + + +class MultiMatchQuery( + ClauseScoringOptions, + FuzzyMatchOptions, + MinimumShouldMatchOption, + ZeroTermsQueryOption, + total=False, +): + """A [`multi_match`](https://docs.opensearch.org/latest/query-dsl/full-text/multi-match/) + full-text clause -- a `match` run across several columns at once.""" + + query: ScalarValue + """Required. The text to match across the listed columns. A string.""" + + fields: List[str] + """Required. The columns to search. Each entry may carry a `^boost` suffix + (e.g. `title^2`).""" + + type: str + """Optional. How the per-field matches are combined: `best_fields` + (default), `most_fields`, `cross_fields`, `phrase`, `phrase_prefix`, or + `bool_prefix`.""" + + operator: str + """Optional. Boolean logic used to combine the analyzed terms: `or` + (default) or `and`.""" + + tie_breaker: float + """Optional. Weight (0-1) applied to non-best field scores in + `best_fields` / `cross_fields`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to each + field's search analyzer.""" + + max_expansions: int + """Optional. Maximum number of terms a fuzzy / prefix expansion will + generate. Default `50`.""" + + slop: int + """Optional. Number of positions allowed between matching terms for the + phrase types.""" + + cutoff_frequency: float + """Optional. Term-frequency threshold above which terms are treated as + low-importance.""" + + auto_generate_synonyms_phrase_query: bool + """Optional. Whether to auto-generate phrase queries for multi-term + synonyms. Default `true`.""" + + lenient: bool + """Optional. When `true`, format-based errors are ignored.""" + + +class SimpleQueryStringQuery( + ClauseScoringOptions, MinimumShouldMatchOption, total=False +): + """A [`simple_query_string`](https://docs.opensearch.org/latest/query-dsl/full-text/simple-query-string/) + full-text clause -- a compact mini-DSL (`+`, `|`, `-`, `"`, `*`, `()`) + parsed leniently across the listed columns.""" + + query: str + """Required. The simple-query-string expression.""" + + fields: List[str] + """Optional. The columns to search. Each entry may carry a `^boost` + suffix. Defaults to the index's default search fields.""" + + default_operator: str + """Optional. Boolean logic used between terms when no explicit operator is + given: `or` (default) or `and`.""" + + flags: str + """Optional. Pipe-delimited list of enabled syntax features (e.g. + `AND|OR|PREFIX`), or `ALL` / `NONE`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to each + field's search analyzer.""" + + analyze_wildcard: bool + """Optional. Whether to analyze wildcard terms. Default `false`. A leading + wildcard with this enabled is rejected (it forces a full index scan).""" + + auto_generate_synonyms_phrase_query: bool + """Optional. Whether to auto-generate phrase queries for multi-term + synonyms. Default `true`.""" + + fuzzy_max_expansions: int + """Optional. Maximum number of terms a fuzzy expansion will generate. + Default `50`.""" + + fuzzy_prefix_length: int + """Optional. Number of leading characters left unchanged when fuzzy + matching.""" + + fuzzy_transpositions: bool + """Optional. Whether to count transpositions (ab -> ba) as a single edit. + Default `true`.""" + + lenient: bool + """Optional. When `true`, format-based errors are ignored.""" + + quote_field_suffix: str + """Optional. Suffix appended to field names for quoted (exact-phrase) + portions of the query.""" + + +class MatchAllQuery(ClauseScoringOptions, total=False): + """A [`match_all`](https://docs.opensearch.org/latest/query-dsl/match-all/) + clause. Matches every document. Use `{"match_all": {}}` to match all + documents.""" + + +class BoolQuery(ClauseScoringOptions, total=False): + """A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) + compound clause -- combines sub-clauses with boolean logic.""" + + must: List["Query"] + """Sub-clauses that must all match (scored). Logical AND.""" + + should: List["Query"] + """Sub-clauses that should match (scored). See `minimum_should_match`.""" + + must_not: List["Query"] + """Sub-clauses that must not match (filter context, not scored).""" + + filter: List["Query"] + """Sub-clauses that must all match in + [filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/) + (not scored).""" + + minimum_should_match: Union[int, str] + """Optional. How many `should` clauses must match. An integer or a + [percentage / formula string](https://docs.opensearch.org/latest/query-dsl/minimum-should-match/).""" + + adjust_pure_negative: bool + """Optional. Whether to automatically add a `match_all` when only negative + clauses are present. Default `true`.""" + + +class DisMaxQuery(ClauseScoringOptions, total=False): + """A [`dis_max`](https://docs.opensearch.org/latest/query-dsl/compound/disjunction-max/) + compound clause. A document matches if any sub-clause matches; its score is + the best single sub-clause score plus `tie_breaker` times the rest.""" + + queries: List["Query"] + """Required. The candidate clauses.""" + + tie_breaker: float + """Optional. Weight (0-1) applied to the scores of the non-best matching + clauses. Default `0.0`.""" + + +class ConstantScoreQuery(ClauseScoringOptions, total=False): + """A [`constant_score`](https://docs.opensearch.org/latest/query-dsl/compound/constant-score/) + compound clause. Wraps a filter and assigns every matching document the + same score (`boost`).""" + + filter: "Query" + """Required. The clause evaluated in + [filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/).""" + + +class BoostingQuery(ClauseScoringOptions, total=False): + """A [`boosting`](https://docs.opensearch.org/latest/query-dsl/compound/boosting/) + compound clause. Returns documents matching `positive`, demoting those that + also match `negative` by `negative_boost`.""" + + positive: "Query" + """Required. The clause documents must match.""" + + negative: "Query" + """Required. The clause whose matches are demoted.""" + + negative_boost: float + """Required. Multiplier (0-1) applied to the score of documents that also + match `negative`.""" + + +class Query(TypedDict, total=False): + """A single [OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) + clause. Exactly one of the keys below may be set -- the set key names the + clause kind. See [query vs. filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/) + and [term-level vs. full-text queries](https://docs.opensearch.org/latest/query-dsl/term-vs-full-text/). + + The field-keyed leaf clauses (`match`, `term`, `range`, ...) are maps whose + key is the column name and whose value is the per-field options object -- + only the long form is accepted (e.g. `{"match": {"title": {"query": "x"}}}`, + not the `{"match": {"title": "x"}}` shorthand). The compound clauses + (`bool`, `dis_max`, ...) nest further Query DSL clauses recursively. + """ + + match: Dict[str, MatchFieldOptions] + """A [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) + full-text clause. Map of column name to its match options.""" + + match_phrase: Dict[str, MatchPhraseFieldOptions] + """A [`match_phrase`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase/) + clause. Map of column name to its phrase options.""" + + match_phrase_prefix: Dict[str, MatchPhrasePrefixFieldOptions] + """A [`match_phrase_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase-prefix/) + clause. Map of column name to its options.""" + + match_bool_prefix: Dict[str, MatchBoolPrefixFieldOptions] + """A [`match_bool_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-bool-prefix/) + clause. Map of column name to its options.""" + + term: Dict[str, TermFieldOptions] + """A [`term`](https://docs.opensearch.org/latest/query-dsl/term/term/) + term-level clause. Map of column name to its term options.""" + + range: Dict[str, RangeFieldOptions] + """A [`range`](https://docs.opensearch.org/latest/query-dsl/term/range/) + term-level clause. Map of column name to its range bounds.""" + + prefix: Dict[str, PrefixFieldOptions] + """A [`prefix`](https://docs.opensearch.org/latest/query-dsl/term/prefix/) + term-level clause. Map of column name to its prefix options.""" + + wildcard: Dict[str, WildcardFieldOptions] + """A [`wildcard`](https://docs.opensearch.org/latest/query-dsl/term/wildcard/) + term-level clause. Map of column name to its wildcard options.""" + + fuzzy: Dict[str, FuzzyFieldOptions] + """A [`fuzzy`](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/) + term-level clause. Map of column name to its fuzzy options.""" + + terms: Dict[str, Any] + """A [`terms`](https://docs.opensearch.org/latest/query-dsl/term/terms/) + term-level clause (matches any of several exact values). Field-keyed: + `{"terms": {"": [v1, v2], "boost": 1.0}}`. The cross-index + `terms`-lookup form is rejected. Untyped because the column name is itself a + key alongside the fixed option keys, which a `TypedDict` cannot express; the + same allowlist is enforced server-side.""" + + exists: ExistsQuery + """An [`exists`](https://docs.opensearch.org/latest/query-dsl/term/exists/) + term-level clause.""" + + multi_match: MultiMatchQuery + """A [`multi_match`](https://docs.opensearch.org/latest/query-dsl/full-text/multi-match/) + full-text clause.""" + + simple_query_string: SimpleQueryStringQuery + """A [`simple_query_string`](https://docs.opensearch.org/latest/query-dsl/full-text/simple-query-string/) + full-text clause.""" + + match_all: MatchAllQuery + """A [`match_all`](https://docs.opensearch.org/latest/query-dsl/match-all/) + clause.""" + + bool: BoolQuery + """A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) + compound clause -- combines sub-clauses with boolean logic.""" + + dis_max: DisMaxQuery + """A [`dis_max`](https://docs.opensearch.org/latest/query-dsl/compound/disjunction-max/) + compound clause.""" + + constant_score: ConstantScoreQuery + """A [`constant_score`](https://docs.opensearch.org/latest/query-dsl/compound/constant-score/) + compound clause.""" + + boosting: BoostingQuery + """A [`boosting`](https://docs.opensearch.org/latest/query-dsl/compound/boosting/) + compound clause.""" + + +class ExtendedBounds(TypedDict, total=False): + """Min/max bounds that force a [`histogram`](https://docs.opensearch.org/latest/aggregations/bucket/histogram/) + or [`date_histogram`](https://docs.opensearch.org/latest/aggregations/bucket/date-histogram/) + to emit buckets across the full range (used as `extended_bounds` or + `hard_bounds`). Bounding the range is what caps the bucket count.""" + + min: ScalarValue + """Lower bound. A number, or a date / date-math string on a + `date_histogram`.""" + + max: ScalarValue + """Upper bound. A number, or a date / date-math string on a + `date_histogram`.""" + + +class HistogramBoundsOptions(TypedDict, total=False): + """The extended/hard bounds options shared by the `histogram` and + `date_histogram` aggregations.""" + + extended_bounds: ExtendedBounds + """Optional. Forces buckets to span at least this min/max range.""" + + hard_bounds: ExtendedBounds + """Optional. Restricts buckets to this min/max range (values outside are + dropped).""" + + +class KeyedBucketOption(TypedDict, total=False): + """The keyed-output option shared by the bucketing aggregations.""" + + keyed: bool + """Optional. Whether to return buckets as a keyed object rather than an + array.""" + + +class MetricAggregation(TypedDict, total=False): + """Common options for the single-value numeric metric aggregations (`avg`, + `max`, `min`, `sum`).""" + + field: str + """Required. The numeric column to aggregate.""" + + format: str + """Optional. Format applied to the result value.""" + + +class MissingValueOption(TypedDict, total=False): + """The missing-value substitution option shared by the metric + aggregations.""" + + missing: ScalarValue + """Optional. Value substituted for documents missing `field`.""" + + +BucketOrder = Union[Dict[str, str], List[Dict[str, str]]] +"""Bucket sort order for a bucketing aggregation -- a `{metric: "asc|desc"}` +object or an array of them, applied in order.""" + + +class TermsAggregation(TypedDict, total=False): + """A [`terms`](https://docs.opensearch.org/latest/aggregations/bucket/terms/) + bucket aggregation -- one bucket per distinct value of `field`.""" + + field: str + """Required. The column to bucket by.""" + + size: int + """Optional. Maximum number of buckets to return. Capped server-side.""" + + shard_size: int + """Optional. Number of candidate buckets collected per shard before the + final reduce. Capped server-side.""" + + min_doc_count: int + """Optional. Minimum document count for a bucket to be returned. Default + `1`.""" + + shard_min_doc_count: int + """Optional. Per-shard minimum document count before a bucket is + considered.""" + + show_term_doc_count_error: bool + """Optional. Whether to return the per-bucket document-count error + bound.""" + + order: BucketOrder + """Optional. Bucket sort order -- a `{metric: "asc|desc"}` object or an + array of them.""" + + include: Union[str, List[ScalarValue]] + """Optional. Terms to include -- a regex string or an array of exact + values.""" + + exclude: Union[str, List[ScalarValue]] + """Optional. Terms to exclude -- a regex string or an array of exact + values.""" + + missing: ScalarValue + """Optional. Bucket value assigned to documents missing `field`.""" + + collect_mode: str + """Optional. `breadth_first` or `depth_first` sub-aggregation collection + strategy.""" + + execution_hint: str + """Optional. Internal execution strategy hint (`map` / + `global_ordinals`).""" + + format: str + """Optional. Format applied to the bucket key in the response.""" + + value_type: str + """Optional. Explicit value type for the field when it cannot be + inferred.""" + + +class HistogramAggregation(KeyedBucketOption, HistogramBoundsOptions, total=False): + """A [`histogram`](https://docs.opensearch.org/latest/aggregations/bucket/histogram/) + bucket aggregation over a numeric column. Must specify `extended_bounds` or + `hard_bounds` so the bucket count is bounded.""" + + field: str + """Required. The numeric column to bucket.""" + + interval: float + """Required. Bucket width. Must be positive.""" + + min_doc_count: int + """Optional. Minimum document count for a bucket to be returned.""" + + offset: float + """Optional. Shifts bucket boundaries by this amount.""" + + order: BucketOrder + """Optional. Bucket sort order -- a `{metric: "asc|desc"}` object or an + array of them.""" + + missing: ScalarValue + """Optional. Bucket value assigned to documents missing `field`.""" + + format: str + """Optional. Format applied to the bucket key in the response.""" + + +class DateHistogramAggregation(KeyedBucketOption, HistogramBoundsOptions, total=False): + """A [`date_histogram`](https://docs.opensearch.org/latest/aggregations/bucket/date-histogram/) + bucket aggregation over a date column. Must specify `extended_bounds` or + `hard_bounds` so the bucket count is bounded.""" + + field: str + """Required. The date column to bucket.""" + + calendar_interval: str + """Optional. Calendar-aware interval (e.g. `month`, `year`). Mutually + exclusive with `fixed_interval`.""" + + fixed_interval: str + """Optional. Fixed-duration interval (e.g. `30d`, `12h`). Mutually + exclusive with `calendar_interval`.""" + + interval: str + """Optional. Legacy interval (use `calendar_interval` / `fixed_interval` + instead).""" + + min_doc_count: int + """Optional. Minimum document count for a bucket to be returned.""" + + offset: str + """Optional. Shifts bucket boundaries by this duration.""" + + time_zone: str + """Optional. UTC offset or IANA zone used to compute bucket boundaries.""" + + order: BucketOrder + """Optional. Bucket sort order -- a `{metric: "asc|desc"}` object or an + array of them.""" + + missing: ScalarValue + """Optional. Bucket value assigned to documents missing `field`.""" + + format: str + """Optional. Date format applied to the bucket key in the response.""" + + +class RangeAggregation(KeyedBucketOption, total=False): + """A [`range`](https://docs.opensearch.org/latest/aggregations/bucket/range/) + bucket aggregation -- one bucket per caller-defined numeric range.""" + + field: str + """Required. The numeric column to bucket.""" + + ranges: List[Dict[str, Any]] + """Required. The [bucket ranges](https://docs.opensearch.org/latest/aggregations/bucket/range/), + each `{"from": , "to": , "key": "