diff --git a/docs/reference/experimental/async/schema_organization.md b/docs/reference/experimental/async/organization.md similarity index 77% rename from docs/reference/experimental/async/schema_organization.md rename to docs/reference/experimental/async/organization.md index 721a45cae..a891981a8 100644 --- a/docs/reference/experimental/async/schema_organization.md +++ b/docs/reference/experimental/async/organization.md @@ -1,8 +1,8 @@ -# SchemaOrganization +# Organization ## API Reference -::: synapseclient.models.SchemaOrganization +::: synapseclient.models.Organization options: inherited_members: true members: diff --git a/docs/reference/experimental/async/search_index.md b/docs/reference/experimental/async/search_index.md new file mode 100644 index 000000000..283073081 --- /dev/null +++ b/docs/reference/experimental/async/search_index.md @@ -0,0 +1,32 @@ +# SearchIndex + +## API reference + +::: synapseclient.models.SearchIndex + options: + inherited_members: true + members: + - store_async + - get_async + - delete_async + - query_async + - autocomplete_async + - get_permissions_async + - get_acl_async + - set_permissions_async + - delete_permissions_async + - list_acl_async + +## Supporting types + +::: synapseclient.models.SearchQuery +::: synapseclient.models.SearchQueryPart +::: synapseclient.models.SearchIndexQuery +::: synapseclient.models.SearchAutocompleteRequest +::: synapseclient.models.SearchHit +::: synapseclient.models.SearchFieldValue +::: synapseclient.models.SearchHighlight + +## OpenSearch query DSL + +::: synapseclient.models.search_dsl.Query diff --git a/docs/reference/experimental/async/search_management.md b/docs/reference/experimental/async/search_management.md new file mode 100644 index 000000000..4358942f6 --- /dev/null +++ b/docs/reference/experimental/async/search_management.md @@ -0,0 +1,46 @@ +# Search Configuration + +Analyzer, synonym, and configuration resources that control how a +[SearchIndex](search_index.md) builds its OpenSearch index. + +## API reference + +::: synapseclient.models.SearchConfiguration + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.TextAnalyzer + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.SynonymSet + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.ColumnAnalyzerOverride + options: + inherited_members: true + members: + - store_async + - get_async + - list_async + +::: synapseclient.models.SearchConfigBinding + options: + inherited_members: true + members: + - store_async + - get_async + - delete_async diff --git a/docs/reference/experimental/sync/schema_organization.md b/docs/reference/experimental/sync/organization.md similarity index 74% rename from docs/reference/experimental/sync/schema_organization.md rename to docs/reference/experimental/sync/organization.md index 2667e9f7b..986978bcb 100644 --- a/docs/reference/experimental/sync/schema_organization.md +++ b/docs/reference/experimental/sync/organization.md @@ -1,8 +1,8 @@ -# SchemaOrganization +# Organization ## API Reference -::: synapseclient.models.SchemaOrganization +::: synapseclient.models.Organization options: inherited_members: true members: diff --git a/docs/reference/experimental/sync/search_index.md b/docs/reference/experimental/sync/search_index.md new file mode 100644 index 000000000..d1f8809a9 --- /dev/null +++ b/docs/reference/experimental/sync/search_index.md @@ -0,0 +1,34 @@ +[](){ #searchindex-reference-sync } +# SearchIndex + +## API reference + +::: synapseclient.models.SearchIndex + options: + inherited_members: true + members: + - store + - get + - delete + - query + - autocomplete + - get_permissions + - get_acl + - set_permissions + - delete_permissions + - list_acl + +## Supporting types + +::: synapseclient.models.SearchQuery +::: synapseclient.models.SearchQueryPart +::: synapseclient.models.SearchIndexQuery +::: synapseclient.models.SearchAutocompleteRequest +::: synapseclient.models.SearchHit +::: synapseclient.models.SearchFieldValue +::: synapseclient.models.SearchHighlight +::: synapseclient.models.protocols.search_index_protocol.SearchIndexSynchronousProtocol + +## OpenSearch query DSL + +::: synapseclient.models.search_dsl.Query diff --git a/docs/reference/experimental/sync/search_management.md b/docs/reference/experimental/sync/search_management.md new file mode 100644 index 000000000..43cd677c3 --- /dev/null +++ b/docs/reference/experimental/sync/search_management.md @@ -0,0 +1,51 @@ +[](){ #search-management-reference-sync } +# Search Configuration + +Analyzer, synonym, and configuration resources that control how a +[SearchIndex][searchindex-reference-sync] builds its OpenSearch index. + +## API reference + +::: synapseclient.models.SearchConfiguration + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.TextAnalyzer + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.SynonymSet + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.ColumnAnalyzerOverride + options: + inherited_members: true + members: + - store + - get + - list + +::: synapseclient.models.SearchConfigBinding + options: + inherited_members: true + members: + - store + - get + - delete + +## Supporting types + +::: synapseclient.models.ColumnAnalyzerOverrideEntry diff --git a/docs/reference/json_schema.md b/docs/reference/json_schema.md index 13dd5a606..e017be3de 100644 --- a/docs/reference/json_schema.md +++ b/docs/reference/json_schema.md @@ -3,7 +3,7 @@ !!! warning "Deprecated — removed in v5.0.0" The legacy `synapseclient.services.json_schema` classes (`JsonSchema`, `JsonSchemaVersion`, `JsonSchemaOrganization`) are deprecated and will be removed in - **v5.0.0**. Use the object-oriented `JSONSchema` and `SchemaOrganization` models + **v5.0.0**. Use the object-oriented `JSONSchema` and `Organization` models instead — see the [Working with JSON Schema tutorial](../tutorials/python/json_schema.md). ::: synapseclient.services.json_schema diff --git a/docs/tutorials/python/tutorial_scripts/json_schema.py b/docs/tutorials/python/tutorial_scripts/json_schema.py index e5bf16942..98ba322f3 100644 --- a/docs/tutorials/python/tutorial_scripts/json_schema.py +++ b/docs/tutorials/python/tutorial_scripts/json_schema.py @@ -4,7 +4,7 @@ import synapseclient from synapseclient.core.utils import make_bogus_data_file -from synapseclient.models import File, Folder, JSONSchema, Project, SchemaOrganization +from synapseclient.models import File, Folder, JSONSchema, Organization, Project # 1. Set up Synapse Python client syn = synapseclient.Synapse() @@ -48,7 +48,7 @@ # 3. Try create test organization and json schema if they do not exist # --8<-- [start:create_org_and_schema] -organization = SchemaOrganization(name=ORG_NAME) +organization = Organization(name=ORG_NAME) try: organization.store() except Exception as e: diff --git a/mkdocs.yml b/mkdocs.yml index daa3cecf6..60aab2d64 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,6 +86,8 @@ nav: - Dataset Collection: reference/experimental/sync/dataset_collection.md - EntityView: reference/experimental/sync/entityview.md - MaterializedView: reference/experimental/sync/materializedview.md + - SearchIndex: reference/experimental/sync/search_index.md + - Search Configuration: reference/experimental/sync/search_management.md - SubmissionView: reference/experimental/sync/submissionview.md - Activity: reference/experimental/sync/activity.md - Team: reference/experimental/sync/team.md @@ -93,7 +95,7 @@ nav: - Curator: reference/experimental/sync/curator.md - Link: reference/experimental/sync/link_entity.md - Functional Interfaces: reference/experimental/functional_interfaces.md - - SchemaOrganization: reference/experimental/sync/schema_organization.md + - Organization: reference/experimental/sync/organization.md - JSONSchema: reference/experimental/sync/json_schema.md - Wiki: reference/experimental/sync/wiki.md - FormGroup and Form: reference/experimental/sync/form.md @@ -117,13 +119,15 @@ nav: - Dataset Collection: reference/experimental/async/dataset_collection.md - EntityView: reference/experimental/async/entityview.md - MaterializedView: reference/experimental/async/materializedview.md + - SearchIndex: reference/experimental/async/search_index.md + - Search Configuration: reference/experimental/async/search_management.md - SubmissionView: reference/experimental/async/submissionview.md - Activity: reference/experimental/async/activity.md - Team: reference/experimental/async/team.md - UserProfile: reference/experimental/async/user_profile.md - Curator: reference/experimental/async/curator.md - Link: reference/experimental/async/link_entity.md - - SchemaOrganization: reference/experimental/async/schema_organization.md + - Organization: reference/experimental/async/organization.md - JSONSchema: reference/experimental/async/json_schema.md - Wiki: reference/experimental/async/wiki.md - FormGroup and Form: reference/experimental/async/form.md diff --git a/synapseclient/api/__init__.py b/synapseclient/api/__init__.py index 4e13bc94a..a0850abb8 100644 --- a/synapseclient/api/__init__.py +++ b/synapseclient/api/__init__.py @@ -146,6 +146,28 @@ get_project_setting, update_project_setting, ) +from .search_services import ( + autocomplete_search, + bind_search_config_to_entity, + clear_search_config_binding, + create_column_analyzer_override, + create_search_configuration, + create_synonym_set, + create_text_analyzer, + get_column_analyzer_override, + get_search_config_binding, + get_search_configuration, + get_synonym_set, + get_text_analyzer, + list_column_analyzer_overrides, + list_search_configurations, + list_synonym_sets, + list_text_analyzers, + update_column_analyzer_override, + update_search_configuration, + update_synonym_set, + update_text_analyzer, +) from .storage_location_services import ( create_storage_location_setting, get_storage_location_setting, @@ -394,4 +416,25 @@ "create_project_setting", "update_project_setting", "delete_project_setting", + # search_services + "create_text_analyzer", + "get_text_analyzer", + "update_text_analyzer", + "list_text_analyzers", + "create_column_analyzer_override", + "get_column_analyzer_override", + "update_column_analyzer_override", + "list_column_analyzer_overrides", + "create_synonym_set", + "get_synonym_set", + "update_synonym_set", + "list_synonym_sets", + "create_search_configuration", + "get_search_configuration", + "update_search_configuration", + "list_search_configurations", + "bind_search_config_to_entity", + "get_search_config_binding", + "clear_search_config_binding", + "autocomplete_search", ] diff --git a/synapseclient/api/entity_factory.py b/synapseclient/api/entity_factory.py index 919af7227..db3472641 100644 --- a/synapseclient/api/entity_factory.py +++ b/synapseclient/api/entity_factory.py @@ -343,6 +343,7 @@ class type. This will also download the file if `download_file` is set to True. MaterializedView, Project, RecordSet, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -379,6 +380,7 @@ class type. This will also download the file if `download_file` is set to True. concrete_types.RECORD_SET_ENTITY: RecordSet, concrete_types.SUBMISSION_VIEW: SubmissionView, concrete_types.VIRTUAL_TABLE: VirtualTable, + concrete_types.SEARCH_INDEX_ENTITY: SearchIndex, concrete_types.LINK_ENTITY: Link, concrete_types.DOCKER_REPOSITORY: DockerRepository, } diff --git a/synapseclient/api/entity_services.py b/synapseclient/api/entity_services.py index 27561da16..af81a9ab4 100644 --- a/synapseclient/api/entity_services.py +++ b/synapseclient/api/entity_services.py @@ -1242,6 +1242,7 @@ async def main(): "submissionview", "dataset", "materializedview", + "searchindex", ] request_body = { diff --git a/synapseclient/api/search_services.py b/synapseclient/api/search_services.py new file mode 100644 index 000000000..839a48224 --- /dev/null +++ b/synapseclient/api/search_services.py @@ -0,0 +1,599 @@ +"""This module is responsible for exposing the services defined at: + + +It covers TextAnalyzer, ColumnAnalyzerOverride, SynonymSet, SearchConfiguration, +SearchConfigBinding, and the synchronous SearchIndex autocomplete endpoint. +The async SearchIndex query endpoint is exposed via the +`SearchIndexQuery.send_job_and_wait_async()` method on the model class +(`models.search_management.SearchIndexQuery`), which uses the shared +`AsynchronousCommunicator` mixin. +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, Optional + +from synapseclient.core.utils import delete_none_keys + +if TYPE_CHECKING: + from synapseclient import Synapse + + +async def create_text_analyzer( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a new TextAnalyzer within the specified Organization. + + + + Arguments: + request: The TextAnalyzer object body. The analyzer's `settings` JSON is + parsed and any `$ref` entries inside its `filter` registry are + checked for qualified-name format and existence. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created TextAnalyzer. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/text/analyzer", body=json.dumps(request) + ) + + +async def get_text_analyzer( + analyzer_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a TextAnalyzer by its ID. + + + + Arguments: + analyzer_id: The numeric ID of the text analyzer to retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested TextAnalyzer. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async(uri=f"/search/text/analyzer/{analyzer_id}") + + +async def update_text_analyzer( + analyzer_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a TextAnalyzer. + + + + Arguments: + analyzer_id: The path ID of the text analyzer (must match the request + body's ID). + request: The updated TextAnalyzer object body. The `organizationName` + and `name` fields are immutable and cannot be modified after + creation. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated TextAnalyzer. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/text/analyzer/{analyzer_id}", body=json.dumps(request) + ) + + +async def list_text_analyzers( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List TextAnalyzer objects, optionally filtered by Organization. + + + + Arguments: + organization_name: If `organizationName` is null, all text analyzers + across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListTextAnalyzersResponse, containing a + page of TextAnalyzers and a nextPageToken if more results exist. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + delete_none_keys(body) + return await client.rest_post_async( + uri="/search/text/analyzer/list", body=json.dumps(body) + ) + + +async def create_column_analyzer_override( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a new ColumnAnalyzerOverride within the specified Organization. + + + + Arguments: + request: The ColumnAnalyzerOverride object body. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created ColumnAnalyzerOverride. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/column/analyzer/override", body=json.dumps(request) + ) + + +async def get_column_analyzer_override( + column_analyzer_override_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a ColumnAnalyzerOverride by its ID. + + + + Arguments: + column_analyzer_override_id: The ID of the column analyzer override to + retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested ColumnAnalyzerOverride. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async( + uri=f"/search/column/analyzer/override/{column_analyzer_override_id}" + ) + + +async def update_column_analyzer_override( + column_analyzer_override_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a ColumnAnalyzerOverride. + + + + Arguments: + column_analyzer_override_id: The path ID (must match the request + body's ID). + request: The updated ColumnAnalyzerOverride object body. Concurrency + is managed via the `etag` field. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated ColumnAnalyzerOverride. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/column/analyzer/override/{column_analyzer_override_id}", + body=json.dumps(request), + ) + + +async def list_column_analyzer_overrides( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List ColumnAnalyzerOverride objects, optionally filtered by Organization. + + + + Arguments: + organization_name: If organizationName is null, all column analyzer + overrides across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListColumnAnalyzerOverridesResponse, + containing a page of ColumnAnalyzerOverrides and a nextPageToken if + more results exist. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + delete_none_keys(body) + return await client.rest_post_async( + uri="/search/column/analyzer/override/list", body=json.dumps(body) + ) + + +async def create_synonym_set( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a new SynonymSet within the specified Organization. + + + + Arguments: + request: The SynonymSet object body. The supplied `definition` is + parsed to confirm it is valid JSON before being passed to AOSS; + supply synonym lists inline via the `synonyms` array rather than + using file-based parameters. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created SynonymSet. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/synonym/set", body=json.dumps(request) + ) + + +async def get_synonym_set( + synonym_set_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a SynonymSet by its ID. + + + + Arguments: + synonym_set_id: The ID of the synonym set to retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested SynonymSet. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async(uri=f"/search/synonym/set/{synonym_set_id}") + + +async def update_synonym_set( + synonym_set_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a SynonymSet. + + + + Arguments: + synonym_set_id: The path ID (must match the request body's ID). + request: The updated SynonymSet object body. The `definition` field is + re-parsed to validate JSON formatting. The `organizationName` and + `name` fields are immutable after creation. Concurrency is managed + via `etag`; mismatches return a 409 Conflict. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated SynonymSet. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/synonym/set/{synonym_set_id}", body=json.dumps(request) + ) + + +async def list_synonym_sets( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List SynonymSet objects, optionally filtered by Organization. + + + + Arguments: + organization_name: If organizationName is null, all synonym sets + across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListSynonymSetsResponse, containing a + page of SynonymSets and a nextPageToken if more results exist. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + delete_none_keys(body) + return await client.rest_post_async( + uri="/search/synonym/set/list", body=json.dumps(body) + ) + + +async def create_search_configuration( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Create a new SearchConfiguration within the specified Organization. + + + + Arguments: + request: The SearchConfiguration object body. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created SearchConfiguration. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/configuration", body=json.dumps(request) + ) + + +async def get_search_configuration( + search_configuration_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get a SearchConfiguration by its ID. + + + + Arguments: + search_configuration_id: The ID of the search configuration to + retrieve. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the requested SearchConfiguration. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async( + uri=f"/search/configuration/{search_configuration_id}" + ) + + +async def update_search_configuration( + search_configuration_id: str, + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Update a SearchConfiguration. + + + + Arguments: + search_configuration_id: The path ID (must match the request body's + ID). + request: The updated SearchConfiguration object body. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the updated SearchConfiguration. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_put_async( + uri=f"/search/configuration/{search_configuration_id}", + body=json.dumps(request), + ) + + +async def list_search_configurations( + organization_name: Optional[str] = None, + next_page_token: Optional[str] = None, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """List SearchConfiguration objects, optionally filtered by Organization. + + + + Arguments: + organization_name: If organizationName is null, all search + configurations across all Organizations are returned. + next_page_token: Results are paginated using a next page token. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the ListSearchConfigurationsResponse, + containing a page of SearchConfigurations and a nextPageToken if more + results exist. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = {"organizationName": organization_name, "nextPageToken": next_page_token} + delete_none_keys(body) + return await client.rest_post_async( + uri="/search/configuration/list", body=json.dumps(body) + ) + + +async def bind_search_config_to_entity( + entity_id: str, + search_configuration_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Bind a SearchConfiguration to an entity (typically a project). The + caller must have EDIT permission on the entity. Replaces any existing + binding on that entity. + + + + Arguments: + entity_id: The ID of the entity to bind to. + search_configuration_id: The ID of the SearchConfiguration to bind. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the created SearchConfigBinding. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + body = { + "entityId": entity_id, + "searchConfigurationId": search_configuration_id, + } + return await client.rest_put_async( + uri=f"/entity/{entity_id}/searchconfig/binding", body=json.dumps(body) + ) + + +async def get_search_config_binding( + entity_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Get the effective SearchConfigBinding for an entity. Walks up the + entity hierarchy (entity -> folder -> project) and returns the first + binding found on the entity or any ancestor. + + + + Arguments: + entity_id: The ID of the entity to look up. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the effective SearchConfigBinding. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_get_async(uri=f"/entity/{entity_id}/searchconfig/binding") + + +async def clear_search_config_binding( + entity_id: str, + *, + synapse_client: Optional["Synapse"] = None, +) -> None: + """Clear the SearchConfigBinding on a specific entity. + + + + Arguments: + entity_id: The ID of the entity whose binding to clear. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + None + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + await client.rest_delete_async(uri=f"/entity/{entity_id}/searchconfig/binding") + + +async def autocomplete_search( + request: Dict[str, Any], + *, + synapse_client: Optional["Synapse"] = None, +) -> Dict[str, Any]: + """Perform a synchronous autocomplete search query against a SearchIndex. + Purpose-built for type-ahead input: no pagination, sorting customization, + or aggregations are available; the server caps every response at 8 hits, + ordered by relevance. For more complex search needs, use the asynchronous + search endpoint instead. + + + + Arguments: + request: The SearchAutocompleteRequest body. Only two top-level keys + are permitted in the `searchQuery`: `query` (required, must use + one of `prefix`, `match_phrase_prefix`, or `match_bool_prefix`) and + `_source` (optional, a source filter to narrow returned fields). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A dictionary representing the SearchQueryResults, capped at 8 hits. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + return await client.rest_post_async( + uri="/search/autocomplete", body=json.dumps(request) + ) diff --git a/synapseclient/core/constants/concrete_types.py b/synapseclient/core/constants/concrete_types.py index 38ec38628..e24ab9da9 100644 --- a/synapseclient/core/constants/concrete_types.py +++ b/synapseclient/core/constants/concrete_types.py @@ -89,8 +89,12 @@ MATERIALIZED_VIEW = "org.sagebionetworks.repo.model.table.MaterializedView" SUBMISSION_VIEW = "org.sagebionetworks.repo.model.table.SubmissionView" VIRTUAL_TABLE = "org.sagebionetworks.repo.model.table.VirtualTable" +SEARCH_INDEX_ENTITY = "org.sagebionetworks.repo.model.search.table.SearchIndex" DOCKER_REPOSITORY = "org.sagebionetworks.repo.model.docker.DockerRepository" +# Search Management +SEARCH_INDEX_QUERY = "org.sagebionetworks.repo.model.search.table.SearchIndexQuery" + # upload requests MULTIPART_UPLOAD_REQUEST = "org.sagebionetworks.repo.model.file.MultipartUploadRequest" MULTIPART_UPLOAD_COPY_REQUEST = ( diff --git a/synapseclient/extensions/curator/schema_management.py b/synapseclient/extensions/curator/schema_management.py index 44b39959a..cf51e71a7 100644 --- a/synapseclient/extensions/curator/schema_management.py +++ b/synapseclient/extensions/curator/schema_management.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from synapseclient import Synapse from synapseclient.models.mixins.json_schema import JSONSchemaBinding - from synapseclient.models.schema_organization import JSONSchema + from synapseclient.models.organization import JSONSchema def register_jsonschema( @@ -129,7 +129,7 @@ async def register_jsonschema_async( ``` """ from synapseclient import Synapse - from synapseclient.models.schema_organization import JSONSchema + from synapseclient.models.organization import JSONSchema syn = Synapse.get_client(synapse_client=synapse_client) diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index ce1a3bf9e..90e08c0ad 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -28,10 +28,30 @@ from synapseclient.models.link import Link from synapseclient.models.materializedview import MaterializedView from synapseclient.models.mixins.table_components import QueryMixin +from synapseclient.models.organization import ( + JSONSchema, + Organization, + SchemaOrganization, +) from synapseclient.models.project import Project from synapseclient.models.project_setting import ProjectSetting from synapseclient.models.recordset import RecordSet -from synapseclient.models.schema_organization import JSONSchema, SchemaOrganization +from synapseclient.models.search_index import SearchIndex +from synapseclient.models.search_management import ( + ColumnAnalyzerOverride, + ColumnAnalyzerOverrideEntry, + SearchAutocompleteRequest, + SearchConfigBinding, + SearchConfiguration, + SearchFieldValue, + SearchHighlight, + SearchHit, + SearchIndexQuery, + SearchQuery, + SearchQueryPart, + SynonymSet, + TextAnalyzer, +) from synapseclient.models.services import FailureStrategy from synapseclient.models.storage_location import ( StorageLocation, @@ -166,6 +186,7 @@ "WikiHistorySnapshot", "WikiHeader", # JSON Schema models + "Organization", "SchemaOrganization", "JSONSchema", # Form models @@ -177,6 +198,21 @@ "UploadType", # Project Setting models "ProjectSetting", + # SearchIndex / Search Management models + "SearchIndex", + "SearchIndexQuery", + "SearchQuery", + "SearchQueryPart", + "SearchAutocompleteRequest", + "SearchHit", + "SearchFieldValue", + "SearchHighlight", + "SearchConfiguration", + "SearchConfigBinding", + "TextAnalyzer", + "ColumnAnalyzerOverride", + "ColumnAnalyzerOverrideEntry", + "SynonymSet", ] # Static methods to expose as functions diff --git a/synapseclient/models/folder.py b/synapseclient/models/folder.py index 02b4f3cc1..e49db55ab 100644 --- a/synapseclient/models/folder.py +++ b/synapseclient/models/folder.py @@ -34,6 +34,7 @@ EntityView, MaterializedView, Project, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -77,6 +78,7 @@ class Folder( datasetcollections: Dataset collections that exist within this folder. materializedviews: Materialized views that exist within this folder. virtualtables: Virtual tables that exist within this folder. + searchindexes: Search indexes that exist within this folder. annotations: Additional metadata associated with the folder. The key is the name of your desired annotations. The value is an object containing a list of values (use empty list to represent no values for key) and the value type @@ -159,6 +161,9 @@ class Folder( virtualtables: List["VirtualTable"] = field(default_factory=list, compare=False) """Virtual tables that exist within this folder.""" + searchindexes: List["SearchIndex"] = field(default_factory=list, compare=False) + """Search indexes that exist within this folder.""" + annotations: Optional[ Dict[ str, diff --git a/synapseclient/models/mixins/__init__.py b/synapseclient/models/mixins/__init__.py index 2e64a1280..f20e4dbb8 100644 --- a/synapseclient/models/mixins/__init__.py +++ b/synapseclient/models/mixins/__init__.py @@ -2,7 +2,6 @@ from synapseclient.models.mixins.access_control import AccessControllable from synapseclient.models.mixins.asynchronous_job import AsynchronousCommunicator -from synapseclient.models.mixins.enum_coercion import EnumCoercionMixin from synapseclient.models.mixins.form import ( FormChangeRequest, FormData, @@ -29,7 +28,6 @@ __all__ = [ "AccessControllable", - "EnumCoercionMixin", "ProjectSettingsMixin", "StorableContainer", "StorageLocationConfigurable", diff --git a/synapseclient/models/mixins/access_control.py b/synapseclient/models/mixins/access_control.py index ccec02aae..5db5f7eca 100644 --- a/synapseclient/models/mixins/access_control.py +++ b/synapseclient/models/mixins/access_control.py @@ -33,6 +33,7 @@ File, Folder, MaterializedView, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -49,6 +50,7 @@ "datasetcollection": "datasetcollections", "materializedview": "materializedviews", "virtualtable": "virtualtables", + "searchindex": "searchindexes", } @@ -203,6 +205,7 @@ class AccessControllable(AccessControllableSynchronousProtocol): datasetcollections: List["DatasetCollection"] = None materializedviews: List["MaterializedView"] = None virtualtables: List["VirtualTable"] = None + searchindexes: List["SearchIndex"] = None async def get_permissions_async( self, diff --git a/synapseclient/models/mixins/asynchronous_job.py b/synapseclient/models/mixins/asynchronous_job.py index cccda8400..31ed18182 100644 --- a/synapseclient/models/mixins/asynchronous_job.py +++ b/synapseclient/models/mixins/asynchronous_job.py @@ -21,6 +21,7 @@ GRID_RECORD_SET_EXPORT_REQUEST, QUERY_BUNDLE_REQUEST, QUERY_TABLE_CSV_REQUEST, + SEARCH_INDEX_QUERY, SYNCHRONIZE_GRID_REQUEST, TABLE_UPDATE_TRANSACTION_REQUEST, UPLOAD_TO_TABLE_PREVIEW_REQUEST, @@ -47,6 +48,7 @@ GRID_CSV_IMPORT_REQUEST: "/grid/import/csv/async", GRID_QUERY_JOB_REQUEST: "/grid/session/query/async", UPLOAD_TO_TABLE_PREVIEW_REQUEST: "/table/upload/csv/preview/async", + SEARCH_INDEX_QUERY: "/search/query/async", } diff --git a/synapseclient/models/mixins/enum_coercion.py b/synapseclient/models/mixins/enum_coercion.py index 77edf6b56..afd227638 100644 --- a/synapseclient/models/mixins/enum_coercion.py +++ b/synapseclient/models/mixins/enum_coercion.py @@ -1,6 +1,7 @@ -"""Mixin for automatic enum coercion in dataclasses.""" +"""Mixins and base classes for enum handling in dataclasses.""" -from typing import Any, ClassVar, Dict +from enum import Enum +from typing import Any, ClassVar, Dict, Optional class EnumCoercionMixin: @@ -31,3 +32,31 @@ def __setattr__(self, name: str, value: Any) -> None: ): value = enum_cls(value) super().__setattr__(name, value) + + +class ForwardCompatibleStrEnum(str, Enum): + """A string enum that accepts values it does not declare. + + Use this for enums whose values are chosen by the Synapse backend. An + undeclared value is kept as a new member rather than raising ``ValueError``, + so a backend that gains a value cannot break deserialization. Comparisons + against the raw string and against declared members both work, and repeated + lookups of the same undeclared value return the same member. + + Example:: + + class MyState(ForwardCompatibleStrEnum): + ACTIVE = "ACTIVE" + + MyState("ACTIVE") is MyState.ACTIVE # True + MyState("BRAND_NEW") == "BRAND_NEW" # True, no exception + """ + + @classmethod + def _missing_(cls, value: object) -> Optional["ForwardCompatibleStrEnum"]: + if not isinstance(value, str): + return None + pseudo_member = str.__new__(cls, value) + pseudo_member._name_ = value + pseudo_member._value_ = value + return cls._value2member_map_.setdefault(value, pseudo_member) diff --git a/synapseclient/models/mixins/storable_container.py b/synapseclient/models/mixins/storable_container.py index df0933a41..39f35af13 100644 --- a/synapseclient/models/mixins/storable_container.py +++ b/synapseclient/models/mixins/storable_container.py @@ -34,6 +34,7 @@ LINK_ENTITY, MATERIALIZED_VIEW, PROJECT_ENTITY, + SEARCH_INDEX_ENTITY, SUBMISSION_VIEW, TABLE_ENTITY, VIRTUAL_TABLE, @@ -72,6 +73,7 @@ File, Folder, MaterializedView, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -95,6 +97,7 @@ class StorableContainer(StorableContainerSynchronousProtocol): - `datasetcollections` - `materializedviews` - `virtualtables` + - `searchindexes` - `_last_persistent_instance` - `_synced_from_synapse` @@ -116,6 +119,7 @@ class StorableContainer(StorableContainerSynchronousProtocol): datasetcollections: List["DatasetCollection"] = None materializedviews: List["MaterializedView"] = None virtualtables: List["VirtualTable"] = None + searchindexes: List["SearchIndex"] = None _last_persistent_instance: None = None _synced_from_synapse: bool = False @@ -223,7 +227,7 @@ async def sync_from_synapse_async( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. + "virtualtable", "searchindex"]`. manifest: Determines whether to generate a manifest CSV file. Options are: - `all` (default): generate `manifest.csv` in every synced directory @@ -530,6 +534,7 @@ async def _sync_from_synapse_async( self.datasetcollections = [] self.materializedviews = [] self.virtualtables = [] + self.searchindexes = [] for child in children: pending_tasks.extend( @@ -885,7 +890,7 @@ async def walk_async( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. The "folder" type is always included so the hierarchy + "virtualtable", "searchindex"]`. The "folder" type is always included so the hierarchy can be traversed. recursive: Whether to recursively traverse subdirectories. Defaults to True. display_ascii_tree: If True, display an ASCII tree representation as the @@ -1017,6 +1022,7 @@ async def my_function(): "datasetcollection", "materializedview", "virtualtable", + "searchindex", ] if follow_link: include_types.append("link") @@ -1169,7 +1175,7 @@ def walk( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. The "folder" type is always included so the hierarchy + "virtualtable", "searchindex"]`. The "folder" type is always included so the hierarchy can be traversed. recursive: Whether to recursively traverse subdirectories. Defaults to True. display_ascii_tree: If True, display an ASCII tree representation as the @@ -1311,7 +1317,7 @@ async def _retrieve_children( Defaults to `["folder", "file", "table", "entityview", "dockerrepo", "submissionview", "dataset", "datasetcollection", "materializedview", - "virtualtable"]`. + "virtualtable", "searchindex"]`. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -1331,6 +1337,7 @@ async def _retrieve_children( "datasetcollection", "materializedview", "virtualtable", + "searchindex", ] if follow_link: include_types.append("link") @@ -1605,6 +1612,17 @@ def _create_task_for_child( ) ) ) + elif synapse_id and child_type == SEARCH_INDEX_ENTITY: + # Lazy import to avoid circular import + from synapseclient.models import SearchIndex + + searchindex = SearchIndex(id=synapse_id, name=name) + self.searchindexes.append(searchindex) + pending_tasks.append( + asyncio.create_task( + wrap_coroutine(searchindex.get_async(synapse_client=synapse_client)) + ) + ) return pending_tasks @@ -1692,6 +1710,7 @@ def _resolve_sync_from_synapse_result( "DatasetCollection", "MaterializedView", "VirtualTable", + "SearchIndex", BaseException, ], failure_strategy: FailureStrategy, @@ -1726,6 +1745,7 @@ def _resolve_sync_from_synapse_result( or result.__class__.__name__ == "DatasetCollection" or result.__class__.__name__ == "MaterializedView" or result.__class__.__name__ == "VirtualTable" + or result.__class__.__name__ == "SearchIndex" ): # Do nothing as the objects are updated in place and the container has # already been updated to append the new objects. diff --git a/synapseclient/models/mixins/table_components.py b/synapseclient/models/mixins/table_components.py index 44591714d..708dce193 100644 --- a/synapseclient/models/mixins/table_components.py +++ b/synapseclient/models/mixins/table_components.py @@ -1263,7 +1263,7 @@ async def main(): await get_from_entity_factory( entity_to_update=self, - version=self.version_number, + version=self.version_number if hasattr(self, "version_number") else None, synapse_id_or_path=entity_id, synapse_client=synapse_client, ) diff --git a/synapseclient/models/schema_organization.py b/synapseclient/models/organization.py similarity index 92% rename from synapseclient/models/schema_organization.py rename to synapseclient/models/organization.py index cfa3f30df..e8ca28eb7 100644 --- a/synapseclient/models/schema_organization.py +++ b/synapseclient/models/organization.py @@ -1,5 +1,5 @@ """ -This file contains the SchemaOrganization and JSONSchema classes. +This file contains the Organization and JSONSchema classes. These are used to manage Organization and JSON Schema entities in Synapse. """ @@ -7,6 +7,8 @@ from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Generator, Optional, Protocol +from deprecated import deprecated + from synapseclient import Synapse from synapseclient.api import ( create_organization, @@ -30,15 +32,13 @@ from synapseclient.models.mixins.json_schema import JSONSchemaVersionInfo -class SchemaOrganizationProtocol(Protocol): +class OrganizationProtocol(Protocol): """ The protocol for methods that are asynchronous but also have a synchronous counterpart that may also be called. """ - def get( - self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Organization": """ Gets the metadata from Synapse for this organization @@ -50,17 +50,17 @@ def get( Returns: Itself - Example: Get an existing SchemaOrganization + Example: Get an existing Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") org.get() print(org) ``` @@ -68,9 +68,7 @@ def get( """ return self - def store( - self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Organization": """ Stores this organization in Synapse @@ -82,17 +80,17 @@ def store( Returns: Itself - Example: Store the SchemaOrganization in Synapse + Example: Store the Organization in Synapse   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.store() print(org) ``` @@ -109,17 +107,17 @@ def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Delete the SchemaOrganization from Synapse + Example: Delete the Organization from Synapse   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.delete() ``` """ @@ -138,17 +136,17 @@ def get_json_schemas( Returns: A Generator containing the JSONSchemas that belong to this organization - Example: Get the JSONSchemas that are part of this SchemaOrganization + Example: Get the JSONSchemas that are part of this Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") js_generator = org.get_json_schemas() for item in js_generator: print(item) @@ -172,17 +170,17 @@ def get_acl(self, *, synapse_client: Optional["Synapse"] = None) -> dict[str, An A dictionary in the form of this response: [AccessControlList]https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/AccessControlList.html - Example: Get the ACL for the SchemaOrganization + Example: Get the ACL for the Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.get_acl() ``` """ @@ -207,17 +205,17 @@ def update_acl( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Update the ACL for the SchemaOrganization + Example: Update the ACL for the Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse syn = Synapse() syn.login() - org = SchemaOrganization("my.org.name") + org = Organization("my.org.name") org.update_acl( principal_id=1, access_type=["READ"] @@ -229,7 +227,7 @@ def update_acl( @dataclass() @async_to_sync -class SchemaOrganization(SchemaOrganizationProtocol): +class Organization(OrganizationProtocol): """ Represents an [Organization](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/Organization.html). @@ -258,7 +256,7 @@ def __post_init__(self) -> None: async def get_async( self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + ) -> "Organization": """ Gets the metadata from Synapse for this organization @@ -273,12 +271,12 @@ async def get_async( Raises: ValueError: If the name has not been set - Example: Get an existing SchemaOrganization + Example: Get an existing Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -287,7 +285,7 @@ async def get_org(): syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") await org.get_async() return org @@ -297,14 +295,14 @@ async def get_org(): ``` """ if not self.name: - raise ValueError("SchemaOrganization must have a name") + raise ValueError("Organization must have a name") response = await get_organization(self.name, synapse_client=synapse_client) self.fill_from_dict(response) return self async def store_async( self, *, synapse_client: Optional["Synapse"] = None - ) -> "SchemaOrganization": + ) -> "Organization": """ Stores this organization in Synapse @@ -319,11 +317,11 @@ async def store_async( Raises: ValueError: If the name has not been set - Example: Store a new SchemaOrganization + Example: Store a new Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -332,7 +330,7 @@ async def store_org(): syn = Synapse() syn.login() - org = SchemaOrganization("my.new.org") + org = Organization("my.new.org") await org.store_async() return org @@ -342,7 +340,7 @@ async def store_org(): ``` """ if not self.name: - raise ValueError("SchemaOrganization must have a name") + raise ValueError("Organization must have a name") response = await create_organization(self.name, synapse_client=synapse_client) self.fill_from_dict(response) return self @@ -356,11 +354,11 @@ async def delete_async(self, *, synapse_client: Optional["Synapse"] = None) -> N `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Delete a SchemaOrganization using a name + Example: Delete a Organization using a name   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -369,17 +367,17 @@ async def delete_org(): syn = Synapse() syn.login() - org = SchemaOrganization("my.org") + org = Organization("my.org") await org.delete_async() asyncio.run(delete_org()) ``` - Example: Delete a SchemaOrganization using an id + Example: Delete a Organization using an id   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -388,7 +386,7 @@ async def delete_org(): syn = Synapse() syn.login() - org = SchemaOrganization(id=1075) + org = Organization(id=1075) await org.delete_async() asyncio.run(delete_org()) @@ -417,11 +415,11 @@ async def get_json_schemas_async( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Get a list of JSONSchemas that belong to the SchemaOrganization + Example: Get a list of JSONSchemas that belong to the Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -430,7 +428,7 @@ async def get_schemas(): syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") js_generator = org.get_json_schemas_async() async for item in js_generator: print(item) @@ -440,7 +438,7 @@ async def get_schemas(): """ if not self.name: - raise ValueError("SchemaOrganization must have a name") + raise ValueError("Organization must have a name") response = list_json_schemas(self.name, synapse_client=synapse_client) async for item in response: yield JSONSchema().fill_from_dict(item) @@ -460,11 +458,11 @@ async def get_acl_async( A dictionary in the form of this response: [AccessControlList]https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/AccessControlList.html - Example: Get the ACL for a SchemaOrganization + Example: Get the ACL for a Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -473,7 +471,7 @@ async def get_acl(): syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") acl = await org.get_acl_async() return acl @@ -512,11 +510,11 @@ async def update_acl_async( `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor - Example: Update the ACL for a SchemaOrganization + Example: Update the ACL for a Organization   ```python - from synapseclient.models import SchemaOrganization + from synapseclient.models import Organization from synapseclient import Synapse import asyncio @@ -525,14 +523,14 @@ async def update_acl() -> None: syn = Synapse() syn.login() - org = SchemaOrganization("dpetest") + org = Organization("dpetest") await org.update_acl_async( principal_id=1, access_type=["READ"] ) asyncio.run(update_acl()) - + ``` """ acl = await self.get_acl_async(synapse_client=synapse_client) @@ -557,7 +555,7 @@ async def update_acl() -> None: synapse_client=synapse_client, ) - def fill_from_dict(self, response: dict[str, Any]) -> "SchemaOrganization": + def fill_from_dict(self, response: dict[str, Any]) -> "Organization": """ Fills in this classes attributes using a Synapse API response @@ -575,6 +573,18 @@ def fill_from_dict(self, response: dict[str, Any]) -> "SchemaOrganization": return self +@deprecated( + version="4.14.0", + reason="SchemaOrganization has been renamed to Organization. " + "This alias will be removed in a future release; use Organization instead.", +) +class SchemaOrganization(Organization): + """Deprecated alias for [Organization][synapseclient.models.Organization]. + + This class will be removed in a future release. Use `Organization` instead. + """ + + class JSONSchemaProtocol(Protocol): """ The protocol for methods that are asynchronous but also @@ -881,7 +891,7 @@ async def get_schema(): # Check that the org exists, # if it doesn't list_json_schemas will unhelpfully return an empty generator. - org = SchemaOrganization(self.organization_name) + org = Organization(self.organization_name) await org.get_async(synapse_client=synapse_client) org_schemas = list_json_schemas( @@ -1396,9 +1406,9 @@ def _create_json_schema_version_from_response( ) -def list_json_schema_organizations( +def list_organizations( synapse_client: Optional["Synapse"] = None, -) -> list[SchemaOrganization]: +) -> list[Organization]: """ Lists all the Schema Organizations currently in Synapse @@ -1408,26 +1418,43 @@ def list_json_schema_organizations( instance from the Synapse class constructor Returns: - A list of SchemaOrganizations + A list of Organizations Example: - from synapseclient.models.schema_organization import list_json_schema_organizations + from synapseclient.models.organization import list_organizations from synapseclient import Synapse syn = Synapse() syn.login() - all_orgs = list_json_schema_organizations() + all_orgs = list_organizations() for item in all_orgs: print(item) """ all_orgs = [ - SchemaOrganization().fill_from_dict(org) + Organization().fill_from_dict(org) for org in list_organizations_sync(synapse_client=synapse_client) ] return all_orgs +@deprecated( + version="4.14.0", + reason="Renamed to list_organizations. " + "This alias will be removed in a future release; use list_organizations instead.", +) +def list_json_schema_organizations( + synapse_client: Optional["Synapse"] = None, +) -> list[Organization]: + """Deprecated alias for + [list_organizations][synapseclient.models.organization.list_organizations]. + + This function will be removed in a future release. Use `list_organizations` + instead. + """ + return list_organizations(synapse_client=synapse_client) + + def _check_org_name(name: str) -> None: """ Checks that the input name is a valid Synapse Organization diff --git a/synapseclient/models/project.py b/synapseclient/models/project.py index b1e95899f..5dd630c20 100644 --- a/synapseclient/models/project.py +++ b/synapseclient/models/project.py @@ -33,6 +33,7 @@ DatasetCollection, EntityView, MaterializedView, + SearchIndex, SubmissionView, Table, VirtualTable, @@ -75,6 +76,7 @@ class Project( datasetcollections: Any dataset collections that are at the root directory of the project. materializedviews: Any materialized views that are at the root directory of the project. virtualtables: Any virtual tables that are at the root directory of the project. + searchindexes: Any search indexes that are at the root directory of the project. annotations: Additional metadata associated with the folder. The key is the name of your desired annotations. The value is an object containing a list of values (use empty list to represent no values for key) and the value type @@ -194,6 +196,9 @@ class Project( virtualtables: List["VirtualTable"] = field(default_factory=list, compare=False) """Any virtual tables that are at the root directory of the project.""" + searchindexes: List["SearchIndex"] = field(default_factory=list, compare=False) + """Any search indexes that are at the root directory of the project.""" + annotations: Optional[ Dict[ str, diff --git a/synapseclient/models/protocols/search_index_protocol.py b/synapseclient/models/protocols/search_index_protocol.py new file mode 100644 index 000000000..1485c8dc7 --- /dev/null +++ b/synapseclient/models/protocols/search_index_protocol.py @@ -0,0 +1,237 @@ +"""Protocol for the specific methods of this class that have synchronous counterparts +generated at runtime.""" + +from typing import TYPE_CHECKING, List, Optional, Protocol + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.models.search_dsl import Query, SourceFilter + +if TYPE_CHECKING: + from synapseclient.models.search_management import ( + SearchHit, + SearchIndexQuery, + SearchQuery, + SearchQueryPart, + ) + + +class SearchIndexSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SearchIndex operations.""" + + def store( + self, + dry_run: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Store metadata about a SearchIndex including the annotations. Creates + a new SearchIndex if `id` is not set, or updates the existing one + otherwise. `defining_sql` must be set before calling this. + + Arguments: + dry_run: If True, will not actually store the SearchIndex but will log + to the console what would be created or updated. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself. + + Raises: + ValueError: If `defining_sql` is not set. + + Example: Create a new SearchIndex. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex( + name="My Search Index", + parent_id="syn12345", + # syn67890 must be a table or a view; + defining_sql="SELECT * FROM syn67890", + ) + index = index.store() + print(f"Created SearchIndex: {index.id}") + ``` + """ + return self + + def get( + self, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """Get the metadata about the SearchIndex from Synapse. Either `id`, or + `name` and `parent_id`, must be set before calling this. + + Arguments: + include_activity: If True, will include the provenance activity on + the returned SearchIndex. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the Synapse response. + + Example: Get a SearchIndex by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + index = SearchIndex(id="syn12345").get() + print(index.name, index.defining_sql) + ``` + """ + return self + + def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: + """Delete the SearchIndex from Synapse. `id` must be set before calling + this. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Example: Delete a SearchIndex by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + + syn = Synapse() + syn.login() + + SearchIndex(id="syn12345").delete() + ``` + """ + return None + + def query( + self, + search_query: "SearchQuery", + response_parts: Optional[List["SearchQueryPart"]] = None, + *, + job_timeout: int = 600, + synapse_client: Optional[Synapse] = None, + ) -> "SearchIndexQuery": + """Query this search index. Unlike a SQL-backed Table, a SearchIndex is + queried with the + [OpenSearch Query DSL](https://docs.opensearch.org/latest/query-dsl/) + carried by a [SearchQuery][synapseclient.models.SearchQuery] — not with + Synapse SQL. See [Query][synapseclient.models.search_dsl.Query] for the + supported clause kinds. + + Arguments: + search_query: The OpenSearch + [`_search`](https://docs.opensearch.org/latest/api-reference/search-apis/search/) + body to execute against this index. + response_parts: Additional response parts to request beyond the + default hits, such as the total hit count or the select columns. + job_timeout: The maximum amount of time to wait for the query job to + complete before raising a `SynapseTimeoutError`. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The completed [SearchIndexQuery][synapseclient.models.SearchIndexQuery], carrying the `hits` and any requested response parts. + + Raises: + ValueError: If the `id` attribute has not been set. + + Example: Query an index for documents mentioning "alzheimer". +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex, SearchQuery, SearchQueryPart + from synapseclient.models.search_dsl import Query + + syn = Synapse() + syn.login() + + results = SearchIndex(id="syn12345").query( + search_query=SearchQuery( + query=Query(match={"title": {"query": "alzheimer"}}), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print(results.total_hits) + for hit in results.hits: + print(hit.row_id, hit.fields) + ``` + """ + from synapseclient.models.search_management import SearchIndexQuery + + return SearchIndexQuery() + + def autocomplete( + self, + query: Query, + source: Optional[SourceFilter] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SearchHit"]: + """Run a synchronous autocomplete search against this index. The + autocomplete endpoint allowlists only prefix-style queries + ([`prefix`](https://docs.opensearch.org/latest/query-dsl/term/prefix/), + [`match_phrase_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase-prefix/), + or [`match_bool_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-bool-prefix/)) + and caps results at 8. + + Arguments: + query: The top-level [OpenSearch Query DSL](https://docs.opensearch.org/latest/query-dsl/) + clause -- see [Query][synapseclient.models.search_dsl.Query]; + restricted server-side to `prefix`, `match_phrase_prefix`, or + `match_bool_prefix`. + source: Optional [source filter](https://docs.opensearch.org/latest/search-plugins/searching-data/retrieve-specific-fields/) + selecting which columns are returned on each hit. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The matching SearchHits, capped at 8. + + Raises: + ValueError: If the ``id`` attribute has not been set. + + Example: Autocomplete titles beginning with "alz". +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchIndex + from synapseclient.models.search_dsl import Query + + syn = Synapse() + syn.login() + + index = SearchIndex(id="syn12345") + hits = index.autocomplete( + query=Query(match_phrase_prefix={"title": {"query": "alz"}}), + ) + for hit in hits: + print(hit.row_id, hit.fields) + ``` + """ + return [] diff --git a/synapseclient/models/protocols/search_management_protocol.py b/synapseclient/models/protocols/search_management_protocol.py new file mode 100644 index 000000000..11d185b7f --- /dev/null +++ b/synapseclient/models/protocols/search_management_protocol.py @@ -0,0 +1,106 @@ +"""Protocol for the specific methods of this class that have synchronous counterparts +generated at runtime.""" + +from typing import TYPE_CHECKING, Optional, Protocol + +from typing_extensions import Self + +if TYPE_CHECKING: + from synapseclient import Synapse + + +class SearchConfigBindingSynchronousProtocol(Protocol): + """Synchronous interface for SearchConfigBinding operations.""" + + def store(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Bind ``search_configuration_id`` to the entity ``object_id``. Replaces + any existing binding on that entity. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the created SearchConfigBinding. + + Raises: + ValueError: If ``object_id`` or ``search_configuration_id`` is not set. + + Example: Bind a SearchConfiguration to a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + binding = SearchConfigBinding( + object_id="syn12345", + search_configuration_id="6789", + ) + binding = binding.store() + print(f"Bound SearchConfiguration {binding.search_configuration_id} " + f"to {binding.object_id}") + ``` + """ + return self + + def get(self, *, synapse_client: Optional["Synapse"] = None) -> "Self": + """Get the effective binding for the entity ``object_id``, resolved by + walking up the entity hierarchy. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Itself, populated from the resolved SearchConfigBinding. + + Raises: + ValueError: If ``object_id`` is not set. + + Example: Get the effective binding for a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + binding = SearchConfigBinding(object_id="syn12345").get() + print(binding.search_configuration_id) + ``` + """ + return self + + def delete(self, *, synapse_client: Optional["Synapse"] = None) -> None: + """Clear the binding on the entity ``object_id``. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Raises: + ValueError: If ``object_id`` is not set. + + Example: Clear the binding on a Project. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import SearchConfigBinding + + syn = Synapse() + syn.login() + + SearchConfigBinding(object_id="syn12345").delete() + ``` + """ + return None diff --git a/synapseclient/models/search_dsl.py b/synapseclient/models/search_dsl.py new file mode 100644 index 000000000..96e132d8b --- /dev/null +++ b/synapseclient/models/search_dsl.py @@ -0,0 +1,1205 @@ +"""Typed shapes for the OpenSearch query DSL accepted by Synapse. + +These `TypedDict` types mirror the server-side schema for a SearchIndex query +one-for-one, so a dict literal written inline against a +[SearchQuery][synapseclient.models.SearchQuery] slot is checked by your IDE +against the exact shape the Synapse REST API expects. + +Every type is `total=False`: all keys are optional to the type checker, and the +per-field docstrings mark which ones the server requires. A `TypedDict` is a +plain `dict` at runtime, so these annotations add no validation and no overhead, +and a raw dict is always an acceptable value. + +Start at [Query][synapseclient.models.search_dsl.Query] for the query clause +kinds and [Aggregation][synapseclient.models.search_dsl.Aggregation] for the +aggregation kinds. Both link out to the relevant +[OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) reference +for each clause. + +Background reading: + +- [OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) +- [Query vs. filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/) +- [Term-level vs. full-text queries](https://docs.opensearch.org/latest/query-dsl/term-vs-full-text/) +- [Aggregations](https://docs.opensearch.org/latest/aggregations/) +- [The `_search` API](https://docs.opensearch.org/latest/api-reference/search-apis/search/) +""" + +from typing import Any, Dict, List, TypedDict, Union + +ScalarValue = Union[str, int, float, bool] +"""A single column value. Which of these is valid depends on the target +column's type in the SearchIndex schema.""" + + +AnalyzerRef = TypedDict("AnalyzerRef", {"$ref": str}, total=False) +"""A reference to a saved [TextAnalyzer][synapseclient.models.TextAnalyzer] or +[SynonymSet][synapseclient.models.SynonymSet] by its qualified name +`{organizationName}-{name}`, written as `{"$ref": "my.org-stemmed_english"}`.""" + + +class ClauseScoringOptions(TypedDict, total=False): + """Scoring options shared by the OpenSearch query clauses: a relevance + `boost` and a `_name` label.""" + + boost: float + """Optional. Multiplier for the relevance score of this clause. + Default `1.0`.""" + + _name: str + """Optional. Label echoed back in matched-queries metadata.""" + + +class FuzzyMatchOptions(TypedDict, total=False): + """Fuzzy-matching parameters shared by the analyzed full-text match clauses + (`match`, `match_bool_prefix`, `multi_match`).""" + + fuzziness: Union[int, str] + """Optional. Allowed [edit distance](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/): + an integer or `AUTO`.""" + + fuzzy_rewrite: str + """Optional. How the fuzzy query is rewritten internally.""" + + fuzzy_transpositions: bool + """Optional. Whether to count transpositions (ab -> ba) as a single edit. + Default `true`.""" + + prefix_length: int + """Optional. Number of leading characters left unchanged when fuzzy + matching.""" + + +class MinimumShouldMatchOption(TypedDict, total=False): + """The minimum-should-match option shared by the analyzed full-text match + clauses.""" + + minimum_should_match: Union[int, str] + """Optional. [Minimum number of terms](https://docs.opensearch.org/latest/query-dsl/minimum-should-match/) + a document must match. An integer or a percentage / formula string.""" + + +class ZeroTermsQueryOption(TypedDict, total=False): + """The zero-terms-query behavior shared by the analyzed full-text match + clauses.""" + + zero_terms_query: str + """Optional. Behavior when the analyzer removes all tokens: `none` + (default) or `all`.""" + + +class MatchFieldOptions( + ClauseScoringOptions, + FuzzyMatchOptions, + MinimumShouldMatchOption, + ZeroTermsQueryOption, + total=False, +): + """Per-field options for a [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) + full-text clause. Carried as the value of the field-keyed `match` map (the + map key is the column name).""" + + query: ScalarValue + """Required. The text (or scalar value on a non-text column) to match. A + string, number, or boolean depending on the target column type.""" + + operator: str + """Optional. Boolean logic used to combine the analyzed query terms: `or` + (default) or `and`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to the + field's search analyzer.""" + + max_expansions: int + """Optional. Maximum number of terms the fuzzy expansion will generate.""" + + cutoff_frequency: float + """Optional. Term-frequency threshold above which terms are treated as + low-importance.""" + + auto_generate_synonyms_phrase_query: bool + """Optional. Whether to auto-generate phrase queries for multi-term + synonyms. Default `true`.""" + + lenient: bool + """Optional. When `true`, format-based errors (e.g. a text value against a + numeric field) are ignored.""" + + +class MatchPhraseFieldOptions(ClauseScoringOptions, ZeroTermsQueryOption, total=False): + """Per-field options for a [`match_phrase`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase/) + clause. Carried as the value of the field-keyed `match_phrase` map (the map + key is the column name).""" + + query: ScalarValue + """Required. The phrase to match. A string (or scalar value on a non-text + column).""" + + analyzer: str + """Optional. Analyzer used to tokenize the phrase. Defaults to the field's + search analyzer.""" + + slop: int + """Optional. Number of positions allowed between matching terms. Default + `0` (exact phrase).""" + + +class MatchPhrasePrefixFieldOptions( + ClauseScoringOptions, ZeroTermsQueryOption, total=False +): + """Per-field options for a [`match_phrase_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase-prefix/) + clause. Carried as the value of the field-keyed `match_phrase_prefix` map + (the map key is the column name).""" + + query: ScalarValue + """Required. The phrase whose last term is treated as a prefix. A + string.""" + + analyzer: str + """Optional. Analyzer used to tokenize the phrase. Defaults to the field's + search analyzer.""" + + slop: int + """Optional. Number of positions allowed between matching terms. Default + `0`.""" + + max_expansions: int + """Optional. Maximum number of terms the last (prefix) term expands into. + Default `50`.""" + + +class MatchBoolPrefixFieldOptions( + ClauseScoringOptions, FuzzyMatchOptions, MinimumShouldMatchOption, total=False +): + """Per-field options for a [`match_bool_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-bool-prefix/) + clause. Carried as the value of the field-keyed `match_bool_prefix` map + (the map key is the column name).""" + + query: ScalarValue + """Required. The text whose terms are matched, with the final term treated + as a prefix. A string.""" + + operator: str + """Optional. Boolean logic used to combine the analyzed terms: `or` + (default) or `and`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to the + field's search analyzer.""" + + max_expansions: int + """Optional. Maximum number of terms the final (prefix) term expands into. + Default `50`.""" + + +class TermFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`term`](https://docs.opensearch.org/latest/query-dsl/term/term/) + term-level clause (exact, non-analyzed match). Carried as the value of the + field-keyed `term` map (the map key is the column name).""" + + value: ScalarValue + """Required. The exact value to match. A string, number, boolean, or date + depending on the target column type.""" + + case_insensitive: bool + """Optional. When `true`, matches the value regardless of case. Default + `false`.""" + + +class RangeFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`range`](https://docs.opensearch.org/latest/query-dsl/term/range/) + term-level clause. Carried as the value of the field-keyed `range` map (the + map key is the column name).""" + + gte: ScalarValue + """Optional. Greater-than-or-equal-to bound. A number or date string, per + the target column type.""" + + gt: ScalarValue + """Optional. Greater-than bound.""" + + lte: ScalarValue + """Optional. Less-than-or-equal-to bound.""" + + lt: ScalarValue + """Optional. Less-than bound.""" + + format: str + """Optional. Date format used to parse the bound values on a date + column.""" + + relation: str + """Optional. How the range relates to range-typed field values: + `INTERSECTS` (default), `CONTAINS`, or `WITHIN`.""" + + time_zone: str + """Optional. UTC offset or IANA zone used to interpret date bounds.""" + + +class PrefixFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`prefix`](https://docs.opensearch.org/latest/query-dsl/term/prefix/) + term-level clause. Carried as the value of the field-keyed `prefix` map + (the map key is the column name). A leading `*` or `?` in `value` is + rejected (it forces a full index scan).""" + + value: ScalarValue + """Required. The prefix the indexed term must start with. A string (or + scalar value on a non-text column).""" + + case_insensitive: bool + """Optional. When `true`, matches the prefix regardless of case. Default + `false`.""" + + rewrite: str + """Optional. How the multi-term query is rewritten internally.""" + + +class WildcardFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`wildcard`](https://docs.opensearch.org/latest/query-dsl/term/wildcard/) + term-level clause. Carried as the value of the field-keyed `wildcard` map + (the map key is the column name). A leading `*` or `?` in the pattern is + rejected (it forces a full index scan).""" + + value: ScalarValue + """Optional. The wildcard pattern (`*` matches any sequence, `?` matches a + single character). A string. Either `value` or `wildcard` supplies the + pattern.""" + + wildcard: ScalarValue + """Optional. Alias for `value` -- the wildcard pattern. A string.""" + + case_insensitive: bool + """Optional. When `true`, matches the pattern regardless of case. Default + `false`.""" + + rewrite: str + """Optional. How the multi-term query is rewritten internally.""" + + +class FuzzyFieldOptions(ClauseScoringOptions, total=False): + """Per-field options for a [`fuzzy`](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/) + term-level clause. Carried as the value of the field-keyed `fuzzy` map (the + map key is the column name).""" + + value: ScalarValue + """Required. The term to match within the allowed edit distance. A string + (or scalar value on a non-text column).""" + + fuzziness: Union[int, str] + """Optional. Allowed [edit distance](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/): + an integer or `AUTO`.""" + + max_expansions: int + """Optional. Maximum number of terms the fuzzy expansion will generate. + Default `50`.""" + + prefix_length: int + """Optional. Number of leading characters left unchanged when fuzzy + matching.""" + + transpositions: bool + """Optional. Whether to count transpositions (ab -> ba) as a single edit. + Default `true`.""" + + rewrite: str + """Optional. How the multi-term query is rewritten internally.""" + + +class ExistsQuery(ClauseScoringOptions, total=False): + """An [`exists`](https://docs.opensearch.org/latest/query-dsl/term/exists/) + term-level clause. Matches documents that have any non-null value for the + given column.""" + + field: str + """Required. The column that must have a value.""" + + +class MultiMatchQuery( + ClauseScoringOptions, + FuzzyMatchOptions, + MinimumShouldMatchOption, + ZeroTermsQueryOption, + total=False, +): + """A [`multi_match`](https://docs.opensearch.org/latest/query-dsl/full-text/multi-match/) + full-text clause -- a `match` run across several columns at once.""" + + query: ScalarValue + """Required. The text to match across the listed columns. A string.""" + + fields: List[str] + """Required. The columns to search. Each entry may carry a `^boost` suffix + (e.g. `title^2`).""" + + type: str + """Optional. How the per-field matches are combined: `best_fields` + (default), `most_fields`, `cross_fields`, `phrase`, `phrase_prefix`, or + `bool_prefix`.""" + + operator: str + """Optional. Boolean logic used to combine the analyzed terms: `or` + (default) or `and`.""" + + tie_breaker: float + """Optional. Weight (0-1) applied to non-best field scores in + `best_fields` / `cross_fields`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to each + field's search analyzer.""" + + max_expansions: int + """Optional. Maximum number of terms a fuzzy / prefix expansion will + generate. Default `50`.""" + + slop: int + """Optional. Number of positions allowed between matching terms for the + phrase types.""" + + cutoff_frequency: float + """Optional. Term-frequency threshold above which terms are treated as + low-importance.""" + + auto_generate_synonyms_phrase_query: bool + """Optional. Whether to auto-generate phrase queries for multi-term + synonyms. Default `true`.""" + + lenient: bool + """Optional. When `true`, format-based errors are ignored.""" + + +class SimpleQueryStringQuery( + ClauseScoringOptions, MinimumShouldMatchOption, total=False +): + """A [`simple_query_string`](https://docs.opensearch.org/latest/query-dsl/full-text/simple-query-string/) + full-text clause -- a compact mini-DSL (`+`, `|`, `-`, `"`, `*`, `()`) + parsed leniently across the listed columns.""" + + query: str + """Required. The simple-query-string expression.""" + + fields: List[str] + """Optional. The columns to search. Each entry may carry a `^boost` + suffix. Defaults to the index's default search fields.""" + + default_operator: str + """Optional. Boolean logic used between terms when no explicit operator is + given: `or` (default) or `and`.""" + + flags: str + """Optional. Pipe-delimited list of enabled syntax features (e.g. + `AND|OR|PREFIX`), or `ALL` / `NONE`.""" + + analyzer: str + """Optional. Analyzer used to tokenize the query text. Defaults to each + field's search analyzer.""" + + analyze_wildcard: bool + """Optional. Whether to analyze wildcard terms. Default `false`. A leading + wildcard with this enabled is rejected (it forces a full index scan).""" + + auto_generate_synonyms_phrase_query: bool + """Optional. Whether to auto-generate phrase queries for multi-term + synonyms. Default `true`.""" + + fuzzy_max_expansions: int + """Optional. Maximum number of terms a fuzzy expansion will generate. + Default `50`.""" + + fuzzy_prefix_length: int + """Optional. Number of leading characters left unchanged when fuzzy + matching.""" + + fuzzy_transpositions: bool + """Optional. Whether to count transpositions (ab -> ba) as a single edit. + Default `true`.""" + + lenient: bool + """Optional. When `true`, format-based errors are ignored.""" + + quote_field_suffix: str + """Optional. Suffix appended to field names for quoted (exact-phrase) + portions of the query.""" + + +class MatchAllQuery(ClauseScoringOptions, total=False): + """A [`match_all`](https://docs.opensearch.org/latest/query-dsl/match-all/) + clause. Matches every document. Use `{"match_all": {}}` to match all + documents.""" + + +class BoolQuery(ClauseScoringOptions, total=False): + """A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) + compound clause -- combines sub-clauses with boolean logic.""" + + must: List["Query"] + """Sub-clauses that must all match (scored). Logical AND.""" + + should: List["Query"] + """Sub-clauses that should match (scored). See `minimum_should_match`.""" + + must_not: List["Query"] + """Sub-clauses that must not match (filter context, not scored).""" + + filter: List["Query"] + """Sub-clauses that must all match in + [filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/) + (not scored).""" + + minimum_should_match: Union[int, str] + """Optional. How many `should` clauses must match. An integer or a + [percentage / formula string](https://docs.opensearch.org/latest/query-dsl/minimum-should-match/).""" + + adjust_pure_negative: bool + """Optional. Whether to automatically add a `match_all` when only negative + clauses are present. Default `true`.""" + + +class DisMaxQuery(ClauseScoringOptions, total=False): + """A [`dis_max`](https://docs.opensearch.org/latest/query-dsl/compound/disjunction-max/) + compound clause. A document matches if any sub-clause matches; its score is + the best single sub-clause score plus `tie_breaker` times the rest.""" + + queries: List["Query"] + """Required. The candidate clauses.""" + + tie_breaker: float + """Optional. Weight (0-1) applied to the scores of the non-best matching + clauses. Default `0.0`.""" + + +class ConstantScoreQuery(ClauseScoringOptions, total=False): + """A [`constant_score`](https://docs.opensearch.org/latest/query-dsl/compound/constant-score/) + compound clause. Wraps a filter and assigns every matching document the + same score (`boost`).""" + + filter: "Query" + """Required. The clause evaluated in + [filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/).""" + + +class BoostingQuery(ClauseScoringOptions, total=False): + """A [`boosting`](https://docs.opensearch.org/latest/query-dsl/compound/boosting/) + compound clause. Returns documents matching `positive`, demoting those that + also match `negative` by `negative_boost`.""" + + positive: "Query" + """Required. The clause documents must match.""" + + negative: "Query" + """Required. The clause whose matches are demoted.""" + + negative_boost: float + """Required. Multiplier (0-1) applied to the score of documents that also + match `negative`.""" + + +class Query(TypedDict, total=False): + """A single [OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) + clause. Exactly one of the keys below may be set -- the set key names the + clause kind. See [query vs. filter context](https://docs.opensearch.org/latest/query-dsl/query-filter-context/) + and [term-level vs. full-text queries](https://docs.opensearch.org/latest/query-dsl/term-vs-full-text/). + + The field-keyed leaf clauses (`match`, `term`, `range`, ...) are maps whose + key is the column name and whose value is the per-field options object -- + only the long form is accepted (e.g. `{"match": {"title": {"query": "x"}}}`, + not the `{"match": {"title": "x"}}` shorthand). The compound clauses + (`bool`, `dis_max`, ...) nest further Query DSL clauses recursively. + """ + + match: Dict[str, MatchFieldOptions] + """A [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) + full-text clause. Map of column name to its match options.""" + + match_phrase: Dict[str, MatchPhraseFieldOptions] + """A [`match_phrase`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase/) + clause. Map of column name to its phrase options.""" + + match_phrase_prefix: Dict[str, MatchPhrasePrefixFieldOptions] + """A [`match_phrase_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-phrase-prefix/) + clause. Map of column name to its options.""" + + match_bool_prefix: Dict[str, MatchBoolPrefixFieldOptions] + """A [`match_bool_prefix`](https://docs.opensearch.org/latest/query-dsl/full-text/match-bool-prefix/) + clause. Map of column name to its options.""" + + term: Dict[str, TermFieldOptions] + """A [`term`](https://docs.opensearch.org/latest/query-dsl/term/term/) + term-level clause. Map of column name to its term options.""" + + range: Dict[str, RangeFieldOptions] + """A [`range`](https://docs.opensearch.org/latest/query-dsl/term/range/) + term-level clause. Map of column name to its range bounds.""" + + prefix: Dict[str, PrefixFieldOptions] + """A [`prefix`](https://docs.opensearch.org/latest/query-dsl/term/prefix/) + term-level clause. Map of column name to its prefix options.""" + + wildcard: Dict[str, WildcardFieldOptions] + """A [`wildcard`](https://docs.opensearch.org/latest/query-dsl/term/wildcard/) + term-level clause. Map of column name to its wildcard options.""" + + fuzzy: Dict[str, FuzzyFieldOptions] + """A [`fuzzy`](https://docs.opensearch.org/latest/query-dsl/term/fuzzy/) + term-level clause. Map of column name to its fuzzy options.""" + + terms: Dict[str, Any] + """A [`terms`](https://docs.opensearch.org/latest/query-dsl/term/terms/) + term-level clause (matches any of several exact values). Field-keyed: + `{"terms": {"": [v1, v2], "boost": 1.0}}`. The cross-index + `terms`-lookup form is rejected. Untyped because the column name is itself a + key alongside the fixed option keys, which a `TypedDict` cannot express; the + same allowlist is enforced server-side.""" + + exists: ExistsQuery + """An [`exists`](https://docs.opensearch.org/latest/query-dsl/term/exists/) + term-level clause.""" + + multi_match: MultiMatchQuery + """A [`multi_match`](https://docs.opensearch.org/latest/query-dsl/full-text/multi-match/) + full-text clause.""" + + simple_query_string: SimpleQueryStringQuery + """A [`simple_query_string`](https://docs.opensearch.org/latest/query-dsl/full-text/simple-query-string/) + full-text clause.""" + + match_all: MatchAllQuery + """A [`match_all`](https://docs.opensearch.org/latest/query-dsl/match-all/) + clause.""" + + bool: BoolQuery + """A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) + compound clause -- combines sub-clauses with boolean logic.""" + + dis_max: DisMaxQuery + """A [`dis_max`](https://docs.opensearch.org/latest/query-dsl/compound/disjunction-max/) + compound clause.""" + + constant_score: ConstantScoreQuery + """A [`constant_score`](https://docs.opensearch.org/latest/query-dsl/compound/constant-score/) + compound clause.""" + + boosting: BoostingQuery + """A [`boosting`](https://docs.opensearch.org/latest/query-dsl/compound/boosting/) + compound clause.""" + + +class ExtendedBounds(TypedDict, total=False): + """Min/max bounds that force a [`histogram`](https://docs.opensearch.org/latest/aggregations/bucket/histogram/) + or [`date_histogram`](https://docs.opensearch.org/latest/aggregations/bucket/date-histogram/) + to emit buckets across the full range (used as `extended_bounds` or + `hard_bounds`). Bounding the range is what caps the bucket count.""" + + min: ScalarValue + """Lower bound. A number, or a date / date-math string on a + `date_histogram`.""" + + max: ScalarValue + """Upper bound. A number, or a date / date-math string on a + `date_histogram`.""" + + +class HistogramBoundsOptions(TypedDict, total=False): + """The extended/hard bounds options shared by the `histogram` and + `date_histogram` aggregations.""" + + extended_bounds: ExtendedBounds + """Optional. Forces buckets to span at least this min/max range.""" + + hard_bounds: ExtendedBounds + """Optional. Restricts buckets to this min/max range (values outside are + dropped).""" + + +class KeyedBucketOption(TypedDict, total=False): + """The keyed-output option shared by the bucketing aggregations.""" + + keyed: bool + """Optional. Whether to return buckets as a keyed object rather than an + array.""" + + +class MetricAggregation(TypedDict, total=False): + """Common options for the single-value numeric metric aggregations (`avg`, + `max`, `min`, `sum`).""" + + field: str + """Required. The numeric column to aggregate.""" + + format: str + """Optional. Format applied to the result value.""" + + +class MissingValueOption(TypedDict, total=False): + """The missing-value substitution option shared by the metric + aggregations.""" + + missing: ScalarValue + """Optional. Value substituted for documents missing `field`.""" + + +BucketOrder = Union[Dict[str, str], List[Dict[str, str]]] +"""Bucket sort order for a bucketing aggregation -- a `{metric: "asc|desc"}` +object or an array of them, applied in order.""" + + +class TermsAggregation(TypedDict, total=False): + """A [`terms`](https://docs.opensearch.org/latest/aggregations/bucket/terms/) + bucket aggregation -- one bucket per distinct value of `field`.""" + + field: str + """Required. The column to bucket by.""" + + size: int + """Optional. Maximum number of buckets to return. Capped server-side.""" + + shard_size: int + """Optional. Number of candidate buckets collected per shard before the + final reduce. Capped server-side.""" + + min_doc_count: int + """Optional. Minimum document count for a bucket to be returned. Default + `1`.""" + + shard_min_doc_count: int + """Optional. Per-shard minimum document count before a bucket is + considered.""" + + show_term_doc_count_error: bool + """Optional. Whether to return the per-bucket document-count error + bound.""" + + order: BucketOrder + """Optional. Bucket sort order -- a `{metric: "asc|desc"}` object or an + array of them.""" + + include: Union[str, List[ScalarValue]] + """Optional. Terms to include -- a regex string or an array of exact + values.""" + + exclude: Union[str, List[ScalarValue]] + """Optional. Terms to exclude -- a regex string or an array of exact + values.""" + + missing: ScalarValue + """Optional. Bucket value assigned to documents missing `field`.""" + + collect_mode: str + """Optional. `breadth_first` or `depth_first` sub-aggregation collection + strategy.""" + + execution_hint: str + """Optional. Internal execution strategy hint (`map` / + `global_ordinals`).""" + + format: str + """Optional. Format applied to the bucket key in the response.""" + + value_type: str + """Optional. Explicit value type for the field when it cannot be + inferred.""" + + +class HistogramAggregation(KeyedBucketOption, HistogramBoundsOptions, total=False): + """A [`histogram`](https://docs.opensearch.org/latest/aggregations/bucket/histogram/) + bucket aggregation over a numeric column. Must specify `extended_bounds` or + `hard_bounds` so the bucket count is bounded.""" + + field: str + """Required. The numeric column to bucket.""" + + interval: float + """Required. Bucket width. Must be positive.""" + + min_doc_count: int + """Optional. Minimum document count for a bucket to be returned.""" + + offset: float + """Optional. Shifts bucket boundaries by this amount.""" + + order: BucketOrder + """Optional. Bucket sort order -- a `{metric: "asc|desc"}` object or an + array of them.""" + + missing: ScalarValue + """Optional. Bucket value assigned to documents missing `field`.""" + + format: str + """Optional. Format applied to the bucket key in the response.""" + + +class DateHistogramAggregation(KeyedBucketOption, HistogramBoundsOptions, total=False): + """A [`date_histogram`](https://docs.opensearch.org/latest/aggregations/bucket/date-histogram/) + bucket aggregation over a date column. Must specify `extended_bounds` or + `hard_bounds` so the bucket count is bounded.""" + + field: str + """Required. The date column to bucket.""" + + calendar_interval: str + """Optional. Calendar-aware interval (e.g. `month`, `year`). Mutually + exclusive with `fixed_interval`.""" + + fixed_interval: str + """Optional. Fixed-duration interval (e.g. `30d`, `12h`). Mutually + exclusive with `calendar_interval`.""" + + interval: str + """Optional. Legacy interval (use `calendar_interval` / `fixed_interval` + instead).""" + + min_doc_count: int + """Optional. Minimum document count for a bucket to be returned.""" + + offset: str + """Optional. Shifts bucket boundaries by this duration.""" + + time_zone: str + """Optional. UTC offset or IANA zone used to compute bucket boundaries.""" + + order: BucketOrder + """Optional. Bucket sort order -- a `{metric: "asc|desc"}` object or an + array of them.""" + + missing: ScalarValue + """Optional. Bucket value assigned to documents missing `field`.""" + + format: str + """Optional. Date format applied to the bucket key in the response.""" + + +class RangeAggregation(KeyedBucketOption, total=False): + """A [`range`](https://docs.opensearch.org/latest/aggregations/bucket/range/) + bucket aggregation -- one bucket per caller-defined numeric range.""" + + field: str + """Required. The numeric column to bucket.""" + + ranges: List[Dict[str, Any]] + """Required. The [bucket ranges](https://docs.opensearch.org/latest/aggregations/bucket/range/), + each `{"from": , "to": , "key": "