diff --git a/app/connectors_service/connectors/sources/servicenow/datasource.py b/app/connectors_service/connectors/sources/servicenow/datasource.py index abc5b783b..414e49639 100644 --- a/app/connectors_service/connectors/sources/servicenow/datasource.py +++ b/app/connectors_service/connectors/sources/servicenow/datasource.py @@ -55,6 +55,7 @@ "change_request": ["admin", "sn_change_read", "itil"], } ACLS_QUERY = "sys_security_acl.operation=read^sys_security_acl.name={table_name}" +PUBLIC_ROLE_NAME = "public" def _prefix_email(email): @@ -69,6 +70,10 @@ def _prefix_user_id(user_id): return prefix_identity("user_id", user_id) +def _prefix_role_id(role_id): + return prefix_identity("role_id", role_id) + + class EndSignal(Enum): SERVICE = "SERVICE_TASK_FINISHED" RECORD = "RECORD_TASK_FINISHED" @@ -97,6 +102,8 @@ def __init__(self, configuration): self.servicenow_mapping = {} self.invalid_services = [] + # Lazy per-sync cache of sys_user_role: (by_name, by_sys_id) + self._roles_maps = None self.task_count = 0 self.queue = MemQueue(maxmemsize=QUEUE_MEM_SIZE, refresh_timeout=120) @@ -176,6 +183,15 @@ def get_default_configuration(cls): "type": "bool", "value": False, }, + "expand_role_members": { + "depends_on": [{"field": "use_document_level_security", "value": True}], + "display": "toggle", + "label": "Expand role members", + "order": 9, + "tooltip": "When enabled, ServiceNow role members are written individually onto each document's access control list. Disable this for large tenants to store compact role tokens on documents instead, and resolve membership during access control syncs. Changing this setting requires a full content sync and access control sync.", + "type": "bool", + "value": True, + }, } def _dls_enabled(self): @@ -192,7 +208,15 @@ def _dls_enabled(self): return self.configuration["use_document_level_security"] - async def _user_access_control_doc(self, user): + def _expand_role_members(self): + """Whether roles are expanded into individual users on document ACLs. + + Default True preserves legacy behavior. When False (compact mode), documents + receive role_id tokens and membership is resolved on identity docs. + """ + return self.configuration.get("expand_role_members", True) + + async def _user_access_control_doc(self, user, role_ids=None): user_id = user.get("_id", "") user_name = user.get("user_name", "") user_email = user.get("email", "") @@ -200,17 +224,37 @@ async def _user_access_control_doc(self, user): _prefixed_user_id = _prefix_user_id(user_id=user_id) _prefixed_user_name = _prefix_username(user=user_name) _prefixed_email = _prefix_email(email=user_email) - return { - "_id": user_id, - "identity": { + access_control = [ + _prefixed_user_id, + _prefixed_user_name, + _prefixed_email, + ] + # role_ids is only set in compact mode; omit the field in legacy mode. + if role_ids is not None: + role_ids_list = [role_id for role_id in role_ids if role_id] + prefixed_role_ids = sorted( + prefixed + for role_id in role_ids_list + if (prefixed := _prefix_role_id(role_id)) is not None + ) + identity = { "user_id": _prefixed_user_id, "display_name": _prefixed_user_name, "email": _prefixed_email, - }, + "role_ids": prefixed_role_ids, + } + access_control.extend(prefixed_role_ids) + else: + identity = { + "user_id": _prefixed_user_id, + "display_name": _prefixed_user_name, + "email": _prefixed_email, + } + return { + "_id": user_id, + "identity": identity, "created_at": user.get("_timestamp"), - } | es_access_control_query( - access_control=[_prefixed_user_id, _prefixed_user_name, _prefixed_email] - ) + } | es_access_control_query(access_control=access_control) async def _fetch_all_users(self): self._logger.debug("Fetching all users.") @@ -227,19 +271,59 @@ async def _fetch_users_by_roles(self, role): ): yield user + async def _fetch_user_roles_map(self): + """Build a map of user sys_id -> set of role sys_ids from sys_user_has_role. + + API fetches are already paginated via ``_table_data_generator`` + (``TABLE_FETCH_SIZE``). Only the user→roles map is held in memory for the + ACL sync: O(role assignments), not O(users × documents). For large tenants + that is tens of MB, versus the GB-scale per-document ACL expansion this + compact mode replaces. + """ + user_roles = {} + async for assignment in self._table_data_generator( + service_name="sys_user_has_role", params={} + ): + user_id = (assignment.get("user") or {}).get("value") + role_id = (assignment.get("role") or {}).get("value") + if not user_id or not role_id: + self._logger.debug( + "Skipping sys_user_has_role row with missing user or role reference" + ) + continue + user_roles.setdefault(user_id, set()).add(role_id) + return user_roles + async def get_access_control(self): if not self._dls_enabled(): self._logger.warning("DLS is not enabled. Skipping") return + user_roles = None + if not self._expand_role_members(): + self._logger.info( + "Enriching identity docs with role memberships from sys_user_has_role" + ) + user_roles = await self._fetch_user_roles_map() + async for user in self._fetch_all_users(): - yield await self._user_access_control_doc(user=user) + if user_roles is None: + yield await self._user_access_control_doc(user=user) + else: + user_id = user.get("_id") or user.get("sys_id") + yield await self._user_access_control_doc( + user=user, role_ids=user_roles.get(user_id, set()) + ) def _decorate_with_access_control(self, document, access_control): - if self._dls_enabled(): - document[ACCESS_CONTROL] = list( - set(document.get(ACCESS_CONTROL, []) + access_control) - ) + if not self._dls_enabled(): + return document + if access_control is None: + # Public table: omit field so DLS must_not exists grants access. + return document + document[ACCESS_CONTROL] = sorted( + set(document.get(ACCESS_CONTROL, []) + access_control) + ) return document async def _remote_validation(self): @@ -411,25 +495,120 @@ async def _fetch_table_data(self, batched_apis, table_access_control): finally: await self.queue.put(EndSignal.RECORD) - async def _fetch_access_controls(self, table_name): - access_control, user_roles, roles = [], [], {} - if table_name in DEFAULT_SERVICE_NAMES.keys(): - async for role in self._table_data_generator( - service_name="sys_user_role", params={} - ): - roles[role.get("name")] = role.get("sys_id") + async def _get_roles_maps(self): + """Return cached (by_name, by_sys_id) maps from sys_user_role for this sync.""" + if self._roles_maps is not None: + return self._roles_maps + by_name, by_sys_id = {}, {} + async for role in self._table_data_generator( + service_name="sys_user_role", params={} + ): + name = role.get("name") + sys_id = role.get("sys_id") + if name and sys_id: + by_name[name] = sys_id + by_sys_id[sys_id] = name + self._roles_maps = (by_name, by_sys_id) + return self._roles_maps + + async def _table_read_role_sys_ids(self, table_name): + """Return role sys_ids that grant read on a custom (non-default) table.""" + self._logger.info(f"Fetching roles of {table_name} with read operation.") + acl_params = { + "sys_security_acl.operation": "read", + "sys_security_acl.name": table_name, + "sys_security_acl.script": "", + "sys_security_acl.condition": "", + } + role_sys_ids = [] + async for acl in self._table_data_generator( + service_name="sys_security_acl_role", params=acl_params + ): + role_sys_id = (acl.get("sys_user_role") or {}).get("value") + if role_sys_id: + role_sys_ids.append(role_sys_id) + return role_sys_ids + + async def _fetch_access_controls_compact(self, table_name): + """Return role_id tokens for a table, or None when the table is public.""" + if table_name in DEFAULT_SERVICE_NAMES: + roles_by_name, _ = await self._get_roles_maps() + role_sys_ids = [] + for role_name in DEFAULT_SERVICE_NAMES.get(table_name, []): + if role_name.lower() == PUBLIC_ROLE_NAME: + self._logger.info( + f"Compact DLS: table {table_name} has public read role — " + "omitting _allow_access_control on content documents" + ) + return None + role_sys_id = roles_by_name.get(role_name) + if role_sys_id: + role_sys_ids.append(role_sys_id) + else: + self._logger.debug( + f"Role '{role_name}' not found while building compact ACL " + f"for table {table_name}" + ) + return self._finalize_compact_access_control(table_name, role_sys_ids) + + # Resolve table ACL roles first so we skip loading sys_user_role when empty. + role_sys_ids = await self._table_read_role_sys_ids(table_name) + if not role_sys_ids: + self._logger.info( + f"Compact DLS: no read roles for table {table_name}; " + "omitting _allow_access_control (treating as world-readable)" + ) + return None + + _, roles_by_sys_id = await self._get_roles_maps() + resolved = [] + for role_sys_id in role_sys_ids: + role_name = roles_by_sys_id.get(role_sys_id) + if not role_name: + self._logger.warning( + f"Skipping unknown role sys_id {role_sys_id} for table {table_name}" + ) + continue + if role_name.lower() == PUBLIC_ROLE_NAME: + self._logger.info( + f"Compact DLS: table {table_name} has public read role — " + "omitting _allow_access_control on content documents" + ) + return None + resolved.append(role_sys_id) + return self._finalize_compact_access_control(table_name, resolved) + + def _finalize_compact_access_control(self, table_name, role_sys_ids): + compact_acl = sorted( + prefixed + for role_id in role_sys_ids + if role_id and (prefixed := _prefix_role_id(role_id)) is not None + ) + if not compact_acl: + self._logger.info( + f"Compact DLS: no read roles for table {table_name}; " + "omitting _allow_access_control (treating as world-readable)" + ) + return None + return compact_acl + async def _fetch_access_controls_legacy(self, table_name): + """Expand role members into individual user_id tokens (legacy behavior).""" + access_control, user_roles = [], [] + roles_by_name, roles_by_sys_id = await self._get_roles_maps() + if table_name in DEFAULT_SERVICE_NAMES.keys(): for role in DEFAULT_SERVICE_NAMES.get(table_name, []): - async for user in self._fetch_users_by_roles(roles[role]): + role_sys_id = roles_by_name.get(role) + if not role_sys_id: + self._logger.warning( + f"Skipping unknown role '{role}' for table {table_name}" + ) + continue + async for user in self._fetch_users_by_roles(role_sys_id): access_control.append( _prefix_user_id(user_id=user.get("user", {}).get("value")) ) else: - async for role in self._table_data_generator( - service_name="sys_user_role", params={} - ): - roles[role.get("sys_id")] = role.get("name") - self._logger.info(f"Fetching roles of {table_name} with read operation.") acl_params = { "sys_security_acl.operation": "read", @@ -443,7 +622,13 @@ async def _fetch_access_controls(self, table_name): user_roles.append(acl.get("sys_user_role", {}).get("value")) for role in user_roles: - if roles.get(role).lower() == "public": + role_name = roles_by_sys_id.get(role) + if not role_name: + self._logger.warning( + f"Skipping unknown role sys_id {role} for table {table_name}" + ) + continue + if role_name.lower() == PUBLIC_ROLE_NAME: self._logger.info( f"Found public role in {table_name}, Fetching all users." ) @@ -456,7 +641,12 @@ async def _fetch_access_controls(self, table_name): access_control.append( _prefix_user_id(user_id=user.get("user", {}).get("value")) ) - return list(set(access_control)) + return sorted(set(access_control)) + + async def _fetch_access_controls(self, table_name): + if self._expand_role_members(): + return await self._fetch_access_controls_legacy(table_name) + return await self._fetch_access_controls_compact(table_name) async def _get_batched_apis(self, service_name, params): table_length = await self.servicenow_client.get_table_length( @@ -546,14 +736,29 @@ async def get_docs(self, filtering=None): advanced_rules_index + TABLE_BATCH_SIZE ) # noqa ] - filter_apis = await self.servicenow_client.get_filter_apis( - rules=batched_advanced_rules, mapping=servicenow_mapping - ) + rules_by_service = {} + for rule in batched_advanced_rules: + rules_by_service.setdefault(rule["service"], []).append(rule) + + for service_label, service_rules in rules_by_service.items(): + table_name = servicenow_mapping[service_label] + table_access_control = None + if self._dls_enabled(): + table_access_control = await self._fetch_access_controls( + table_name=table_name + ) + filter_apis = await self.servicenow_client.get_filter_apis( + rules=service_rules, mapping=servicenow_mapping + ) - await self.fetchers.put( - partial(self._fetch_table_data, filter_apis, []) - ) - self.task_count += 1 + await self.fetchers.put( + partial( + self._fetch_table_data, + filter_apis, + table_access_control, + ) + ) + self.task_count += 1 else: if ( @@ -569,7 +774,7 @@ async def get_docs(self, filtering=None): for service_name in ( self.servicenow_mapping.values() or DEFAULT_SERVICE_NAMES.keys() ): - table_access_control = [] + table_access_control = None if self._dls_enabled(): table_access_control = await self._fetch_access_controls( table_name=service_name diff --git a/app/connectors_service/tests/sources/fixtures/servicenow/connector.json b/app/connectors_service/tests/sources/fixtures/servicenow/connector.json index 9d13c203a..d5df4ef05 100644 --- a/app/connectors_service/tests/sources/fixtures/servicenow/connector.json +++ b/app/connectors_service/tests/sources/fixtures/servicenow/connector.json @@ -123,6 +123,26 @@ "ui_restrictions": [], "validations": [], "value": false + }, + "expand_role_members": { + "default_value": true, + "depends_on": [ + { + "field": "use_document_level_security", + "value": true + } + ], + "display": "toggle", + "label": "Expand role members", + "options": [], + "order": 9, + "required": true, + "sensitive": false, + "tooltip": "When enabled, ServiceNow role members are written individually onto each document's access control list. Disable this for large tenants to store compact role tokens on documents instead, and resolve membership during access control syncs. Changing this setting requires a full content sync and access control sync.", + "type": "bool", + "ui_restrictions": [], + "validations": [], + "value": true } } } \ No newline at end of file diff --git a/app/connectors_service/tests/sources/test_servicenow.py b/app/connectors_service/tests/sources/test_servicenow.py index e385c3218..243e7e780 100644 --- a/app/connectors_service/tests/sources/test_servicenow.py +++ b/app/connectors_service/tests/sources/test_servicenow.py @@ -12,12 +12,13 @@ import pytest from aiohttp.client_exceptions import ServerDisconnectedError from connectors_sdk.filtering.validation import Filter, SyncRuleValidationResult -from connectors_sdk.source import ConfigurableFieldValueError +from connectors_sdk.source import ConfigurableFieldValueError, DataSourceConfiguration -from connectors.access_control import DLS_QUERY +from connectors.access_control import ACCESS_CONTROL, DLS_QUERY from connectors.sources.servicenow.client import InvalidResponse, ServiceNowClient from connectors.sources.servicenow.datasource import ( ServiceNowDataSource, + _prefix_role_id, ) from connectors.sources.servicenow.validator import ServiceNowAdvancedRulesValidator from tests.commons import AsyncIterator @@ -28,7 +29,9 @@ @asynccontextmanager -async def create_service_now_source(use_text_extraction_service=False): +async def create_service_now_source( + use_text_extraction_service=False, expand_role_members=True +): async with create_source( ServiceNowDataSource, url="http://127.0.0.1:1234", @@ -36,6 +39,7 @@ async def create_service_now_source(use_text_extraction_service=False): password="changeme", services="*", use_text_extraction_service=use_text_extraction_service, + expand_role_members=expand_role_members, ) as source: yield source @@ -930,7 +934,9 @@ async def test_get_docs_with_advanced_rules_pagination(filtering): async for response in source.get_docs(filtering): response_list.append(response[0]) - mock_fetch_table_data.assert_called_once_with(expected_filter_apis, []) + mock_fetch_table_data.assert_called_once_with( + expected_filter_apis, None + ) assert [ { "sys_updated_on": "2023-10-10 05:21:45", @@ -966,7 +972,7 @@ async def test_get_access_control(): } }, } - async with create_service_now_source() as source: + async with create_service_now_source(expand_role_members=True) as source: source.servicenow_client.get_table_length = mock.AsyncMock(return_value=2) source._dls_enabled = Mock(return_value=True) with mock.patch.object( @@ -1002,8 +1008,78 @@ async def test_get_access_control_dls_disabled(): @pytest.mark.asyncio -async def test_fetch_access_control(): - async with create_service_now_source() as source: +async def test_get_access_control_includes_role_ids(): + async with create_service_now_source(expand_role_members=False) as source: + source._dls_enabled = Mock(return_value=True) + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator( + [ + { + "user": {"value": "id_1"}, + "role": {"value": "role_id_1"}, + } + ] + ), + AsyncIterator( + [ + { + "sys_updated_on": "2023-10-10 05:21:45", + "sys_id": "id_1", + "email": "admin@email.com", + "user_name": "demo.user", + "_id": "id_1", + "_timestamp": "2023-10-10T05:21:45", + } + ] + ), + ], + ): + users = [user async for user in source.get_access_control()] + + assert len(users) == 1 + assert users[0]["identity"]["role_ids"] == ["role_id:role_id_1"] + assert ( + "role_id:role_id_1" + in users[0]["query"]["template"]["params"]["access_control"] + ) + + +@pytest.mark.asyncio +async def test_get_access_control_legacy_does_not_fetch_roles(): + async with create_service_now_source(expand_role_members=True) as source: + source._dls_enabled = Mock(return_value=True) + source._fetch_user_roles_map = mock.AsyncMock( + side_effect=AssertionError("legacy mode must not fetch roles") + ) + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + return_value=AsyncIterator( + [ + { + "sys_updated_on": "2023-10-10 05:21:45", + "sys_id": "id_1", + "email": "admin@email.com", + "user_name": "demo.user", + "_id": "id_1", + "_timestamp": "2023-10-10T05:21:45", + } + ] + ), + ): + users = [user async for user in source.get_access_control()] + + assert len(users) == 1 + assert "role_ids" not in users[0]["identity"] + source._fetch_user_roles_map.assert_not_called() + + +@pytest.mark.asyncio +async def test_fetch_access_controls_legacy_expands_users(): + async with create_service_now_source(expand_role_members=True) as source: with mock.patch.object( ServiceNowDataSource, "_table_data_generator", @@ -1031,8 +1107,36 @@ async def test_fetch_access_control(): @pytest.mark.asyncio -async def test_fetch_access_control_for_public(): - async with create_service_now_source() as source: +async def test_fetch_access_controls_legacy_skips_deleted_role(): + async with create_service_now_source(expand_role_members=True) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator( + [ + { + "sys_id": "role_id_1", + "name": "role_1", + }, + ] + ), + AsyncIterator( + [ + {"sys_user_role": {"value": "deleted_role_id"}}, + {"sys_user_role": {"value": "role_id_1"}}, + ] + ), + AsyncIterator([{"user": {"value": "user_id_1"}}]), + ], + ): + access_control = await source._fetch_access_controls("service_name") + assert access_control == ["user_id:user_id_1"] + + +@pytest.mark.asyncio +async def test_fetch_access_controls_legacy_for_public(): + async with create_service_now_source(expand_role_members=True) as source: with mock.patch.object( ServiceNowDataSource, "_table_data_generator", @@ -1077,6 +1181,384 @@ async def test_fetch_access_control_for_public(): ) +@pytest.mark.asyncio +async def test_fetch_access_controls_compact_returns_role_tokens(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator( + [ + { + "sys_user_role": {"value": "role_id_1"}, + }, + ] + ), + AsyncIterator( + [ + { + "sys_id": "role_id_1", + "name": "role_1", + }, + ] + ), + ], + ) as mock_generator: + access_control = await source._fetch_access_controls("service_name") + assert access_control == ["role_id:role_id_1"] + # Table ACL roles + sys_user_role only — no user expansion pass + assert mock_generator.call_count == 2 + + +@pytest.mark.asyncio +async def test_fetch_access_controls_compact_public_returns_none(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator( + [ + { + "sys_user_role": {"value": "role_id_1"}, + }, + ] + ), + AsyncIterator( + [ + { + "sys_id": "role_id_1", + "name": "public", + }, + ] + ), + ], + ): + access_control = await source._fetch_access_controls("service_name") + assert access_control is None + + +@pytest.mark.asyncio +async def test_fetch_access_controls_compact_default_table_returns_role_tokens(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator( + [ + {"sys_id": "admin_id", "name": "admin"}, + {"sys_id": "itil_id", "name": "itil"}, + { + "sys_id": "sn_incident_read_id", + "name": "sn_incident_read", + }, + {"sys_id": "ml_report_user_id", "name": "ml_report_user"}, + {"sys_id": "ml_admin_id", "name": "ml_admin"}, + ] + ), + ], + ): + access_control = await source._fetch_access_controls("incident") + assert access_control == sorted( + [ + "role_id:admin_id", + "role_id:itil_id", + "role_id:sn_incident_read_id", + "role_id:ml_report_user_id", + "role_id:ml_admin_id", + ] + ) + + +@pytest.mark.asyncio +async def test_fetch_access_controls_compact_filters_deleted_roles(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator( + [ + {"sys_user_role": {"value": "deleted_role_id"}}, + {"sys_user_role": {"value": "role_id_1"}}, + ] + ), + AsyncIterator( + [ + { + "sys_id": "role_id_1", + "name": "role_1", + }, + ] + ), + ], + ): + access_control = await source._fetch_access_controls("service_name") + assert access_control == ["role_id:role_id_1"] + + +@pytest.mark.asyncio +async def test_fetch_access_controls_compact_no_acl_entries_returns_none(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[AsyncIterator([])], + ) as mock_generator: + access_control = await source._fetch_access_controls("service_name") + assert access_control is None + # Empty ACL short-circuits before loading sys_user_role + assert mock_generator.call_count == 1 + assert ( + mock_generator.call_args.kwargs["service_name"] + == "sys_security_acl_role" + ) + + +@pytest.mark.asyncio +async def test_roles_maps_cached_per_sync(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator([{"sys_user_role": {"value": "role_id_1"}}]), + AsyncIterator( + [ + {"sys_id": "role_id_1", "name": "role_1"}, + {"sys_id": "role_id_2", "name": "role_2"}, + ] + ), + AsyncIterator([{"sys_user_role": {"value": "role_id_2"}}]), + ], + ) as mock_generator: + first = await source._fetch_access_controls("table_a") + second = await source._fetch_access_controls("table_b") + + assert first == ["role_id:role_id_1"] + assert second == ["role_id:role_id_2"] + # One ACL fetch per table + one cached sys_user_role fetch + assert mock_generator.call_count == 3 + assert ( + mock_generator.call_args_list[0].kwargs["service_name"] + == "sys_security_acl_role" + ) + assert ( + mock_generator.call_args_list[1].kwargs["service_name"] == "sys_user_role" + ) + + +@pytest.mark.asyncio +async def test_decorate_skips_field_for_public(): + async with create_service_now_source() as source: + source._dls_enabled = Mock(return_value=True) + document = {"_id": "id_1"} + result = source._decorate_with_access_control(document, None) + assert ACCESS_CONTROL not in result + + +@pytest.mark.asyncio +async def test_decorate_with_role_tokens(): + async with create_service_now_source() as source: + source._dls_enabled = Mock(return_value=True) + document = {"_id": "id_1"} + result = source._decorate_with_access_control( + document, [_prefix_role_id("role_id_b"), _prefix_role_id("role_id_a")] + ) + assert result[ACCESS_CONTROL] == ["role_id:role_id_a", "role_id:role_id_b"] + + +@pytest.mark.asyncio +async def test_decorate_denies_all_for_empty_acl(): + async with create_service_now_source() as source: + source._dls_enabled = Mock(return_value=True) + document = {"_id": "id_1"} + result = source._decorate_with_access_control(document, []) + assert result[ACCESS_CONTROL] == [] + + +@pytest.mark.asyncio +async def test_fetch_access_controls_compact_empty_roles_returns_none(): + async with create_service_now_source(expand_role_members=False) as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + side_effect=[ + AsyncIterator([]), + ], + ): + access_control = await source._fetch_access_controls("incident") + assert access_control is None + + +@pytest.mark.asyncio +async def test_fetch_user_roles_map_skips_rows_without_role(): + async with create_service_now_source() as source: + with mock.patch.object( + ServiceNowDataSource, + "_table_data_generator", + return_value=AsyncIterator( + [ + {"user": {"value": "user_id_1"}}, + { + "user": {"value": "user_id_2"}, + "role": {"value": "role_id_1"}, + }, + ] + ), + ): + user_roles = await source._fetch_user_roles_map() + assert user_roles == {"user_id_2": {"role_id_1"}} + + +@pytest.mark.asyncio +async def test_expand_role_members_defaults_true_when_missing(): + config = ServiceNowDataSource.get_default_configuration() + config.pop("expand_role_members") + async with create_source( + ServiceNowDataSource, + url="http://127.0.0.1:1234", + username="admin", + password="changeme", + services="*", + ) as source: + source.configuration = DataSourceConfiguration(config) + source.configuration.set_defaults( + ServiceNowDataSource.get_default_configuration() + ) + assert source._expand_role_members() is True + + +@pytest.mark.asyncio +async def test_get_docs_advanced_rules_with_dls(): + filtering = Filter( + { + ADVANCED_SNIPPET: { + "value": [ + {"service": "Incident", "query": "user_nameSTARTSWITHa"}, + ] + } + } + ) + async with create_service_now_source(expand_role_members=False) as source: + source.servicenow_client.services = ["custom"] + source._dls_enabled = Mock(return_value=True) + source._fetch_access_controls = mock.AsyncMock( + return_value=["role_id:role_id_1"] + ) + source.servicenow_client._api_call = mock.AsyncMock( + return_value=MockResponse( + res=SAMPLE_RESPONSE, + headers={"Content-Type": "application/json", "x-total-count": 2}, + ) + ) + + response_list = [] + with mock.patch.object( + ServiceNowClient, + "filter_services", + return_value=({"Incident": "incident"}, []), + ): + with mock.patch.object( + ServiceNowClient, + "get_data", + side_effect=[ + AsyncIterator( + [ + [ + { + "sys_id": "id_1", + "sys_updated_on": "1212-12-12 12:12:12", + "sys_class_name": "incident", + "sys_user": "abc", + "type": "table_record", + }, + ] + ] + ), + AsyncIterator( + [ + [ + { + "sys_id": "id_2", + "table_sys_id": "id_1", + "sys_updated_on": "1212-12-12 12:12:12", + "sys_class_name": "incident", + "sys_user": "abc", + "type": "attachment_metadata", + }, + ] + ] + ), + ], + ): + async for response in source.get_docs(filtering): + response_list.append(response[0]) + + source._fetch_access_controls.assert_awaited_once_with(table_name="incident") + assert response_list[0][ACCESS_CONTROL] == ["role_id:role_id_1"] + assert response_list[1][ACCESS_CONTROL] == ["role_id:role_id_1"] + + +@pytest.mark.asyncio +async def test_get_docs_compact_stamps_roles_on_records(): + async with create_service_now_source(expand_role_members=False) as source: + source._dls_enabled = Mock(return_value=True) + source._fetch_access_controls = mock.AsyncMock( + return_value=["role_id:role_id_1"] + ) + source.servicenow_client._api_call = mock.AsyncMock( + return_value=MockResponse( + res=SAMPLE_RESPONSE, + headers={"Content-Type": "application/json", "x-total-count": 2}, + ) + ) + + response_list = [] + with mock.patch( + "connectors.sources.servicenow.datasource.DEFAULT_SERVICE_NAMES", + {"incident": ["sn_incident_read"]}, + ): + with mock.patch.object( + ServiceNowClient, + "get_data", + side_effect=[ + AsyncIterator( + [ + [ + { + "sys_id": "id_1", + "sys_updated_on": "1212-12-12 12:12:12", + "sys_class_name": "incident", + "sys_user": "admin", + "type": "table_record", + } + ] + ] + ), + Exception("Something went wrong"), + ], + ): + async for response in source.get_docs(): + response_list.append(response) + + assert ( + { + "_allow_access_control": ["role_id:role_id_1"], + "_id": "id_1", + "_timestamp": "1212-12-12T12:12:12", + "sys_id": "id_1", + "sys_updated_on": "1212-12-12 12:12:12", + "sys_class_name": "incident", + "sys_user": "admin", + "type": "table_record", + }, + None, + ) in response_list + + @pytest.mark.asyncio async def test_end_signal_is_added_to_queue_in_case_of_exception(): END_SIGNAL = "RECORD_TASK_FINISHED"