From 18dec37b320bc9d5515639541a34ec789513395c Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Fri, 16 Oct 2026 13:55:55 -0700 Subject: [PATCH] Deconflict DSS entity test IDs --- .basedpyright/baseline.json | 10 --- monitoring/prober/decode_id.py | 12 +-- monitoring/prober/infrastructure.py | 57 +++++++++++--- monitoring/prober/utils.py | 18 +++-- monitoring/uss_qualifier/README.md | 4 + .../resources/interuss/id_generator.py | 77 +++++++++++++++++-- .../suites/astm/utm/f3548_21.yaml | 18 ++++- .../IDGeneratorProviderSpecification.json | 16 ++++ .../IDGeneratorSpecification.json | 6 +- 9 files changed, 178 insertions(+), 40 deletions(-) create mode 100644 schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorProviderSpecification.json diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index 0d1417fc49e..a3cc15ee10e 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -5242,16 +5242,6 @@ } } ], - "./monitoring/uss_qualifier/resources/interuss/id_generator.py": [ - { - "code": "reportAssignmentType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - } - ], "./monitoring/uss_qualifier/resources/interuss/mock_uss/client.py": [ { "code": "reportArgumentType", diff --git a/monitoring/prober/decode_id.py b/monitoring/prober/decode_id.py index 341f1afef0d..d03cee9b944 100644 --- a/monitoring/prober/decode_id.py +++ b/monitoring/prober/decode_id.py @@ -4,6 +4,7 @@ import sys import monitoring.prober +import monitoring.uss_qualifier.scenarios from monitoring.prober import infrastructure @@ -26,9 +27,6 @@ def import_submodules(package, recursive=True): return results -import_submodules(monitoring.prober) - - def parse_args(argv: list[str]): parser = argparse.ArgumentParser(description="Decode a test ID") parser.add_argument( @@ -39,8 +37,12 @@ def parse_args(argv: list[str]): if __name__ == "__main__": args = parse_args(sys.argv[1:]) - owner_name, id_code = infrastructure.IDFactory.decode(args.id) - print(f"Owner: {owner_name}") + import_submodules(monitoring.prober) + import_submodules(monitoring.uss_qualifier.scenarios) + decoded = infrastructure.IDFactory.decode_details(args.id) + id_code = decoded.resource_type + print(f"Owner: {decoded.owner_name}") + print(f"Instance index: {decoded.instance_index}") if id_code in infrastructure.resource_type_code_descriptions: print( f"Resource type: {infrastructure.resource_type_code_descriptions[id_code]}" diff --git a/monitoring/prober/infrastructure.py b/monitoring/prober/infrastructure.py index 344a63575bb..d9519f81758 100644 --- a/monitoring/prober/infrastructure.py +++ b/monitoring/prober/infrastructure.py @@ -1,6 +1,7 @@ import functools import inspect import os +from dataclasses import dataclass import pytest @@ -111,7 +112,7 @@ def register_resource_type(code: int, description: str) -> ResourceType: """ test_filename = inspect.stack()[1].filename this_folder = os.path.dirname(os.path.abspath(__file__)) - test = test_filename[len(this_folder) + 1 :] + test = os.path.relpath(test_filename, this_folder) full_description = f"{test}: {description}" if code in resource_type_code_descriptions: raise ValueError( @@ -121,10 +122,17 @@ def register_resource_type(code: int, description: str) -> ResourceType: return code +@dataclass(frozen=True) +class DecodedTestID: + owner_name: str + resource_type: ResourceType + instance_index: int + + class IDFactory: """Creates UUIDv4 (as described in RFC4122) formatted IDs encoding the kind of ID and owner. - Format: 0000XXXX-YYYY-4ZYY-8YYY-YYYYYYYY0000 + Format: 000HXXXX-YYYY-4LYY-8YYY-YYYYYYYY0000 Note that a UUID V4 needs to conform to the following regexp, which is why we set the first digit of the fourth group to 8. @@ -133,35 +141,62 @@ class IDFactory: XXXX encodes the kind of ID according to id_codes. YYYYYYYYYYYYYYYYY encodes the owner/creator of the resource having the ID and - consists of 11 characters encoded as 6-bit groups. - Z is reserved and currently set to 0. + consists of 11 complete characters and 2 bits of a twelfth character, + encoded as 6-bit groups. Decoding marks the incomplete character with '?'. + HL encodes an instance index from 0 to 255 in hexadecimal. Index 0 produces + the original ID format; other indices separate repeated invocations of a test. """ owner_id: str + _instance_index: int - def __init__(self, test_owner: str): + def __init__(self, test_owner: str, instance_index: int = 0): + self.validate_instance_index(instance_index) self.owner_id = utils.encode_owner(test_owner) + self._instance_index = instance_index + + @staticmethod + def validate_instance_index(instance_index: int) -> None: + if type(instance_index) is not int or not 0 <= instance_index <= 255: + raise ValueError("instance_index must be an integer from 0 to 255") + + @property + def instance_index(self) -> int: + return self._instance_index def make_id(self, resource_type: ResourceType): """Make a test ID with the specified resource type code""" - return f"0000{utils.encode_resource_type_code(resource_type)}-{self.owner_id[0:4]}-40{self.owner_id[4:6]}-8{self.owner_id[6:9]}-{self.owner_id[9:17]}0000" + return f"000{self.instance_index >> 4:x}{utils.encode_resource_type_code(resource_type)}-{self.owner_id[0:4]}-4{self.instance_index & 0xF:x}{self.owner_id[4:6]}-8{self.owner_id[6:9]}-{self.owner_id[9:17]}0000" @classmethod def decode(cls, id: str) -> tuple[str, ResourceType]: - hex_digits = id.replace("-", "") + """Decode an ID, retaining the original two-item return value.""" + decoded = cls.decode_details(id) + return decoded.owner_name, decoded.resource_type + + @classmethod + def decode_details(cls, id: str) -> DecodedTestID: + """Decode the owner, resource type, and instance index of a test ID.""" + hex_digits = id.replace("-", "").lower() if len(hex_digits) != 32: raise ValueError(f"ID {id} has the wrong number of characters for a UUID") - if hex_digits[0:4] != "0000" or hex_digits[-4:] != "0000": + if any(c not in "0123456789abcdef" for c in hex_digits): + raise ValueError(f"ID {id} contains non-hexadecimal characters") + if hex_digits[0:3] != "000" or hex_digits[-4:] != "0000": raise ValueError( f"ID {id} does not have the leading and trailing zeros indicating a test ID" ) - if hex_digits[12:14] != "40": + if hex_digits[12] != "4" or hex_digits[16] != "8": raise ValueError(f"ID {id} is not formatted like a v4 test ID") x = hex_digits[4:8] - y = hex_digits[8:12] + hex_digits[14:28] + y = hex_digits[8:12] + hex_digits[14:16] + hex_digits[17:28] resource_type_code = ResourceType(int(x, 16)) owner_name = utils.decode_owner(y) - return owner_name, resource_type_code + return DecodedTestID( + owner_name=owner_name, + resource_type=resource_type_code, + instance_index=int(hex_digits[3] + hex_digits[13], 16), + ) check_scd_write_subscription_id = register_resource_type( diff --git a/monitoring/prober/utils.py b/monitoring/prober/utils.py index 441967f070e..a8f1973e97b 100644 --- a/monitoring/prober/utils.py +++ b/monitoring/prober/utils.py @@ -62,8 +62,14 @@ def encode_resource_type_code(resource_type: int) -> str: def decode_owner(owner_id: str) -> str: - """Decode an owner name from an 18-character hexidecimal string""" - if len(owner_id) != 18: + """Decode an owner name, marking a truncated final character as unknown. + + IDFactory preserves only the first 17 of the original 18 hexadecimal digits. + """ + truncated = len(owner_id) == 17 + if truncated: + owner_id += "0" + elif len(owner_id) != 18: raise ValueError("Invalid owner id.") hex_splits = split_by(owner_id, num=2) bits = "" @@ -73,15 +79,17 @@ def decode_owner(owner_id: str) -> str: for seq in split_by(bits, 6): num = bin_to_dec(seq) test_owner += get_ascii_val_from_bit_value(num) + if truncated: + test_owner = test_owner[:-1] + "?" if test_owner[0] != "?": return ( test_owner[: math.ceil(MAX_OWNER_LENGTH / 2)] + ".." + test_owner[-math.floor(MAX_OWNER_LENGTH / 2) :] ) - while test_owner[0] == "?": - test_owner = test_owner[1:] - return test_owner + if truncated: + return test_owner[:-1].lstrip("?") + "?" + return test_owner.lstrip("?") def decode_resouce_type(id: str) -> int: diff --git a/monitoring/uss_qualifier/README.md b/monitoring/uss_qualifier/README.md index 6df5981b84f..24f24997d5d 100644 --- a/monitoring/uss_qualifier/README.md +++ b/monitoring/uss_qualifier/README.md @@ -61,6 +61,10 @@ For a concrete example of how to invoke uss_qualifier in the manner described ab uss_qualifier is generally designed to skip test components when some prerequisites for those components are not met to enable testing of other parts of the systems under test until those prerequisites can be met. Components are generally skipped when a necessary resource is not provided. If you believe you have provided the resource that is claimed not to be provided, it is possible the resource could not be created due to another prerequisite not being met. See console warnings at the beginning of test execution for more information, and/or set [`stop_when_resource_not_created`](./configurations/configuration.py) to `true` in your configuration's `execution` configuration to stop test execution when such a prerequisite is not met. +### Orphaned DSS entities + +If a test entity (subscription, etc) is orphaned in the DSS, its origin can be determined by running (from the repository root) `uv run ./monitoring/prober/decode_id.py `. This displays the owner, instance index, and registered resource description, including qualifier scenarios. Owner encoding truncates its final character; the decoder displays `?` for that character. + ## Architecture ### Structure diff --git a/monitoring/uss_qualifier/resources/interuss/id_generator.py b/monitoring/uss_qualifier/resources/interuss/id_generator.py index 00e8d0859eb..e48aa669247 100644 --- a/monitoring/uss_qualifier/resources/interuss/id_generator.py +++ b/monitoring/uss_qualifier/resources/interuss/id_generator.py @@ -2,18 +2,22 @@ from monitoring.prober.infrastructure import IDFactory from monitoring.uss_qualifier.resources.communications import ClientIdentityResource -from monitoring.uss_qualifier.resources.resource import Resource +from monitoring.uss_qualifier.resources.resource import ( + Resource, + ResourceProvidingResource, + SupportedKeysNotSpecifiedError, +) class IDGeneratorSpecification(ImplicitDict): - """No fields required for the ID generator""" + instance_index: int = 0 + """Namespace for this invocation, from 0 to 255. Zero preserves legacy IDs.""" class IDGeneratorResource(Resource[IDGeneratorSpecification]): _client_identity: ClientIdentityResource - - # Not initialised before it's actually used - _id_factory: IDFactory = None + _instance_index: int + _id_factory: IDFactory | None def __init__( self, @@ -22,12 +26,73 @@ def __init__( client_identity: ClientIdentityResource, ): super().__init__(specification, resource_origin) + IDFactory.validate_instance_index(specification.instance_index) self._client_identity = client_identity + self._instance_index = specification.instance_index + self._id_factory = None + + @property + def instance_index(self) -> int: + return self._instance_index + + def with_instance_index( + self, instance_index: int, resource_origin: str | None = None + ) -> "IDGeneratorResource": + """Derive an independent generator, replacing rather than adding an index.""" + return IDGeneratorResource( + specification=IDGeneratorSpecification(instance_index=instance_index), + resource_origin=(resource_origin or f"Instance index {instance_index}") + + f" derived from {self.resource_origin}", + client_identity=self._client_identity, + ) @property def id_factory(self) -> IDFactory: # Not thread safe, but the consequences here are acceptable if self._id_factory is None: - self._id_factory = IDFactory(self._client_identity.subject()) + self._id_factory = IDFactory( + self._client_identity.subject(), self.instance_index + ) return self._id_factory + + +class IDGeneratorProviderSpecification(ImplicitDict): + first_instance_index: int = 1 + """First allocated instance index; environment index i uses first_instance_index + i. + + Allocated indices must fit in 0..255. Independent providers used concurrently + with the same client identity must be configured with disjoint ranges. + """ + + +class IDGeneratorProviderResource( + ResourceProvidingResource[IDGeneratorProviderSpecification, IDGeneratorResource] +): + """Provide deterministic ID namespaces for enumerated resource environments.""" + + base_resource: IDGeneratorResource + _first_instance_index: int + + def __init__( + self, + specification: IDGeneratorProviderSpecification, + resource_origin: str, + base_resource: IDGeneratorResource, + ): + super().__init__(specification, resource_origin) + IDFactory.validate_instance_index(specification.first_instance_index) + self.base_resource = base_resource + self._first_instance_index = specification.first_instance_index + + def provide_resource_for(self, **kwargs) -> IDGeneratorResource: + index = kwargs.get("index") + if type(index) is not int or index < 0: + raise SupportedKeysNotSpecifiedError("index must be a nonnegative integer") + instance_index = self._first_instance_index + index + return self.base_resource.with_instance_index( + instance_index, + resource_origin=self._provided_resource_origin( + f"index {index} (instance index {instance_index})" + ), + ) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index b3141e7692f..cf90481ee32 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -24,12 +24,26 @@ resources: second_utm_auth: resources.communications.AuthAdapterResource? utm_client_identity: resources.communications.ClientIdentityResource mock_uss: resources.interuss.mock_uss.client.MockUSSResource? - id_generator: resources.interuss.IDGeneratorResource + + # Zero or one of the following two resources must be provided + id_generator: resources.interuss.id_generator.IDGeneratorResource? + id_generator_provider: resources.interuss.id_generator.IDGeneratorProviderResource? + planning_area_provider: resources.resource.ResourceProvidingResource[resources.planning_area.PlanningAreaResource] problematically_big_area: resources.VolumeResource system_identity: resources.versioning.SystemIdentityResource? test_exclusions: resources.dev.TestExclusionsResource? local_resources: + id_generator: + resource_type: resources.interuss.id_generator.IDGeneratorResource + dependencies: + client_identity: utm_client_identity + specification: {} + id_generator_provider: + resource_type: resources.interuss.id_generator.IDGeneratorProviderResource + dependencies: + base_resource: id_generator + specification: {} conflicting_flights: resource_type: resources.resource.SingularResource[resources.flight_planning.FlightIntentsResource] dependencies: @@ -50,6 +64,7 @@ local_resources: dependencies: environments: dss_environments planning_area: planning_area_provider + id_generator: id_generator_provider # TODO: populate flight_intents from non_conflicting_flights_provider specification: {} priority_preemption_combinations: @@ -112,7 +127,6 @@ actions: second_utm_auth: second_utm_auth? utm_client_identity: utm_client_identity flight_intents: non_conflicting_flights - id_generator: id_generator problematically_big_area: problematically_big_area test_exclusions: test_exclusions? specification: diff --git a/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorProviderSpecification.json b/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorProviderSpecification.json new file mode 100644 index 00000000000..2423de0e5a2 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorProviderSpecification.json @@ -0,0 +1,16 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorProviderSpecification.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.resources.interuss.id_generator.IDGeneratorProviderSpecification, as defined in monitoring/uss_qualifier/resources/interuss/id_generator.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "first_instance_index": { + "description": "First allocated instance index; environment index i uses first_instance_index + i.\n\nAllocated indices must fit in 0..255. Independent providers used concurrently\nwith the same client identity must be configured with disjoint ranges.", + "type": "integer" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorSpecification.json b/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorSpecification.json index 4ad29547e11..579089a1e4b 100644 --- a/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorSpecification.json +++ b/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorSpecification.json @@ -1,11 +1,15 @@ { "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/resources/interuss/id_generator/IDGeneratorSpecification.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "No fields required for the ID generator\n\nmonitoring.uss_qualifier.resources.interuss.id_generator.IDGeneratorSpecification, as defined in monitoring/uss_qualifier/resources/interuss/id_generator.py", + "description": "monitoring.uss_qualifier.resources.interuss.id_generator.IDGeneratorSpecification, as defined in monitoring/uss_qualifier/resources/interuss/id_generator.py", "properties": { "$ref": { "description": "Path to content that replaces the $ref", "type": "string" + }, + "instance_index": { + "description": "Namespace for this invocation, from 0 to 255. Zero preserves legacy IDs.", + "type": "integer" } }, "type": "object"