From fdd787bc595166ae249d7fa2562b1f111d7dd7b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 02:08:11 +0000 Subject: [PATCH 1/3] Add --target_url/--target to override the test-specified target service Tests specify which component (ars, ara, ...) to run against, and those components are resolved to deployed services via the SmartAPI registry. That makes it impossible to run a suite against a service that isn't deployed yet, e.g. a locally running ARA being validated before release. Add a pair of CLI flags that override the target: - --target_url: send every query straight to this URL, ignoring the components in the tests and skipping SmartAPI registry retrieval - --target: the infores of that service (with or without the infores: prefix); anything other than ars is queried as a single service via POST /query, while ars keeps the ARS submit/poll flow With an override, the overall pass/fail status is driven by the override agent instead of the ARS, the result collector reports on just that agent (unless the override is an ARS), and performance tests are pointed at the override URL with the matching locust user type. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016M8hanncqPqAhshbb1Vod2 --- README.md | 18 +++++ test_harness/main.py | 28 +++++++- test_harness/performance_test_runner.py | 18 +++-- test_harness/result_collector.py | 20 +++++- test_harness/run.py | 35 +++++++--- test_harness/runner/query_runner.py | 91 ++++++++++++++++++------- tests/test_run.py | 52 ++++++++++++++ 7 files changed, 222 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index ff911ef..5582fb3 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,21 @@ You don't have to use `--local` to get local files: if Slack isn't configured saved to `--output_dir` automatically. Likewise, if the Information Radiator isn't configured (no `ZE_BASE_URL` / `ZE_REFRESH_TOKEN`), the harness falls back to a local reporter. + +### Overriding the target service +Tests specify which component to run against (`ars`, `ara`, ...), and the +harness normally resolves those components to deployed services through the +SmartAPI registry. To run the tests against a service that isn't deployed yet — +for example a locally running ARA you want to check before releasing — you can +override the target with `--target_url` and `--target`: +- `test-harness --local --target_url http://localhost:8080 --target aragorn download ` + +With an override in place: +- Every query is sent directly to `--target_url`, regardless of the `components` + specified in the tests, and the SmartAPI registry is not consulted. +- `--target` is the infores identifier of the service (with or without the + `infores:` prefix). Any target other than `ars` is treated as a single + service and queried with a `POST` to `/query`; use `--target ars` + to run a local ARS with the usual submit/poll flow. +- Pass/fail results are reported for the override target (instead of being + driven by the ARS), and performance tests are pointed at the override URL. diff --git a/test_harness/main.py b/test_harness/main.py index cba4363..4b38100 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -35,6 +35,10 @@ def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] logger = get_logger(qid, args["log_level"]) + if bool(args.get("target_url")) != bool(args.get("target")): + return logger.error( + "--target_url and --target must be provided together." + ) tests = [] if "tests_url" in args: tests = download_tests(args["suite"], args["tests_url"], logger) @@ -79,7 +83,7 @@ def main(args): slacker = LocalSlacker(output_dir=output_dir, logger=logger) else: slacker = Slacker() - collector = ResultCollector(test_env, logger) + collector = ResultCollector(test_env, logger, target=args.get("target")) queried_envs = set() for test in tests.values(): queried_envs.add(test.test_env) @@ -188,6 +192,28 @@ def cli(): help="Have the Test Harness send the test results to the Testing Dashboard", ) + parser.add_argument( + "--target_url", + type=url_type, + help=( + "Override the target service specified in the tests and send all " + "queries to this URL instead, e.g. http://localhost:8080 for a " + "locally running service. Must be used with --target." + ), + ) + + parser.add_argument( + "--target", + type=str, + help=( + "The infores identifier of the service at --target_url, with or " + "without the 'infores:' prefix, e.g. aragorn or infores:aragorn. " + "Anything other than ars is queried directly as a single service " + "(POST to /query); ars uses the normal ARS " + "submit/poll flow. Must be used with --target_url." + ), + ) + parser.add_argument( "--trapi_version", type=str, diff --git a/test_harness/performance_test_runner.py b/test_harness/performance_test_runner.py index 03f4ea0..dbdd58b 100644 --- a/test_harness/performance_test_runner.py +++ b/test_harness/performance_test_runner.py @@ -2,7 +2,7 @@ import logging import time -from typing import Dict, List +from typing import Dict, List, Optional import gevent from gevent import GreenletExit @@ -312,9 +312,19 @@ def _record_query_size( } -def run_performance_test(test: PerformanceTestCase, test_query: Dict, host: str): - """Wrapper function to run load tests with custom parameters""" - target = test.components[0] +def run_performance_test( + test: PerformanceTestCase, + test_query: Dict, + host: str, + target: Optional[str] = None, +): + """Wrapper function to run load tests with custom parameters. + + ``target`` overrides the component specified in the test case, so the + load test can be pointed at eg a locally running ARA regardless of what + the test says. Any target that isn't the ARS is queried as an ARA. + """ + target = target or test.components[0] results = run_locust_tests( host, diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index 2c0fa8d..4b3d7a8 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -201,8 +201,20 @@ def _summarize_response_sizes( class ResultCollector: """Collect results for easy dissemination.""" - def __init__(self, test_env: Optional[TestEnvEnum], logger: logging.Logger): - """Initialize the Collector.""" + def __init__( + self, + test_env: Optional[TestEnvEnum], + logger: logging.Logger, + target: Optional[str] = None, + ): + """Initialize the Collector. + + ``target`` is the infores of an override target service (with or + without the ``infores:`` prefix). A non-ARS override queries a single + service directly, so results are collected for that one agent instead + of the ARS + ARA roster; an ARS override still fans out to ARAs and + keeps the usual roster. + """ self.logger = logger self.has_acceptance_results = False self.has_performance_results = False @@ -222,6 +234,10 @@ def __init__(self, test_env: Optional[TestEnvEnum], logger: logging.Logger): "shepherd-arax", "shepherd-bte", ] + if target is not None: + target = target.split("infores:")[-1] + if target != "ars": + agents = [target] self.agents = agents self.query_types = ["TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"] self.acceptance_report = {status_type.value: 0 for status_type in AgentStatus} diff --git a/test_harness/run.py b/test_harness/run.py index 064489b..ea1d537 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -41,9 +41,17 @@ def run_tests( ) -> None: """Send tests through the Test Runners.""" logger.info(f"Running {len(tests)} queries...") - query_runner = QueryRunner(logger) + target_url = args.get("target_url") + target = args.get("target") + query_runner = QueryRunner(logger, target_url=target_url, target=target) logger.info("Runner is getting service registry") query_runner.retrieve_registry(trapi_version=args["trapi_version"]) + # The overall test status is normally driven by the ARS. When the target + # service specified in the tests is overridden (eg to run against a + # locally running ARA), it is driven by the override target instead. + status_agent = "ars" + if target_url is not None and target is not None: + status_agent = target.split("infores:")[-1] # loop over all tests for test in tqdm(list(tests.values())): # check if acceptance test @@ -179,12 +187,14 @@ def run_tests( agent_report.status = AgentStatus.FAILED agent_report.message = "Test Error" - # The overall test status is driven by ARS. If ARS didn't - # produce a result, the whole test is considered skipped. - if "ars" not in report.result: + # The overall test status is driven by the status agent + # (the ARS, or the override target when one is given). If + # it didn't produce a result, the whole test is considered + # skipped. + if status_agent not in report.result: status = AgentStatus.SKIPPED else: - status = report.result["ars"].status + status = report.result[status_agent].status # When the test is skipped, every agent is skipped too: the # query never really ran, so the incidental per-ARA @@ -285,11 +295,18 @@ def run_tests( ) status = AgentStatus.FAILED else: - host = query_runner.registry[env_map[test.test_env]][ - test.components[0] - ][0]["url"] + if target_url is not None: + host = query_runner.target_url + perf_target = status_agent + else: + host = query_runner.registry[env_map[test.test_env]][ + test.components[0] + ][0]["url"] + perf_target = None try: - results = run_performance_test(test, test_query, host) + results = run_performance_test( + test, test_query, host, target=perf_target + ) collector.collect_performance_result( test, asset, diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index f80968b..23c1e5c 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -2,7 +2,7 @@ import logging import time -from typing import Dict, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import httpx from translator_testing_model.datamodel.pydanticmodel import ( @@ -28,11 +28,36 @@ class QueryRunner: """Translator Test Query Runner.""" - def __init__(self, logger: logging.Logger): + def __init__( + self, + logger: logging.Logger, + target_url: Optional[str] = None, + target: Optional[str] = None, + ): + """Initialize the Query Runner. + + ``target_url`` and ``target`` override the target service specified in + the tests themselves: when given, every query is sent to ``target_url`` + instead of the services registered for the test's components, and the + service is identified as ``target`` (an infores curie, with or without + the ``infores:`` prefix). This is how the harness is pointed at a + locally running ARA/ARS before it is released and deployed. + """ self.registry = {} self.logger = logger + self.target_url = target_url.rstrip("/") if target_url else None + if target is not None and not target.startswith("infores:"): + target = f"infores:{target}" + self.target_infores = target def retrieve_registry(self, trapi_version: str): + if self.target_url is not None: + # all queries go to the override target, so the registry of + # deployed services is never consulted + self.logger.info( + "Target override in use; skipping SmartAPI registry retrieval." + ) + return self.registry = retrieve_registry_from_smartapi(trapi_version) def run_query( @@ -290,29 +315,47 @@ def run_queries( except Exception as e: self.logger.warning(e) - # send queries to a single type of component at a time - for component in test_case.components: - # component = "ara" - # loop over all specified components, i.e. ars, ara, kp, utilities + if self.target_url is not None: + # ignore the components specified in the test case and send every + # query straight to the override target self.logger.info( - f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" + f"Overriding test-specified components; sending queries to {self.target_infores} at {self.target_url}" ) - try: - all_responses = [] - for service in self.registry[env_map[test_case.test_env]][component]: - for query_hash, query in queries.items(): - all_responses.append( - self.run_query( - query_hash, - query["query"], - service["url"], - service["infores"], - ) - ) - for query_hash, responses, pks in all_responses: - queries[query_hash]["responses"].update(responses) - queries[query_hash]["pks"].update(pks) - except Exception as e: - self.logger.error(f"Something went wrong with the queries: {e}") + self._send_queries( + [{"url": self.target_url, "infores": self.target_infores}], + queries, + ) + else: + # send queries to a single type of component at a time + for component in test_case.components: + # component = "ara" + # loop over all specified components, i.e. ars, ara, kp, utilities + self.logger.info( + f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" + ) + self._send_queries( + self.registry[env_map[test_case.test_env]][component], + queries, + ) return queries, normalized_curies + + def _send_queries(self, services: List[Dict[str, str]], queries: Dict[int, dict]): + """Send all generated queries to each of the given services.""" + try: + all_responses = [] + for service in services: + for query_hash, query in queries.items(): + all_responses.append( + self.run_query( + query_hash, + query["query"], + service["url"], + service["infores"], + ) + ) + for query_hash, responses, pks in all_responses: + queries[query_hash]["responses"].update(responses) + queries[query_hash]["pks"].update(pks) + except Exception as e: + self.logger.error(f"Something went wrong with the queries: {e}") diff --git a/tests/test_run.py b/tests/test_run.py index 72e9822..3478834 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -56,3 +56,55 @@ def test_run_tests(mocker, httpx_mock: HTTPXMock): "trapi_version": "1.6.0", }, ) + + +def test_run_tests_with_target_override(httpx_mock: HTTPXMock): + """Test that a target override sends queries to the given service. + + The example tests specify the ars component, but with an override every + query should go straight to the local service, the SmartAPI registry + should never be fetched, and results should be collected for the + override agent only. + """ + httpx_mock.add_response( + url="http://localhost:8080/query", + json=kp_response, + ) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + json={ + "MONDO:0010794": None, + "DRUGBANK:DB00313": None, + "MESH:D001463": None, + "CHEBI:18295": None, + "CHEBI:31690": None, + "CL:0000097": None, + "MONDO:0004979": None, + "NCBIGene:3815": None, + "NCBIGene:4254": None, + "PR:000049994": None, + }, + ) + collector = MockResultCollector("ci", logger, target="aragorn") + assert collector.agents == ["aragorn"] + run_tests( + tests=example_test_cases, + reporter=MockReporter( + base_url="http://test", + ), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.6.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + }, + ) + # every asset was recorded against the override agent + total_results = sum( + count + for query_type in collector.acceptance_stats["aragorn"].values() + for count in query_type.values() + ) + assert total_results > 0 From 4ec0faf14f1470a4f73866d6edef18385df6ea99 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:13:30 +0000 Subject: [PATCH 2/3] Prefix saved result filenames with the override target When running with --target_url/--target, prepend the target's infores id to the acceptance CSV/JSON results, performance artifacts, and the --json_output report (e.g. aragorn_test_report.json), so results from runs against different local services stay distinguishable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016M8hanncqPqAhshbb1Vod2 --- README.md | 4 ++++ test_harness/main.py | 15 ++++++++++---- tests/test_local.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5582fb3..6ecd0df 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,7 @@ With an override in place: to run a local ARS with the usual submit/poll flow. - Pass/fail results are reported for the override target (instead of being driven by the ARS), and performance tests are pointed at the override URL. +- Saved result filenames (CSV/JSON results, performance artifacts, and the + `--json_output` report) are prefixed with the target, e.g. + `aragorn_test_report.json`, so runs against different services stay + distinguishable. diff --git a/test_harness/main.py b/test_harness/main.py index 4b38100..587a40b 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -54,6 +54,12 @@ def main(args): output_dir = args.get("output_dir") or "test_results" + # prefix saved/uploaded result filenames with the override target so runs + # against different local services don't produce indistinguishable files + target_prefix = "" + if args.get("target"): + target_prefix = f"{args['target'].split('infores:')[-1]}_" + # Run fully locally when asked to, or fall back to local stand-ins when the # respective service isn't configured, so developers can run the harness # without an Information Radiator or Slack workspace. @@ -108,22 +114,23 @@ def main(args): ) if collector.has_acceptance_results: slacker.upload_test_results_file( - reporter.test_name, + f"{target_prefix}{reporter.test_name}", "json", collector.acceptance_stats, ) slacker.upload_test_results_file( - reporter.test_name, + f"{target_prefix}{reporter.test_name}", "csv", collector.acceptance_csv, ) if collector.has_performance_results: slacker.upload_test_results_file( - reporter.test_name, + f"{target_prefix}{reporter.test_name}", "json", collector.performance_stats, ) for filename, content in collector.render_performance_artifacts(): + filename = f"{target_prefix}{filename}" try: slacker.upload_binary_file(filename, content) except Exception as e: @@ -134,7 +141,7 @@ def main(args): if args["json_output"]: os.makedirs(output_dir, exist_ok=True) - report_path = os.path.join(output_dir, "test_report.json") + report_path = os.path.join(output_dir, f"{target_prefix}test_report.json") logger.info(f"Saving report as JSON to {report_path}...") with open(report_path, "w") as f: json.dump(collector.acceptance_report, f) diff --git a/tests/test_local.py b/tests/test_local.py index af86788..2f5eeb8 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -119,3 +119,50 @@ def test_main_local_flag_uses_local_stand_ins(mocker, monkeypatch, tmp_path): # The real (networked) reporter/slacker are never constructed in local mode. reporter_cls.assert_not_called() slacker_cls.assert_not_called() + + +def test_target_override_prefixes_saved_filenames(mocker, tmp_path): + """With a target override, every saved filename is prefixed with the target.""" + + def fake_run_tests(tests, reporter, collector, logger, args): + collector.has_acceptance_results = True + + mocker.patch("test_harness.main.run_tests", side_effect=fake_run_tests) + + main( + { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": True, + "log_level": "ERROR", + "local": True, + "output_dir": str(tmp_path), + "target_url": "http://localhost:8080", + "target": "infores:aragorn", + } + ) + + saved = os.listdir(tmp_path) + # acceptance json + csv and the json report were all saved with the prefix + assert len(saved) == 3 + assert all(name.startswith("aragorn_") for name in saved) + assert "aragorn_test_report.json" in saved + + +def test_target_override_requires_both_flags(mocker, tmp_path): + """--target_url and --target must be given together.""" + run_tests = mocker.patch("test_harness.main.run_tests") + + args = { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": False, + "log_level": "ERROR", + "local": True, + "output_dir": str(tmp_path), + } + main({**args, "target_url": "http://localhost:8080"}) + main({**args, "target": "aragorn"}) + run_tests.assert_not_called() From c46bd2875f92e833224d017ba532376ec9730efc Mon Sep 17 00:00:00 2001 From: Max Wang Date: Fri, 24 Jul 2026 07:48:40 -0400 Subject: [PATCH 3/3] Run black --- test_harness/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_harness/main.py b/test_harness/main.py index 587a40b..c227e24 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -36,9 +36,7 @@ def main(args): qid = str(uuid4())[:8] logger = get_logger(qid, args["log_level"]) if bool(args.get("target_url")) != bool(args.get("target")): - return logger.error( - "--target_url and --target must be provided together." - ) + return logger.error("--target_url and --target must be provided together.") tests = [] if "tests_url" in args: tests = download_tests(args["suite"], args["tests_url"], logger)