diff --git a/README.md b/README.md index ff911ef..6ecd0df 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,25 @@ You don't have to use `--local` to get local files: if Slack isn't configured saved to `--output_dir` automatically. Likewise, if the Information Radiator isn't configured (no `ZE_BASE_URL` / `ZE_REFRESH_TOKEN`), the harness falls back to a local reporter. + +### Overriding the target service +Tests specify which component to run against (`ars`, `ara`, ...), and the +harness normally resolves those components to deployed services through the +SmartAPI registry. To run the tests against a service that isn't deployed yet — +for example a locally running ARA you want to check before releasing — you can +override the target with `--target_url` and `--target`: +- `test-harness --local --target_url http://localhost:8080 --target aragorn download ` + +With an override in place: +- Every query is sent directly to `--target_url`, regardless of the `components` + specified in the tests, and the SmartAPI registry is not consulted. +- `--target` is the infores identifier of the service (with or without the + `infores:` prefix). Any target other than `ars` is treated as a single + service and queried with a `POST` to `/query`; use `--target ars` + to run a local ARS with the usual submit/poll flow. +- Pass/fail results are reported for the override target (instead of being + driven by the ARS), and performance tests are pointed at the override URL. +- Saved result filenames (CSV/JSON results, performance artifacts, and the + `--json_output` report) are prefixed with the target, e.g. + `aragorn_test_report.json`, so runs against different services stay + distinguishable. diff --git a/test_harness/main.py b/test_harness/main.py index cba4363..c227e24 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -35,6 +35,8 @@ def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] logger = get_logger(qid, args["log_level"]) + if bool(args.get("target_url")) != bool(args.get("target")): + return logger.error("--target_url and --target must be provided together.") tests = [] if "tests_url" in args: tests = download_tests(args["suite"], args["tests_url"], logger) @@ -50,6 +52,12 @@ def main(args): output_dir = args.get("output_dir") or "test_results" + # prefix saved/uploaded result filenames with the override target so runs + # against different local services don't produce indistinguishable files + target_prefix = "" + if args.get("target"): + target_prefix = f"{args['target'].split('infores:')[-1]}_" + # Run fully locally when asked to, or fall back to local stand-ins when the # respective service isn't configured, so developers can run the harness # without an Information Radiator or Slack workspace. @@ -79,7 +87,7 @@ def main(args): slacker = LocalSlacker(output_dir=output_dir, logger=logger) else: slacker = Slacker() - collector = ResultCollector(test_env, logger) + collector = ResultCollector(test_env, logger, target=args.get("target")) queried_envs = set() for test in tests.values(): queried_envs.add(test.test_env) @@ -104,22 +112,23 @@ def main(args): ) if collector.has_acceptance_results: slacker.upload_test_results_file( - reporter.test_name, + f"{target_prefix}{reporter.test_name}", "json", collector.acceptance_stats, ) slacker.upload_test_results_file( - reporter.test_name, + f"{target_prefix}{reporter.test_name}", "csv", collector.acceptance_csv, ) if collector.has_performance_results: slacker.upload_test_results_file( - reporter.test_name, + f"{target_prefix}{reporter.test_name}", "json", collector.performance_stats, ) for filename, content in collector.render_performance_artifacts(): + filename = f"{target_prefix}{filename}" try: slacker.upload_binary_file(filename, content) except Exception as e: @@ -130,7 +139,7 @@ def main(args): if args["json_output"]: os.makedirs(output_dir, exist_ok=True) - report_path = os.path.join(output_dir, "test_report.json") + report_path = os.path.join(output_dir, f"{target_prefix}test_report.json") logger.info(f"Saving report as JSON to {report_path}...") with open(report_path, "w") as f: json.dump(collector.acceptance_report, f) @@ -188,6 +197,28 @@ def cli(): help="Have the Test Harness send the test results to the Testing Dashboard", ) + parser.add_argument( + "--target_url", + type=url_type, + help=( + "Override the target service specified in the tests and send all " + "queries to this URL instead, e.g. http://localhost:8080 for a " + "locally running service. Must be used with --target." + ), + ) + + parser.add_argument( + "--target", + type=str, + help=( + "The infores identifier of the service at --target_url, with or " + "without the 'infores:' prefix, e.g. aragorn or infores:aragorn. " + "Anything other than ars is queried directly as a single service " + "(POST to /query); ars uses the normal ARS " + "submit/poll flow. Must be used with --target_url." + ), + ) + parser.add_argument( "--trapi_version", type=str, diff --git a/test_harness/performance_test_runner.py b/test_harness/performance_test_runner.py index 03f4ea0..dbdd58b 100644 --- a/test_harness/performance_test_runner.py +++ b/test_harness/performance_test_runner.py @@ -2,7 +2,7 @@ import logging import time -from typing import Dict, List +from typing import Dict, List, Optional import gevent from gevent import GreenletExit @@ -312,9 +312,19 @@ def _record_query_size( } -def run_performance_test(test: PerformanceTestCase, test_query: Dict, host: str): - """Wrapper function to run load tests with custom parameters""" - target = test.components[0] +def run_performance_test( + test: PerformanceTestCase, + test_query: Dict, + host: str, + target: Optional[str] = None, +): + """Wrapper function to run load tests with custom parameters. + + ``target`` overrides the component specified in the test case, so the + load test can be pointed at eg a locally running ARA regardless of what + the test says. Any target that isn't the ARS is queried as an ARA. + """ + target = target or test.components[0] results = run_locust_tests( host, diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index 2c0fa8d..4b3d7a8 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -201,8 +201,20 @@ def _summarize_response_sizes( class ResultCollector: """Collect results for easy dissemination.""" - def __init__(self, test_env: Optional[TestEnvEnum], logger: logging.Logger): - """Initialize the Collector.""" + def __init__( + self, + test_env: Optional[TestEnvEnum], + logger: logging.Logger, + target: Optional[str] = None, + ): + """Initialize the Collector. + + ``target`` is the infores of an override target service (with or + without the ``infores:`` prefix). A non-ARS override queries a single + service directly, so results are collected for that one agent instead + of the ARS + ARA roster; an ARS override still fans out to ARAs and + keeps the usual roster. + """ self.logger = logger self.has_acceptance_results = False self.has_performance_results = False @@ -222,6 +234,10 @@ def __init__(self, test_env: Optional[TestEnvEnum], logger: logging.Logger): "shepherd-arax", "shepherd-bte", ] + if target is not None: + target = target.split("infores:")[-1] + if target != "ars": + agents = [target] self.agents = agents self.query_types = ["TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"] self.acceptance_report = {status_type.value: 0 for status_type in AgentStatus} diff --git a/test_harness/run.py b/test_harness/run.py index 064489b..ea1d537 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -41,9 +41,17 @@ def run_tests( ) -> None: """Send tests through the Test Runners.""" logger.info(f"Running {len(tests)} queries...") - query_runner = QueryRunner(logger) + target_url = args.get("target_url") + target = args.get("target") + query_runner = QueryRunner(logger, target_url=target_url, target=target) logger.info("Runner is getting service registry") query_runner.retrieve_registry(trapi_version=args["trapi_version"]) + # The overall test status is normally driven by the ARS. When the target + # service specified in the tests is overridden (eg to run against a + # locally running ARA), it is driven by the override target instead. + status_agent = "ars" + if target_url is not None and target is not None: + status_agent = target.split("infores:")[-1] # loop over all tests for test in tqdm(list(tests.values())): # check if acceptance test @@ -179,12 +187,14 @@ def run_tests( agent_report.status = AgentStatus.FAILED agent_report.message = "Test Error" - # The overall test status is driven by ARS. If ARS didn't - # produce a result, the whole test is considered skipped. - if "ars" not in report.result: + # The overall test status is driven by the status agent + # (the ARS, or the override target when one is given). If + # it didn't produce a result, the whole test is considered + # skipped. + if status_agent not in report.result: status = AgentStatus.SKIPPED else: - status = report.result["ars"].status + status = report.result[status_agent].status # When the test is skipped, every agent is skipped too: the # query never really ran, so the incidental per-ARA @@ -285,11 +295,18 @@ def run_tests( ) status = AgentStatus.FAILED else: - host = query_runner.registry[env_map[test.test_env]][ - test.components[0] - ][0]["url"] + if target_url is not None: + host = query_runner.target_url + perf_target = status_agent + else: + host = query_runner.registry[env_map[test.test_env]][ + test.components[0] + ][0]["url"] + perf_target = None try: - results = run_performance_test(test, test_query, host) + results = run_performance_test( + test, test_query, host, target=perf_target + ) collector.collect_performance_result( test, asset, diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index f80968b..23c1e5c 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -2,7 +2,7 @@ import logging import time -from typing import Dict, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import httpx from translator_testing_model.datamodel.pydanticmodel import ( @@ -28,11 +28,36 @@ class QueryRunner: """Translator Test Query Runner.""" - def __init__(self, logger: logging.Logger): + def __init__( + self, + logger: logging.Logger, + target_url: Optional[str] = None, + target: Optional[str] = None, + ): + """Initialize the Query Runner. + + ``target_url`` and ``target`` override the target service specified in + the tests themselves: when given, every query is sent to ``target_url`` + instead of the services registered for the test's components, and the + service is identified as ``target`` (an infores curie, with or without + the ``infores:`` prefix). This is how the harness is pointed at a + locally running ARA/ARS before it is released and deployed. + """ self.registry = {} self.logger = logger + self.target_url = target_url.rstrip("/") if target_url else None + if target is not None and not target.startswith("infores:"): + target = f"infores:{target}" + self.target_infores = target def retrieve_registry(self, trapi_version: str): + if self.target_url is not None: + # all queries go to the override target, so the registry of + # deployed services is never consulted + self.logger.info( + "Target override in use; skipping SmartAPI registry retrieval." + ) + return self.registry = retrieve_registry_from_smartapi(trapi_version) def run_query( @@ -290,29 +315,47 @@ def run_queries( except Exception as e: self.logger.warning(e) - # send queries to a single type of component at a time - for component in test_case.components: - # component = "ara" - # loop over all specified components, i.e. ars, ara, kp, utilities + if self.target_url is not None: + # ignore the components specified in the test case and send every + # query straight to the override target self.logger.info( - f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" + f"Overriding test-specified components; sending queries to {self.target_infores} at {self.target_url}" ) - try: - all_responses = [] - for service in self.registry[env_map[test_case.test_env]][component]: - for query_hash, query in queries.items(): - all_responses.append( - self.run_query( - query_hash, - query["query"], - service["url"], - service["infores"], - ) - ) - for query_hash, responses, pks in all_responses: - queries[query_hash]["responses"].update(responses) - queries[query_hash]["pks"].update(pks) - except Exception as e: - self.logger.error(f"Something went wrong with the queries: {e}") + self._send_queries( + [{"url": self.target_url, "infores": self.target_infores}], + queries, + ) + else: + # send queries to a single type of component at a time + for component in test_case.components: + # component = "ara" + # loop over all specified components, i.e. ars, ara, kp, utilities + self.logger.info( + f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" + ) + self._send_queries( + self.registry[env_map[test_case.test_env]][component], + queries, + ) return queries, normalized_curies + + def _send_queries(self, services: List[Dict[str, str]], queries: Dict[int, dict]): + """Send all generated queries to each of the given services.""" + try: + all_responses = [] + for service in services: + for query_hash, query in queries.items(): + all_responses.append( + self.run_query( + query_hash, + query["query"], + service["url"], + service["infores"], + ) + ) + for query_hash, responses, pks in all_responses: + queries[query_hash]["responses"].update(responses) + queries[query_hash]["pks"].update(pks) + except Exception as e: + self.logger.error(f"Something went wrong with the queries: {e}") diff --git a/tests/test_local.py b/tests/test_local.py index af86788..2f5eeb8 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -119,3 +119,50 @@ def test_main_local_flag_uses_local_stand_ins(mocker, monkeypatch, tmp_path): # The real (networked) reporter/slacker are never constructed in local mode. reporter_cls.assert_not_called() slacker_cls.assert_not_called() + + +def test_target_override_prefixes_saved_filenames(mocker, tmp_path): + """With a target override, every saved filename is prefixed with the target.""" + + def fake_run_tests(tests, reporter, collector, logger, args): + collector.has_acceptance_results = True + + mocker.patch("test_harness.main.run_tests", side_effect=fake_run_tests) + + main( + { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": True, + "log_level": "ERROR", + "local": True, + "output_dir": str(tmp_path), + "target_url": "http://localhost:8080", + "target": "infores:aragorn", + } + ) + + saved = os.listdir(tmp_path) + # acceptance json + csv and the json report were all saved with the prefix + assert len(saved) == 3 + assert all(name.startswith("aragorn_") for name in saved) + assert "aragorn_test_report.json" in saved + + +def test_target_override_requires_both_flags(mocker, tmp_path): + """--target_url and --target must be given together.""" + run_tests = mocker.patch("test_harness.main.run_tests") + + args = { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": False, + "log_level": "ERROR", + "local": True, + "output_dir": str(tmp_path), + } + main({**args, "target_url": "http://localhost:8080"}) + main({**args, "target": "aragorn"}) + run_tests.assert_not_called() diff --git a/tests/test_run.py b/tests/test_run.py index 72e9822..3478834 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -56,3 +56,55 @@ def test_run_tests(mocker, httpx_mock: HTTPXMock): "trapi_version": "1.6.0", }, ) + + +def test_run_tests_with_target_override(httpx_mock: HTTPXMock): + """Test that a target override sends queries to the given service. + + The example tests specify the ars component, but with an override every + query should go straight to the local service, the SmartAPI registry + should never be fetched, and results should be collected for the + override agent only. + """ + httpx_mock.add_response( + url="http://localhost:8080/query", + json=kp_response, + ) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + json={ + "MONDO:0010794": None, + "DRUGBANK:DB00313": None, + "MESH:D001463": None, + "CHEBI:18295": None, + "CHEBI:31690": None, + "CL:0000097": None, + "MONDO:0004979": None, + "NCBIGene:3815": None, + "NCBIGene:4254": None, + "PR:000049994": None, + }, + ) + collector = MockResultCollector("ci", logger, target="aragorn") + assert collector.agents == ["aragorn"] + run_tests( + tests=example_test_cases, + reporter=MockReporter( + base_url="http://test", + ), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.6.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + }, + ) + # every asset was recorded against the override agent + total_results = sum( + count + for query_type in collector.acceptance_stats["aragorn"].values() + for count in query_type.values() + ) + assert total_results > 0