From fc1ba4b717abc0c975f2ceedc3950964ffc4b149 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:56:23 +0000 Subject: [PATCH 1/2] Add local mode to run without Reporter and Slacker Developers can now run the harness without an Information Radiator or a Slack workspace via a new `--local` CLI switch. In local mode the harness makes no network calls to either service: a LocalReporter hands out sequential ids and logs instead of uploading, and a LocalSlacker logs notifications and saves the CSV/JSON results and performance artifacts to disk under `--output_dir` (default `test_results/`). The stand-ins also kick in automatically when a service isn't configured: if Slack has no webhook/token/channel, results are saved locally; if the Information Radiator has no base URL/refresh token, the local reporter is used. `Reporter.is_configured` / `Slacker.is_configured` centralize that detection. Local result filenames are sanitized and de-duplicated so, for example, the acceptance and performance JSON summaries don't clobber each other. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01QKBHvKbt1zt5hGN215VtNv --- .gitignore | 2 + README.md | 17 ++++++ test_harness/main.py | 61 +++++++++++++++++--- test_harness/reporter.py | 79 +++++++++++++++++++++++++ test_harness/slacker.py | 78 +++++++++++++++++++++++++ tests/test_local.py | 121 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 350 insertions(+), 8 deletions(-) create mode 100644 tests/test_local.py diff --git a/.gitignore b/.gitignore index 2cc92da..6a03c19 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ logs/ scripts/ .env .vscode +test_results/ +test_report.json diff --git a/README.md b/README.md index c524445..ff911ef 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,20 @@ The Test Harness is a CLI that you need to install: Once everything is installed, you can call - `test-harness -h` to see available options + +### Running locally +By default the Test Harness reports results to an Information Radiator and +posts them to Slack. To run everything locally without those services (for +example while developing), pass `--local`: +- `test-harness --local download ` + +In local mode the harness makes no network calls to the Information Radiator or +Slack. The test results (CSV and JSON) and any performance artifacts are saved +to a local directory instead (`test_results/` by default, configurable with +`--output_dir`). + +You don't have to use `--local` to get local files: if Slack isn't configured +(no `SLACK_WEBHOOK_URL` / `SLACK_TOKEN` / `SLACK_CHANNEL`), the results are +saved to `--output_dir` automatically. Likewise, if the Information Radiator +isn't configured (no `ZE_BASE_URL` / `ZE_REFRESH_TOKEN`), the harness falls +back to a local reporter. diff --git a/test_harness/main.py b/test_harness/main.py index dc323a0..58fc37a 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -5,6 +5,7 @@ monkey.patch_all() import json +import os import time from argparse import ArgumentParser from urllib.parse import urlparse @@ -14,10 +15,10 @@ from test_harness.download import download_tests from test_harness.logger import get_logger, setup_logger -from test_harness.reporter import Reporter +from test_harness.reporter import LocalReporter, Reporter from test_harness.result_collector import ResultCollector from test_harness.run import run_tests -from test_harness.slacker import Slacker +from test_harness.slacker import LocalSlacker, Slacker setproctitle("TestHarness") setup_logger() @@ -47,16 +48,39 @@ def main(args): if len(tests) < 1: return logger.warning("No tests to run. Exiting.") - # Create test run in the Information Radiator - reporter = Reporter( + output_dir = args.get("output_dir") or "test_results" + + # Run fully locally when asked to, or fall back to local stand-ins when the + # respective service isn't configured, so developers can run the harness + # without an Information Radiator or Slack workspace. + local = args.get("local", False) + + use_local_reporter = local or not Reporter.is_configured( base_url=args.get("reporter_url"), refresh_token=args.get("reporter_access_token"), - logger=logger, ) + if use_local_reporter: + logger.info("Running without the Information Radiator (local reporter).") + reporter = LocalReporter(logger=logger) + else: + # Create test run in the Information Radiator + reporter = Reporter( + base_url=args.get("reporter_url"), + refresh_token=args.get("reporter_access_token"), + logger=logger, + ) reporter.get_auth() test_env = next(iter(tests.values())).test_env reporter.create_test_run(test_env, args["suite"]) - slacker = Slacker() + + use_local_slacker = local or not Slacker.is_configured() + if use_local_slacker: + logger.info( + f"Running without Slack; results will be saved to '{output_dir}'." + ) + slacker = LocalSlacker(output_dir=output_dir, logger=logger) + else: + slacker = Slacker() collector = ResultCollector(test_env, logger) queried_envs = set() for test in tests.values(): @@ -107,8 +131,10 @@ def main(args): reporter.finish_test_run() if args["json_output"]: - # logger.info("Saving report as JSON...") - with open("test_report.json", "w") as f: + os.makedirs(output_dir, exist_ok=True) + report_path = os.path.join(output_dir, "test_report.json") + logger.info(f"Saving report as JSON to {report_path}...") + with open(report_path, "w") as f: json.dump(collector.acceptance_report, f) return logger.info("All tests have completed!") @@ -177,6 +203,25 @@ def cli(): help="Save the test results locally in json", ) + parser.add_argument( + "--local", + action="store_true", + help=( + "Run entirely locally without an Information Radiator or Slack. " + "Test results (CSV/JSON) and artifacts are saved to --output_dir." + ), + ) + + parser.add_argument( + "--output_dir", + type=str, + default="test_results", + help=( + "Directory to save local test results and artifacts when Slack is " + "not configured or --local is used." + ), + ) + parser.add_argument( "--log_level", type=str, diff --git a/test_harness/reporter.py b/test_harness/reporter.py index 9d7b63f..2a0bdc9 100644 --- a/test_harness/reporter.py +++ b/test_harness/reporter.py @@ -242,3 +242,82 @@ def finish_test_run(self): res.raise_for_status() res_json = res.json() return res_json["status"] + + @staticmethod + def is_configured(base_url=None, refresh_token=None): + """Return True if enough config exists to talk to the Information Radiator. + + Falls back to the same environment variables the constructor uses, so + callers can decide whether to use a real Reporter or a LocalReporter + without instantiating one first. + """ + has_url = bool(base_url or os.getenv("ZE_BASE_URL")) + has_token = bool(refresh_token or os.getenv("ZE_REFRESH_TOKEN")) + return has_url and has_token + + +class LocalReporter(Reporter): + """A Reporter that keeps everything local and makes no network calls. + + Lets developers run the harness without an Information Radiator. It mirrors + the Reporter interface but hands out sequential ids and logs instead of + uploading, so the rest of the harness is oblivious to the difference. + """ + + def __init__( + self, + base_url=None, + refresh_token=None, + logger: logging.Logger = logging.getLogger(), + ): + super().__init__( + base_url=base_url or "http://localhost", + refresh_token=refresh_token, + logger=logger, + ) + self._next_test_id = 0 + + def get_auth(self): + """No authentication needed when running locally.""" + pass + + def create_test_run(self, test_env, suite_name): + """Create a local, in-memory test run.""" + self.test_name = f"{suite_name}: {datetime.now().strftime('%Y_%m_%d_%H_%M')}" + self.test_run_id = "local" + self.logger.info(f"[local] Created test run '{self.test_name}'") + return self.test_run_id + + def create_test(self, test, asset): + """Hand out a sequential id so downstream URLs still format.""" + self._next_test_id += 1 + return self._next_test_id + + def upload_labels(self, test_id, labels): + """No-op: labels are not persisted when running locally.""" + pass + + def upload_logs(self, test_id, logs): + """No-op: logs are not persisted when running locally.""" + pass + + def upload_artifact_references(self, test_id, artifact_references): + """No-op: artifact references are not persisted when running locally.""" + pass + + def upload_screenshot(self, test_id, screenshot): + """No-op: screenshots are not persisted when running locally.""" + pass + + def upload_log(self, test_id, message): + """No-op: logs are not persisted when running locally.""" + pass + + def finish_test(self, test_id, result): + """Nothing to send; just echo the result back.""" + return result + + def finish_test_run(self): + """Nothing to finish remotely when running locally.""" + self.logger.info("[local] Finished test run") + return None diff --git a/test_harness/slacker.py b/test_harness/slacker.py index bc6d9c0..6b869e3 100644 --- a/test_harness/slacker.py +++ b/test_harness/slacker.py @@ -3,6 +3,7 @@ import json import logging import os +import re import tempfile import httpx @@ -50,6 +51,21 @@ def __init__(self, url=None, token=None, slack_channel=None): self.client = WebClient(slack_token) self.logger = logging.getLogger(__name__) + @staticmethod + def is_configured(url=None, token=None, slack_channel=None): + """Return True if enough config exists to talk to Slack. + + Falls back to the same environment variables the constructor uses, so + callers can decide whether to use a real Slacker or a LocalSlacker + without instantiating one first. A webhook URL is required to post + notifications and a token + channel to upload result files. + """ + has_webhook = bool(url or os.getenv("SLACK_WEBHOOK_URL")) + has_uploads = bool(token or os.getenv("SLACK_TOKEN")) and bool( + slack_channel or os.getenv("SLACK_CHANNEL") + ) + return has_webhook and has_uploads + def post_notification(self, messages=[]): """Post a notification to Slack.""" # https://gist.github.com/mrjk/079b745c4a8a118df756b127d6499aa0 @@ -108,3 +124,65 @@ def upload_binary_file(self, filename, content, initial_comment=None, title=None file=tmp_path, initial_comment=initial_comment or "Performance report:", ) + + +def _slugify_filename(name): + """Make ``name`` safe to use as a filename (no spaces/colons/slashes).""" + slug = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("_") + return slug or "test_results" + + +class LocalSlacker(Slacker): + """A Slacker that writes results to disk instead of posting to Slack. + + Lets developers run the harness without a Slack workspace. Notifications + are logged and result/artifact files are saved under ``output_dir`` so the + CSV, JSON, and performance artifacts are still available locally. + """ + + def __init__(self, output_dir="test_results", logger=None): + # Intentionally skip Slacker.__init__: no Slack client/config needed. + self.output_dir = output_dir + self.logger = logger if logger is not None else logging.getLogger(__name__) + + def _unique_path(self, filename): + """Return a path in ``output_dir`` that doesn't clobber an existing file. + + Several results can share a base name (eg the acceptance and + performance JSON summaries), so append a counter rather than silently + overwriting a previously saved file. + """ + os.makedirs(self.output_dir, exist_ok=True) + base, ext = os.path.splitext(filename) + candidate = os.path.join(self.output_dir, filename) + counter = 1 + while os.path.exists(candidate): + candidate = os.path.join(self.output_dir, f"{base}_{counter}{ext}") + counter += 1 + return candidate + + def post_notification(self, messages=[]): + """Log notifications instead of posting them to Slack.""" + for message in messages: + self.logger.info(message) + + def upload_test_results_file(self, filename, extension, results): + """Save a results file locally instead of uploading it to Slack.""" + path = self._unique_path(f"{_slugify_filename(filename)}.{extension}") + with open(path, "w") as f: + if extension == "csv": + f.write(results) + elif extension == "json": + json.dump(results, f, indent=2) + else: + f.write(str(results)) + self.logger.info(f"Saved test results to {path}") + return path + + def upload_binary_file(self, filename, content, initial_comment=None, title=None): + """Save a binary artifact locally instead of uploading it to Slack.""" + path = self._unique_path(_slugify_filename(filename)) + with open(path, "wb") as f: + f.write(content) + self.logger.info(f"Saved artifact to {path}") + return path diff --git a/tests/test_local.py b/tests/test_local.py new file mode 100644 index 0000000..af86788 --- /dev/null +++ b/tests/test_local.py @@ -0,0 +1,121 @@ +"""Tests for running the harness locally without a Reporter or Slacker. + +These cover the ``--local`` switch and the fall-back to local stand-ins when +the Information Radiator / Slack aren't configured, including saving the CSV +and JSON results to disk. +""" + +import json +import os + +from test_harness.main import main +from test_harness.reporter import LocalReporter, Reporter +from test_harness.slacker import LocalSlacker, Slacker + +from .helpers.example_tests import example_test_cases + + +def test_slacker_is_configured(monkeypatch): + """Slack counts as configured only with a webhook plus token and channel.""" + monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False) + monkeypatch.delenv("SLACK_TOKEN", raising=False) + monkeypatch.delenv("SLACK_CHANNEL", raising=False) + assert not Slacker.is_configured() + assert not Slacker.is_configured(url="http://hook") # missing token/channel + assert Slacker.is_configured(url="http://hook", token="t", slack_channel="c") + + monkeypatch.setenv("SLACK_WEBHOOK_URL", "http://hook") + monkeypatch.setenv("SLACK_TOKEN", "t") + monkeypatch.setenv("SLACK_CHANNEL", "c") + assert Slacker.is_configured() + + +def test_reporter_is_configured(monkeypatch): + """The reporter needs both a base URL and a refresh token.""" + monkeypatch.delenv("ZE_BASE_URL", raising=False) + monkeypatch.delenv("ZE_REFRESH_TOKEN", raising=False) + assert not Reporter.is_configured() + assert not Reporter.is_configured(base_url="http://ir") # missing token + assert Reporter.is_configured(base_url="http://ir", refresh_token="tok") + + +def test_local_reporter_makes_no_network_calls(): + """The LocalReporter hands out sequential ids and never authenticates.""" + reporter = LocalReporter() + reporter.get_auth() # no-op, must not raise + assert reporter.authenticated_client is None + reporter.create_test_run("ci", "my-suite") + assert reporter.test_run_id == "local" + assert reporter.test_name.startswith("my-suite:") + assert reporter.create_test(None, None) == 1 + assert reporter.create_test(None, None) == 2 + assert reporter.finish_test("2", "PASSED") == "PASSED" + assert reporter.finish_test_run() is None + + +def test_local_slacker_saves_results_to_disk(tmp_path): + """CSV/JSON results and binary artifacts are written under output_dir.""" + slacker = LocalSlacker(output_dir=str(tmp_path)) + slacker.post_notification(["hello"]) # logged, not posted + + csv_path = slacker.upload_test_results_file("my-suite: 2026", "csv", "a,b\n1,2\n") + json_path = slacker.upload_test_results_file( + "my-suite: 2026", "json", {"passed": 3} + ) + bin_path = slacker.upload_binary_file("chart.png", b"\x89PNG") + + assert os.path.exists(csv_path) + assert os.path.exists(json_path) + assert os.path.exists(bin_path) + # filenames are sanitized (no spaces/colons) + assert " " not in os.path.basename(csv_path) + assert ":" not in os.path.basename(csv_path) + with open(csv_path) as f: + assert f.read() == "a,b\n1,2\n" + with open(json_path) as f: + assert json.load(f) == {"passed": 3} + + +def test_local_slacker_does_not_clobber_same_name(tmp_path): + """Two results sharing a base name both survive (eg acceptance + perf json).""" + slacker = LocalSlacker(output_dir=str(tmp_path)) + first = slacker.upload_test_results_file("run", "json", {"a": 1}) + second = slacker.upload_test_results_file("run", "json", {"b": 2}) + assert first != second + assert os.path.exists(first) and os.path.exists(second) + + +def test_main_local_flag_uses_local_stand_ins(mocker, monkeypatch, tmp_path): + """`--local` forces local reporter/slacker even when services are configured.""" + monkeypatch.setenv("SLACK_WEBHOOK_URL", "http://hook") + monkeypatch.setenv("SLACK_TOKEN", "t") + monkeypatch.setenv("SLACK_CHANNEL", "c") + monkeypatch.setenv("ZE_BASE_URL", "http://ir") + monkeypatch.setenv("ZE_REFRESH_TOKEN", "tok") + + run_tests = mocker.patch("test_harness.main.run_tests", return_value={}) + reporter_cls = mocker.patch("test_harness.main.Reporter", wraps=Reporter) + slacker_cls = mocker.patch("test_harness.main.Slacker", wraps=Slacker) + local_reporter = mocker.patch( + "test_harness.main.LocalReporter", wraps=LocalReporter + ) + local_slacker = mocker.patch("test_harness.main.LocalSlacker", wraps=LocalSlacker) + + main( + { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": False, + "log_level": "ERROR", + "local": True, + "output_dir": str(tmp_path), + } + ) + + run_tests.assert_called_once() + local_reporter.assert_called_once() + local_slacker.assert_called_once() + # The real (networked) reporter/slacker are never constructed in local mode. + reporter_cls.assert_not_called() + slacker_cls.assert_not_called() From 4b106a96e36a05f2906692b7f6a9a59812938d41 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Fri, 17 Jul 2026 10:25:57 -0400 Subject: [PATCH 2/2] Run black --- test_harness/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_harness/main.py b/test_harness/main.py index 58fc37a..cba4363 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -75,9 +75,7 @@ def main(args): use_local_slacker = local or not Slacker.is_configured() if use_local_slacker: - logger.info( - f"Running without Slack; results will be saved to '{output_dir}'." - ) + logger.info(f"Running without Slack; results will be saved to '{output_dir}'.") slacker = LocalSlacker(output_dir=output_dir, logger=logger) else: slacker = Slacker()