From 0bf43340591dda8368d4ef824783cedb92c39d5e Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Thu, 20 Aug 2026 14:21:15 -0300 Subject: [PATCH 1/5] Add rescan-tests command for in-process test collection discovery Adds a th-cli rescan-tests command and the corresponding API client method that call the backend's new POST /api/v1/test_collections/rescan endpoint, letting side-loaded custom test scripts be picked up without restarting the backend container. Fixes project-chip/certification-tool#1083 --- tests/test_rescan_tests.py | 104 ++++++++++++++++++ .../api/test_collections_api.py | 21 ++++ th_cli/commands/__init__.py | 2 + th_cli/commands/rescan_tests.py | 57 ++++++++++ th_cli/main.py | 11 +- 5 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 tests/test_rescan_tests.py create mode 100644 th_cli/commands/rescan_tests.py diff --git a/tests/test_rescan_tests.py b/tests/test_rescan_tests.py new file mode 100644 index 0000000..dba43b2 --- /dev/null +++ b/tests/test_rescan_tests.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2026 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for the rescan_tests command.""" + +from unittest.mock import Mock, patch + +import pytest +from click.testing import CliRunner + +from th_cli.api_lib_autogen import models as api_models +from th_cli.api_lib_autogen.exceptions import UnexpectedResponse +from th_cli.commands.rescan_tests import rescan_tests +from th_cli.exceptions import ConfigurationError + + +@pytest.mark.unit +@pytest.mark.cli +class TestRescanTestsCommand: + """Test cases for the rescan_tests command.""" + + def test_rescan_tests_success( + self, + cli_runner: CliRunner, + mock_sync_apis: Mock, + mock_api_client: Mock, + sample_test_collections: api_models.TestCollections, + ) -> None: + """Test successful rescan of test collections.""" + # Arrange + api = mock_sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post + api.return_value = sample_test_collections + + with patch("th_cli.commands.rescan_tests.get_client", return_value=mock_api_client): + with patch("th_cli.commands.rescan_tests.SyncApis", return_value=mock_sync_apis): + # Act + result = cli_runner.invoke(rescan_tests) + + # Assert + assert result.exit_code == 0 + assert "Rescanned test collections successfully" in result.output + api.assert_called_once() + mock_api_client.close.assert_called_once() + + def test_rescan_tests_configuration_error(self, cli_runner: CliRunner) -> None: + """Test rescan_tests with configuration error.""" + with patch( + "th_cli.commands.rescan_tests.get_client", side_effect=ConfigurationError("Could not connect to server") + ): + result = cli_runner.invoke(rescan_tests) + + assert result.exit_code == 1 + assert "Error: Could not connect to server" in result.output + + def test_rescan_tests_api_error(self, cli_runner: CliRunner, mock_sync_apis: Mock, mock_api_client: Mock) -> None: + """Test rescan_tests when the server reports the Test Engine is busy.""" + api_exception = UnexpectedResponse( + status_code=409, + content=b"Test Engine is busy.", + ) + api = mock_sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post + api.side_effect = api_exception + + with patch("th_cli.commands.rescan_tests.get_client", return_value=mock_api_client): + with patch("th_cli.commands.rescan_tests.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(rescan_tests) + + assert result.exit_code == 1 + assert "Error: Failed to rescan test collections (Status: 409) - Test Engine is busy." in result.output + mock_api_client.close.assert_called_once() + + def test_rescan_tests_generic_exception( + self, cli_runner: CliRunner, mock_sync_apis: Mock, mock_api_client: Mock + ) -> None: + """Test rescan_tests with an unexpected error.""" + api = mock_sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post + api.side_effect = Exception("Unexpected error") + + with patch("th_cli.commands.rescan_tests.get_client", return_value=mock_api_client): + with patch("th_cli.commands.rescan_tests.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(rescan_tests) + + assert result.exit_code == 1 + assert "Could not rescan test collections" in result.output + mock_api_client.close.assert_called_once() + + def test_rescan_tests_help_message(self, cli_runner: CliRunner) -> None: + """Test the help message for the rescan_tests command.""" + result = cli_runner.invoke(rescan_tests, ["--help"]) + + assert result.exit_code == 0 + assert "Rescan available test collections" in result.output diff --git a/th_cli/api_lib_autogen/api/test_collections_api.py b/th_cli/api_lib_autogen/api/test_collections_api.py index f3c414b..c439214 100644 --- a/th_cli/api_lib_autogen/api/test_collections_api.py +++ b/th_cli/api_lib_autogen/api/test_collections_api.py @@ -33,6 +33,14 @@ def _build_for_read_test_collections_api_v1_test_collections__get(self) -> Corou """ return self.api_client.request(type_=m.TestCollections, method="GET", url="/api/v1/test_collections/") + def _build_for_rescan_test_collections_api_v1_test_collections_rescan_post( + self, + ) -> Coroutine[Any, Any, m.TestCollections]: + """ + Rescan Test Collections + """ + return self.api_client.request(type_=m.TestCollections, method="POST", url="/api/v1/test_collections/rescan") + class AsyncTestCollectionsApi(_TestCollectionsApi): async def read_test_collections_api_v1_test_collections__get(self) -> m.TestCollections: @@ -41,6 +49,12 @@ async def read_test_collections_api_v1_test_collections__get(self) -> m.TestColl """ return await self._build_for_read_test_collections_api_v1_test_collections__get() + async def rescan_test_collections_api_v1_test_collections_rescan_post(self) -> m.TestCollections: + """ + Rescan Test Collections + """ + return await self._build_for_rescan_test_collections_api_v1_test_collections_rescan_post() + class SyncTestCollectionsApi(_TestCollectionsApi): def read_test_collections_api_v1_test_collections__get(self) -> m.TestCollections: @@ -49,3 +63,10 @@ def read_test_collections_api_v1_test_collections__get(self) -> m.TestCollection """ coroutine = self._build_for_read_test_collections_api_v1_test_collections__get() return get_event_loop().run_until_complete(coroutine) + + def rescan_test_collections_api_v1_test_collections_rescan_post(self) -> m.TestCollections: + """ + Rescan Test Collections + """ + coroutine = self._build_for_rescan_test_collections_api_v1_test_collections_rescan_post() + return get_event_loop().run_until_complete(coroutine) diff --git a/th_cli/commands/__init__.py b/th_cli/commands/__init__.py index 73ca737..1c6f3b9 100644 --- a/th_cli/commands/__init__.py +++ b/th_cli/commands/__init__.py @@ -16,6 +16,7 @@ from .abort_testing import abort_testing from .available_tests import available_tests from .project import project +from .rescan_tests import rescan_tests from .run_tests import run_tests from .test_run_execution import test_run_execution from .test_runner_status import test_runner_status @@ -24,6 +25,7 @@ "abort_testing", "available_tests", "project", + "rescan_tests", "run_tests", "test_run_execution", "test_runner_status", diff --git a/th_cli/commands/rescan_tests.py b/th_cli/commands/rescan_tests.py new file mode 100644 index 0000000..bfd2312 --- /dev/null +++ b/th_cli/commands/rescan_tests.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2026 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import click + +from th_cli.api_lib_autogen.api_client import SyncApis +from th_cli.api_lib_autogen.exceptions import UnexpectedResponse +from th_cli.client import get_client +from th_cli.colorize import colorize_cmd_help, colorize_help, colorize_success +from th_cli.exceptions import CLIError, handle_api_error + + +@click.command( + short_help=colorize_help("Rescan available test collections"), + help=colorize_cmd_help( + "rescan_tests", + "Re-run test collection discovery on the backend, picking up newly " + "added or edited side-loaded test scripts without restarting it", + ), +) +def rescan_tests() -> None: + """Rescan available test collections""" + client = None + try: + client = get_client() + sync_apis: SyncApis = SyncApis(client) + test_collections = sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post() + + if test_collections is None: + raise CLIError("Server did not return test_collections") + + collection_count = len(test_collections.test_collections) + click.echo(colorize_success(f"Rescanned test collections successfully ({collection_count} found)")) + except CLIError: + raise # Re-raise CLI Errors as-is + except UnexpectedResponse as e: + handle_api_error(e, "rescan test collections") + except Exception as e: + raise CLIError( + f"Could not rescan test collections: {e}. Please check if the API server is running and accessible." + ) + finally: + if client: + client.close() diff --git a/th_cli/main.py b/th_cli/main.py index fe71981..2e88f9a 100644 --- a/th_cli/main.py +++ b/th_cli/main.py @@ -18,7 +18,15 @@ import click from th_cli.colorize import colorize_cmd_help, colorize_error, colorize_key_value -from th_cli.commands import abort_testing, available_tests, project, run_tests, test_run_execution, test_runner_status +from th_cli.commands import ( + abort_testing, + available_tests, + project, + rescan_tests, + run_tests, + test_run_execution, + test_runner_status, +) from th_cli.utils import get_cli_sha, get_cli_version, get_versions @@ -52,6 +60,7 @@ def root() -> None: root.add_command(abort_testing) root.add_command(available_tests) root.add_command(project) +root.add_command(rescan_tests) root.add_command(run_tests) root.add_command(test_run_execution) root.add_command(test_runner_status) From 88418ac05a9366ce43999556a0c072a7b1b9ec51 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Tue, 25 Aug 2026 10:09:41 -0300 Subject: [PATCH 2/5] Fix rescan_tests help-message test asserting on the wrong help text CliRunner.invoke(cmd, ["--help"]) renders the click.command's help= text, not short_help=. test_rescan_tests_help_message asserted on the short_help text ('Rescan available test collections'), which never appears in --help output, so the test failed in CI. Assert on a substring of the actual help= text instead. Addresses CodeRabbit review comment / CI failure on #110. --- tests/test_rescan_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_rescan_tests.py b/tests/test_rescan_tests.py index dba43b2..d1c7c6a 100644 --- a/tests/test_rescan_tests.py +++ b/tests/test_rescan_tests.py @@ -101,4 +101,4 @@ def test_rescan_tests_help_message(self, cli_runner: CliRunner) -> None: result = cli_runner.invoke(rescan_tests, ["--help"]) assert result.exit_code == 0 - assert "Rescan available test collections" in result.output + assert "Re-run test collection discovery on the backend" in result.output From 453222effe276c3c861cb015d09c147b4bfefe32 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Tue, 25 Aug 2026 10:18:43 -0300 Subject: [PATCH 3/5] Add missing test coverage for rescan_tests' None-response branch available_tests.py has a dedicated test for the 'server returned None' branch, but rescan_tests.py's equivalent branch was untested, likely tripping the project's --cov-fail-under=85 threshold and contributing to the CI failure alongside the help-text assertion bug fixed previously. --- tests/test_rescan_tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_rescan_tests.py b/tests/test_rescan_tests.py index d1c7c6a..aed7656 100644 --- a/tests/test_rescan_tests.py +++ b/tests/test_rescan_tests.py @@ -54,6 +54,24 @@ def test_rescan_tests_success( api.assert_called_once() mock_api_client.close.assert_called_once() + def test_rescan_tests_no_response( + self, + cli_runner: CliRunner, + mock_sync_apis: Mock, + mock_api_client: Mock, + ) -> None: + """Test handling of a None response from the server.""" + api = mock_sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post + api.return_value = None + + with patch("th_cli.commands.rescan_tests.get_client", return_value=mock_api_client): + with patch("th_cli.commands.rescan_tests.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(rescan_tests) + + assert result.exit_code == 1 + assert "Error: Server did not return test_collections" in result.output + mock_api_client.close.assert_called_once() + def test_rescan_tests_configuration_error(self, cli_runner: CliRunner) -> None: """Test rescan_tests with configuration error.""" with patch( From 4f70a145ae7e83e67ea85f588038b91c6747c2ba Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Tue, 25 Aug 2026 10:31:36 -0300 Subject: [PATCH 4/5] Fix flaky help-text assertion breaking on CI's terminal width CI actually failed with: AssertionError: assert 'Re-run test collection discovery on the backend' in 'Usage: rescan-tests [OPTIONS]\n\n rescan_tests: Re-run test collection discovery on the\n backend, picking up newly added or edited side-loaded test scripts without\n restarting it\n\n...' Click wraps help= text at a column width that depends on the runner's terminal width, which differs between local runs and CI. The wrap point happened to land between 'the' and 'backend', splitting the substring my assertion checked for across two lines. Collapse whitespace/newlines in the captured output before asserting, so the check is robust to any wrap width. --- tests/test_rescan_tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_rescan_tests.py b/tests/test_rescan_tests.py index aed7656..05b0621 100644 --- a/tests/test_rescan_tests.py +++ b/tests/test_rescan_tests.py @@ -119,4 +119,8 @@ def test_rescan_tests_help_message(self, cli_runner: CliRunner) -> None: result = cli_runner.invoke(rescan_tests, ["--help"]) assert result.exit_code == 0 - assert "Re-run test collection discovery on the backend" in result.output + # Click wraps the help text at a terminal-width-dependent column, so + # collapse whitespace/newlines before checking for the substring — + # otherwise the wrap point can land inside the expected text. + normalized_output = " ".join(result.output.split()) + assert "Re-run test collection discovery on the backend" in normalized_output From ed471e7fde996d7ed5c6024ffdf0aac524834f0a Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Tue, 25 Aug 2026 18:04:14 -0300 Subject: [PATCH 5/5] Fix rescan-tests raising a confusing error on client-side timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported by user testing the CLI against a real backend: $ th-cli rescan-tests Error: Could not rescan test collections: Error handling response: . Please check if the API server is running and accessible. Despite the error, a subsequent test run picked up the updated side-loaded script — the backend had rescanned successfully, but the CLI gave up first. Root cause: rescan_test_collections_...() previously used httpx's default 5s read timeout. Rescanning regenerates the Python test JSON files via the SDK container, which routinely takes much longer than 5s, so httpx.ReadTimeout fires client-side while the backend keeps running the rescan to completion in the background. The resulting ResponseHandlingException stringifies ReadTimeout('') as an empty string, producing the confusing 'Error handling response: .' message verbatim (reproduced and confirmed locally). Fixes it the same way abort_testing.py already works around this for its own (much faster) operation: - Extends the client's timeout to 120s (rescanning is far slower than aborting a test run, since it starts an SDK container). - Catches ResponseHandlingException and, when the underlying error is an httpx.TimeoutException, reports success-with-caveat instead of an error, since the backend keeps processing regardless. - Non-timeout ResponseHandlingExceptions (e.g. connection refused) still surface as a real CLIError. Note: abort_testing.py's existing equivalent branch checks e.source, which ResponseHandlingException doesn't define (only e.error) — that looks like a pre-existing bug, not something copied here. --- tests/test_rescan_tests.py | 32 +++++++++++++++++++++++++++++++- th_cli/commands/rescan_tests.py | 18 +++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/test_rescan_tests.py b/tests/test_rescan_tests.py index 05b0621..a61f905 100644 --- a/tests/test_rescan_tests.py +++ b/tests/test_rescan_tests.py @@ -19,9 +19,10 @@ import pytest from click.testing import CliRunner +from httpx import ConnectError, ReadTimeout from th_cli.api_lib_autogen import models as api_models -from th_cli.api_lib_autogen.exceptions import UnexpectedResponse +from th_cli.api_lib_autogen.exceptions import ResponseHandlingException, UnexpectedResponse from th_cli.commands.rescan_tests import rescan_tests from th_cli.exceptions import ConfigurationError @@ -99,6 +100,35 @@ def test_rescan_tests_api_error(self, cli_runner: CliRunner, mock_sync_apis: Moc assert "Error: Failed to rescan test collections (Status: 409) - Test Engine is busy." in result.output mock_api_client.close.assert_called_once() + def test_rescan_tests_timeout(self, cli_runner: CliRunner, mock_sync_apis: Mock, mock_api_client: Mock) -> None: + """A client-side timeout should not be reported as a failure: rescanning can + outlast the CLI's timeout while the backend keeps running it to completion.""" + api = mock_sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post + api.side_effect = ResponseHandlingException(ReadTimeout("timed out")) + + with patch("th_cli.commands.rescan_tests.get_client", return_value=mock_api_client): + with patch("th_cli.commands.rescan_tests.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(rescan_tests) + + assert result.exit_code == 0 + assert "Rescan request sent (backend may still be processing)" in result.output + mock_api_client.close.assert_called_once() + + def test_rescan_tests_response_handling_error_non_timeout( + self, cli_runner: CliRunner, mock_sync_apis: Mock, mock_api_client: Mock + ) -> None: + """A non-timeout response-handling error should still be reported as a failure.""" + api = mock_sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post + api.side_effect = ResponseHandlingException(ConnectError("connection refused")) + + with patch("th_cli.commands.rescan_tests.get_client", return_value=mock_api_client): + with patch("th_cli.commands.rescan_tests.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(rescan_tests) + + assert result.exit_code == 1 + assert "Could not rescan test collections" in result.output + mock_api_client.close.assert_called_once() + def test_rescan_tests_generic_exception( self, cli_runner: CliRunner, mock_sync_apis: Mock, mock_api_client: Mock ) -> None: diff --git a/th_cli/commands/rescan_tests.py b/th_cli/commands/rescan_tests.py index bfd2312..6990bab 100644 --- a/th_cli/commands/rescan_tests.py +++ b/th_cli/commands/rescan_tests.py @@ -15,13 +15,18 @@ # import click +from httpx import Timeout, TimeoutException from th_cli.api_lib_autogen.api_client import SyncApis -from th_cli.api_lib_autogen.exceptions import UnexpectedResponse +from th_cli.api_lib_autogen.exceptions import ResponseHandlingException, UnexpectedResponse from th_cli.client import get_client from th_cli.colorize import colorize_cmd_help, colorize_help, colorize_success from th_cli.exceptions import CLIError, handle_api_error +# Rescanning regenerates the Python test JSON files via the SDK container, +# which can take significantly longer than httpx's 5s default read timeout. +RESCAN_TIMEOUT = Timeout(120.0, connect=10.0) # 120s total, 10s connect + @click.command( short_help=colorize_help("Rescan available test collections"), @@ -36,6 +41,7 @@ def rescan_tests() -> None: client = None try: client = get_client() + client._async_client.timeout = RESCAN_TIMEOUT sync_apis: SyncApis = SyncApis(client) test_collections = sync_apis.test_collections_api.rescan_test_collections_api_v1_test_collections_rescan_post() @@ -46,6 +52,16 @@ def rescan_tests() -> None: click.echo(colorize_success(f"Rescanned test collections successfully ({collection_count} found)")) except CLIError: raise # Re-raise CLI Errors as-is + except ResponseHandlingException as e: + # Rescanning can outlast even the extended timeout above (e.g. a + # slow SDK container pull). The backend keeps running the rescan to + # completion regardless of whether the CLI is still waiting on it. + if isinstance(e.error, TimeoutException): + click.echo(colorize_success("Rescan request sent (backend may still be processing)")) + else: + raise CLIError( + f"Could not rescan test collections: {e}. Please check if the API server is running and accessible." + ) except UnexpectedResponse as e: handle_api_error(e, "rescan test collections") except Exception as e: