diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b00d7622..7236b724 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -118,6 +118,45 @@ jobs: run: | dotnet test --configuration Debug --no-build --logger:"console;verbosity=detailed" api-validator-dotnet + - name: Create TestRun + run: | + pip3 install testit-cli + testit testrun create --token ${{ env.TMS_PRIVATE_TOKEN }} --output ${{ env.TEMP_FILE }} + echo "TMS_TEST_RUN_ID=$(<${{ env.TEMP_FILE }})" >> $GITHUB_ENV + pip3 uninstall -y -r <(pip freeze) + + - name: Setup environment import realtime false + run: | + dotnet build --configuration Debug --property WarningLevel=0 api-validator-dotnet + pip3 install -r python-examples/${{ matrix.project_name }}/requirements_ci.txt + pip3 install ./testit-python-commons + pip3 install ./${{ matrix.adapter_name }} + + + - name: Test import realtime false + run: | + export TMS_IMPORT_REALTIME=false + cd python-examples/${{ matrix.project_name }} + + # restart sync storage (binary was downloaded in the previous Test step) + pkill -f 'syncstorage-linux-amd64' || true + fuser -k 49152/tcp 2>/dev/null || true + sleep 1 + nohup .caches/syncstorage-linux-amd64 --testRunId ${{ env.TMS_TEST_RUN_ID }} --port 49152 \ + --baseURL ${{ env.TMS_URL }} --privateToken ${{ env.TMS_PRIVATE_TOKEN }} > service.log 2>&1 & + curl -v http://127.0.0.1:49152/health || true + + eval "${{ matrix.test_command }}" || true #ignore error code + + sleep 1 + curl -v http://127.0.0.1:49152/wait-completion?testRunId=${{ env.TMS_TEST_RUN_ID }} --max-time 100 || true + cat service.log + + - name: Validate import realtime false + run: | + dotnet test --configuration Debug --no-build --logger:"console;verbosity=detailed" api-validator-dotnet + + - name: Test Adapter Mode 2 run: | diff --git a/testit-adapter-behave/setup.py b/testit-adapter-behave/setup.py index 23a9bd7b..a0371f33 100644 --- a/testit-adapter-behave/setup.py +++ b/testit-adapter-behave/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.5" +VERSION = "4.2.6rc1" setup( name='testit-adapter-behave', diff --git a/testit-adapter-nose/setup.py b/testit-adapter-nose/setup.py index cf7f3df5..1133032b 100644 --- a/testit-adapter-nose/setup.py +++ b/testit-adapter-nose/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = "4.2.5" +VERSION = "4.2.6rc1" setup( name='testit-adapter-nose', diff --git a/testit-adapter-pytest/setup.py b/testit-adapter-pytest/setup.py index 2e9857f0..976ed17f 100644 --- a/testit-adapter-pytest/setup.py +++ b/testit-adapter-pytest/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.5" +VERSION = "4.2.6rc1" setup( name='testit-adapter-pytest', diff --git a/testit-adapter-robotframework/setup.py b/testit-adapter-robotframework/setup.py index 3af307ae..5b84b052 100644 --- a/testit-adapter-robotframework/setup.py +++ b/testit-adapter-robotframework/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.5" +VERSION = "4.2.6rc1" setup( name='testit-adapter-robotframework', diff --git a/testit-python-commons/realtime-import-spec.md b/testit-python-commons/realtime-import-spec.md new file mode 100644 index 00000000..2651e467 --- /dev/null +++ b/testit-python-commons/realtime-import-spec.md @@ -0,0 +1,101 @@ +# Real-Time Import (`importRealtime=true`) Specification + +This document describes how the Python adapter imports test results in real time and documents a known issue with nested test steps that appeared when the run finished. + +## Overview + +When `importRealtime=true` (CLI: `--testit-import-realtime`, config: `importrealtime`), each test result is sent to Test IT immediately after the test completes. At the end of the session, the adapter performs an additional update to attach fixture setup/teardown steps that were not available during the per-test upload. + +When `importRealtime=false`, all test results are sent once after the session finishes (`write_tests_after_all`). + +## Real-Time Flow + +1. **Per test** (`pytest_runtest_logfinish` → `AdapterManager.write_test` → `ApiClientWorker.write_test`): + - Autotest metadata is created or updated (including nested steps in the autotest model). + - Test result is posted via `set_auto_test_results_for_test_run` with full nested `step_results` converted by `Converter.step_results_to_attachment_put_model_autotest_step_results_model`. + +2. **After session** (`pytest_sessionfinish` → `AdapterManager.write_tests` → `ApiClientWorker.update_test_results`): + - Only fixture setup/teardown steps are uploaded for tests that were already sent in real time. + - Test result IDs are stored in `AdapterManager.__test_result_map` during the per-test upload. + +## Bug: Nested Steps Disappear After Run Completion + +### Symptoms + +- Nested steps are visible on the **autotest** card (correct). +- Nested steps are visible on the **test result** while the run is still in progress. +- After the run completes, only **top-level** steps remain on the test result. +- Reproducible with `importRealtime=true` (default since adapter 4.x). +- Not reproducible with `importRealtime=false`. + +### Root Cause + +At session finish, `update_test_results` used to: + +1. `GET` the test result from the API (`get_test_result_by_id`). +2. Build a PUT request from the GET response via `convert_test_result_model_to_test_results_id_put_request`. +3. Overwrite `setup_results` and `teardown_results` with fixture steps. +4. `PUT` the full model back, **including `step_results` from the GET response**. + +The problem is a model mismatch: + +| Operation | `step_results` model | Nested structure | +|-----------|---------------------|------------------| +| POST (create result) | `AttachmentPutModelAutoTestStepResultsModel` | Full tree via recursive `step_results` | +| GET (read result) | `StepResultApiModel` | References only (`step_id`, `outcome`, …) — **no nested `step_results`** | +| PUT (update result) | `StepResultApiModel` | Same flat reference list | + +Re-sending `step_results` from GET in the final PUT replaced the full nested tree (written during real-time import) with a flat list of top-level step references. Autotest steps were unaffected because they are written on a separate API path during `write_test`. + +### Fix + +The final session update must **not** send `step_results`. It should only attach fixture setup/teardown steps. + +**Before:** + +```python +test_result_response = self.get_test_result_by_id(test_result.get_test_result_id()) +model = Converter.convert_test_result_model_to_test_results_id_put_request(test_result_response) +model.setup_results = Converter.step_results_to_auto_test_step_result_update_request(...) +model.teardown_results = Converter.step_results_to_auto_test_step_result_update_request(...) +self.__test_results_api.api_v2_test_results_id_put(...) +``` + +**After:** + +```python +model = Converter.convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request( + test_result) +self.__test_results_api.api_v2_test_results_id_put(...) +``` + +`convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request` builds a PUT request with only: + +- `setup_results` — recursive `AutoTestStepResultUpdateRequest` (nested fixture steps preserved) +- `teardown_results` — same + +The `step_results` field is omitted, so the nested test steps written during real-time import are not overwritten. + +### Affected Code + +| File | Responsibility | +|------|----------------| +| `services/adapter_manager.py` | Calls `update_test_results` when `importRealtime=true` | +| `client/api_client.py` | `update_test_results` — final PUT after session | +| `client/converter.py` | `convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request` | + +### Verification + +1. Run pytest with nested `@testit.step` / `with testit.step(...)` and `importRealtime=true`. +2. Confirm nested steps on the test result **during** the run. +3. Wait for session finish (`pytest_sessionfinish`). +4. Confirm nested steps are still present on the test result after the run completes. + +Unit test: `tests/client/test_converter_update_test_results.py` — asserts the final PUT model does not include `step_results` and preserves nested setup steps. + +## Related Configuration + +| Setting | Default (4.x+) | Effect | +|---------|----------------|--------| +| `importRealtime` / `importrealtime` | `false` in config, real-time path used when enabled | Per-test upload + fixture update at end | +| `adapterMode` | varies | Parallel execution / Sync Storage coordination (separate from this issue) | diff --git a/testit-python-commons/setup.py b/testit-python-commons/setup.py index 0a6e6bc4..abd0ce26 100644 --- a/testit-python-commons/setup.py +++ b/testit-python-commons/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.5" +VERSION = "4.2.6rc1" setup( name='testit-python-commons', diff --git a/testit-python-commons/src/testit_python_commons/client/api_client.py b/testit-python-commons/src/testit_python_commons/client/api_client.py index 25922b48..5bc95bd5 100644 --- a/testit-python-commons/src/testit_python_commons/client/api_client.py +++ b/testit-python-commons/src/testit_python_commons/client/api_client.py @@ -499,13 +499,8 @@ def update_test_results(self, fixtures_containers: dict, test_result_ids: dict) fixtures_containers, test_result_ids) for test_result in test_results: - test_result_response = self.get_test_result_by_id(test_result.get_test_result_id()) - model = Converter.convert_test_result_model_to_test_results_id_put_request(test_result_response) - - model.setup_results = Converter.step_results_to_auto_test_step_result_update_request( - test_result.get_setup_results()) - model.teardown_results = Converter.step_results_to_auto_test_step_result_update_request( - test_result.get_teardown_results()) + model = Converter.convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request( + test_result) try: self.__test_results_api.api_v2_test_results_id_put( diff --git a/testit-python-commons/src/testit_python_commons/client/converter.py b/testit-python-commons/src/testit_python_commons/client/converter.py index fbaad29c..09a7d34a 100644 --- a/testit-python-commons/src/testit_python_commons/client/converter.py +++ b/testit-python-commons/src/testit_python_commons/client/converter.py @@ -316,9 +316,9 @@ def convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put cls, test_result: TestResultWithAllFixtureStepResults) -> ApiV2TestResultsIdPutRequest: return ApiV2TestResultsIdPutRequest( - setup_results=cls.step_results_to_attachment_put_model_autotest_step_results_model( + setup_results=cls.step_results_to_auto_test_step_result_update_request( test_result.get_setup_results()), - teardown_results=cls.step_results_to_attachment_put_model_autotest_step_results_model( + teardown_results=cls.step_results_to_auto_test_step_result_update_request( test_result.get_teardown_results())) @classmethod diff --git a/testit-python-commons/tests/client/test_converter_update_test_results.py b/testit-python-commons/tests/client/test_converter_update_test_results.py new file mode 100644 index 00000000..153d474f --- /dev/null +++ b/testit-python-commons/tests/client/test_converter_update_test_results.py @@ -0,0 +1,22 @@ +from testit_python_commons.client.converter import Converter +from testit_python_commons.models.outcome_type import OutcomeType +from testit_python_commons.models.step_result import StepResult +from testit_python_commons.models.test_result_with_all_fixture_step_results_model import ( + TestResultWithAllFixtureStepResults, +) + + +def test_setup_teardown_put_request_does_not_include_step_results(): + test_result = TestResultWithAllFixtureStepResults('result-id') + parent_step = StepResult().set_title('parent').set_outcome(OutcomeType.PASSED) + child_step = StepResult().set_title('child').set_outcome(OutcomeType.PASSED) + parent_step.set_step_results([child_step]) + test_result.set_setup_results([parent_step]) + + model = Converter.convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request( + test_result) + + assert 'step_results' not in model + assert len(model.setup_results) == 1 + assert len(model.setup_results[0].step_results) == 1 + assert model.setup_results[0].step_results[0].title == 'child' diff --git a/update_versions.sh b/update_versions.sh index 6974bed0..4025e9b4 100644 --- a/update_versions.sh +++ b/update_versions.sh @@ -1,6 +1,6 @@ #!/bin/bash -NEW_VERSION="4.2.5" +NEW_VERSION="4.2.6rc1" TESTIT_API_CLIENT_VERSION="7.5.10" echo "Updating all adapters to version: $NEW_VERSION"