diff --git a/testit-adapter-behave/setup.py b/testit-adapter-behave/setup.py index a0371f3..03cb2b3 100644 --- a/testit-adapter-behave/setup.py +++ b/testit-adapter-behave/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.6rc1" +VERSION = "4.2.6rc2" setup( name='testit-adapter-behave', diff --git a/testit-adapter-nose/setup.py b/testit-adapter-nose/setup.py index 1133032..9916b71 100644 --- a/testit-adapter-nose/setup.py +++ b/testit-adapter-nose/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = "4.2.6rc1" +VERSION = "4.2.6rc2" setup( name='testit-adapter-nose', diff --git a/testit-adapter-pytest/setup.py b/testit-adapter-pytest/setup.py index 976ed17..3e88aa8 100644 --- a/testit-adapter-pytest/setup.py +++ b/testit-adapter-pytest/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.6rc1" +VERSION = "4.2.6rc2" setup( name='testit-adapter-pytest', diff --git a/testit-adapter-robotframework/setup.py b/testit-adapter-robotframework/setup.py index 5b84b05..e0e4a81 100644 --- a/testit-adapter-robotframework/setup.py +++ b/testit-adapter-robotframework/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.6rc1" +VERSION = "4.2.6rc2" setup( name='testit-adapter-robotframework', diff --git a/testit-python-commons/realtime-import-spec.md b/testit-python-commons/realtime-import-spec.md index 2651e46..e16963c 100644 --- a/testit-python-commons/realtime-import-spec.md +++ b/testit-python-commons/realtime-import-spec.md @@ -27,8 +27,15 @@ When `importRealtime=false`, all test results are sent once after the session fi - After the run completes, only **top-level** steps remain on the test result. - Reproducible with `importRealtime=true` (default since adapter 4.x). - Not reproducible with `importRealtime=false`. +- **With Sync Storage enabled:** only the **first** test in the run is affected (the one held as `InProgress`). +- Tests that are not the first one keep nested steps. +- Without Sync Storage the issue does not reproduce (for the adapter-only PUT bug below). -### Root Cause +There are **two independent causes**; both can flatten nested steps on the test result. + +--- + +### Cause A (adapter): final session PUT overwrote `step_results` At session finish, `update_test_results` used to: @@ -76,7 +83,89 @@ self.__test_results_api.api_v2_test_results_id_put(...) The `step_results` field is omitted, so the nested test steps written during real-time import are not overwritten. -### Affected Code +--- + +### Cause B (Sync Storage + adapter): first test finalized without step tree + +When Sync Storage is running and the worker is **master**, the **first** completed test takes a special path in `AdapterManager.__write_test_realtime` → `on_master_no_already_in_progress`: + +1. Adapter sends **`TestResultCutApiModel`** to Sync Storage (`POST /in_progress_test_result`). + - Payload contains only: `projectId`, `autoTestExternalId`, `statusCode`, `statusType`, `startedOn`. + - **No `step_results`, no attachments, no nested data.** +2. Adapter sets outcome to `InProgress` and posts the test result to Test IT via `_write_test_realtime_internal`. + - This POST **does** include the full nested `step_results` tree. +3. Method returns `True` — the test is **not** written again with the real outcome through the adapter. + +All subsequent tests skip step 1–3 (Sync Storage `is_already_in_progress` flag is set) and go through normal `_write_test_realtime_internal` with the real outcome and full steps. + +At the end of the run, CI (or tooling) typically calls: + +```bash +curl http://127.0.0.1:49152/wait-completion?testRunId=... +``` + +Sync Storage then finalizes the held in-progress result and pushes the **real** status to Test IT. It only ever received the **cut** model, so the final TMS update cannot restore nested `step_results`. That explains the observed pattern: + +| Scenario | Nested steps on test result | +|----------|----------------------------| +| First test + Sync Storage + `importRealtime=true` | Visible during run, **lost after** `wait-completion` | +| Later tests in the same run | OK (normal real-time path) | +| Without Sync Storage | OK (no finalize overwrite) | +| Single test only | Always the first → always broken with Sync Storage | + +```mermaid +sequenceDiagram + participant Pytest + participant Adapter + participant SyncStorage + participant TMS + + Pytest->>Adapter: test 1 finished + Adapter->>SyncStorage: POST cut model (no steps) + Adapter->>TMS: POST InProgress + full step tree + Note over Adapter: return early, no real-outcome write + + Pytest->>Adapter: test 2..N finished + Adapter->>TMS: POST real outcome + full step tree + + Pytest->>Adapter: sessionfinish / write_tests + Adapter->>TMS: PUT setup/teardown only + + Note over CI: curl wait-completion + SyncStorage->>TMS: finalize test 1 (cut data, no nested steps) + Note over TMS: nested steps on test 1 lost +``` + +#### Recommended fixes (Cause B) + +**Option 1 — adapter (preferred, no Sync Storage release required):** + +After Sync Storage completion, re-send the full test result for the held test: + +1. In `on_master_no_already_in_progress`, **store a copy** of `TestResult` with the real outcome and full `step_results` before mutating outcome to `InProgress`. +2. In `write_tests` (or after `wait_completion`): + - Call `wait_completion` on Sync Storage if the adapter owns the lifecycle, **or** document that finalize must run after external `wait-completion`. + - `PUT` the stored result to Test IT by `test_result_id` from `__test_result_map`, including recursive `step_results` via `step_results_to_auto_test_step_result_update_request`. + +**Option 2 — Sync Storage:** + +On finalize, either do not send `step_results` in the TMS PUT (update status/duration only), or accept and persist the full `AutoTestResultsForTestRunModel` (as in the Java spec) instead of `TestResultCutApiModel`. + +**Option 3 — adapter (coordination-only):** + +Send cut data to Sync Storage for coordination but still write the real outcome to TMS immediately (do not skip `_write_test_realtime_internal` with real data). Requires validation that parallel worker coordination still works. + +#### Affected Code (Cause B) + +| File | Responsibility | +|------|----------------| +| `services/adapter_manager.py` | `on_master_no_already_in_progress`, `__write_test_realtime` | +| `services/sync_storage/sync_storage_runner.py` | `send_in_progress_test_result`, `test_result_to_test_result_cut_api_model` | +| Sync Storage binary | `/wait-completion` → TMS finalize | + +--- + +### Affected Code (Cause A) | File | Responsibility | |------|----------------| diff --git a/testit-python-commons/setup.py b/testit-python-commons/setup.py index abd0ce2..1038e7f 100644 --- a/testit-python-commons/setup.py +++ b/testit-python-commons/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.2.6rc1" +VERSION = "4.2.6rc2" setup( name='testit-python-commons', diff --git a/testit-python-commons/src/testit_python_commons/services/sync_storage/sync_storage_runner.py b/testit-python-commons/src/testit_python_commons/services/sync_storage/sync_storage_runner.py index 1393b39..35fdf7f 100644 --- a/testit-python-commons/src/testit_python_commons/services/sync_storage/sync_storage_runner.py +++ b/testit-python-commons/src/testit_python_commons/services/sync_storage/sync_storage_runner.py @@ -40,7 +40,7 @@ class SyncStorageRunner: across multiple workers. """ - SYNC_STORAGE_VERSION = "v0.3.5-tms-5.7" + SYNC_STORAGE_VERSION = "v0.3.6-tms-5.7" SYNC_STORAGE_REPO_URL = ( "https://github.com/testit-tms/sync-storage-public/releases/download/" ) diff --git a/update_versions.sh b/update_versions.sh index 4025e9b..4b17ceb 100644 --- a/update_versions.sh +++ b/update_versions.sh @@ -1,6 +1,6 @@ #!/bin/bash -NEW_VERSION="4.2.6rc1" +NEW_VERSION="4.2.6rc2" TESTIT_API_CLIENT_VERSION="7.5.10" echo "Updating all adapters to version: $NEW_VERSION"