Improve unit test times with isolated parallel workers (#225)

* Parallelize unit tests with isolated workspaces

* [Automated] Updated coverage badge

* Add language hint to test command

* Shard E2E tests with configurable count

* [Automated] Updated coverage badge

* Expand E2E sharding to ten workers

Add a shared ten-shard assignment while preserving the existing three-shard fallback.

Run ten E2E shards per platform and fail explicitly when the public E2E collection drifts from the checked-in assignments.

* Run representative E2E coverage on Windows and macOS

---------

Co-authored-by: GitHub Actions <actions@github.com>

Author: rolandocortez

.github/workflows/end2end_tests.yaml

@@ -15,13 +15,61 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_tests:
-    name: E2E Tests on ${{ matrix.os }} / Python ${{ matrix.version }}
+  ubuntu_full:
+    name: E2E full on ubuntu-latest / Python ${{ matrix.version }} / shard ${{ matrix.shard }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
         version: ['3.10']
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+        ref: ${{ github.event.pull_request.head.sha }}
+        submodules: recursive
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.version }}
+        cache: pip
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        CONSTRAINTS_FILE="$GITHUB_WORKSPACE/constraints.txt"
+        echo "setuptools<82" > "$CONSTRAINTS_FILE"
+        export PIP_CONSTRAINT="$CONSTRAINTS_FILE"
+        pip install -e .[dev]
+
+    - name: Run full public tests
+      run: pytest tests/test_end2end.py --e2e-suite full --e2e-shard-index ${{ matrix.shard }} --e2e-shard-count 10 --download-weights -s -v
+
+    - name: Authenticate to Google Cloud
+      if: matrix.shard == 0 && github.event.pull_request.head.repo.full_name == github.repository
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+        create_credentials_file: true
+        export_environment_variables: true
+
+    - name: Run private model tests
+      if: matrix.shard == 0 && github.event.pull_request.head.repo.full_name == github.repository
+      run: pytest tests/test_end2end.py::test_private_model_conversion --test-private --delete-weights-now -s -v
+
+  cross_platform_representative:
+    name: E2E representative on ${{ matrix.os }} / Python ${{ matrix.version }} / shard ${{ matrix.shard }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macOS-latest]
+        version: ['3.10']
+        shard: [0, 1]
 
     runs-on: ${{ matrix.os }}
 
@@ -31,7 +79,7 @@ jobs:
       with:
         repository: ${{ github.event.pull_request.head.repo.full_name }}
         ref: ${{ github.event.pull_request.head.sha }}
-        submodules: recursive  # Ensures submodules are cloned
+        submodules: recursive
 
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -47,16 +95,17 @@ jobs:
         export PIP_CONSTRAINT="$CONSTRAINTS_FILE"
         pip install -e .[dev]
 
-    - name: Run tests
-      run: pytest tests/test_end2end.py --download-weights --delete-weights-now -s -v
+    - name: Run representative public tests
+      run: pytest tests/test_end2end.py --e2e-suite representative --e2e-shard-index ${{ matrix.shard }} --e2e-shard-count 2 --download-weights -s -v
 
     - name: Authenticate to Google Cloud
+      if: matrix.shard == 0 && github.event.pull_request.head.repo.full_name == github.repository
       uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
         create_credentials_file: true
         export_environment_variables: true
 
     - name: Run private model tests
-      if: github.event.pull_request.head.repo.full_name == github.repository
+      if: matrix.shard == 0 && github.event.pull_request.head.repo.full_name == github.repository
       run: pytest tests/test_end2end.py::test_private_model_conversion --test-private --delete-weights-now -s -v

.github/workflows/unittests.yaml

@@ -80,19 +80,19 @@ jobs:
     - name: Run tests with coverage [Ubuntu]
       if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10'
       working-directory: ${{ inputs.tools_ref != '' && 'tools' || '' }}
-      env:
-        COVERAGE_PROCESS_START: .coveragerc
       run: |
+        export COVERAGE_PROCESS_START="$PWD/.coveragerc"
+        export COVERAGE_FILE="$PWD/.coverage"
         coverage erase
-        coverage run -m pytest tests/test_unittests.py --download-weights --delete-weights-now -s -v --junit-xml pytest.xml
+        coverage run -m pytest tests/test_unittests.py -n 2 --dist load --download-weights -s -v --junit-xml pytest.xml
         coverage combine
         coverage xml -o coverage.xml -i # Recent changes to coverage 7.13 make it stricter where it was previously passing https://coverage.readthedocs.io/en/7.13.0/messages.html#error-no-source
 
 
     - name: Run tests [Windows, macOS]
       if: matrix.os != 'ubuntu-latest' || matrix.version != '3.10'
       working-directory: ${{ inputs.tools_ref != '' && 'tools' || '' }}
-      run: pytest tests/test_unittests.py --download-weights --delete-weights-now -s -v --junit-xml pytest.xml
+      run: pytest tests/test_unittests.py -n 2 --dist load --download-weights -s -v --junit-xml pytest.xml
 
     - name: Generate coverage badge [Ubuntu]
       if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' && inputs.tools_ref == '' && inputs.ml_ref == ''

media/coverage_badge.svg

@@ -1,4 +1,5 @@
 pytest
+pytest-xdist>=3.6.1
 pytest-subtests
 pre-commit>=4.1.0
 pytest-cov>=4.1.0

requirements-dev.txt

@@ -13,13 +13,13 @@ Unit tests download the preset weights from original repositories and check if t
                     If set then test only that specific yolo version
 --test-case=TEST_CASE
                     If set then test only that specific test case
---delete-weights-now  Clean weights after every test to save space - but longer test time.
+--delete-weights-now  Clean weights after every test to save disk space, at the cost of longer runtime.
 ```
 
 Here is an example of the call to run:
 
-```
-pytest --download-weights --log-cli-level=INFO --log-file=out.log --log-file-level=DEBUG .
+```bash
+pytest tests/test_unittests.py -n 2 --dist load --download-weights --log-cli-level=INFO --log-file=out.log --log-file-level=DEBUG
 ```
 
-This will run the full test suite on all the supported models and store the DEBUG logs into the out.log file.
+This will run the full unit test suite on all supported models and store the DEBUG logs in the out.log file.

tests/README.md

@@ -1,9 +1,12 @@
 import logging
 import os
 import shutil
+from pathlib import Path
 
 import pytest
 
+pytest_plugins = ["e2e_shards"]
+
 logger = logging.getLogger()
 
 os.environ.setdefault("LUXONIS_TELEMETRY_ENABLED", "false")
@@ -68,21 +71,42 @@ def test_config(pytestconfig):
     }
 
 
+@pytest.fixture(scope="session")
+def test_workspace(tmp_path_factory, request) -> Path:
+    """Create one isolated conversion workspace per pytest worker."""
+    worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master")
+    workspace = tmp_path_factory.mktemp(f"tools-tests-{worker_id}")
+    (workspace / "weights").mkdir(exist_ok=True)
+    return workspace
+
+
+def _artifact_path(request: pytest.FixtureRequest, name: str) -> Path:
+    """Resolve cleanup paths without changing E2E behavior."""
+    if "test_workspace" in request.fixturenames:
+        workspace = request.getfixturevalue("test_workspace")
+        return Path(workspace) / name
+
+    return Path(name)
+
+
 @pytest.fixture(scope="function", autouse=True)
-def cleanup_output_after_tests(test_config):
+def cleanup_output_after_tests(test_config, request):
     yield  # Tests run here
     if test_config["delete_output"]:
-        folder_to_delete = "shared_with_container"
-        if os.path.exists(folder_to_delete):
+        folder_to_delete = _artifact_path(
+            request,
+            "shared_with_container",
+        )
+        if folder_to_delete.exists():
             shutil.rmtree(folder_to_delete)
             logger.info(f"Removed test artifacts from {folder_to_delete}")
 
 
 @pytest.fixture(scope="function", autouse=True)
-def cleanup_weights_after_tests(test_config):
+def cleanup_weights_after_tests(test_config, request):
     yield  # Tests run here
     if test_config["delete_weights_now"]:
-        folder_to_delete = "weights"
-        if os.path.exists(folder_to_delete):
+        folder_to_delete = _artifact_path(request, "weights")
+        if folder_to_delete.exists():
             shutil.rmtree(folder_to_delete)
             logger.info(f"Removed test artifacts from {folder_to_delete}")