Skip to content

Commit f77bed7

Browse files
rolandocortezrolandocortez
committed
Shard E2E tests with configurable count
1 parent caf29f5 commit f77bed7

4 files changed

Lines changed: 251 additions & 3 deletions

File tree

‎.github/workflows/end2end_tests.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@ concurrency:
1616

1717
jobs:
1818
run_tests:
19-
name: E2E Tests on ${{ matrix.os }} / Python ${{ matrix.version }}
19+
name: E2E Tests on ${{ matrix.os }} / Python ${{ matrix.version }} / shard ${{ matrix.shard }}
2020
strategy:
2121
fail-fast: false
2222
matrix:
2323
os: [ubuntu-latest, windows-latest, macOS-latest]
2424
version: ['3.10']
25+
shard: [0, 1, 2]
2526

2627
runs-on: ${{ matrix.os }}
2728

@@ -48,15 +49,16 @@ jobs:
4849
pip install -e .[dev]
4950
5051
- name: Run tests
51-
run: pytest tests/test_end2end.py --download-weights --delete-weights-now -s -v
52+
run: pytest tests/test_end2end.py --e2e-shard-index ${{ matrix.shard }} --e2e-shard-count 3 --download-weights -s -v
5253

5354
- name: Authenticate to Google Cloud
55+
if: matrix.shard == 0 && github.event.pull_request.head.repo.full_name == github.repository
5456
uses: google-github-actions/auth@v2
5557
with:
5658
credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
5759
create_credentials_file: true
5860
export_environment_variables: true
5961

6062
- name: Run private model tests
61-
if: github.event.pull_request.head.repo.full_name == github.repository
63+
if: matrix.shard == 0 && github.event.pull_request.head.repo.full_name == github.repository
6264
run: pytest tests/test_end2end.py::test_private_model_conversion --test-private --delete-weights-now -s -v

‎tests/conftest.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
import logging
2+
import os
23
import shutil
34
from pathlib import Path
45

56
import pytest
67

8+
pytest_plugins = ["e2e_shards"]
9+
710
logger = logging.getLogger()
811

912
os.environ.setdefault("LUXONIS_TELEMETRY_ENABLED", "false")

‎tests/e2e_shards.py‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
# Pytest plugin for deterministic E2E test sharding.
2+
#
3+
# The default assignment is generated from the timing profile captured during
4+
# Task 10. The workflow passes both the shard index and shard count explicitly
5+
# so the CI topology is easy to review.
6+
#
7+
# To try another shard count later, regenerate/add another entry in
8+
# E2E_SHARD_ASSIGNMENTS_BY_COUNT and keep the collection tests green.
9+
10+
from __future__ import annotations
11+
12+
import pytest
13+
14+
DEFAULT_E2E_SHARD_COUNT = 3
15+
16+
E2E_SHARD_ASSIGNMENTS_BY_COUNT: dict[int, tuple[frozenset[str], ...]] = {
17+
3: (
18+
frozenset(
19+
{
20+
"tests/test_end2end.py::test_cli_conversion[yolo26m]",
21+
"tests/test_end2end.py::test_cli_conversion[yolo26n-pose]",
22+
"tests/test_end2end.py::test_cli_conversion[yolo26n-seg]",
23+
"tests/test_end2end.py::test_cli_conversion[yolo26x]",
24+
"tests/test_end2end.py::test_cli_conversion[yoloe-v8l-seg]",
25+
"tests/test_end2end.py::test_cli_conversion[yolov10s]",
26+
"tests/test_end2end.py::test_cli_conversion[yolov11n]",
27+
"tests/test_end2end.py::test_cli_conversion[yolov11s]",
28+
"tests/test_end2end.py::test_cli_conversion[yolov12s]",
29+
"tests/test_end2end.py::test_cli_conversion[yolov12x]",
30+
"tests/test_end2end.py::test_cli_conversion[yolov5l6]",
31+
"tests/test_end2end.py::test_cli_conversion[yolov5lu]",
32+
"tests/test_end2end.py::test_cli_conversion[yolov5m6]",
33+
"tests/test_end2end.py::test_cli_conversion[yolov5m]",
34+
"tests/test_end2end.py::test_cli_conversion[yolov5mu]",
35+
"tests/test_end2end.py::test_cli_conversion[yolov6mr21]",
36+
"tests/test_end2end.py::test_cli_conversion[yolov6mr3]",
37+
"tests/test_end2end.py::test_cli_conversion[yolov6nr1]",
38+
"tests/test_end2end.py::test_cli_conversion[yolov6nr2]",
39+
"tests/test_end2end.py::test_cli_conversion[yolov6sr21]",
40+
"tests/test_end2end.py::test_cli_conversion[yolov6sr3]",
41+
"tests/test_end2end.py::test_cli_conversion[yolov6sr4]",
42+
"tests/test_end2end.py::test_cli_conversion[yolov6tr1]",
43+
"tests/test_end2end.py::test_cli_conversion[yolov8n-cls]",
44+
"tests/test_end2end.py::test_cli_conversion[yolov8n-obb]",
45+
"tests/test_end2end.py::test_cli_conversion[yolov8n-pose]",
46+
"tests/test_end2end.py::test_cli_conversion[yolov8s]",
47+
"tests/test_end2end.py::test_cli_conversion[yolov9c]",
48+
"tests/test_end2end.py::test_cli_conversion[yolov9e]",
49+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolo26n-pose]",
50+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolo26n-seg]",
51+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov11n]",
52+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov8n-pose]",
53+
}
54+
),
55+
frozenset(
56+
{
57+
"tests/test_end2end.py::test_cli_conversion[yolo26l]",
58+
"tests/test_end2end.py::test_cli_conversion[yolo26n-sem]",
59+
"tests/test_end2end.py::test_cli_conversion[yolo26n]",
60+
"tests/test_end2end.py::test_cli_conversion[yolo26s]",
61+
"tests/test_end2end.py::test_cli_conversion[yoloe-11s-seg]",
62+
"tests/test_end2end.py::test_cli_conversion[yoloe-v8m-seg]",
63+
"tests/test_end2end.py::test_cli_conversion[yoloe-v8s-seg]",
64+
"tests/test_end2end.py::test_cli_conversion[yolov10b]",
65+
"tests/test_end2end.py::test_cli_conversion[yolov10m]",
66+
"tests/test_end2end.py::test_cli_conversion[yolov10n]",
67+
"tests/test_end2end.py::test_cli_conversion[yolov11m]",
68+
"tests/test_end2end.py::test_cli_conversion[yolov11n-seg]",
69+
"tests/test_end2end.py::test_cli_conversion[yolov12l]",
70+
"tests/test_end2end.py::test_cli_conversion[yolov26_nms]",
71+
"tests/test_end2end.py::test_cli_conversion[yolov5m6u]",
72+
"tests/test_end2end.py::test_cli_conversion[yolov5n6]",
73+
"tests/test_end2end.py::test_cli_conversion[yolov5n]",
74+
"tests/test_end2end.py::test_cli_conversion[yolov5nu]",
75+
"tests/test_end2end.py::test_cli_conversion[yolov5s6u]",
76+
"tests/test_end2end.py::test_cli_conversion[yolov5x]",
77+
"tests/test_end2end.py::test_cli_conversion[yolov6lr21]",
78+
"tests/test_end2end.py::test_cli_conversion[yolov6lr2]",
79+
"tests/test_end2end.py::test_cli_conversion[yolov6lr4]",
80+
"tests/test_end2end.py::test_cli_conversion[yolov6mr2]",
81+
"tests/test_end2end.py::test_cli_conversion[yolov6nr3]",
82+
"tests/test_end2end.py::test_cli_conversion[yolov6nr4]",
83+
"tests/test_end2end.py::test_cli_conversion[yolov7]",
84+
"tests/test_end2end.py::test_cli_conversion[yolov8m]",
85+
"tests/test_end2end.py::test_cli_conversion[yolov8n]",
86+
"tests/test_end2end.py::test_cli_conversion[yolov8x]",
87+
"tests/test_end2end.py::test_cli_conversion[yolov9s]",
88+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolo26n]",
89+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov11n-seg]",
90+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov8n]",
91+
"tests/test_end2end.py::test_yolo26_semseg_nnarchive_head",
92+
}
93+
),
94+
frozenset(
95+
{
96+
"tests/test_end2end.py::test_cli_conversion[yoloe-11l-seg]",
97+
"tests/test_end2end.py::test_cli_conversion[yoloe-11m-seg]",
98+
"tests/test_end2end.py::test_cli_conversion[yolov10l]",
99+
"tests/test_end2end.py::test_cli_conversion[yolov10x]",
100+
"tests/test_end2end.py::test_cli_conversion[yolov11l]",
101+
"tests/test_end2end.py::test_cli_conversion[yolov11n-cls]",
102+
"tests/test_end2end.py::test_cli_conversion[yolov11n-obb]",
103+
"tests/test_end2end.py::test_cli_conversion[yolov11n-pose]",
104+
"tests/test_end2end.py::test_cli_conversion[yolov11x]",
105+
"tests/test_end2end.py::test_cli_conversion[yolov12m]",
106+
"tests/test_end2end.py::test_cli_conversion[yolov12n]",
107+
"tests/test_end2end.py::test_cli_conversion[yolov5l6u]",
108+
"tests/test_end2end.py::test_cli_conversion[yolov5l]",
109+
"tests/test_end2end.py::test_cli_conversion[yolov5n6u]",
110+
"tests/test_end2end.py::test_cli_conversion[yolov5s6]",
111+
"tests/test_end2end.py::test_cli_conversion[yolov5s]",
112+
"tests/test_end2end.py::test_cli_conversion[yolov5su]",
113+
"tests/test_end2end.py::test_cli_conversion[yolov6lr3]",
114+
"tests/test_end2end.py::test_cli_conversion[yolov6mr4]",
115+
"tests/test_end2end.py::test_cli_conversion[yolov6nr21]",
116+
"tests/test_end2end.py::test_cli_conversion[yolov6sr1]",
117+
"tests/test_end2end.py::test_cli_conversion[yolov6sr2]",
118+
"tests/test_end2end.py::test_cli_conversion[yolov6tr2]",
119+
"tests/test_end2end.py::test_cli_conversion[yolov7t]",
120+
"tests/test_end2end.py::test_cli_conversion[yolov7x]",
121+
"tests/test_end2end.py::test_cli_conversion[yolov8l]",
122+
"tests/test_end2end.py::test_cli_conversion[yolov8n-seg]",
123+
"tests/test_end2end.py::test_cli_conversion[yolov9m]",
124+
"tests/test_end2end.py::test_cli_conversion[yolov9t]",
125+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov11n-pose]",
126+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov12n]",
127+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov8n-seg]",
128+
"tests/test_end2end.py::test_n_variant_nnarchive_outputs[yolov9t]",
129+
}
130+
),
131+
),
132+
}
133+
134+
135+
def pytest_addoption(parser: pytest.Parser) -> None:
136+
group = parser.getgroup("e2e-shards")
137+
group.addoption(
138+
"--e2e-shard-index",
139+
type=int,
140+
default=None,
141+
help=(
142+
"Run only one deterministic public E2E shard. "
143+
"Use together with --e2e-shard-count."
144+
),
145+
)
146+
group.addoption(
147+
"--e2e-shard-count",
148+
type=int,
149+
default=DEFAULT_E2E_SHARD_COUNT,
150+
help=(
151+
"Total number of public E2E shards. "
152+
"Currently supported counts are listed in "
153+
"E2E_SHARD_ASSIGNMENTS_BY_COUNT."
154+
),
155+
)
156+
157+
158+
def supported_e2e_shard_counts() -> tuple[int, ...]:
159+
return tuple(sorted(E2E_SHARD_ASSIGNMENTS_BY_COUNT))
160+
161+
162+
def get_e2e_shard_assignment(shard_count: int) -> tuple[frozenset[str], ...]:
163+
try:
164+
return E2E_SHARD_ASSIGNMENTS_BY_COUNT[shard_count]
165+
except KeyError as exc:
166+
supported = ", ".join(str(count) for count in supported_e2e_shard_counts())
167+
raise pytest.UsageError(
168+
f"No E2E shard assignment exists for count={shard_count}. "
169+
f"Supported counts: {supported}. "
170+
"Regenerate the shard assignment from the timing profile before "
171+
"using this count."
172+
) from exc
173+
174+
175+
def get_e2e_shard_items(shard_index: int, shard_count: int) -> frozenset[str]:
176+
assignment = get_e2e_shard_assignment(shard_count)
177+
if shard_index < 0 or shard_index >= shard_count:
178+
raise pytest.UsageError(
179+
f"Invalid E2E shard index {shard_index} for count={shard_count}."
180+
)
181+
if len(assignment) != shard_count:
182+
raise pytest.UsageError(
183+
f"E2E shard assignment for count={shard_count} has "
184+
f"{len(assignment)} shards."
185+
)
186+
187+
return assignment[shard_index]
188+
189+
190+
def pytest_collection_modifyitems(
191+
config: pytest.Config,
192+
items: list[pytest.Item],
193+
) -> None:
194+
shard_index = config.getoption("e2e_shard_index")
195+
if shard_index is None:
196+
return
197+
198+
shard_count = config.getoption("e2e_shard_count")
199+
selected_nodeids = get_e2e_shard_items(shard_index, shard_count)
200+
201+
selected_items: list[pytest.Item] = []
202+
deselected_items: list[pytest.Item] = []
203+
204+
for item in items:
205+
if item.nodeid in selected_nodeids:
206+
selected_items.append(item)
207+
else:
208+
deselected_items.append(item)
209+
210+
if deselected_items:
211+
config.hook.pytest_deselected(items=deselected_items)
212+
213+
items[:] = selected_items

‎tests/test_unittests.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,3 +251,33 @@ def test_explicit_version_detection(test_workspace: Path):
251251
result = _run_tools(command, test_workspace)
252252
if result.returncode != 0:
253253
pytest.fail(f"Exit code: {result.returncode}, Output: {result.stdout}")
254+
255+
256+
def test_e2e_shard_count_three_manifest_matches_profile():
257+
import importlib
258+
259+
e2e_shards = importlib.import_module("e2e_shards")
260+
261+
assignment = e2e_shards.get_e2e_shard_assignment(3)
262+
counts = tuple(len(shard) for shard in assignment)
263+
public_union = set().union(*assignment)
264+
public_overlap = (
265+
(assignment[0] & assignment[1])
266+
| (assignment[0] & assignment[2])
267+
| (assignment[1] & assignment[2])
268+
)
269+
270+
assert e2e_shards.DEFAULT_E2E_SHARD_COUNT == 3
271+
assert e2e_shards.supported_e2e_shard_counts() == (3,)
272+
assert counts == (33, 35, 33)
273+
assert len(public_union) == 101
274+
assert public_overlap == set()
275+
276+
277+
def test_e2e_shard_count_requires_checked_in_assignment():
278+
import importlib
279+
280+
e2e_shards = importlib.import_module("e2e_shards")
281+
282+
with pytest.raises(pytest.UsageError, match="Regenerate the shard assignment"):
283+
e2e_shards.get_e2e_shard_assignment(2)

0 commit comments

Comments
 (0)