perf(inference): batch `predict()`'s per-image GPU->CPU transfers into one sync #3731

Workflow file for this run

.github/workflows/ci-integrations.yml at 8e85e09

	name: Smoke Tests

	on:
	push:
	branches: ["main", "release/*", "develop"]
	pull_request:
	branches: ["main", "release/*", "develop"]

	permissions:
	contents: read

	env:
	# UV_TORCH_BACKEND only works with 'uv pip' commands, not 'uv sync'.
	# Used by 'uv pip install torch torchvision' step to install CPU-only PyTorch.
	UV_TORCH_BACKEND: "cpu"

	jobs:
	try-all-models:
	runs-on: ${{ matrix.os }}
	concurrency:
	group: model-smoke-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}
	strategy:
	fail-fast: false
	matrix:
	os: ["ubuntu-latest", "macos-latest", "windows-latest"]
	python-version: ["3.10", "3.13"]
	timeout-minutes: 15
	steps:
	- name: 📥 Checkout the repository
	uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

	- name: 🐍 Install uv and set Python version ${{ matrix.python-version }}
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
	with:
	version: "0.9.26"
	python-version: ${{ matrix.python-version }}
	activate-environment: true

	- name: 🚀 Install Packages (plus extras)
	timeout-minutes: 5
	# Install PyTorch CPU-only first (UV_TORCH_BACKEND=cpu works with 'uv pip')
	run: uv pip install -e .[plus]

	- name: 🔎 Smoke-test model instantiation, downloads, and inference
	run: python tests/run_smoke_all_models.py

	- name: Minimize uv cache
	run: uv cache prune --ci

	executorch-parity:
	name: ExecuTorch parity
	# The real numerical-parity test exports an RFDETRNano to .pte and asserts the
	# XNNPACK-delegated runtime matches eager PyTorch. XNNPACK is a plain-CPU backend,
	# so no special hardware/SDK is required — ubuntu-latest is sufficient.
	runs-on: ${{ matrix.os }}
	concurrency:
	group: pytest-executorch-${{ matrix.python-version }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}
	timeout-minutes: 25
	strategy:
	fail-fast: false
	matrix:
	os: ["ubuntu-latest"]
	# executorch 1.3.1 ships manylinux_2_28 x86_64 wheels for cp310–cp313 and
	# requires_python is >=3.10,<3.14. 3.10 (floor) and 3.13 (ceiling) bracket
	# the supported range; widen the matrix if the [executorch] pin changes.
	python-version: ["3.10", "3.13"]
	steps:
	- name: 📥 Checkout the repository
	uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

	- name: 🐍 Install uv and set Python version ${{ matrix.python-version }}
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
	with:
	version: "0.9.26"
	python-version: ${{ matrix.python-version }}
	activate-environment: true

	- name: 🚀 Install Packages (executorch extra)
	timeout-minutes: 8
	# The end-to-end parity test builds and exports the full RFDETRNano stack, so
	# pull the same extras as the CPU test job plus the [executorch] backend.
	# --group ci-executorch-pin locks torch to a version whose C10 ABI matches
	# executorch 1.3.1's prebuilt wheel (see pyproject.toml for the empirically
	# validated range and removal condition).
	run: uv pip install -e ".[executorch,train,augment,cli,visual]" --group tests --group ci-executorch-pin

	- name: 🔎 Verify executorch runtime imports
	# Guard against a silently-skipped parity test: the test class is gated by a
	# skipif on executorch availability, so a broken/missing wheel would skip it
	# (false green). Fail here (red) instead if the runtime cannot be imported.
	# require_runtime=True also surfaces an actionable ABI-compatibility message
	# (rather than a bare "undefined symbol") if the pin above ever drifts stale.
	# shell: python — activate-environment: true (setup-uv, above) puts the venv's
	# python on PATH, so this runs the script through it directly (no uv run needed).
	run: \|
	from rfdetr.export._executorch import _IS_EXECUTORCH_AVAILABLE
	from rfdetr.export._executorch.converter import _check_executorch_available

	_check_executorch_available(require_runtime=True)
	assert _IS_EXECUTORCH_AVAILABLE, "executorch installed but not detected as available"
	print("executorch runtime import OK")
	shell: python

	- name: 🧪 Run ExecuTorch parity tests
	# executorch export is heavy; run single-worker as advised for this suite.
	run: \|
	uv run --no-sync pytest tests/export/test_executorch_export.py \
	-m e2e_executorch -n 1 \
	--timeout=600 \
	--durations=20

	- name: Minimize uv cache
	continue-on-error: true
	run: uv cache prune --ci

	coreml-parity:
	name: CoreML parity
	# coremltools + the CoreML runtime are macOS-only, so unlike executorch-parity (portable
	# XNNPACK, runs on ubuntu-latest) this job requires macos-latest.
	runs-on: macos-latest
	concurrency:
	group: pytest-coreml-${{ matrix.python-version }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}
	timeout-minutes: 25
	strategy:
	fail-fast: false
	matrix:
	# 3.11 not 3.10 for this job specifically (bumped alongside the coreml extra's torch<2.12
	# pin, see pyproject.toml) — other CI jobs keep testing 3.10 as the project floor.
	python-version: ["3.11", "3.13"]
	steps:
	- name: 📥 Checkout the repository
	uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

	- name: 🐍 Install uv and set Python version ${{ matrix.python-version }}
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
	with:
	version: "0.9.26"
	python-version: ${{ matrix.python-version }}
	activate-environment: true

	- name: 🚀 Install Packages (coreml extra)
	timeout-minutes: 8
	run: uv pip install -e ".[coreml,train,augment,cli,visual]" --group tests

	- name: 🔎 Verify coremltools import
	# Guard against a silently-skipped parity test: TestCoreMLEndToEnd is gated by a
	# skipif on coremltools availability, so a broken/missing wheel would skip it (false
	# green) instead of failing red.
	run: \|
	from rfdetr.export._coreml import _IS_COREMLTOOLS_AVAILABLE

	assert _IS_COREMLTOOLS_AVAILABLE, "coremltools installed but not detected as available"
	print("coremltools import OK")
	shell: python

	- name: 🧪 Run CoreML end-to-end parity tests
	# Real ct.convert + mlmodel.predict + download_assets is heavy/network; single-worker,
	# opt-in marker (see pyproject.toml `e2e_coreml`) — deliberately excluded from the
	# ci-tests-cpu.yml default filter (`and not e2e_coreml`) so it only runs here.
	run: \|
	uv run --no-sync pytest tests/export/test_coreml_export.py \
	-m e2e_coreml -n 1 \
	--timeout=600 \
	--durations=20

	- name: Minimize uv cache
	continue-on-error: true
	run: uv cache prune --ci

	tensorrt-parity:
	name: TensorRT parity
	# TensorRT engine build + runtime inference need a real GPU + CUDA driver, so unlike the CPU
	# integration jobs this runs on the self-hosted GPU runner (same as ci-tests-gpu.yml).
	runs-on: Roboflow-GPU-VM-Runner
	timeout-minutes: 25
	concurrency:
	group: pytest-tensorrt-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}
	env:
	# UV_TORCH_BACKEND=auto (GPU) works only with uv pip, not uv sync; overrides the workflow-level
	# "cpu" default which is wrong for this GPU job.
	UV_TORCH_BACKEND: "auto"
	steps:
	- name: 🖥️ Print GPU information
	run: nvidia-smi
	- name: 📥 Checkout the repository
	uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

	- name: 🐍 Install uv and set Python version
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
	with:
	version: "0.9.26"
	python-version: "3.12"
	activate-environment: true

	- name: 🚀 Install Packages (tensorrt + onnx extras)
	timeout-minutes: 8
	# [tensorrt] provides tensorrt/polygraphy for the engine build; [onnx] provides the ONNX
	# export toolchain (the e2e exports RFDETRNano to ONNX first). --group ci-gpu-pin matches the GPU
	# runner's CUDA driver (torch<2.11), same as ci-tests-gpu.yml.
	run: uv pip install -e ".[tensorrt,onnx,train,augment,cli,visual]" --group tests --group ci-gpu-pin

	- name: 🔎 Verify tensorrt import
	# Guard against a silently-skipped parity test: TestTensorRTEndToEnd is gated by a skipif on
	# tensorrt availability, so a broken/missing wheel would skip it (false green) instead of failing red.
	run: \|
	from rfdetr.export._tensorrt import _IS_TENSORRT_AVAILABLE

	assert _IS_TENSORRT_AVAILABLE, "tensorrt/polygraphy installed but not detected as available"
	print("tensorrt import OK")
	shell: python

	- name: 🔗 Expose CUDA runtime (libcudart.so) to the loader
	# The engine build succeeds, but polygraphy's TrtRunner separately dlopen's the unversioned
	# `libcudart.so` for CUDA stream management. The GPU wheels ship it only as `libcudart.so.12`
	# (e.g. under torch/lib), which the dynamic loader does not resolve by bare name — hence
	# `OSError: libcudart.so: cannot open shared object file` at runtime. Symlink the versioned
	# library to `libcudart.so` in a job-local dir and prepend it to LD_LIBRARY_PATH.
	run: \|
	import glob
	import os
	import pathlib
	import site

	roots = list(site.getsitepackages())
	venv = os.environ.get("VIRTUAL_ENV")
	if venv:
	roots.append(os.path.join(venv, "lib"))
	candidates = []
	for root in roots:
	candidates += glob.glob(os.path.join(root, "*", "libcudart.so"), recursive=True)
	candidates += glob.glob("/usr/local/cuda/lib64/libcudart.so")
	candidates = sorted({c for c in candidates if os.path.exists(c)}, key=len)
	assert candidates, "libcudart.so* not found in installed wheels or system CUDA"
	source = candidates[0]

	lib_dir = pathlib.Path(os.environ["RUNNER_TEMP"]) / "cudalibs"
	lib_dir.mkdir(parents=True, exist_ok=True)
	link = lib_dir / "libcudart.so"
	if link.is_symlink() or link.exists():
	link.unlink()
	link.symlink_to(source)
	print(f"linked {link} -> {source}")

	with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file:
	env_file.write(f"LD_LIBRARY_PATH={lib_dir}{os.pathsep}{os.environ.get('LD_LIBRARY_PATH', '')}\n")
	shell: python

	- name: 🧪 Run TensorRT end-to-end parity tests
	# Real ONNX->engine build + GPU inference is heavy; single-worker, opt-in marker (see pyproject.toml
	# `e2e_tensorrt`) — excluded from the ci-tests-gpu.yml default filter (`and not e2e_tensorrt`) so it
	# only runs here.
	run: \|
	uv run --no-sync pytest tests/export/test_tensorrt_export.py \
	-m e2e_tensorrt -n 1 \
	--timeout=600 \
	--durations=20

	- name: Minimize uv cache
	continue-on-error: true
	run: uv cache prune --ci

	roboflow-deploy-e2e:
	name: Roboflow deploy e2e
	runs-on: ubuntu-latest
	# deploy_to_roboflow() delegates the actual upload to a swallow-and-print try/except deep in
	# the roboflow SDK (see tests/export/test_roboflow_deploy_e2e.py module docstring), so a failed
	# server-side upload is otherwise indistinguishable from success. This job generates a fresh
	# dataset version, deploys against a real Roboflow workspace/project (version auto-resolved to
	# that fresh latest), and independently polls that version's trained-model status.
	# Live external service + real account state: advisory only, not a required check (mirrors
	# ci-legacy-checkpoints.yml). Restricted to the upstream repo — forks never have the secret, so
	# running there would just burn CI minutes on a guaranteed skip.
	if: >-
	github.repository == 'roboflow/rf-detr' &&
	(github.event_name != 'pull_request' \|\|
	github.event.pull_request.head.repo.full_name == github.repository)
	concurrency:
	# Repo-wide group, not per-ref: two parallel runs against the same Roboflow project race —
	# each generates its own fresh version, so run A's auto-latest deploy can land on run B's
	# newer version while A polls its own, guaranteeing a 300s timeout false-negative. Queue
	# instead (and never cancel mid-upload, which would strand a half-deployed version).
	group: roboflow-deploy-e2e
	cancel-in-progress: false
	timeout-minutes: 20
	steps:
	- name: 📥 Checkout the repository
	uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

	- name: 🐍 Install uv and set Python version
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
	with:
	version: "0.9.26"
	python-version: "3.13"
	activate-environment: true

	- name: 🚀 Install Packages (train extra, for the roboflow SDK)
	timeout-minutes: 5
	run: uv pip install -e ".[train]" --group tests

	- name: 🔑 Verify required secrets are present
	run: \|
	missing=()
	[[ -z "${{ secrets.ROBOFLOW_API_KEY }}" ]] && missing+=(ROBOFLOW_API_KEY)
	[[ -z "${{ secrets.ROBOFLOW_TEST_WORKSPACE }}" ]] && missing+=(ROBOFLOW_TEST_WORKSPACE)
	[[ -z "${{ secrets.ROBOFLOW_TEST_PROJECT }}" ]] && missing+=(ROBOFLOW_TEST_PROJECT)
	if [[ ${#missing[@]} -gt 0 ]]; then
	echo "::error::Required secrets are not set: ${missing[*]}"
	exit 1
	fi

	- name: 🔎 Deploy a real model and poll for trained status
	env:
	ROBOFLOW_API_KEY: ${{ secrets.ROBOFLOW_API_KEY }}
	ROBOFLOW_TEST_WORKSPACE: ${{ secrets.ROBOFLOW_TEST_WORKSPACE }}
	ROBOFLOW_TEST_PROJECT: ${{ secrets.ROBOFLOW_TEST_PROJECT }}
	run: \|
	uv run --no-sync pytest . -m e2e_roboflow -n 1 \
	--timeout=600 \
	--durations=20

	- name: Minimize uv cache
	continue-on-error: true
	run: uv cache prune --ci

	openvino-parity:
	name: OpenVINO parity
	# openvino ships portable manylinux wheels (no macOS/GPU-only constraint like coreml/tensorrt),
	# so this can run on ubuntu-latest like executorch-parity.
	runs-on: ubuntu-latest
	concurrency:
	group: pytest-openvino-${{ matrix.python-version }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}
	timeout-minutes: 25
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.10", "3.13"]
	steps:
	- name: 📥 Checkout the repository
	uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

	- name: 🐍 Install uv and set Python version ${{ matrix.python-version }}
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
	with:
	version: "0.9.26"
	python-version: ${{ matrix.python-version }}
	activate-environment: true

	- name: 🚀 Install Packages (openvino extra)
	timeout-minutes: 8
	run: uv pip install -e ".[openvino,train,augment,cli,visual]" --group tests

	- name: 🔎 Verify openvino import
	# Guard against a silently-skipped parity test: TestOpenVINOEndToEnd is gated by
	# pytest.importorskip("openvino"), so a broken/missing wheel would skip it (false
	# green) instead of failing red.
	run: \|
	from rfdetr.export._openvino import _IS_OPENVINO_AVAILABLE

	assert _IS_OPENVINO_AVAILABLE, "openvino installed but not detected as available"
	print("openvino import OK")
	shell: python

	- name: 🧪 Run OpenVINO end-to-end parity tests
	# Real convert_model + save_model + Core().compile_model is heavy; single-worker,
	# opt-in marker (see pyproject.toml `e2e_openvino`) — deliberately excluded from the
	# ci-tests-cpu.yml default filter (`and not e2e_openvino`) so it only runs here.
	run: \|
	uv run --no-sync pytest -m e2e_openvino -n 1 \
	--timeout=600 --durations=20

	- name: Minimize uv cache
	continue-on-error: true
	run: uv cache prune --ci

	export-priority-guardian:
	runs-on: ubuntu-latest
	needs: [executorch-parity, coreml-parity, openvino-parity, tensorrt-parity]
	if: always()
	steps:
	- name: 📋 Display parity job results
	run: \|
	echo "executorch-parity: ${{ needs.executorch-parity.result }}"
	echo "coreml-parity: ${{ needs.coreml-parity.result }}"
	echo "openvino-parity: ${{ needs.openvino-parity.result }}"
	echo "tensorrt-parity: ${{ needs.tensorrt-parity.result }}"
	# Fail explicitly on any non-success result (failure, cancelled, or skipped) instead of
	# relying on timeout behavior, so a cancelled or skipped parity job still blocks the gate.
	- name: ❌ Fail guardian unless every parity job succeeded
	if: >-
	needs.executorch-parity.result != 'success' \|\|
	needs.coreml-parity.result != 'success' \|\|
	needs.openvino-parity.result != 'success' \|\|
	needs.tensorrt-parity.result != 'success'
	run: \|
	echo "One or more export parity jobs did not succeed; failing explicitly."
	exit 1
	- name: ✅ all parity jobs succeeded
	run: echo "All export parity jobs (executorch, coreml, openvino, tensorrt) completed successfully."

Navigation Menu

perf(inference): batch `predict()`'s per-image GPU->CPU transfers into one sync #3731

Workflow file

perf(inference): batch `predict()`'s per-image GPU->CPU transfers into one sync #3731

Uh oh!

Workflow file for this run

perf(inference): batch predict()'s per-image GPU->CPU transfers into one sync #3731

Workflow file

Workflow file for this run

perf(inference): batch `predict()`'s per-image GPU->CPU transfers into one sync #3731