Merge branch 'main' into add-testing-315 #18064
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | ||
|
Check failure on line 1 in .github/workflows/unittest.yml
|
||
| pull_request: | ||
| branches: | ||
| - main | ||
| - preview | ||
| # Trigger workflow on GitHub merge queue events | ||
| # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group | ||
| merge_group: | ||
| types: [checks_requested] | ||
| name: unittest | ||
| permissions: | ||
| contents: read | ||
| env: | ||
| PACKAGE_DIRS: packages preview-packages | ||
| NOX_REUSE_EXISTING_VIRTUALENVS: "true" | ||
| NOX_ENVDIR: "/tmp/shared_nox_envs" | ||
| NOX_DEFAULT_VENV_BACKEND: "uv" | ||
| UV_VENV_SEED: "1" | ||
| # Note: PARALLEL_WORKERS must remain "1" for unit tests because running pytest concurrently | ||
| # on the same runner VM can cause socket/IPC/process deadlocks across packages. | ||
| PARALLEL_WORKERS: "1" | ||
| jobs: | ||
| initialize: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| matrix: ${{ steps.set-matrix.outputs.matrix }} | ||
| is_full_run: ${{ steps.check-label.outputs.is_full_run }} | ||
| env: | ||
| MAX_SHARDS: 16 | ||
| # Define weights for long-running unit tests to balance shard execution time | ||
| # Each weight is roughly 1 minute of expected execution time | ||
| # Default for unset packages is 1 | ||
| PACKAGE_WEIGHTS: | | ||
| bigframes: 6 | ||
| google-ai-generativelanguage: 4 | ||
| google-auth: 5 | ||
| google-cloud-compute: 12 | ||
| google-cloud-compute-v1beta: 12 | ||
| google-cloud-dialogflow: 6 | ||
| google-cloud-dialogflow-cx: 6 | ||
| google-cloud-discoveryengine: 8 | ||
| google-cloud-retail: 5 | ||
| google-shopping-merchant-accounts: 4 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 | ||
| with: | ||
| fetch-depth: 2 | ||
| persist-credentials: false | ||
| - name: Check for unit_test:all_packages label | ||
| id: check-label | ||
| run: | | ||
| if [[ "${{ contains(github.event.pull_request.labels.*.name, 'unit_test:all_packages') }}" == "true" ]]; then | ||
| echo "is_full_run=true" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "is_full_run=false" >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Setup Python | ||
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Get package shards | ||
| id: set-matrix | ||
| env: | ||
| BUILD_TYPE: presubmit | ||
| TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} | ||
| TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }} | ||
| MAX_SHARDS: ${{ env.MAX_SHARDS }} | ||
| PACKAGE_WEIGHTS: ${{ env.PACKAGE_WEIGHTS }} | ||
| run: | | ||
| if [ -n "$TARGET_BRANCH" ]; then | ||
| git fetch origin "$TARGET_BRANCH" --depth=1 || true | ||
| fi | ||
| python3 ci/get_package_shards.py | ||
| unit: | ||
| needs: initialize | ||
| if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != '' | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: true | ||
| matrix: | ||
| python: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] | ||
| package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }} | ||
| name: ${{ matrix.package_shard.is_sharded && format('unit ({0}, {1})', matrix.python, matrix.package_shard.name) || format('unit ({0})', matrix.python) }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | ||
| # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` | ||
| # See https://github.com/googleapis/google-cloud-python/issues/12013 | ||
| # and https://github.com/actions/checkout#checkout-head. | ||
| with: | ||
| fetch-depth: 2 | ||
| persist-credentials: false | ||
| - name: Setup Python | ||
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 | ||
| with: | ||
| python-version: ${{ matrix.python }} | ||
| cache: 'pip' | ||
| allow-prereleases: true | ||
| - name: Setup uv | ||
| uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: 'packages/**/testing/constraints*.txt' | ||
| - name: Install nox | ||
| run: | | ||
| python -m pip install --upgrade setuptools pip wheel | ||
| python -m pip install nox | ||
| - name: Run unit tests | ||
| uv pip install --system nox nox-uv | ||
| # Caches compiled wheels locally to prevent building heavy libraries | ||
| # such as grpcio, which we build from scratch on every run for Python 3.15+. | ||
| # Follow https://github.com/grpc/grpc/issues/41010 for updates. | ||
| - name: Cache Built Python 3.15 Wheels | ||
| if: matrix.python == '3.15' | ||
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | ||
| with: | ||
| path: .uv-wheel-cache | ||
| key: ${{ runner.os }}-uv-wheels-3.15-${{ hashFiles('packages/**/testing/constraints*.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-uv-wheels-3.15- | ||
| # Pre-cache heavy dependencies sequentially before running parallel tests | ||
| - name: Pre-cache heavy dependencies (grpcio) | ||
| if: matrix.python == '3.15' | ||
| run: | | ||
| # Create the directory so uv doesn't crash if there is a `cache miss` | ||
| mkdir -p .uv-wheel-cache | ||
| # Download and compile grpcio sequentially so that parallel test workers | ||
| # immediately hit the warm cache instead of deadlocking on compilation. | ||
| uv pip install --system grpcio --find-links .uv-wheel-cache | ||
| - name: Run unit tests for ${{ matrix.package_shard.description }} | ||
| env: | ||
| BUILD_TYPE: presubmit | ||
| TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} | ||
| TEST_TYPE: unit | ||
| PY_VERSION: ${{ matrix.python }} | ||
| PACKAGE_LIST: ${{ matrix.package_shard.packages }} | ||
| run: | | ||
| ci/run_conditional_tests.sh | ||
| - name: Upload coverage results | ||
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | ||
| with: | ||
| name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }} | ||
| path: .coverage.${{ matrix.python }}.* | ||
| include-hidden-files: true | ||
| all-tests: | ||
| needs: [initialize, unit] | ||
| if: always() | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Check unit test results | ||
| run: | | ||
| # 1. Check initialize job | ||
| if [[ "${{ needs.initialize.result }}" != "success" ]]; then | ||
| echo "Error: The initialize job status was: ${{ needs.initialize.result }}" | ||
| exit 1 | ||
| fi | ||
| # 2. Check unit test shards | ||
| if [[ "${{ needs.unit.result }}" != "success" && "${{ needs.unit.result }}" != "skipped" ]]; then | ||
| echo "Unit tests failed" | ||
| exit 1 | ||
| fi | ||
| echo "All unit tests passed or were skipped" | ||
| cover: | ||
| if: always() && !cancelled() && needs.all-tests.result == 'success' | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - all-tests | ||
| - initialize | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | ||
| # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` | ||
| # See https://github.com/googleapis/google-cloud-python/issues/12013 | ||
| # and https://github.com/actions/checkout#checkout-head. | ||
| with: | ||
| fetch-depth: 2 | ||
| persist-credentials: false | ||
| - name: Setup Python | ||
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Determine if coverage evaluation is required | ||
| id: packages | ||
| env: | ||
| TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }} | ||
| TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} | ||
| run: | | ||
| if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then | ||
| echo "should_evaluate_coverage=true" >> "$GITHUB_OUTPUT" | ||
| else | ||
| TARGET_BRANCH="${TARGET_BRANCH:-main}" | ||
| git fetch origin "$TARGET_BRANCH" --depth=1 || true | ||
| num_files_changed=$(git diff --name-only "origin/${TARGET_BRANCH}" -- ${PACKAGE_DIRS} | wc -l | tr -d ' ') | ||
| if [[ "${num_files_changed}" -gt 0 ]]; then | ||
| echo "should_evaluate_coverage=true" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "should_evaluate_coverage=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| fi | ||
| - name: Install coverage | ||
| if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }} | ||
| run: | | ||
| python -m pip install --upgrade setuptools pip wheel | ||
| python -m pip install coverage | ||
| - name: Download coverage results | ||
| if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }} | ||
| uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 | ||
| with: | ||
| path: /dev/shm/.coverage-results/ | ||
| merge-multiple: true | ||
| - name: Report coverage results | ||
| if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }} | ||
| env: | ||
| # TODO: default to 100% coverage after next gapic-generator release | ||
| # https://github.com/googleapis/google-cloud-python/issues/17459 | ||
| DEFAULT_FAIL_UNDER: 99 | ||
| TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }} | ||
| TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} | ||
| BUILD_TYPE: presubmit | ||
| run: | | ||
| ci/report_coverage.sh | ||
