From 5fa1916db3e56208465425bf5198c4429a525016 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Thu, 8 Oct 2026 00:41:40 +0545 Subject: [PATCH 1/4] ci: pin runner images to ubuntu-24.04, macos-26, windows-2025 --- .github/workflows/_build-packages.yml | 4 ++-- .github/workflows/_legacy-checkpoints.yml | 4 ++-- .github/workflows/ci-pkg-install.yml | 2 +- .github/workflows/ci-tests-fabric.yml | 4 ++-- .github/workflows/ci-tests-pytorch.yml | 4 ++-- .github/workflows/code-checks.yml | 2 +- .github/workflows/docs-build.yml | 4 ++-- .github/workflows/docs-tutorials.yml | 2 +- .github/workflows/release-nightly.yml | 2 +- .github/workflows/release-pkg.yml | 8 ++++---- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/_build-packages.yml b/.github/workflows/_build-packages.yml index 5a780c1d33854..b3e2a5e042b16 100644 --- a/.github/workflows/_build-packages.yml +++ b/.github/workflows/_build-packages.yml @@ -20,7 +20,7 @@ defaults: jobs: build-packages: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: pkg-name: ${{ fromJSON(inputs.pkg-names) }} @@ -49,7 +49,7 @@ jobs: merge-artifacts: needs: build-packages - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: # download all build artifacts diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 1b893005a91f1..5e8b9e9d4c118 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -53,7 +53,7 @@ defaults: jobs: create-legacy-ckpts: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: pl-version: ${{ steps.decide-version.outputs.pl-version }} steps: @@ -138,7 +138,7 @@ jobs: aws s3 cp $AWS_RUN checkpoints.zip s3://pl-public-data/legacy/ --acl public-read add-ckpt-test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: inputs.create_pr needs: create-legacy-ckpts env: diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index 62ef2737bc9c9..8bd8af101a0e7 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-22.04", "macOS-14", "windows-2022"] + os: ["ubuntu-24.04", "macos-26", "windows-2025"] pkg-name: ["fabric", "pytorch", "lightning", "notset"] python-version: ["3.10", "3.11"] env: diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index 1f55b944d8f64..2d394509f88da 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macOS-14, ubuntu-22.04, windows-2022] + os: [macos-26, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.6" } @@ -68,7 +68,7 @@ jobs: FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/" # TODO: Remove this - Enable running MPS tests on this platform - DISABLE_MPS: ${{ matrix.os == 'macOS-14' && '1' || '0' }} + DISABLE_MPS: ${{ matrix.os == 'macos-26' && '1' || '0' }} UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer UV_EXCLUDE_NEWER: "2 days" diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 4d91f6a56c860..607acaa9a676b 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macOS-14, ubuntu-22.04, windows-2022] + os: [macos-26, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.6" } @@ -72,7 +72,7 @@ jobs: TORCH_URL: "https://download.pytorch.org/whl/cpu/" FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} # TODO: Remove this - Enable running MPS tests on this platform - DISABLE_MPS: ${{ matrix.os == 'macOS-14' && '1' || '0' }} + DISABLE_MPS: ${{ matrix.os == 'macos-26' && '1' || '0' }} UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer UV_EXCLUDE_NEWER: "2 days" diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index dc172bd0c48b1..59449f25778cf 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -26,7 +26,7 @@ defaults: jobs: mypy: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index 022a970f171a4..e4c754dc454a5 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -54,7 +54,7 @@ env: jobs: docs-make: if: github.event.pull_request.draft == false - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -151,7 +151,7 @@ jobs: deploy-docs: needs: docs-make if: github.repository_owner == 'Lightning-AI' && github.event_name != 'pull_request' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/docs-tutorials.yml b/.github/workflows/docs-tutorials.yml index 517fc0116c011..856c4bc872c29 100644 --- a/.github/workflows/docs-tutorials.yml +++ b/.github/workflows/docs-tutorials.yml @@ -16,7 +16,7 @@ defaults: jobs: docs-update: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index dfa0dbdcdbdd7..561b799c44427 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -18,7 +18,7 @@ defaults: jobs: build-packages: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: PKG_NAME: "lightning" # Supply-chain guard: skip PyPI releases newer than this (ISO 8601 duration). See https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-uploaded-prior-to diff --git a/.github/workflows/release-pkg.yml b/.github/workflows/release-pkg.yml index e04be38241082..9f2e84cd58763 100644 --- a/.github/workflows/release-pkg.yml +++ b/.github/workflows/release-pkg.yml @@ -32,7 +32,7 @@ jobs: artifact-name: dist-packages-${{ github.sha }} upload-packages: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: build-packages if: github.event_name == 'release' steps: @@ -49,7 +49,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} release-version: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: tag: ${{ steps.lai-package.outputs.version }} steps: @@ -66,7 +66,7 @@ jobs: run: python -c "import lightning ; print(f'version={lightning.__version__}')" >> $GITHUB_OUTPUT signaling: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: ["release-version", "build-packages"] if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' env: @@ -130,7 +130,7 @@ jobs: borda publish-packages: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [build-packages] if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' permissions: From 27f45bce2b05bdde8c69228bf9c27eacc8f2e5b8 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Thu, 8 Oct 2026 00:42:22 +0545 Subject: [PATCH 2/4] ci: add runner images reference --- .github/workflows/ci-tests-pytorch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 607acaa9a676b..342b53aa67ccc 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -42,6 +42,7 @@ jobs: strategy: fail-fast: false matrix: + # Runner images: https://github.com/actions/runner-images os: [macos-26, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch From 096c7f711a44fe95c56d473d21777f05e17ff150 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Mon, 12 Oct 2026 14:27:27 +0545 Subject: [PATCH 3/4] ci: run macOS test matrices on macos-15 and relax hooks doctest macos-14 is retired on 2026-11-02, but on macos-26 gloo/spawned-process startup takes ~35s per test with torch>=2.7, pushing pl-cpu/fabric-cpu past their timeouts. Use macos-15 for the test matrices for now and leave a TODO to move to macos-26 in a follow-up. Also stop asserting the rich progress-bar width in hooks.rst; it depends on the it/s text and so on runner speed. --- .github/workflows/ci-tests-fabric.yml | 6 ++++-- .github/workflows/ci-tests-pytorch.yml | 6 ++++-- docs/source-pytorch/common/hooks.rst | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index 2d394509f88da..8141e99e575c2 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -38,7 +38,9 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-26, ubuntu-24.04, windows-2025] + # TODO: move macOS to macos-26 in a follow-up; on that image gloo/spawned-process startup + # takes ~35s per test with torch>=2.7, which pushes the jobs past their timeout + os: [macos-15, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.6" } @@ -68,7 +70,7 @@ jobs: FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/" # TODO: Remove this - Enable running MPS tests on this platform - DISABLE_MPS: ${{ matrix.os == 'macos-26' && '1' || '0' }} + DISABLE_MPS: ${{ matrix.os == 'macos-15' && '1' || '0' }} UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer UV_EXCLUDE_NEWER: "2 days" diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 342b53aa67ccc..a86dfe3f5f256 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -43,7 +43,9 @@ jobs: fail-fast: false matrix: # Runner images: https://github.com/actions/runner-images - os: [macos-26, ubuntu-24.04, windows-2025] + # TODO: move macOS to macos-26 in a follow-up; on that image gloo/spawned-process startup + # takes ~35s per test with torch>=2.7, which pushes the jobs past their timeout + os: [macos-15, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.6" } @@ -73,7 +75,7 @@ jobs: TORCH_URL: "https://download.pytorch.org/whl/cpu/" FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} # TODO: Remove this - Enable running MPS tests on this platform - DISABLE_MPS: ${{ matrix.os == 'macos-26' && '1' || '0' }} + DISABLE_MPS: ${{ matrix.os == 'macos-15' && '1' || '0' }} UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer UV_EXCLUDE_NEWER: "2 days" diff --git a/docs/source-pytorch/common/hooks.rst b/docs/source-pytorch/common/hooks.rst index 73b05f7765e53..90a1351e7d37f 100644 --- a/docs/source-pytorch/common/hooks.rst +++ b/docs/source-pytorch/common/hooks.rst @@ -59,7 +59,7 @@ important to understand. The following order is always used: ... Callback: Training is starting! Model: Training is starting! - Epoch 0/0 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 64/64 ... + Epoch 0/0 ... .. note:: From ee017a6dfb017e1f3680c555b95142af9051ff60 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Mon, 12 Oct 2026 16:29:57 +0545 Subject: [PATCH 4/4] ci: run macOS tests as root to avoid ~35s DNS stall per process group; move to macos-26 On the macos-15 and macos-26 runner images, macOS Local Network Privacy blocks mDNS queries from non-root processes, so any hostname lookup or reverse-DNS lookup (even of 127.0.0.1 / ::ffff:127.0.0.1) blocks for ~35s before falling back. torch's TCPStore formats the peer address with getnameinfo() on every client connect (c10d socket.cpp, SocketImpl ctor), so each ddp_spawn/gloo test paid ~35s in rendezvous (TCPStore 35.0s vs 0.05s), regardless of torch version, USE_LIBUV, MASTER_ADDR or /etc/hosts entries. Root processes are exempt: run the test command via sudo on macOS (TCPStore 0.05s, fabric spawn tests 2-5s again). See actions/runner-images#14568 and #10924. --- .github/workflows/ci-tests-fabric.yml | 11 ++++++----- .github/workflows/ci-tests-pytorch.yml | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index 8141e99e575c2..ddf0816307eba 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -38,9 +38,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO: move macOS to macos-26 in a follow-up; on that image gloo/spawned-process startup - # takes ~35s per test with torch>=2.7, which pushes the jobs past their timeout - os: [macos-15, ubuntu-24.04, windows-2025] + os: [macos-26, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.6" } @@ -70,7 +68,7 @@ jobs: FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/" # TODO: Remove this - Enable running MPS tests on this platform - DISABLE_MPS: ${{ matrix.os == 'macos-15' && '1' || '0' }} + DISABLE_MPS: ${{ startsWith(matrix.os, 'macos') && '1' || '0' }} UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer UV_EXCLUDE_NEWER: "2 days" @@ -156,9 +154,12 @@ jobs: - name: Testing Fabric working-directory: tests/tests_fabric # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 + # On macOS 15+ runners, Local Network Privacy blocks mDNS for non-root processes, so every + # hostname/reverse-DNS lookup (e.g. torch's TCPStore connect) stalls ~35s; root is exempt. + # See https://github.com/actions/runner-images/issues/14568 and https://github.com/actions/runner-images/issues/10924 run: | echo $GITHUB_RUN_ID - python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \ + ${{ runner.os == 'macOS' && 'sudo -E env "PATH=$PATH"' || '' }} python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \ -m pytest -v --timeout=60 --durations=50 --random-order-seed=$GITHUB_RUN_ID \ --junitxml=junit.xml -o junit_family=legacy # NOTE: for Codecov's test results diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index a86dfe3f5f256..5021ccc67eede 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -43,9 +43,7 @@ jobs: fail-fast: false matrix: # Runner images: https://github.com/actions/runner-images - # TODO: move macOS to macos-26 in a follow-up; on that image gloo/spawned-process startup - # takes ~35s per test with torch>=2.7, which pushes the jobs past their timeout - os: [macos-15, ubuntu-24.04, windows-2025] + os: [macos-26, ubuntu-24.04, windows-2025] config: # Test unified "lightning" package at the oldest supported and the latest PyTorch - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.6" } @@ -75,7 +73,7 @@ jobs: TORCH_URL: "https://download.pytorch.org/whl/cpu/" FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} # TODO: Remove this - Enable running MPS tests on this platform - DISABLE_MPS: ${{ matrix.os == 'macos-15' && '1' || '0' }} + DISABLE_MPS: ${{ startsWith(matrix.os, 'macos') && '1' || '0' }} UV_TORCH_BACKEND: "cpu" # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer UV_EXCLUDE_NEWER: "2 days" @@ -187,9 +185,12 @@ jobs: - name: Testing PyTorch working-directory: tests/tests_pytorch # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 + # On macOS 15+ runners, Local Network Privacy blocks mDNS for non-root processes, so every + # hostname/reverse-DNS lookup (e.g. torch's TCPStore connect) stalls ~35s; root is exempt. + # See https://github.com/actions/runner-images/issues/14568 and https://github.com/actions/runner-images/issues/10924 run: | echo $GITHUB_RUN_ID - python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \ + ${{ runner.os == 'macOS' && 'sudo -E env "PATH=$PATH"' || '' }} python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \ -m pytest . -v --timeout=90 --durations=50 --random-order-seed=$GITHUB_RUN_ID \ --junitxml=junit.xml -o junit_family=legacy # NOTE: for Codecov's test results