Skip to content
Python

feat: expose staged index segment transactions #4

Sign in to view logs

feat: expose staged index segment transactions

feat: expose staged index segment transactions #4

Workflow file for this run

.github/workflows/python.yml at 75c5c79
name: Python
on:
push:
branches:
- main
- release/**
pull_request:
branches:
- main
- release/**
paths:
- Cargo.*
- python/**
- rust/**
- protos/**
- .github/workflows/python.yml
- .github/workflows/build_linux_wheel/**
- .github/workflows/build_mac_wheel/**
- .github/workflows/run_tests/**
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
# This env var is used by Swatinem/rust-cache@v2 for the cache
# key, so we set it to make sure it is always consistent.
CARGO_TERM_COLOR: always
RUST_BACKTRACE: "1"
CI: "true"
# Color output for pytest is off by default.
PYTEST_ADDOPTS: "--color=yes"
FORCE_COLOR: "1"
# Change this to bust all caches (may be needed periodically if the caches accumulate
# a lot of cruft).
CACHE_PREFIX: "2"
jobs:
lint:
timeout-minutes: 45
runs-on: "ubuntu-24.04-4x"
defaults:
run:
shell: bash
working-directory: python
env:
# Need up-to-date compilers for kernels
CC: clang-18
CXX: clang++-18
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: 3.11 # Ray does not support 3.12 yet.
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
workspaces: python
prefix-key: ${{ env.CACHE_PREFIX }}
- name: Install linting tools
run: |
pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas] lance-namespace
pip install torch --index-url https://download.pytorch.org/whl/cpu
- name: Lint Python
run: |
ruff format --check --diff python
ruff check python
pyright
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Lint Rust
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
cargo fmt --all -- --check
cargo clippy --profile ci --locked --features ${ALL_FEATURES} --tests -- -D warnings
- name: Build
run: |
python -m venv venv
source venv/bin/activate
pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu
pip install maturin==1.13.3 uv
maturin develop --uv --profile ci --locked --extras tests,ray
- name: Run doctest
run: |
source venv/bin/activate
pytest --doctest-modules python/lance
linux:
timeout-minutes: 45
strategy:
matrix:
python-minor-version: ["10", "13"]
name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
runs-on: "ubuntu-24.04-4x"
defaults:
run:
shell: bash
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: 3.${{ matrix.python-minor-version }}
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
workspaces: python
prefix-key: ${{ env.CACHE_PREFIX }}
- uses: ./.github/workflows/build_linux_wheel
with:
args: "--profile ci"
- uses: ./.github/workflows/run_tests
with:
memtest: true
- name: Upload wheels as artifacts
if: ${{ matrix.python-minor-version == '13' }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: linux-wheels
path: python/target/wheels/pylance-*.whl
compat:
needs: linux
timeout-minutes: 60
runs-on: ubuntu-24.04
name: Compatibility Tests
defaults:
run:
shell: bash
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: 3.13
- name: Download wheels
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: linux-wheels
path: python/wheels
- name: Install dependencies
run: |
pip install $(ls wheels/pylance-*.whl)[tests,ray]
- name: Run compatibility tests
run: |
make compattest
env:
COMPAT_TEMP_VENV: 1
# Cross-version index maintenance-sequence search (see tests/compat/compat_sequence.py).
# Ages an index under the latest release of each of the two previous majors and exercises
# it under this commit, searching op sequences for panics or correctness divergence. The
# reader (HEAD) reuses the wheel the `linux` job already built rather than recompiling.
# Post-merge only: the search is slower than the rest of the suite, so it runs on pushes
# to main/release rather than blocking every PR (same gating as rust-benchmark).
compat-sequence:
needs: linux
if: github.event_name != 'pull_request'
timeout-minutes: 60
runs-on: ubuntu-24.04
name: Index Sequence Compat
defaults:
run:
shell: bash
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: 3.13
- name: Download wheels
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: linux-wheels
path: python/wheels
- name: Install host deps
run: pip install pytest pytest-xdist pyarrow packaging
# Age under each of the two previous majors (writer wheels from PyPI) and read under
# this commit's prebuilt wheel. Length 5 is the shallowest depth that reaches the
# ENT-1662 sequence; -n oversubscribes cores since the work is I/O-bound.
- name: Run sequence search (previous two majors -> HEAD)
env:
COMPAT_MAX_LENGTH: "5"
COMPAT_TO_REF: HEAD
COMPAT_PREBUILT_REF: HEAD
run: |
set -euo pipefail
wheel=$(ls "$PWD"/wheels/pylance-*.whl | head -1)
refs=$(PYTHONPATH=python/tests python -c "
from compat.compat_decorator import pylance_stable_versions
latest = {}
for v in pylance_stable_versions():
latest[v.major] = v
print(' '.join(str(latest[m]) for m in sorted(latest)[-2:]))
")
echo "reader wheel: $wheel"
echo "writer refs : $refs"
status=0
for from_ref in $refs; do
echo "::group::$from_ref -> HEAD"
COMPAT_FROM_REF="$from_ref" COMPAT_PREBUILT_WHEEL="$wheel" \
python -m pytest python/tests/compat/test_index_sequence.py \
--run-compat -n "$(( $(nproc) * 4 ))" -v --no-header || status=1
echo "::endgroup::"
done
exit $status
linux-arm:
timeout-minutes: 45
runs-on: ubuntu-24.04-arm64-4x
name: Python Linux 3.13 ARM
defaults:
run:
shell: bash
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: 3.13
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
workspaces: python
prefix-key: ${{ env.CACHE_PREFIX }}
- uses: ./.github/workflows/build_linux_wheel
with:
arm-build: "true"
manylinux: "2_28"
args: "--profile ci"
- name: Install dependencies
run: |
sudo apt update -y -qq
sudo apt install -y libhdf5-dev
- uses: ./.github/workflows/run_tests
mac:
timeout-minutes: 45
name: Python macOS 3.13 ARM
runs-on: "warp-macos-14-arm64-6x"
defaults:
run:
shell: bash
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: "3.13"
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
workspaces: python
prefix-key: ${{ env.CACHE_PREFIX }}
- uses: ./.github/workflows/build_mac_wheel
with:
args: "--profile ci"
- uses: ./.github/workflows/run_tests
with:
skip-torch: "true"
windows:
runs-on: windows-latest-4x
timeout-minutes: 90
defaults:
run:
shell: powershell
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
workspaces: python
prefix-key: ${{ env.CACHE_PREFIX }}
- uses: ./.github/workflows/build_windows_wheel
with:
args: "--profile ci"
- uses: ./.github/workflows/run_tests
aws-integtest:
timeout-minutes: 45
runs-on: "ubuntu-24.04-4x"
defaults:
run:
shell: bash
working-directory: python
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: "3.11" # TODO: upgrade when ray supports 3.12
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
workspaces: python
prefix-key: ${{ env.CACHE_PREFIX }}
- uses: ./.github/workflows/build_linux_wheel
with:
args: "--profile ci"
- name: Install dependencies
run: |
pip install ray[data]
pip install torch --index-url https://download.pytorch.org/whl/cpu
- uses: ./.github/workflows/run_integtests