python-ci.yml

name: Python CI

on:
  push:
    paths:
      - 'python/**'
      - 'cpp/**'
      - '.github/workflows/python-ci.yml'
  pull_request:
    paths:
      - 'python/**'
      - 'cpp/**'
      - '.github/workflows/python-ci.yml'

env:
  CONAN_VERSION: "2.25.1"

jobs:
  lint:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Set up uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        working-directory: ./python
        run: make install

      - name: Lint
        working-directory: ./python
        run: make lint

  test:
    runs-on: ubuntu-22.04
    timeout-minutes: 180
    strategy:
      matrix:
        python-version: ['3.11']
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Set up uv
        uses: astral-sh/setup-uv@v4

      # libaio-dev: Conan 2.x CMakeDeps doesn't propagate system_libs through shared
      # library targets, so libaio (required by folly) must be explicitly installed.
      - name: Install C++ dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y cmake libaio-dev
          pip install conan==${{ env.CONAN_VERSION }}

      - name: Cache Conan packages
        uses: actions/cache@v4
        with:
          path: ~/.conan2
          key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
          restore-keys: |
            conan-cpp-

      - name: Setup Conan remote
        run: |
          conan profile detect --force
          # || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
          conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
          conan remote list

      - name: Setup Rust
        id: rust-toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: rust build cache
        uses: actions/cache@v4
        with:
          path: |
            cpp/build/Release/cargo/build
          key: storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('cpp/src/format/bridge/rust/Cargo.lock', 'cpp/src/format/bridge/rust/Cargo.toml') }}
          restore-keys: |
            storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-

      - name: Build C++ library for Python
        working-directory: ./cpp
        run: |
          make python-lib

      - name: Set library path
        run: |
          BUILD_DIR="$GITHUB_WORKSPACE/cpp/build/Release"
          BUILD_LIB_DIR="$BUILD_DIR/libs"

          # Use the dependency libraries copied for this build. Scanning the
          # whole Conan cache can put stale/incompatible libraries before the
          # ones that match libmilvus-storage.so.
          echo "LD_LIBRARY_PATH=$BUILD_LIB_DIR:$BUILD_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> $GITHUB_ENV

          # Preload the liblzma copied for this build to override system liblzma.
          BUILD_LZMA=$(find "$BUILD_LIB_DIR" -name "liblzma.so*" -type f 2>/dev/null | sort | head -n 1)

          # Build LD_PRELOAD list for libraries that need to be forced at runtime
          PRELOAD_LIBS=""

          if [ -n "$BUILD_LZMA" ]; then
            echo "Found build liblzma at: $BUILD_LZMA"
            echo "Checking symbol in $BUILD_LZMA:"
            nm -D "$BUILD_LZMA" | grep lzma_index_uncompressed_size || echo "Symbol NOT FOUND in build lib"
            PRELOAD_LIBS="$BUILD_LZMA"
          else
            echo "WARNING: build liblzma.so* not found! Verify xz_utils:shared=True"
          fi

          echo "Checking system liblzma symbol:"
          nm -D /usr/lib/x86_64-linux-gnu/liblzma.so.5 2>/dev/null | grep lzma_index_uncompressed_size || echo "Symbol NOT FOUND in System lib"

          # Preload libaio: folly (shared) uses io_submit but doesn't have libaio in DT_NEEDED
          LIBAIO=$(find /usr/lib -name "libaio.so*" 2>/dev/null | head -n 1)
          if [ -n "$LIBAIO" ]; then
            echo "Found libaio at: $LIBAIO"
            PRELOAD_LIBS="${PRELOAD_LIBS:+$PRELOAD_LIBS:}$LIBAIO"
          else
            echo "WARNING: libaio.so not found!"
          fi

          if [ -n "$PRELOAD_LIBS" ]; then
            echo "LD_PRELOAD=$PRELOAD_LIBS" >> $GITHUB_ENV
          fi

          echo "Configured LD_LIBRARY_PATH and LD_PRELOAD for runtime"

      - name: Install Python package
        working-directory: ./python
        run: make install

      - name: Verify library dependencies
        run: |
          set -o pipefail

          echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
          echo "libmilvus-storage dependency resolution:"
          ldd -r cpp/build/Release/libmilvus-storage.so 2>&1 | tee /tmp/libmilvus-storage.ldd
          if grep -q "not found\\|undefined symbol" /tmp/libmilvus-storage.ldd; then
            echo "Runtime dependency resolution failed"
            exit 1
          fi
          # Check if liblzma is available
          ldconfig -p | grep lzma || echo "liblzma not in ldconfig, checking file system..."
          ls -la /usr/lib/x86_64-linux-gnu/liblzma* || true
          # Check glog dependencies
          find cpp/build/Release/libs -name "libglog.so*" -exec ldd {} \; 2>/dev/null | head -20 || true

      - name: Run tests
        working-directory: ./python
        run: make test

      - name: Upload coverage to Codecov
        if: ${{ github.repository == 'milvus-io/milvus-storage' }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.MILVUS_STORAGE_CODECOV_TOKEN }}
          files: ./python/coverage.xml
          flags: python
          name: python-coverage
          fail_ci_if_error: false