[Cylon] Arrow 14.0.2 Integration

.github/workflows/c-cpp.yml

@@ -21,8 +21,11 @@ jobs:
       run: sudo apt-get update && sudo apt-get install -y --no-install-recommends --no-install-suggests g++ python3 python3-dev python3-pip maven libnuma-dev libc-dev python3-venv python3-numpy openmpi-bin libopenmpi-dev
     - name: Installing UCX
       run: wget https://github.com/openucx/ucx/releases/download/v1.13.1/ucx-1.13.1.tar.gz -P $HOME/ucx && (cd $HOME/ucx && tar xzf ucx-1.13.1.tar.gz) && (cd $HOME/ucx/ucx-1.13.1 && ./contrib/configure-release --prefix=$PWD/install --with-go=no && make -j8 install)
+    - name: Export Arrow Path
+      run: export LD_LIBRARY_PATH=$PWD/build/arrow/install/lib64:$PWD/build/glog/install/lib64:$$PWD/build/lib64:$PWD/build/lib:$LD_LIBRARY_PATH
+
     - name: Build and test
-      run: python3 -m venv ENV && ./build.sh -pyenv $(pwd)/ENV -bpath $(pwd)/build --cpp --test --python --pytest --cmake-flags "-DCYLON_UCX=ON -DUCX_LIBDIR=$HOME/ucx/ucx-1.13.1/install/lib -DUCX_INCLUDEDIR=$HOME/ucx/ucx-1.13.1/install/include"
+      run: python3 -m venv ENV && ./build.sh -pyenv $(pwd)/ENV -bpath $(pwd)/build --cpp --test --python --pytest --cmake-flags "-DCYLON_UCX=ON -DUCX_LIBDIR=$HOME/ucx/ucx-1.13.1/install/lib -DUCX_INCLUDEDIR=$HOME/ucx/ucx-1.13.1/install/include -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx)"
     - name: Build java
       run: ./build.sh -pyenv $(pwd)/ENV -bpath $(pwd)/build --java
 

.github/workflows/macos.yml

@@ -24,10 +24,13 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Remove link for preventing an error
-        run: rm -f /usr/local/bin/2to3
+      #- name: Remove link for preventing an error
+      #  run: rm -f /usr/local/bin/2to3
       - name: Install dependencies
         run: brew install re2 automake boost brotli c-ares ccache flatbuffers grpc llvm lz4 minio ninja openssl@1.1 protobuf rapidjson snappy thrift wget zstd
+
+      - name: Include $CONDA in $PATH
+        run: export PATH="$CONDA/condabin:$PATH"
 
       - uses: conda-incubator/setup-miniconda@v2
         with:

build.py

@@ -18,6 +18,7 @@
 import platform
 import subprocess
 import sys
+import pyarrow as pa
 from pathlib import Path
 
 logging.basicConfig(format='[%(levelname)s] %(message)s')
@@ -217,6 +218,34 @@ def build_cpp():
     verb = '-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON' if args.verbose else ''
     clean = '--clean-first' if args.clean else ''
 
+    pyarrow_location = os.path.dirname(pa.__file__)
+    conda_prefix = check_conda_prefix()
+    if OS_NAME == 'Linux' or OS_NAME == 'Darwin':
+        if not os.path.exists(f"{conda_prefix}/lib/libarrow_python.so"):
+            sym_generator = f"ln -s {pyarrow_location}/libarrow_python.so {conda_prefix}/lib/libarrow_python.so"
+            logger.info(f"Generate Symbolic link: {sym_generator}")
+            res = subprocess.call(sym_generator, cwd=BUILD_DIR, shell=True)
+            check_status(res, "Generate Symbolic link")
+
+        os.environ['CC']=f"{conda_prefix}/bin/mpicc" 
+        logger.info(f"export cc: : {os.getenv('CC')}")
+
+        os.environ['CXX']=f"{conda_prefix}/bin/mpicxx" 
+        logger.info(f"export cxx: : {os.getenv('CXX')}")
+
+        os.environ['MPI_CC']=f"{conda_prefix}/bin/mpicc" 
+        logger.info(f"export mpicc: : {os.getenv('MPI_CC')}")
+
+        os.environ['MPI_CXX']=f"{conda_prefix}/bin/mpicxx" 
+        logger.info(f"export mpicxx: : {os.getenv('MPI_CXX')}")
+    else:
+        if not os.path.exists(f"{conda_prefix}\lib\libarrow_python.so"):
+            sym_generator = f"mklink {pyarrow_location}\libarrow_python.so {conda_prefix}\lib\libarrow_python.so"
+            logger.info(f"Generate Symbolic link: {sym_generator}")
+            res = subprocess.call(sym_generator, cwd=BUILD_DIR, shell=True)
+            check_status(res, "Generate Symbolic link")
+
+
     cmake_command = f"cmake -DPYCYLON_BUILD={on_off(BUILD_PYTHON)} {win_cmake_args} " \
                     f"-DCMAKE_BUILD_TYPE={CPP_BUILD_MODE} " \
                     f"-DCYLON_WITH_TEST={on_off(RUN_CPP_TESTS)} " \

build.sh

@@ -262,6 +262,8 @@ build_cpp_with_custom_arrow() {
   print_line
   source "${PYTHON_ENV_PATH}"/bin/activate || exit 1
   read_python_requirements
+  pip install pyarrow==16.1.0 || exit 1
+  pip install parquet || exit 1
   ARROW_LIB=$(python3 -c 'import pyarrow as pa; import os; print(os.path.dirname(pa.__file__))') || exit 1
   ARROW_INC=$(python3 -c 'import pyarrow as pa; import os; print(os.path.join(os.path.dirname(pa.__file__), "include"))') || exit 1
   echo "ARROW_LIB: $ARROW_LIB"
@@ -300,6 +302,11 @@ build_cpp_conda() {
   print_line
   echo "Building Conda CPP in ${BUILD_MODE} mode"
   print_line
+
+  export CC=`which mpicc`
+  export CXX=`which mpicxx`
+  export MPI_CC=`which mpicc`
+  export MPI_CXX=`which mpicxx`
 
   # set install path to conda directory if not already set
   INSTALL_PATH=${INSTALL_PATH:=${PREFIX:=${CONDA_PREFIX}}}
@@ -314,10 +321,15 @@ build_cpp_conda() {
   for SO_FILE in "${ARROW_LIB}/libarrow.so" "${ARROW_LIB}/libarrow_python.so"; do
     if [ ! -f "$SO_FILE" ]; then
       echo "$SO_FILE does not exist! Trying to create a symlink"
-      ln -sf "$(ls "$SO_FILE".*)" "$SO_FILE" || exit 1
+      PYARROW_LIB=$(python3 -c 'import pyarrow as pa; import os; print(os.path.dirname(pa.__file__))') || exit 1
+      ln -sf ${PYARROW_LIB}/libarrow_python.so "$SO_FILE" || exit 1
     fi
   done
 
+
+
+  #ln -s ${ARROW_LIB}/libarrow_python.so ${PYARROW_LIB}/libarrow_python.so
+
   echo "SOURCE_DIR: ${SOURCE_DIR}"
   BUILD_PATH=$(pwd)/build
   mkdir -p ${BUILD_PATH}
@@ -344,6 +356,7 @@ build_gcylon() {
 
   # set install path to conda directory if not already set
   INSTALL_PATH=${INSTALL_PATH:=${PREFIX:=${CONDA_PREFIX}}}
+  export CUDA_HOME=$CONDA_PREFIX
 
   echo "SOURCE_DIR: ${SOURCE_DIR}"
   BUILD_PATH=$(pwd)/build
@@ -380,7 +393,7 @@ build_python_pyarrow() {
   echo "Building Pycylon"
   source "${PYTHON_ENV_PATH}"/bin/activate || exit 1
   read_python_requirements
-  pip install pyarrow==9.0.0 || exit 1
+  #pip install pyarrow==14.0.2 || exit 1
 
   ARROW_LIB=$(python3 -c 'import pyarrow as pa; import os; print(os.path.dirname(pa.__file__))') || exit 1
   LD_LIBRARY_PATH="${ARROW_LIB}:${BUILD_PATH}/lib:${LD_LIBRARY_PATH}" || exit 1
@@ -526,7 +539,7 @@ fi
 
 if [ "${PYTHON_BUILD}" = "ON" ]; then
   export_info
-  build_pyarrow
+  #build_pyarrow
   check_pyarrow_installation
   build_python
   check_pycylon_installation

conda/environments/cylon.yml

@@ -3,19 +3,23 @@ channels:
   - conda-forge
   - defaults
 dependencies:
+  #- python>=3.8
   - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  - cmake>=3.23.1
+  #- arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
-  - openmpi=4.1.3=ha1ae619_105
+  - openmpi=4.1.4
   - ucx>=1.12.1
   - cython>=0.29.31,<3
   - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
+  - pandas>=1.0
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it
   - pytest
   - pytest-mpi
   - mpi4py
+  - gcc=11.4
+  - gxx=11.4
+  - gxx_linux-64=11.4

conda/environments/cylon_MacOS.yml

@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python>=3.9,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  #- arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - openmpi>=4.1.2
   - cython>=0.29.31,<3
@@ -18,3 +18,6 @@ dependencies:
   - pytest
   - pytest-mpi
   - mpi4py
+  #- gcc=11.4
+  #- gxx=11.4
+  #- gxx_linux-64=11.4

conda/environments/cylon_NoUCX.yml

@@ -5,10 +5,10 @@ channels:
 dependencies:
   - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  #- arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
-  - openmpi=4.1.3=ha1ae619_105
+  - openmpi=4.1.4
   - cython>=0.29.31,<3
   - numpy<1.24.4
   - pandas>=1.0,<2.0.0
@@ -18,3 +18,6 @@ dependencies:
   - pytest
   - pytest-mpi
   - mpi4py
+  - gcc=11.4
+  - gxx=11.4
+  - gxx_linux-64=11.4

conda/environments/cylon_rivanna_1.yml

@@ -3,12 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8
+  - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  #- arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
-  - openmpi=4.1.3=ha1ae619_105
+  - openmpi=4.1.4
   - ucx>=1.12.1
   - cython>=0.29.31,<3
   - numpy
@@ -19,6 +19,6 @@ dependencies:
   - pytest
   - pytest-mpi
   - mpi4py
-  - gcc
-  - gxx
-  - gxx_linux-64
+  - gcc=11.4
+  - gxx=11.4
+  - gxx_linux-64=11.4

conda/environments/cylon_rivanna_2.yml

@@ -3,12 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.10
+  - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  #- arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
-  #- openmpi=4.1.3=ha1ae619_105
+  - openmpi=4.1.4
   - ucx>=1.12.1
   - cython>=0.29.31,<3
   - numpy

conda/environments/gcylon.yml

@@ -5,22 +5,29 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
-  - cython>=0.29.31,<3
-  - cudf=22.12.01
-  - cudatoolkit=11.5
+  - cudf
+  - cuda-toolkit=11.8
+  - cuda-runtime=11.8
+  - cuda-nvcc=11.8
+  - cuda-cudart=11.8
+  - python=3.11
+  - cmake
+  #- arrow-cpp=9
+  #- pyarrow=9.0.0
+  - cython
+  #- cuda-version=12
   - glog
-  - openmpi=4.1.3=ha1ae619_105
+  - openmpi=4.1.4
   - ucx>=1.12.1
-  - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
-  - fsspec>=0.6.0
+  - numpy
+  - pandas
+  - fsspec
   - setuptools
   # these are for running tests only,
   # they are not needed for using pygcylon or compiling it
   - pytest
   - pytest-mpi
   - mpi4py
+  - gcc=11.4
+  - gxx=11.4
+  - gxx_linux-64=11.4

conda/environments/windows.yml

@@ -4,17 +4,20 @@ channels:
   - defaults
 dependencies:
   - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  - cmake>=3.23.1
+  #g- arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - msmpi
   - cython>=0.29.31,<3
   - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
+  - pandas>=1.0
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it
   - pytest
   - pytest-mpi
   - mpi4py
+  #- gcc=11.4
+  #- gxx=11.4
+  #- gxx_linux-64=11.4

cpp/CMake/Modules/ConfigureArrow.cmake

@@ -96,11 +96,11 @@ message(STATUS "Arrow libs dir: " ${ARROW_LIBRARY_DIR})
 message(STATUS "Arrow include dir: " ${ARROW_INCLUDE_DIR})
 
 # find packages with the help of arrow Find*.cmake files
-find_package(Arrow REQUIRED HINTS "${ARROW_LIBRARY_DIR}/cmake/arrow" CONFIGS FindArrow.cmake)
+find_package(Arrow REQUIRED HINTS "${ARROW_ROOT}/cmake/arrow" CONFIGS FindArrow.cmake)
 message(STATUS "Arrow lib: ${ARROW_SHARED_LIB}")
 set(ARROW_LIB ${ARROW_SHARED_LIB})
 
-find_package(Parquet REQUIRED HINTS "${ARROW_LIBRARY_DIR}/cmake/arrow" CONFIGS FindParquet.cmake)
+find_package(Parquet REQUIRED HINTS "${ARROW_ROOT}/cmake/arrow" CONFIGS FindParquet.cmake)
 message(STATUS "Parquet lib: ${PARQUET_SHARED_LIB}")
 set(PARQUET_LIB ${PARQUET_SHARED_LIB})
 

cpp/CMakeLists.txt

@@ -24,17 +24,17 @@
 cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-project(CYLON VERSION 0.6.0)
+project(CYLON VERSION 0.7.0)
 
-set(CYLON_VERSION 0.6.0)
+set(CYLON_VERSION 0.7.0)
 
 ## defaults to release build
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif ()
 
 # cmake modules directories
-set(CYLON_ARROW_VERSION 9.0.0)
+set(CYLON_ARROW_VERSION 16.1.0)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake/Modules/" ${CMAKE_MODULE_PATH})
 list(APPEND CMAKE_MODULE_PATH ${CYLON_SOURCE_DIR}/CMake)
 
@@ -74,14 +74,16 @@ else ()
 endif ()
 
 # C++ standard
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(GCC_ABI_COMPILE_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")
 if (NOT HAVE_EXCLUSIVE_SCAN)
     add_definitions(-DNEED_EXCLUSIVE_SCAN)
 endif ()
 
+
+
 # check for conda environment
 if ("$ENV{CONDA_BUILD}" STREQUAL "1")
     message("CONDA_BUILD detected. BUILD_PREFIX: $ENV{BUILD_PREFIX} PREFIX: $ENV{PREFIX}")
@@ -404,11 +406,11 @@ if (${ARROW_BUILD_TYPE} STREQUAL "SYSTEM")
 
 elseif (${ARROW_BUILD_TYPE} STREQUAL "SOURCE")
     message("Building Arrow from SOURCE")
+    set(Arrow_DIR ${ARROW_ROOT}/cmake/arrow)
     include(ConfigureArrow)
     include_directories(SYSTEM "${ARROW_INCLUDE_DIR}")
 elseif (${ARROW_BUILD_TYPE} STREQUAL "CUSTOM")
     message("Using CUSTOM Arrow installation")
-
     if (NOT ARROW_LIB_DIR)
         message(FATAL_ERROR "ARROW_BUILD_TYPE is set to CUSTOM, ARROW_LIB_DIR should be set")
     endif ()

cpp/src/cylon/arrow/arrow_comparator.cpp

@@ -150,7 +150,7 @@ struct CompareFunc<ArrowT, Asc, arrow::enable_if_floating_point<ArrowT>> {
 template<typename ArrowT, bool Asc>
 struct CompareFunc<ArrowT, Asc, arrow::enable_if_has_string_view<ArrowT>> {
 
-  static int compare(const arrow::util::string_view &v1, const arrow::util::string_view &v2) {
+  static int compare(const std::string_view &v1, const std::string_view &v2) {
     if (Asc) {
       return v1.compare(v2);
     } else {
@@ -220,7 +220,7 @@ class EmptyIndexComparator : public ArrayIndexComparator {
 /*
  * Single implementation for both numeric and binary comparators. array->GetView(idx) method is
  * used here. For Numeric arrays, this would translate to value by pointer offset. For binary,
- * this will take arrow::util::string_view.
+ * this will take std::string_view.
  */
 template<typename ArrowT, bool Asc, bool NullOrder>
 class ArrayIndexComparatorWithNulls : public ArrayIndexComparator {