diff --git a/.github/workflows/tests-external-cfitsio.yml b/.github/workflows/tests-external-cfitsio.yml index 324190a7..81542678 100644 --- a/.github/workflows/tests-external-cfitsio.yml +++ b/.github/workflows/tests-external-cfitsio.yml @@ -140,7 +140,7 @@ jobs: cd .. cd .. - - name: test non-bundled build + - name: build non-bundled fitsio run: | export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" @@ -149,19 +149,22 @@ jobs: --config-settings="--global-option=--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \ --config-settings="--global-option=--system-fitsio-libdir=$HOME/cfitsio-static-install/lib" - python -c "import fitsio; assert not fitsio.cfitsio_has_bzip2_support()" + - name: test non-bundled build + run: | + python -c "import fitsio; assert not fitsio.backend_has_bzip2_support()" if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()" + python -c "import fitsio; assert not fitsio.backend_has_curl_support()" else - python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + python -c "import fitsio; assert fitsio.backend_has_curl_support()" fi if [[ "${{ matrix.config.cf}}" != "" ]]; then - python -c "import fitsio; assert fitsio.cfitsio_is_reentrant()" + python -c "import fitsio; assert fitsio.backend_is_reentrant()" else - python -c "import fitsio; assert not fitsio.cfitsio_is_reentrant()" + python -c "import fitsio; assert not fitsio.backend_is_reentrant()" fi + python -c "import fitsio; assert fitsio.fitsio_backend() == 'cfitsio'" - pytest -vv --durations=20 ${PRP_FLAGS} fitsio + pytest --durations=20 ${PRP_FLAGS} fitsio - name: install bzip2 on linux if: matrix.os == 'ubuntu-latest' @@ -202,7 +205,7 @@ jobs: cd .. cd .. - - name: test non-bundled build w/ env vars w/ bzip2 + - name: build non-bundled fitsio w/ env vars w/ bzip2 run: | export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" @@ -214,11 +217,14 @@ jobs: export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib pip install ${PIP_OPTIONS} -e . - python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" + - name: test non-bundled build w/ env vars w/ bzip2 + run: | + python -c "import fitsio; assert fitsio.backend_has_bzip2_support()" if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()" + python -c "import fitsio; assert not fitsio.backend_has_curl_support()" else - python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + python -c "import fitsio; assert fitsio.backend_has_curl_support()" fi + python -c "import fitsio; assert fitsio.fitsio_backend() == 'cfitsio'" - pytest -vv --durations=20 ${{ matrix.config.doslow }} ${PRP_FLAGS} fitsio + pytest --durations=20 ${{ matrix.config.doslow }} ${PRP_FLAGS} fitsio diff --git a/.github/workflows/tests-pypi.yml b/.github/workflows/tests-pypi.yml index 5982f0fc..526aaa4e 100644 --- a/.github/workflows/tests-pypi.yml +++ b/.github/workflows/tests-pypi.yml @@ -60,7 +60,8 @@ jobs: - name: test fitsio run: | - pytest -vv --durations=20 --slow fitsio - python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" - python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" - python -c "import fitsio; assert fitsio.cfitsio_is_reentrant()" + pytest --durations=20 --slow fitsio + python -c "import fitsio; assert fitsio.backend_has_bzip2_support()" + python -c "import fitsio; assert fitsio.backend_has_curl_support()" + python -c "import fitsio; assert fitsio.backend_is_reentrant()" + python -c "import fitsio; assert fitsio.fitsio_backend() == 'cfitsio'" diff --git a/.github/workflows/tests-rsfitsio.yml b/.github/workflows/tests-rsfitsio.yml new file mode 100644 index 00000000..979b98b9 --- /dev/null +++ b/.github/workflows/tests-rsfitsio.yml @@ -0,0 +1,102 @@ +name: tests-rsfitsio + +on: + push: + branches: + - master + pull_request: null + +env: + PY_COLORS: "1" + # These compiler flags force the tests to fail if arrays are + # accessed at the C level from an unaligned location. + TEST_CFLAGS: "-fsanitize=alignment -fno-sanitize-recover=alignment" + +defaults: + run: + shell: bash -leo pipefail {0} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests-rsfitsio: + name: tests-rsfitsio + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + config: + # - { pyver: "3.10", npver: "1", ft: ""} + # - { pyver: "3.11", npver: "1.26", ft: "python-gil"} + # - { pyver: "3.12", npver: "1.26", ft: "python-gil"} + # - { pyver: "3.11", npver: "2", ft: "python-gil"} + # - { pyver: "3.12", npver: "2", ft: "python-gil"} + # - { pyver: "3.13", npver: "2", ft: "python-gil"} + - { pyver: "3.14", npver: "2", ft: "python-gil"} + - { pyver: "3.14", npver: "2", ft: "python-freethreading"} + + runs-on: ${{ matrix.os }} + env: + PIP_OPTIONS: "--no-cache-dir --no-deps --no-build-isolation -v" + + steps: + - name: free disk space + if: matrix.os == 'ubuntu-latest' + uses: endersonmenezes/free-disk-space@v3 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + rm_cmd: "rmz" + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 #v3.0.0 + with: + environment-name: fitsio-dev + environment-file: environment.yml + condarc: | + channels: + - conda-forge + show_channel_urls: true + channel_priority: strict + always_yes: true + create-args: >- + python=${{ matrix.config.pyver }} + numpy=${{ matrix.config.npver }} + ${{ matrix.config.ft }} + rust + + - name: set parallel testing flags + run: | + if [[ "${{ matrix.config.pyver }}" == "3.14" ]]; then + echo "PRP_FLAGS=--parallel-threads 4 --iterations 4" >> ${GITHUB_ENV} + fi + + - name: build rsfitsio + run: | + git clone https://github.com/cruzzil/rsfitsio.git + pushd rsfitsio + RUSTFLAGS="--sysroot=${CONDA_PREFIX}" cargo build --release + popd + + - name: build fitsio + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + export FITSIO_USE_RSFITSIO=`pwd`/rsfitsio/target/release + + pip install ${PIP_OPTIONS} -e . + + - name: test fitsio + run: | + pytest --durations=20 ${PRP_FLAGS} fitsio + python -c "import fitsio; assert fitsio.backend_has_bzip2_support()" + python -c "import fitsio; assert fitsio.backend_has_curl_support()" + python -c "import fitsio; assert not fitsio.backend_is_bundled()" + python -c "import fitsio; assert fitsio.backend_is_reentrant()" + python -c "import fitsio; assert fitsio.fitsio_backend() == 'rsfitsio'" diff --git a/.github/workflows/tests-windows.yaml b/.github/workflows/tests-windows.yaml index 10fc0ab9..09981d9b 100644 --- a/.github/workflows/tests-windows.yaml +++ b/.github/workflows/tests-windows.yaml @@ -32,13 +32,14 @@ jobs: - name: make nmake available uses: ilammy/msvc-dev-cmd@v1 - - name: install code + - name: build fitsio run: | python -m pip install --upgrade pip pip install -v -e .[dev] python -c "import fitsio" - - name: test + - name: test fitsio run: | - python -c "import fitsio; assert not fitsio.cfitsio_is_reentrant()" - pytest -vv --durations=20 --parallel-threads 4 --iterations 4 --ignore-gil-enabled fitsio + python -c "import fitsio; assert not fitsio.backend_is_reentrant()" + python -c "import fitsio; assert fitsio.fitsio_backend() == 'cfitsio'" + pytest --durations=20 --parallel-threads 4 --iterations 4 --ignore-gil-enabled fitsio diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 19a706b1..9766cb3b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,7 @@ jobs: echo "PRP_FLAGS=--parallel-threads 4 --iterations 4" >> ${GITHUB_ENV} fi - - name: test bundled build + - name: build bundled fitsio run: | export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" @@ -86,16 +86,20 @@ jobs: rm -rf $HOME/cfitsio-static-install find . -name "*.so" -type f -delete pip install ${PIP_OPTIONS} -e . + + - name: test bundled build + run: | if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - pytest -vv --durations=20 ${PRP_FLAGS} fitsio - python -c "import fitsio; assert not fitsio.cfitsio_has_bzip2_support()" - python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()" + pytest --durations=20 ${PRP_FLAGS} fitsio + python -c "import fitsio; assert not fitsio.backend_has_bzip2_support()" + python -c "import fitsio; assert not fitsio.backend_has_curl_support()" else - pytest -vv --durations=20 ${PRP_FLAGS} fitsio - python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" - python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + pytest --durations=20 ${PRP_FLAGS} fitsio + python -c "import fitsio; assert fitsio.backend_has_bzip2_support()" + python -c "import fitsio; assert fitsio.backend_has_curl_support()" fi - python -c "import fitsio; assert fitsio.cfitsio_is_reentrant()" + python -c "import fitsio; assert fitsio.backend_is_reentrant()" + python -c "import fitsio; assert fitsio.fitsio_backend() == 'cfitsio'" - name: install bzip2 and curl on linux if: matrix.os == 'ubuntu-latest' @@ -121,7 +125,7 @@ jobs: cd - pip uninstall fitsio --yes - - name: test sdist + - name: build fitsio from sdist run: | export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" @@ -132,6 +136,9 @@ jobs: rm -rf dist python setup.py sdist + + - name: test fitsio from sdist + run: | pushd dist/ fname=$(ls fitsio*.gz) @@ -140,10 +147,11 @@ jobs: pushd $dname pip install ${PIP_OPTIONS} -e . - pytest -vv --durations=20 ${PRP_FLAGS} fitsio - python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" - python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" - python -c "import fitsio; assert fitsio.cfitsio_is_reentrant()" + pytest --durations=20 ${PRP_FLAGS} fitsio + python -c "import fitsio; assert fitsio.backend_has_bzip2_support()" + python -c "import fitsio; assert fitsio.backend_has_curl_support()" + python -c "import fitsio; assert fitsio.backend_is_reentrant()" + python -c "import fitsio; assert fitsio.fitsio_backend() == 'cfitsio'" popd popd diff --git a/.gitignore b/.gitignore index 67cbbd92..697353cd 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,5 @@ prof/ .DS_Store cfitsio-*/ zlib/ + +rsfitsio/ diff --git a/CHANGES.md b/CHANGES.md index 3e9e1071..a2de396b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ # Changes +## 1.5.0 (unreleased) + +Changes + +- Added support for `rsfitsio` as a backend in addition to `cfitsio`. +- Added function `fitsio_backend` to return the current backend. +- Renamed `cfitsio_*` functions to `backend_*` functions for clarity. +- Added constant `CFITSIO_BACKEND` and `RSFITSIO_BACKEND` which are + the return values of `fitsio_backend`. + ## 1.4.3 Bug Fixes diff --git a/fitsio/__init__.py b/fitsio/__init__.py index 7c37fb3b..a84693f3 100644 --- a/fitsio/__init__.py +++ b/fitsio/__init__.py @@ -41,10 +41,23 @@ FITSRuntimeWarning, cfitsio_is_bundled, ) + +backend_version = cfitsio_version +backend_is_bundled = cfitsio_is_bundled from ._fitsio_wrap import ( cfitsio_has_bzip2_support, cfitsio_has_curl_support, cfitsio_is_reentrant, + fitsio_backend, ) +backend_has_bzip2_support = cfitsio_has_bzip2_support +backend_has_curl_support = cfitsio_has_curl_support +backend_is_reentrant = cfitsio_is_reentrant + +# return values of fitsio_backend, here to help make code +# clearer and avoid mispelling errors in testing strings +CFITSIO_BACKEND = "cfitsio" +RSFITSIO_BACKEND = "rsfitsio" + from .fits_exceptions import FITSFormatError diff --git a/fitsio/fitsio_pywrap.c b/fitsio/fitsio_pywrap.c index 82f85ec6..3e8e35e8 100644 --- a/fitsio/fitsio_pywrap.c +++ b/fitsio/fitsio_pywrap.c @@ -21,12 +21,27 @@ */ #include "fitsio.h" -#include "fitsio2.h" #include #include // #include "fitsio_pywrap_lists.h" #include +#ifdef FITSIO_BACKEND_RSFITSIO +/* random extras needed by fitsio */ +int CFITS_API fits_unset_compression_request(fitsfile *fptr, int *status); +int ffgbytoff(fitsfile *fptr, long gsize, long ngroups, long offset, + void *buffer, int *status); +int ffseek(FITSfile *fptr, LONGLONG position); +int ffread(FITSfile *fptr, long nbytes, void *buffer, int *status); +#define REPORT_EOF 0 +#define FSTRCMP(a, b) \ + ((a)[0]<(b)[0] ? -1 : (a)[0]>(b)[0] ? 1 : strcmp((a), (b))) +#endif + +#ifdef FITSIO_BACKEND_CFITSIO +#include "fitsio2.h" +#endif + // this is not defined anywhere in cfitsio except in // the fits file structure #define CFITSIO_MAX_ARRAY_DIMS 99 @@ -6347,9 +6362,11 @@ static PyObject *PyFITS_cfitsio_version(void) { static PyObject *PyFITS_cfitsio_is_bundled(void) { #ifdef FITSIO_USING_SYSTEM_FITSIO Py_RETURN_FALSE; -#else - Py_RETURN_TRUE; #endif +#ifdef FITSIO_BACKEND_RSFITSIO + Py_RETURN_FALSE; +#endif + Py_RETURN_TRUE; } static PyObject *PyFITS_cfitsio_has_bzip2_support(void) { @@ -6380,6 +6397,18 @@ static PyObject *PyFITS_cfitsio_is_reentrant(void) { } } +static PyObject *PyFITS_fitsio_backend(void) { +#ifdef FITSIO_BACKEND_CFITSIO + return PyUnicode_FromString("cfitsio"); +#endif +#ifdef FITSIO_BACKEND_RSFITSIO + return PyUnicode_FromString("rsfitsio"); +#endif + PyErr_SetString(PyExc_ValueError, + "No valid fitsio backend specified in C layer!"); + return NULL; +} + /* 'C', 'L', 'I', 'F' 'X' @@ -6685,6 +6714,9 @@ static PyMethodDef fitstype_methods[] = { METH_NOARGS, "cfitsio_is_reentrant\n\nReturn True if cfitsio was compiled with " "reentrant support."}, + {"fitsio_backend", (PyCFunction)PyFITS_fitsio_backend, METH_NOARGS, + "fitsio_backend\n\nReturn the backend FITS library (e.g., 'cfitsio', " + "'rsfitsio')."}, {"parse_card", (PyCFunction)PyFITS_parse_card, METH_VARARGS, "parse_card\n\nparse the card to get the key name, value (as a string), " "data type and comment."}, diff --git a/fitsio/tests/makedata.py b/fitsio/tests/makedata.py index 83e0394e..4c9fa5a4 100644 --- a/fitsio/tests/makedata.py +++ b/fitsio/tests/makedata.py @@ -2,9 +2,14 @@ import numpy as np from functools import lru_cache -from ..util import cfitsio_version +from .. import ( + backend_version, + fitsio_backend, + CFITSIO_BACKEND, + RSFITSIO_BACKEND, +) -CFITSIO_VERSION = cfitsio_version(asfloat=True) +BACKEND_VERSION = backend_version(asfloat=True) lorem_ipsum = ( 'Lorem ipsum dolor sit amet, consectetur adipiscing ' @@ -65,7 +70,9 @@ def make_data(): ('Sarr', Sdtype, ashape), ] - if CFITSIO_VERSION > 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: dtype += [ ('u8scalar', 'u8'), ('u8vec', 'u8', nvec), @@ -75,7 +82,9 @@ def make_data(): # cfitsio 3 or earlier does not # handle non-space padded strings # properly - if CFITSIO_VERSION >= 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: dtype += [ ('Sscalar_nopad', Sdtype), ('Svec_nopad', Sdtype, nvec), @@ -92,7 +101,9 @@ def make_data(): # cfitsio 3 or earlier does not # handle non-space padded strings # properly - if CFITSIO_VERSION >= 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: dtype += [ ('Uscalar_nopad', Udtype), ('Uvec_nopad', Udtype, nvec), @@ -121,7 +132,9 @@ def make_data(): 'c8', 'c16', ] - if CFITSIO_VERSION > 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: dtypes += ["u8"] for t in dtypes: @@ -175,7 +188,9 @@ def make_data(): # cfitsio 3 or earlier does not # handle non-space padded strings # properly - if CFITSIO_VERSION >= 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: data['Sscalar_nopad'] = ['hello', 'world', 'good', 'bye'] data['Svec_nopad'][:, 0] = 'hello' data['Svec_nopad'][:, 1] = 'world' @@ -198,7 +213,9 @@ def make_data(): # cfitsio 3 or earlier does not # handle non-space padded strings # properly - if CFITSIO_VERSION >= 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: data['Uscalar_nopad'] = ['hello', 'world', 'good', 'bye'] data['Uvec_nopad'][:, 0] = 'hello' data['Uvec_nopad'][:, 1] = 'world' @@ -343,7 +360,9 @@ def make_data(): ('Svec', Sdtype, nvec), ('Sarr', Sdtype, ashape), ] - if CFITSIO_VERSION > 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: dtype += [ ('u8vec', 'u8', nvec), ('u8arr', 'u8', ashape), @@ -362,7 +381,9 @@ def make_data(): vardata = np.zeros(nrows, dtype=dtype) _dtypes = ['u1', 'i1', 'u2', 'i2', 'u4', 'i4', 'i8', 'f4', 'f8'] - if CFITSIO_VERSION > 4: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4 + ) or fitsio_backend() == RSFITSIO_BACKEND: _dtypes += ["u8"] for t in _dtypes: vardata[t + 'scalar'] = 1 + np.arange(nrows, dtype=t) diff --git a/fitsio/tests/test_header.py b/fitsio/tests/test_header.py index b19d1694..dd355d56 100644 --- a/fitsio/tests/test_header.py +++ b/fitsio/tests/test_header.py @@ -10,9 +10,14 @@ from ..fitslib import FITS, read_header, write from ..header import FITSHDR from ..hdu.base import INVALID_HDR_CHARS -from ..util import cfitsio_version +from .. import ( + backend_version, + fitsio_backend, + CFITSIO_BACKEND, + RSFITSIO_BACKEND, +) -CFITSIO_VERSION = cfitsio_version(asfloat=True) +BACKEND_VERSION = backend_version(asfloat=True) def test_free_form_string(): @@ -106,7 +111,9 @@ def test_header_write_read(): 'unders': '1_000_000', # test string with underscore 'longs': lorem_ipsum, } - if CFITSIO_VERSION > 4.02: + if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 4.02 + ) or fitsio_backend() == RSFITSIO_BACKEND: # force hierarch + continue header["long_keyword_name"] = lorem_ipsum diff --git a/fitsio/tests/test_image.py b/fitsio/tests/test_image.py index 6eed47b9..2090d4fa 100644 --- a/fitsio/tests/test_image.py +++ b/fitsio/tests/test_image.py @@ -5,13 +5,21 @@ # import warnings from .checks import check_header, compare_array -from ..util import cfitsio_version, cfitsio_is_bundled +from .. import ( + backend_version, + backend_is_bundled, + fitsio_backend, + CFITSIO_BACKEND, + RSFITSIO_BACKEND, +) import numpy as np from ..fitslib import FITS -CFITSIO_VERSION = cfitsio_version(asfloat=True) +BACKEND_VERSION = backend_version(asfloat=True) DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] -if CFITSIO_VERSION > 3.44: +if ( + BACKEND_VERSION > 3.44 and fitsio_backend() == CFITSIO_BACKEND +) or fitsio_backend() == RSFITSIO_BACKEND: DTYPES += ["u8"] @@ -74,9 +82,9 @@ def test_image_write_read_unaligned(dtype, with_nan): by hand to fix a bug. """ - if ( - dtype == ">f4" or ("f" in dtype and with_nan) - ) and not cfitsio_is_bundled(): + if (dtype == ">f4" or ("f" in dtype and with_nan)) and ( + fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled() + ): pytest.xfail( reason=( "Non-bundled cfitsio libraries have a bug for " @@ -561,7 +569,7 @@ def test_image_write_subset_2d( with_nan and with_nan_base_img and partial_overlap_str in partial_overlap_str_cases - and not cfitsio_is_bundled() + and (fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled()) and compress_kws and compress_kws.get("qlevel", 0) > 0 ): @@ -785,7 +793,7 @@ def test_image_read_write_ulonglong(): with FITS(fname, 'rw') as fits: data = np.arange(5 * 20, dtype='u8').reshape(5, 20) header = {'DTYPE': 'u8', 'NBYTES': data.dtype.itemsize} - if CFITSIO_VERSION < 3.45: + if BACKEND_VERSION < 3.45 and fitsio_backend() == CFITSIO_BACKEND: with pytest.raises(TypeError) as e: fits.write_image(data, header=header) assert ( @@ -801,6 +809,8 @@ def test_image_read_write_ulonglong(): rh = fits[-1].read_header() check_header(header, rh) - if CFITSIO_VERSION >= 3.45: + if ( + BACKEND_VERSION >= 3.45 and fitsio_backend() == CFITSIO_BACKEND + ) or fitsio_backend() == RSFITSIO_BACKEND: with FITS(fname) as fits: assert not fits[0].is_compressed(), 'not compressed' diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 5ca191e5..9bf21bb7 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -14,14 +14,11 @@ write, RICE_1, SUBTRACTIVE_DITHER_1, - GZIP_1, - GZIP_2, - PLIO_1, - HCOMPRESS_1, ) -from ..util import cfitsio_is_bundled, cfitsio_version +from .. import backend_is_bundled, backend_version +from .. import fitsio_backend, CFITSIO_BACKEND -CFITSIO_VERSION = cfitsio_version(asfloat=True) +BACKEND_VERSION = backend_version(asfloat=True) @pytest.mark.parametrize("with_nan", [False, True]) @@ -125,11 +122,10 @@ def test_compressed_write_read_fitsobj(compress, dtype, with_nan): In this version, keep the fits object open """ - if ( "gzip" in compress and dtype in ["u2", "i2", "u4", "i4"] - and not cfitsio_is_bundled() + and (fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled()) ): pytest.xfail( reason=( @@ -205,7 +201,10 @@ def test_compressed_write_read_fitsobj(compress, dtype, with_nan): @pytest.mark.skipif(sys.version_info < (3, 9), reason='importlib bug in 3.8') -@pytest.mark.skipif(CFITSIO_VERSION < 3.49, reason='bug in cfitsio < 3.49') +@pytest.mark.skipif( + BACKEND_VERSION < 3.49 and fitsio_backend() == CFITSIO_BACKEND, + reason='bug in cfitsio < 3.49', +) def test_gzip_tile_compressed_read_lossless_astropy(): """ Test reading an image gzip compressed by astropy (fixed by cfitsio 3.49) @@ -516,11 +515,11 @@ def test_image_compression_raises_on_python_set(kw, val, set_val_to_none): @pytest.mark.parametrize( "compress", [ - RICE_1, - GZIP_1, - GZIP_2, - PLIO_1, - HCOMPRESS_1, + "rice", + "gzip", + "gzip_2", + "plio", + "hcompress", ], ) @pytest.mark.parametrize( @@ -536,7 +535,7 @@ def test_image_compression_raises_on_python_set(kw, val, set_val_to_none): ) @pytest.mark.parametrize("fname", ["mem://", "test.fits"]) def test_image_compression_inmem_lossess_int(compress, dtype, fname): - if not cfitsio_is_bundled(): + if fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled(): pytest.xfail( reason=( "Non-bundled cfitsio libraries have a bug. " @@ -544,7 +543,7 @@ def test_image_compression_inmem_lossess_int(compress, dtype, fname): "and https://github.com/HEASARC/cfitsio/pull/99." ), ) - if compress == PLIO_1 and dtype in [np.int16, np.uint32, np.int32]: + if compress == "plio" and dtype in [np.int16, np.uint32, np.int32]: pytest.skip( reason="PLIO lossless compression of int16, uint32, and " "int32 types is not supported by cfitsio", @@ -629,10 +628,12 @@ def test_image_mem_reopen_noop(): 2, marks=[ pytest.mark.xfail( - condition=CFITSIO_VERSION < 4.04, + condition=BACKEND_VERSION < 4.04 + and fitsio_backend() == CFITSIO_BACKEND + and fitsio_backend() == CFITSIO_BACKEND, reason=( "Writing compressed binary tables exceeding " - "2**32 bytes fails for cfitsio < 4.040!" + "2**32 bytes fails for cfitsio < 4.04!" ), ), pytest.mark.slow, @@ -800,7 +801,10 @@ def test_image_compression_nulls_patches_with_subnormal( # however, on read, we send nullcheck=NAN and as a side # effect the subnormal float value in this row gets truncated # to zero - if not cfitsio_is_bundled(): + if ( + fitsio_backend() == CFITSIO_BACKEND + and not backend_is_bundled() + ): np.testing.assert_array_equal(read_data[1, 1:], 0.0) np.testing.assert_array_equal(read_slice1[1:], 0.0) @@ -852,7 +856,9 @@ def test_image_compression_read_chunks(): @pytest.mark.xfail( - condition=not cfitsio_is_bundled(), + condition=( + fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled() + ), reason=( "system cfitsio must be compiled without FMA instructions to " "enable reproducible lossy float compression" @@ -885,7 +891,9 @@ def test_image_compression_write_read_comp_to_osx_arm64(): @pytest.mark.xfail( - condition=not cfitsio_is_bundled(), + condition=( + fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled() + ), reason=( "system cfitsio must be compiled without FMA instructions to " "enable reproducible lossy float compression" @@ -925,12 +933,31 @@ def test_image_compression_gzip_subnormal_cast_to_zero(): ) back = read(fn) - if cfitsio_is_bundled(): + if fitsio_backend() == CFITSIO_BACKEND and backend_is_bundled(): assert not back.ravel()[0] == 0, back.ravel() else: assert back.ravel()[0] == 0, back.ravel() +def test_image_compression_simple(): + data = np.arange(10).reshape(5, 2).astype(np.float32) + + fname = "test.fits" + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(data, compress="RICE", qlevel=1, dither_seed=10) + + with FITS(fpth, "r") as fits: + rdata = fits[0].read() + + assert not np.array_equal(data, rdata) + + if __name__ == '__main__': test_compressed_seed( compress='rice', diff --git a/fitsio/tests/test_image_compression_defaults.py b/fitsio/tests/test_image_compression_defaults.py index c927be3e..5bdd79f2 100644 --- a/fitsio/tests/test_image_compression_defaults.py +++ b/fitsio/tests/test_image_compression_defaults.py @@ -157,7 +157,7 @@ def test_compression_qlevel_default(): bigimg = np.random.uniform(size=(H, W)) # Default qlevel with fitsio.FITS(fn, 'rw', clobber=True) as fits: - fits.write(bigimg, compress='GZIP') + fits.write(bigimg, compress='GZIP', dither_seed=42) size_def = os.stat(fn).st_size hdr = fitsio.read_header(fn, ext=1) print(hdr) @@ -168,15 +168,15 @@ def test_compression_qlevel_default(): assert hdr[key] == val # qlevel=0 with fitsio.FITS(fn, 'rw', clobber=True) as fits: - fits.write(bigimg, compress='GZIP', qlevel=0) + fits.write(bigimg, compress='GZIP', qlevel=0, dither_seed=42) size_0 = os.stat(fn).st_size # qlevel=4 with fitsio.FITS(fn, 'rw', clobber=True) as fits: - fits.write(bigimg, compress='GZIP', qlevel=4) + fits.write(bigimg, compress='GZIP', qlevel=4, dither_seed=42) size_4 = os.stat(fn).st_size # qlevel=16 with fitsio.FITS(fn, 'rw', clobber=True) as fits: - fits.write(bigimg, compress='GZIP', qlevel=16) + fits.write(bigimg, compress='GZIP', qlevel=16, dither_seed=42) size_16 = os.stat(fn).st_size # zero means NO COMPRESSION assert size_0 > size_4 diff --git a/fitsio/tests/test_segfault.py b/fitsio/tests/test_segfault.py index 21522547..491e3014 100644 --- a/fitsio/tests/test_segfault.py +++ b/fitsio/tests/test_segfault.py @@ -4,6 +4,7 @@ import numpy as np import fitsio +from .. import fitsio_backend, backend_is_bundled, CFITSIO_BACKEND import pytest @@ -53,7 +54,7 @@ def _run_mixed(n): @pytest.mark.slow @pytest.mark.skipif( - not fitsio.util.cfitsio_is_bundled(), + (fitsio_backend() == CFITSIO_BACKEND and not backend_is_bundled()), reason=( "small images cause a memory corruption w/ PLIO " "compression (see https://github.com/heasarc/cfitsio/issues/136)" diff --git a/fitsio/tests/test_table.py b/fitsio/tests/test_table.py index c2ef84f8..a94976f7 100644 --- a/fitsio/tests/test_table.py +++ b/fitsio/tests/test_table.py @@ -15,11 +15,19 @@ from .makedata import make_data from ..fitslib import FITS, write, read from .. import util -from .. import cfitsio_has_bzip2_support +from .. import ( + backend_has_bzip2_support, + fitsio_backend, + backend_version, + CFITSIO_BACKEND, + RSFITSIO_BACKEND, +) -CFITSIO_VERSION = util.cfitsio_version(asfloat=True) +BACKEND_VERSION = backend_version(asfloat=True) DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] -if CFITSIO_VERSION > 4: +if ( + BACKEND_VERSION > 4 and fitsio_backend() == CFITSIO_BACKEND +) or fitsio_backend() == RSFITSIO_BACKEND: DTYPES += ["u8"] @@ -570,6 +578,13 @@ def test_ascii_table_write_read(): Test write and read for an ascii table """ + if fitsio_backend() == RSFITSIO_BACKEND: + tol = 1e-14 + elif fitsio_backend() == CFITSIO_BACKEND: + tol = 2.15e-16 + else: + assert False, "No valid backend specified! got " + fitsio_backend() + adata = make_data() ascii_data = adata['ascii_data'] @@ -590,9 +605,10 @@ def test_ascii_table_write_read(): d = fits[1].read_column(f) if d.dtype == np.float64: # note we should be able to do 1.11e-16 in principle, but - # in practice we get more like 2.15e-16 + # in practice we get more like 2.15e-16 for cfitsio and + # closer to 6e-16 for rsfitsio compare_array_tol( - ascii_data[f], d, 2.15e-16, "table field read '%s'" % f + ascii_data[f], d, tol, "table field read '%s'" % f ) else: compare_array( @@ -606,7 +622,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f][rows], d, - 2.15e-16, + tol, "table field read subrows '%s'" % f, ) else: @@ -624,7 +640,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f][beg:end], d, - 2.15e-16, + tol, "table field read slice '%s'" % f, ) else: @@ -643,7 +659,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f], d, - 2.15e-16, + tol, "table subcol, '%s'" % f, ) else: @@ -658,7 +674,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f], d, - 2.15e-16, + tol, "table subcol, '%s'" % f, ) else: @@ -675,7 +691,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f][rows], d, - 2.15e-16, + tol, "table subcol, '%s'" % f, ) else: @@ -692,7 +708,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f][rows], d, - 2.15e-16, + tol, "table subcol/row, '%s'" % f, ) else: @@ -710,7 +726,7 @@ def test_ascii_table_write_read(): compare_array_tol( ascii_data[f][beg:end], d, - 2.15e-16, + tol, "table subcol/slice, '%s'" % f, ) else: @@ -888,7 +904,9 @@ def test_table_resize(): add_data['u4scalar'] = 2**31 add_data['u4vec'] = 2**31 add_data['u4arr'] = 2**31 - if CFITSIO_VERSION > 4: + if ( + BACKEND_VERSION > 4 and fitsio_backend() == CFITSIO_BACKEND + ) or fitsio_backend() == RSFITSIO_BACKEND: add_data['u8scalar'] = 2**63 add_data['u8vec'] = 2**63 add_data['u8arr'] = 2**63 @@ -1097,7 +1115,7 @@ def test_gz_write_read(): @pytest.mark.skipif( - not cfitsio_has_bzip2_support(), + not backend_has_bzip2_support(), reason='cfitsio was not built with bzip2 support', ) def test_bz2_read(): @@ -1116,29 +1134,29 @@ def test_bz2_read(): bzfname = fname + '.bz2' try: - fits = FITS(fname, 'rw') - fits.write_table(data, header=adata['keys'], extname='mytable') - fits.close() + with FITS(fname, 'rw') as fits: + fits.write_table(data, header=adata['keys'], extname='mytable') os.system('bzip2 %s' % fname) - f2 = FITS(bzfname) - d = f2[1].read() - compare_rec(data, d, "bzip2 read") - - h = f2[1].read_header() - for entry in adata['keys']: - name = entry['name'].upper() - value = entry['value'] - hvalue = h[name] - if isinstance(hvalue, str): - hvalue = hvalue.strip() - - assert value == hvalue, "testing header key '%s'" % name - - if 'comment' in entry: - assert ( - entry['comment'].strip() == h.get_comment(name).strip() - ), "testing comment for header key '%s'" % name + with FITS(bzfname) as f2: + d = f2[1].read() + compare_rec(data, d, "bzip2 read") + + h = f2[1].read_header() + for entry in adata['keys']: + name = entry['name'].upper() + value = entry['value'] + hvalue = h[name] + if isinstance(hvalue, str): + hvalue = hvalue.strip() + + assert value == hvalue, "testing header key '%s'" % name + + if 'comment' in entry: + assert ( + entry['comment'].strip() + == h.get_comment(name).strip() + ), "testing comment for header key '%s'" % name except Exception: import traceback @@ -1570,7 +1588,9 @@ def test_table_big_col(table_type): pth = os.path.join(tmpdir, "test.fits") # v3 cfitsio that is not bundled fails for big # columns - if table_type == "ascii" or CFITSIO_VERSION < 4: + if table_type == "ascii" or ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION < 4 + ): with pytest.raises(OSError) as e: write(pth, d, table_type=table_type) assert "FITSIO status = 236: column exceeds width of table" in str( @@ -1586,7 +1606,7 @@ def test_table_big_col(table_type): @pytest.mark.xfail( - condition=CFITSIO_VERSION < 4, + condition=fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION < 4, reason=( "cfitsio versions < 4 do not easily support null-terminated strings" ), @@ -1617,7 +1637,7 @@ def test_table_read_write_ulonglong(): fname = os.path.join(tmpdir, 'test.fits') with FITS(fname, 'rw') as fits: - if CFITSIO_VERSION < 3.45: + if fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION < 3.45: with pytest.raises(IOError) as e: fits.write_table( adata, diff --git a/fitsio/tests/test_threading.py b/fitsio/tests/test_threading.py index a7a7dd97..7c982fa5 100644 --- a/fitsio/tests/test_threading.py +++ b/fitsio/tests/test_threading.py @@ -1,5 +1,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import os +import sys import tempfile import time @@ -61,7 +62,10 @@ def _read_file(i): pass -@pytest.mark.xfail(reason="Threading performance might be flaky!") +@pytest.mark.xfail( + reason="threading performance might be flaky", + condition=sys.version_info < (3, 13) or not fitsio.backend_is_reentrant(), +) @pytest.mark.parallel_threads_limit(1) @pytest.mark.iterations(1) @pytest.mark.parametrize( @@ -104,50 +108,61 @@ def _remove_files(): for fname in filenames: create_file(fname) - t0 = time.time() - if not read_only: - create_file(filenames[0]) - if not write_only: - read_file(filenames[0]) - t0_one = time.time() - t0 - print("one file time:", t0_one, flush=True) - if not read_only: - _remove_files() + n_trials = 3 + t0_threads = 0 + t0_one = 0 + t0_serial = 0 + + for _ in range(n_trials): + t0 = time.time() + if not read_only: + create_file(filenames[0]) + if not write_only: + read_file(filenames[0]) + t0_one += time.time() - t0 + if not read_only: + _remove_files() + + t0 = time.time() + with klass(max_workers=nt) as pool: + if not read_only: + futs = [ + pool.submit(create_file, filenames[i]) + for i in range(nt * fac) + ] + for fut in as_completed(futs): + fut.result() + if not write_only: + futs = [ + pool.submit(read_file, filenames[i]) + for i in range(nt * fac) + ] + for fut in as_completed(futs): + fut.result() + t0_threads += time.time() - t0 + if not read_only: + _remove_files() - t0 = time.time() - with klass(max_workers=nt) as pool: + t0 = time.time() if not read_only: - futs = [ - pool.submit(create_file, filenames[i]) - for i in range(nt * fac) - ] - for fut in as_completed(futs): - fut.result() + for fname in filenames: + create_file(fname) if not write_only: - futs = [ - pool.submit(read_file, filenames[i]) - for i in range(nt * fac) - ] - for fut in as_completed(futs): - fut.result() - t0_threads = time.time() - t0 + for fname in filenames: + read_file(fname) + t0_serial += time.time() - t0 + + t0_one /= n_trials + t0_serial /= n_trials + t0_threads /= n_trials + + print("one file time:", t0_one, flush=True) print( "parallel time / one file time", t0_threads / t0_one, "(perfect is %d)" % fac, flush=True, ) - if not read_only: - _remove_files() - - t0 = time.time() - if not read_only: - for fname in filenames: - create_file(fname) - if not write_only: - for fname in filenames: - read_file(fname) - t0_serial = time.time() - t0 print( "serial time / one file time:", t0_serial / t0_one, @@ -155,13 +170,16 @@ def _remove_files(): flush=True, ) - assert t0_threads < t0_serial / 1.5, ( + assert t0_threads < t0_serial, ( "Threading should be faster than serial! (%f < %f)" % (t0_threads, t0_serial) ) -@pytest.mark.xfail(reason="Threading performance might be flaky!") +@pytest.mark.xfail( + reason="threading performance might be flaky", + condition=sys.version_info < (3, 13) or not fitsio.backend_is_reentrant(), +) @pytest.mark.parallel_threads_limit(1) @pytest.mark.iterations(1) def test_threading_read_one_file(): @@ -179,28 +197,39 @@ def _read_file(fname): assert (fits[1].read() == -1).all() return True - t0 = time.time() - _read_file(fname) - t0_one = time.time() - t0 - print("\none file time:", t0_one, flush=True) + n_trials = 3 + t0_threads = 0 + t0_one = 0 + t0_serial = 0 - t0 = time.time() - with ThreadPoolExecutor(max_workers=nt) as pool: - futs = [pool.submit(_read_file, fname) for _ in range(nt)] + for _ in range(n_trials): + t0 = time.time() + _read_file(fname) + t0_one += time.time() - t0 + + t0 = time.time() + with ThreadPoolExecutor(max_workers=nt) as pool: + futs = [pool.submit(_read_file, fname) for _ in range(nt)] - assert all([fut.result() for fut in as_completed(futs)]) - t0_threads = time.time() - t0 + assert all([fut.result() for fut in as_completed(futs)]) + t0_threads += time.time() - t0 + + t0 = time.time() + for _ in range(nt): + _read_file(fname) + t0_serial += time.time() - t0 + + t0_one /= n_trials + t0_serial /= n_trials + t0_threads /= n_trials + + print("\none file time:", t0_one, flush=True) print( "parallel time / one file time", t0_threads / t0_one, "(perfect is 1)", flush=True, ) - - t0 = time.time() - for _ in range(nt): - _read_file(fname) - t0_serial = time.time() - t0 print( "serial time / one file time:", t0_serial / t0_one, @@ -208,7 +237,7 @@ def _read_file(fname): flush=True, ) - assert t0_threads < t0_serial / 2, ( + assert t0_threads < t0_serial, ( "Threading should be faster than serial! (%f < %f)" % (t0_threads, t0_serial) ) diff --git a/fitsio/tests/test_util.py b/fitsio/tests/test_util.py index 7fd76b91..3bb651cc 100644 --- a/fitsio/tests/test_util.py +++ b/fitsio/tests/test_util.py @@ -5,13 +5,20 @@ from ..util import ( _nonfinite_as_cfitsio_floating_null_value, - cfitsio_version, _FLOATING_NULL_VALUE, ) +from .. import ( + fitsio_backend, + backend_version, + CFITSIO_BACKEND, + RSFITSIO_BACKEND, +) -CFITSIO_VERSION = cfitsio_version(asfloat=True) +BACKEND_VERSION = backend_version(asfloat=True) DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] -if CFITSIO_VERSION > 3.44: +if ( + fitsio_backend() == CFITSIO_BACKEND and BACKEND_VERSION > 3.44 +) or fitsio_backend() == RSFITSIO_BACKEND: DTYPES += ["u8"] diff --git a/setup.py b/setup.py index 78fb7aca..33c23afc 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,25 @@ else: FITSIO_FAIL_ON_BAD_PATCHES = True +if "--use-rsfitsio" in sys.argv or any( + a.startswith("--use-rsfitsio=") for a in sys.argv +): + if "--use-rsfitsio" in sys.argv: + USE_RSFITSIO = True + ind = sys.argv.index("--use-rsfitsio") + del sys.argv[ind] + else: + for ind in range(len(sys.argv)): + if sys.argv[ind].startswith("--use-rsfitsio="): + break + USE_RSFITSIO = sys.argv[ind].split("=", 1)[1] + del sys.argv[ind] +else: + USE_RSFITSIO = os.environ.get( + "FITSIO_USE_RSFITSIO", + False, + ) + if "--use-system-fitsio" in sys.argv: del sys.argv[sys.argv.index("--use-system-fitsio")] USE_SYSTEM_FITSIO = True @@ -150,6 +169,12 @@ def finalize_options(self): for pth in SYSTEM_FITSIO_LIBDIR.split(os.pathsep): _print_msg(f"Adding lib directory '{pth}'") self.library_dirs.insert(0, pth) + elif USE_RSFITSIO: + if isinstance(USE_RSFITSIO, str): + self.include_dirs.insert( + 0, os.path.join(USE_RSFITSIO, "..", "..", "c") + ) + self.library_dirs.insert(0, USE_RSFITSIO) else: if os.name == "nt": self.include_dirs.insert( @@ -177,7 +202,21 @@ def run(self): build_ext.run(self) def build_extensions(self): - if not USE_SYSTEM_FITSIO: + if not USE_RSFITSIO: + self.compiler.define_macro('FITSIO_BACKEND_CFITSIO') + else: + self.compiler.define_macro('FITSIO_BACKEND_RSFITSIO') + + if USE_RSFITSIO: + if isinstance(USE_RSFITSIO, str): + self.compiler.include_dirs.insert( + 0, os.path.join(USE_RSFITSIO, "..", "..", "c") + ) + self.compiler.library_dirs.insert(0, USE_RSFITSIO) + self.compiler.add_library('rsfitsio') + self.compiler.define_macro('FITSIO_HAS_CURL_SUPPORT') + self.compiler.define_macro('FITSIO_HAS_BZIP2_SUPPORT') + elif not USE_SYSTEM_FITSIO: # Use the compiler for building python to build cfitsio # for maximized compatibility. if os.name != "nt": @@ -220,8 +259,8 @@ def build_extensions(self): self.compiler.add_library('z') # fitsio requires libm as well, but do not need to link it - # explicitly on windows - if os.name != "nt": + # explicitly on windows or when using rust + if os.name != "nt" and not USE_RSFITSIO: self.compiler.add_library('m') # call the original build_extensions @@ -581,7 +620,16 @@ def check_system_cfitsio_objects(self, obj_name): sources = ["fitsio/fitsio_pywrap.c"] -ext = Extension("fitsio._fitsio_wrap", sources, include_dirs=['numpy']) +kwargs = {} +if USE_RSFITSIO and isinstance(USE_RSFITSIO, str): + kwargs["extra_link_args"] = [ + "-Wl,-rpath," + USE_RSFITSIO, + "-L" + USE_RSFITSIO, + ] + +ext = Extension( + "fitsio._fitsio_wrap", sources, include_dirs=['numpy'], **kwargs +) setup( ext_modules=[ext],