diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6b7e6bb7..1e2735b9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,8 +20,12 @@ jobs: fail-fast: false matrix: os: [macos-latest, ubuntu-latest] - pyver: ["3.12", "3.11"] - npver: ["1.26", "2.0"] + config: + - { pyver: "3.11", npver: "1.26"} + - { pyver: "3.12", npver: "1.26"} + - { pyver: "3.11", npver: "2.2.0"} + - { pyver: "3.12", npver: "2.2.0"} + - { pyver: "3.13", npver: "2.2.0"} runs-on: ${{ matrix.os }} @@ -40,10 +44,7 @@ jobs: shell: bash -l {0} run: | conda list - mamba install numpy=${{ matrix.npver }} nose cython wget make pytest flake8 - if [ "${{ matrix.pyver }}" == "3.8" ] || [ "${{ matrix.pyver }}" == "3.9" ] || [ "${{ matrix.pyver }}" == "3.10" ]; then - mamba install importlib_resources - fi + conda install numpy=${{ matrix.config.npver }} nose cython wget make pytest flake8 setuptools - name: install bzip2 on linux shell: bash -l {0} diff --git a/CHANGES.md b/CHANGES.md index 2bdaa3ca..2e89161d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ version 1.2.6 (not yet released) Bug Fixes - Fix bug parsing header cards with free-form strings + - Fix bug for numpy 2 that did not support writing length-1 vector + columns for strings. A side effect of this is that for + numpy 2 we can round-trip the shape for these columns, unlike + for numbers. version 1.2.5 ------------- diff --git a/fitsio/__init__.py b/fitsio/__init__.py index 54ae40b7..3ec3f105 100644 --- a/fitsio/__init__.py +++ b/fitsio/__init__.py @@ -5,7 +5,7 @@ usage. """ -__version__ = '1.2.5' +__version__ = '1.2.6' from . import fitslib diff --git a/fitsio/hdu/table.py b/fitsio/hdu/table.py index dc6eea87..fc930b60 100644 --- a/fitsio/hdu/table.py +++ b/fitsio/hdu/table.py @@ -368,17 +368,24 @@ def _verify_column_data(self, colnum, data): verify the input data is of the correct type and shape """ this_dt = data.dtype.descr[0] + npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) + is_string = npy_type[0] in ('S', 'U') if len(data.shape) > 2: this_shape = data.shape[1:] - elif len(data.shape) == 2 and data.shape[1] > 1: + elif len(data.shape) == 2 and (data.shape[1] > 1 or is_string): + # strings are special case for vector size 1, because they are + # always represented as vectors, due to the need to include the + # string length in the definition. This means a 1-d vector column + # can be written with TDIM with length 2, which means we can ensure + # the shape is compatible on read, unlike for numbers for which the + # TDIM would have a length of 1, which is illegal. this_shape = data.shape[1:] else: this_shape = () this_npy_type = this_dt[1][1:] - npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) info = self._info['colinfo'][colnum] if npy_type[0] in ['>', '<', '|']: @@ -387,7 +394,9 @@ def _verify_column_data(self, colnum, data): col_name = info['name'] col_tdim = info['tdim'] col_shape = _tdim2shape( - col_tdim, col_name, is_string=(npy_type[0] == 'S')) + col_tdim, col_name, + is_string=is_string + ) if col_shape is None: if this_shape == (): @@ -1242,45 +1251,6 @@ def _fix_tbit_dtype(self, array, colnums): return array.view(descr) - def _get_simple_dtype_and_shape(self, colnum, rows=None): - """ - When reading a single column, we want the basic data - type and the shape of the array. - - for scalar columns, shape is just nrows, otherwise - it is (nrows, dim1, dim2) - - Note if rows= is sent and only a single row is requested, - the shape will be (dim2,dim2) - """ - - # basic datatype - npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) - info = self._info['colinfo'][colnum] - name = info['name'] - - if rows is None: - nrows = self._info['nrows'] - else: - nrows = rows.size - - shape = None - tdim = info['tdim'] - - shape = _tdim2shape(tdim, name, is_string=(npy_type[0] == 'S')) - if shape is not None: - if nrows > 1: - if not isinstance(shape, tuple): - # vector - shape = (nrows, shape) - else: - # multi-dimensional - shape = tuple([nrows] + list(shape)) - else: - # scalar - shape = nrows - return npy_type, shape - def get_rec_column_descr(self, colnum, vstorage): """ Get a descriptor entry for the specified column. @@ -1333,7 +1303,8 @@ def get_rec_column_descr(self, colnum, vstorage): tdim = self._info['colinfo'][colnum]['tdim'] shape = _tdim2shape( tdim, name, - is_string=(npy_type[0] == 'S' or npy_type[0] == 'U')) + is_string=(npy_type[0] in ('S', 'U')) + ) if shape is not None: descr = (name, npy_type, shape) else: diff --git a/fitsio/tests/test_table.py b/fitsio/tests/test_table.py index 516fb2b3..2c984ef5 100644 --- a/fitsio/tests/test_table.py +++ b/fitsio/tests/test_table.py @@ -18,6 +18,11 @@ DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] +if np.lib.NumpyVersion(np.__version__) >= "2.0.0": + IS_NP2 = True +else: + IS_NP2 = False + def test_table_read_write(): @@ -86,6 +91,133 @@ def test_table_read_write(): ) +@pytest.mark.parametrize('nvec', [2, 1]) +def test_table_read_write_vec1(nvec): + """ + ensure the data for vec length 1 gets round-tripped, even though + the shape is not preserved + """ + dtype = [('x', 'f4', (nvec,))] + num = 10 + data = np.zeros(num, dtype=dtype) + data['x'] = np.arange(num * nvec).reshape(num, nvec) + assert data['x'].shape == (num, nvec) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data) + + d = fits[1].read() + if nvec == 1: + assert d['x'].shape == (num,) + compare_array( + data['x'].ravel(), d['x'].ravel(), + "table single field read 'x'" + ) + + # see if our convenience functions are working + write( + fname, + data, + extname="newext", + ) + d = read(fname, ext='newext') + if nvec == 1: + assert d['x'].shape == (num,) + compare_array(data['x'].ravel(), d['x'].ravel(), "table data2") + + # now test read_column + with FITS(fname) as fits: + + d = fits[1].read_column('x') + if nvec == 1: + assert d.shape == (num,) + compare_array( + data['x'].ravel(), d.ravel(), + "table single field read 'x'" + ) + + +@pytest.mark.parametrize('nvec', [2, 1]) +def test_table_read_write_uvec1(nvec): + """ + ensure the data for U string vec length 1 gets round-tripped, even though + the shape is not preserved. Also test 2 for consistency + """ + + dtype = [('string', 'U10', (nvec,))] + num = 10 + data = np.zeros(num, dtype=dtype) + sravel = data['string'].ravel() + sravel[:] = ['%-10s' % i for i in range(num * nvec)] + assert data['string'].shape == (num, nvec) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data) + + d = fits[1].read() + + if IS_NP2: + # We can only get the right shape back for numpy 2 + compare_array( + data['string'], d['string'], + "table single field read 'string'" + ) + else: + # for numpy 1.X we can't get the right shape for + # length 1 vector, need to flatted for comparision + if nvec == 1: + assert d['string'].shape == (num,) + + compare_array( + data['string'].ravel(), d['string'].ravel(), + "table single field read 'string'" + ) + + # see if our convenience functions are working + write( + fname, + data, + extname="newext", + ) + d = read(fname, ext='newext') + + if IS_NP2: + compare_array( + data['string'], d['string'], + "table single field read 'string'" + ) + else: + if nvec == 1: + assert d['string'].shape == (num,) + compare_array( + data['string'].ravel(), d['string'].ravel(), "table data2", + ) + + # now test read_column + with FITS(fname) as fits: + + d = fits[1].read_column('string') + + if IS_NP2: + compare_array( + data['string'], d, + "table single field read 'string'" + ) + else: + if nvec == 1: + assert d.shape == (num,) + compare_array( + data['string'].ravel(), d.ravel(), + "table single field read 'string'" + ) + + def test_table_column_index_scalar(): """ Test a basic table write, data and a header, then reading back in to diff --git a/setup.py b/setup.py index 56d1944b..7de67a7b 100644 --- a/setup.py +++ b/setup.py @@ -318,7 +318,7 @@ def check_system_cfitsio_objects(self, obj_name): setup( name="fitsio", - version="1.2.5", + version="1.2.6", description=description, long_description=long_description, long_description_content_type='text/markdown; charset=UTF-8; variant=GFM',