Make sure string column with vec size 1 work

.github/workflows/tests.yml

@@ -20,8 +20,12 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest]
-        pyver: ["3.12", "3.11"]
-        npver: ["1.26", "2.0"]
+        config:
+            - { pyver: "3.11", npver: "1.26"}
+            - { pyver: "3.12", npver: "1.26"}
+            - { pyver: "3.11", npver: "2.2.0"}
+            - { pyver: "3.12", npver: "2.2.0"}
+            - { pyver: "3.13", npver: "2.2.0"}
 
     runs-on: ${{ matrix.os }}
 
@@ -40,10 +44,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda list
-          mamba install numpy=${{ matrix.npver }} nose cython wget make pytest flake8
-          if [ "${{ matrix.pyver }}" == "3.8" ] || [ "${{ matrix.pyver }}" == "3.9" ] || [ "${{ matrix.pyver }}" == "3.10" ]; then
-            mamba install importlib_resources
-          fi
+          conda install numpy=${{ matrix.config.npver }} nose cython wget make pytest flake8 setuptools
 
       - name: install bzip2 on linux
         shell: bash -l {0}

CHANGES.md

@@ -4,6 +4,10 @@ version 1.2.6 (not yet released)
 Bug Fixes
 
     - Fix bug parsing header cards with free-form strings
+    - Fix bug for numpy 2 that did not support writing length-1 vector
+      columns for strings.  A side effect of this is that for
+      numpy 2 we can round-trip the shape for these columns, unlike
+      for numbers.
 
 version 1.2.5
 -------------

fitsio/__init__.py

@@ -5,7 +5,7 @@
 usage.
 """
 
-__version__ = '1.2.5'
+__version__ = '1.2.6'
 
 from . import fitslib
 

fitsio/hdu/table.py

@@ -368,17 +368,24 @@ def _verify_column_data(self, colnum, data):
         verify the input data is of the correct type and shape
         """
         this_dt = data.dtype.descr[0]
+        npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum)
+        is_string = npy_type[0] in ('S', 'U')
 
         if len(data.shape) > 2:
             this_shape = data.shape[1:]
-        elif len(data.shape) == 2 and data.shape[1] > 1:
+        elif len(data.shape) == 2 and (data.shape[1] > 1 or is_string):
+            # strings are special case for vector size 1, because they are
+            # always represented as vectors, due to the need to include the
+            # string length in the definition.  This means a 1-d vector column
+            # can be written with TDIM with length 2, which means we can ensure
+            # the shape is compatible on read, unlike for numbers for which the
+            # TDIM would have a length of 1, which is illegal.
             this_shape = data.shape[1:]
         else:
             this_shape = ()
 
         this_npy_type = this_dt[1][1:]
 
-        npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum)
         info = self._info['colinfo'][colnum]
 
         if npy_type[0] in ['>', '<', '|']:
@@ -387,7 +394,9 @@ def _verify_column_data(self, colnum, data):
         col_name = info['name']
         col_tdim = info['tdim']
         col_shape = _tdim2shape(
-            col_tdim, col_name, is_string=(npy_type[0] == 'S'))
+            col_tdim, col_name,
+            is_string=is_string
+        )
 
         if col_shape is None:
             if this_shape == ():
@@ -1242,45 +1251,6 @@ def _fix_tbit_dtype(self, array, colnums):
 
         return array.view(descr)
 
-    def _get_simple_dtype_and_shape(self, colnum, rows=None):
-        """
-        When reading a single column, we want the basic data
-        type and the shape of the array.
-
-        for scalar columns, shape is just nrows, otherwise
-        it is (nrows, dim1, dim2)
-
-        Note if rows= is sent and only a single row is requested,
-        the shape will be (dim2,dim2)
-        """
-
-        # basic datatype
-        npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum)
-        info = self._info['colinfo'][colnum]
-        name = info['name']
-
-        if rows is None:
-            nrows = self._info['nrows']
-        else:
-            nrows = rows.size
-
-        shape = None
-        tdim = info['tdim']
-
-        shape = _tdim2shape(tdim, name, is_string=(npy_type[0] == 'S'))
-        if shape is not None:
-            if nrows > 1:
-                if not isinstance(shape, tuple):
-                    # vector
-                    shape = (nrows, shape)
-                else:
-                    # multi-dimensional
-                    shape = tuple([nrows] + list(shape))
-        else:
-            # scalar
-            shape = nrows
-        return npy_type, shape
-
     def get_rec_column_descr(self, colnum, vstorage):
         """
         Get a descriptor entry for the specified column.
@@ -1333,7 +1303,8 @@ def get_rec_column_descr(self, colnum, vstorage):
             tdim = self._info['colinfo'][colnum]['tdim']
             shape = _tdim2shape(
                 tdim, name,
-                is_string=(npy_type[0] == 'S' or npy_type[0] == 'U'))
+                is_string=(npy_type[0] in ('S', 'U'))
+            )
             if shape is not None:
                 descr = (name, npy_type, shape)
             else:

fitsio/tests/test_table.py

@@ -18,6 +18,11 @@
 
 DTYPES = ['u1', 'i1', 'u2', 'i2', '<u4', 'i4', 'i8', '>f4', 'f8']
 
+if np.lib.NumpyVersion(np.__version__) >= "2.0.0":
+    IS_NP2 = True
+else:
+    IS_NP2 = False
+
 
 def test_table_read_write():
 
@@ -86,6 +91,133 @@ def test_table_read_write():
                         )
 
 
+@pytest.mark.parametrize('nvec', [2, 1])
+def test_table_read_write_vec1(nvec):
+    """
+    ensure the data for vec length 1 gets round-tripped, even though
+    the shape is not preserved
+    """
+    dtype = [('x', 'f4', (nvec,))]
+    num = 10
+    data = np.zeros(num, dtype=dtype)
+    data['x'] = np.arange(num * nvec).reshape(num, nvec)
+    assert data['x'].shape == (num, nvec)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        fname = os.path.join(tmpdir, 'test.fits')
+
+        with FITS(fname, 'rw') as fits:
+            fits.write_table(data)
+
+            d = fits[1].read()
+            if nvec == 1:
+                assert d['x'].shape == (num,)
+            compare_array(
+                data['x'].ravel(), d['x'].ravel(),
+                "table single field read 'x'"
+            )
+
+        # see if our convenience functions are working
+        write(
+            fname,
+            data,
+            extname="newext",
+        )
+        d = read(fname, ext='newext')
+        if nvec == 1:
+            assert d['x'].shape == (num,)
+        compare_array(data['x'].ravel(), d['x'].ravel(), "table data2")
+
+        # now test read_column
+        with FITS(fname) as fits:
+
+            d = fits[1].read_column('x')
+            if nvec == 1:
+                assert d.shape == (num,)
+            compare_array(
+                data['x'].ravel(), d.ravel(),
+                "table single field read 'x'"
+            )
+
+
+@pytest.mark.parametrize('nvec', [2, 1])
+def test_table_read_write_uvec1(nvec):
+    """
+    ensure the data for U string vec length 1 gets round-tripped, even though
+    the shape is not preserved.  Also test 2 for consistency
+    """
+
+    dtype = [('string', 'U10', (nvec,))]
+    num = 10
+    data = np.zeros(num, dtype=dtype)
+    sravel = data['string'].ravel()
+    sravel[:] = ['%-10s' % i for i in range(num * nvec)]
+    assert data['string'].shape == (num, nvec)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        fname = os.path.join(tmpdir, 'test.fits')
+
+        with FITS(fname, 'rw') as fits:
+            fits.write_table(data)
+
+            d = fits[1].read()
+
+            if IS_NP2:
+                # We can only get the right shape back for numpy 2
+                compare_array(
+                    data['string'], d['string'],
+                    "table single field read 'string'"
+                )
+            else:
+                # for numpy 1.X we can't get the right shape for
+                # length 1 vector, need to flatted for comparision
+                if nvec == 1:
+                    assert d['string'].shape == (num,)
+
+                compare_array(
+                    data['string'].ravel(), d['string'].ravel(),
+                    "table single field read 'string'"
+                )
+
+        # see if our convenience functions are working
+        write(
+            fname,
+            data,
+            extname="newext",
+        )
+        d = read(fname, ext='newext')
+
+        if IS_NP2:
+            compare_array(
+                data['string'], d['string'],
+                "table single field read 'string'"
+            )
+        else:
+            if nvec == 1:
+                assert d['string'].shape == (num,)
+            compare_array(
+                data['string'].ravel(), d['string'].ravel(), "table data2",
+            )
+
+        # now test read_column
+        with FITS(fname) as fits:
+
+            d = fits[1].read_column('string')
+
+            if IS_NP2:
+                compare_array(
+                    data['string'], d,
+                    "table single field read 'string'"
+                )
+            else:
+                if nvec == 1:
+                    assert d.shape == (num,)
+                compare_array(
+                    data['string'].ravel(), d.ravel(),
+                    "table single field read 'string'"
+                )
+
+
 def test_table_column_index_scalar():
     """
     Test a basic table write, data and a header, then reading back in to

setup.py

@@ -318,7 +318,7 @@ def check_system_cfitsio_objects(self, obj_name):
 
 setup(
     name="fitsio",
-    version="1.2.5",
+    version="1.2.6",
     description=description,
     long_description=long_description,
     long_description_content_type='text/markdown; charset=UTF-8; variant=GFM',