Unidata · WardF · Aug 1, 2025 · Feb 17, 2025 · Feb 17, 2025 · Apr 15, 2025
@@ -16,16 +16,18 @@ permissions: {}
 ####
 # Serial Testing
 ###
-jobs:  
-  netcdf-tests-serial:
-    name: Docker-Based NetCDF-C, Fortran Regression Testing (Serial)
+jobs:
+
+  netcdf-tests-s3-serial:
+    name: Docker-Based NetCDF-C and Fortran Regression Testing, S3 (Internal) and Zarr (Serial)
     runs-on: ubuntu-latest
+
     strategy:
       matrix:
         repo-type: [ 'c' ]
         c-compiler: [ 'gcc', 'clang' ]
-        h5ver: [ '1.12.1', '1.14.6' ]
-        fortran-branch: [ 'v4.6.2', 'main' ]
+        h5ver: [ '1.14.6' ]
+        fortran-branch: [ 'main' ]
         buildsystem: [ 'both' ]
 
     steps:
@@ -41,13 +43,40 @@ jobs:
           c-compiler: '${{ matrix.c-compiler }}'
           run-fortran: 'TRUE'
           fortran-branch: '${{ matrix.fortran-branch }}'
+          autotools-c-options: '--enable-s3 --enable-s3-internal'
+          cmake-c-options: '-DNETCDF_ENABLE_S3_INTERNAL=TRUE'
+
+  netcdf-test-matrix-serial:
+      name: Docker-Based NetCDF-C, Fortran Regression Testing (Serial)
+      runs-on: ubuntu-latest
+      strategy:
+        matrix:
+          repo-type: [ 'c' ]
+          c-compiler: [ 'gcc', 'clang' ]
+          h5ver: [ '1.12.1', '1.14.6' ]
+          fortran-branch: [ 'v4.6.2', 'main' ]
+          buildsystem: [ 'both' ]
+
+      steps:
+        - uses: actions/checkout@v4
+        - name: Pull and Run netCDF Regression Tests
+          uses: WardF/netcdf-test-action@v1
+          with:
+            repo-type: '${{ matrix.repo-type }}'
+            run-c: 'TRUE'
+            build-system: '${{ matrix.buildsystem }}'
+            hdf5-version: '${{ matrix.h5ver }}'
+            ctest-repeat: 3
+            c-compiler: '${{ matrix.c-compiler }}'
+            run-fortran: 'TRUE'
+            fortran-branch: '${{ matrix.fortran-branch }}'
 
 ###
 # Parallel Testing
 ###          
-  netcdf-tests-parallel:
+  netcdf-test-matrix-parallel:
 
-    needs: netcdf-tests-serial
+    needs: netcdf-test-matrix-serial
 
     name: Docker-Based NetCDF-C, Fortran Regression Testing (Parallel)
     runs-on: ubuntu-latest
@@ -75,11 +104,11 @@ jobs:
           run-fortran: 'TRUE'
           fortran-branch: '${{ matrix.fortran-branch }}'
 
-###
-# Test with Java
-###
+  ###
+  # Test with Java
+  ###
   netcdf-tests-java:
-    needs: netcdf-tests-serial
+    needs: netcdf-test-matrix-serial
 
     name: Docker-Based NetCDF-C, Java Regression Testing (Serial, Java Only)
     runs-on: ubuntu-latest

@@ -70,6 +70,7 @@ jobs:
         use_nc4: [ nc3, nc4 ]
         use_dap: [ dap_off, dap_on ]
         use_nczarr: [ nczarr_off, nczarr_on ]
+        use_s3: [ s3_off, s3_on ]
     steps:
 
       - uses: actions/checkout@v4
@@ -97,6 +98,12 @@ jobs:
         if: matrix.use_nczarr == 'nczarr_off'
       - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV
         if: matrix.use_nczarr == 'nczarr_on'
+      - run: echo "NETCDF_ENABLE_NCZARR=--disable-s3" >> $GITHUB_ENV
+        if: matrix.use_s3 == 's3_off'
+      - run: |
+          echo "NETCDF_ENABLE_NCZARR=--enable-s3" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_NCZARR=--enable-s3-internal" >> $GITHUB_ENV
+        if: matrix.use_s3 == 's3_on'
 
       ###
       # Fetch Cache
@@ -176,6 +183,7 @@ jobs:
         use_nc4: [ nc3, nc4 ]
         use_dap: [ dap_off, dap_on ]
         use_nczarr: [ nczarr_off, nczarr_on ]
+        use_s3: [ s3_off, s3_on ]
     steps:
 
       - uses: actions/checkout@v4
@@ -202,6 +210,12 @@ jobs:
         if: matrix.use_nczarr == 'nczarr_off'
       - run: echo "NETCDF_ENABLE_NCZARR=ON" >> $GITHUB_ENV
         if: matrix.use_nczarr == 'nczarr_on'
+      - run: echo "NETCDF_ENABLE_S3=OFF" >> $GITHUB_ENV
+        if: matrix.use_s3 == 's3_off'
+      - run: |
+          echo "NETCDF_ENABLE_S3=ON" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_S3_INTERNAL=ON" >> $GITHUB_ENV
+        if: matrix.use_s3 == 's3_on'
       - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
 
     ###

@@ -659,7 +659,7 @@ jobs:
         hdf5: [1.10.8, 1.12.2, 1.14.3 ]
         use_nc4: [ nc3, nc4 ]
         use_dap: [ dap_off, dap_on ]
-        use_nczarr: [ nczarr_off, nczarr_on ]
+        use_nczarr_and_s3: [ nczarr_off, nczarr_on, nczarr_on_s3_on ]
         runs-on: [ ubuntu-22.04 ]
 
     runs-on: ${{ matrix.runs-on }}
@@ -710,9 +710,14 @@ jobs:
       - run: echo "NETCDF_ENABLE_DAP=--enable-dap" >> $GITHUB_ENV
         if: matrix.use_dap == 'dap_on'
       - run: echo "NETCDF_ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV
-        if: matrix.use_nczarr == 'nczarr_off'
+        if: matrix.use_nczarr_and_s3 == 'nczarr_off'
       - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV
-        if: matrix.use_nczarr == 'nczarr_on'
+        if: matrix.use_nczarr_and_s3 == 'nczarr_on'
+      - run: |
+          echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_S3=--enable-s3" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_S3_INTERNAL=--enable-s3-internal" >> $GITHUB_ENV
+        if: matrix.use_nczarr_and_s3 == 'nczarr_on_s3_on'
 
       ###
       # Fetch Cache
@@ -759,7 +764,7 @@ jobs:
           done
           current_directory="$(pwd)"
           mkdir ../build
-          cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${NETCDF_ENABLE_HDF5} ${NETCDF_ENABLE_DAP} ${NETCDF_ENABLE_NCZARR}
+          cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${NETCDF_ENABLE_HDF5} ${NETCDF_ENABLE_DAP} ${NETCDF_ENABLE_NCZARR} ${NETCDF_ENABLE_S3} ${NETCDF_ENABLE_S3_INTERNAL}
         if: ${{ success() }}
 
       - name: Look at config.log if error
@@ -827,6 +832,7 @@ jobs:
         use_nc4: [ nc3, nc4 ]
         use_dap: [ dap_off, dap_on ]
         use_nczarr: [ nczarr_off, nczarr_on ]
+        use_s3: [ s3_off, s3_on ]
     steps:
 
       - uses: actions/checkout@v4
@@ -855,6 +861,12 @@ jobs:
         if: matrix.use_nczarr == 'nczarr_off'
       - run: echo "NETCDF_ENABLE_NCZARR=ON" >> $GITHUB_ENV
         if: matrix.use_nczarr == 'nczarr_on'
+      - run: echo "NETCDF_ENABLE_S3=OFF" >> $GITHUB_ENV
+        if: matrix.use_s3 == 's3_off'
+      - run: |
+          echo "NETCDF_ENABLE_S3=ON" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_S3_INTERNAL=ON" >> $GITHUB_ENV
+        if: matrix.use_s3 == 's3_on'
       - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
 
       ###
@@ -881,7 +893,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DNETCDF_ENABLE_DAP=${NETCDF_ENABLE_DAP} -DNETCDF_ENABLE_HDF5=${NETCDF_ENABLE_HDF5} -DNETCDF_ENABLE_NCZARR=${NETCDF_ENABLE_NCZARR}
+          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DNETCDF_ENABLE_DAP=${NETCDF_ENABLE_DAP} -DNETCDF_ENABLE_HDF5=${NETCDF_ENABLE_HDF5} -DNETCDF_ENABLE_NCZARR=${NETCDF_ENABLE_NCZARR} -DNETCDF_ENABLE_S3=${NETCDF_ENABLE_S3} -DNETCDF_ENABLE_S3_INTERNAL=${NETCDF_ENABLE_S3_INTERNAL}
 
       - name: Print Summary
         shell: bash -l {0}

@@ -1,3 +1,4 @@
+
 name: Run Cygwin-based tests
 
 on: [pull_request,workflow_dispatch]
@@ -24,7 +25,7 @@ jobs:
     strategy:
       matrix:
         plugin_dir_option: ["", "--without-plugin-dir"]
-
+        use_nczarr_and_s3: ["nczarr_off", "nczarr_on", "nczarr_on_s3_on" ]
     steps:
       - name: Fix line endings
         shell: pwsh
@@ -41,7 +42,7 @@ jobs:
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
             libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel
-            libssl-devel libcrypt-devel attr libattr-devel
+            libssl-devel libcrypt-devel attr libattr-devel unzip zip
 
       - name: (Autotools) Run autoconf and friends
         run: |
@@ -50,14 +51,25 @@ jobs:
           /bin/dash /usr/bin/libtoolize --force --copy --verbose
           /usr/bin/autoreconf-2.69 --force --install --verbose --debug
 
+###
+# Setup environment variables
+###
+      - run: echo "NETCDF_ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV
+        if: matrix.use_nczarr_and_s3 == 'nczarr_off'
+      - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV
+        if: matrix.use_nczarr_and_s3 == 'nczarr_on'
+      - run: |
+          echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_S3=--enable-s3" >> $GITHUB_ENV
+          echo "NETCDF_ENABLE_S3_INTERNAL=--enable-s3-internal" >> $GITHUB_ENV
+        if: matrix.use_nczarr_and_s3 == 'nczarr_on_s3_on'
+
       - name: (Autotools) Configure in-tree build
         run: >-
           /bin/dash ./configure --enable-hdf5 --enable-shared
           --disable-static --enable-dap --disable-dap-remote-tests
           --enable-plugins ${{ matrix.plugin_dir_option }}
-          --disable-nczarr --disable-nczarr-filters
-          --disable-s3 --with-s3-testing=no
-
+          ${NETCDF_ENABLE_NCZARR} ${NETCDF_ENABLE_S3} ${NETCDF_ENABLE_S3_INTERNAL}
 
       - name: Look at config.log if error
         if: ${{ failure() }}
@@ -87,11 +99,16 @@ jobs:
         timeout-minutes: 30
         run: |
           make check -j$(nproc) SHELL=/bin/dash
+
+      - name: (Autotools) Examine test failures.
+        run: |
+          cat $(find . -name 'test-suite.log') SHELL=/bin/dash
+        if: ${{ failure() }}
 
       - name: Upload autoconf test results
         uses: actions/upload-artifact@v4
         with:
-          name: cygwin-autotools-test-logs-${{ matrix.plugin_dir_option }}
+          name: cygwin-autotools-test-logs-${{ matrix.plugin_dir_option }}-${{ matrix.use_nczarr_and_s3 }}
           path: |
             */*.log
             */*.trs
@@ -102,6 +119,10 @@ jobs:
     defaults:
       run:
         shell: C:/cygwin/bin/bash.exe -eo pipefail -o igncr "{0}"
+
+    strategy:
+      matrix:
+        enable_nczarr_and_s3: ["OFF", "ON"]      
 
     steps:
 
@@ -117,7 +138,7 @@ jobs:
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
             libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel
-            libssl-devel libcrypt-devel cmake ninja make m4 diffutils unzip
+            libssl-devel libcrypt-devel cmake ninja make m4 diffutils zip unzip
 
 ###
 # Configure and build
@@ -139,7 +160,9 @@ jobs:
           -DNETCDF_ENABLE_TESTS=ON \
           -DNETCDF_ENABLE_HDF5=ON \
           -DNETCDF_ENABLE_NCZARR=TRUE \
-          -DNETCDF_ENABLE_PLUGINS=ON
+          -DNETCDF_ENABLE_PLUGINS=ON \
+          -DNETCDF_ENABLE_S3=${{ matrix.enable_nczarr_and_s3 }} \
+          -DNETCDF_ENABLE_S3_INTERNAL=${{ matrix.enable_nczarr_and_s3 }} 
         if: ${{ success() }}
 
       - name: (CMake) Look at CMakeCache.txt if error

@@ -772,6 +772,12 @@ if (NOT NETCDF_ENABLE_S3 AND NETCDF_ENABLE_NCZARR_S3)
 endif()
 unset(NETCDF_ENABLE_NCZARR_S3)
 
+# If NETCDF_ENABLE_S3 is ON/TRUE, then enable NETCDF_ENABLE_NCZARR, in order to keep 
+# things congruent with autotools.
+if (NETCDF_ENABLE_S3)
+  set(NETCDF_ENABLE_NCZARR ON CACHE BOOL "Enable NCZarr Client" FORCE)
+endif(NETCDF_ENABLE_S3)
+
 if(NOT NETCDF_ENABLE_REMOTE_FUNCTIONALITY)
   message(WARNING "NETCDF_ENABLE_REMOTE_FUNCTIONALITY=NO => disable all s3 functionality")
   set(NETCDF_ENABLE_S3 OFF CACHE BOOL "" FORCE)

@@ -10,6 +10,7 @@ This file contains a high-level description of this package's evolution. Release
 * Clean up the S3 API for all non-libnczarr code. This continues the splitting of PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
 See [Github #3090](https://github.com/Unidata/netcdf-c/pull/3090) for more information.
 * Step 1 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Update ncjson.[ch] and ncproplist.[ch]. Also fix references to old API. Also fix include/netcdf_ncjson.h and include/netcdf_proplist.h builds. See [Github #3086](https://github.com/Unidata/netcdf-c/pull/3086) for more information.
+* Refactor drc.c to move many of its purely utility functions into dutil.c. Also change the NC_mktmp signature. Change other files to match. See [Github #3094](https://github.com/Unidata/netcdf-c/pull/3094) for more information.
 * Provide an auxilliary function, `ncaux_parse_provenance()`, that allows users to parse the _NCProperties attribute into a collection of character pointers. See [Github #3088](https://github.com/Unidata/netcdf-c/pull/3088) for more information.
 * Step 2 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Fix a namespace problem in tinyxml2.cpp. Note that this is a visual studio problem hence use of _MSC_VER. Also turn off DAP4 tests against Hyrax server until DAP4 spec problems are fixed. See [Github #3089](https://github.com/Unidata/netcdf-c/pull/3089) for more information.
 

@@ -969,13 +969,13 @@ else
 
    # Set enable_s3 instead of enable_nczarr_s3
    if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then
-   enable_s3=yes # back compatibility
+      enable_s3=yes # back compatibility
    fi
    unset enable_nczarr_s3
 
    if test "x$enable_remote_functionality" = xno ; then
    AC_MSG_WARN([--disable-remote-functionality => --disable-s3])
-   enable_s3=no
+      enable_s3=no
    fi
 
    # Note we check for the library after checking for enable_s3

@@ -10,7 +10,7 @@
 
 include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h	\
 netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h 	\
-netcdf_dispatch.h
+netcdf_dispatch.h netcdf_vutils.h
 
 include_HEADERS += netcdf_json.h netcdf_proplist.h
 
@@ -25,7 +25,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h		\
 ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h	\
 ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h	\
 hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h          \
-ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h
+ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h ncutil.h
 
 if USE_DAP
 noinst_HEADERS += ncdap.h

@@ -12,10 +12,10 @@ and accessing rc files (e.g. .daprc).
 #define NCRC_H
 
 /* Need these support includes */
+#include <stdio.h>
 #include "ncuri.h"
 #include "nclist.h"
 #include "ncbytes.h"
-#include <stdio.h>
 
 /* getenv() keys */
 #define NCRCENVIGNORE "NCRCENV_IGNORE"
@@ -46,6 +46,7 @@ typedef struct NCRCinfo {
 
 /* Opaque structures */
 struct NCS3INFO;
+enum NCS3SVC;
 
 #if defined(__cplusplus)
 extern "C" {
@@ -67,23 +68,6 @@ EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t);
 EXTERNL void NC_rcclear(NCRCinfo* info);
 EXTERNL void NC_rcclear(NCRCinfo* info);
 
-/* From dutil.c (Might later move to e.g. nc.h */
-EXTERNL int NC__testurl(const char* path, char** basenamep);
-EXTERNL int NC_isLittleEndian(void);
-EXTERNL char* NC_entityescape(const char* s);
-EXTERNL int NC_readfile(const char* filename, NCbytes* content);
-EXTERNL int NC_readfilen(const char* filename, NCbytes* content, long long len);
-EXTERNL int NC_readfileF(FILE* fp, NCbytes* content, long long len);
-EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
-EXTERNL char* NC_mktmp(const char* base);
-EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp);
-EXTERNL int NC_testmode(NCURI* uri, const char* tag);
-EXTERNL int NC_testpathmode(const char* path, const char* tag);
-EXTERNL int NC_addmodetag(NCURI* uri, const char* tag);
-EXTERNL int NC_split_delim(const char* path, char delim, NClist* segments);
-EXTERNL int NC_join(struct NClist* segments, char** pathp);
-EXTERNL int NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* suffix, char** pathp);
-
 #if defined(__cplusplus)
 }
 #endif