From 505f940af75b386cb4f8902c466762d9fc062677 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 15 Apr 2025 12:04:38 +0200 Subject: [PATCH 1/9] Finish refactoring NC_s3sdkclose --- libdispatch/dhttp.c | 2 +- libdispatch/ncs3sdk_aws.cpp | 14 +++----------- libnczarr/zmap_s3sdk.c | 4 ++-- unit_test/aws_config.c | 2 +- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/libdispatch/dhttp.c b/libdispatch/dhttp.c index b0aedd9cf1..81dd574c4f 100644 --- a/libdispatch/dhttp.c +++ b/libdispatch/dhttp.c @@ -164,7 +164,7 @@ nc_http_close(NC_HTTP_STATE* state) #ifdef NETCDF_ENABLE_S3 case HTTPS3: { if(state->s3.s3client) - NC_s3sdkclose(state->s3.s3client, state->s3.info, 0, NULL); + NC_s3sdkclose(state->s3.s3client, NULL); NC_s3clear(state->s3.info); nullfree(state->s3.info); state->s3.s3client = NULL; diff --git a/libdispatch/ncs3sdk_aws.cpp b/libdispatch/ncs3sdk_aws.cpp index e1bceb7835..cb74c3e553 100644 --- a/libdispatch/ncs3sdk_aws.cpp +++ b/libdispatch/ncs3sdk_aws.cpp @@ -548,21 +548,13 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s } EXTERNL int -NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp) +NC_s3sdkclose(void* s3client0, char** errmsgp) { int stat = NC_NOERR; - NCTRACE(11,"info=%s rootkey=%s deleteit=%d",dumps3info(info),deleteit); - AWSS3CLIENT s3client = (AWSS3CLIENT)s3client0; - if(deleteit) { - /* Delete the root key; ok it if does not exist */ - switch (stat = NC_s3sdkdeletekey(s3client0,info->bucket,info->rootkey,errmsgp)) { - case NC_NOERR: break; - case NC_EEMPTY: case NC_ENOTFOUND: stat = NC_NOERR; break; - default: break; - } - } + NCTRACE(11,"info=%s y=%s deleteit=%d",s3client.); + #ifdef TRANSFER delete s3client; #else diff --git a/libnczarr/zmap_s3sdk.c b/libnczarr/zmap_s3sdk.c index 7ae6f0848f..2c2cbce5a4 100644 --- a/libnczarr/zmap_s3sdk.c +++ b/libnczarr/zmap_s3sdk.c @@ -261,7 +261,7 @@ zs3truncate(const char *s3url) if((s3client = NC_s3sdkcreateclient(&info))==NULL) {stat = NC_ES3; goto done;} if((stat = s3clear(s3client,info.bucket,info.rootkey))) goto done; done: - if(s3client) {stat=NC_s3sdkclose(s3client,&info,1,NULL);} + if(s3client) {stat=NC_s3sdkclose(s3client,NULL);} ncurifree(url); ncurifree(purl); (void)NC_s3clear(&info); @@ -407,7 +407,7 @@ zs3close(NCZMAP* map, int deleteit) if(deleteit) s3clear(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey); if(z3map->s3client && z3map->s3.bucket && z3map->s3.rootkey) { - NC_s3sdkclose(z3map->s3client, &z3map->s3, deleteit, &z3map->errmsg); + NC_s3sdkclose(z3map->s3client, &z3map->errmsg); } reporterr(z3map); z3map->s3client = NULL; diff --git a/unit_test/aws_config.c b/unit_test/aws_config.c index c9764ee18b..11fd96ba55 100644 --- a/unit_test/aws_config.c +++ b/unit_test/aws_config.c @@ -35,7 +35,7 @@ static void cleanup(void) { if(s3client) - NC_s3sdkclose(s3client, &s3info, 0/*deleteit*/, NULL); + NC_s3sdkclose(s3client, NULL); s3client = NULL; NC_s3clear(&s3info); } From ff9710fea1f06f805d055643b930d70aa06fb4ac Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 15 Apr 2025 12:05:18 +0200 Subject: [PATCH 2/9] Finish refactoring NC_s3sdklist & NC_s3sdklistall --- libdispatch/ncs3sdk_aws.cpp | 2 +- libnczarr/zmap_s3sdk.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libdispatch/ncs3sdk_aws.cpp b/libdispatch/ncs3sdk_aws.cpp index cb74c3e553..3f6d409911 100644 --- a/libdispatch/ncs3sdk_aws.cpp +++ b/libdispatch/ncs3sdk_aws.cpp @@ -646,7 +646,7 @@ Return a list of full keys of legal objects immediately below a specified key. Not necessarily sorted. */ EXTERNL int -NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) +NC_s3sdklist(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) { return getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp); } diff --git a/libnczarr/zmap_s3sdk.c b/libnczarr/zmap_s3sdk.c index 2c2cbce5a4..fe1da9fda6 100644 --- a/libnczarr/zmap_s3sdk.c +++ b/libnczarr/zmap_s3sdk.c @@ -227,7 +227,7 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m /* Search the root for content */ content = nclistnew(); - if((stat = NC_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg))) + if((stat = NC_s3sdklist(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg))) goto done; if(nkeys == 0) { /* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */ @@ -443,7 +443,7 @@ zs3search(NCZMAP* map, const char* prefix, NClist* matches) if((stat = maketruekey(z3map->s3.rootkey,prefix,&trueprefix))) goto done; if(*trueprefix != '/') return NC_EINTERNAL; - if((stat = NC_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,trueprefix,&nkeys,&list,&z3map->errmsg))) + if((stat = NC_s3sdklist(z3map->s3client,z3map->s3.bucket,trueprefix,&nkeys,&list,&z3map->errmsg))) goto done; if(nkeys > 0) { size_t tplen = strlen(trueprefix); @@ -505,7 +505,7 @@ s3clear(void* s3client, const char* bucket, const char* rootkey) size_t nkeys = 0; if(s3client && bucket && rootkey) { - if((stat = NC_s3sdksearch(s3client, bucket, rootkey, &nkeys, &list, NULL))) + if((stat = NC_s3sdklistall(s3client, bucket, rootkey, &nkeys, &list, NULL))) goto done; if(list != NULL) { size_t i; From 0949a064b09d90e1196d986808dcf7b39b7cab89 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 24 Jun 2025 10:47:31 +0200 Subject: [PATCH 3/9] Add NC_freeenvv to libdispatch --- libdispatch/dutil.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index f6f0eecc34..0bc9fd4f09 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -539,3 +539,47 @@ NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* s ncbytesfree(buf); return stat; } + +static int +lexical_compare(const void* arg1, const void* arg2) +{ + char* s1 = *((char**)arg1); + char* s2 = *((char**)arg2); + size_t slen1 = nulllen(s1); + size_t slen2 = nulllen(s2); + if(slen1 != slen2) return (slen1 - slen2); + return strcmp(s1,s2); +} + +/** +Sort a vector of strings. +@param n Number of strings to sort +@param env vector of strings to sort +*/ +void +NC_sortenvv(size_t n, char** envv) +{ + if(n <= 1) return; + qsort(envv, (int)n, sizeof(char*), lexical_compare); +} + +/** +Sort a nclist of strings. +@param l NClist of strings +*/ +void +NC_sortlist(NClist* l) +{ + if(l == NULL || nclistlength(l) == 0) return; + NC_sortenvv(nclistlength(l),(char**)nclistcontents(l)); +} + +/* Free up a vector of strings */ +void +NC_freeenvv(size_t nkeys, char** keys) +{ + size_t i; + for(i=0;i Date: Tue, 24 Jun 2025 10:56:40 +0200 Subject: [PATCH 4/9] Rename & move ncutil.h to include/netcdf_vutils.h --- include/Makefile.am | 2 +- .../ncutil.h => include/netcdf_vutils.h | 33 +++++++++++-------- libdispatch/Makefile.am | 2 +- 3 files changed, 21 insertions(+), 16 deletions(-) rename libdispatch/ncutil.h => include/netcdf_vutils.h (89%) diff --git a/include/Makefile.am b/include/Makefile.am index 9253e61932..07e663779d 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -10,7 +10,7 @@ include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \ netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h \ -netcdf_dispatch.h +netcdf_dispatch.h netcdf_vutils.h include_HEADERS += netcdf_json.h netcdf_proplist.h diff --git a/libdispatch/ncutil.h b/include/netcdf_vutils.h similarity index 89% rename from libdispatch/ncutil.h rename to include/netcdf_vutils.h index 44ab508b36..0324bade45 100644 --- a/libdispatch/ncutil.h +++ b/include/netcdf_vutils.h @@ -1,8 +1,8 @@ -/* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc. +/* Copyright 2018, UCAR/Unidata See the COPYRIGHT file for more information. */ -#ifndef UTILS_H -#define UTILS_H 1 +#ifndef NCVUTILS_H +#define NCVUTILS_H 1 /* Define a header-only simple version of a dynamically expandable list and byte buffer */ /* To be used in code that should be independent of libnetcdf */ @@ -15,8 +15,8 @@ typedef struct VList { typedef struct VString { int nonextendible; /* 1 => fail if an attempt is made to extend this string*/ - unsigned int alloc; - unsigned int length; + unsigned alloc; + unsigned length; char* content; } VString; @@ -55,7 +55,7 @@ static void vlistexpand(VList* l) { void** newcontent = NULL; - size_t newsz; + unsigned newsz; if(l == NULL) return; newsz = (l->length * 2) + 1; /* basically double allocated space */ @@ -132,11 +132,11 @@ static void vsexpand(VString* vs) { char* newcontent = NULL; - size_t newsz; + unsigned newsz; if(vs == NULL) return; assert(vs->nonextendible == 0); - newsz = (vs->alloc + VSTRALLOC); /* basically double allocated space */ + newsz = (vs->alloc + VSTRALLOC); /* increase allocated space */ if(vs->alloc >= newsz) return; /* space already allocated */ newcontent=(char*)calloc(1,newsz+1);/* always room for nul term */ assert(newcontent != NULL); @@ -154,7 +154,7 @@ vsappendn(VString* vs, const char* elem, unsigned n) { size_t need; assert(vs != NULL && elem != NULL); - if(n == 0) {n = strlen(elem);} + if(n == 0) {n = (unsigned)strlen(elem);} need = vs->length + n; if(vs->nonextendible) { /* Space must already be available */ @@ -166,7 +166,7 @@ vsappendn(VString* vs, const char* elem, unsigned n) memcpy(&vs->content[vs->length],elem,n); vs->length += n; if(!vs->nonextendible) - vs->content[vs->length] = '\0'; + vs->content[vs->length] = '\0'; /* guarantee nul term */ } static void @@ -196,7 +196,12 @@ static char* vsextract(VString* vs) { char* x = NULL; - if(vs == NULL || vs->content == NULL) return NULL; + if(vs == NULL) return NULL; + if(vs->content == NULL) { + /* guarantee content existence and nul terminated */ + if((vs->content = calloc(1,sizeof(char)))==NULL) return NULL; + vs->length = 0; + } x = vs->content; vs->content = NULL; vs->length = 0; @@ -229,14 +234,14 @@ util_initialize(void) /* Following are always "in-lined"*/ #define vlistcontents(l) ((l)==NULL?NULL:(l)->content) -#define vlistlength(l) ((l)==NULL?0:(int)(l)->length) +#define vlistlength(l) ((l)==NULL?0:(l)->length) #define vlistclear(l) vlistsetlength(l,0) #define vlistsetlength(l,len) do{if((l)!=NULL) (l)->length=len;} while(0) #define vscontents(vs) ((vs)==NULL?NULL:(vs)->content) -#define vslength(vs) ((vs)==NULL?0:(int)(vs)->length) +#define vslength(vs) ((vs)==NULL?0:(vs)->length) #define vscat(vs,s) vsappendn(vs,s,0) #define vsclear(vs) vssetlength(vs,0) #define vssetlength(vs,len) do{if((vs)!=NULL) (vs)->length=len;} while(0) -#endif /*UTILS_H*/ +#endif /*NCVUTIL_H*/ diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index 8d35269ea3..9001ed4c25 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -51,7 +51,7 @@ endif # NETCDF_ENABLE_BYTERANGE if NETCDF_ENABLE_S3 if NETCDF_ENABLE_S3_INTERNAL # Renamed to avoid conflicts with the HDF5 files -libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h ncutil.h nccurl_setup.h \ +libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h nccurl_setup.h \ nccurl_sha256.c nccurl_sha256.h nccurl_hmac.c nccurl_hmac.h AM_CPPFLAGS += -I$(top_srcdir)/libncxml libdispatch_la_CPPFLAGS += ${AM_CPPFLAGS} From 355e453d3aaa28cba849012f87a7940b2c4c0c4e Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Mon, 16 Jun 2025 16:12:47 +0200 Subject: [PATCH 5/9] CI: Enable S3 on the CI --- .github/workflows/run_tests_osx.yml | 14 ++++++++++++++ .github/workflows/run_tests_ubuntu.yml | 24 ++++++++++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index bcf0e7c6e4..f81f55d27f 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -70,6 +70,7 @@ jobs: use_nc4: [ nc3, nc4 ] use_dap: [ dap_off, dap_on ] use_nczarr: [ nczarr_off, nczarr_on ] + use_s3: [ s3_off, s3_on ] steps: - uses: actions/checkout@v4 @@ -97,6 +98,12 @@ jobs: if: matrix.use_nczarr == 'nczarr_off' - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV if: matrix.use_nczarr == 'nczarr_on' + - run: echo "NETCDF_ENABLE_NCZARR=--disable-s3" >> $GITHUB_ENV + if: matrix.use_s3 == 's3_off' + - run: | + echo "NETCDF_ENABLE_NCZARR=--enable-s3" >> $GITHUB_ENV + echo "NETCDF_ENABLE_NCZARR=--enable-s3-internal" >> $GITHUB_ENV + if: matrix.use_s3 == 's3_on' ### # Fetch Cache @@ -176,6 +183,7 @@ jobs: use_nc4: [ nc3, nc4 ] use_dap: [ dap_off, dap_on ] use_nczarr: [ nczarr_off, nczarr_on ] + use_s3: [ s3_off, s3_on ] steps: - uses: actions/checkout@v4 @@ -202,6 +210,12 @@ jobs: if: matrix.use_nczarr == 'nczarr_off' - run: echo "NETCDF_ENABLE_NCZARR=ON" >> $GITHUB_ENV if: matrix.use_nczarr == 'nczarr_on' + - run: echo "NETCDF_ENABLE_S3=OFF" >> $GITHUB_ENV + if: matrix.use_s3 == 's3_off' + - run: | + echo "NETCDF_ENABLE_S3=ON" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3_INTERNAL=ON" >> $GITHUB_ENV + if: matrix.use_s3 == 's3_on' - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV ### diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 6b78cb478b..a2397d1a11 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [workflow_dispatch] +on: [pull_request, workflow_dispatch] env: REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} @@ -659,7 +659,7 @@ jobs: hdf5: [1.10.8, 1.12.2, 1.14.3 ] use_nc4: [ nc3, nc4 ] use_dap: [ dap_off, dap_on ] - use_nczarr: [ nczarr_off, nczarr_on ] + use_nczarr_and_s3: [ nczarr_off, nczarr_on, nczarr_on_s3_on ] runs-on: [ ubuntu-22.04 ] runs-on: ${{ matrix.runs-on }} @@ -710,9 +710,14 @@ jobs: - run: echo "NETCDF_ENABLE_DAP=--enable-dap" >> $GITHUB_ENV if: matrix.use_dap == 'dap_on' - run: echo "NETCDF_ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_off' + if: matrix.use_nczarr_and_s3 == 'nczarr_off' - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_on' + if: matrix.use_nczarr_and_s3 == 'nczarr_on' + - run: | + echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3=--enable-s3" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3_INTERNAL=--enable-s3-internal" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_on_s3_on' ### # Fetch Cache @@ -759,7 +764,7 @@ jobs: done current_directory="$(pwd)" mkdir ../build - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${NETCDF_ENABLE_HDF5} ${NETCDF_ENABLE_DAP} ${NETCDF_ENABLE_NCZARR} + cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${NETCDF_ENABLE_HDF5} ${NETCDF_ENABLE_DAP} ${NETCDF_ENABLE_NCZARR} ${NETCDF_ENABLE_S3} ${NETCDF_ENABLE_S3_INTERNAL} if: ${{ success() }} - name: Look at config.log if error @@ -827,6 +832,7 @@ jobs: use_nc4: [ nc3, nc4 ] use_dap: [ dap_off, dap_on ] use_nczarr: [ nczarr_off, nczarr_on ] + use_s3: [ s3_off, s3_on ] steps: - uses: actions/checkout@v4 @@ -855,6 +861,12 @@ jobs: if: matrix.use_nczarr == 'nczarr_off' - run: echo "NETCDF_ENABLE_NCZARR=ON" >> $GITHUB_ENV if: matrix.use_nczarr == 'nczarr_on' + - run: echo "NETCDF_ENABLE_S3=OFF" >> $GITHUB_ENV + if: matrix.use_s3 == 's3_off' + - run: | + echo "NETCDF_ENABLE_S3=ON" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3_INTERNAL=ON" >> $GITHUB_ENV + if: matrix.use_s3 == 's3_on' - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV ### @@ -881,7 +893,7 @@ jobs: run: | mkdir build cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DNETCDF_ENABLE_DAP=${NETCDF_ENABLE_DAP} -DNETCDF_ENABLE_HDF5=${NETCDF_ENABLE_HDF5} -DNETCDF_ENABLE_NCZARR=${NETCDF_ENABLE_NCZARR} + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DNETCDF_ENABLE_DAP=${NETCDF_ENABLE_DAP} -DNETCDF_ENABLE_HDF5=${NETCDF_ENABLE_HDF5} -DNETCDF_ENABLE_NCZARR=${NETCDF_ENABLE_NCZARR} -DNETCDF_ENABLE_S3=${NETCDF_ENABLE_S3} -DNETCDF_ENABLE_S3_INTERNAL=${NETCDF_ENABLE_S3_INTERNAL} - name: Print Summary shell: bash -l {0} From d24059346fae34847b20e98a7f6d8920e2eb9a75 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Thu, 26 Jun 2025 11:07:34 +0200 Subject: [PATCH 6/9] CI: Enable S3 for CYGWIN workflow --- .github/workflows/run_tests_win_cygwin.yml | 27 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 54e8d0affb..dfbec1763a 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: plugin_dir_option: ["", "--without-plugin-dir"] - + use_nczarr_and_s3: ["nczarr_off", "nczarr_on", "nczarr_on_s3_on" ] steps: - name: Fix line endings shell: pwsh @@ -50,14 +50,25 @@ jobs: /bin/dash /usr/bin/libtoolize --force --copy --verbose /usr/bin/autoreconf-2.69 --force --install --verbose --debug +### +# Setup environment variables +### + - run: echo "NETCDF_ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_off' + - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_on' + - run: | + echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3=--enable-s3" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3_INTERNAL=--enable-s3-internal" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_on_s3_on' + - name: (Autotools) Configure in-tree build run: >- /bin/dash ./configure --enable-hdf5 --enable-shared --disable-static --enable-dap --disable-dap-remote-tests --enable-plugins ${{ matrix.plugin_dir_option }} - --disable-nczarr --disable-nczarr-filters - --disable-s3 --with-s3-testing=no - + ${NETCDF_ENABLE_NCZARR} ${NETCDF_ENABLE_S3} ${NETCDF_ENABLE_S3_INTERNAL} - name: Look at config.log if error if: ${{ failure() }} @@ -102,6 +113,10 @@ jobs: defaults: run: shell: C:/cygwin/bin/bash.exe -eo pipefail -o igncr "{0}" + + strategy: + matrix: + enable_nczarr_and_s3: ["OFF", "ON"] steps: @@ -139,7 +154,9 @@ jobs: -DNETCDF_ENABLE_TESTS=ON \ -DNETCDF_ENABLE_HDF5=ON \ -DNETCDF_ENABLE_NCZARR=TRUE \ - -DNETCDF_ENABLE_PLUGINS=ON + -DNETCDF_ENABLE_PLUGINS=ON \ + -DNETCDF_ENABLE_S3=${{ matrix.enable_nczarr_and_s3 }} \ + -DNETCDF_ENABLE_S3_INTERNAL=${{ matrix.enable_nczarr_and_s3 }} if: ${{ success() }} - name: (CMake) Look at CMakeCache.txt if error From 8a4284a923878a468c1c7afa97b61edd820e52c7 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Thu, 26 Jun 2025 16:04:34 +0200 Subject: [PATCH 7/9] CI: Enable S3 for MINGW workflow --- .github/workflows/run_tests_win_mingw.yml | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index be3034f9af..a7d59e66ff 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -24,6 +24,7 @@ jobs: strategy: matrix: msystem: [ MINGW64, UCRT64] #, CLANG64 ] + use_nczarr_and_s3: ["nczarr_off", "nczarr_on", "nczarr_on_s3_on" ] defaults: run: shell: msys2 {0} @@ -38,15 +39,31 @@ jobs: pacboy: cc:p autotools:p hdf5:p curl:p libxml2:p zlib:p blosc:p bzip2:p install: diffutils m4 git unzip +### +# Setup environment variables +### + - run: echo "NETCDF_ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_off' + - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_on' + - run: | + echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3=--enable-s3" >> $GITHUB_ENV + echo "NETCDF_ENABLE_S3_INTERNAL=--enable-s3-internal" >> $GITHUB_ENV + if: matrix.use_nczarr_and_s3 == 'nczarr_on_s3_on' + ### # Configure and build ### - name: (Autotools) Run autoconf run: autoreconf -if - - name: (Autotools) Configure Build - run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-s3 + run: >- + ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests + --disable-static --disable-byterange --disable-dap-remote-tests + --disable-logging --enable-plugins ${NETCDF_ENABLE_NCZARR} + ${NETCDF_ENABLE_S3} ${NETCDF_ENABLE_S3_INTERNAL} if: ${{ success() }} - name: (Autotools) Look at config.log if error @@ -84,6 +101,7 @@ jobs: strategy: matrix: msystem: [ MINGW64, UCRT64] #, CLANG64 ] + use_s3: [ "s3_off", "s3_on" ] defaults: run: shell: msys2 {0} @@ -120,6 +138,8 @@ jobs: -DNETCDF_ENABLE_NCZARR=TRUE \ -DNETCDF_ENABLE_DAP_LONG_TESTS=TRUE \ -DNETCDF_ENABLE_PLUGINS=ON + -DNETCDF_ENABLE_S3=${{ matrix.use_s3 }}\ + -DNETCDF_ENABLE_S3_INTERNAL=${{ matrix.use_s3}} \ if: ${{ success() }} - name: (CMake) Look at CMakeCache.txt if error From 97a6b90eef8bef15d6f96cf42a47e22f9c110c49 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Thu, 26 Jun 2025 16:28:52 +0200 Subject: [PATCH 8/9] Revert "CI: Enable S3 for MINGW workflow" This reverts commit 8a4284a923878a468c1c7afa97b61edd820e52c7. --- .github/workflows/run_tests_win_mingw.yml | 24 ++--------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index a7d59e66ff..be3034f9af 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -24,7 +24,6 @@ jobs: strategy: matrix: msystem: [ MINGW64, UCRT64] #, CLANG64 ] - use_nczarr_and_s3: ["nczarr_off", "nczarr_on", "nczarr_on_s3_on" ] defaults: run: shell: msys2 {0} @@ -39,31 +38,15 @@ jobs: pacboy: cc:p autotools:p hdf5:p curl:p libxml2:p zlib:p blosc:p bzip2:p install: diffutils m4 git unzip -### -# Setup environment variables -### - - run: echo "NETCDF_ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV - if: matrix.use_nczarr_and_s3 == 'nczarr_off' - - run: echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV - if: matrix.use_nczarr_and_s3 == 'nczarr_on' - - run: | - echo "NETCDF_ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV - echo "NETCDF_ENABLE_S3=--enable-s3" >> $GITHUB_ENV - echo "NETCDF_ENABLE_S3_INTERNAL=--enable-s3-internal" >> $GITHUB_ENV - if: matrix.use_nczarr_and_s3 == 'nczarr_on_s3_on' - ### # Configure and build ### - name: (Autotools) Run autoconf run: autoreconf -if + - name: (Autotools) Configure Build - run: >- - ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests - --disable-static --disable-byterange --disable-dap-remote-tests - --disable-logging --enable-plugins ${NETCDF_ENABLE_NCZARR} - ${NETCDF_ENABLE_S3} ${NETCDF_ENABLE_S3_INTERNAL} + run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-s3 if: ${{ success() }} - name: (Autotools) Look at config.log if error @@ -101,7 +84,6 @@ jobs: strategy: matrix: msystem: [ MINGW64, UCRT64] #, CLANG64 ] - use_s3: [ "s3_off", "s3_on" ] defaults: run: shell: msys2 {0} @@ -138,8 +120,6 @@ jobs: -DNETCDF_ENABLE_NCZARR=TRUE \ -DNETCDF_ENABLE_DAP_LONG_TESTS=TRUE \ -DNETCDF_ENABLE_PLUGINS=ON - -DNETCDF_ENABLE_S3=${{ matrix.use_s3 }}\ - -DNETCDF_ENABLE_S3_INTERNAL=${{ matrix.use_s3}} \ if: ${{ success() }} - name: (CMake) Look at CMakeCache.txt if error From 9eb36ef1cd24d6219b1a480ca9820e67c7748d67 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Wed, 2 Jul 2025 17:18:09 +0200 Subject: [PATCH 9/9] CMake: Fix plugin install variable --- plugins/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index fbaeb21304..036965a4b4 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -119,7 +119,7 @@ else() endif() # Installation -if(ENABLE_PLUGIN_INSTALL) +if(NETCDF_PLUGIN_INSTALL) macro(installplugin PLUG) set(INSTALLED_PLUGIN_LIB "${PLUGINPRE}${PLUG}.${PLUGINEXT}") @@ -155,5 +155,5 @@ if(MSVC) file(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/) endif() -endif(ENABLE_PLUGIN_INSTALL) +endif(NETCDF_PLUGIN_INSTALL) endif(NETCDF_ENABLE_PLUGINS)