From c594bb24488b51b039f9e972f3fa05b1521a27f9 Mon Sep 17 00:00:00 2001
From: Dennis Heimbigner <dennis.heimbigner@gmail.com>
Date: Wed, 12 Feb 2025 18:48:29 -0700
Subject: [PATCH 1/2] Fix various problems with S3 related code

re: PR https://github.com/Unidata/netcdf-c/pull/3068

Part of splitting PR 3068.

General goal is to fix minor bugs and issues involving
S3 code outside of libnczarr.

1. libsrc/s3io.c:
    - Fix handling of error output of NC_s3sdkinfo.
    - Implement delete arg to s3io_close
2. Add support for Zarr-Over-HTTP (ZOH) protocol, but leave disabled until given the go-ahead from Manuel Reis.
3. Change the ncs3sdk API to light of eventual addition of Zarr V3 support.
4. Modify s3cleanup.in to catch and remove some previously overlooked entries in the Unidata S3 test bucket.
5. Modify s3gc.in to catch and properly remove Unidata S3 test bucket entries with old UIDs.
6. Modify ds3util.c:
    - Support use of a non-standard port -- H/T Manuel Reis
    - Clean up some memory leaks
    - Add disabled ZOH support
    - Better doc on how URLs are interpreted.
    - Document how .aws/config and .aws/credentials files are parsed.
7. Repair bugs in the internal S3 reader/writer module.
    - Allow API functions to return an http code (see also nch5s3comms.h).
    - Fix some casting warnings.
8. Fixes to ncs3sdk_h5.c, the dispatch wrapper for the internal S3 reader/writer
    - Move httptonc to this file.
    - Make conform to API modifications
    - Refactor the key search code.
    - Rename getkeys to list and searchkeys to listall.
9. Fixes to nczarr_test/s3util.c
    - Reflect ds3util.c API changes: e.g search->listall.
10. Add s3util.c to v3_nczarr_directory
11. Extend unit_test/test_s3dk.c to reflect S3 API changes.
12. Extend unit_test/run_s3sdk.sh to add new tests involving S3 API changes.
---
 RELEASE_NOTES.md            |   5 +
 dap4_test/CMakeLists.txt    |   2 +-
 dap4_test/Makefile.am       |   2 +-
 dap4_test/test_hyrax.sh     |   4 +-
 include/ncs3sdk.h           |  17 +-
 libdispatch/ds3util.c       | 130 ++++++++---
 libdispatch/nch5s3comms.c   |  82 ++-----
 libdispatch/nch5s3comms.h   |   8 +-
 libdispatch/ncs3sdk_h5.c    | 113 +++++++--
 libncxml/ncxml_tinyxml2.cpp |   4 +
 libsrc/s3io.c               |   6 +-
 nczarr_test/s3util.c        |  15 +-
 s3cleanup.in                |   6 +-
 s3gc.in                     |  36 +--
 unit_test/run_s3sdk.sh      |  11 +-
 unit_test/test_s3sdk.c      |  99 +++++---
 v3_nczarr_test/s3util.c     | 442 ++++++++++++++++++++++++++++++++++++
 17 files changed, 800 insertions(+), 182 deletions(-)
 create mode 100644 v3_nczarr_test/s3util.c

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index a1a0d87667..ff275a116e 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -5,6 +5,11 @@ Release Notes       {#RELEASE_NOTES}
 
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 
+## 4.9.4 - TBD
+
+* Clean up the S3 API for all non-libnczarr code. This continues the splitting of PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
+See [Github #????](https://github.com/Unidata/netcdf-c/pull/????) for more information.
+
 ## 4.9.3 - February 7, 2025
 
 ## Known Issues
diff --git a/dap4_test/CMakeLists.txt b/dap4_test/CMakeLists.txt
index cd9be513c3..5d538ccda1 100644
--- a/dap4_test/CMakeLists.txt
+++ b/dap4_test/CMakeLists.txt
@@ -52,7 +52,7 @@ ENDIF()
 
   IF(NETCDF_ENABLE_DAP_REMOTE_TESTS)
       add_sh_test(dap4_test test_remote)
-      add_sh_test(dap4_test test_hyrax)
+#Suppress until reconciled with hyrax: add_sh_test(dap4_test test_hyrax)
       add_sh_test(dap4_test test_dap4url)
     IF(RUN_MANUAL_TESTS)
         # The following test can only be run by hand.
diff --git a/dap4_test/Makefile.am b/dap4_test/Makefile.am
index a6b1c9f566..418238d12a 100644
--- a/dap4_test/Makefile.am
+++ b/dap4_test/Makefile.am
@@ -47,7 +47,7 @@ if NETCDF_ENABLE_DAP_REMOTE_TESTS
 
 TESTS += test_remote.sh
 TESTS += test_constraints.sh
-TESTS += test_hyrax.sh
+#Suppress until reconciled with hyrax: TESTS += test_hyrax.sh
 TESTS += test_dap4url.sh
 
 # The following test can only be run by hand.
diff --git a/dap4_test/test_hyrax.sh b/dap4_test/test_hyrax.sh
index c370e41150..5360f8192f 100755
--- a/dap4_test/test_hyrax.sh
+++ b/dap4_test/test_hyrax.sh
@@ -3,6 +3,7 @@
 if test "x$srcdir" = "x"; then srcdir=`dirname $0`; fi
 export srcdir;
 
+
 . ../test_common.sh
 
 . ${srcdir}/d4test_common.sh
@@ -61,7 +62,8 @@ hyraxsplit() {
     PREFIX=`dirname $P`
 }
 
-if test "x${RESET}" = x1 ; then rm -fr ${BASELINEHY}/*.hyrax ; fi
+# Turn on only if regenerating test baseline
+#if test "x${RESET}" = x1 ; then rm -fr ${BASELINEHY}/*.hyrax ; fi
 for f in $F ; do
   hyraxsplit $f
   makehyraxurl
diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h
index adc7e456be..f81f670efc 100644
--- a/include/ncs3sdk.h
+++ b/include/ncs3sdk.h
@@ -14,10 +14,16 @@
 
 /* Track the server type, if known */
 typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
-                 NCS3=1,    /* s3.amazon.aws */
-		 NCS3GS=2   /* storage.googleapis.com */
+	NCS3=1,     /* s3.amazon.aws */
+	NCS3GS=2,   /* storage.googleapis.com */
+#ifdef NETCDF_ENABLE_ZOH
+	NCS3ZOH=4,  /* ZoH Server */
+#endif
 } NCS3SVC;
 
+/* Opaque Handles */
+struct NClist;
+
 typedef struct NCS3INFO {
     char* host; /* non-null if other*/
     char* region; /* region */
@@ -55,9 +61,10 @@ EXTERNL int NC_s3sdkbucketdelete(void* s3client, NCS3INFO* info, char** errmsgp)
 EXTERNL int NC_s3sdkinfo(void* client0, const char* bucket, const char* pathkey, unsigned long long* lenp, char** errmsgp);
 EXTERNL int NC_s3sdkread(void* client0, const char* bucket, const char* pathkey, unsigned long long start, unsigned long long count, void* content, char** errmsgp);
 EXTERNL int NC_s3sdkwriteobject(void* client0, const char* bucket, const char* pathkey, unsigned long long count, const void* content, char** errmsgp);
-EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp);
-EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
-EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
+EXTERNL int NC_s3sdkclose(void* s3client0, char** errmsgp);
+EXTERNL int NC_s3sdktruncate(void* s3client0, const char* bucket, const char* prefix, char** errmsgp);
+EXTERNL int NC_s3sdklist(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
+EXTERNL int NC_s3sdklistall(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
 EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
 
 /* From ds3util.c */
diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c
index 2b81f342bc..9401ace692 100644
--- a/libdispatch/ds3util.c
+++ b/libdispatch/ds3util.c
@@ -25,6 +25,7 @@
 #include "nc4internal.h"
 #include "ncuri.h"
 #include "nclist.h"
+#include "ncbytes.h"
 #include "ncrc.h"
 #include "nclog.h"
 #include "ncs3sdk.h"
@@ -34,9 +35,6 @@
 /* Alternate .aws directory location */
 #define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR"
 
-#define AWSHOST ".amazonaws.com"
-#define GOOGLEHOST "storage.googleapis.com"
-
 enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4};
 
 /* Read these files in order and later overriding earlier */
@@ -47,6 +45,7 @@ static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
 /* Forward */
 
 static int endswith(const char* s, const char* suffix);
+static void freeprofile(struct AWSprofile* profile);
 static void freeentry(struct AWSentry* e);
 static int awsparse(const char* text, NClist* profiles);
 
@@ -78,9 +77,9 @@ NC_s3sdkenvironment(void)
 /*
 Rebuild an S3 url into a canonical path-style url.
 If region is not in the host, then use specified region
-if provided, otherwise leave blank and let the S3 server deal with it.
-@param url (in) the current url
-@param s3	(in/out) NCS3INFO structure
+if provided, otherwise us-east-1.
+@param url      (in) the current url
+@param s3       (in/out) NCS3INFO structure
 @param pathurlp (out) the resulting pathified url string
 */
 
@@ -309,7 +308,13 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
 
     /* Rebuild the URL to path format and get a usable region and optional bucket*/
     if((stat = NC_s3urlrebuild(url,s3,&url2))) goto done;
-    s3->host = strdup(url2->host);
+    if(url2->port){
+	char hostport[8192];
+	snprintf(hostport,sizeof(hostport),"%s:%s",url2->host,url2->port);
+	s3->host = strdup(hostport);
+    }else{
+        s3->host = strdup(url2->host);
+    }
     /* construct the rootkey minus the leading bucket */
     pathsegments = nclistnew();
     if((stat = NC_split_delim(url2->path,'/',pathsegments))) goto done;
@@ -358,19 +363,29 @@ NC_s3clear(NCS3INFO* s3)
 }
 
 /*
-Check if a url has indicators that signal an S3 or Google S3 url.
+Check if a url has indicators that signal an S3 or Google S3 url or ZoH S3 url.
+The rules are as follows:
+1. If the protocol is "s3" or "gs3" or "zoh", then return (true,s3|gs3|zoh).
+2. If the mode contains "s3" or "gs3" or "zoh", then return (true,s3|gs3|zoh).
+3. Check the host name:
+3.1 If the host ends with ".amazonaws.com", then return (true,s3).
+3.1 If the host is "storage.googleapis.com", then return (true,gs3).
+4. Otherwise return (false,unknown).
 */
 
 int
-NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
+NC_iss3(NCURI* uri, NCS3SVC* svcp)
 {
     int iss3 = 0;
     NCS3SVC svc = NCS3UNK;
 
     if(uri == NULL) goto done; /* not a uri */
-    /* is the protocol "s3" or "gs3" ? */
+    /* is the protocol "s3" or "gs3" or "zoh" ? */
     if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; svc = NCS3; goto done;}
     if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; svc = NCS3GS; goto done;}
+#ifdef NETCDF_ENABLE_ZOH
+    if(strcasecmp(uri->protocol,"zoh")==0) {iss3 = 1; svc = NCS3ZOH; goto done;}
+#endif
     /* Is "s3" or "gs3" in the mode list? */
     if(NC_testmode(uri,"s3")) {iss3 = 1; svc = NCS3; goto done;}
     if(NC_testmode(uri,"gs3")) {iss3 = 1; svc = NCS3GS; goto done;}    
@@ -384,18 +399,71 @@ NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
     return iss3;
 }
 
-const char*
-NC_s3dumps3info(NCS3INFO* info)
+/**************************************************/
+/**
+The .aws/config and .aws/credentials files
+are in INI format (https://en.wikipedia.org/wiki/INI_file).
+This format is not well defined, so the grammar used
+here is restrictive. Here, the term "profile" is the same
+as the INI term "section".
+
+The grammar used is as follows:
+
+Grammar:
+
+inifile: profilelist ;
+profilelist: profile | profilelist profile ;
+profile: '[' profilename ']' EOL entries ;
+entries: empty | entries entry ;
+entry:  WORD = WORD EOL ;
+profilename: WORD ;
+Lexical:
+WORD    sequence of printable characters - [ \[\]=]+
+EOL	'\n' | ';'
+
+Note:
+1. The semicolon at beginning of a line signals a comment.
+2. # comments are not allowed
+3. Duplicate profiles or keys are ignored.
+4. Escape characters are not supported.
+*/
+
+#define AWS_EOF (-1)
+#define AWS_ERR (0)
+#define AWS_WORD (0x10001)
+#define AWS_EOL (0x10002)
+
+typedef struct AWSparser {
+    char* text;
+    char* pos;
+    size_t yylen; /* |yytext| */
+    NCbytes* yytext;
+    int token; /* last token found */
+    int pushback; /* allow 1-token pushback */
+} AWSparser;
+
+#ifdef LEXDEBUG
+static const char*
+tokenname(int token)
 {
-    static char text[8192];
-    snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
-		(info->host?info->host:"null"),
-		(info->region?info->region:"null"),
-		(info->bucket?info->bucket:"null"),
-		(info->rootkey?info->rootkey:"null"),
-		(info->profile?info->profile:"null"));
-    return text;
+    static char num[32];
+    switch(token) {
+    case AWS_EOF: return "EOF";
+    case AWS_ERR: return "ERR";
+    case AWS_WORD: return "WORD";
+    default: snprintf(num,sizeof(num),"%d",token); return num;
+    }
+    return "UNKNOWN";
 }
+#endif
+
+/*
+@param text of the aws credentials file
+@param profiles list of form struct AWSprofile (see ncauth.h)
+*/
+
+#define LBR '['
+#define RBR ']'
 
 static void
 freeprofile(struct AWSprofile* profile)
@@ -426,6 +494,19 @@ NC_s3freeprofilelist(NClist* profiles)
     }
 }
 
+const char*
+NC_s3dumps3info(NCS3INFO* info)
+{
+    static char text[8192];
+    snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
+		(info->host?info->host:"null"),
+		(info->region?info->region:"null"),
+		(info->bucket?info->bucket:"null"),
+		(info->rootkey?info->rootkey:"null"),
+		(info->profile?info->profile:"null"));
+    return text;
+}
+
 /* Find, load, and parse the aws config &/or credentials file */
 int
 NC_aws_load_credentials(NCglobalstate* gstate)
@@ -707,15 +788,6 @@ tokenname(int token)
 }
 #endif
 
-typedef struct AWSparser {
-    char* text;
-    char* pos;
-    size_t yylen; /* |yytext| */
-    NCbytes* yytext;
-    int token; /* last token found */
-    int pushback; /* allow 1-token pushback */
-} AWSparser;
-
 static int
 awslex(AWSparser* parser)
 {
diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c
index 29aedea326..0fc97deb6f 100644
--- a/libdispatch/nch5s3comms.c
+++ b/libdispatch/nch5s3comms.c
@@ -92,7 +92,7 @@
 
 #include "netcdf.h"
 #include "ncuri.h"
-#include "ncutil.h"
+#include "netcdf_vutils.h"
 
 /*****************/
 
@@ -292,7 +292,6 @@ static int build_range(size_t offset, size_t len, char** rangep);
 static const char* verbtext(HTTPVerb verb);
 static int trace(CURL* curl, int onoff);
 static int sortheaders(VList* headers);
-static int httptonc(long httpcode);
 static void hrb_node_free(hrb_node_t *node);
 
 #if S3COMMS_DEBUG_HRB
@@ -789,7 +788,7 @@ NCH5_s3comms_s3r_close(s3r_t *handle)
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
+NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep, long* httpcodep)
 {
     int ret_value      = SUCCEED;
     char* contentlength = NULL;
@@ -806,7 +805,8 @@ NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
     if((ret_value = NCH5_s3comms_s3r_head(handle, url, "Content-Length", NULL, &httpcode, &contentlength)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "NCH5_s3comms_s3r_head failed.");
 
-    if((ret_value = httptonc(httpcode))) goto done;
+    /* Content-Length will not be defined if object does not exist */
+    if(httpcode == 404) goto done;
 
     /******************
      * PARSE RESPONSE *
@@ -816,7 +816,7 @@ NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
     if(value == NULL)
         HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not find content length value");
     value++;
-    content_length = strtoumax(value, NULL, 0);
+    content_length = (long long)strtoumax(value, NULL, 0);
     if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX)
         HGOTO_ERROR(H5E_ARGS, NC_ERANGE, FAIL, "content_length overflows size_t");
 
@@ -825,9 +825,9 @@ NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
                     "could not convert found \"Content-Length\" response (\"%s\")",
                     contentlength); /* range is null-terminated, remember */
 
-    if(sizep) {*sizep = (long long)content_length;}
-
 done:
+    if(sizep) {*sizep = (long long)content_length;}
+    if(httpcodep) *httpcodep = httpcode;    
     nullfree(contentlength);
     return UNTRACEX(ret_value,"size=%lld",(sizep?-1:*sizep));
 } /* NCH5_s3comms_s3r_getsize */
@@ -859,17 +859,13 @@ NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep)
 
     if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPDELETE, NULL, NULL, NULL, &httpcode, data)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-
-    /******************
-     * RESPONSE *
-     ******************/
-    if((ret_value = httptonc(httpcode))) goto done;
-    if(httpcode != 204) 
-        HGOTO_ERROR(H5E_ARGS, NC_ECANTREMOVE, FAIL, "deletekey failed.");     
+    
+    /* Apparently, aws delivers a 204 response if it successfully deletes the key */
+    if(httpcode == 204) httpcode = 200; /* treat 204 as success */
 
 done:
-    vsfree(data);
     if(httpcodep) *httpcodep = httpcode;
+    vsfree(data);
     return UNTRACEX(ret_value,"httpcode=%d",INULL(httpcodep));
 } /* NCH5_s3comms_s3r_getsize */
 
@@ -911,12 +907,8 @@ NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const
     if (SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPHEAD, NULL, header, NULL, &httpcode, data))
         HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem in reading during getsize.");
 
-    if((ret_value = httptonc(httpcode))) goto done;
-
     if(header != NULL) {
-        if(vslength(data) == 0)
-            HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata: header=%s; not found",header);
-        else if (vslength(data) > CURL_MAX_HTTP_HEADER)
+        if (vslength(data) > CURL_MAX_HTTP_HEADER)
             HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata buffer overrun");
 #if S3COMMS_DEBUG
         else
@@ -930,7 +922,7 @@ NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const
 
     if(header != NULL) {
         char* content;
-	content = vsextract(data);
+	content = vsextract(data); /* guaranteed to exist */
         if(valuep) {*valuep = content;}
     }
 
@@ -1222,7 +1214,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* dest)
+NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* dest, long* httpcodep)
 {
     char              *rangebytesstr = NULL;
     int                ret_value = SUCCEED;
@@ -1251,9 +1243,9 @@ NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len,
 
     if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPGET, rangebytesstr, NULL, NULL, &httpcode, wrap)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-    if((ret_value = httptonc(httpcode))) goto done;
 
 done:
+    if(httpcodep) *httpcodep = httpcode;
     (void)vsextract(wrap);
     vsfree(wrap);
     /* clean any malloc'd resources */
@@ -1271,7 +1263,7 @@ NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len,
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data)
+NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data, long* httpcodep)
 {
     int ret_value = SUCCEED;
     VList* otherheaders = vlistnew();
@@ -1301,9 +1293,10 @@ NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data)
     vssetlength(wrap,data->count);
     if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPPUT, NULL, NULL, (const char**)vlistcontents(otherheaders), &httpcode, wrap)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-    if((ret_value = httptonc(httpcode))) goto done;
+
     
 done:
+    if(httpcodep) *httpcodep = httpcode;
     (void)vsextract(wrap);
     vsfree(wrap);
     /* clean any malloc'd resources */
@@ -1321,7 +1314,7 @@ NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data)
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response)
+NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response, long* httpcodep)
 {
     int ret_value = SUCCEED;
     const char* otherheaders[3] = {"Content-Type", "application/xml", NULL};
@@ -1340,13 +1333,13 @@ NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response)
 
     if((SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPGET, NULL, NULL, otherheaders, &httpcode, content)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-    if((ret_value = httptonc(httpcode))) goto done;
     if(response) {
 	response->count = vslength(content);
 	response->content = vsextract(content);
     }
 
 done:
+    if(httpcodep) *httpcodep = httpcode;
     vsfree(content);
     /* clean any malloc'd resources */
     curl_reset(handle);
@@ -1421,7 +1414,7 @@ NCH5_s3comms_aws_canonical_request(VString* canonical_request_dest, VString* sig
 {
     hrb_node_t *node         = NULL;
     int      ret_value    = SUCCEED;
-    int i;
+    size_t i;
     
     const char* sverb = verbtext(verb);
     const char* query_params = (query?query:"");
@@ -1568,7 +1561,7 @@ NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m
 #else
     if(CURLE_OK != Curl_hmacit(Curl_HMAC_SHA256,
                      key, key_len,
-                     msg, msg_len,
+                     (const unsigned char*)msg, msg_len,
                      md))
         HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "Curl_hmacit failure.");
 #endif
@@ -2052,7 +2045,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re
     HMAC(EVP_sha256(), (const unsigned char *)dateregionservicekey, SHA256_DIGEST_LENGTH,
          (const unsigned char *)"aws4_request", 12, md, NULL);
 #else
-    Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)AWS4_secret, (int)nulllen(AWS4_secret),
+    Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)AWS4_secret, nulllen(AWS4_secret),
          (const unsigned char *)iso8601now, 8, /* 8 --> length of 8 --> "yyyyMMDD"  */
          datekey);
     Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)datekey, SHA256_DIGEST_LENGTH, (const unsigned char *)region,
@@ -2390,7 +2383,8 @@ build_request(s3r_t* handle, NCURI* purl,
               VString* payload,
               HTTPVerb verb)
 {
-    int i,ret_value = SUCCEED;
+    int ret_value = SUCCEED;
+    size_t i;
     struct curl_slist *curlheaders   = NULL;
     hrb_node_t        *node          = NULL;
     hrb_t             *request       = NULL;
@@ -2788,32 +2782,6 @@ sortheaders(VList* headers)
     return (ret_value);
 }
 
-static int
-httptonc(long httpcode)
-{
-    int stat = NC_NOERR;
-    if(httpcode <= 99) stat = NC_EINTERNAL; /* should never happen */
-    else if(httpcode <= 199)
-        stat = NC_NOERR; /* I guess */
-    else if(httpcode <= 299) {
-        switch (httpcode) {
-        default: stat = NC_NOERR; break;
-        }
-    } else if(httpcode <= 399)
-        stat = NC_NOERR; /* ? */
-    else if(httpcode <= 499) {
-        switch (httpcode) {
-        case 400: stat = NC_EINVAL; break;
-        case 401: case 402: case 403:
-            stat = NC_EAUTH; break;
-        case 404: stat = NC_EEMPTY; break;
-        default: stat = NC_EINVAL; break;
-        }
-    } else
-        stat = NC_ES3;
-    return stat;
-}
-
 /**************************************************/
 /* Request Tracing */
 
diff --git a/libdispatch/nch5s3comms.h b/libdispatch/nch5s3comms.h
index 7cc482df66..f7c383e037 100644
--- a/libdispatch/nch5s3comms.h
+++ b/libdispatch/nch5s3comms.h
@@ -506,13 +506,13 @@ EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char*
 
 EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle);
 
-EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* data);
+EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* data, long* httpcodep);
 
-EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data);
+EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data, long* httpcodep);
 
-EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response);
+EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response, long* httpcodep);
 
-EXTERNL int NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long * sizep);
+EXTERNL int NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long * sizep, long* httpcodep);
 
 EXTERNL int NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep);
 
diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c
index 359ab2f8b3..27bb0e2038 100644
--- a/libdispatch/ncs3sdk_h5.c
+++ b/libdispatch/ncs3sdk_h5.c
@@ -101,6 +101,7 @@ static int s3objectsinfo(NClist* contents, NClist* keys, NClist* lens);
 static int s3commonprefixes(NClist* list, NClist* keys);
 static int mergekeysets(NClist*,NClist*,NClist*);
 static int rawtokeys(s3r_buf_t* response, NClist* keys, NClist* lengths, struct LISTOBJECTSV2** listv2p);
+static int httptonc(long httpcode);
 
 static int queryadd(NClist* query, const char* key, const char* value);
 static int queryend(NClist* query, char** querystring);
@@ -193,7 +194,7 @@ NC_s3sdkcreateclient(NCS3INFO* info)
 done:
     nullfree(urlroot);
     if(stat && s3client) {
-        NC_s3sdkclose(s3client,info,0,NULL);
+        NC_s3sdkclose(s3client,NULL);
 	s3client = NULL;
     }
     NCNILTRACE(NC_NOERR);
@@ -213,6 +214,7 @@ NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** e
 
     if((stat = makes3fullpath(s3client->rooturl,bucket,NULL,NULL,url))) goto done;
     if((stat = NCH5_s3comms_s3r_head(s3client->h5s3client, ncbytescontents(url), NULL, NULL, &httpcode, NULL))) goto done;
+    stat = httptonc(httpcode);
 
     if(existsp) {*existsp = (stat == 0 && httpcode == 200);}
 done:
@@ -255,7 +257,7 @@ NC_s3sdkbucketdelete(void* s3client0, NCS3INFO* info, char** errmsgp)
 
 /*
 @return NC_NOERR if key points to a content-bearing object.
-@return NC_EEMPTY if object at key has no content.
+@return NC_ENOOBJECT if object at key does not exist
 @return NC_EXXX return true error
 */
 EXTERNL int
@@ -265,13 +267,15 @@ NC_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t*
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
     NCbytes* url = ncbytesnew();
     long long len = -1;
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
 
     if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
-    if((stat = NCH5_s3comms_s3r_getsize(s3client->h5s3client, ncbytescontents(url), &len))) goto done;
+    if((stat = NCH5_s3comms_s3r_getsize(s3client->h5s3client, ncbytescontents(url), &len, &httpcode))) goto done;
+    stat = httptonc(httpcode);
 
-    if(lenp) {*lenp = len;}
+    if(lenp) {*lenp = (size64_t)len;}
 
 done:
     ncbytesfree(url);
@@ -289,6 +293,7 @@ NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
     NCbytes* url = ncbytesnew();
     struct s3r_buf_t data = {0,NULL};
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s pathkey=%s start=%llu count=%llu content=%p",bucket,pathkey,start,count,content);
 
@@ -297,8 +302,8 @@ NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t
     /* Read the data */
     data.count = count;
     data.content = content;
-    if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,&data))) goto done;
-    
+    if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,&data,&httpcode))) goto done;
+    stat = httptonc(httpcode);    
 done:
     ncbytesfree(url);
     return NCUNTRACE(stat);
@@ -315,6 +320,7 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
     NCbytes* url = ncbytesnew();
     s3r_buf_t data;
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s pathkey=%s count=%llu content=%p",bucket,pathkey,count,content);
 
@@ -323,7 +329,8 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
     /* Write the data */
     data.count = count;
     data.content = (void*)content;
-    if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),&data))) goto done;
+    if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),&data,&httpcode))) goto done;
+    stat = httptonc(httpcode);
     
 done:
     ncbytesfree(url);
@@ -331,28 +338,49 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
 }
 
 EXTERNL int
-NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp)
+NC_s3sdkclose(void* s3client0, char** errmsgp)
 {
     int stat = NC_NOERR;
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
 
-    NCTRACE(11,"info=%s deleteit=%d",NC_s3dumps3info(info),deleteit);
-    
-    if(deleteit) {
-        /* Delete the root key; ok it if does not exist */
-        switch (stat = NC_s3sdkdeletekey(s3client0,info->bucket,info->rootkey,errmsgp)) {
-        case NC_NOERR: break;
-        case NC_EEMPTY: case NC_ENOTFOUND: stat = NC_NOERR; break;
-        default: break;
+    NCTRACE(11,"");
+    s3client_destroy(s3client);
+    return NCUNTRACE(stat);
+}
+
+EXTERNL int
+NC_s3sdktruncate(void* s3client0, const char* bucket, const char* prefix, char** errmsgp)
+{
+    int stat = NC_NOERR;
+    char* errmsg = NULL;
+    size_t nkeys;
+    char** keys = NULL;
+    NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
+
+    NCTRACE(11,"bucket=%s prefix=%s",bucket,prefix);
+
+    if((stat = NC_s3sdklistall(s3client0,bucket,prefix,&nkeys,&keys,&errmsg))) goto done;
+
+    if(nkeys > 0 && keys != NULL) {
+	size_t i;
+	/* Sort the list -- shortest first */
+	NC_sortenvv(nkeys,keys);
+	for(i=0;i<nkeys;i++) {
+	    if((stat = NC_s3sdkdeletekey(s3client, bucket, keys[i], NULL))) goto done;
         }
     }
-    s3client_destroy(s3client);
+
+    if(errmsgp) {*errmsgp = errmsg; errmsg = NULL;}
+
+done:
+    nullfree(errmsg);
+    NC_freeenvv(nkeys,keys);    
     return NCUNTRACE(stat);
 }
 
 /*
-Common code for getkeys and searchkeys.
-Return a list of names of legal objects immediately below a specified key.
+Common code for list and listall.
+Return a list of names of legal objects immediately or anywhere below a specified key.
 In theory, the returned list should be sorted in lexical order,
 but it possible that it is not.
 */
@@ -371,6 +399,7 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
     int istruncated = 0;
     char* continuetoken = NULL;
     s3r_buf_t response = {0,NULL};
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
     
@@ -401,7 +430,8 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
         ncbytescat(listurl,"?");
         ncbytescat(listurl,querystring);
 
-        if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), &response))) goto done;
+        if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), &response, &httpcode))) goto done;
+	if((stat = httptonc(httpcode))) goto done;
         if((stat = rawtokeys(&response,allkeys,NULL,&listv2))) goto done;
 	istruncated = (strcasecmp(listv2->istruncated,"true")==0?1:0);
 	nullfree(continuetoken);
@@ -430,19 +460,19 @@ In theory, the returned list should be sorted in lexical order,
 but it possible that it is not.
 */
 EXTERNL int
-NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
+NC_s3sdklist(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
 {
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
     return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp));
 }
 
 /*
-Return a list of full keys  of legal objects immediately below a specified key.
+Return a list of full keys  of legal objects below a specified key.
 Not necessarily sorted.
 Essentially same as getkeys, but with no delimiter.
 */
 EXTERNL int
-NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
+NC_s3sdklistall(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
 {
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
     return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp));
@@ -461,6 +491,11 @@ NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char
     if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
 
     if((stat = NCH5_s3comms_s3r_deletekey(s3client->h5s3client, ncbytescontents(url), &httpcode))) goto done;
+    switch(stat = httptonc(httpcode)) {
+    case NC_NOERR: break;
+    case NC_ENOOBJECT: break; /* does not exist */
+    default: goto done;
+    }
 
 done:
     ncbytesfree(url);
@@ -700,8 +735,8 @@ parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV
     if(resultp) {*resultp = result; result = NULL;}
 
 done:
-    if(result) reclaim_listobjectsv2(result);
     if(doc) ncxml_free(doc);
+    if(result) reclaim_listobjectsv2(result);
     return NCTHROW(stat);
 }
 
@@ -1061,3 +1096,33 @@ queryinsert(NClist* list, char* ekey, char* evalue)
     return NCTHROW(stat);
 }
 
+static int
+httptonc(long httpcode)
+{
+    int stat = NC_NOERR;
+    if(httpcode == 0)
+        stat = NC_NOERR;
+    else if(httpcode <= 99)
+        stat = NC_EINTERNAL; /* should never happen */
+    else if(httpcode <= 199)
+        stat = NC_NOERR; /* I guess */
+    else if(httpcode <= 299) {
+        switch (httpcode) {
+	case 200: stat = NC_NOERR; break;
+	case 204: stat = NC_ENOOBJECT; break;
+        default: stat = NC_NOERR; break;
+        }
+    } else if(httpcode <= 399)
+        stat = NC_NOERR; /* ? */
+    else if(httpcode <= 499) {
+        switch (httpcode) {
+        case 400: stat = NC_EINVAL; break;
+        case 401: case 402: case 403:
+            stat = NC_EAUTH; break;
+        case 404: stat = NC_ENOOBJECT; break;
+        default: stat = NC_EINVAL; break;
+        }
+    } else
+        stat = NC_ES3;
+    return stat;
+}
diff --git a/libncxml/ncxml_tinyxml2.cpp b/libncxml/ncxml_tinyxml2.cpp
index 4307fa630a..3f150d26df 100644
--- a/libncxml/ncxml_tinyxml2.cpp
+++ b/libncxml/ncxml_tinyxml2.cpp
@@ -14,6 +14,10 @@
 
 using namespace tinyxml2;
 
+#ifdef _MSC_VER /*Do not use _WIN32 since this is a visual studio issue */
+#define XMLDocument tinyxml2::XMLDocument
+#endif
+
 static int ncxml_initialized = 0;
 
 void
diff --git a/libsrc/s3io.c b/libsrc/s3io.c
index f8bd941b42..d4381da7a9 100644
--- a/libsrc/s3io.c
+++ b/libsrc/s3io.c
@@ -184,7 +184,7 @@ s3io_open(const char* path,
     /* Get the size */
     switch (status = NC_s3sdkinfo(s3io->s3client,s3io->s3.bucket,s3io->s3.rootkey,(long long unsigned*)&s3io->size,&s3io->errmsg)) {
     case NC_NOERR: break;
-    case NC_EEMPTY:
+    case NC_ENOOBJECT:
         s3io->size = 0;
 	goto done;
     default:
@@ -253,7 +253,9 @@ s3io_close(ncio* nciop, int deleteit)
     assert(s3io != NULL);
 
     if(s3io->s3client && s3io->s3.bucket && s3io->s3.rootkey) {
-        NC_s3sdkclose(s3io->s3client, &s3io->s3, deleteit, &s3io->errmsg);
+	if(deleteit)
+	    NC_s3sdktruncate(s3io->s3client, s3io->s3.bucket, s3io->s3.rootkey, &s3io->errmsg);
+        NC_s3sdkclose(s3io->s3client, &s3io->errmsg);
     }
     s3io->s3client = NULL;
     NC_s3clear(&s3io->s3);
diff --git a/nczarr_test/s3util.c b/nczarr_test/s3util.c
index 9f0c433f73..56fc0cda54 100644
--- a/nczarr_test/s3util.c
+++ b/nczarr_test/s3util.c
@@ -83,7 +83,7 @@ static void printcontent(size64_t len, const char* content, nc_type nctype);
 static void
 usage(void)
 {
-    fprintf(stderr,"usage: s3util  list|print|upload|download|clear -u <url> [-k <key.] [-f <filename>]\n");
+    fprintf(stderr,"usage: s3util  list|print|upload|download|clear -u <url> [-k <key>] [-f <filename>]\n");
     exit(1);
 }
 
@@ -112,7 +112,10 @@ static int
 s3shutdown(int deleteit)
 {
     int stat = NC_NOERR;
-    stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.s3, deleteit, &s3sdk.errmsg);
+    if(deleteit) {
+        stat = s3clear();
+    }
+    stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.errmsg);
     return stat;
 }
 
@@ -258,13 +261,13 @@ s3list(void)
 
     if(s3setup()) goto done;
 
-    stat = NC_s3sdksearch(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
+    stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
     if(stat) goto done;
 
     if(nkeys > 0) {
         size_t i;
 	/* Sort the list -- shortest first */
-	nczm_sortenvv(nkeys,keys);
+	NC_sortenvv(nkeys,keys);
 	for(i=0;i<nkeys;i++) {
 	    printf("[%u] %s\n",(unsigned)i,keys[i]);
 	}
@@ -287,12 +290,12 @@ s3clear(void)
 
     if(s3setup()) goto done;
 
-    if((stat = NC_s3sdksearch(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done;
+    if((stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done;
 
     if(nkeys > 0 && keys != NULL) {
 	size_t i;
 	/* Sort the list -- shortest first */
-	nczm_sortenvv(nkeys,keys);
+	NC_sortenvv(nkeys,keys);
 	if(dumpoptions.verbose) {
             printf("deleted keys:\n");
 	    for(i=0;i<nkeys;i++) {
diff --git a/s3cleanup.in b/s3cleanup.in
index 22504717e0..248c7d3037 100755
--- a/s3cleanup.in
+++ b/s3cleanup.in
@@ -67,7 +67,11 @@ if test "x$VERBOSE" = x1 ; then set +x; fi
 for key in $ALLKEYS ; do
     for uid in $UIDS ; do
         case "$key" in
-            "$S3TESTSUBTREE/testset_${uid}"*)
+            "$S3TESTSUBTREE/nczarr_test_${uid}"*)
+               # capture the key'
+               MATCHKEYS="$MATCHKEYS $key"
+               ;;
+            "$S3TESTSUBTREE/v3_nczarr_test_${uid}"*)
                 # capture the key'
                 MATCHKEYS="$MATCHKEYS $key"
                 ;;
diff --git a/s3gc.in b/s3gc.in
index a63345c6c7..543e151943 100755
--- a/s3gc.in
+++ b/s3gc.in
@@ -74,24 +74,32 @@ done < s3gc.keys
 # Look at each key and see if it is less than lastdate.
 # If so, then record that key
 
+captureuid() {
+    PFX="$1"
+    key="$2"
+    # Capture the uid for this key         
+    s3uid=`echo $key | sed -e "s|$S3TESTSUBTREE/${PFX}_\([0-9][0-9]*\)/.*|\1|"`
+    # check that we got a uid       
+    if test "x$s3uid" != x ; then
+        # Test age of the uid
+        if test $((s3uid < lastdate)) = 1; then
+    	    MATCHKEYS="${MATCHKEYS} $key"
+	fi
+    else
+        if test "x$VERBOSE" = x1 ; then echo "Excluding \"$key\""; fi
+    fi
+}
+
 # Capture the keys with old uids to delete
 unset MATCHKEYS
 for key in $ALLKEYS ; do
     case "$key" in
-        "$S3TESTSUBTREE/testset_"*)
-             # Capture the uid for this key         
-             s3uid=`echo $key | sed -e "s|$S3TESTSUBTREE/testset_\([0-9][0-9]*\)/.*|\1|"`
-             # check that we got a uid       
-             if test "x$s3uid" != x ; then
-                # Test age of the uid
-                if test $((s3uid < lastdate)) = 1; then
-                    MATCHKEYS="${MATCHKEYS} $key"
-                fi
-             else
-                 if test "x$VERBOSE" = x1 ; then echo "Excluding \"$key\""; fi
-             fi
-	     ;;
-        *) if test "x$VERBOSE" = x1; then echo "Ignoring \"$key\""; fi ;;
+        "$S3TESTSUBTREE/alltests_"*) captureuid "alltests" "${key}" ;;
+        "$S3TESTSUBTREE/nczarr_test_"*) captureuid "nczarr_test" "${key}" ;;
+        "$S3TESTSUBTREE/v3_nczarr_test_"*) captureuid "v3_nczarr_test" "${key}" ;;
+        "$S3TESTSUBTREE/v3_tests_"*) captureuid "v3_tests" "${key}" ;;
+        "$S3TESTSUBTREE/testset_"*) captureuid "testset" "${key}" ;;
+    *) if test "x$VERBOSE" = x1; then echo "Ignoring \"$key\""; fi ;;
     esac
 done
 
diff --git a/unit_test/run_s3sdk.sh b/unit_test/run_s3sdk.sh
index 18073e4570..ce7fb9d092 100755
--- a/unit_test/run_s3sdk.sh
+++ b/unit_test/run_s3sdk.sh
@@ -7,14 +7,14 @@ set -e
 
 #CMD="valgrind --leak-check=full"
 
-URL="https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}"
+URL="https://${S3ENDPOINT}/${S3TESTBUCKET}"
 
 isolate "testdir_uts3sdk"
 
 # Create an isolation path for S3; build on the isolation directory
 S3ISODIR="$ISODIR"
 S3ISOPATH="/${S3TESTSUBTREE}"
-S3ISOPATH="/${S3ISOPATH}/$S3ISODIR"
+S3ISOPATH="${S3ISOPATH}/$S3ISODIR"
 
 test_cleanup() {
 ${CMD} ${execdir}/../nczarr_test/s3util -u "${URL}" -k "${S3ISOPATH}" clear
@@ -47,13 +47,17 @@ ${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}"                list
 echo "Status: $?"
 
 echo -e "\to Checking search command for ${URL}"
-${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}"                search
+${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "/object_store"               listall
 echo "Status: $?"
 
 echo -e "\to Checking delete command for ${URL}/test_s3sdk.txt"
 ${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}/test_s3sdk.txt" delete
 echo "Status: $?"
 
+echo -e "\to Checking delete command for non-existent ${URL}/test_s3sdk_x.txt"
+${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}/test_s3sdk_x.txt" delete
+echo "Status: $?"
+
 if test "x$FEATURE_LARGE_TESTS" = xyes ; then
     echo -e "\to Checking longlist command for ${URL}"
     ${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}"                longlist
@@ -62,7 +66,6 @@ fi
 
 echo -e "Finished"
 
-exit
 if test "x$GITHUB_ACTIONS" = xtrue; then
 # Cleanup on exit
 test_cleanup
diff --git a/unit_test/test_s3sdk.c b/unit_test/test_s3sdk.c
index 93557a18f8..896f703c07 100644
--- a/unit_test/test_s3sdk.c
+++ b/unit_test/test_s3sdk.c
@@ -30,7 +30,7 @@
 
 #define LONGCOUNT 1010
 
-enum Actions {ERROR_ACTION, EXISTS_ACTION, SIZE_ACTION, READ_ACTION, WRITE_ACTION, DELETE_ACTION, LIST_ACTION, LONGLIST_ACTION, SEARCH_ACTION};
+enum Actions {ERROR_ACTION, EXISTS_ACTION, SIZE_ACTION, READ_ACTION, WRITE_ACTION, DELETE_ACTION, LIST_ACTION, LONGLIST_ACTION, LISTALL_ACTION};
 
 struct Options {
     int debug;
@@ -78,7 +78,7 @@ actionfor(const char* s)
     else if(strcasecmp(s,"write")==0) return WRITE_ACTION;
     else if(strcasecmp(s,"list")==0) return LIST_ACTION;
     else if(strcasecmp(s,"longlist")==0) return LONGLIST_ACTION;
-    else if(strcasecmp(s,"search")==0) return SEARCH_ACTION;
+    else if(strcasecmp(s,"listall")==0) return LISTALL_ACTION;
     else if(strcasecmp(s,"delete")==0) return DELETE_ACTION;
     return ERROR_ACTION;
 }
@@ -126,6 +126,39 @@ profilesetup(const char* url)
     return stat;
 }
 
+#if 0
+static int
+s3treesetup(void)
+{
+    int stat = NC_NOERR;
+    size64_t size = 0;
+    void* content = NULL;
+    char key[8192];
+
+    seturl("https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}", "/netcdf-c/test_s3",!FORCE);
+
+    CHECK(profilesetup(dumpoptions.url));
+    newurl = ncuribuild(purl,NULL,NULL,NCURIALL);
+#ifdef DEBUG
+    printf("url=%s {url=%s bucket=%s region=%s profile=%s}\n",
+               dumpoptions.url,newurl,s3info.bucket,s3info.region,activeprofile);
+#endif
+    if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
+
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x/y");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x/y/z");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x/y/w");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+done:
+    cleanup();
+    return stat;
+}
+#endif
+
 static int
 testbucketexists(void)
 {
@@ -236,7 +269,7 @@ testwrite(void)
 }
 
 static int
-testgetkeys(void)
+testlist(void)
 {
     int stat = NC_NOERR;
     size_t i,nkeys = 0;
@@ -250,8 +283,8 @@ testgetkeys(void)
     printf("url=%s => info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info));
 #endif
     if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
-    CHECK(NC_s3sdkgetkeys(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
-    printf("testgetkeys: nkeys=%u; keys:\n",(unsigned)nkeys);
+    CHECK(NC_s3sdklist(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
+    printf("testlist: nkeys=%u; keys:\n",(unsigned)nkeys);
     for(i=0;i<nkeys;i++) {
         printf("\t|%s|\n",keys[i]);
     }
@@ -265,7 +298,7 @@ testgetkeys(void)
 }
 
 static int
-testgetkeyslong(void)
+testlistlong(void)
 {
     int stat = NC_NOERR;
     size_t i,nkeys = 0;
@@ -290,8 +323,8 @@ testgetkeyslong(void)
         snprintf(path,sizeof(path),"%s/getkey_%d",dumpoptions.key,(int)i);
         CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, path, strlen(uploaddata), uploaddata, NULL));
     }
-    CHECK(NC_s3sdkgetkeys(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
-    printf("testgetkeys: nkeys=%u; keys:\n",(unsigned)nkeys);
+    CHECK(NC_s3sdklist(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
+    printf("testlistlong: nkeys=%u; keys:\n",(unsigned)nkeys);
     if(nkeys != LONGCOUNT) {
         fprintf(stderr,"*** nkeys mismatch: create=%d found=%d\n",LONGCOUNT,(int)nkeys);
     }
@@ -320,9 +353,9 @@ testgetkeyslong(void)
         printf("\tkey=%s: ",keys[i]);
         stat = NC_s3sdkdeletekey(s3client, s3info.bucket, keys[i], NULL);
         switch (stat) {
-        case NC_NOERR:  printf("deleted\n");        break;
-        case NC_EEMPTY: printf("does not exist\n"); break;
-        default:        printf("failed\n");         break;
+        case NC_NOERR:     printf("deleted\n");        break;
+        case NC_ENOOBJECT: printf("does not exist\n"); break;
+        default:           printf("failed\n");         break;
         }
         stat = NC_NOERR; /* reset */
     }    
@@ -336,13 +369,13 @@ testgetkeyslong(void)
 }
 
 static int
-testsearch(void)
+testlistall(void)
 {
     int stat = NC_NOERR;
     size_t i,nkeys = 0;
     char** keys = NULL;
 
-    seturl("https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}", "/object_store/dir1",!FORCE);
+    seturl("https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}", "/object_store",!FORCE);
 
     CHECK(profilesetup(dumpoptions.url));
     newurl = ncuribuild(purl,NULL,NULL,NCURIALL);
@@ -350,8 +383,8 @@ testsearch(void)
     printf("url=%s => info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info));
 #endif
     if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
-    CHECK(NC_s3sdksearch(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
-    printf("testsearch: nkeys=%u; keys:\n",(unsigned)nkeys);
+    CHECK(NC_s3sdklistall(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
+    printf("testlistall: nkeys=%u; keys:\n",(unsigned)nkeys);
     for(i=0;i<nkeys;i++) {
         printf("\t|%s|\n",keys[i]);
     }
@@ -381,21 +414,21 @@ testdeletekey(void)
     if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
     stat = NC_s3sdkdeletekey(s3client, s3info.bucket, dumpoptions.key, NULL);
 
-    printf("testdeletekey: url %s: ",newurl);
+    printf("testdeletekey: url %s%s: ",newurl,dumpoptions.key);
     switch (stat) {
-    case NC_NOERR:  printf("deleted\n");        break;
-    case NC_EEMPTY: printf("does not exist\n"); break;
-    default:        printf("failed\n");         break;
+    case NC_NOERR:     printf("deleted\n");        break;
+    case NC_ENOOBJECT: printf("does not exist\n"); break;
+    default:           printf("failed\n");         break;
     }
     stat = NC_NOERR; /* reset */
     
     /* Verify deleted and size */
     stat = NC_s3sdkinfo(s3client, s3info.bucket, dumpoptions.key, &size, NULL);
-    printf("testdeletekey.info: url %s: ",newurl);
+    printf("testdeletekey: url %s%s: ",newurl,dumpoptions.key);
     switch (stat) {
-    case NC_NOERR:  printf("not deleted; size=%d\n",(int)size); break;
-    case NC_EEMPTY: printf("deleted\n"); break;
-    default:        printf("failed\n"); goto done;
+    case NC_NOERR:     printf("not deleted; size=%d\n",(int)size); break;
+    case NC_ENOOBJECT: printf("deleted\n"); break;
+    default:           printf("failed\n"); goto done;
     }
     stat = NC_NOERR; /* reset */
 
@@ -408,7 +441,7 @@ static void
 cleanup(void)
 {
     if(s3client)
-        NC_s3sdkclose(s3client, &s3info, 0/*deleteit*/, NULL);
+        NC_s3sdkclose(s3client, NULL);
     s3client = NULL;
     NC_s3clear(&s3info);
     ncurifree(purl); purl = NULL;
@@ -464,12 +497,12 @@ main(int argc, char** argv)
         if((stat = testread())) goto done;
         printf("Test: testinfo\n");
         if((stat = testinfo())) goto done;
-        printf("Test: testgetkeys\n");
-        if((stat = testgetkeys())) goto done;
-        printf("Test: testgetkeyslong\n");
-        if((stat = testgetkeyslong())) goto done;
-        printf("Test: testsearch\n");
-        if((stat = testsearch())) goto done;
+        printf("Test: testlist\n");
+        if((stat = testlist())) goto done;
+        printf("Test: testlistlong\n");
+        if((stat = testlistlong())) goto done;
+        printf("Test: testlistall\n");
+        if((stat = testlistall())) goto done;
         printf("Test: testdeletekey\n");
         if((stat = testdeletekey())) goto done;
     } else {
@@ -498,9 +531,9 @@ main(int argc, char** argv)
         case SIZE_ACTION: stat = testinfo(); break;
         case READ_ACTION: stat = testread(); break;
         case WRITE_ACTION: stat = testwrite(); break;
-        case LIST_ACTION: stat = testgetkeys(); break;
-        case LONGLIST_ACTION: stat = testgetkeyslong(); break;
-        case SEARCH_ACTION: stat = testsearch(); break;
+        case LIST_ACTION: stat = testlist(); break;
+        case LONGLIST_ACTION: stat = testlistlong(); break;
+        case LISTALL_ACTION: stat = testlistall(); break;
         case DELETE_ACTION: stat = testdeletekey(); break;
         case ERROR_ACTION: /* fall thru */
         default: fprintf(stderr,"Illegal action\n"); exit(1);
diff --git a/v3_nczarr_test/s3util.c b/v3_nczarr_test/s3util.c
new file mode 100644
index 0000000000..56fc0cda54
--- /dev/null
+++ b/v3_nczarr_test/s3util.c
@@ -0,0 +1,442 @@
+/*
+ *	Copyright 2018, University Corporation for Atmospheric Research
+ *      See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+#if defined(_WIN32) && !defined(__MINGW32__)
+#include "XGetopt.h"
+#endif
+
+#include "zincludes.h"
+#include "ncs3sdk.h"
+#include "ncpathmgr.h"
+#include "nclog.h"
+#include "ncuri.h"
+#include "netcdf_aux.h"
+
+#undef NODELETE
+
+#undef DEBUG
+
+#define DATANAME "data"
+
+typedef enum S3op {
+S3OP_NONE=0,
+S3OP_LIST=1,
+S3OP_CLEAR=2,
+S3OP_PRINT=3,
+S3OP_UPLOAD=4,
+S3OP_DOWNLOAD=5,
+} S3op;
+
+static struct S3ops {
+    S3op s3op;
+    const char* opnames[3];
+} s3ops[] = {
+{S3OP_LIST,{"list","l",NULL}},
+{S3OP_CLEAR,{"clear","c",NULL}},
+{S3OP_PRINT,{"print","p",NULL}},
+{S3OP_UPLOAD,{"upload","u",NULL}},
+{S3OP_DOWNLOAD,{"download","d",NULL}},
+{S3OP_NONE,{NULL,NULL,NULL}},
+};
+
+/* Command line options */
+struct Dumpptions {
+    int debug;
+    int verbose;
+    S3op s3op;
+    NCURI* url;
+    char* key; /* via -k flag */    
+    char* rootkey; /* from url | key */
+    nc_type nctype; /* for printing content */
+    char* filename;
+    char* profile;
+} dumpoptions;
+
+struct S3SDK {
+    NCS3INFO s3;
+    void* s3client;
+    char* errmsg;
+} s3sdk;
+
+/* Forward */
+static int s3list(void);
+static int s3clear(void);
+static int s3print(void);
+static int s3upload(void);
+static int s3download(void);
+static nc_type typefor(const char* t);
+static void printcontent(size64_t len, const char* content, nc_type nctype);
+
+static void
+usage(void)
+{
+    fprintf(stderr,"usage: s3util  list|print|upload|download|clear -u <url> [-k <key>] [-f <filename>]\n");
+    exit(1);
+}
+
+static S3op
+decodeop(const char* name)
+{
+    struct S3ops* s3op = s3ops;
+    const char** s = NULL;
+    for(;s3op->opnames[0] != NULL;s3op++) {
+        for(s=s3op->opnames;*s;s++) {
+	    if(strcasecmp(*s,name)==0) return s3op->s3op;
+	}
+    }
+    return S3OP_NONE;
+}
+
+static int
+s3setup(void)
+{
+    int stat = NC_NOERR;
+    s3sdk.s3client = NC_s3sdkcreateclient(&s3sdk.s3);
+    return stat;
+}
+
+static int
+s3shutdown(int deleteit)
+{
+    int stat = NC_NOERR;
+    if(deleteit) {
+        stat = s3clear();
+    }
+    stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.errmsg);
+    return stat;
+}
+
+int
+main(int argc, char** argv)
+{
+    int stat = NC_NOERR;
+    int c;
+    char* tmp = NULL;
+
+    nc_initialize();
+
+    memset((void*)&dumpoptions,0,sizeof(dumpoptions));
+
+    dumpoptions.nctype = NC_UBYTE; /* default */
+
+    while ((c = getopt(argc, argv, "df:hk:p:t:T:u:v")) != EOF) {
+	switch(c) {
+	case 'd': 
+	    dumpoptions.debug = 1;	    
+	    break;
+	case 'f':
+	    dumpoptions.filename = strdup(optarg);
+	    break;
+	case 'h':
+	    usage();
+	    return 0;
+	case 'k': {
+	    size_t len = strlen(optarg);
+	    dumpoptions.key = (char*)malloc(len+1+1);
+	    if(*optarg != '/') {
+	        fprintf(stderr,"warning: -k option does not start with '/': %s",optarg);
+		dumpoptions.key[0] = '/';
+		memcpy(dumpoptions.key+1,optarg,len);
+		len++;
+	    } else
+		memcpy(dumpoptions.key,optarg,strlen(optarg));
+	    dumpoptions.key[len] = '\0';
+	    } break;
+	case 'p': 
+	    dumpoptions.profile = strdup(optarg);
+	    break;
+	case 't': 
+	    dumpoptions.nctype = typefor(optarg);
+	    break;
+	case 'u': {
+ 	        char* p = NC_shellUnescape(optarg);
+		ncuriparse(p,&dumpoptions.url);
+		nullfree(p);
+		if(dumpoptions.url == NULL) {
+		    fprintf(stderr,"malformed -u option: %s",optarg);
+		    stat = NC_EINVAL;
+		    goto done;
+		}
+	    } break;
+	case 'v': 
+	    dumpoptions.verbose = 1;
+	    break;
+	case 'T':
+	    nctracelevel(atoi(optarg));
+	    break;
+	case '?':
+	   fprintf(stderr,"unknown option\n");
+	   stat = NC_EINVAL;
+	   goto done;
+	}
+    }
+
+    /* get command argument */
+    argc -= optind;
+    argv += optind;
+
+    if (argc > 1) {
+	int j;
+	fprintf(stderr, "s3util: only one command argument permitted:");
+	for(j=0;j<argc;j++) fprintf(stderr," %s",argv[j]);
+	fprintf(stderr, "\n");
+	stat = NC_EINVAL;
+	goto done;
+    }
+    if (argc == 0) {
+	fprintf(stderr, "s3util: no command specified\n");
+	stat = NC_EINVAL;
+	goto done;
+    }
+
+    dumpoptions.s3op = decodeop(argv[0]);
+
+    memset(&s3sdk,0,sizeof(s3sdk));
+
+    if((stat = NC_s3urlprocess(dumpoptions.url, &s3sdk.s3, NULL))) goto done;
+    if(s3sdk.s3.rootkey != NULL && dumpoptions.key != NULL) {
+	/* Make the root key be the concatenation of rootkey+dumpoptions.key */
+        if((stat = nczm_concat(s3sdk.s3.rootkey,dumpoptions.key,&tmp))) goto done;
+	nullfree(s3sdk.s3.rootkey);
+	s3sdk.s3.rootkey = tmp; tmp = NULL;
+    } else if(dumpoptions.key != NULL) {
+	s3sdk.s3.rootkey = dumpoptions.key;
+	dumpoptions.key = NULL;
+    }
+    if(s3sdk.s3.rootkey == NULL || strlen(s3sdk.s3.rootkey)==0)
+        s3sdk.s3.rootkey = strdup("/");
+
+    switch (dumpoptions.s3op) {
+    default:
+	fprintf(stderr,"Default action: list\n");
+	/* fall thru */
+    case S3OP_LIST:
+	if((stat = s3list())) goto done;
+	break;
+    case S3OP_CLEAR:
+	if((stat = s3clear())) goto done;
+	break;
+    case S3OP_PRINT:
+	if((stat = s3print())) goto done;
+	break;
+    case S3OP_UPLOAD:
+	if((stat = s3upload())) goto done;
+	break;
+    case S3OP_DOWNLOAD:
+	if((stat = s3download())) goto done;
+	break;
+    }    
+
+done:
+    /* Reclaim dumpoptions */
+    ncurifree(dumpoptions.url);
+    nullfree(dumpoptions.rootkey);
+    nullfree(tmp);
+    NC_s3clear(&s3sdk.s3);
+    nc_finalize();
+    if(stat)
+	fprintf(stderr,"fail: %s\n",nc_strerror(stat));
+    return (stat ? 1 : 0);    
+}
+
+static int
+s3list(void)
+{
+    int stat = NC_NOERR;
+    size_t nkeys = 0;
+    char** keys = NULL;
+
+    if(s3setup()) goto done;
+
+    stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
+    if(stat) goto done;
+
+    if(nkeys > 0) {
+        size_t i;
+	/* Sort the list -- shortest first */
+	NC_sortenvv(nkeys,keys);
+	for(i=0;i<nkeys;i++) {
+	    printf("[%u] %s\n",(unsigned)i,keys[i]);
+	}
+
+    } else
+	printf("<empty>\n");
+
+done:
+    s3shutdown(0);
+    NCZ_freeenvv(nkeys,keys);    
+    return stat;
+}
+
+static int
+s3clear(void)
+{
+    int stat = NC_NOERR;
+    size_t nkeys = 0;
+    char** keys = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done;
+
+    if(nkeys > 0 && keys != NULL) {
+	size_t i;
+	/* Sort the list -- shortest first */
+	NC_sortenvv(nkeys,keys);
+	if(dumpoptions.verbose) {
+            printf("deleted keys:\n");
+	    for(i=0;i<nkeys;i++) {
+                printf("\t%s\n",keys[i]);
+            }
+        }
+#ifndef NODELETE
+	for(i=0;i<nkeys;i++) {
+	    if((stat = NC_s3sdkdeletekey(s3sdk.s3client, s3sdk.s3.bucket, keys[i], &s3sdk.errmsg)))
+		goto done;
+        }
+#endif
+    }
+
+done:
+    NCZ_freeenvv(nkeys,keys);    
+    s3shutdown(0);
+    return stat;
+}
+
+static int
+s3print(void)
+{
+    int stat = NC_NOERR;
+    size_t nkeys = 0;
+    char** keys = NULL;
+    size64_t count;
+    char* content = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = NC_s3sdkinfo(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &count,&s3sdk.errmsg)))
+	goto done;
+
+    if((content = (char*)calloc(1,count+1))==NULL)
+        {stat = NC_ENOMEM; goto done;}
+
+    if((stat = NC_s3sdkread(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, 0, count, (void*) content, &s3sdk.errmsg)))
+	goto done;
+
+    printcontent(count,content,dumpoptions.nctype);
+
+done:
+    if(content) free(content);
+    s3shutdown(0);
+    NCZ_freeenvv(nkeys,keys);    
+    return stat;
+}
+
+static int
+s3upload(void)
+{
+    int stat = NC_NOERR;
+    size_t red = 0;
+    void* content = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = ncaux_readfile(dumpoptions.filename,&red,&content)))
+        goto done;
+
+    if((stat = NC_s3sdkwriteobject(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, red, content, &s3sdk.errmsg)))
+	goto done;
+
+done:
+    if(content) free(content);
+    s3shutdown(0);
+    return stat;
+}
+
+static int
+s3download(void)
+{
+    int stat = NC_NOERR;
+    size64_t count;
+    char* content = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = NC_s3sdkinfo(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &count,&s3sdk.errmsg)))
+	goto done;
+
+    if((content = (char*)calloc(1,count))==NULL)
+        {stat = NC_ENOMEM; goto done;}
+
+    if((stat = NC_s3sdkread(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, 0, count, (void*) content, &s3sdk.errmsg)))
+	goto done;
+
+    if((stat = ncaux_writefile(dumpoptions.filename,count,content)))
+        goto done;
+
+done:
+    if(content) free(content);
+    s3shutdown(0);
+    return stat;
+}
+
+static void
+printcontent(size64_t len, const char* content, nc_type nctype)
+{
+    size64_t i;
+
+    if(len == 0) {
+        printf("<empty>\n");
+	return;
+    }
+
+    if(nctype == NC_STRING) printf("|");
+    for(i=0;i<len;i++) {
+	    if(nctype != NC_STRING && i > 0) printf(", ");
+	    switch(nctype) {
+	    case NC_BYTE: printf("%d",((char*)content)[i]); break;
+	    case NC_SHORT: printf("%d",((short*)content)[i]); break;		
+	    case NC_INT: printf("%d",((int*)content)[i]); break;		
+	    case NC_INT64: printf("%lld",((long long*)content)[i]); break;		
+	    case NC_UBYTE: printf("%u",((unsigned char*)content)[i]); break;
+	    case NC_USHORT: printf("%u",((unsigned short*)content)[i]); break;		
+	    case NC_UINT: printf("%u",((unsigned int*)content)[i]); break;		
+	    case NC_UINT64: printf("%llu",((unsigned long long*)content)[i]); break;		
+	    case NC_FLOAT: printf("%f",((float*)content)[i]); break;		
+	    case NC_DOUBLE: printf("%lf",((double*)content)[i]); break;		
+	    case NC_STRING: putc(content[i],stdout); break;
+	    default: abort();
+	    }
+    }
+    if(nctype == NC_STRING) printf("|\n");
+}
+
+static nc_type
+typefor(const char* t)
+{
+    if(strcmp(t,"NC_BYTE")==0) return NC_BYTE;
+    else if(strcmp(t,"NC_SHORT")==0) return NC_SHORT;
+    else if(strcmp(t,"NC_INT")==0) return NC_INT;
+    else if(strcmp(t,"NC_INT64")==0) return NC_INT64;
+    else if(strcmp(t,"NC_UBYTE")==0) return NC_UBYTE;
+    else if(strcmp(t,"NC_USHORT")==0) return NC_USHORT;
+    else if(strcmp(t,"NC_UINT")==0) return NC_UINT;
+    else if(strcmp(t,"NC_UINT64")==0) return NC_UINT64;
+    else if(strcmp(t,"NC_FLOAT")==0) return NC_FLOAT;
+    else if(strcmp(t,"NC_DOUBLE")==0) return NC_DOUBLE;
+    else if(strcmp(t,"NC_STRING")==0) return NC_STRING;
+    return NC_NAT;
+}

From 9ae353fffa1ca15ba42785f55223eec0a3ba268c Mon Sep 17 00:00:00 2001
From: Dennis Heimbigner <dennis.heimbigner@gmail.com>
Date: Wed, 12 Feb 2025 19:48:37 -0700
Subject: [PATCH 2/2] Update RELEASE NOTES

---
 RELEASE_NOTES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index ff275a116e..ecf56810f8 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -8,7 +8,7 @@ This file contains a high-level description of this package's evolution. Release
 ## 4.9.4 - TBD
 
 * Clean up the S3 API for all non-libnczarr code. This continues the splitting of PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
-See [Github #????](https://github.com/Unidata/netcdf-c/pull/????) for more information.
+See [Github #3090](https://github.com/Unidata/netcdf-c/pull/3090) for more information.
 
 ## 4.9.3 - February 7, 2025