diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 7001adaa95..75b3378414 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -5,6 +5,10 @@ Release Notes       {#RELEASE_NOTES}
 
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 
+## 4.9.4 - TBD
+
+* Clean up the S3 API for all non-libnczarr code. This continues the splitting of PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
+See [Github #3090](https://github.com/Unidata/netcdf-c/pull/3090) for more information.
 * Step 1 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Update ncjson.[ch] and ncproplist.[ch]. Also fix references to old API. Also fix include/netcdf_ncjson.h and include/netcdf_proplist.h builds. See [Github #3086](https://github.com/Unidata/netcdf-c/pull/3086) for more information.
 * Provide an auxilliary function, `ncaux_parse_provenance()`, that allows users to parse the _NCProperties attribute into a collection of character pointers. See [Github #3088](https://github.com/Unidata/netcdf-c/pull/3088) for more information.
 * Step 2 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Fix a namespace problem in tinyxml2.cpp. Note that this is a visual studio problem hence use of _MSC_VER. Also turn off DAP4 tests against Hyrax server until DAP4 spec problems are fixed. See [Github #3089](https://github.com/Unidata/netcdf-c/pull/3089) for more information.
diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h
index adc7e456be..f81f670efc 100644
--- a/include/ncs3sdk.h
+++ b/include/ncs3sdk.h
@@ -14,10 +14,16 @@
 
 /* Track the server type, if known */
 typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
-                 NCS3=1,    /* s3.amazon.aws */
-		 NCS3GS=2   /* storage.googleapis.com */
+	NCS3=1,     /* s3.amazon.aws */
+	NCS3GS=2,   /* storage.googleapis.com */
+#ifdef NETCDF_ENABLE_ZOH
+	NCS3ZOH=4,  /* ZoH Server */
+#endif
 } NCS3SVC;
 
+/* Opaque Handles */
+struct NClist;
+
 typedef struct NCS3INFO {
     char* host; /* non-null if other*/
     char* region; /* region */
@@ -55,9 +61,10 @@ EXTERNL int NC_s3sdkbucketdelete(void* s3client, NCS3INFO* info, char** errmsgp)
 EXTERNL int NC_s3sdkinfo(void* client0, const char* bucket, const char* pathkey, unsigned long long* lenp, char** errmsgp);
 EXTERNL int NC_s3sdkread(void* client0, const char* bucket, const char* pathkey, unsigned long long start, unsigned long long count, void* content, char** errmsgp);
 EXTERNL int NC_s3sdkwriteobject(void* client0, const char* bucket, const char* pathkey, unsigned long long count, const void* content, char** errmsgp);
-EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp);
-EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
-EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
+EXTERNL int NC_s3sdkclose(void* s3client0, char** errmsgp);
+EXTERNL int NC_s3sdktruncate(void* s3client0, const char* bucket, const char* prefix, char** errmsgp);
+EXTERNL int NC_s3sdklist(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
+EXTERNL int NC_s3sdklistall(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
 EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
 
 /* From ds3util.c */
diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c
index 2b81f342bc..9401ace692 100644
--- a/libdispatch/ds3util.c
+++ b/libdispatch/ds3util.c
@@ -25,6 +25,7 @@
 #include "nc4internal.h"
 #include "ncuri.h"
 #include "nclist.h"
+#include "ncbytes.h"
 #include "ncrc.h"
 #include "nclog.h"
 #include "ncs3sdk.h"
@@ -34,9 +35,6 @@
 /* Alternate .aws directory location */
 #define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR"
 
-#define AWSHOST ".amazonaws.com"
-#define GOOGLEHOST "storage.googleapis.com"
-
 enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4};
 
 /* Read these files in order and later overriding earlier */
@@ -47,6 +45,7 @@ static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
 /* Forward */
 
 static int endswith(const char* s, const char* suffix);
+static void freeprofile(struct AWSprofile* profile);
 static void freeentry(struct AWSentry* e);
 static int awsparse(const char* text, NClist* profiles);
 
@@ -78,9 +77,9 @@ NC_s3sdkenvironment(void)
 /*
 Rebuild an S3 url into a canonical path-style url.
 If region is not in the host, then use specified region
-if provided, otherwise leave blank and let the S3 server deal with it.
-@param url (in) the current url
-@param s3	(in/out) NCS3INFO structure
+if provided, otherwise us-east-1.
+@param url      (in) the current url
+@param s3       (in/out) NCS3INFO structure
 @param pathurlp (out) the resulting pathified url string
 */
 
@@ -309,7 +308,13 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
 
     /* Rebuild the URL to path format and get a usable region and optional bucket*/
     if((stat = NC_s3urlrebuild(url,s3,&url2))) goto done;
-    s3->host = strdup(url2->host);
+    if(url2->port){
+	char hostport[8192];
+	snprintf(hostport,sizeof(hostport),"%s:%s",url2->host,url2->port);
+	s3->host = strdup(hostport);
+    }else{
+        s3->host = strdup(url2->host);
+    }
     /* construct the rootkey minus the leading bucket */
     pathsegments = nclistnew();
     if((stat = NC_split_delim(url2->path,'/',pathsegments))) goto done;
@@ -358,19 +363,29 @@ NC_s3clear(NCS3INFO* s3)
 }
 
 /*
-Check if a url has indicators that signal an S3 or Google S3 url.
+Check if a url has indicators that signal an S3 or Google S3 url or ZoH S3 url.
+The rules are as follows:
+1. If the protocol is "s3" or "gs3" or "zoh", then return (true,s3|gs3|zoh).
+2. If the mode contains "s3" or "gs3" or "zoh", then return (true,s3|gs3|zoh).
+3. Check the host name:
+3.1 If the host ends with ".amazonaws.com", then return (true,s3).
+3.1 If the host is "storage.googleapis.com", then return (true,gs3).
+4. Otherwise return (false,unknown).
 */
 
 int
-NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
+NC_iss3(NCURI* uri, NCS3SVC* svcp)
 {
     int iss3 = 0;
     NCS3SVC svc = NCS3UNK;
 
     if(uri == NULL) goto done; /* not a uri */
-    /* is the protocol "s3" or "gs3" ? */
+    /* is the protocol "s3" or "gs3" or "zoh" ? */
     if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; svc = NCS3; goto done;}
     if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; svc = NCS3GS; goto done;}
+#ifdef NETCDF_ENABLE_ZOH
+    if(strcasecmp(uri->protocol,"zoh")==0) {iss3 = 1; svc = NCS3ZOH; goto done;}
+#endif
     /* Is "s3" or "gs3" in the mode list? */
     if(NC_testmode(uri,"s3")) {iss3 = 1; svc = NCS3; goto done;}
     if(NC_testmode(uri,"gs3")) {iss3 = 1; svc = NCS3GS; goto done;}    
@@ -384,18 +399,71 @@ NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
     return iss3;
 }
 
-const char*
-NC_s3dumps3info(NCS3INFO* info)
+/**************************************************/
+/**
+The .aws/config and .aws/credentials files
+are in INI format (https://en.wikipedia.org/wiki/INI_file).
+This format is not well defined, so the grammar used
+here is restrictive. Here, the term "profile" is the same
+as the INI term "section".
+
+The grammar used is as follows:
+
+Grammar:
+
+inifile: profilelist ;
+profilelist: profile | profilelist profile ;
+profile: '[' profilename ']' EOL entries ;
+entries: empty | entries entry ;
+entry:  WORD = WORD EOL ;
+profilename: WORD ;
+Lexical:
+WORD    sequence of printable characters - [ \[\]=]+
+EOL	'\n' | ';'
+
+Note:
+1. The semicolon at beginning of a line signals a comment.
+2. # comments are not allowed
+3. Duplicate profiles or keys are ignored.
+4. Escape characters are not supported.
+*/
+
+#define AWS_EOF (-1)
+#define AWS_ERR (0)
+#define AWS_WORD (0x10001)
+#define AWS_EOL (0x10002)
+
+typedef struct AWSparser {
+    char* text;
+    char* pos;
+    size_t yylen; /* |yytext| */
+    NCbytes* yytext;
+    int token; /* last token found */
+    int pushback; /* allow 1-token pushback */
+} AWSparser;
+
+#ifdef LEXDEBUG
+static const char*
+tokenname(int token)
 {
-    static char text[8192];
-    snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
-		(info->host?info->host:"null"),
-		(info->region?info->region:"null"),
-		(info->bucket?info->bucket:"null"),
-		(info->rootkey?info->rootkey:"null"),
-		(info->profile?info->profile:"null"));
-    return text;
+    static char num[32];
+    switch(token) {
+    case AWS_EOF: return "EOF";
+    case AWS_ERR: return "ERR";
+    case AWS_WORD: return "WORD";
+    default: snprintf(num,sizeof(num),"%d",token); return num;
+    }
+    return "UNKNOWN";
 }
+#endif
+
+/*
+@param text of the aws credentials file
+@param profiles list of form struct AWSprofile (see ncauth.h)
+*/
+
+#define LBR '['
+#define RBR ']'
 
 static void
 freeprofile(struct AWSprofile* profile)
@@ -426,6 +494,19 @@ NC_s3freeprofilelist(NClist* profiles)
     }
 }
 
+const char*
+NC_s3dumps3info(NCS3INFO* info)
+{
+    static char text[8192];
+    snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
+		(info->host?info->host:"null"),
+		(info->region?info->region:"null"),
+		(info->bucket?info->bucket:"null"),
+		(info->rootkey?info->rootkey:"null"),
+		(info->profile?info->profile:"null"));
+    return text;
+}
+
 /* Find, load, and parse the aws config &/or credentials file */
 int
 NC_aws_load_credentials(NCglobalstate* gstate)
@@ -707,15 +788,6 @@ tokenname(int token)
 }
 #endif
 
-typedef struct AWSparser {
-    char* text;
-    char* pos;
-    size_t yylen; /* |yytext| */
-    NCbytes* yytext;
-    int token; /* last token found */
-    int pushback; /* allow 1-token pushback */
-} AWSparser;
-
 static int
 awslex(AWSparser* parser)
 {
diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c
index 29aedea326..0fc97deb6f 100644
--- a/libdispatch/nch5s3comms.c
+++ b/libdispatch/nch5s3comms.c
@@ -92,7 +92,7 @@
 
 #include "netcdf.h"
 #include "ncuri.h"
-#include "ncutil.h"
+#include "netcdf_vutils.h"
 
 /*****************/
 
@@ -292,7 +292,6 @@ static int build_range(size_t offset, size_t len, char** rangep);
 static const char* verbtext(HTTPVerb verb);
 static int trace(CURL* curl, int onoff);
 static int sortheaders(VList* headers);
-static int httptonc(long httpcode);
 static void hrb_node_free(hrb_node_t *node);
 
 #if S3COMMS_DEBUG_HRB
@@ -789,7 +788,7 @@ NCH5_s3comms_s3r_close(s3r_t *handle)
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
+NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep, long* httpcodep)
 {
     int ret_value      = SUCCEED;
     char* contentlength = NULL;
@@ -806,7 +805,8 @@ NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
     if((ret_value = NCH5_s3comms_s3r_head(handle, url, "Content-Length", NULL, &httpcode, &contentlength)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "NCH5_s3comms_s3r_head failed.");
 
-    if((ret_value = httptonc(httpcode))) goto done;
+    /* Content-Length will not be defined if object does not exist */
+    if(httpcode == 404) goto done;
 
     /******************
      * PARSE RESPONSE *
@@ -816,7 +816,7 @@ NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
     if(value == NULL)
         HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not find content length value");
     value++;
-    content_length = strtoumax(value, NULL, 0);
+    content_length = (long long)strtoumax(value, NULL, 0);
     if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX)
         HGOTO_ERROR(H5E_ARGS, NC_ERANGE, FAIL, "content_length overflows size_t");
 
@@ -825,9 +825,9 @@ NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
                     "could not convert found \"Content-Length\" response (\"%s\")",
                     contentlength); /* range is null-terminated, remember */
 
-    if(sizep) {*sizep = (long long)content_length;}
-
 done:
+    if(sizep) {*sizep = (long long)content_length;}
+    if(httpcodep) *httpcodep = httpcode;    
     nullfree(contentlength);
     return UNTRACEX(ret_value,"size=%lld",(sizep?-1:*sizep));
 } /* NCH5_s3comms_s3r_getsize */
@@ -859,17 +859,13 @@ NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep)
 
     if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPDELETE, NULL, NULL, NULL, &httpcode, data)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-
-    /******************
-     * RESPONSE *
-     ******************/
-    if((ret_value = httptonc(httpcode))) goto done;
-    if(httpcode != 204) 
-        HGOTO_ERROR(H5E_ARGS, NC_ECANTREMOVE, FAIL, "deletekey failed.");     
+    
+    /* Apparently, aws delivers a 204 response if it successfully deletes the key */
+    if(httpcode == 204) httpcode = 200; /* treat 204 as success */
 
 done:
-    vsfree(data);
     if(httpcodep) *httpcodep = httpcode;
+    vsfree(data);
     return UNTRACEX(ret_value,"httpcode=%d",INULL(httpcodep));
 } /* NCH5_s3comms_s3r_getsize */
 
@@ -911,12 +907,8 @@ NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const
     if (SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPHEAD, NULL, header, NULL, &httpcode, data))
         HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem in reading during getsize.");
 
-    if((ret_value = httptonc(httpcode))) goto done;
-
     if(header != NULL) {
-        if(vslength(data) == 0)
-            HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata: header=%s; not found",header);
-        else if (vslength(data) > CURL_MAX_HTTP_HEADER)
+        if (vslength(data) > CURL_MAX_HTTP_HEADER)
             HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata buffer overrun");
 #if S3COMMS_DEBUG
         else
@@ -930,7 +922,7 @@ NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const
 
     if(header != NULL) {
         char* content;
-	content = vsextract(data);
+	content = vsextract(data); /* guaranteed to exist */
         if(valuep) {*valuep = content;}
     }
 
@@ -1222,7 +1214,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* dest)
+NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* dest, long* httpcodep)
 {
     char              *rangebytesstr = NULL;
     int                ret_value = SUCCEED;
@@ -1251,9 +1243,9 @@ NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len,
 
     if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPGET, rangebytesstr, NULL, NULL, &httpcode, wrap)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-    if((ret_value = httptonc(httpcode))) goto done;
 
 done:
+    if(httpcodep) *httpcodep = httpcode;
     (void)vsextract(wrap);
     vsfree(wrap);
     /* clean any malloc'd resources */
@@ -1271,7 +1263,7 @@ NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len,
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data)
+NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data, long* httpcodep)
 {
     int ret_value = SUCCEED;
     VList* otherheaders = vlistnew();
@@ -1301,9 +1293,10 @@ NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data)
     vssetlength(wrap,data->count);
     if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPPUT, NULL, NULL, (const char**)vlistcontents(otherheaders), &httpcode, wrap)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-    if((ret_value = httptonc(httpcode))) goto done;
+
     
 done:
+    if(httpcodep) *httpcodep = httpcode;
     (void)vsextract(wrap);
     vsfree(wrap);
     /* clean any malloc'd resources */
@@ -1321,7 +1314,7 @@ NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data)
  *----------------------------------------------------------------------------
  */
 int
-NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response)
+NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response, long* httpcodep)
 {
     int ret_value = SUCCEED;
     const char* otherheaders[3] = {"Content-Type", "application/xml", NULL};
@@ -1340,13 +1333,13 @@ NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response)
 
     if((SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPGET, NULL, NULL, otherheaders, &httpcode, content)))
         HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
-    if((ret_value = httptonc(httpcode))) goto done;
     if(response) {
 	response->count = vslength(content);
 	response->content = vsextract(content);
     }
 
 done:
+    if(httpcodep) *httpcodep = httpcode;
     vsfree(content);
     /* clean any malloc'd resources */
     curl_reset(handle);
@@ -1421,7 +1414,7 @@ NCH5_s3comms_aws_canonical_request(VString* canonical_request_dest, VString* sig
 {
     hrb_node_t *node         = NULL;
     int      ret_value    = SUCCEED;
-    int i;
+    size_t i;
     
     const char* sverb = verbtext(verb);
     const char* query_params = (query?query:"");
@@ -1568,7 +1561,7 @@ NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m
 #else
     if(CURLE_OK != Curl_hmacit(Curl_HMAC_SHA256,
                      key, key_len,
-                     msg, msg_len,
+                     (const unsigned char*)msg, msg_len,
                      md))
         HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "Curl_hmacit failure.");
 #endif
@@ -2052,7 +2045,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re
     HMAC(EVP_sha256(), (const unsigned char *)dateregionservicekey, SHA256_DIGEST_LENGTH,
          (const unsigned char *)"aws4_request", 12, md, NULL);
 #else
-    Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)AWS4_secret, (int)nulllen(AWS4_secret),
+    Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)AWS4_secret, nulllen(AWS4_secret),
          (const unsigned char *)iso8601now, 8, /* 8 --> length of 8 --> "yyyyMMDD"  */
          datekey);
     Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)datekey, SHA256_DIGEST_LENGTH, (const unsigned char *)region,
@@ -2390,7 +2383,8 @@ build_request(s3r_t* handle, NCURI* purl,
               VString* payload,
               HTTPVerb verb)
 {
-    int i,ret_value = SUCCEED;
+    int ret_value = SUCCEED;
+    size_t i;
     struct curl_slist *curlheaders   = NULL;
     hrb_node_t        *node          = NULL;
     hrb_t             *request       = NULL;
@@ -2788,32 +2782,6 @@ sortheaders(VList* headers)
     return (ret_value);
 }
 
-static int
-httptonc(long httpcode)
-{
-    int stat = NC_NOERR;
-    if(httpcode <= 99) stat = NC_EINTERNAL; /* should never happen */
-    else if(httpcode <= 199)
-        stat = NC_NOERR; /* I guess */
-    else if(httpcode <= 299) {
-        switch (httpcode) {
-        default: stat = NC_NOERR; break;
-        }
-    } else if(httpcode <= 399)
-        stat = NC_NOERR; /* ? */
-    else if(httpcode <= 499) {
-        switch (httpcode) {
-        case 400: stat = NC_EINVAL; break;
-        case 401: case 402: case 403:
-            stat = NC_EAUTH; break;
-        case 404: stat = NC_EEMPTY; break;
-        default: stat = NC_EINVAL; break;
-        }
-    } else
-        stat = NC_ES3;
-    return stat;
-}
-
 /**************************************************/
 /* Request Tracing */
 
diff --git a/libdispatch/nch5s3comms.h b/libdispatch/nch5s3comms.h
index 7cc482df66..f7c383e037 100644
--- a/libdispatch/nch5s3comms.h
+++ b/libdispatch/nch5s3comms.h
@@ -506,13 +506,13 @@ EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char*
 
 EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle);
 
-EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* data);
+EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* data, long* httpcodep);
 
-EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data);
+EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data, long* httpcodep);
 
-EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response);
+EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response, long* httpcodep);
 
-EXTERNL int NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long * sizep);
+EXTERNL int NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long * sizep, long* httpcodep);
 
 EXTERNL int NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep);
 
diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c
index 359ab2f8b3..27bb0e2038 100644
--- a/libdispatch/ncs3sdk_h5.c
+++ b/libdispatch/ncs3sdk_h5.c
@@ -101,6 +101,7 @@ static int s3objectsinfo(NClist* contents, NClist* keys, NClist* lens);
 static int s3commonprefixes(NClist* list, NClist* keys);
 static int mergekeysets(NClist*,NClist*,NClist*);
 static int rawtokeys(s3r_buf_t* response, NClist* keys, NClist* lengths, struct LISTOBJECTSV2** listv2p);
+static int httptonc(long httpcode);
 
 static int queryadd(NClist* query, const char* key, const char* value);
 static int queryend(NClist* query, char** querystring);
@@ -193,7 +194,7 @@ NC_s3sdkcreateclient(NCS3INFO* info)
 done:
     nullfree(urlroot);
     if(stat && s3client) {
-        NC_s3sdkclose(s3client,info,0,NULL);
+        NC_s3sdkclose(s3client,NULL);
 	s3client = NULL;
     }
     NCNILTRACE(NC_NOERR);
@@ -213,6 +214,7 @@ NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** e
 
     if((stat = makes3fullpath(s3client->rooturl,bucket,NULL,NULL,url))) goto done;
     if((stat = NCH5_s3comms_s3r_head(s3client->h5s3client, ncbytescontents(url), NULL, NULL, &httpcode, NULL))) goto done;
+    stat = httptonc(httpcode);
 
     if(existsp) {*existsp = (stat == 0 && httpcode == 200);}
 done:
@@ -255,7 +257,7 @@ NC_s3sdkbucketdelete(void* s3client0, NCS3INFO* info, char** errmsgp)
 
 /*
 @return NC_NOERR if key points to a content-bearing object.
-@return NC_EEMPTY if object at key has no content.
+@return NC_ENOOBJECT if object at key does not exist
 @return NC_EXXX return true error
 */
 EXTERNL int
@@ -265,13 +267,15 @@ NC_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t*
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
     NCbytes* url = ncbytesnew();
     long long len = -1;
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
 
     if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
-    if((stat = NCH5_s3comms_s3r_getsize(s3client->h5s3client, ncbytescontents(url), &len))) goto done;
+    if((stat = NCH5_s3comms_s3r_getsize(s3client->h5s3client, ncbytescontents(url), &len, &httpcode))) goto done;
+    stat = httptonc(httpcode);
 
-    if(lenp) {*lenp = len;}
+    if(lenp) {*lenp = (size64_t)len;}
 
 done:
     ncbytesfree(url);
@@ -289,6 +293,7 @@ NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
     NCbytes* url = ncbytesnew();
     struct s3r_buf_t data = {0,NULL};
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s pathkey=%s start=%llu count=%llu content=%p",bucket,pathkey,start,count,content);
 
@@ -297,8 +302,8 @@ NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t
     /* Read the data */
     data.count = count;
     data.content = content;
-    if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,&data))) goto done;
-    
+    if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,&data,&httpcode))) goto done;
+    stat = httptonc(httpcode);    
 done:
     ncbytesfree(url);
     return NCUNTRACE(stat);
@@ -315,6 +320,7 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
     NCbytes* url = ncbytesnew();
     s3r_buf_t data;
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s pathkey=%s count=%llu content=%p",bucket,pathkey,count,content);
 
@@ -323,7 +329,8 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
     /* Write the data */
     data.count = count;
     data.content = (void*)content;
-    if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),&data))) goto done;
+    if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),&data,&httpcode))) goto done;
+    stat = httptonc(httpcode);
     
 done:
     ncbytesfree(url);
@@ -331,28 +338,49 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
 }
 
 EXTERNL int
-NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp)
+NC_s3sdkclose(void* s3client0, char** errmsgp)
 {
     int stat = NC_NOERR;
     NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
 
-    NCTRACE(11,"info=%s deleteit=%d",NC_s3dumps3info(info),deleteit);
-    
-    if(deleteit) {
-        /* Delete the root key; ok it if does not exist */
-        switch (stat = NC_s3sdkdeletekey(s3client0,info->bucket,info->rootkey,errmsgp)) {
-        case NC_NOERR: break;
-        case NC_EEMPTY: case NC_ENOTFOUND: stat = NC_NOERR; break;
-        default: break;
+    NCTRACE(11,"");
+    s3client_destroy(s3client);
+    return NCUNTRACE(stat);
+}
+
+EXTERNL int
+NC_s3sdktruncate(void* s3client0, const char* bucket, const char* prefix, char** errmsgp)
+{
+    int stat = NC_NOERR;
+    char* errmsg = NULL;
+    size_t nkeys;
+    char** keys = NULL;
+    NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
+
+    NCTRACE(11,"bucket=%s prefix=%s",bucket,prefix);
+
+    if((stat = NC_s3sdklistall(s3client0,bucket,prefix,&nkeys,&keys,&errmsg))) goto done;
+
+    if(nkeys > 0 && keys != NULL) {
+	size_t i;
+	/* Sort the list -- shortest first */
+	NC_sortenvv(nkeys,keys);
+	for(i=0;i<nkeys;i++) {
+	    if((stat = NC_s3sdkdeletekey(s3client, bucket, keys[i], NULL))) goto done;
         }
     }
-    s3client_destroy(s3client);
+
+    if(errmsgp) {*errmsgp = errmsg; errmsg = NULL;}
+
+done:
+    nullfree(errmsg);
+    NC_freeenvv(nkeys,keys);    
     return NCUNTRACE(stat);
 }
 
 /*
-Common code for getkeys and searchkeys.
-Return a list of names of legal objects immediately below a specified key.
+Common code for list and listall.
+Return a list of names of legal objects immediately or anywhere below a specified key.
 In theory, the returned list should be sorted in lexical order,
 but it possible that it is not.
 */
@@ -371,6 +399,7 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
     int istruncated = 0;
     char* continuetoken = NULL;
     s3r_buf_t response = {0,NULL};
+    long httpcode = 0;
 
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
     
@@ -401,7 +430,8 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
         ncbytescat(listurl,"?");
         ncbytescat(listurl,querystring);
 
-        if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), &response))) goto done;
+        if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), &response, &httpcode))) goto done;
+	if((stat = httptonc(httpcode))) goto done;
         if((stat = rawtokeys(&response,allkeys,NULL,&listv2))) goto done;
 	istruncated = (strcasecmp(listv2->istruncated,"true")==0?1:0);
 	nullfree(continuetoken);
@@ -430,19 +460,19 @@ In theory, the returned list should be sorted in lexical order,
 but it possible that it is not.
 */
 EXTERNL int
-NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
+NC_s3sdklist(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
 {
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
     return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp));
 }
 
 /*
-Return a list of full keys  of legal objects immediately below a specified key.
+Return a list of full keys  of legal objects below a specified key.
 Not necessarily sorted.
 Essentially same as getkeys, but with no delimiter.
 */
 EXTERNL int
-NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
+NC_s3sdklistall(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
 {
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
     return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp));
@@ -461,6 +491,11 @@ NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char
     if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
 
     if((stat = NCH5_s3comms_s3r_deletekey(s3client->h5s3client, ncbytescontents(url), &httpcode))) goto done;
+    switch(stat = httptonc(httpcode)) {
+    case NC_NOERR: break;
+    case NC_ENOOBJECT: break; /* does not exist */
+    default: goto done;
+    }
 
 done:
     ncbytesfree(url);
@@ -700,8 +735,8 @@ parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV
     if(resultp) {*resultp = result; result = NULL;}
 
 done:
-    if(result) reclaim_listobjectsv2(result);
     if(doc) ncxml_free(doc);
+    if(result) reclaim_listobjectsv2(result);
     return NCTHROW(stat);
 }
 
@@ -1061,3 +1096,33 @@ queryinsert(NClist* list, char* ekey, char* evalue)
     return NCTHROW(stat);
 }
 
+static int
+httptonc(long httpcode)
+{
+    int stat = NC_NOERR;
+    if(httpcode == 0)
+        stat = NC_NOERR;
+    else if(httpcode <= 99)
+        stat = NC_EINTERNAL; /* should never happen */
+    else if(httpcode <= 199)
+        stat = NC_NOERR; /* I guess */
+    else if(httpcode <= 299) {
+        switch (httpcode) {
+	case 200: stat = NC_NOERR; break;
+	case 204: stat = NC_ENOOBJECT; break;
+        default: stat = NC_NOERR; break;
+        }
+    } else if(httpcode <= 399)
+        stat = NC_NOERR; /* ? */
+    else if(httpcode <= 499) {
+        switch (httpcode) {
+        case 400: stat = NC_EINVAL; break;
+        case 401: case 402: case 403:
+            stat = NC_EAUTH; break;
+        case 404: stat = NC_ENOOBJECT; break;
+        default: stat = NC_EINVAL; break;
+        }
+    } else
+        stat = NC_ES3;
+    return stat;
+}
diff --git a/libsrc/s3io.c b/libsrc/s3io.c
index f8bd941b42..d4381da7a9 100644
--- a/libsrc/s3io.c
+++ b/libsrc/s3io.c
@@ -184,7 +184,7 @@ s3io_open(const char* path,
     /* Get the size */
     switch (status = NC_s3sdkinfo(s3io->s3client,s3io->s3.bucket,s3io->s3.rootkey,(long long unsigned*)&s3io->size,&s3io->errmsg)) {
     case NC_NOERR: break;
-    case NC_EEMPTY:
+    case NC_ENOOBJECT:
         s3io->size = 0;
 	goto done;
     default:
@@ -253,7 +253,9 @@ s3io_close(ncio* nciop, int deleteit)
     assert(s3io != NULL);
 
     if(s3io->s3client && s3io->s3.bucket && s3io->s3.rootkey) {
-        NC_s3sdkclose(s3io->s3client, &s3io->s3, deleteit, &s3io->errmsg);
+	if(deleteit)
+	    NC_s3sdktruncate(s3io->s3client, s3io->s3.bucket, s3io->s3.rootkey, &s3io->errmsg);
+        NC_s3sdkclose(s3io->s3client, &s3io->errmsg);
     }
     s3io->s3client = NULL;
     NC_s3clear(&s3io->s3);
diff --git a/nczarr_test/s3util.c b/nczarr_test/s3util.c
index 9f0c433f73..56fc0cda54 100644
--- a/nczarr_test/s3util.c
+++ b/nczarr_test/s3util.c
@@ -83,7 +83,7 @@ static void printcontent(size64_t len, const char* content, nc_type nctype);
 static void
 usage(void)
 {
-    fprintf(stderr,"usage: s3util  list|print|upload|download|clear -u <url> [-k <key.] [-f <filename>]\n");
+    fprintf(stderr,"usage: s3util  list|print|upload|download|clear -u <url> [-k <key>] [-f <filename>]\n");
     exit(1);
 }
 
@@ -112,7 +112,10 @@ static int
 s3shutdown(int deleteit)
 {
     int stat = NC_NOERR;
-    stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.s3, deleteit, &s3sdk.errmsg);
+    if(deleteit) {
+        stat = s3clear();
+    }
+    stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.errmsg);
     return stat;
 }
 
@@ -258,13 +261,13 @@ s3list(void)
 
     if(s3setup()) goto done;
 
-    stat = NC_s3sdksearch(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
+    stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
     if(stat) goto done;
 
     if(nkeys > 0) {
         size_t i;
 	/* Sort the list -- shortest first */
-	nczm_sortenvv(nkeys,keys);
+	NC_sortenvv(nkeys,keys);
 	for(i=0;i<nkeys;i++) {
 	    printf("[%u] %s\n",(unsigned)i,keys[i]);
 	}
@@ -287,12 +290,12 @@ s3clear(void)
 
     if(s3setup()) goto done;
 
-    if((stat = NC_s3sdksearch(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done;
+    if((stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done;
 
     if(nkeys > 0 && keys != NULL) {
 	size_t i;
 	/* Sort the list -- shortest first */
-	nczm_sortenvv(nkeys,keys);
+	NC_sortenvv(nkeys,keys);
 	if(dumpoptions.verbose) {
             printf("deleted keys:\n");
 	    for(i=0;i<nkeys;i++) {
diff --git a/s3cleanup.in b/s3cleanup.in
index 22504717e0..248c7d3037 100755
--- a/s3cleanup.in
+++ b/s3cleanup.in
@@ -67,7 +67,11 @@ if test "x$VERBOSE" = x1 ; then set +x; fi
 for key in $ALLKEYS ; do
     for uid in $UIDS ; do
         case "$key" in
-            "$S3TESTSUBTREE/testset_${uid}"*)
+            "$S3TESTSUBTREE/nczarr_test_${uid}"*)
+               # capture the key'
+               MATCHKEYS="$MATCHKEYS $key"
+               ;;
+            "$S3TESTSUBTREE/v3_nczarr_test_${uid}"*)
                 # capture the key'
                 MATCHKEYS="$MATCHKEYS $key"
                 ;;
diff --git a/s3gc.in b/s3gc.in
index a63345c6c7..543e151943 100755
--- a/s3gc.in
+++ b/s3gc.in
@@ -74,24 +74,32 @@ done < s3gc.keys
 # Look at each key and see if it is less than lastdate.
 # If so, then record that key
 
+captureuid() {
+    PFX="$1"
+    key="$2"
+    # Capture the uid for this key         
+    s3uid=`echo $key | sed -e "s|$S3TESTSUBTREE/${PFX}_\([0-9][0-9]*\)/.*|\1|"`
+    # check that we got a uid       
+    if test "x$s3uid" != x ; then
+        # Test age of the uid
+        if test $((s3uid < lastdate)) = 1; then
+    	    MATCHKEYS="${MATCHKEYS} $key"
+	fi
+    else
+        if test "x$VERBOSE" = x1 ; then echo "Excluding \"$key\""; fi
+    fi
+}
+
 # Capture the keys with old uids to delete
 unset MATCHKEYS
 for key in $ALLKEYS ; do
     case "$key" in
-        "$S3TESTSUBTREE/testset_"*)
-             # Capture the uid for this key         
-             s3uid=`echo $key | sed -e "s|$S3TESTSUBTREE/testset_\([0-9][0-9]*\)/.*|\1|"`
-             # check that we got a uid       
-             if test "x$s3uid" != x ; then
-                # Test age of the uid
-                if test $((s3uid < lastdate)) = 1; then
-                    MATCHKEYS="${MATCHKEYS} $key"
-                fi
-             else
-                 if test "x$VERBOSE" = x1 ; then echo "Excluding \"$key\""; fi
-             fi
-	     ;;
-        *) if test "x$VERBOSE" = x1; then echo "Ignoring \"$key\""; fi ;;
+        "$S3TESTSUBTREE/alltests_"*) captureuid "alltests" "${key}" ;;
+        "$S3TESTSUBTREE/nczarr_test_"*) captureuid "nczarr_test" "${key}" ;;
+        "$S3TESTSUBTREE/v3_nczarr_test_"*) captureuid "v3_nczarr_test" "${key}" ;;
+        "$S3TESTSUBTREE/v3_tests_"*) captureuid "v3_tests" "${key}" ;;
+        "$S3TESTSUBTREE/testset_"*) captureuid "testset" "${key}" ;;
+    *) if test "x$VERBOSE" = x1; then echo "Ignoring \"$key\""; fi ;;
     esac
 done
 
diff --git a/unit_test/run_s3sdk.sh b/unit_test/run_s3sdk.sh
index 18073e4570..ce7fb9d092 100755
--- a/unit_test/run_s3sdk.sh
+++ b/unit_test/run_s3sdk.sh
@@ -7,14 +7,14 @@ set -e
 
 #CMD="valgrind --leak-check=full"
 
-URL="https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}"
+URL="https://${S3ENDPOINT}/${S3TESTBUCKET}"
 
 isolate "testdir_uts3sdk"
 
 # Create an isolation path for S3; build on the isolation directory
 S3ISODIR="$ISODIR"
 S3ISOPATH="/${S3TESTSUBTREE}"
-S3ISOPATH="/${S3ISOPATH}/$S3ISODIR"
+S3ISOPATH="${S3ISOPATH}/$S3ISODIR"
 
 test_cleanup() {
 ${CMD} ${execdir}/../nczarr_test/s3util -u "${URL}" -k "${S3ISOPATH}" clear
@@ -47,13 +47,17 @@ ${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}"                list
 echo "Status: $?"
 
 echo -e "\to Checking search command for ${URL}"
-${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}"                search
+${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "/object_store"               listall
 echo "Status: $?"
 
 echo -e "\to Checking delete command for ${URL}/test_s3sdk.txt"
 ${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}/test_s3sdk.txt" delete
 echo "Status: $?"
 
+echo -e "\to Checking delete command for non-existent ${URL}/test_s3sdk_x.txt"
+${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}/test_s3sdk_x.txt" delete
+echo "Status: $?"
+
 if test "x$FEATURE_LARGE_TESTS" = xyes ; then
     echo -e "\to Checking longlist command for ${URL}"
     ${CMD} ${execdir}/test_s3sdk -u "${URL}" -k "${S3ISOPATH}"                longlist
@@ -62,7 +66,6 @@ fi
 
 echo -e "Finished"
 
-exit
 if test "x$GITHUB_ACTIONS" = xtrue; then
 # Cleanup on exit
 test_cleanup
diff --git a/unit_test/test_s3sdk.c b/unit_test/test_s3sdk.c
index 93557a18f8..896f703c07 100644
--- a/unit_test/test_s3sdk.c
+++ b/unit_test/test_s3sdk.c
@@ -30,7 +30,7 @@
 
 #define LONGCOUNT 1010
 
-enum Actions {ERROR_ACTION, EXISTS_ACTION, SIZE_ACTION, READ_ACTION, WRITE_ACTION, DELETE_ACTION, LIST_ACTION, LONGLIST_ACTION, SEARCH_ACTION};
+enum Actions {ERROR_ACTION, EXISTS_ACTION, SIZE_ACTION, READ_ACTION, WRITE_ACTION, DELETE_ACTION, LIST_ACTION, LONGLIST_ACTION, LISTALL_ACTION};
 
 struct Options {
     int debug;
@@ -78,7 +78,7 @@ actionfor(const char* s)
     else if(strcasecmp(s,"write")==0) return WRITE_ACTION;
     else if(strcasecmp(s,"list")==0) return LIST_ACTION;
     else if(strcasecmp(s,"longlist")==0) return LONGLIST_ACTION;
-    else if(strcasecmp(s,"search")==0) return SEARCH_ACTION;
+    else if(strcasecmp(s,"listall")==0) return LISTALL_ACTION;
     else if(strcasecmp(s,"delete")==0) return DELETE_ACTION;
     return ERROR_ACTION;
 }
@@ -126,6 +126,39 @@ profilesetup(const char* url)
     return stat;
 }
 
+#if 0
+static int
+s3treesetup(void)
+{
+    int stat = NC_NOERR;
+    size64_t size = 0;
+    void* content = NULL;
+    char key[8192];
+
+    seturl("https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}", "/netcdf-c/test_s3",!FORCE);
+
+    CHECK(profilesetup(dumpoptions.url));
+    newurl = ncuribuild(purl,NULL,NULL,NCURIALL);
+#ifdef DEBUG
+    printf("url=%s {url=%s bucket=%s region=%s profile=%s}\n",
+               dumpoptions.url,newurl,s3info.bucket,s3info.region,activeprofile);
+#endif
+    if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
+
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x/y");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x/y/z");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+    snprintf(key,sizeof(key),"%s/%s",dumpoptions.key,"x/y/w");
+    CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, key, 0, NULL, NULL));
+done:
+    cleanup();
+    return stat;
+}
+#endif
+
 static int
 testbucketexists(void)
 {
@@ -236,7 +269,7 @@ testwrite(void)
 }
 
 static int
-testgetkeys(void)
+testlist(void)
 {
     int stat = NC_NOERR;
     size_t i,nkeys = 0;
@@ -250,8 +283,8 @@ testgetkeys(void)
     printf("url=%s => info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info));
 #endif
     if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
-    CHECK(NC_s3sdkgetkeys(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
-    printf("testgetkeys: nkeys=%u; keys:\n",(unsigned)nkeys);
+    CHECK(NC_s3sdklist(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
+    printf("testlist: nkeys=%u; keys:\n",(unsigned)nkeys);
     for(i=0;i<nkeys;i++) {
         printf("\t|%s|\n",keys[i]);
     }
@@ -265,7 +298,7 @@ testgetkeys(void)
 }
 
 static int
-testgetkeyslong(void)
+testlistlong(void)
 {
     int stat = NC_NOERR;
     size_t i,nkeys = 0;
@@ -290,8 +323,8 @@ testgetkeyslong(void)
         snprintf(path,sizeof(path),"%s/getkey_%d",dumpoptions.key,(int)i);
         CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, path, strlen(uploaddata), uploaddata, NULL));
     }
-    CHECK(NC_s3sdkgetkeys(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
-    printf("testgetkeys: nkeys=%u; keys:\n",(unsigned)nkeys);
+    CHECK(NC_s3sdklist(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
+    printf("testlistlong: nkeys=%u; keys:\n",(unsigned)nkeys);
     if(nkeys != LONGCOUNT) {
         fprintf(stderr,"*** nkeys mismatch: create=%d found=%d\n",LONGCOUNT,(int)nkeys);
     }
@@ -320,9 +353,9 @@ testgetkeyslong(void)
         printf("\tkey=%s: ",keys[i]);
         stat = NC_s3sdkdeletekey(s3client, s3info.bucket, keys[i], NULL);
         switch (stat) {
-        case NC_NOERR:  printf("deleted\n");        break;
-        case NC_EEMPTY: printf("does not exist\n"); break;
-        default:        printf("failed\n");         break;
+        case NC_NOERR:     printf("deleted\n");        break;
+        case NC_ENOOBJECT: printf("does not exist\n"); break;
+        default:           printf("failed\n");         break;
         }
         stat = NC_NOERR; /* reset */
     }    
@@ -336,13 +369,13 @@ testgetkeyslong(void)
 }
 
 static int
-testsearch(void)
+testlistall(void)
 {
     int stat = NC_NOERR;
     size_t i,nkeys = 0;
     char** keys = NULL;
 
-    seturl("https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}", "/object_store/dir1",!FORCE);
+    seturl("https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}", "/object_store",!FORCE);
 
     CHECK(profilesetup(dumpoptions.url));
     newurl = ncuribuild(purl,NULL,NULL,NCURIALL);
@@ -350,8 +383,8 @@ testsearch(void)
     printf("url=%s => info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info));
 #endif
     if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
-    CHECK(NC_s3sdksearch(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
-    printf("testsearch: nkeys=%u; keys:\n",(unsigned)nkeys);
+    CHECK(NC_s3sdklistall(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL));
+    printf("testlistall: nkeys=%u; keys:\n",(unsigned)nkeys);
     for(i=0;i<nkeys;i++) {
         printf("\t|%s|\n",keys[i]);
     }
@@ -381,21 +414,21 @@ testdeletekey(void)
     if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);}
     stat = NC_s3sdkdeletekey(s3client, s3info.bucket, dumpoptions.key, NULL);
 
-    printf("testdeletekey: url %s: ",newurl);
+    printf("testdeletekey: url %s%s: ",newurl,dumpoptions.key);
     switch (stat) {
-    case NC_NOERR:  printf("deleted\n");        break;
-    case NC_EEMPTY: printf("does not exist\n"); break;
-    default:        printf("failed\n");         break;
+    case NC_NOERR:     printf("deleted\n");        break;
+    case NC_ENOOBJECT: printf("does not exist\n"); break;
+    default:           printf("failed\n");         break;
     }
     stat = NC_NOERR; /* reset */
     
     /* Verify deleted and size */
     stat = NC_s3sdkinfo(s3client, s3info.bucket, dumpoptions.key, &size, NULL);
-    printf("testdeletekey.info: url %s: ",newurl);
+    printf("testdeletekey: url %s%s: ",newurl,dumpoptions.key);
     switch (stat) {
-    case NC_NOERR:  printf("not deleted; size=%d\n",(int)size); break;
-    case NC_EEMPTY: printf("deleted\n"); break;
-    default:        printf("failed\n"); goto done;
+    case NC_NOERR:     printf("not deleted; size=%d\n",(int)size); break;
+    case NC_ENOOBJECT: printf("deleted\n"); break;
+    default:           printf("failed\n"); goto done;
     }
     stat = NC_NOERR; /* reset */
 
@@ -408,7 +441,7 @@ static void
 cleanup(void)
 {
     if(s3client)
-        NC_s3sdkclose(s3client, &s3info, 0/*deleteit*/, NULL);
+        NC_s3sdkclose(s3client, NULL);
     s3client = NULL;
     NC_s3clear(&s3info);
     ncurifree(purl); purl = NULL;
@@ -464,12 +497,12 @@ main(int argc, char** argv)
         if((stat = testread())) goto done;
         printf("Test: testinfo\n");
         if((stat = testinfo())) goto done;
-        printf("Test: testgetkeys\n");
-        if((stat = testgetkeys())) goto done;
-        printf("Test: testgetkeyslong\n");
-        if((stat = testgetkeyslong())) goto done;
-        printf("Test: testsearch\n");
-        if((stat = testsearch())) goto done;
+        printf("Test: testlist\n");
+        if((stat = testlist())) goto done;
+        printf("Test: testlistlong\n");
+        if((stat = testlistlong())) goto done;
+        printf("Test: testlistall\n");
+        if((stat = testlistall())) goto done;
         printf("Test: testdeletekey\n");
         if((stat = testdeletekey())) goto done;
     } else {
@@ -498,9 +531,9 @@ main(int argc, char** argv)
         case SIZE_ACTION: stat = testinfo(); break;
         case READ_ACTION: stat = testread(); break;
         case WRITE_ACTION: stat = testwrite(); break;
-        case LIST_ACTION: stat = testgetkeys(); break;
-        case LONGLIST_ACTION: stat = testgetkeyslong(); break;
-        case SEARCH_ACTION: stat = testsearch(); break;
+        case LIST_ACTION: stat = testlist(); break;
+        case LONGLIST_ACTION: stat = testlistlong(); break;
+        case LISTALL_ACTION: stat = testlistall(); break;
         case DELETE_ACTION: stat = testdeletekey(); break;
         case ERROR_ACTION: /* fall thru */
         default: fprintf(stderr,"Illegal action\n"); exit(1);
diff --git a/v3_nczarr_test/s3util.c b/v3_nczarr_test/s3util.c
new file mode 100644
index 0000000000..56fc0cda54
--- /dev/null
+++ b/v3_nczarr_test/s3util.c
@@ -0,0 +1,442 @@
+/*
+ *	Copyright 2018, University Corporation for Atmospheric Research
+ *      See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+#if defined(_WIN32) && !defined(__MINGW32__)
+#include "XGetopt.h"
+#endif
+
+#include "zincludes.h"
+#include "ncs3sdk.h"
+#include "ncpathmgr.h"
+#include "nclog.h"
+#include "ncuri.h"
+#include "netcdf_aux.h"
+
+#undef NODELETE
+
+#undef DEBUG
+
+#define DATANAME "data"
+
+typedef enum S3op {
+S3OP_NONE=0,
+S3OP_LIST=1,
+S3OP_CLEAR=2,
+S3OP_PRINT=3,
+S3OP_UPLOAD=4,
+S3OP_DOWNLOAD=5,
+} S3op;
+
+static struct S3ops {
+    S3op s3op;
+    const char* opnames[3];
+} s3ops[] = {
+{S3OP_LIST,{"list","l",NULL}},
+{S3OP_CLEAR,{"clear","c",NULL}},
+{S3OP_PRINT,{"print","p",NULL}},
+{S3OP_UPLOAD,{"upload","u",NULL}},
+{S3OP_DOWNLOAD,{"download","d",NULL}},
+{S3OP_NONE,{NULL,NULL,NULL}},
+};
+
+/* Command line options */
+struct Dumpptions {
+    int debug;
+    int verbose;
+    S3op s3op;
+    NCURI* url;
+    char* key; /* via -k flag */    
+    char* rootkey; /* from url | key */
+    nc_type nctype; /* for printing content */
+    char* filename;
+    char* profile;
+} dumpoptions;
+
+struct S3SDK {
+    NCS3INFO s3;
+    void* s3client;
+    char* errmsg;
+} s3sdk;
+
+/* Forward */
+static int s3list(void);
+static int s3clear(void);
+static int s3print(void);
+static int s3upload(void);
+static int s3download(void);
+static nc_type typefor(const char* t);
+static void printcontent(size64_t len, const char* content, nc_type nctype);
+
+static void
+usage(void)
+{
+    fprintf(stderr,"usage: s3util  list|print|upload|download|clear -u <url> [-k <key>] [-f <filename>]\n");
+    exit(1);
+}
+
+static S3op
+decodeop(const char* name)
+{
+    struct S3ops* s3op = s3ops;
+    const char** s = NULL;
+    for(;s3op->opnames[0] != NULL;s3op++) {
+        for(s=s3op->opnames;*s;s++) {
+	    if(strcasecmp(*s,name)==0) return s3op->s3op;
+	}
+    }
+    return S3OP_NONE;
+}
+
+static int
+s3setup(void)
+{
+    int stat = NC_NOERR;
+    s3sdk.s3client = NC_s3sdkcreateclient(&s3sdk.s3);
+    return stat;
+}
+
+static int
+s3shutdown(int deleteit)
+{
+    int stat = NC_NOERR;
+    if(deleteit) {
+        stat = s3clear();
+    }
+    stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.errmsg);
+    return stat;
+}
+
+int
+main(int argc, char** argv)
+{
+    int stat = NC_NOERR;
+    int c;
+    char* tmp = NULL;
+
+    nc_initialize();
+
+    memset((void*)&dumpoptions,0,sizeof(dumpoptions));
+
+    dumpoptions.nctype = NC_UBYTE; /* default */
+
+    while ((c = getopt(argc, argv, "df:hk:p:t:T:u:v")) != EOF) {
+	switch(c) {
+	case 'd': 
+	    dumpoptions.debug = 1;	    
+	    break;
+	case 'f':
+	    dumpoptions.filename = strdup(optarg);
+	    break;
+	case 'h':
+	    usage();
+	    return 0;
+	case 'k': {
+	    size_t len = strlen(optarg);
+	    dumpoptions.key = (char*)malloc(len+1+1);
+	    if(*optarg != '/') {
+	        fprintf(stderr,"warning: -k option does not start with '/': %s",optarg);
+		dumpoptions.key[0] = '/';
+		memcpy(dumpoptions.key+1,optarg,len);
+		len++;
+	    } else
+		memcpy(dumpoptions.key,optarg,strlen(optarg));
+	    dumpoptions.key[len] = '\0';
+	    } break;
+	case 'p': 
+	    dumpoptions.profile = strdup(optarg);
+	    break;
+	case 't': 
+	    dumpoptions.nctype = typefor(optarg);
+	    break;
+	case 'u': {
+ 	        char* p = NC_shellUnescape(optarg);
+		ncuriparse(p,&dumpoptions.url);
+		nullfree(p);
+		if(dumpoptions.url == NULL) {
+		    fprintf(stderr,"malformed -u option: %s",optarg);
+		    stat = NC_EINVAL;
+		    goto done;
+		}
+	    } break;
+	case 'v': 
+	    dumpoptions.verbose = 1;
+	    break;
+	case 'T':
+	    nctracelevel(atoi(optarg));
+	    break;
+	case '?':
+	   fprintf(stderr,"unknown option\n");
+	   stat = NC_EINVAL;
+	   goto done;
+	}
+    }
+
+    /* get command argument */
+    argc -= optind;
+    argv += optind;
+
+    if (argc > 1) {
+	int j;
+	fprintf(stderr, "s3util: only one command argument permitted:");
+	for(j=0;j<argc;j++) fprintf(stderr," %s",argv[j]);
+	fprintf(stderr, "\n");
+	stat = NC_EINVAL;
+	goto done;
+    }
+    if (argc == 0) {
+	fprintf(stderr, "s3util: no command specified\n");
+	stat = NC_EINVAL;
+	goto done;
+    }
+
+    dumpoptions.s3op = decodeop(argv[0]);
+
+    memset(&s3sdk,0,sizeof(s3sdk));
+
+    if((stat = NC_s3urlprocess(dumpoptions.url, &s3sdk.s3, NULL))) goto done;
+    if(s3sdk.s3.rootkey != NULL && dumpoptions.key != NULL) {
+	/* Make the root key be the concatenation of rootkey+dumpoptions.key */
+        if((stat = nczm_concat(s3sdk.s3.rootkey,dumpoptions.key,&tmp))) goto done;
+	nullfree(s3sdk.s3.rootkey);
+	s3sdk.s3.rootkey = tmp; tmp = NULL;
+    } else if(dumpoptions.key != NULL) {
+	s3sdk.s3.rootkey = dumpoptions.key;
+	dumpoptions.key = NULL;
+    }
+    if(s3sdk.s3.rootkey == NULL || strlen(s3sdk.s3.rootkey)==0)
+        s3sdk.s3.rootkey = strdup("/");
+
+    switch (dumpoptions.s3op) {
+    default:
+	fprintf(stderr,"Default action: list\n");
+	/* fall thru */
+    case S3OP_LIST:
+	if((stat = s3list())) goto done;
+	break;
+    case S3OP_CLEAR:
+	if((stat = s3clear())) goto done;
+	break;
+    case S3OP_PRINT:
+	if((stat = s3print())) goto done;
+	break;
+    case S3OP_UPLOAD:
+	if((stat = s3upload())) goto done;
+	break;
+    case S3OP_DOWNLOAD:
+	if((stat = s3download())) goto done;
+	break;
+    }    
+
+done:
+    /* Reclaim dumpoptions */
+    ncurifree(dumpoptions.url);
+    nullfree(dumpoptions.rootkey);
+    nullfree(tmp);
+    NC_s3clear(&s3sdk.s3);
+    nc_finalize();
+    if(stat)
+	fprintf(stderr,"fail: %s\n",nc_strerror(stat));
+    return (stat ? 1 : 0);    
+}
+
+static int
+s3list(void)
+{
+    int stat = NC_NOERR;
+    size_t nkeys = 0;
+    char** keys = NULL;
+
+    if(s3setup()) goto done;
+
+    stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
+    if(stat) goto done;
+
+    if(nkeys > 0) {
+        size_t i;
+	/* Sort the list -- shortest first */
+	NC_sortenvv(nkeys,keys);
+	for(i=0;i<nkeys;i++) {
+	    printf("[%u] %s\n",(unsigned)i,keys[i]);
+	}
+
+    } else
+	printf("<empty>\n");
+
+done:
+    s3shutdown(0);
+    NCZ_freeenvv(nkeys,keys);    
+    return stat;
+}
+
+static int
+s3clear(void)
+{
+    int stat = NC_NOERR;
+    size_t nkeys = 0;
+    char** keys = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done;
+
+    if(nkeys > 0 && keys != NULL) {
+	size_t i;
+	/* Sort the list -- shortest first */
+	NC_sortenvv(nkeys,keys);
+	if(dumpoptions.verbose) {
+            printf("deleted keys:\n");
+	    for(i=0;i<nkeys;i++) {
+                printf("\t%s\n",keys[i]);
+            }
+        }
+#ifndef NODELETE
+	for(i=0;i<nkeys;i++) {
+	    if((stat = NC_s3sdkdeletekey(s3sdk.s3client, s3sdk.s3.bucket, keys[i], &s3sdk.errmsg)))
+		goto done;
+        }
+#endif
+    }
+
+done:
+    NCZ_freeenvv(nkeys,keys);    
+    s3shutdown(0);
+    return stat;
+}
+
+static int
+s3print(void)
+{
+    int stat = NC_NOERR;
+    size_t nkeys = 0;
+    char** keys = NULL;
+    size64_t count;
+    char* content = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = NC_s3sdkinfo(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &count,&s3sdk.errmsg)))
+	goto done;
+
+    if((content = (char*)calloc(1,count+1))==NULL)
+        {stat = NC_ENOMEM; goto done;}
+
+    if((stat = NC_s3sdkread(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, 0, count, (void*) content, &s3sdk.errmsg)))
+	goto done;
+
+    printcontent(count,content,dumpoptions.nctype);
+
+done:
+    if(content) free(content);
+    s3shutdown(0);
+    NCZ_freeenvv(nkeys,keys);    
+    return stat;
+}
+
+static int
+s3upload(void)
+{
+    int stat = NC_NOERR;
+    size_t red = 0;
+    void* content = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = ncaux_readfile(dumpoptions.filename,&red,&content)))
+        goto done;
+
+    if((stat = NC_s3sdkwriteobject(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, red, content, &s3sdk.errmsg)))
+	goto done;
+
+done:
+    if(content) free(content);
+    s3shutdown(0);
+    return stat;
+}
+
+static int
+s3download(void)
+{
+    int stat = NC_NOERR;
+    size64_t count;
+    char* content = NULL;
+
+    if(s3setup()) goto done;
+
+    if((stat = NC_s3sdkinfo(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &count,&s3sdk.errmsg)))
+	goto done;
+
+    if((content = (char*)calloc(1,count))==NULL)
+        {stat = NC_ENOMEM; goto done;}
+
+    if((stat = NC_s3sdkread(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, 0, count, (void*) content, &s3sdk.errmsg)))
+	goto done;
+
+    if((stat = ncaux_writefile(dumpoptions.filename,count,content)))
+        goto done;
+
+done:
+    if(content) free(content);
+    s3shutdown(0);
+    return stat;
+}
+
+static void
+printcontent(size64_t len, const char* content, nc_type nctype)
+{
+    size64_t i;
+
+    if(len == 0) {
+        printf("<empty>\n");
+	return;
+    }
+
+    if(nctype == NC_STRING) printf("|");
+    for(i=0;i<len;i++) {
+	    if(nctype != NC_STRING && i > 0) printf(", ");
+	    switch(nctype) {
+	    case NC_BYTE: printf("%d",((char*)content)[i]); break;
+	    case NC_SHORT: printf("%d",((short*)content)[i]); break;		
+	    case NC_INT: printf("%d",((int*)content)[i]); break;		
+	    case NC_INT64: printf("%lld",((long long*)content)[i]); break;		
+	    case NC_UBYTE: printf("%u",((unsigned char*)content)[i]); break;
+	    case NC_USHORT: printf("%u",((unsigned short*)content)[i]); break;		
+	    case NC_UINT: printf("%u",((unsigned int*)content)[i]); break;		
+	    case NC_UINT64: printf("%llu",((unsigned long long*)content)[i]); break;		
+	    case NC_FLOAT: printf("%f",((float*)content)[i]); break;		
+	    case NC_DOUBLE: printf("%lf",((double*)content)[i]); break;		
+	    case NC_STRING: putc(content[i],stdout); break;
+	    default: abort();
+	    }
+    }
+    if(nctype == NC_STRING) printf("|\n");
+}
+
+static nc_type
+typefor(const char* t)
+{
+    if(strcmp(t,"NC_BYTE")==0) return NC_BYTE;
+    else if(strcmp(t,"NC_SHORT")==0) return NC_SHORT;
+    else if(strcmp(t,"NC_INT")==0) return NC_INT;
+    else if(strcmp(t,"NC_INT64")==0) return NC_INT64;
+    else if(strcmp(t,"NC_UBYTE")==0) return NC_UBYTE;
+    else if(strcmp(t,"NC_USHORT")==0) return NC_USHORT;
+    else if(strcmp(t,"NC_UINT")==0) return NC_UINT;
+    else if(strcmp(t,"NC_UINT64")==0) return NC_UINT64;
+    else if(strcmp(t,"NC_FLOAT")==0) return NC_FLOAT;
+    else if(strcmp(t,"NC_DOUBLE")==0) return NC_DOUBLE;
+    else if(strcmp(t,"NC_STRING")==0) return NC_STRING;
+    return NC_NAT;
+}