Fix various problems with S3 related code

RELEASE_NOTES.md

@@ -5,6 +5,10 @@ Release Notes       {#RELEASE_NOTES}
 
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 
+## 4.9.4 - TBD
+
+* Clean up the S3 API for all non-libnczarr code. This continues the splitting of PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
+See [Github #3090](https://github.com/Unidata/netcdf-c/pull/3090) for more information.
 * Step 1 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Update ncjson.[ch] and ncproplist.[ch]. Also fix references to old API. Also fix include/netcdf_ncjson.h and include/netcdf_proplist.h builds. See [Github #3086](https://github.com/Unidata/netcdf-c/pull/3086) for more information.
 * Provide an auxilliary function, `ncaux_parse_provenance()`, that allows users to parse the _NCProperties attribute into a collection of character pointers. See [Github #3088](https://github.com/Unidata/netcdf-c/pull/3088) for more information.
 * Step 2 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Fix a namespace problem in tinyxml2.cpp. Note that this is a visual studio problem hence use of _MSC_VER. Also turn off DAP4 tests against Hyrax server until DAP4 spec problems are fixed. See [Github #3089](https://github.com/Unidata/netcdf-c/pull/3089) for more information.

include/ncs3sdk.h

@@ -14,10 +14,16 @@
 
 /* Track the server type, if known */
 typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
-                 NCS3=1,    /* s3.amazon.aws */
-		 NCS3GS=2   /* storage.googleapis.com */
+	NCS3=1,     /* s3.amazon.aws */
+	NCS3GS=2,   /* storage.googleapis.com */
+#ifdef NETCDF_ENABLE_ZOH
+	NCS3ZOH=4,  /* ZoH Server */
+#endif
 } NCS3SVC;
 
+/* Opaque Handles */
+struct NClist;
+
 typedef struct NCS3INFO {
     char* host; /* non-null if other*/
     char* region; /* region */
@@ -55,9 +61,10 @@ EXTERNL int NC_s3sdkbucketdelete(void* s3client, NCS3INFO* info, char** errmsgp)
 EXTERNL int NC_s3sdkinfo(void* client0, const char* bucket, const char* pathkey, unsigned long long* lenp, char** errmsgp);
 EXTERNL int NC_s3sdkread(void* client0, const char* bucket, const char* pathkey, unsigned long long start, unsigned long long count, void* content, char** errmsgp);
 EXTERNL int NC_s3sdkwriteobject(void* client0, const char* bucket, const char* pathkey, unsigned long long count, const void* content, char** errmsgp);
-EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp);
-EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
-EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
+EXTERNL int NC_s3sdkclose(void* s3client0, char** errmsgp);
+EXTERNL int NC_s3sdktruncate(void* s3client0, const char* bucket, const char* prefix, char** errmsgp);
+EXTERNL int NC_s3sdklist(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
+EXTERNL int NC_s3sdklistall(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
 EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
 
 /* From ds3util.c */

libdispatch/ds3util.c

@@ -25,6 +25,7 @@
 #include "nc4internal.h"
 #include "ncuri.h"
 #include "nclist.h"
+#include "ncbytes.h"
 #include "ncrc.h"
 #include "nclog.h"
 #include "ncs3sdk.h"
@@ -34,9 +35,6 @@
 /* Alternate .aws directory location */
 #define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR"
 
-#define AWSHOST ".amazonaws.com"
-#define GOOGLEHOST "storage.googleapis.com"
-
 enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4};
 
 /* Read these files in order and later overriding earlier */
@@ -47,6 +45,7 @@ static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
 /* Forward */
 
 static int endswith(const char* s, const char* suffix);
+static void freeprofile(struct AWSprofile* profile);
 static void freeentry(struct AWSentry* e);
 static int awsparse(const char* text, NClist* profiles);
 
@@ -78,9 +77,9 @@ NC_s3sdkenvironment(void)
 /*
 Rebuild an S3 url into a canonical path-style url.
 If region is not in the host, then use specified region
-if provided, otherwise leave blank and let the S3 server deal with it.
-@param url (in) the current url
-@param s3	(in/out) NCS3INFO structure
+if provided, otherwise us-east-1.
+@param url      (in) the current url
+@param s3       (in/out) NCS3INFO structure
 @param pathurlp (out) the resulting pathified url string
 */
 
@@ -309,7 +308,13 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
 
     /* Rebuild the URL to path format and get a usable region and optional bucket*/
     if((stat = NC_s3urlrebuild(url,s3,&url2))) goto done;
-    s3->host = strdup(url2->host);
+    if(url2->port){
+	char hostport[8192];
+	snprintf(hostport,sizeof(hostport),"%s:%s",url2->host,url2->port);
+	s3->host = strdup(hostport);
+    }else{
+        s3->host = strdup(url2->host);
+    }
     /* construct the rootkey minus the leading bucket */
     pathsegments = nclistnew();
     if((stat = NC_split_delim(url2->path,'/',pathsegments))) goto done;
@@ -358,19 +363,29 @@ NC_s3clear(NCS3INFO* s3)
 }
 
 /*
-Check if a url has indicators that signal an S3 or Google S3 url.
+Check if a url has indicators that signal an S3 or Google S3 url or ZoH S3 url.
+The rules are as follows:
+1. If the protocol is "s3" or "gs3" or "zoh", then return (true,s3|gs3|zoh).
+2. If the mode contains "s3" or "gs3" or "zoh", then return (true,s3|gs3|zoh).
+3. Check the host name:
+3.1 If the host ends with ".amazonaws.com", then return (true,s3).
+3.1 If the host is "storage.googleapis.com", then return (true,gs3).
+4. Otherwise return (false,unknown).
 */
 
 int
-NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
+NC_iss3(NCURI* uri, NCS3SVC* svcp)
 {
     int iss3 = 0;
     NCS3SVC svc = NCS3UNK;
 
     if(uri == NULL) goto done; /* not a uri */
-    /* is the protocol "s3" or "gs3" ? */
+    /* is the protocol "s3" or "gs3" or "zoh" ? */
     if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; svc = NCS3; goto done;}
     if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; svc = NCS3GS; goto done;}
+#ifdef NETCDF_ENABLE_ZOH
+    if(strcasecmp(uri->protocol,"zoh")==0) {iss3 = 1; svc = NCS3ZOH; goto done;}
+#endif
     /* Is "s3" or "gs3" in the mode list? */
     if(NC_testmode(uri,"s3")) {iss3 = 1; svc = NCS3; goto done;}
     if(NC_testmode(uri,"gs3")) {iss3 = 1; svc = NCS3GS; goto done;}    
@@ -384,18 +399,71 @@ NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
     return iss3;
 }
 
-const char*
-NC_s3dumps3info(NCS3INFO* info)
+/**************************************************/
+/**
+The .aws/config and .aws/credentials files
+are in INI format (https://en.wikipedia.org/wiki/INI_file).
+This format is not well defined, so the grammar used
+here is restrictive. Here, the term "profile" is the same
+as the INI term "section".
+
+The grammar used is as follows:
+
+Grammar:
+
+inifile: profilelist ;
+profilelist: profile | profilelist profile ;
+profile: '[' profilename ']' EOL entries ;
+entries: empty | entries entry ;
+entry:  WORD = WORD EOL ;
+profilename: WORD ;
+Lexical:
+WORD    sequence of printable characters - [ \[\]=]+
+EOL	'\n' | ';'
+
+Note:
+1. The semicolon at beginning of a line signals a comment.
+2. # comments are not allowed
+3. Duplicate profiles or keys are ignored.
+4. Escape characters are not supported.
+*/
+
+#define AWS_EOF (-1)
+#define AWS_ERR (0)
+#define AWS_WORD (0x10001)
+#define AWS_EOL (0x10002)
+
+typedef struct AWSparser {
+    char* text;
+    char* pos;
+    size_t yylen; /* |yytext| */
+    NCbytes* yytext;
+    int token; /* last token found */
+    int pushback; /* allow 1-token pushback */
+} AWSparser;
+
+#ifdef LEXDEBUG
+static const char*
+tokenname(int token)
 {
-    static char text[8192];
-    snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
-		(info->host?info->host:"null"),
-		(info->region?info->region:"null"),
-		(info->bucket?info->bucket:"null"),
-		(info->rootkey?info->rootkey:"null"),
-		(info->profile?info->profile:"null"));
-    return text;
+    static char num[32];
+    switch(token) {
+    case AWS_EOF: return "EOF";
+    case AWS_ERR: return "ERR";
+    case AWS_WORD: return "WORD";
+    default: snprintf(num,sizeof(num),"%d",token); return num;
+    }
+    return "UNKNOWN";
 }
+#endif
+
+/*
+@param text of the aws credentials file
+@param profiles list of form struct AWSprofile (see ncauth.h)
+*/
+
+#define LBR '['
+#define RBR ']'
 
 static void
 freeprofile(struct AWSprofile* profile)
@@ -426,6 +494,19 @@ NC_s3freeprofilelist(NClist* profiles)
     }
 }
 
+const char*
+NC_s3dumps3info(NCS3INFO* info)
+{
+    static char text[8192];
+    snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
+		(info->host?info->host:"null"),
+		(info->region?info->region:"null"),
+		(info->bucket?info->bucket:"null"),
+		(info->rootkey?info->rootkey:"null"),
+		(info->profile?info->profile:"null"));
+    return text;
+}
+
 /* Find, load, and parse the aws config &/or credentials file */
 int
 NC_aws_load_credentials(NCglobalstate* gstate)
@@ -707,15 +788,6 @@ tokenname(int token)
 }
 #endif
 
-typedef struct AWSparser {
-    char* text;
-    char* pos;
-    size_t yylen; /* |yytext| */
-    NCbytes* yytext;
-    int token; /* last token found */
-    int pushback; /* allow 1-token pushback */
-} AWSparser;
-
 static int
 awslex(AWSparser* parser)
 {