Read + Write Zarr with consolidated metadata

include/ncuri.h

@@ -59,6 +59,12 @@ typedef struct NCURI {
 extern "C" {
 #endif
 
+#ifndef HAVE_STRNDUP
+#define strndup ncstrndup
+/* Not all systems have strndup, so provide one*/
+char *ncstrndup(const char *s, size_t len);
+#endif
+
 EXTERNL int ncuriparse(const char* s, NCURI** ncuri);
 EXTERNL void ncurifree(NCURI* ncuri);
 

include/netcdf.h

@@ -531,8 +531,10 @@ by the desired type. */
 #define NC_EOBJECT       (-140)    /**< Some object exists when it should not */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
+#define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
+#define NC_EZARRMETA     (-144)    /**< Malformed (NC)Zarr file consolidated metadata */
 
-#define NC4_LAST_ERROR   (-142)    /**< @internal All netCDF errors > this. */
+#define NC4_LAST_ERROR   (-144)    /**< @internal All netCDF errors > this. */
 
 /*
  * Don't forget to update docs/all-error-codes.md if adding new error codes here!

libdispatch/ncuri.c

@@ -79,7 +79,6 @@ static const char* userpwdallow =
 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!$&'()*+,-.;=_~?#/";
 
 #ifndef HAVE_STRNDUP
-#define strndup ncstrndup
 /* Not all systems have strndup, so provide one*/
 char*
 ncstrndup(const char* s, size_t len)

libnczarr/CMakeLists.txt

@@ -23,6 +23,8 @@ zgrp.c
 zinternal.c
 zmap.c
 zmap_file.c
+zmetadata2.c
+zmetadata.c
 zodom.c
 zopen.c
 zprov.c
@@ -39,6 +41,7 @@ zdispatch.h
 zincludes.h
 zinternal.h
 zmap.h
+zmetadata.h
 zodom.h
 zprovenance.h
 zplugins.h

libnczarr/Makefile.am

@@ -43,6 +43,8 @@ zgrp.c \
 zinternal.c \
 zmap.c \
 zmap_file.c \
+zmetadata2.c \
+zmetadata.c \
 zodom.c \
 zopen.c \
 zprov.c \
@@ -59,6 +61,7 @@ zdispatch.h \
 zincludes.h \
 zinternal.h \
 zmap.h \
+zmetadata.h \
 zodom.h \
 zprovenance.h \
 zplugins.h \

libnczarr/zarr.c

@@ -79,6 +79,10 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls)
     if((stat = nczmap_create(zinfo->controls.mapimpl,nc->path,nc->mode,zinfo->controls.flags,NULL,&zinfo->map)))
 	goto done;
 
+    /* Initialize metadata handle */
+    assert(zinfo->map != NULL);
+    if((stat = NCZMD_set_metadata_handler(zinfo))) goto done;
+
 done:
     ncurifree(uri);
     NCJreclaim(json);
@@ -143,6 +147,10 @@ ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls)
     if((stat = nczmap_open(zinfo->controls.mapimpl,nc->path,mode,zinfo->controls.flags,NULL,&zinfo->map)))
 	goto done;
 
+    /* Initialize metadata handle */
+    assert(zinfo->map != NULL);
+    NCZMD_set_metadata_handler(zinfo);
+
     /* Ok, try to read superblock */
     if((stat = ncz_read_superblock(file,&nczarr_version,&zarr_format))) goto done;
 
@@ -268,6 +276,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
     /* Process the modelist first */
     zinfo->controls.mapimpl = NCZM_DEFAULT;
     zinfo->controls.flags |= FLAG_XARRAYDIMS; /* Always support XArray convention where possible */
+    zinfo->controls.flags |= (NCZARR_CONSOLIDATED_DEFAULT ? FLAG_CONSOLIDATED : 0); /* Default to consolidated metadata */
     for(i=0;i<nclistlength(modelist);i++) {
         const char* p = nclistget(modelist,i);
 	if(strcasecmp(p,PUREZARRCONTROL)==0)
@@ -279,6 +288,9 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
 	else if(strcasecmp(p,"zip")==0) zinfo->controls.mapimpl = NCZM_ZIP;
 	else if(strcasecmp(p,"file")==0) zinfo->controls.mapimpl = NCZM_FILE;
 	else if(strcasecmp(p,"s3")==0) zinfo->controls.mapimpl = NCZM_S3;
+    else if(strcasecmp(p,"consolidate") == 0 || strcasecmp(p,"consolidated") == 0) {
+        zinfo->controls.flags |= FLAG_CONSOLIDATED;
+    }
     }
     /* Apply negative controls by turning off negative flags */
     /* This is necessary to avoid order dependence of mode flags when both positive and negative flags are defined */

libnczarr/zarr.h

@@ -69,7 +69,7 @@ EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* ty
 EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
 EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
 EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
-EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
+EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json);
 EXTERNL int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
 EXTERNL int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist);
 EXTERNL int NCZ_grpname_full(int gid, char** pathp);

libnczarr/zclose.c

@@ -51,6 +51,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort)
 	goto done;
     nclistfreeall(zinfo->controllist);
     NC_authfree(zinfo->auth);
+    NCZMD_free_metadata_handler(&(zinfo->metadata));
     nullfree(zinfo);
 
 done:

libnczarr/zincludes.h

@@ -51,6 +51,7 @@ extern "C" {
 #include "ncutil.h"
 
 #include "zmap.h"
+#include "zmetadata.h"
 #include "zinternal.h"
 #include "zdispatch.h"
 #include "zprovenance.h"

libnczarr/zinternal.h

@@ -13,6 +13,7 @@
 #define ZINTERNAL_H
 
 #define ZARRVERSION "2"
+#define ZARRFORMAT2 2
 
 /* NCZARRVERSION is independent of Zarr version,
    but NCZARRVERSION => ZARRVERSION */
@@ -22,6 +23,11 @@
 #define NCZ_CHUNKSIZE_FACTOR (10)
 #define NCZ_MIN_CHUNK_SIZE (2)
 
+/* The name of the env var for controlling .zmetadata use*/
+#define NCZARR_CONSOLIDATED_KEY_ENV "NCZARR_METADATA_CONSOLIDATED_KEY"
+#define NCZARR_CONSOLIDATED_ENV "NCZARR_CONSOLIDATED"
+#define NCZARR_CONSOLIDATED_DEFAULT 0 /* default to consolidated metadata */
+
 /**************************************************/
 /* Constants */
 
@@ -38,11 +44,12 @@
 #  endif
 #endif
 
-#define ZMETAROOT "/.zgroup"
-#define ZMETAATTR "/.zattrs"
-#define ZGROUP ".zgroup"
-#define ZATTRS ".zattrs"
-#define ZARRAY ".zarray"
+/* V2 Reserved Objects */
+#define Z2METAROOT "/.zgroup"
+#define Z2ATTSROOT "/.zattrs"
+#define Z2GROUP ".zgroup"
+#define Z2ATTRS ".zattrs"
+#define Z2ARRAY ".zarray"
 
 /* V2 Reserved Attributes */
 /*
@@ -141,8 +148,10 @@ typedef struct NCZ_FILE_INFO {
 #		define FLAG_LOGGING     4
 #		define FLAG_XARRAYDIMS  8
 #		define FLAG_NCZARR_KEY  16 /* _nczarr_xxx keys are stored in object and not in _nczarr_attrs */
+#       define FLAG_CONSOLIDATED 32 /* Use/set consolidated metadata */
 	NCZM_IMPL mapimpl;
     } controls;
+    struct NCZ_Metadata metadata;
     int default_maxstrlen; /* default max str size for variables of type string */
 } NCZ_FILE_INFO_T;
 

libnczarr/zmap_s3sdk.c

@@ -225,15 +225,6 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
         stat = NC_ES3; goto done;
     }
 
-    /* Search the root for content */
-    content = nclistnew();
-    if((stat = NC_s3sdklist(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
-	goto done;
-    if(nkeys == 0) {
-	/* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
-	stat = NC_ENOOBJECT;
-	goto done;
-    }
     if(mapp) *mapp = (NCZMAP*)z3map;    
 
 done:

libnczarr/zmetadata.c

@@ -0,0 +1,161 @@
+/*********************************************************************
+ *   Copyright 2018, UCAR/Unidata
+ *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ *********************************************************************/
+
+#include "zincludes.h"
+
+static int
+cmpstrings(const void* a1, const void* a2)
+{
+    const char** s1 = (const char**)a1;
+    const char** s2 = (const char**)a2;
+    return strcmp(*s1,*s2);
+}
+
+int NCZMD_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars)
+{
+    int stat = NC_NOERR;
+    if((stat = zfile->metadata.list_nodes(zfile,key, groups, vars))){
+        return stat;
+    }
+    qsort(groups->content, groups->length, sizeof(char*), cmpstrings);
+    qsort(vars->content, vars->length, sizeof(char*), cmpstrings);
+    return stat;
+}
+
+int NCZMD_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *subgrpnames)
+{
+    int stat = NC_NOERR;
+    if((stat = zfile->metadata.list_groups(zfile,key, subgrpnames))){
+        return stat;
+    }
+    qsort(subgrpnames->content, subgrpnames->length, sizeof(char*), cmpstrings);
+    return stat;
+}
+
+int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist *varnames)
+{
+	int stat = NC_NOERR;
+    if((stat = zfile->metadata.list_variables(zfile, key, varnames))){
+        return stat;
+    }
+    qsort(varnames->content, varnames->length, sizeof(char*), cmpstrings);
+    return stat;
+}
+
+int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jgroup) {
+	return zfile->metadata.fetch_json_content(zfile, NCZMD_GROUP, key, jgroup);
+}
+
+int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jattrs) {
+	return  zfile->metadata.fetch_json_content(zfile, NCZMD_ATTRS, key, jattrs);
+}
+
+int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jarray) {
+	return zfile->metadata.fetch_json_content(zfile, NCZMD_ARRAY, key, jarray);
+}
+
+int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jgroup) {
+	return zfile->metadata.update_json_content(zfile, NCZMD_GROUP, key, jgroup);
+}
+
+int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jattrs) {
+	return zfile->metadata.update_json_content(zfile, NCZMD_ATTRS, key , jattrs);
+}
+
+int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jarray) {
+	return zfile->metadata.update_json_content(zfile, NCZMD_ARRAY, key, jarray);
+}
+
+int NCZMD_consolidate(NCZ_FILE_INFO_T *zfile) {
+	int stat = NC_NOERR;
+	if (zfile->creating == 1 && zfile->metadata.jcsl !=NULL){
+		stat = NCZ_uploadjson(zfile->map, Z2METADATA ,zfile->metadata.jcsl);
+	}
+	return stat;
+}
+
+int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile)
+{
+	NCZ_Metadata *zmd = &(zfile->metadata);
+	if (zmd == NULL ||
+		zmd->jcsl == NULL ||
+		NCJsort(zmd->jcsl) != NCJ_DICT ||
+		!(zmd->flags & ZARR_CONSOLIDATED))
+	{
+		return NC_ENOOBJECT;
+	}
+	return NC_NOERR;
+}
+
+int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat)
+{
+    NCZ_Metadata *zmd = &(zfile->metadata);
+
+	if (zmd->zarr_format >= ZARRFORMAT2)
+	{
+		*zarrformat = zmd->zarr_format;
+		return NC_NOERR;
+	}
+
+	if (!nczmap_exists(zfile->map, "/" Z2ATTRS) && !nczmap_exists(zfile->map, "/" Z2GROUP) && !nczmap_exists(zfile->map, "/" Z2ARRAY))
+	{
+		return NC_ENOTZARR;
+	}
+
+	*zarrformat = ZARRFORMAT2;
+	return NC_NOERR;
+}
+
+
+int use_consolidated_metadata(NCZ_FILE_INFO_T *zfile)
+{
+    int use_consolidated = NCZARR_CONSOLIDATED_DEFAULT || (zfile->controls.flags & FLAG_CONSOLIDATED);
+    const char *e = getenv(NCZARR_CONSOLIDATED_ENV);
+
+    int env_use_consolidated = (e != NULL)  && (
+            (atoi(e) > 0) ||(strcasecmp(e, "true") == 0) || (strcasecmp(e, "yes") == 0)
+        );
+
+    return use_consolidated || env_use_consolidated ;
+}
+
+int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile)
+{
+    NCjson *jcsl = NULL;
+
+    int use_consolidated = use_consolidated_metadata(zfile);
+    if (!use_consolidated){
+        nclog(NCLOGNOTE, "Not using consolidated metadata! Doing so could improve reading performance");
+    }
+
+    if (use_consolidated && zfile->creating) {
+        zfile->metadata = *NCZ_csl_metadata_handler2;
+        return NC_NOERR;
+    }
+
+	zfile->metadata = *NCZ_metadata_handler2;
+    if (!use_consolidated)
+        return NC_NOERR;
+
+    if (NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl) || jcsl == NULL) {
+        nclog(NCLOGNOTE, "Dataset not consolidated! Doing so will improve performance");
+        return NC_NOERR;
+    }
+
+    if (NCZ_csl_metadata_handler2->validate_consolidated(jcsl) != NC_NOERR) {
+        nclog(NCLOGWARN,"Consolidated metadata is invalid, ignoring it!");
+        return NC_EZARRMETA;
+    }
+
+    zfile->metadata = *NCZ_csl_metadata_handler2;
+    zfile->metadata.jcsl = jcsl;
+    return NC_NOERR;
+}
+
+void NCZMD_free_metadata_handler(NCZ_Metadata * zmd){
+	if (zmd == NULL) return;
+	NCJreclaim(zmd->jcsl);
+    zmd->jcsl = NULL;
+}