diff --git a/Makefile b/Makefile index 0b988399..b0a2a0b8 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ clean: rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check indent: - clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/*.[ch] + clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/jsonpull.h jsonpull/jsonpull.cpp TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) diff --git a/attribute.cpp b/attribute.cpp index 53c9dae5..e4deb932 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -42,10 +42,10 @@ void set_attribute_accum(std::unordered_map &attribut void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv) { if (*arg == '{') { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: -E%s: %s\n", *argv, arg, jp->error); exit(EXIT_JSON); } @@ -55,24 +55,21 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - json_object *k = o->value.object.keys[i]; - json_object *v = o->value.object.values[i]; - - if (k->type != JSON_STRING) { + size_t i = 0; + for (const auto &e : o->entries()) { + if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: key %zu not a string\n", *argv, arg, i); exit(EXIT_JSON); } - if (v->type != JSON_STRING) { + if (e.value->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: value %zu not a string\n", *argv, arg, i); exit(EXIT_JSON); } - set_attribute_accum(attribute_accum, k->value.string.string, v->value.string.string); + set_attribute_accum(attribute_accum, e.key->string().c_str(), e.value->string().c_str()); + i++; } - json_free(o); - json_end(jp); return; } diff --git a/dirtiles.cpp b/dirtiles.cpp index 98138bd5..452ecf9e 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -248,9 +248,9 @@ sqlite3 *dirmeta2tmp(const char *fname) { if (f == NULL) { perror(name.c_str()); } else { - json_pull *jp = json_begin_file(f); - json_object *o = json_read_tree(jp); - if (o == NULL) { + json_pull_ptr jp = json_begin_file(f); + json_object_ptr o = json_read_tree(jp); + if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", name.c_str(), jp->error); exit(EXIT_JSON); } @@ -260,19 +260,18 @@ sqlite3 *dirmeta2tmp(const char *fname) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { + for (const auto &e : o->entries()) { + if (e.key->type != JSON_STRING || e.value->type != JSON_STRING) { fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); } - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->value.object.keys[i]->value.string.string, o->value.object.values[i]->value.string.string); + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", e.key->string().c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->value.object.keys[i]->value.string.string, err); + fprintf(stderr, "set %s in metadata: %s\n", e.key->string().c_str(), err); } sqlite3_free(sql); } - json_end(jp); fclose(f); } diff --git a/evaluator.cpp b/evaluator.cpp index 38566ae2..ce1e576c 100644 --- a/evaluator.cpp +++ b/evaluator.cpp @@ -17,7 +17,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { return false; // string vs non-string } - return strcmp(one.c_str(), two->value.string.string); + return strcmp(one.c_str(), two->string().c_str()); case mvt_double: case mvt_float: @@ -52,9 +52,9 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { exit(EXIT_IMPOSSIBLE); } - if (v < two->value.number.number) { + if (v < two->number()) { return -1; - } else if (v > two->value.number.number) { + } else if (v > two->number()) { return 1; } else { return 0; @@ -92,7 +92,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { // 1: true // -1: incomparable (sql null), treated as false in final output static int eval(std::function feature, json_object *f, std::set &exclude_attributes, std::vector const &unidecode_data) { - if (f != NULL) { + if (f != nullptr) { if (f->type == JSON_TRUE) { return 1; } else if (f->type == JSON_FALSE) { @@ -102,7 +102,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_NUMBER) { - if (f->value.number.number == 0) { + if (f->number() == 0) { return 0; } else { return 1; @@ -110,7 +110,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_STRING) { - if (f->value.string.string[0] == '\0') { + if (f->string().empty()) { return 0; } else { return 1; @@ -118,131 +118,129 @@ static int eval(std::function feature, json_obje } } - if (f == NULL || f->type != JSON_ARRAY) { - fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f)); + if (f == nullptr || f->type != JSON_ARRAY) { + fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.length < 1) { - fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + if (f->array().size() < 1) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[0]->type != JSON_STRING) { - fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f)); + if (f->array()[0]->type != JSON_STRING) { + fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { - if (f->value.array.length != 2) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + const std::string &op = f->array()[0]->string(); + + if (op == "has" || + op == "!has") { + if (f->array().size() != 2) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0) { - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f)); + if (op == "has") { + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key; + return feature(f->array()[1]->string()).type != mvt_no_such_key; } - if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); + if (op == "!has") { + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key; + return feature(f->array()[1]->string()).type == mvt_no_such_key; } } - if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!=") == 0 || - strcmp(f->value.array.array[0]->value.string.string, ">") == 0 || - strcmp(f->value.array.array[0]->value.string.string, ">=") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "<") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) { - if (f->value.array.length != 3) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + if (op == "==" || + op == "!=" || + op == ">" || + op == ">=" || + op == "<" || + op == "<=") { + if (f->array().size() != 3) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f)); + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + mvt_value ff = feature(f->array()[1]->string()); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return true; // attributes that aren't found are not equal } return false; // not found: comparison is false } bool fail = false; - int cmp = compare(ff, f->value.array.array[2], fail); + int cmp = compare(ff, f->array()[2].get(), fail); if (fail) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return true; // mismatched types are not equal } return false; } - if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0) { + if (op == "==") { return cmp == 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return cmp != 0; } - if (strcmp(f->value.array.array[0]->value.string.string, ">") == 0) { + if (op == ">") { return cmp > 0; } - if (strcmp(f->value.array.array[0]->value.string.string, ">=") == 0) { + if (op == ">=") { return cmp >= 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "<") == 0) { + if (op == "<") { return cmp < 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) { + if (op == "<=") { return cmp <= 0; } - fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f)); + fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f).c_str()); exit(EXIT_IMPOSSIBLE); } - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "any") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "none") == 0) { + if (op == "all" || + op == "any" || + op == "none") { bool v; - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) { + if (op == "all") { v = true; } else { v = false; } - for (size_t i = 1; i < f->value.array.length; i++) { - int out = eval(feature, f->value.array.array[i], exclude_attributes, unidecode_data); + for (size_t i = 1; i < f->array().size(); i++) { + int out = eval(feature, f->array()[i].get(), exclude_attributes, unidecode_data); if (out >= 0) { // nulls are ignored in boolean and/or expressions - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) { + if (op == "all") { v = v && out; if (!v) { break; @@ -256,51 +254,47 @@ static int eval(std::function feature, json_obje } } - if (strcmp(f->value.array.array[0]->value.string.string, "none") == 0) { + if (op == "none") { return !v; } else { return v; } } - if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) { - if (f->value.array.length < 2) { - fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + if (op == "in" || + op == "!in") { + if (f->array().size() < 2) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f)); + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + mvt_value ff = feature(f->array()[1]->string()); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) { + if (op == "!in") { return true; // attributes that aren't found are not in } return false; // not found: comparison is false } bool found = false; - for (size_t i = 2; i < f->value.array.length; i++) { + for (size_t i = 2; i < f->array().size(); i++) { bool fail = false; - int cmp = compare(ff, f->value.array.array[i], fail); + int cmp = compare(ff, f->array()[i].get(), fail); if (fail) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str()); warned = true; } cmp = 1; @@ -312,39 +306,39 @@ static int eval(std::function feature, json_obje } } - if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0) { + if (op == "in") { return found; } else { return !found; } } - if (strcmp(f->value.array.array[0]->value.string.string, "attribute-filter") == 0) { - if (f->value.array.length != 3) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + if (op == "attribute-filter") { + if (f->array().size() != 3) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f)); + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - bool ok = eval(feature, f->value.array.array[2], exclude_attributes, unidecode_data) > 0; + bool ok = eval(feature, f->array()[2].get(), exclude_attributes, unidecode_data) > 0; if (!ok) { - exclude_attributes.insert(f->value.array.array[1]->value.string.string); + exclude_attributes.insert(f->array()[1]->string()); } return true; } - fprintf(stderr, "Unknown filter %s\n", json_stringify(f)); + fprintf(stderr, "Unknown filter %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } -bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { - if (filter == NULL || filter->type != JSON_HASH) { - fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter)); +static bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { + if (filter == nullptr || filter->type != JSON_HASH) { + fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter).c_str()); exit(EXIT_JSON); } @@ -352,47 +346,43 @@ bool evaluate(std::function feature, std::string json_object *f; f = json_hash_get(filter, layer.c_str()); - if (ok && f != NULL) { + if (ok && f != nullptr) { ok = eval(feature, f, exclude_attributes, unidecode_data) > 0; } f = json_hash_get(filter, "*"); - if (ok && f != NULL) { + if (ok && f != nullptr) { ok = eval(feature, f, exclude_attributes, unidecode_data) > 0; } return ok; } -json_object *read_filter(const char *fname) { +json_object_ptr read_filter(const char *fname) { FILE *fp = fopen(fname, "r"); if (fp == NULL) { perror(fname); exit(EXIT_OPEN); } - json_pull *jp = json_begin_file(fp); - json_object *filter = json_read_tree(jp); - if (filter == NULL) { + json_pull_ptr jp = json_begin_file(fp); + json_object_ptr filter = json_read_tree(jp); + if (filter == nullptr) { fprintf(stderr, "%s: %s\n", fname, jp->error); exit(EXIT_JSON); } - json_disconnect(filter); - json_end(jp); fclose(fp); return filter; } -json_object *parse_filter(const char *s) { - json_pull *jp = json_begin_string(s); - json_object *filter = json_read_tree(jp); - if (filter == NULL) { +json_object_ptr parse_filter(const char *s) { + json_pull_ptr jp = json_begin_string(s); + json_object_ptr filter = json_read_tree(jp); + if (filter == nullptr) { fprintf(stderr, "Could not parse filter %s\n", s); fprintf(stderr, "%s\n", jp->error); exit(EXIT_JSON); } - json_disconnect(filter); - json_end(jp); return filter; } diff --git a/evaluator.hpp b/evaluator.hpp index bc6be00c..4be1582d 100644 --- a/evaluator.hpp +++ b/evaluator.hpp @@ -1,5 +1,5 @@ #ifndef EVALUATOR_HPP -#define EVALUATOR HPP +#define EVALUATOR_HPP #include #include @@ -7,9 +7,13 @@ #include "jsonpull/jsonpull.h" #include "mvt.hpp" +// The `filter` parameters take a borrowed pointer; the caller (in +// main.cpp, tile-join, overzoom) keeps the json_object_ptr alive +// across every per-feature evaluate() call. A raw pointer avoids +// touching unique_ptr at all on this hot path. bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data); -json_object *parse_filter(const char *s); -json_object *read_filter(const char *fname); +json_object_ptr parse_filter(const char *s); +json_object_ptr read_filter(const char *fname); bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data); diff --git a/geobuf.cpp b/geobuf.cpp index 02d8ded3..6481695e 100644 --- a/geobuf.cpp +++ b/geobuf.cpp @@ -394,28 +394,25 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vectorsecond.s.c_str()); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(tip->second.s.c_str()); + json_object_ptr o = json_read_tree(jp); - if (o != NULL) { + if (o != nullptr) { json_object *min = json_hash_get(o, "minzoom"); - if (min != NULL && (min->type == JSON_NUMBER)) { - sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); + if (min != nullptr && (min->type == JSON_NUMBER)) { + sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } json_object *max = json_hash_get(o, "maxzoom"); - if (max != NULL && (max->type == JSON_NUMBER)) { - sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); + if (max != nullptr && (max->type == JSON_NUMBER)) { + sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } json_object *tlayer = json_hash_get(o, "layer"); - if (tlayer != NULL && (tlayer->type == JSON_STRING)) { - layername = tlayer->value.string.string; + if (tlayer != nullptr && (tlayer->type == JSON_STRING)) { + layername = tlayer->string(); } } - - json_free(o); - json_end(jp); } serialize_feature(sst, sf, layername); diff --git a/geojson-loop.cpp b/geojson-loop.cpp index 199c6531..e4f2f40b 100644 --- a/geojson-loop.cpp +++ b/geojson-loop.cpp @@ -26,34 +26,34 @@ static const char *geometry_names[GEOM_TYPES] = { // XXX duplicated static void json_context(json_object *j) { - char *s = json_stringify(j); + std::string s = json_stringify(j); - if (strlen(s) >= 500) { - snprintf(s + 497, strlen(s) + 1 - 497, "..."); + if (s.size() >= 500) { + s.resize(497); + s.append("..."); } - fprintf(stderr, "in JSON object %s\n", s); - free(s); // stringify + fprintf(stderr, "in JSON object %s\n", s.c_str()); } -void parse_json(json_feature_action *jfa, json_pull *jp) { +void parse_json(json_feature_action *jfa, json_pull_ptr &jp) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; while (1) { json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "%s:%d: %s: ", jfa->fname.c_str(), jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); + if (jp->root != nullptr) { + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } } - json_free(jp->root); + jp->root.reset(); break; } @@ -66,7 +66,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + if (type == nullptr || type->type != JSON_STRING) { continue; } @@ -74,25 +74,25 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { int i; int is_geometry = 0; for (i = 0; i < GEOM_TYPES; i++) { - if (strcmp(type->value.string.string, geometry_names[i]) == 0) { + if (type->string() == geometry_names[i]) { is_geometry = 1; break; } } if (is_geometry) { - if (j->parent != NULL) { - if (j->parent->type == JSON_ARRAY && j->parent->parent != NULL) { + if (j->parent != nullptr) { + if (j->parent->type == JSON_ARRAY && j->parent->parent != nullptr) { if (j->parent->parent->type == JSON_HASH) { json_object *geometries = json_hash_get(j->parent->parent, "geometries"); - if (geometries != NULL) { + if (geometries != nullptr) { // Parent of Parent must be a GeometryCollection is_geometry = 0; } } } else if (j->parent->type == JSON_HASH) { json_object *geometry = json_hash_get(j->parent, "geometry"); - if (geometry != NULL) { + if (geometry != nullptr) { // Parent must be a Feature is_geometry = 0; } @@ -102,8 +102,8 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { if (is_geometry) { json_object *jo = j; - while (jo != NULL) { - if (jo->parent != NULL && jo->parent->type == JSON_HASH) { + while (jo != nullptr) { + if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { if (json_hash_get(jo->parent, "properties") == jo) { // Ancestor is the value corresponding to a properties key is_geometry = 0; @@ -120,14 +120,14 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } found_geometries++; - jfa->add_feature(j, false, NULL, NULL, NULL, j); + jfa->add_feature(j, false, nullptr, nullptr, nullptr, j); json_free(j); continue; } } - if (strcmp(type->value.string.string, "Feature") != 0) { - if (strcmp(type->value.string.string, "FeatureCollection") == 0) { + if (type->string() != "Feature") { + if (type->string() == "FeatureCollection") { jfa->check_crs(j); json_free(j); } @@ -141,7 +141,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { found_features++; json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + if (geometry == nullptr) { fprintf(stderr, "%s:%d: feature with no geometry: ", jfa->fname.c_str(), jp->line); json_context(j); json_free(j); @@ -149,7 +149,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "%s:%d: feature without properties hash: ", jfa->fname.c_str(), jp->line); json_context(j); json_free(j); @@ -159,8 +159,8 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { bool is_feature = true; { json_object *jo = j; - while (jo != NULL) { - if (jo->parent != NULL && jo->parent->type == JSON_HASH) { + while (jo != nullptr) { + if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { if (json_hash_get(jo->parent, "properties") == jo) { // Ancestor is the value corresponding to a properties key is_feature = false; @@ -178,7 +178,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { json_object *id = json_hash_get(j, "id"); json_object *geometries = json_hash_get(geometry, "geometries"); - if (geometries != NULL && geometries->type == JSON_ARRAY) { + if (geometries != nullptr && geometries->type == JSON_ARRAY) { jfa->add_feature(geometries, true, properties, id, tippecanoe, j); } else { jfa->add_feature(geometry, false, properties, id, tippecanoe, j); diff --git a/geojson-loop.hpp b/geojson-loop.hpp index 3d82be8e..acdb43d7 100644 --- a/geojson-loop.hpp +++ b/geojson-loop.hpp @@ -8,4 +8,4 @@ struct json_feature_action { virtual void check_crs(json_object *j) = 0; }; -void parse_json(json_feature_action *action, json_pull *jp); +void parse_json(json_feature_action *action, json_pull_ptr &jp); diff --git a/geojson.cpp b/geojson.cpp index 3798d14b..8997c5a2 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -42,7 +42,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string const &layername) { json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + if (geometry_type == nullptr) { static int warned = 0; if (!warned) { fprintf(stderr, "%s:%d: null geometry (additional not reported): ", sst->fname, sst->line); @@ -60,7 +60,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom } json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: feature without coordinates array: ", sst->fname, sst->line); json_context(feature); return 0; @@ -68,12 +68,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->value.string.string); + fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->string().c_str()); json_context(feature); return 0; } @@ -82,48 +82,48 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int tippecanoe_maxzoom = -1; std::string tippecanoe_layername = layername; - if (tippecanoe != NULL) { + if (tippecanoe != nullptr) { json_object *min = json_hash_get(tippecanoe, "minzoom"); - if (min != NULL && (min->type == JSON_NUMBER)) { - tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); + if (min != nullptr && (min->type == JSON_NUMBER)) { + tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } json_object *max = json_hash_get(tippecanoe, "maxzoom"); - if (max != NULL && (max->type == JSON_NUMBER)) { - tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); + if (max != nullptr && (max->type == JSON_NUMBER)) { + tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } json_object *ln = json_hash_get(tippecanoe, "layer"); - if (ln != NULL && (ln->type == JSON_STRING)) { - tippecanoe_layername = std::string(ln->value.string.string); + if (ln != nullptr && (ln->type == JSON_STRING)) { + tippecanoe_layername = ln->string(); } } bool has_id = false; unsigned long long id_value = 0; - if (id != NULL) { + if (id != nullptr) { if (id->type == JSON_NUMBER) { - if (id->value.number.number >= 0) { + if (id->number() >= 0) { char *err = NULL; - std::string id_number = milo::dtoa_milo(id->value.number.number); + std::string id_number = milo::dtoa_milo(id->number()); id_value = strtoull(id_number.c_str(), &err, 10); - if (id->value.number.large_unsigned != 0) { - id_value = id->value.number.large_unsigned; + if (id->large_unsigned() != 0) { + id_value = id->large_unsigned(); } if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned_frac = true; } - } else if (id->value.number.large_unsigned == 0 && std::to_string(id_value) != milo::dtoa_milo(id->value.number.number)) { + } else if (id->large_unsigned() == 0 && std::to_string(id_value) != milo::dtoa_milo(id->number())) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned = true; } } else { @@ -133,7 +133,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom static bool warned_neg = false; if (!warned_neg) { - fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned_neg = true; } } @@ -142,20 +142,20 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom if (additional[A_CONVERT_NUMERIC_IDS] && id->type == JSON_STRING) { char *err = NULL; - id_value = strtoull(id->value.string.string, &err, 10); + id_value = strtoull(id->string().c_str(), &err, 10); if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->value.string.string); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->string().c_str()); warned_frac = true; } - } else if (std::to_string(id_value) != id->value.string.string) { + } else if (std::to_string(id_value) != id->string()) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->value.string.string); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->string().c_str()); warned = true; } } else { @@ -168,33 +168,29 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom static bool warned_nan = false; if (!warned_nan) { - char *s = json_stringify(id); - fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", s); - free(s); // stringify + fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", json_stringify(id).c_str()); warned_nan = true; } } } } - size_t nprop = 0; - if (properties != NULL && properties->type == JSON_HASH) { - nprop = properties->value.object.length; - } - std::vector> full_keys; std::vector values; - - full_keys.reserve(nprop); - values.reserve(nprop); key_pool key_pool; - for (size_t i = 0; i < nprop; i++) { - if (properties->value.object.keys[i]->type == JSON_STRING) { - serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature); + if (properties != nullptr && properties->type == JSON_HASH) { + const auto &entries = properties->entries(); + full_keys.reserve(entries.size()); + values.reserve(entries.size()); - full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string)); - values.push_back(std::move(sv)); + for (const auto &e : entries) { + if (e.key->type == JSON_STRING) { + serial_val sv = stringify_value(e.value.get(), sst->fname, sst->line, feature); + + full_keys.emplace_back(key_pool.pool(e.key->string().c_str())); + values.push_back(std::move(sv)); + } } } @@ -220,14 +216,14 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom void check_crs(json_object *j, const char *reading) { json_object *crs = json_hash_get(j, "crs"); - if (crs != NULL) { + if (crs != nullptr) { json_object *properties = json_hash_get(crs, "properties"); - if (properties != NULL) { + if (properties != nullptr) { json_object *name = json_hash_get(properties, "name"); - if (name != NULL && name->type == JSON_STRING) { - if (strcmp(name->value.string.string, projection->alias) != 0) { + if (name != nullptr && name->type == JSON_STRING) { + if (name->string() != projection->alias) { if (!quiet) { - fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->value.string.string, projection->alias); + fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->string().c_str(), projection->alias); fprintf(stderr, "%s: If \"%s\" is not the expected projection, use -s to specify the right one.\n", reading, projection->alias); } } @@ -245,8 +241,8 @@ struct json_serialize_action : json_feature_action { sst->line = geometry->parser->line; if (geometrycollection) { int ret = 1; - for (size_t g = 0; g < geometry->value.array.length; g++) { - ret &= serialize_geojson_feature(sst, geometry->value.array.array[g], properties, id, layer, tippecanoe, feature, layername); + for (size_t g = 0; g < geometry->array().size(); g++) { + ret &= serialize_geojson_feature(sst, geometry->array()[g].get(), properties, id, layer, tippecanoe, feature, layername); } return ret; } else { @@ -259,7 +255,7 @@ struct json_serialize_action : json_feature_action { } }; -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) { +void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername) { json_serialize_action jsa; jsa.fname = sst->fname; jsa.sst = sst; @@ -296,7 +292,7 @@ ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) { return n; } -struct json_pull *json_begin_map(char *map, long long len) { +json_pull_ptr json_begin_map(char *map, long long len) { struct jsonmap *jm = new jsonmap; if (jm == NULL) { perror("Out of memory"); @@ -310,7 +306,11 @@ struct json_pull *json_begin_map(char *map, long long len) { return json_begin(json_map_read, jm); } -void json_end_map(struct json_pull *jp) { +void json_end_map(json_pull_ptr &jp) { + if (jp == nullptr) { + return; + } delete (struct jsonmap *) jp->source; + jp->source = nullptr; json_end(jp); } diff --git a/geojson.hpp b/geojson.hpp index 664ea2e8..8c63318c 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -10,21 +10,21 @@ #include "serial.hpp" struct parse_json_args { - json_pull *jp; + json_pull_ptr jp; int layer; std::string *layername; struct serialization_state *sst; - parse_json_args(json_pull *jp1, int layer1, std::string *layername1, struct serialization_state *sst1) + parse_json_args(json_pull_ptr jp1, int layer1, std::string *layername1, struct serialization_state *sst1) : jp(jp1), layer(layer1), layername(layername1), sst(sst1) { } }; -struct json_pull *json_begin_map(char *map, long long len); -void json_end_map(struct json_pull *jp); +json_pull_ptr json_begin_map(char *map, long long len); +void json_end_map(json_pull_ptr &jp); -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername); +void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername); void *run_parse_json(void *v); #endif diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 04ca0f29..8b23ba75 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -5,50 +5,19 @@ #include #include #include +#include +#include +#include #include "jsonpull.h" #include "../milo/milo.h" #define BUFFER 10000 -struct string { - char *buf; - size_t n; - size_t nalloc; -}; - -static void string_init(struct string *s); -static void string_free(struct string *s); - -json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source) { - json_pull *j = malloc(sizeof(json_pull)); - if (j == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - j->error = NULL; - j->line = 1; - j->container = NULL; - j->root = NULL; - +json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source) { + auto j = std::make_shared(); j->read = read; j->source = source; - j->buffer_head = 0; - j->buffer_tail = 0; - - j->buffer = malloc(BUFFER); - if (j->buffer == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - j->number_buffer = malloc(sizeof(struct string)); - if (j->number_buffer == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - string_init(j->number_buffer); - + j->buffer.resize(BUFFER); return j; } @@ -57,7 +26,7 @@ static inline int peek(json_pull *j) { return (unsigned char) j->buffer[j->buffer_head]; } else { j->buffer_head = 0; - j->buffer_tail = j->read(j, j->buffer, BUFFER); + j->buffer_tail = j->read(j, j->buffer.data(), BUFFER); if (j->buffer_head >= j->buffer_tail) { return EOF; } @@ -70,7 +39,7 @@ static inline int next(json_pull *j) { return (unsigned char) j->buffer[j->buffer_head++]; } else { j->buffer_head = 0; - j->buffer_tail = j->read(j, j->buffer, BUFFER); + j->buffer_tail = j->read(j, j->buffer.data(), BUFFER); if (j->buffer_head >= j->buffer_tail) { return EOF; } @@ -79,15 +48,15 @@ static inline int next(json_pull *j) { } static ssize_t read_file(json_pull *j, char *buffer, size_t n) { - return fread(buffer, 1, n, j->source); + return fread(buffer, 1, n, (FILE *) j->source); } -json_pull *json_begin_file(FILE *f) { +json_pull_ptr json_begin_file(FILE *f) { return json_begin(read_file, f); } static ssize_t read_string(json_pull *j, char *buffer, size_t n) { - const char *cp = j->source; + const char *cp = (const char *) j->source; size_t out = 0; while (out < n && cp[out] != '\0') { @@ -99,17 +68,12 @@ static ssize_t read_string(json_pull *j, char *buffer, size_t n) { return out; } -json_pull *json_begin_string(const char *s) { +json_pull_ptr json_begin_string(const char *s) { return json_begin(read_string, (void *) s); } -void json_end(json_pull *p) { - string_free(p->number_buffer); - free(p->number_buffer); - - json_free(p->root); - free(p->buffer); - free(p); +void json_end(json_pull_ptr &p) { + p.reset(); } static inline int read_wrap(json_pull *j) { @@ -122,196 +86,119 @@ static inline int read_wrap(json_pull *j) { return c; } -#define SIZE_FOR(i, size) ((size_t) ((((i) + 7) & ~7) * size)) - -static json_object *fabricate_object(json_pull *jp, json_object *parent, json_type type) { - json_object *o = malloc(sizeof(struct json_object)); - if (o == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); +// Construct an instance of the right subclass for the given type. +// JSON_TRUE / JSON_FALSE / JSON_NULL and the parse-token types are bare +// json_objects; the value-bearing types each get their own subclass. +// +// Returns a json_object_ptr (unique_ptr with a type-dispatching deleter, +// see jsonpull.h), so the caller doesn't have to remember which subclass +// was constructed when it eventually deletes. +static json_object_ptr make_object(json_type type, json_object *parent, json_pull *jp) { + switch (type) { + case JSON_NUMBER: + return json_object_ptr(new json_number(parent, jp)); + case JSON_STRING: + return json_object_ptr(new json_string(parent, jp)); + case JSON_ARRAY: + return json_object_ptr(new json_array(parent, jp)); + case JSON_HASH: + return json_object_ptr(new json_hash(parent, jp)); + default: + return json_object_ptr(new json_object(type, parent, jp)); } - o->type = type; - o->parent = parent; - o->parser = jp; +} - if (type == JSON_ARRAY) { - o->value.array.array = NULL; - o->value.array.length = 0; - } else if (type == JSON_HASH) { - o->value.object.keys = NULL; - o->value.object.values = NULL; - o->value.object.length = 0; - } +static json_object_ptr fabricate_object(json_pull *jp, json_object *parent, json_type type) { + return make_object(type, parent, jp); +} - return o; +static inline json_pull::parse_frame *current_frame(json_pull *j) { + return j->container_stack.empty() ? nullptr : &j->container_stack.back(); } +// Construct a new node of `type` and install it as a child of the +// current container (or as the parser's root, if the container stack +// is empty). Returns a borrowed pointer into the parser-owned tree; +// the unique_ptr that owns the node lives in whichever vector slot +// we just pushed it into. Returns nullptr on error after setting +// j->error. static json_object *add_object(json_pull *j, json_type type) { - json_object *c = j->container; - json_object *o = fabricate_object(j, c, type); + json_pull::parse_frame *f = current_frame(j); + json_object *c = f ? f->container : nullptr; + json_object_ptr o = make_object(type, c, j); + json_object *raw = o.get(); - if (c != NULL) { + if (f != nullptr) { if (c->type == JSON_ARRAY) { - if (c->expect == JSON_ITEM) { - if (SIZE_FOR(c->value.array.length + 1, sizeof(json_object *)) != SIZE_FOR(c->value.array.length, sizeof(json_object *))) { - if (SIZE_FOR(c->value.array.length + 1, sizeof(json_object *)) < SIZE_FOR(c->value.array.length, sizeof(json_object *))) { - fprintf(stderr, "Array size overflow\n"); - exit(EXIT_FAILURE); - } - c->value.array.array = realloc(c->value.array.array, SIZE_FOR(c->value.array.length + 1, sizeof(json_object *))); - if (c->value.array.array == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - c->value.array.array[c->value.array.length++] = o; - c->expect = JSON_COMMA; + if (f->expect == JSON_ITEM) { + c->array().push_back(std::move(o)); + f->expect = JSON_COMMA; } else { j->error = "Expected a comma, not a list item"; - free(o); - return NULL; + return nullptr; } } else if (c->type == JSON_HASH) { - if (c->expect == JSON_VALUE) { - c->value.object.values[c->value.object.length - 1] = o; - c->expect = JSON_COMMA; - } else if (c->expect == JSON_KEY) { + if (f->expect == JSON_VALUE) { + c->entries().back().value = std::move(o); + f->expect = JSON_COMMA; + } else if (f->expect == JSON_KEY) { if (type != JSON_STRING) { j->error = "Hash key is not a string"; - free(o); - return NULL; - } - - if (SIZE_FOR(c->value.object.length + 1, sizeof(json_object *)) != SIZE_FOR(c->value.object.length, sizeof(json_object *))) { - if (SIZE_FOR(c->value.object.length + 1, sizeof(json_object *)) < SIZE_FOR(c->value.object.length, sizeof(json_object *))) { - fprintf(stderr, "Hash size overflow\n"); - exit(EXIT_FAILURE); - } - c->value.object.keys = realloc(c->value.object.keys, SIZE_FOR(c->value.object.length + 1, sizeof(json_object *))); - c->value.object.values = realloc(c->value.object.values, SIZE_FOR(c->value.object.length + 1, sizeof(json_object *))); - if (c->value.object.keys == NULL || c->value.object.values == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } + return nullptr; } - c->value.object.keys[c->value.object.length] = o; - c->value.object.values[c->value.object.length] = NULL; - c->value.object.length++; - c->expect = JSON_COLON; + c->entries().push_back({std::move(o), nullptr}); + f->expect = JSON_COLON; } else { j->error = "Expected a comma or colon"; - free(o); - return NULL; + return nullptr; } } } else { - if (j->root != NULL) { - json_free(j->root); - } - - j->root = o; + // Replacing the parser's root destroys the previous top-level + // value (if no one called json_disconnect / json_read_tree to + // take ownership of it). + j->root = std::move(o); } - return o; + return raw; } json_object *json_hash_get(json_object *o, const char *s) { - if (o == NULL || o->type != JSON_HASH) { - return NULL; - } - - size_t i; - for (i = 0; i < o->value.object.length; i++) { - if (o->value.object.keys[i] != NULL && o->value.object.keys[i]->type == JSON_STRING) { - if (strcmp(o->value.object.keys[i]->value.string.string, s) == 0) { - return o->value.object.values[i]; - } - } - } - - return NULL; -} - -static void string_init(struct string *s) { - s->nalloc = 500; - s->buf = malloc(s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - s->n = 0; - s->buf[0] = '\0'; -} - -static void string_append(struct string *s, char c) { - if (s->n + 2 >= s->nalloc) { - size_t prev = s->nalloc; - s->nalloc += 500; - if (s->nalloc <= prev) { - fprintf(stderr, "String size overflowed\n"); - exit(EXIT_FAILURE); - } - s->buf = realloc(s->buf, s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } + if (o == nullptr || o->type != JSON_HASH) { + return nullptr; } - s->buf[s->n++] = c; - s->buf[s->n] = '\0'; -} - -static void string_append_string(struct string *s, char *add) { - size_t len = strlen(add); - - if (s->n + len + 1 >= s->nalloc) { - size_t prev = s->nalloc; - s->nalloc += 500 + len; - if (s->nalloc <= prev) { - fprintf(stderr, "String size overflowed\n"); - exit(EXIT_FAILURE); - } - s->buf = realloc(s->buf, s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); + for (const auto &e : o->entries()) { + if (e.key != nullptr && e.key->type == JSON_STRING && e.key->string() == s) { + return e.value.get(); } } - for (; *add != '\0'; add++) { - s->buf[s->n++] = *add; - } - - s->buf[s->n] = '\0'; + return nullptr; } -static void string_free(struct string *s) { - free(s->buf); +json_object *json_hash_get(const json_object_ptr &o, const char *s) { + return json_hash_get(o.get(), s); } -json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state) { +json_object *json_read_separators(json_pull_ptr &jp, json_separator_callback cb, void *state) { int c; + json_pull *j = jp.get(); // In case there is an error at the top level - if (j->container == NULL) { - if (j->root != NULL) { - json_free(j->root); - } - - j->root = NULL; + if (j->container_stack.empty()) { + j->root.reset(); } again: c = read_wrap(j); if (c == EOF) { - if (j->container != NULL) { + if (!j->container_stack.empty()) { j->error = "Reached EOF without all containers being closed"; } - return NULL; + return nullptr; } switch (c) { @@ -329,7 +216,7 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void } } j->error = "Corrupt byte-order mark found"; - return NULL; + return nullptr; } /////////////////////////// Whitespace @@ -345,13 +232,14 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void case '[': { json_object *o = add_object(j, JSON_ARRAY); - if (o == NULL) { - return NULL; + if (o == nullptr) { + return nullptr; } - j->container = o; - j->container->expect = JSON_ITEM; + // add_object already installed `o` in the parent (or the + // parser's root) as a unique_ptr; the frame just borrows. + j->container_stack.push_back({o, JSON_ITEM}); - if (cb != NULL) { + if (cb != nullptr) { cb(JSON_ARRAY, j, state); } @@ -359,39 +247,41 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void } case ']': { - if (j->container == NULL) { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { j->error = "Found ] at top level"; - return NULL; + return nullptr; } - if (j->container->type != JSON_ARRAY) { + json_object *cc = f->container; + if (cc->type != JSON_ARRAY) { j->error = "Found ] not in an array"; - return NULL; + return nullptr; } - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_ITEM && j->container->value.array.length == 0)) { + if (f->expect != JSON_COMMA) { + if (!(f->expect == JSON_ITEM && cc->array().size() == 0)) { j->error = "Found ] without final element"; - return NULL; + return nullptr; } } - json_object *ret = j->container; - j->container = ret->parent; - return ret; + // Pop the frame; ownership of `cc` stays with whatever + // surrounding container (or jp->root) installed it. + j->container_stack.pop_back(); + return cc; } /////////////////////////// Hashes case '{': { json_object *o = add_object(j, JSON_HASH); - if (o == NULL) { - return NULL; + if (o == nullptr) { + return nullptr; } - j->container = o; - j->container->expect = JSON_KEY; + j->container_stack.push_back({o, JSON_KEY}); - if (cb != NULL) { + if (cb != nullptr) { cb(JSON_HASH, j, state); } @@ -399,26 +289,27 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void } case '}': { - if (j->container == NULL) { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { j->error = "Found } at top level"; - return NULL; + return nullptr; } - if (j->container->type != JSON_HASH) { + json_object *cc = f->container; + if (cc->type != JSON_HASH) { j->error = "Found } not in a hash"; - return NULL; + return nullptr; } - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_KEY && j->container->value.object.length == 0)) { + if (f->expect != JSON_COMMA) { + if (!(f->expect == JSON_KEY && cc->entries().size() == 0)) { j->error = "Found } without final element"; - return NULL; + return nullptr; } } - json_object *ret = j->container; - j->container = ret->parent; - return ret; + j->container_stack.pop_back(); + return cc; } /////////////////////////// Null @@ -426,7 +317,7 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void case 'n': { if (read_wrap(j) != 'u' || read_wrap(j) != 'l' || read_wrap(j) != 'l') { j->error = "Found misspelling of null"; - return NULL; + return nullptr; } return add_object(j, JSON_NULL); @@ -437,11 +328,11 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void case 'N': { if (read_wrap(j) != 'a' || read_wrap(j) != 'N') { j->error = "Found misspelling of NaN"; - return NULL; + return nullptr; } j->error = "JSON does not allow NaN"; - return NULL; + return nullptr; } /////////////////////////// Infinity @@ -451,11 +342,11 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void read_wrap(j) != 'n' || read_wrap(j) != 'i' || read_wrap(j) != 't' || read_wrap(j) != 'y') { j->error = "Found misspelling of Infinity"; - return NULL; + return nullptr; } j->error = "JSON does not allow Infinity"; - return NULL; + return nullptr; } /////////////////////////// True @@ -463,7 +354,7 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void case 't': { if (read_wrap(j) != 'r' || read_wrap(j) != 'u' || read_wrap(j) != 'e') { j->error = "Found misspelling of true"; - return NULL; + return nullptr; } return add_object(j, JSON_TRUE); @@ -474,7 +365,7 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void case 'f': { if (read_wrap(j) != 'a' || read_wrap(j) != 'l' || read_wrap(j) != 's' || read_wrap(j) != 'e') { j->error = "Found misspelling of false"; - return NULL; + return nullptr; } return add_object(j, JSON_FALSE); @@ -483,20 +374,21 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void /////////////////////////// Comma case ',': { - if (j->container != NULL) { - if (j->container->expect != JSON_COMMA) { + json_pull::parse_frame *f = current_frame(j); + if (f != nullptr) { + if (f->expect != JSON_COMMA) { j->error = "Found unexpected comma"; - return NULL; + return nullptr; } - if (j->container->type == JSON_HASH) { - j->container->expect = JSON_KEY; + if (f->container->type == JSON_HASH) { + f->expect = JSON_KEY; } else { - j->container->expect = JSON_ITEM; + f->expect = JSON_ITEM; } } - if (cb != NULL) { + if (cb != nullptr) { cb(JSON_COMMA, j, state); } @@ -506,19 +398,20 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void /////////////////////////// Colon case ':': { - if (j->container == NULL) { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { j->error = "Found colon at top level"; - return NULL; + return nullptr; } - if (j->container->expect != JSON_COLON) { + if (f->expect != JSON_COLON) { j->error = "Found unexpected colon"; - return NULL; + return nullptr; } - j->container->expect = JSON_VALUE; + f->expect = JSON_VALUE; - if (cb != NULL) { + if (cb != nullptr) { cb(JSON_COLON, j, state); } @@ -538,87 +431,84 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void case '7': case '8': case '9': { - j->number_buffer->n = 0; + j->number_buffer.clear(); int decimal = 0; if (c == '-') { - string_append(j->number_buffer, c); + j->number_buffer.push_back(c); c = read_wrap(j); } if (c == '0') { - string_append(j->number_buffer, c); + j->number_buffer.push_back(c); } else if (c >= '1' && c <= '9') { - string_append(j->number_buffer, c); + j->number_buffer.push_back(c); c = peek(j); while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); + j->number_buffer.push_back(read_wrap(j)); c = peek(j); } } if (peek(j) == '.') { - string_append(j->number_buffer, read_wrap(j)); + j->number_buffer.push_back(read_wrap(j)); decimal = 1; c = peek(j); if (c < '0' || c > '9') { j->error = "Decimal point without digits"; - string_free(j->number_buffer); - return NULL; + return nullptr; } while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); + j->number_buffer.push_back(read_wrap(j)); c = peek(j); } } c = peek(j); if (c == 'e' || c == 'E') { - string_append(j->number_buffer, read_wrap(j)); + j->number_buffer.push_back(read_wrap(j)); decimal = 1; c = peek(j); if (c == '+' || c == '-') { - string_append(j->number_buffer, read_wrap(j)); + j->number_buffer.push_back(read_wrap(j)); } c = peek(j); if (c < '0' || c > '9') { j->error = "Exponent without digits"; - string_free(j->number_buffer); - return NULL; + return nullptr; } while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); + j->number_buffer.push_back(read_wrap(j)); c = peek(j); } } json_object *n = add_object(j, JSON_NUMBER); - if (n != NULL) { - n->value.number.number = atof(j->number_buffer->buf); - n->value.number.large_signed = 0; - n->value.number.large_unsigned = 0; + if (n != nullptr) { + double d = atof(j->number_buffer.c_str()); + n->set_number(d); #define MAX_SAFE_INTEGER 9007199254740991.0 #define MIN_SAFE_INTEGER -9007199254740991.0 - if (!decimal && n->value.number.number > MAX_SAFE_INTEGER) { + if (!decimal && d > MAX_SAFE_INTEGER) { errno = 0; - char *err = NULL; - unsigned long long ull = strtoull(j->number_buffer->buf, &err, 10); - if (errno == 0 && (err == NULL || *err == '\0')) { - n->value.number.large_unsigned = ull; + char *err = nullptr; + unsigned long long ull = strtoull(j->number_buffer.c_str(), &err, 10); + if (errno == 0 && (err == nullptr || *err == '\0')) { + n->set_large_unsigned(ull); } } - if (!decimal && n->value.number.number < MIN_SAFE_INTEGER) { + if (!decimal && d < MIN_SAFE_INTEGER) { errno = 0; - char *err = NULL; - long long ll = strtoll(j->number_buffer->buf, &err, 10); - if (errno == 0 && (err == NULL || *err == '\0')) { - n->value.number.large_signed = ll; + char *err = nullptr; + long long ll = strtoll(j->number_buffer.c_str(), &err, 10); + if (errno == 0 && (err == nullptr || *err == '\0')) { + n->set_large_signed(ll); } } } @@ -628,16 +518,19 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void /////////////////////////// Strings case '"': { - struct string val; - string_init(&val); + // Reuse the parser-wide string buffer so we don't construct a + // fresh std::string (with its inevitable SSO->heap promotion + // and capacity doublings) for every JSON_STRING token. + std::string &val = j->string_buffer; + val.clear(); int surrogate = -1; while ((c = read_wrap(j)) != EOF) { if (c == '"') { if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); surrogate = -1; } @@ -652,25 +545,24 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void hex[i] = read_wrap(j); if (hex[i] < '0' || (hex[i] > '9' && hex[i] < 'A') || (hex[i] > 'F' && hex[i] < 'a') || hex[i] > 'f') { j->error = "Invalid \\u hex character"; - string_free(&val); - return NULL; + return nullptr; } } - unsigned long ch = strtoul(hex, NULL, 16); + unsigned long ch = strtoul(hex, nullptr, 16); if (ch >= 0xd800 && ch <= 0xdbff) { if (surrogate < 0) { surrogate = ch; } else { // Impossible surrogate, so output the first half, // keep what might be a legitimate new first half. - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); surrogate = ch; } continue; - } else if (ch >= 0xdc00 && c <= 0xdfff) { + } else if (ch >= 0xdc00 && ch <= 0xdfff) { if (surrogate >= 0) { long c1 = surrogate - 0xd800; long c2 = ch - 0xdc00; @@ -680,317 +572,325 @@ json_object *json_read_separators(json_pull *j, json_separator_callback cb, void } if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); surrogate = -1; } if (ch <= 0x7F) { - string_append(&val, ch); + val.push_back(ch); } else if (ch <= 0x7FF) { - string_append(&val, 0xC0 | (ch >> 6)); - string_append(&val, 0x80 | (ch & 0x3F)); + val.push_back(0xC0 | (ch >> 6)); + val.push_back(0x80 | (ch & 0x3F)); } else if (ch < 0xFFFF) { - string_append(&val, 0xE0 | (ch >> 12)); - string_append(&val, 0x80 | ((ch >> 6) & 0x3F)); - string_append(&val, 0x80 | (ch & 0x3F)); + val.push_back(0xE0 | (ch >> 12)); + val.push_back(0x80 | ((ch >> 6) & 0x3F)); + val.push_back(0x80 | (ch & 0x3F)); } else { - string_append(&val, 0xF0 | (ch >> 18)); - string_append(&val, 0x80 | ((ch >> 12) & 0x3F)); - string_append(&val, 0x80 | ((ch >> 6) & 0x3F)); - string_append(&val, 0x80 | (ch & 0x3F)); + val.push_back(0xF0 | (ch >> 18)); + val.push_back(0x80 | ((ch >> 12) & 0x3F)); + val.push_back(0x80 | ((ch >> 6) & 0x3F)); + val.push_back(0x80 | (ch & 0x3F)); } } else { if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); surrogate = -1; } if (c == '"') { - string_append(&val, '"'); + val.push_back('"'); } else if (c == '\\') { - string_append(&val, '\\'); + val.push_back('\\'); } else if (c == '/') { - string_append(&val, '/'); + val.push_back('/'); } else if (c == 'b') { - string_append(&val, '\b'); + val.push_back('\b'); } else if (c == 'f') { - string_append(&val, '\f'); + val.push_back('\f'); } else if (c == 'n') { - string_append(&val, '\n'); + val.push_back('\n'); } else if (c == 'r') { - string_append(&val, '\r'); + val.push_back('\r'); } else if (c == 't') { - string_append(&val, '\t'); + val.push_back('\t'); } else { j->error = "Found backslash followed by unknown character"; - string_free(&val); - return NULL; + return nullptr; } } } else if (c < ' ') { j->error = "Found control character in string"; - string_free(&val); - return NULL; + return nullptr; } else { if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); surrogate = -1; } - string_append(&val, c); + val.push_back(c); } } if (c == EOF) { j->error = "String without closing quote mark"; - string_free(&val); - return NULL; + return nullptr; } json_object *s = add_object(j, JSON_STRING); - if (s != NULL) { - s->value.string.string = val.buf; - s->value.string.refcon = NULL; - } else { - string_free(&val); + if (s != nullptr) { + // Copy (don't move) so j->string_buffer retains its + // grown capacity for the next token. The copy is a + // single right-sized allocation plus one memcpy, which + // is cheaper than the multiple capacity doublings the + // per-token std::string would otherwise incur. + s->string() = val; } return s; } } j->error = "Found unexpected character"; - return NULL; + return nullptr; } -json_object *json_read(json_pull *j) { - return json_read_separators(j, NULL, NULL); +json_object *json_read(json_pull_ptr &j) { + return json_read_separators(j, nullptr, nullptr); } -json_object *json_read_tree(json_pull *p) { +// Forward declaration so json_read_tree can clear back-pointers on +// the tree it hands out -- this lets callers (like the filter loaders) +// keep the returned tree past the parser's lifetime without having to +// follow up with a separate json_disconnect call. +static void clear_back_pointers(json_object *o); + +json_object_ptr json_read_tree(json_pull_ptr &p) { json_object *j; - while ((j = json_read(p)) != NULL) { - if (j->parent == NULL) { - return j; + while ((j = json_read(p)) != nullptr) { + if (j->parent == nullptr) { + // The parser owns the top-level value via p->root; + // transfer ownership out to the caller and detach + // the subtree from the parser so the caller can + // outlive the json_pull. + json_object_ptr tree = std::move(p->root); + clear_back_pointers(tree.get()); + return tree; } } - return NULL; + return nullptr; } -void json_free(json_object *o) { - size_t i; - - if (o == NULL) { - return; - } - - // Free any data linked from here - - if (o->type == JSON_ARRAY) { - json_object **a = o->value.array.array; - size_t n = o->value.array.length; - - o->value.array.array = NULL; - o->value.array.length = 0; - - for (i = 0; i < n; i++) { - json_free(a[i]); +// Take ownership of `o` away from its parent (or from the parser's +// root) by moving the owning json_object_ptr out of whatever vector +// slot or hash entry holds it. Returns the unique_ptr to the caller, +// who is now solely responsible for it. Returns an empty +// json_object_ptr if `o` is not currently owned by a parent or by +// the parser (e.g. already detached, or only borrowed from somewhere +// untracked). +// +// For a hash, removing a single key or value individually would +// disturb the surrounding key/value pairing, so we replace the +// extracted half with a fresh JSON_NULL placeholder and only erase +// the entry once both halves have been detached. This matches the +// historical json_disconnect semantics for partially-disconnected +// pairs. +static json_object_ptr take_from_owner(json_object *o) { + if (o == nullptr) { + return nullptr; + } + + json_object *parent = o->parent; + if (parent == nullptr) { + // Top-level value: the parser owns it via root, unless the + // caller already moved it out. + json_pull *parser = o->parser; + if (parser != nullptr && parser->root.get() == o) { + return std::move(parser->root); + } + return nullptr; + } + + if (parent->type == JSON_ARRAY) { + auto &arr = parent->array(); + for (size_t i = 0; i < arr.size(); i++) { + if (arr[i].get() == o) { + json_object_ptr taken = std::move(arr[i]); + arr.erase(arr.begin() + i); + return taken; + } } - - free(a); - } else if (o->type == JSON_HASH) { - json_object **k = o->value.object.keys; - json_object **v = o->value.object.values; - size_t n = o->value.object.length; - - o->value.object.keys = NULL; - o->value.object.values = NULL; - o->value.object.length = 0; - - for (i = 0; i < n; i++) { - json_free(k[i]); - json_free(v[i]); + } else if (parent->type == JSON_HASH) { + auto &entries = parent->entries(); + for (size_t i = 0; i < entries.size(); i++) { + auto &e = entries[i]; + if (e.key.get() == o) { + json_object_ptr taken = std::move(e.key); + e.key = fabricate_object(parent->parser, parent, JSON_NULL); + if (e.value != nullptr && e.value->type == JSON_NULL && e.key->type == JSON_NULL) { + entries.erase(entries.begin() + i); + } + return taken; + } + if (e.value.get() == o) { + json_object_ptr taken = std::move(e.value); + e.value = fabricate_object(parent->parser, parent, JSON_NULL); + if (e.key != nullptr && e.key->type == JSON_NULL && e.value->type == JSON_NULL) { + entries.erase(entries.begin() + i); + } + return taken; + } } - - free(k); - free(v); - } else if (o->type == JSON_STRING) { - free(o->value.string.string); - } else if (o->type == JSON_NUMBER) { - ; } - json_disconnect(o); + return nullptr; +} - free(o); +// json_free splices `o` out of its parent (if any), or out of the +// parser's root (if `o` is the most recently completed top-level +// value), and destroys the subtree. After this call, `o` is a +// dangling pointer and must not be used. +// +// geojson-loop.cpp relies on this to release each feature after it +// has been serialized, so that already-serialized features don't sit +// in memory while subsequent features are parsed. +// +// Unlike json_disconnect, this does NOT walk the subtree clearing +// parent/parser back-pointers, because the subtree is about to be +// destroyed and those pointers will never be observed again -- the +// unique_ptr returned by take_from_owner goes out of scope at the end +// of this function and runs the type-dispatching deleter. +void json_free(json_object *o) { + (void) take_from_owner(o); } -static void json_disconnect_parser(json_object *o) { +// Walk the subtree clearing parent/parser back-pointers so the detached +// subtree can outlive the original parser. +static void clear_back_pointers(json_object *o) { + if (o == nullptr) { + return; + } + if (o->type == JSON_HASH) { - size_t i; - for (i = 0; i < o->value.object.length; i++) { - json_disconnect_parser(o->value.object.keys[i]); - json_disconnect_parser(o->value.object.values[i]); + for (const auto &e : o->entries()) { + clear_back_pointers(e.key.get()); + clear_back_pointers(e.value.get()); } } else if (o->type == JSON_ARRAY) { - size_t i; - for (i = 0; i < o->value.array.length; i++) { - json_disconnect_parser(o->value.array.array[i]); + const auto &arr = o->array(); + for (size_t i = 0; i < arr.size(); i++) { + clear_back_pointers(arr[i].get()); } } - o->parser = NULL; + o->parent = nullptr; + o->parser = nullptr; } -void json_disconnect(json_object *o) { - // Expunge references to this as an array element - // or a hash key or value. - - if (o->parent != NULL) { - if (o->parent->type == JSON_ARRAY) { - size_t i; - - for (i = 0; i < o->parent->value.array.length; i++) { - if (o->parent->value.array.array[i] == o) { - break; - } - } - - if (i < o->parent->value.array.length) { - memmove(o->parent->value.array.array + i, o->parent->value.array.array + i + 1, o->parent->value.array.length - i - 1); - o->parent->value.array.length--; - } - } - - if (o->parent->type == JSON_HASH) { - size_t i; - - for (i = 0; i < o->parent->value.object.length; i++) { - if (o->parent->value.object.keys[i] == o) { - o->parent->value.object.keys[i] = fabricate_object(o->parser, o->parent, JSON_NULL); - break; - } - if (o->parent->value.object.values[i] == o) { - o->parent->value.object.values[i] = fabricate_object(o->parser, o->parent, JSON_NULL); - break; - } - } - - if (i < o->parent->value.object.length) { - if (o->parent->value.object.keys[i] != NULL && o->parent->value.object.keys[i]->type == JSON_NULL) { - if (o->parent->value.object.values[i] != NULL && o->parent->value.object.values[i]->type == JSON_NULL) { - free(o->parent->value.object.keys[i]); - free(o->parent->value.object.values[i]); - - memmove(o->parent->value.object.keys + i, o->parent->value.object.keys + i + 1, o->parent->value.object.length - i - 1); - memmove(o->parent->value.object.values + i, o->parent->value.object.values + i + 1, o->parent->value.object.length - i - 1); - o->parent->value.object.length--; - } - } - } - } +json_object_ptr json_disconnect(json_object *o) { + json_object_ptr taken = take_from_owner(o); + if (taken != nullptr) { + clear_back_pointers(taken.get()); } + return taken; +} - if (o->parser != NULL && o->parser->root == o) { - o->parser->root = NULL; - } +static void string_append_c(std::string &val, char c) { + val.push_back(c); +} - json_disconnect_parser(o); - o->parent = NULL; +static void string_append(std::string &val, const char *add) { + val.append(add); } -static void json_print_one(struct string *val, json_object *o) { - if (o == NULL) { - string_append_string(val, "..."); +static void json_print_one(std::string &val, const json_object *o) { + if (o == nullptr) { + string_append(val, "..."); } else if (o->type == JSON_STRING) { - string_append(val, '\"'); + string_append_c(val, '\"'); - char *cp; - for (cp = o->value.string.string; *cp != '\0'; cp++) { + for (const char *cp = o->string().c_str(); *cp != '\0'; cp++) { if (*cp == '\\' || *cp == '"') { - string_append(val, '\\'); - string_append(val, *cp); + string_append_c(val, '\\'); + string_append_c(val, *cp); } else if (*cp >= 0 && *cp < ' ') { char *s; if (asprintf(&s, "\\u%04x", *cp) >= 0) { - string_append_string(val, s); + string_append(val, s); free(s); } } else { - string_append(val, *cp); + string_append_c(val, *cp); } } - string_append(val, '\"'); + string_append_c(val, '\"'); } else if (o->type == JSON_NUMBER) { - if (o->value.number.large_signed != 0) { + if (o->large_signed() != 0) { char s[65]; - sprintf(s, "%lld", o->value.number.large_signed); - string_append_string(val, s); - } else if (o->value.number.large_unsigned != 0) { + snprintf(s, sizeof(s), "%lld", o->large_signed()); + string_append(val, s); + } else if (o->large_unsigned() != 0) { char s[65]; - sprintf(s, "%llu", o->value.number.large_unsigned); - string_append_string(val, s); + snprintf(s, sizeof(s), "%llu", o->large_unsigned()); + string_append(val, s); } else { - char *s = dtoa_milo(o->value.number.number); - string_append_string(val, s); + char *s = dtoa_milo(o->number()); + string_append(val, s); free(s); } } else if (o->type == JSON_NULL) { - string_append_string(val, "null"); + string_append(val, "null"); } else if (o->type == JSON_TRUE) { - string_append_string(val, "true"); + string_append(val, "true"); } else if (o->type == JSON_FALSE) { - string_append_string(val, "false"); + string_append(val, "false"); } else if (o->type == JSON_HASH) { - string_append(val, '}'); + string_append_c(val, '}'); } else if (o->type == JSON_ARRAY) { - string_append(val, ']'); + string_append_c(val, ']'); } } -static void json_print(struct string *val, json_object *o) { - if (o == NULL) { +static void json_print(std::string &val, const json_object *o) { + if (o == nullptr) { // Hash value in incompletely read hash - string_append_string(val, "..."); + string_append(val, "..."); } else if (o->type == JSON_HASH) { - string_append(val, '{'); - - size_t i; - for (i = 0; i < o->value.object.length; i++) { - json_print(val, o->value.object.keys[i]); - string_append(val, ':'); - json_print(val, o->value.object.values[i]); - if (i + 1 < o->value.object.length) { - string_append(val, ','); + string_append_c(val, '{'); + + const auto &entries = o->entries(); + for (size_t i = 0; i < entries.size(); i++) { + json_print(val, entries[i].key.get()); + string_append_c(val, ':'); + json_print(val, entries[i].value.get()); + if (i + 1 < entries.size()) { + string_append_c(val, ','); } } - string_append(val, '}'); + string_append_c(val, '}'); } else if (o->type == JSON_ARRAY) { - string_append(val, '['); - size_t i; - for (i = 0; i < o->value.array.length; i++) { - json_print(val, o->value.array.array[i]); - if (i + 1 < o->value.array.length) { - string_append(val, ','); + string_append_c(val, '['); + const auto &arr = o->array(); + for (size_t i = 0; i < arr.size(); i++) { + json_print(val, arr[i].get()); + if (i + 1 < arr.size()) { + string_append_c(val, ','); } } - string_append(val, ']'); + string_append_c(val, ']'); } else { json_print_one(val, o); } } -char *json_stringify(json_object *o) { - struct string val; - string_init(&val); - json_print(&val, o); - - return val.buf; +std::string json_stringify(const json_object *o) { + std::string val; + json_print(val, o); + return val; } diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index b19e0e8e..059dde0d 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -1,9 +1,12 @@ #ifndef JSONPULL_H #define JSONPULL_H -#ifdef __cplusplus -extern "C" { -#endif +#include +#include +#include +#include +#include +#include typedef enum json_type { // These types can be returned by json_read() @@ -25,74 +28,369 @@ typedef enum json_type { JSON_VALUE, } json_type; -typedef struct json_object { - struct json_object *parent; - struct json_pull *parser; - - union { - struct { - double number; - unsigned long long large_unsigned; - long long large_signed; - } number; - - struct { - char *string; - void *refcon; // reference constant for caller's use - } string; - - struct { - struct json_object **array; - size_t length; - } array; - - struct { - struct json_object **keys; - struct json_object **values; - size_t length; - } object; - } value; +struct json_object; +struct json_pull; + +// json_object is non-virtual so that JSON_TRUE / JSON_FALSE / JSON_NULL +// nodes don't have to pay for a vptr, but the typed subclasses +// (json_number, json_string, json_array, json_hash) have non-trivial +// destructors that need to run to free their std::vector / std::string +// members. So json_object_ptr is given a custom empty deleter that +// dispatches on `type` and static_casts to the right subclass before +// `delete`. The deleter is stateless, so the unique_ptr stays one +// pointer wide. +struct json_object_deleter { + void operator()(json_object *p) const noexcept; +}; + +// Ownership of a JSON subtree is unique: every node has a single owner, +// which is either its parent (via a json_object_ptr in the parent's +// vector or hash entry) or, for the root, the parser (via jp->root) or +// the caller (after json_read_tree / json_disconnect). +// +// Callers receive borrowed `json_object *` views from json_read, +// json_hash_get, etc.; those pointers stay valid as long as the owning +// container is intact (which, for json_read results, means "until the +// next json_read, json_free, or json_disconnect call on that subtree"). +// +// json_pull_ptr stays a shared_ptr because the parser is created once +// and freed once and the cost of shared_ptr there is irrelevant. +typedef std::unique_ptr json_object_ptr; +typedef std::shared_ptr json_pull_ptr; + +// A single key/value pair inside a JSON_HASH. The pairs are stored in +// insertion order in a single std::vector on json_hash, so +// callers can range-for over `o->entries()` with structured bindings +// (`for (auto &[k, v] : o->entries()) ...`) while still preserving the +// order keys appeared in the source document. +struct json_entry { + json_object_ptr key; + json_object_ptr value; +}; + +// json_object is a small base type that just records the JSON type and +// the back-pointers to its parent and parser. The actual value payload +// lives in a type-specific subclass (json_number, json_string, json_array, +// json_hash), so that JSON_TRUE / JSON_FALSE / JSON_NULL nodes pay only +// the base-class cost and a JSON_HASH does not also drag along a string +// or a number field. Type-tagged accessor methods on the base class +// downcast and return references to the underlying subclass storage. +// +// Children are owned by their parent (via std::vector +// inside json_array / json_hash); the raw `parent` and `parser` +// back-pointers stay valid as long as the node is attached to the tree. +// json_disconnect() splices a node out of its parent and walks the +// detached subtree clearing those back-pointers so the subtree can +// outlive the original parser. +// +// json_object intentionally has no virtual functions and no virtual +// destructor; the json_object_ptr deleter (see below in this header) +// switches on `type` and static_casts to the correct subclass before +// `delete`, so each subclass's destructor still runs without costing +// a vptr per node. Dispatch on `type` is what the rest of the code +// already does. The accessor methods assert at debug time that the +// type matches before downcasting. + +struct json_object { + json_object *parent = nullptr; + json_pull *parser = nullptr; json_type type; - int expect; -} json_object; -typedef struct json_pull { - char *error; - int line; + json_object(json_type t) + : type(t) { + } + json_object(json_type t, json_object *p, json_pull *pl) + : parent(p), parser(pl), type(t) { + } - ssize_t (*read)(struct json_pull *, char *buf, size_t n); - void *source; - char *buffer; - ssize_t buffer_tail; - ssize_t buffer_head; + // Type-tagged accessors. Each one asserts that the receiver is of + // the right kind, then downcasts to the storage in the appropriate + // subclass. Inline so the assert and cast disappear at -O. + inline std::string &string(); + inline const std::string &string() const; - json_object *container; - json_object *root; + // Numbers are stored in a discriminated union (double / unsigned / + // signed) so a json_number is only 40 bytes instead of 48. The + // large_*() accessors return 0 when the number is not currently + // stored in that representation, matching the prior convention + // where "0" meant "not set, fall through to the next slot". + inline double number() const; + inline unsigned long long large_unsigned() const; + inline long long large_signed() const; + inline void set_number(double d); + inline void set_large_unsigned(unsigned long long u); + inline void set_large_signed(long long s); - struct string *number_buffer; -} json_pull; + inline std::vector &array(); + inline const std::vector &array() const; -json_pull *json_begin_file(FILE *f); -json_pull *json_begin_string(const char *s); + inline std::vector &entries(); + inline const std::vector &entries() const; +}; -json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); -void json_end(json_pull *p); +struct json_number : json_object { + enum repr_t { REPR_DOUBLE, + REPR_LARGE_UNSIGNED, + REPR_LARGE_SIGNED }; -typedef void (*json_separator_callback)(json_type type, json_pull *j, void *state); + repr_t repr = REPR_DOUBLE; + union value_t { + double d; + unsigned long long u; + long long s; + value_t() + : d(0) { + } + } value; -json_object *json_read_tree(json_pull *j); -json_object *json_read(json_pull *j); -json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state); -void json_free(json_object *j); -void json_disconnect(json_object *j); + json_number() + : json_object(JSON_NUMBER) { + } + json_number(json_object *p, json_pull *pl) + : json_object(JSON_NUMBER, p, pl) { + } +}; -json_object *json_hash_get(json_object *o, const char *s); +struct json_string : json_object { + std::string string_value; + + json_string() + : json_object(JSON_STRING) { + } + json_string(json_object *p, json_pull *pl) + : json_object(JSON_STRING, p, pl) { + } +}; + +struct json_array : json_object { + std::vector array_value; -char *json_stringify(json_object *o); + // Coordinate-heavy GeoJSON dominates the parse workload, and every + // `[x, y]` (or `[x, y, z]`) pair would otherwise force the inner + // vector through 0 -> 1 -> 2 -> 4 growths plus the matching + // shared_ptr copies. Reserving 2 slots up front eliminates those + // reallocations for the common case and adds only a single small + // allocation for larger rings (which still grow geometrically). + json_array() + : json_object(JSON_ARRAY) { + array_value.reserve(2); + } + json_array(json_object *p, json_pull *pl) + : json_object(JSON_ARRAY, p, pl) { + array_value.reserve(2); + } +}; -#ifdef __cplusplus +struct json_hash : json_object { + std::vector entries_value; + + // Most GeoJSON property hashes have a handful of keys (type, id, + // properties, geometry, plus a few attribute fields). Reserving 4 + // slots avoids the 0 -> 1 -> 2 -> 4 growth chain for the typical + // case while only modestly over-allocating for one-key hashes. + json_hash() + : json_object(JSON_HASH) { + entries_value.reserve(4); + } + json_hash(json_object *p, json_pull *pl) + : json_object(JSON_HASH, p, pl) { + entries_value.reserve(4); + } +}; + +inline std::string &json_object::string() { + assert(type == JSON_STRING); + return static_cast(this)->string_value; } -#endif +inline const std::string &json_object::string() const { + assert(type == JSON_STRING); + return static_cast(this)->string_value; +} + +inline double json_object::number() const { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + switch (n->repr) { + case json_number::REPR_LARGE_UNSIGNED: + return static_cast(n->value.u); + case json_number::REPR_LARGE_SIGNED: + return static_cast(n->value.s); + case json_number::REPR_DOUBLE: + default: + return n->value.d; + } +} +inline unsigned long long json_object::large_unsigned() const { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + return n->repr == json_number::REPR_LARGE_UNSIGNED ? n->value.u : 0; +} +inline long long json_object::large_signed() const { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + return n->repr == json_number::REPR_LARGE_SIGNED ? n->value.s : 0; +} +inline void json_object::set_number(double d) { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + n->repr = json_number::REPR_DOUBLE; + n->value.d = d; +} +inline void json_object::set_large_unsigned(unsigned long long u) { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + n->repr = json_number::REPR_LARGE_UNSIGNED; + n->value.u = u; +} +inline void json_object::set_large_signed(long long s) { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + n->repr = json_number::REPR_LARGE_SIGNED; + n->value.s = s; +} + +inline std::vector &json_object::array() { + assert(type == JSON_ARRAY); + return static_cast(this)->array_value; +} +inline const std::vector &json_object::array() const { + assert(type == JSON_ARRAY); + return static_cast(this)->array_value; +} + +inline std::vector &json_object::entries() { + assert(type == JSON_HASH); + return static_cast(this)->entries_value; +} +inline const std::vector &json_object::entries() const { + assert(type == JSON_HASH); + return static_cast(this)->entries_value; +} + +inline void json_object_deleter::operator()(json_object *p) const noexcept { + if (p == nullptr) { + return; + } + // Dispatch on the discriminator so the correct subclass destructor + // runs. json_object has no virtual destructor, so a bare `delete p` + // would skip the std::vector / std::string members of the subclass. + switch (p->type) { + case JSON_NUMBER: + delete static_cast(p); + break; + case JSON_STRING: + delete static_cast(p); + break; + case JSON_ARRAY: + delete static_cast(p); + break; + case JSON_HASH: + delete static_cast(p); + break; + default: + // JSON_TRUE / JSON_FALSE / JSON_NULL (and the parse-token + // types, which never appear as owned nodes) are bare + // json_objects with no extra fields. + delete p; + break; + } +} + +struct json_pull { + const char *error = nullptr; // points at a string literal; no allocation + int line = 1; + + ssize_t (*read)(struct json_pull *, char *buf, size_t n) = nullptr; + void *source = nullptr; + std::vector buffer; + ssize_t buffer_tail = 0; + ssize_t buffer_head = 0; + + // Stack of currently-open containers; the top is the innermost + // container being parsed. Each frame also remembers what token is + // expected next (an item, a comma, a key, a colon, or a value). + // The frame's `container` is a borrowed raw pointer; actual + // ownership of the in-progress container lives in either the + // surrounding container's vector (for nested containers) or + // `root` (for the outermost container). + struct parse_frame { + json_object *container; + json_type expect; + }; + std::vector container_stack; + + // The most recently completed top-level value. The parser owns + // it (as a unique_ptr) until either: the next top-level value + // starts parsing (the old root is destroyed), the caller calls + // json_read_tree (ownership is transferred out), or the caller + // calls json_free / json_disconnect (the parser's reference is + // dropped explicitly). + json_object_ptr root; + + // Scratch buffers reused across tokens so we don't reallocate per + // number/string. number_buffer accumulates raw digits before atof(); + // string_buffer accumulates decoded bytes before being copied into + // the final json_string. Both are cleared (capacity preserved) at + // the start of each token, so once they grow to the largest seen + // size they stop reallocating entirely. + std::string number_buffer; + std::string string_buffer; +}; + +json_pull_ptr json_begin_file(FILE *f); +json_pull_ptr json_begin_string(const char *s); + +json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); + +// json_end is a thin convenience that resets the caller's json_pull_ptr. +// The parser (and any tree it still owns) is freed when the last +// shared_ptr to it is dropped, so calling json_end is optional if the +// json_pull_ptr will go out of scope on its own. +void json_end(json_pull_ptr &p); + +typedef void (*json_separator_callback)(json_type type, json_pull *j, void *state); + +// json_read returns a borrowed pointer to the next completed JSON node +// in the stream. The returned pointer is valid until the next call that +// extends or trims the parser's tree (the next json_read on the same +// parser, a json_free on the same node, or a json_disconnect that +// extracts the node). Returns nullptr at end of input or on error. +// +// For top-level values, ownership stays with the parser (via jp->root); +// for nested values, ownership stays with the enclosing container. +json_object *json_read(json_pull_ptr &j); +json_object *json_read_separators(json_pull_ptr &j, json_separator_callback cb, void *state); + +// json_read_tree drains the next top-level value out of the parser +// and hands ownership to the caller. After it returns, jp->root is +// empty, the parent/parser back-pointers throughout the subtree have +// been cleared, and the caller's json_object_ptr is the only thing +// keeping the tree alive. The returned tree can outlive the +// json_pull it was parsed from. +json_object_ptr json_read_tree(json_pull_ptr &j); + +// json_free splices `o` out of its parent (if any), or clears the +// parser's root if `o` is the parser's current top-level value, and +// destroys the subtree. After this call, `o` is a dangling pointer +// that must not be used. Safe to call with nullptr. +void json_free(json_object *o); + +// Splice `o` out of its parent's array/object (or out of the parser's +// root), walk the detached subtree clearing parent/parser back-pointers, +// and return ownership of the subtree to the caller as a +// json_object_ptr. After this returns, the parser no longer references +// any node in the subtree, and the subtree can outlive the original +// parser. +json_object_ptr json_disconnect(json_object *o); + +// Look up `s` in the hash `o`. Returns a borrowed pointer; ownership +// stays with the hash. nullptr if `o` is not a hash, or `s` is absent, +// or the matching value is null. Accepts a json_object_ptr by reference +// as a convenience so callers don't have to write `.get()`. +json_object *json_hash_get(const json_object_ptr &o, const char *s); +json_object *json_hash_get(json_object *o, const char *s); + +std::string json_stringify(const json_object *o); #endif diff --git a/jsontool.cpp b/jsontool.cpp index bb5341b8..014b1f6e 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -146,17 +146,16 @@ void out(std::string const &s, int type, json_object *properties) { bool found = false; json_object *o = json_hash_get(properties, extract); - if (o != NULL) { + if (o != nullptr) { found = true; - if (o->type == JSON_STRING || o->type == JSON_NUMBER) { - extracted = sort_quote(o->value.string.string); + if (o->type == JSON_STRING) { + extracted = sort_quote(o->string().c_str()); } else { - // Don't really know what to do about sort quoting - // for arbitrary objects - - const char *out = json_stringify(o); - extracted = sort_quote(out); - free((void *) out); + // Numbers, booleans, null, and any other non-string + // values are rendered via json_stringify(); calling + // o->string() here would assert because the type-tagged + // accessor requires JSON_STRING. + extracted = sort_quote(json_stringify(o).c_str()); } } @@ -232,13 +231,13 @@ void join_csv(json_object *j) { } json_object *properties = json_hash_get(j, "properties"); - json_object *key = NULL; + json_object *key = nullptr; - if (properties != NULL) { + if (properties != nullptr) { key = json_hash_get(properties, header[0].c_str()); } - if (key == NULL) { + if (key == nullptr) { static bool warned = false; if (!warned) { fprintf(stderr, "Warning: couldn't find CSV key \"%s\" in JSON\n", header[0].c_str()); @@ -249,13 +248,11 @@ void join_csv(json_object *j) { std::string joinkey; if (key->type == JSON_STRING) { - joinkey = key->value.string.string; + joinkey = key->string(); } else if (key->type == JSON_NUMBER) { - joinkey = milo::dtoa_milo(key->value.number.number); + joinkey = milo::dtoa_milo(key->number()); } else { - const char *s = json_stringify(key); - joinkey = s; - free((void *) s); + joinkey = json_stringify(key); } if (joinkey < prev_joinkey) { @@ -305,14 +302,7 @@ void join_csv(json_object *j) { } if (fields.size() > 0 && joinkey == fields[0]) { - // This knows more about the structure of JSON objects than it ought to - // The 8 is to round up at least as much as SIZE_FOR in json_pull.c - properties->value.object.keys = (json_object **) realloc((void *) properties->value.object.keys, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *)); - properties->value.object.values = (json_object **) realloc((void *) properties->value.object.values, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *)); - if (properties->value.object.keys == NULL || properties->value.object.values == NULL) { - perror("realloc"); - exit(EXIT_MEMORY); - } + properties->entries().reserve(properties->entries().size() + fields.size()); for (size_t i = 1; i < fields.size(); i++) { std::string k = header[i]; @@ -330,46 +320,21 @@ void join_csv(json_object *j) { } if (attr_type != JSON_NULL) { - // This knows more about the structure of JSON objects than it ought to - - json_object *ko = (json_object *) malloc(sizeof(json_object)); - json_object *vo = (json_object *) malloc(sizeof(json_object)); - if (ko == NULL || vo == NULL) { - perror("malloc"); - exit(EXIT_MEMORY); - } - - ko->type = JSON_STRING; - ko->parent = properties; - ko->parser = properties->parser; - - ko->value.string.string = strdup(k.c_str()); - if (ko->value.string.string == NULL) { - perror("strdup"); - exit(EXIT_MEMORY); - } - - vo->type = attr_type; - vo->parent = properties; - vo->parser = properties->parser; + json_object_ptr ko(new json_string(properties, properties->parser)); + ko->string() = k; + json_object_ptr vo; if (attr_type == JSON_STRING) { - vo->value.string.string = strdup(v.c_str()); - if (vo->value.string.string == NULL) { - perror("strdup"); - exit(EXIT_MEMORY); - } + vo = json_object_ptr(new json_string(properties, properties->parser)); + vo->string() = v; } else if (attr_type == JSON_NUMBER) { - vo->value.number.number = atof(v.c_str()); - vo->value.number.large_unsigned = 0; - vo->value.number.large_signed = 0; + vo = json_object_ptr(new json_number(properties, properties->parser)); + vo->set_number(atof(v.c_str())); } else { abort(); } - properties->value.object.keys[properties->value.object.length] = ko; - properties->value.object.values[properties->value.object.length] = vo; - properties->value.object.length++; + properties->entries().push_back({std::move(ko), std::move(vo)}); } } } @@ -382,13 +347,9 @@ struct json_join_action : json_feature_action { join_csv(feature); } - char *s = json_stringify(feature); - out(s, 1, json_hash_get(feature, "properties")); - free(s); + out(json_stringify(feature), 1, json_hash_get(feature, "properties")); } else { - char *s = json_stringify(geometry); - out(s, 2, NULL); - free(s); + out(json_stringify(geometry), 2, nullptr); } return 1; @@ -399,12 +360,11 @@ struct json_join_action : json_feature_action { }; void process(FILE *fp, const char *fname) { - json_pull *jp = json_begin_file(fp); + json_pull_ptr jp = json_begin_file(fp); json_join_action jja; jja.fname = fname; parse_json(&jja, jp); - json_end(jp); } int main(int argc, char **argv) { diff --git a/main.cpp b/main.cpp index 1f8b8f83..c11c3456 100644 --- a/main.cpp +++ b/main.cpp @@ -586,7 +586,7 @@ struct STREAM { } } - json_pull *json_begin() { + json_pull_ptr json_begin() { return ::json_begin(read_stream, this); } }; @@ -1818,7 +1818,7 @@ std::pair read_input(std::vector &sources, char *fname, i // Plain serial reading std::atomic layer_seq(overall_offset); - json_pull *jp = fp->json_begin(); + json_pull_ptr jp = fp->json_begin(); struct serialization_state sst; sst.fname = reading.c_str(); @@ -1845,7 +1845,6 @@ std::pair read_input(std::vector &sources, char *fname, i sst.attribute_types = attribute_types; parse_json(&sst, jp, layer, sources[layer].layer); - json_end(jp); overall_offset = layer_seq; checkdisk(&readers); } @@ -2873,10 +2872,10 @@ void set_attribute_type(std::unordered_map &attribute_types, c void set_attribute_value(const char *arg) { if (*arg == '{') { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: --set-attribute %s: %s\n", *av, arg, jp->error); exit(EXIT_JSON); } @@ -2886,21 +2885,18 @@ void set_attribute_value(const char *arg) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - json_object *k = o->value.object.keys[i]; - json_object *v = o->value.object.values[i]; - - if (k->type != JSON_STRING) { + size_t i = 0; + for (const auto &e : o->entries()) { + if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: --set-attribute %s: key %zu not a string\n", *av, arg, i); exit(EXIT_JSON); } - serial_val val = stringify_value(v, "json", 1, o); - set_attributes.emplace(k->value.string.string, val); + serial_val val = stringify_value(e.value.get(), "json", 1, o.get()); + set_attributes.emplace(e.key->string(), val); + i++; } - json_free(o); - json_end(jp); return; } @@ -2925,10 +2921,10 @@ void set_attribute_value(const char *arg) { } void parse_json_source(const char *arg, struct source &src) { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: -L%s: %s\n", *av, arg, jp->error); exit(EXIT_JSON); } @@ -2939,30 +2935,27 @@ void parse_json_source(const char *arg, struct source &src) { } json_object *fname = json_hash_get(o, "file"); - if (fname == NULL || fname->type != JSON_STRING) { + if (fname == nullptr || fname->type != JSON_STRING) { fprintf(stderr, "%s: -L%s: requires \"file\": filename\n", *av, arg); exit(EXIT_JSON); } - src.file = std::string(fname->value.string.string); + src.file = fname->string(); json_object *layer = json_hash_get(o, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - src.layer = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + src.layer = layer->string(); } json_object *description = json_hash_get(o, "description"); - if (description != NULL && description->type == JSON_STRING) { - src.description = std::string(description->value.string.string); + if (description != nullptr && description->type == JSON_STRING) { + src.description = description->string(); } json_object *format = json_hash_get(o, "format"); - if (format != NULL && format->type == JSON_STRING) { - src.format = std::string(format->value.string.string); + if (format != nullptr && format->type == JSON_STRING) { + src.format = format->string(); } - - json_free(o); - json_end(jp); } int main(int argc, char **argv) { @@ -3008,7 +3001,7 @@ int main(int argc, char **argv) { int exclude_all = 0; int read_parallel = 0; int files_open_at_start; - json_object *filter = NULL; + json_object_ptr filter; memsize = calc_memsize(); @@ -3853,7 +3846,7 @@ int main(int argc, char **argv) { auto input_ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, - maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); + maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter.get(), droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); ret = std::get<0>(input_ret); @@ -3876,9 +3869,7 @@ int main(int argc, char **argv) { exit(EXIT_IMPOSSIBLE); } - if (filter != NULL) { - json_free(filter); - } + filter.reset(); return ret; } diff --git a/mvt.cpp b/mvt.cpp index 18f239d2..8042db7c 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -407,7 +407,7 @@ std::string mvt_tile::encode() { std::string feature_string; protozero::pbf_writer feature_writer(feature_string); - if (layers[i].features[f].type >= 0) + if (layers[i].features[f].type >= 0) feature_writer.add_enum(3, layers[i].features[f].type); std::vector sorted_tags = layers[i].features[f].tags; diff --git a/overzoom.cpp b/overzoom.cpp index cdef1dd8..05dce3e7 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -238,7 +238,7 @@ int main(int argc, char **argv) { std::string out; { - json_object *json_filter = NULL; + json_object_ptr json_filter; if (filter.size() > 0) { json_filter = parse_filter(filter.c_str()); } @@ -264,7 +264,7 @@ int main(int argc, char **argv) { its.push_back(std::move(t)); } - out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector(), "", "", SIZE_MAX, std::vector(), deduplicate_by_id); + out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter.get(), preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector(), "", "", SIZE_MAX, std::vector(), deduplicate_by_id); } FILE *f = fopen(outfile, "wb"); diff --git a/plugin.cpp b/plugin.cpp index cfe06593..7e07dea3 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -27,9 +27,7 @@ #include "errors.hpp" #include "thread.hpp" -extern "C" { #include "jsonpull/jsonpull.h" -} #include "plugin.hpp" #include "write_json.hpp" @@ -145,53 +143,58 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } // Reads from the prefilter -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { +serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { serial_feature sf; while (1) { + // json_read returns each token as the parser produces it, including + // intermediate (still incomplete) container nodes. We must NOT free + // these intermediates here: they belong to the larger feature hash + // still being assembled, and freeing them would splice them out of + // the parent and corrupt the in-progress tree. We only free `j` + // after we have successfully processed a complete Feature hash + // (just before returning), or `jp->root` when the stream ends. json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); + if (jp->root != nullptr) { + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } exit(EXIT_JSON); } - json_free(jp->root); + jp->root.reset(); sf.t = -1; return sf; } json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + if (type == nullptr || type->type != JSON_STRING) { continue; } - if (strcmp(type->value.string.string, "Feature") != 0) { + if (type->string() != "Feature") { continue; } json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); exit(EXIT_JSON); @@ -204,7 +207,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: feature without coordinates array: ", jp->line); json_context(j); exit(EXIT_JSON); @@ -212,12 +215,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str()); json_context(j); exit(EXIT_JSON); } @@ -253,29 +256,29 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: std::string layername = "unknown"; json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - if (tippecanoe != NULL) { + if (tippecanoe != nullptr) { json_object *layer = json_hash_get(tippecanoe, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - layername = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + layername = layer->string(); } json_object *index = json_hash_get(tippecanoe, "index"); - if (index != NULL && index->type == JSON_NUMBER) { - sf.index = index->value.number.number; + if (index != nullptr && index->type == JSON_NUMBER) { + sf.index = index->number(); } json_object *sequence = json_hash_get(tippecanoe, "sequence"); - if (sequence != NULL && sequence->type == JSON_NUMBER) { - sf.seq = sequence->value.number.number; + if (sequence != nullptr && sequence->type == JSON_NUMBER) { + sf.seq = sequence->number(); } json_object *extent = json_hash_get(tippecanoe, "extent"); - if (extent != NULL && extent->type == JSON_NUMBER) { - sf.extent = extent->value.number.number; + if (extent != nullptr && extent->type == JSON_NUMBER) { + sf.extent = extent->number(); } json_object *dropped = json_hash_get(tippecanoe, "dropped"); - if (dropped != NULL && dropped->type == JSON_TRUE) { + if (dropped != nullptr && dropped->type == JSON_TRUE) { sf.dropped = FEATURE_DROPPED; // dropped } else { sf.dropped = FEATURE_KEPT; // kept @@ -300,10 +303,10 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } json_object *id = json_hash_get(j, "id"); - if (id != NULL && id->type == JSON_NUMBER) { - sf.id = id->value.number.number; - if (id->value.number.large_unsigned > 0) { - sf.id = id->value.number.large_unsigned; + if (id != nullptr && id->type == JSON_NUMBER) { + sf.id = id->number(); + if (id->large_unsigned() > 0) { + sf.id = id->large_unsigned(); } sf.has_id = true; } @@ -347,18 +350,20 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } } - for (size_t i = 0; i < properties->value.object.length; i++) { - serial_val v = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); + if (properties->type == JSON_HASH) { + for (const auto &e : properties->entries()) { + serial_val v = stringify_value(e.value.get(), "Filter output", jp->line, j); - // Nulls can be excluded here because the expression evaluation filter - // would have already run before prefiltering + // Nulls can be excluded here because the expression evaluation filter + // would have already run before prefiltering - if (v.type != mvt_null) { - sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string))); - sf.full_values.push_back(v); + if (v.type != mvt_null) { + sf.full_keys.push_back(key_pool.pool(e.key->string())); + sf.full_values.push_back(v); - if (!postfilter) { - add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), v); + if (!postfilter) { + add_to_tilestats(ts->second.tilestats, e.key->string(), v); + } } } } diff --git a/plugin.hpp b/plugin.hpp index fc901daf..5ad99092 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,4 +1,4 @@ struct key_pool; std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); +serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index e86e538b..57695b1a 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -397,9 +397,9 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { exit(EXIT_OPEN); } - json_pull *jp = json_begin_string(decompressed_json.c_str()); - json_object *o = json_read_tree(jp); - if (o == NULL) { + json_pull_ptr jp = json_begin_string(decompressed_json.c_str()); + json_object_ptr o = json_read_tree(jp); + if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", fname, jp->error); exit(EXIT_JSON); } @@ -415,45 +415,44 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { state.nospace = true; state.json_write_hash(); - for (size_t i = 0; i < o->value.object.length; i++) { - const char *key = o->value.object.keys[i]->value.string.string; - if (strcmp(key, "vector_layers") == 0 && o->value.object.values[i]->type == JSON_ARRAY) { + for (const auto &e : o->entries()) { + const std::string &key = e.key->string(); + if (key == "vector_layers" && e.value->type == JSON_ARRAY) { has_json = true; state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; - state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (strcmp(key, "tilestats") == 0 && o->value.object.values[i]->type == JSON_HASH) { + state.json_write_json(json_stringify(e.value.get())); + } else if (key == "tilestats" && e.value->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; - state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (strcmp(key, "strategies") == 0 && o->value.object.values[i]->type == JSON_ARRAY) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->value.object.values[i])); + state.json_write_json(json_stringify(e.value.get())); + } else if (key == "strategies" && e.value->type == JSON_ARRAY) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (strcmp(key, "tippecanoe_decisions") == 0 && o->value.object.values[i]->type == JSON_HASH) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i])); + } else if (key == "tippecanoe_decisions" && e.value->type == JSON_HASH) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { - fprintf(stderr, "%s\n", key); + } else if (e.key->type != JSON_STRING || e.value->type != JSON_STRING) { + fprintf(stderr, "%s\n", key.c_str()); fprintf(stderr, "%s: non-string in metadata\n", fname); } else { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key, o->value.object.values[i]->value.string.string); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } } - json_end(jp); state.nospace = true; state.json_end_hash(); diff --git a/read_json.cpp b/read_json.cpp index 63329acb..13798ee6 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -43,18 +43,18 @@ int mb_geometry[GEOM_TYPES] = { }; void json_context(json_object *j) { - char *s = json_stringify(j); + std::string s = json_stringify(j); - if (strlen(s) >= 500) { - snprintf(s + 497, strlen(s) + 1 - 497, "..."); + if (s.size() >= 500) { + s.resize(497); + s.append("..."); } - fprintf(stderr, "in JSON object %s\n", s); - free(s); // stringify + fprintf(stderr, "in JSON object %s\n", s.c_str()); } void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature) { - if (j == NULL || j->type != JSON_ARRAY) { + if (j == nullptr || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for geometry type %d: ", fname, line, t); json_context(feature); return; @@ -63,7 +63,7 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * int within = geometry_within[t]; if (within >= 0) { size_t i; - for (i = 0; i < j->value.array.length; i++) { + for (i = 0; i < j->array().size(); i++) { if (within == GEOM_POINT) { if (i == 0 || mb_geometry[t] == VT_POINT) { op = VT_MOVETO; @@ -72,16 +72,16 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * } } - parse_coordinates(within, j->value.array.array[i], out, op, fname, line, feature); + parse_coordinates(within, j->array()[i].get(), out, op, fname, line, feature); } } else { - if (j->value.array.length >= 2 && j->value.array.array[0]->type == JSON_NUMBER && j->value.array.array[1]->type == JSON_NUMBER) { + if (j->array().size() >= 2 && j->array()[0]->type == JSON_NUMBER && j->array()[1]->type == JSON_NUMBER) { long long x, y; - double lon = j->value.array.array[0]->value.number.number; - double lat = j->value.array.array[1]->value.number.number; + double lon = j->array()[0]->number(); + double lat = j->array()[1]->number(); projection->project(lon, lat, 32, &x, &y); - if (j->value.array.length > 2) { + if (j->array().size() > 2) { static int warned = 0; if (!warned) { @@ -124,12 +124,12 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature) { serial_val sv; - if (value != NULL) { + if (value != nullptr) { int vt = value->type; if (vt == JSON_STRING) { sv.type = mvt_string; - sv.s = value->value.string.string; + sv.s = value->string(); std::string err = check_utf8(sv.s); if (err.size() > 0) { @@ -140,12 +140,12 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js } else if (vt == JSON_NUMBER) { sv.type = mvt_double; - if (value->value.number.large_unsigned != 0) { - sv.s = std::to_string(value->value.number.large_unsigned); - } else if (value->value.number.large_signed != 0) { - sv.s = std::to_string(value->value.number.large_signed); + if (value->large_unsigned() != 0) { + sv.s = std::to_string(value->large_unsigned()); + } else if (value->large_signed() != 0) { + sv.s = std::to_string(value->large_signed()); } else { - sv.s = milo::dtoa_milo(value->value.number.number); + sv.s = milo::dtoa_milo(value->number()); } } else if (vt == JSON_TRUE) { sv.type = mvt_bool; @@ -158,9 +158,7 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js sv.s = "null"; } else { sv.type = mvt_string; - const char *v = json_stringify(value); - sv.s = std::string(v); - free((void *) v); // stringify + sv.s = json_stringify(value); } } @@ -178,10 +176,10 @@ static std::vector to_feature(drawvec &geom) { return out; } -std::pair parse_geometry(json_object *geometry, json_pull *jp, json_object *j, +std::pair parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style) { json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); exit(EXIT_JSON); @@ -194,7 +192,7 @@ std::pair parse_geometry(json_object *geometry, json_pull *jp, jso } json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: geometry without coordinates array: ", jp->line); json_context(j); exit(EXIT_JSON); @@ -202,12 +200,12 @@ std::pair parse_geometry(json_object *geometry, json_pull *jp, jso int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str()); json_context(j); exit(EXIT_JSON); } @@ -305,47 +303,51 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int std::map ret; std::shared_ptr tile_stringpool = std::make_shared(); - json_pull *jp = json_begin_file(fp); + json_pull_ptr jp = json_begin_file(fp); while (1) { json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); + if (jp->root != nullptr) { + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } exit(EXIT_JSON); } - json_free(jp->root); + jp->root.reset(); break; } + // json_read returns each parser token in sequence, including + // intermediate (still-incomplete) container nodes. Freeing those + // here would splice them out of the feature hash being built + // up, so only free `j` once we have processed a complete + // Feature (or `jp->root` when the stream ends). json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + if (type == nullptr || type->type != JSON_STRING) { continue; } - if (strcmp(type->value.string.string, "Feature") != 0) { + if (type->string() != "Feature") { continue; } json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } std::string layername = "unknown"; json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - json_object *layer = NULL; - if (tippecanoe != NULL) { + json_object *layer = nullptr; + if (tippecanoe != nullptr) { layer = json_hash_get(tippecanoe, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - layername = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + layername = layer->string(); } } @@ -360,10 +362,9 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int auto l = ret.find(layername); json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } @@ -378,23 +379,25 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int feature.geometry = to_feature(dv); json_object *id = json_hash_get(j, "id"); - if (id != NULL && id->type == JSON_NUMBER) { - feature.id = id->value.number.number; - if (id->value.number.large_unsigned > 0) { - feature.id = id->value.number.large_unsigned; + if (id != nullptr && id->type == JSON_NUMBER) { + feature.id = id->number(); + if (id->large_unsigned() > 0) { + feature.id = id->large_unsigned(); } feature.has_id = true; } - for (size_t i = 0; i < properties->value.object.length; i++) { - serial_val sv = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); + if (properties->type == JSON_HASH) { + for (const auto &e : properties->entries()) { + serial_val sv = stringify_value(e.value.get(), "Filter output", jp->line, j); - // Nulls can be excluded here because this is the postfilter - // and it is nearly time to create the vector representation + // Nulls can be excluded here because this is the postfilter + // and it is nearly time to create the vector representation - if (sv.type != mvt_null) { - mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v); + if (sv.type != mvt_null) { + mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); + l->second.tag(feature, e.key->string(), v); + } } } @@ -404,8 +407,6 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int json_free(j); } - json_end(jp); - std::vector final; for (auto a : ret) { final.push_back(a.second); diff --git a/read_json.hpp b/read_json.hpp index a5d5d7b2..f254e354 100644 --- a/read_json.hpp +++ b/read_json.hpp @@ -12,7 +12,7 @@ extern int mb_geometry[GEOM_TYPES]; void json_context(json_object *j); void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature); -std::pair parse_geometry(json_object *geometry, json_pull *jp, json_object *j, +std::pair parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style); std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int extent, bool fix_longitudes); diff --git a/tile-join.cpp b/tile-join.cpp index 1b057577..03464714 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -335,7 +335,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map> &tasks, std::vector< } void handle_strategies(const unsigned char *s, std::vector *st) { - json_pull *jp = json_begin_string((const char *) s); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string((const char *) s); + json_object_ptr o = json_read_tree(jp); - if (o != NULL && o->type == JSON_ARRAY) { - for (size_t i = 0; i < o->value.array.length; i++) { - json_object *h = o->value.array.array[i]; + if (o != nullptr && o->type == JSON_ARRAY) { + for (size_t i = 0; i < o->array().size(); i++) { + const json_object_ptr &h = o->array()[i]; if (h->type == JSON_HASH) { - for (size_t j = 0; j < h->value.object.length; j++) { - json_object *k = h->value.object.keys[j]; - json_object *v = h->value.object.values[j]; - - if (k->type != JSON_STRING) { + size_t j = 0; + for (const auto &kv : h->entries()) { + if (kv.key->type != JSON_STRING) { fprintf(stderr, "Key %zu of %zu is not a string: %s\n", j, i, s); - } else if (v->type != JSON_NUMBER) { + } else if (kv.value->type != JSON_NUMBER) { fprintf(stderr, "Value %zu of %zu is not a number: %s\n", j, i, s); } else { if (i >= st->size()) { st->resize(i + 1); } - if (strcmp(k->value.string.string, "dropped_by_rate") == 0) { - (*st)[i].dropped_by_rate += v->value.number.number; - } else if (strcmp(k->value.string.string, "dropped_by_gamma") == 0) { - (*st)[i].dropped_by_gamma += v->value.number.number; - } else if (strcmp(k->value.string.string, "dropped_as_needed") == 0) { - (*st)[i].dropped_as_needed += v->value.number.number; - } else if (strcmp(k->value.string.string, "coalesced_as_needed") == 0) { - (*st)[i].coalesced_as_needed += v->value.number.number; - } else if (strcmp(k->value.string.string, "truncated_zooms") == 0) { - (*st)[i].truncated_zooms += v->value.number.number; - } else if (strcmp(k->value.string.string, "detail_reduced") == 0) { - (*st)[i].detail_reduced += v->value.number.number; - } else if (strcmp(k->value.string.string, "tiny_polygons") == 0) { - (*st)[i].tiny_polygons += v->value.number.number; - } else if (strcmp(k->value.string.string, "tile_size_desired") == 0) { - (*st)[i].tile_size += v->value.number.number; - } else if (strcmp(k->value.string.string, "feature_count_desired") == 0) { - (*st)[i].feature_count += v->value.number.number; + const std::string &key = kv.key->string(); + if (key == "dropped_by_rate") { + (*st)[i].dropped_by_rate += kv.value->number(); + } else if (key == "dropped_by_gamma") { + (*st)[i].dropped_by_gamma += kv.value->number(); + } else if (key == "dropped_as_needed") { + (*st)[i].dropped_as_needed += kv.value->number(); + } else if (key == "coalesced_as_needed") { + (*st)[i].coalesced_as_needed += kv.value->number(); + } else if (key == "truncated_zooms") { + (*st)[i].truncated_zooms += kv.value->number(); + } else if (key == "detail_reduced") { + (*st)[i].detail_reduced += kv.value->number(); + } else if (key == "tiny_polygons") { + (*st)[i].tiny_polygons += kv.value->number(); + } else if (key == "tile_size_desired") { + (*st)[i].tile_size += kv.value->number(); + } else if (key == "feature_count_desired") { + (*st)[i].feature_count += kv.value->number(); } } + j++; } } else { fprintf(stderr, "Element %zu is not a hash: %s\n", i, s); } } - json_free(o); } - - json_end(jp); } void handle_vector_layers(json_object *vector_layers, std::map &layermap, std::map &attribute_descriptions) { - if (vector_layers != NULL && vector_layers->type == JSON_ARRAY) { - for (size_t i = 0; i < vector_layers->value.array.length; i++) { - if (vector_layers->value.array.array[i]->type == JSON_HASH) { - json_object *id = json_hash_get(vector_layers->value.array.array[i], "id"); - json_object *desc = json_hash_get(vector_layers->value.array.array[i], "description"); + if (vector_layers != nullptr && vector_layers->type == JSON_ARRAY) { + for (size_t i = 0; i < vector_layers->array().size(); i++) { + if (vector_layers->array()[i]->type == JSON_HASH) { + json_object *id = json_hash_get(vector_layers->array()[i].get(), "id"); + json_object *desc = json_hash_get(vector_layers->array()[i].get(), "description"); - if (id != NULL && desc != NULL && id->type == JSON_STRING && desc->type == JSON_STRING) { - std::string sid = id->value.string.string; - std::string sdesc = desc->value.string.string; + if (id != nullptr && desc != nullptr && id->type == JSON_STRING && desc->type == JSON_STRING) { + const std::string &sid = id->string(); + const std::string &sdesc = desc->string(); if (sdesc.size() != 0) { auto f = layermap.find(sid); @@ -1035,17 +1032,18 @@ void handle_vector_layers(json_object *vector_layers, std::mapvalue.array.array[i], "fields"); - if (fields != NULL && fields->type == JSON_HASH) { - for (size_t j = 0; j < fields->value.object.length; j++) { - if (fields->value.object.keys[j]->type == JSON_STRING && fields->value.object.values[j]->type) { - const char *desc2 = fields->value.object.values[j]->value.string.string; - - if (strcmp(desc2, "Number") != 0 && - strcmp(desc2, "String") != 0 && - strcmp(desc2, "Boolean") != 0 && - strcmp(desc2, "Mixed") != 0) { - attribute_descriptions.insert(std::pair(fields->value.object.keys[j]->value.string.string, desc2)); + json_object *fields = json_hash_get(vector_layers->array()[i].get(), "fields"); + if (fields != nullptr && fields->type == JSON_HASH) { + for (const auto &e : fields->entries()) { + if (e.key != nullptr && e.key->type == JSON_STRING && + e.value != nullptr && e.value->type == JSON_STRING) { + const std::string &desc2 = e.value->string(); + + if (desc2 != "Number" && + desc2 != "String" && + desc2 != "Boolean" && + desc2 != "Mixed") { + attribute_descriptions.insert(std::pair(e.key->string(), desc2)); } } } @@ -1205,17 +1203,14 @@ void decode(struct tileset_reader *readers, std::maptype == JSON_HASH) { + if (o != nullptr && o->type == JSON_HASH) { json_object *vector_layers = json_hash_get(o, "vector_layers"); handle_vector_layers(vector_layers, layermap, attribute_descriptions); - json_free(o); } - - json_end(jp); } } @@ -1266,7 +1261,7 @@ int main(int argc, char **argv) { int force = 0; int ifmatched = 0; int filearg = 0; - json_object *filter = NULL; + json_object_ptr filter; std::string join_sqlite_fname; @@ -1596,7 +1591,7 @@ int main(int argc, char **argv) { std::string generator_options; std::vector strategies; - decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); + decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter.get(), attribute_descriptions, generator_options, &strategies); if (set_attribution.size() != 0) { attribution = set_attribution; @@ -1648,9 +1643,7 @@ int main(int argc, char **argv) { mbtiles_close(outdb, argv[0]); } - if (filter != NULL) { - json_free(filter); - } + filter.reset(); if (pmtiles_has_suffix(out_mbtiles)) { mbtiles_map_image_to_pmtiles(out_mbtiles, m, !pC, quiet, false); diff --git a/tile.cpp b/tile.cpp index a4f12c75..7d951bf1 100644 --- a/tile.cpp +++ b/tile.cpp @@ -48,11 +48,7 @@ #include "attribute.hpp" #include "thread.hpp" #include "shared_borders.hpp" - -extern "C" { #include "jsonpull/jsonpull.h" -} - #include "plugin.hpp" #define CMD_BITS 3 @@ -835,7 +831,7 @@ static double choose_minattribute(std::vector &attribute_values, double if (descending) { // For descending: drop features > threshold, keep features <= threshold // ix points at the last value to keep - size_t ix = (size_t)((attribute_values.size() - 1) * f); + size_t ix = (size_t) ((attribute_values.size() - 1) * f); while (ix > 0 && attribute_values[ix] >= existing_attribute) { ix--; } @@ -848,7 +844,7 @@ static double choose_minattribute(std::vector &attribute_values, double } else { // For ascending: drop features < threshold, keep features >= threshold // ix points at the first value to keep - size_t ix = (size_t)ceil((double)(attribute_values.size() - 1) * (1 - f)); + size_t ix = (size_t) ceil((double) (attribute_values.size() - 1) * (1 - f)); if (ix >= attribute_values.size()) { ix = attribute_values.size() - 1; } @@ -1775,7 +1771,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch pthread_t prefilter_writer; run_prefilter_args rpa; // here so it stays in scope until joined FILE *prefilter_read_fp = NULL; - json_pull *prefilter_jp = NULL; + json_pull_ptr prefilter_jp; if (z < minzoom) { prefilter = NULL; @@ -2110,8 +2106,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (attr_valid) { add_sample_to(attribute_values, attr_numeric, attribute_values_increment, seq); bool should_drop = arg->drop_by_attribute_descending - ? (minattribute != HUGE_VAL && attr_numeric > minattribute) - : (minattribute != -HUGE_VAL && attr_numeric < minattribute); + ? (minattribute != HUGE_VAL && attr_numeric > minattribute) + : (minattribute != -HUGE_VAL && attr_numeric < minattribute); if (should_drop) { can_stop_early = false; if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) { @@ -2774,7 +2770,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } } else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) { minattribute_fraction = minattribute_fraction * - adjusted_max_tile_features / adjusted_feature_count * 0.75; + adjusted_max_tile_features / adjusted_feature_count * 0.75; if (minattribute_fraction > 0.80) { if (!quiet) { fprintf(stderr, @@ -3456,8 +3452,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: again = true; } bool attr_propagate = drop_by_attribute_descending - ? args[thread].minattribute_out < zoom_minattribute - : args[thread].minattribute_out > zoom_minattribute; + ? args[thread].minattribute_out < zoom_minattribute + : args[thread].minattribute_out > zoom_minattribute; if (attr_propagate) { zoom_minattribute = args[thread].minattribute_out; again = true; diff --git a/tile.hpp b/tile.hpp index 8f266c0b..8b72e3fb 100644 --- a/tile.hpp +++ b/tile.hpp @@ -62,7 +62,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/unit.cpp b/unit.cpp index fff6b7c4..e30ae227 100644 --- a/unit.cpp +++ b/unit.cpp @@ -6,6 +6,7 @@ #include "mvt.hpp" #include "projection.hpp" #include "geometry.hpp" +#include "jsonpull/jsonpull.h" #include #include @@ -136,3 +137,112 @@ TEST_CASE("line_is_too_small") { dv.emplace_back(VT_LINETO, -51864809, 2683873977); REQUIRE(line_is_too_small(dv, 0, 10)); } + +// Regression test for the surrogate-decoding bug that compared the leftover +// outer-loop byte `c` against `0xdfff` instead of the parsed code unit `ch`. +// For a string like "\uD83D\uE000" (a valid high surrogate followed by a +// non-surrogate BMP code point) the buggy version would mis-classify +// U+E000 as a low surrogate and combine the two units into the four-byte +// UTF-8 sequence F0 9F 90 80 (U+1F400). The fixed version flushes the +// stale high surrogate as standalone CESU-8 (ED A0 BD) and then encodes +// U+E000 normally as EE 80 80. +TEST_CASE("jsonpull surrogate-pair regression", "[jsonpull][surrogate]") { + json_pull_ptr jp = json_begin_string("\"\\uD83D\\uE000\""); + json_object_ptr o = json_read_tree(jp); + + REQUIRE(jp->error == nullptr); + REQUIRE(o != nullptr); + REQUIRE(o->type == JSON_STRING); + + const std::string expected = "\xED\xA0\xBD\xEE\x80\x80"; + REQUIRE(o->string() == expected); + + // Sanity check: the buggy output (a single 4-byte UTF-8 sequence for + // U+1F400) must not be what we got. + const std::string buggy = "\xF0\x9F\x90\x80"; + REQUIRE(o->string() != buggy); +} + +// geojson-loop.cpp calls json_free(j) after jfa->add_feature has +// serialized the feature, intending to drop the JSON subtree from the +// in-progress parse tree so that already-serialized features don't sit +// in memory while subsequent features are parsed. That intent was +// never tested; this test pins it down. The pre-fix behavior of +// json_free was a bare unique_ptr/shared_ptr reset that only dropped +// the caller's local reference; the parent container kept the subtree +// alive, so memory grew until the top-level parse completed. +TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") { + json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]"); + + json_object *outer = nullptr; + int arrays_seen = 0; + + json_object *j; + while ((j = json_read(jp)) != nullptr) { + if (j->type != JSON_ARRAY) { + continue; + } + arrays_seen++; + if (arrays_seen == 2) { + // This is [3, 4]; verify, then ask the parser to drop it. + REQUIRE(j->array().size() == 2); + REQUIRE(j->array()[0]->number() == 3); + REQUIRE(j->array()[1]->number() == 4); + json_free(j); + } else if (j->parent == nullptr) { + // The completed outer array; the parser still owns it + // via jp->root, so the borrowed pointer stays valid. + outer = j; + break; + } + } + + REQUIRE(outer != nullptr); + REQUIRE(outer->type == JSON_ARRAY); + REQUIRE(outer->array().size() == 2); + + // First surviving element: [1, 2]. + REQUIRE(outer->array()[0]->type == JSON_ARRAY); + REQUIRE(outer->array()[0]->array().size() == 2); + REQUIRE(outer->array()[0]->array()[0]->number() == 1); + REQUIRE(outer->array()[0]->array()[1]->number() == 2); + + // Second surviving element (previously third): [5, 6]. + REQUIRE(outer->array()[1]->type == JSON_ARRAY); + REQUIRE(outer->array()[1]->array().size() == 2); + REQUIRE(outer->array()[1]->array()[0]->number() == 5); + REQUIRE(outer->array()[1]->array()[1]->number() == 6); +} + +// The companion case to the pruning test above: in a line-delimited +// stream, each feature returned by json_read is a top-level value +// with no parent, but the parser still owns it via jp->root. +// json_free must drop that parser reference too, otherwise the +// just-serialized feature would sit in memory until the next feature +// started parsing. Under the unique_ptr ownership model, the only +// owner is jp->root, so verifying that jp->root is empty after the +// json_free call is also a guarantee that the subtree itself has +// been destroyed. +TEST_CASE("json_free releases a top-level value held by the parser", "[jsonpull][memory]") { + json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": [2, 3]})"); + + // json_read streams atoms first (1, 2, 3, [2,3], ...); the top-level + // hash is returned by the final `}` token. + json_object *top = nullptr; + json_object *j; + while ((j = json_read(jp)) != nullptr) { + if (j->parent == nullptr) { + top = j; + break; + } + } + + REQUIRE(top != nullptr); + REQUIRE(top->type == JSON_HASH); + REQUIRE(jp->root.get() == top); + + json_free(top); + // top is dangling now; do not dereference. + + REQUIRE(jp->root == nullptr); +}