From 9381158165ee0a6ff34ba855e3f35a21b171846d Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 17:48:21 -0700 Subject: [PATCH 01/13] Clear for merge --- jsonpull/jsonpull.cpp | 996 ------------------------------------------ 1 file changed, 996 deletions(-) diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 04ca0f296..e69de29bb 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -1,996 +0,0 @@ -#define _GNU_SOURCE // for asprintf() -#include -#include -#include -#include -#include -#include -#include "jsonpull.h" -#include "../milo/milo.h" - -#define BUFFER 10000 - -struct string { - char *buf; - size_t n; - size_t nalloc; -}; - -static void string_init(struct string *s); -static void string_free(struct string *s); - -json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source) { - json_pull *j = malloc(sizeof(json_pull)); - if (j == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - j->error = NULL; - j->line = 1; - j->container = NULL; - j->root = NULL; - - j->read = read; - j->source = source; - j->buffer_head = 0; - j->buffer_tail = 0; - - j->buffer = malloc(BUFFER); - if (j->buffer == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - j->number_buffer = malloc(sizeof(struct string)); - if (j->number_buffer == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - string_init(j->number_buffer); - - return j; -} - -static inline int peek(json_pull *j) { - if (j->buffer_head < j->buffer_tail) { - return (unsigned char) j->buffer[j->buffer_head]; - } else { - j->buffer_head = 0; - j->buffer_tail = j->read(j, j->buffer, BUFFER); - if (j->buffer_head >= j->buffer_tail) { - return EOF; - } - return (unsigned char) j->buffer[j->buffer_head]; - } -} - -static inline int next(json_pull *j) { - if (j->buffer_head < j->buffer_tail) { - return (unsigned char) j->buffer[j->buffer_head++]; - } else { - j->buffer_head = 0; - j->buffer_tail = j->read(j, j->buffer, BUFFER); - if (j->buffer_head >= j->buffer_tail) { - return EOF; - } - return (unsigned char) j->buffer[j->buffer_head++]; - } -} - -static ssize_t read_file(json_pull *j, char *buffer, size_t n) { - return fread(buffer, 1, n, j->source); -} - -json_pull *json_begin_file(FILE *f) { - return json_begin(read_file, f); -} - -static ssize_t read_string(json_pull *j, char *buffer, size_t n) { - const char *cp = j->source; - size_t out = 0; - - while (out < n && cp[out] != '\0') { - buffer[out] = cp[out]; - out++; - } - - j->source = (void *) (cp + out); - return out; -} - -json_pull *json_begin_string(const char *s) { - return json_begin(read_string, (void *) s); -} - -void json_end(json_pull *p) { - string_free(p->number_buffer); - free(p->number_buffer); - - json_free(p->root); - free(p->buffer); - free(p); -} - -static inline int read_wrap(json_pull *j) { - int c = next(j); - - if (c == '\n') { - j->line++; - } - - return c; -} - -#define SIZE_FOR(i, size) ((size_t) ((((i) + 7) & ~7) * size)) - -static json_object *fabricate_object(json_pull *jp, json_object *parent, json_type type) { - json_object *o = malloc(sizeof(struct json_object)); - if (o == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - o->type = type; - o->parent = parent; - o->parser = jp; - - if (type == JSON_ARRAY) { - o->value.array.array = NULL; - o->value.array.length = 0; - } else if (type == JSON_HASH) { - o->value.object.keys = NULL; - o->value.object.values = NULL; - o->value.object.length = 0; - } - - return o; -} - -static json_object *add_object(json_pull *j, json_type type) { - json_object *c = j->container; - json_object *o = fabricate_object(j, c, type); - - if (c != NULL) { - if (c->type == JSON_ARRAY) { - if (c->expect == JSON_ITEM) { - if (SIZE_FOR(c->value.array.length + 1, sizeof(json_object *)) != SIZE_FOR(c->value.array.length, sizeof(json_object *))) { - if (SIZE_FOR(c->value.array.length + 1, sizeof(json_object *)) < SIZE_FOR(c->value.array.length, sizeof(json_object *))) { - fprintf(stderr, "Array size overflow\n"); - exit(EXIT_FAILURE); - } - c->value.array.array = realloc(c->value.array.array, SIZE_FOR(c->value.array.length + 1, sizeof(json_object *))); - if (c->value.array.array == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - c->value.array.array[c->value.array.length++] = o; - c->expect = JSON_COMMA; - } else { - j->error = "Expected a comma, not a list item"; - free(o); - return NULL; - } - } else if (c->type == JSON_HASH) { - if (c->expect == JSON_VALUE) { - c->value.object.values[c->value.object.length - 1] = o; - c->expect = JSON_COMMA; - } else if (c->expect == JSON_KEY) { - if (type != JSON_STRING) { - j->error = "Hash key is not a string"; - free(o); - return NULL; - } - - if (SIZE_FOR(c->value.object.length + 1, sizeof(json_object *)) != SIZE_FOR(c->value.object.length, sizeof(json_object *))) { - if (SIZE_FOR(c->value.object.length + 1, sizeof(json_object *)) < SIZE_FOR(c->value.object.length, sizeof(json_object *))) { - fprintf(stderr, "Hash size overflow\n"); - exit(EXIT_FAILURE); - } - c->value.object.keys = realloc(c->value.object.keys, SIZE_FOR(c->value.object.length + 1, sizeof(json_object *))); - c->value.object.values = realloc(c->value.object.values, SIZE_FOR(c->value.object.length + 1, sizeof(json_object *))); - if (c->value.object.keys == NULL || c->value.object.values == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - c->value.object.keys[c->value.object.length] = o; - c->value.object.values[c->value.object.length] = NULL; - c->value.object.length++; - c->expect = JSON_COLON; - } else { - j->error = "Expected a comma or colon"; - free(o); - return NULL; - } - } - } else { - if (j->root != NULL) { - json_free(j->root); - } - - j->root = o; - } - - return o; -} - -json_object *json_hash_get(json_object *o, const char *s) { - if (o == NULL || o->type != JSON_HASH) { - return NULL; - } - - size_t i; - for (i = 0; i < o->value.object.length; i++) { - if (o->value.object.keys[i] != NULL && o->value.object.keys[i]->type == JSON_STRING) { - if (strcmp(o->value.object.keys[i]->value.string.string, s) == 0) { - return o->value.object.values[i]; - } - } - } - - return NULL; -} - -static void string_init(struct string *s) { - s->nalloc = 500; - s->buf = malloc(s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - s->n = 0; - s->buf[0] = '\0'; -} - -static void string_append(struct string *s, char c) { - if (s->n + 2 >= s->nalloc) { - size_t prev = s->nalloc; - s->nalloc += 500; - if (s->nalloc <= prev) { - fprintf(stderr, "String size overflowed\n"); - exit(EXIT_FAILURE); - } - s->buf = realloc(s->buf, s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - s->buf[s->n++] = c; - s->buf[s->n] = '\0'; -} - -static void string_append_string(struct string *s, char *add) { - size_t len = strlen(add); - - if (s->n + len + 1 >= s->nalloc) { - size_t prev = s->nalloc; - s->nalloc += 500 + len; - if (s->nalloc <= prev) { - fprintf(stderr, "String size overflowed\n"); - exit(EXIT_FAILURE); - } - s->buf = realloc(s->buf, s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - for (; *add != '\0'; add++) { - s->buf[s->n++] = *add; - } - - s->buf[s->n] = '\0'; -} - -static void string_free(struct string *s) { - free(s->buf); -} - -json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state) { - int c; - - // In case there is an error at the top level - if (j->container == NULL) { - if (j->root != NULL) { - json_free(j->root); - } - - j->root = NULL; - } - -again: - c = read_wrap(j); - if (c == EOF) { - if (j->container != NULL) { - j->error = "Reached EOF without all containers being closed"; - } - - return NULL; - } - - switch (c) { - /////////////////////////// Byte order mark - - case 0xEF: { - int c2 = peek(j); - if (c2 == 0xBB) { - c2 = read_wrap(j); - c2 = peek(j); - if (c2 == 0xBF) { - c2 = read_wrap(j); - c = ' '; - goto again; - } - } - j->error = "Corrupt byte-order mark found"; - return NULL; - } - - /////////////////////////// Whitespace - - case ' ': - case '\t': - case '\r': - case '\n': - case 0x1E: - goto again; - - /////////////////////////// Arrays - - case '[': { - json_object *o = add_object(j, JSON_ARRAY); - if (o == NULL) { - return NULL; - } - j->container = o; - j->container->expect = JSON_ITEM; - - if (cb != NULL) { - cb(JSON_ARRAY, j, state); - } - - goto again; - } - - case ']': { - if (j->container == NULL) { - j->error = "Found ] at top level"; - return NULL; - } - - if (j->container->type != JSON_ARRAY) { - j->error = "Found ] not in an array"; - return NULL; - } - - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_ITEM && j->container->value.array.length == 0)) { - j->error = "Found ] without final element"; - return NULL; - } - } - - json_object *ret = j->container; - j->container = ret->parent; - return ret; - } - - /////////////////////////// Hashes - - case '{': { - json_object *o = add_object(j, JSON_HASH); - if (o == NULL) { - return NULL; - } - j->container = o; - j->container->expect = JSON_KEY; - - if (cb != NULL) { - cb(JSON_HASH, j, state); - } - - goto again; - } - - case '}': { - if (j->container == NULL) { - j->error = "Found } at top level"; - return NULL; - } - - if (j->container->type != JSON_HASH) { - j->error = "Found } not in a hash"; - return NULL; - } - - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_KEY && j->container->value.object.length == 0)) { - j->error = "Found } without final element"; - return NULL; - } - } - - json_object *ret = j->container; - j->container = ret->parent; - return ret; - } - - /////////////////////////// Null - - case 'n': { - if (read_wrap(j) != 'u' || read_wrap(j) != 'l' || read_wrap(j) != 'l') { - j->error = "Found misspelling of null"; - return NULL; - } - - return add_object(j, JSON_NULL); - } - - /////////////////////////// NaN - - case 'N': { - if (read_wrap(j) != 'a' || read_wrap(j) != 'N') { - j->error = "Found misspelling of NaN"; - return NULL; - } - - j->error = "JSON does not allow NaN"; - return NULL; - } - - /////////////////////////// Infinity - - case 'I': { - if (read_wrap(j) != 'n' || read_wrap(j) != 'f' || read_wrap(j) != 'i' || - read_wrap(j) != 'n' || read_wrap(j) != 'i' || read_wrap(j) != 't' || - read_wrap(j) != 'y') { - j->error = "Found misspelling of Infinity"; - return NULL; - } - - j->error = "JSON does not allow Infinity"; - return NULL; - } - - /////////////////////////// True - - case 't': { - if (read_wrap(j) != 'r' || read_wrap(j) != 'u' || read_wrap(j) != 'e') { - j->error = "Found misspelling of true"; - return NULL; - } - - return add_object(j, JSON_TRUE); - } - - /////////////////////////// False - - case 'f': { - if (read_wrap(j) != 'a' || read_wrap(j) != 'l' || read_wrap(j) != 's' || read_wrap(j) != 'e') { - j->error = "Found misspelling of false"; - return NULL; - } - - return add_object(j, JSON_FALSE); - } - - /////////////////////////// Comma - - case ',': { - if (j->container != NULL) { - if (j->container->expect != JSON_COMMA) { - j->error = "Found unexpected comma"; - return NULL; - } - - if (j->container->type == JSON_HASH) { - j->container->expect = JSON_KEY; - } else { - j->container->expect = JSON_ITEM; - } - } - - if (cb != NULL) { - cb(JSON_COMMA, j, state); - } - - goto again; - } - - /////////////////////////// Colon - - case ':': { - if (j->container == NULL) { - j->error = "Found colon at top level"; - return NULL; - } - - if (j->container->expect != JSON_COLON) { - j->error = "Found unexpected colon"; - return NULL; - } - - j->container->expect = JSON_VALUE; - - if (cb != NULL) { - cb(JSON_COLON, j, state); - } - - goto again; - } - - /////////////////////////// Numbers - - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - j->number_buffer->n = 0; - int decimal = 0; - - if (c == '-') { - string_append(j->number_buffer, c); - c = read_wrap(j); - } - - if (c == '0') { - string_append(j->number_buffer, c); - } else if (c >= '1' && c <= '9') { - string_append(j->number_buffer, c); - c = peek(j); - - while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); - c = peek(j); - } - } - - if (peek(j) == '.') { - string_append(j->number_buffer, read_wrap(j)); - decimal = 1; - - c = peek(j); - if (c < '0' || c > '9') { - j->error = "Decimal point without digits"; - string_free(j->number_buffer); - return NULL; - } - while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); - c = peek(j); - } - } - - c = peek(j); - if (c == 'e' || c == 'E') { - string_append(j->number_buffer, read_wrap(j)); - decimal = 1; - - c = peek(j); - if (c == '+' || c == '-') { - string_append(j->number_buffer, read_wrap(j)); - } - - c = peek(j); - if (c < '0' || c > '9') { - j->error = "Exponent without digits"; - string_free(j->number_buffer); - return NULL; - } - while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); - c = peek(j); - } - } - - json_object *n = add_object(j, JSON_NUMBER); - if (n != NULL) { - n->value.number.number = atof(j->number_buffer->buf); - n->value.number.large_signed = 0; - n->value.number.large_unsigned = 0; - -#define MAX_SAFE_INTEGER 9007199254740991.0 -#define MIN_SAFE_INTEGER -9007199254740991.0 - - if (!decimal && n->value.number.number > MAX_SAFE_INTEGER) { - errno = 0; - char *err = NULL; - unsigned long long ull = strtoull(j->number_buffer->buf, &err, 10); - if (errno == 0 && (err == NULL || *err == '\0')) { - n->value.number.large_unsigned = ull; - } - } - if (!decimal && n->value.number.number < MIN_SAFE_INTEGER) { - errno = 0; - char *err = NULL; - long long ll = strtoll(j->number_buffer->buf, &err, 10); - if (errno == 0 && (err == NULL || *err == '\0')) { - n->value.number.large_signed = ll; - } - } - } - return n; - } - - /////////////////////////// Strings - - case '"': { - struct string val; - string_init(&val); - - int surrogate = -1; - while ((c = read_wrap(j)) != EOF) { - if (c == '"') { - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - break; - } else if (c == '\\') { - c = read_wrap(j); - - if (c == 'u') { - char hex[5] = "aaaa"; - int i; - for (i = 0; i < 4; i++) { - hex[i] = read_wrap(j); - if (hex[i] < '0' || (hex[i] > '9' && hex[i] < 'A') || (hex[i] > 'F' && hex[i] < 'a') || hex[i] > 'f') { - j->error = "Invalid \\u hex character"; - string_free(&val); - return NULL; - } - } - - unsigned long ch = strtoul(hex, NULL, 16); - if (ch >= 0xd800 && ch <= 0xdbff) { - if (surrogate < 0) { - surrogate = ch; - } else { - // Impossible surrogate, so output the first half, - // keep what might be a legitimate new first half. - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = ch; - } - continue; - } else if (ch >= 0xdc00 && c <= 0xdfff) { - if (surrogate >= 0) { - long c1 = surrogate - 0xd800; - long c2 = ch - 0xdc00; - ch = ((c1 << 10) | c2) + 0x010000; - surrogate = -1; - } - } - - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - if (ch <= 0x7F) { - string_append(&val, ch); - } else if (ch <= 0x7FF) { - string_append(&val, 0xC0 | (ch >> 6)); - string_append(&val, 0x80 | (ch & 0x3F)); - } else if (ch < 0xFFFF) { - string_append(&val, 0xE0 | (ch >> 12)); - string_append(&val, 0x80 | ((ch >> 6) & 0x3F)); - string_append(&val, 0x80 | (ch & 0x3F)); - } else { - string_append(&val, 0xF0 | (ch >> 18)); - string_append(&val, 0x80 | ((ch >> 12) & 0x3F)); - string_append(&val, 0x80 | ((ch >> 6) & 0x3F)); - string_append(&val, 0x80 | (ch & 0x3F)); - } - } else { - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - if (c == '"') { - string_append(&val, '"'); - } else if (c == '\\') { - string_append(&val, '\\'); - } else if (c == '/') { - string_append(&val, '/'); - } else if (c == 'b') { - string_append(&val, '\b'); - } else if (c == 'f') { - string_append(&val, '\f'); - } else if (c == 'n') { - string_append(&val, '\n'); - } else if (c == 'r') { - string_append(&val, '\r'); - } else if (c == 't') { - string_append(&val, '\t'); - } else { - j->error = "Found backslash followed by unknown character"; - string_free(&val); - return NULL; - } - } - } else if (c < ' ') { - j->error = "Found control character in string"; - string_free(&val); - return NULL; - } else { - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - string_append(&val, c); - } - } - if (c == EOF) { - j->error = "String without closing quote mark"; - string_free(&val); - return NULL; - } - - json_object *s = add_object(j, JSON_STRING); - if (s != NULL) { - s->value.string.string = val.buf; - s->value.string.refcon = NULL; - } else { - string_free(&val); - } - return s; - } - } - - j->error = "Found unexpected character"; - return NULL; -} - -json_object *json_read(json_pull *j) { - return json_read_separators(j, NULL, NULL); -} - -json_object *json_read_tree(json_pull *p) { - json_object *j; - - while ((j = json_read(p)) != NULL) { - if (j->parent == NULL) { - return j; - } - } - - return NULL; -} - -void json_free(json_object *o) { - size_t i; - - if (o == NULL) { - return; - } - - // Free any data linked from here - - if (o->type == JSON_ARRAY) { - json_object **a = o->value.array.array; - size_t n = o->value.array.length; - - o->value.array.array = NULL; - o->value.array.length = 0; - - for (i = 0; i < n; i++) { - json_free(a[i]); - } - - free(a); - } else if (o->type == JSON_HASH) { - json_object **k = o->value.object.keys; - json_object **v = o->value.object.values; - size_t n = o->value.object.length; - - o->value.object.keys = NULL; - o->value.object.values = NULL; - o->value.object.length = 0; - - for (i = 0; i < n; i++) { - json_free(k[i]); - json_free(v[i]); - } - - free(k); - free(v); - } else if (o->type == JSON_STRING) { - free(o->value.string.string); - } else if (o->type == JSON_NUMBER) { - ; - } - - json_disconnect(o); - - free(o); -} - -static void json_disconnect_parser(json_object *o) { - if (o->type == JSON_HASH) { - size_t i; - for (i = 0; i < o->value.object.length; i++) { - json_disconnect_parser(o->value.object.keys[i]); - json_disconnect_parser(o->value.object.values[i]); - } - } else if (o->type == JSON_ARRAY) { - size_t i; - for (i = 0; i < o->value.array.length; i++) { - json_disconnect_parser(o->value.array.array[i]); - } - } - - o->parser = NULL; -} - -void json_disconnect(json_object *o) { - // Expunge references to this as an array element - // or a hash key or value. - - if (o->parent != NULL) { - if (o->parent->type == JSON_ARRAY) { - size_t i; - - for (i = 0; i < o->parent->value.array.length; i++) { - if (o->parent->value.array.array[i] == o) { - break; - } - } - - if (i < o->parent->value.array.length) { - memmove(o->parent->value.array.array + i, o->parent->value.array.array + i + 1, o->parent->value.array.length - i - 1); - o->parent->value.array.length--; - } - } - - if (o->parent->type == JSON_HASH) { - size_t i; - - for (i = 0; i < o->parent->value.object.length; i++) { - if (o->parent->value.object.keys[i] == o) { - o->parent->value.object.keys[i] = fabricate_object(o->parser, o->parent, JSON_NULL); - break; - } - if (o->parent->value.object.values[i] == o) { - o->parent->value.object.values[i] = fabricate_object(o->parser, o->parent, JSON_NULL); - break; - } - } - - if (i < o->parent->value.object.length) { - if (o->parent->value.object.keys[i] != NULL && o->parent->value.object.keys[i]->type == JSON_NULL) { - if (o->parent->value.object.values[i] != NULL && o->parent->value.object.values[i]->type == JSON_NULL) { - free(o->parent->value.object.keys[i]); - free(o->parent->value.object.values[i]); - - memmove(o->parent->value.object.keys + i, o->parent->value.object.keys + i + 1, o->parent->value.object.length - i - 1); - memmove(o->parent->value.object.values + i, o->parent->value.object.values + i + 1, o->parent->value.object.length - i - 1); - o->parent->value.object.length--; - } - } - } - } - } - - if (o->parser != NULL && o->parser->root == o) { - o->parser->root = NULL; - } - - json_disconnect_parser(o); - o->parent = NULL; -} - -static void json_print_one(struct string *val, json_object *o) { - if (o == NULL) { - string_append_string(val, "..."); - } else if (o->type == JSON_STRING) { - string_append(val, '\"'); - - char *cp; - for (cp = o->value.string.string; *cp != '\0'; cp++) { - if (*cp == '\\' || *cp == '"') { - string_append(val, '\\'); - string_append(val, *cp); - } else if (*cp >= 0 && *cp < ' ') { - char *s; - if (asprintf(&s, "\\u%04x", *cp) >= 0) { - string_append_string(val, s); - free(s); - } - } else { - string_append(val, *cp); - } - } - - string_append(val, '\"'); - } else if (o->type == JSON_NUMBER) { - if (o->value.number.large_signed != 0) { - char s[65]; - sprintf(s, "%lld", o->value.number.large_signed); - string_append_string(val, s); - } else if (o->value.number.large_unsigned != 0) { - char s[65]; - sprintf(s, "%llu", o->value.number.large_unsigned); - string_append_string(val, s); - } else { - char *s = dtoa_milo(o->value.number.number); - string_append_string(val, s); - free(s); - } - } else if (o->type == JSON_NULL) { - string_append_string(val, "null"); - } else if (o->type == JSON_TRUE) { - string_append_string(val, "true"); - } else if (o->type == JSON_FALSE) { - string_append_string(val, "false"); - } else if (o->type == JSON_HASH) { - string_append(val, '}'); - } else if (o->type == JSON_ARRAY) { - string_append(val, ']'); - } -} - -static void json_print(struct string *val, json_object *o) { - if (o == NULL) { - // Hash value in incompletely read hash - string_append_string(val, "..."); - } else if (o->type == JSON_HASH) { - string_append(val, '{'); - - size_t i; - for (i = 0; i < o->value.object.length; i++) { - json_print(val, o->value.object.keys[i]); - string_append(val, ':'); - json_print(val, o->value.object.values[i]); - if (i + 1 < o->value.object.length) { - string_append(val, ','); - } - } - string_append(val, '}'); - } else if (o->type == JSON_ARRAY) { - string_append(val, '['); - size_t i; - for (i = 0; i < o->value.array.length; i++) { - json_print(val, o->value.array.array[i]); - if (i + 1 < o->value.array.length) { - string_append(val, ','); - } - } - string_append(val, ']'); - } else { - json_print_one(val, o); - } -} - -char *json_stringify(json_object *o) { - struct string val; - string_init(&val); - json_print(&val, o); - - return val.buf; -} From 3da03c60752307e72d73fe54ce4cce02926c0f67 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 09:27:37 -0700 Subject: [PATCH 02/13] Convert jsonpull to C++ with shared_ptr and std::vector/std::string Replace the manual malloc/realloc/free memory management in jsonpull with std::shared_ptr ownership. Each json_object now owns its children through std::vector; raw back-pointers to parent and parser remain valid by structural invariant and are cleared on json_disconnect so detached subtrees can outlive their parser. Strings become std::string, child arrays become std::vector, and the old union becomes a struct so non-trivial members can coexist while preserving the existing o->value.xxx access paths. The old jsonpull.c is replaced by jsonpull.cpp, json_stringify now returns std::string, and all callers across tippecanoe, tile-join, tippecanoe-decode, tippecanoe-json-tool, tippecanoe-overzoom and the unit tests are updated to use json_object_ptr / json_pull_ptr. Co-authored-by: Cursor --- Makefile | 2 +- attribute.cpp | 16 +- clip.cpp | 4 +- dirtiles.cpp | 13 +- evaluator.cpp | 172 +++++---- evaluator.hpp | 8 +- geobuf.cpp | 21 +- geojson-loop.cpp | 80 ++--- geojson-loop.hpp | 6 +- geojson.cpp | 78 ++-- geojson.hpp | 10 +- geometry.hpp | 4 +- jsonpull/jsonpull.cpp | 819 ++++++++++++++++++++++++++++++++++++++++++ jsonpull/jsonpull.h | 118 +++--- jsontool.cpp | 86 ++--- main.cpp | 60 ++-- overzoom.cpp | 2 +- plugin.cpp | 81 ++--- plugin.hpp | 2 +- pmtiles_file.cpp | 33 +- read_json.cpp | 94 +++-- read_json.hpp | 8 +- tile-join.cpp | 95 +++-- tile.cpp | 12 +- tile.hpp | 2 +- 25 files changed, 1295 insertions(+), 531 deletions(-) diff --git a/Makefile b/Makefile index 0b9883993..b0a2a0b8d 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ clean: rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check indent: - clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/*.[ch] + clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/jsonpull.h jsonpull/jsonpull.cpp TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) diff --git a/attribute.cpp b/attribute.cpp index 53c9dae52..bf53d399d 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -42,10 +42,10 @@ void set_attribute_accum(std::unordered_map &attribut void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv) { if (*arg == '{') { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: -E%s: %s\n", *argv, arg, jp->error); exit(EXIT_JSON); } @@ -55,9 +55,9 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - json_object *k = o->value.object.keys[i]; - json_object *v = o->value.object.values[i]; + for (size_t i = 0; i < o->value.object.keys.size(); i++) { + json_object_ptr k = o->value.object.keys[i]; + json_object_ptr v = o->value.object.values[i]; if (k->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: key %zu not a string\n", *argv, arg, i); @@ -68,11 +68,9 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - set_attribute_accum(attribute_accum, k->value.string.string, v->value.string.string); + set_attribute_accum(attribute_accum, k->value.string.string.c_str(), v->value.string.string.c_str()); } - json_free(o); - json_end(jp); return; } diff --git a/clip.cpp b/clip.cpp index 72a1fb3a6..a50a7eda6 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1221,7 +1221,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object *filter, bool preserve_input_order, + bool demultiply, json_object_ptr filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, @@ -1457,7 +1457,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object *filter, bool preserve_input_order, + bool demultiply, json_object_ptr filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, diff --git a/dirtiles.cpp b/dirtiles.cpp index 98138bd5a..da7c82af2 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -248,9 +248,9 @@ sqlite3 *dirmeta2tmp(const char *fname) { if (f == NULL) { perror(name.c_str()); } else { - json_pull *jp = json_begin_file(f); - json_object *o = json_read_tree(jp); - if (o == NULL) { + json_pull_ptr jp = json_begin_file(f); + json_object_ptr o = json_read_tree(jp); + if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", name.c_str(), jp->error); exit(EXIT_JSON); } @@ -260,19 +260,18 @@ sqlite3 *dirmeta2tmp(const char *fname) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { + for (size_t i = 0; i < o->value.object.keys.size(); i++) { if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); } - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->value.object.keys[i]->value.string.string, o->value.object.values[i]->value.string.string); + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->value.object.keys[i]->value.string.string.c_str(), o->value.object.values[i]->value.string.string.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->value.object.keys[i]->value.string.string, err); + fprintf(stderr, "set %s in metadata: %s\n", o->value.object.keys[i]->value.string.string.c_str(), err); } sqlite3_free(sql); } - json_end(jp); fclose(f); } diff --git a/evaluator.cpp b/evaluator.cpp index 38566ae23..0d99409c1 100644 --- a/evaluator.cpp +++ b/evaluator.cpp @@ -9,7 +9,7 @@ #include "milo/dtoa_milo.h" #include "text.hpp" -int compare(mvt_value const &one, json_object *two, bool &fail) { +int compare(mvt_value const &one, json_object_ptr two, bool &fail) { switch (one.type) { case mvt_string: if (two->type != JSON_STRING) { @@ -17,7 +17,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { return false; // string vs non-string } - return strcmp(one.c_str(), two->value.string.string); + return strcmp(one.c_str(), two->value.string.string.c_str()); case mvt_double: case mvt_float: @@ -91,8 +91,8 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { // 0: false // 1: true // -1: incomparable (sql null), treated as false in final output -static int eval(std::function feature, json_object *f, std::set &exclude_attributes, std::vector const &unidecode_data) { - if (f != NULL) { +static int eval(std::function feature, json_object_ptr f, std::set &exclude_attributes, std::vector const &unidecode_data) { + if (f != nullptr) { if (f->type == JSON_TRUE) { return 1; } else if (f->type == JSON_FALSE) { @@ -110,7 +110,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_STRING) { - if (f->value.string.string[0] == '\0') { + if (f->value.string.string.empty()) { return 0; } else { return 1; @@ -118,70 +118,70 @@ static int eval(std::function feature, json_obje } } - if (f == NULL || f->type != JSON_ARRAY) { - fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f)); + if (f == nullptr || f->type != JSON_ARRAY) { + fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.length < 1) { - fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + if (f->value.array.array.size() < 1) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } if (f->value.array.array[0]->type != JSON_STRING) { - fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f)); + fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { - if (f->value.array.length != 2) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + const std::string &op = f->value.array.array[0]->value.string.string; + + if (op == "has" || + op == "!has") { + if (f->value.array.array.size() != 2) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0) { + if (op == "has") { if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f)); + fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key; + return feature(f->value.array.array[1]->value.string.string).type != mvt_no_such_key; } - if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { + if (op == "!has") { if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); + fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key; + return feature(f->value.array.array[1]->value.string.string).type == mvt_no_such_key; } } - if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!=") == 0 || - strcmp(f->value.array.array[0]->value.string.string, ">") == 0 || - strcmp(f->value.array.array[0]->value.string.string, ">=") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "<") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) { - if (f->value.array.length != 3) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + if (op == "==" || + op == "!=" || + op == ">" || + op == ">=" || + op == "<" || + op == "<=") { + if (f->value.array.array.size() != 3) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f)); + fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + mvt_value ff = feature(f->value.array.array[1]->value.string.string); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return true; // attributes that aren't found are not equal } return false; // not found: comparison is false @@ -193,56 +193,54 @@ static int eval(std::function feature, json_obje if (fail) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return true; // mismatched types are not equal } return false; } - if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0) { + if (op == "==") { return cmp == 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return cmp != 0; } - if (strcmp(f->value.array.array[0]->value.string.string, ">") == 0) { + if (op == ">") { return cmp > 0; } - if (strcmp(f->value.array.array[0]->value.string.string, ">=") == 0) { + if (op == ">=") { return cmp >= 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "<") == 0) { + if (op == "<") { return cmp < 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) { + if (op == "<=") { return cmp <= 0; } - fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f)); + fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f).c_str()); exit(EXIT_IMPOSSIBLE); } - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "any") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "none") == 0) { + if (op == "all" || + op == "any" || + op == "none") { bool v; - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) { + if (op == "all") { v = true; } else { v = false; } - for (size_t i = 1; i < f->value.array.length; i++) { + for (size_t i = 1; i < f->value.array.array.size(); i++) { int out = eval(feature, f->value.array.array[i], exclude_attributes, unidecode_data); if (out >= 0) { // nulls are ignored in boolean and/or expressions - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) { + if (op == "all") { v = v && out; if (!v) { break; @@ -256,51 +254,47 @@ static int eval(std::function feature, json_obje } } - if (strcmp(f->value.array.array[0]->value.string.string, "none") == 0) { + if (op == "none") { return !v; } else { return v; } } - if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) { - if (f->value.array.length < 2) { - fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + if (op == "in" || + op == "!in") { + if (f->value.array.array.size() < 2) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f)); + fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + mvt_value ff = feature(f->value.array.array[1]->value.string.string); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) { + if (op == "!in") { return true; // attributes that aren't found are not in } return false; // not found: comparison is false } bool found = false; - for (size_t i = 2; i < f->value.array.length; i++) { + for (size_t i = 2; i < f->value.array.array.size(); i++) { bool fail = false; int cmp = compare(ff, f->value.array.array[i], fail); if (fail) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str()); warned = true; } cmp = 1; @@ -312,21 +306,21 @@ static int eval(std::function feature, json_obje } } - if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0) { + if (op == "in") { return found; } else { return !found; } } - if (strcmp(f->value.array.array[0]->value.string.string, "attribute-filter") == 0) { - if (f->value.array.length != 3) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + if (op == "attribute-filter") { + if (f->value.array.array.size() != 3) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f)); + fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } @@ -338,65 +332,63 @@ static int eval(std::function feature, json_obje return true; } - fprintf(stderr, "Unknown filter %s\n", json_stringify(f)); + fprintf(stderr, "Unknown filter %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } -bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { - if (filter == NULL || filter->type != JSON_HASH) { - fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter)); +bool evaluate(std::function feature, std::string const &layer, json_object_ptr filter, std::set &exclude_attributes, std::vector const &unidecode_data) { + if (filter == nullptr || filter->type != JSON_HASH) { + fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter).c_str()); exit(EXIT_JSON); } bool ok = true; - json_object *f; + json_object_ptr f; f = json_hash_get(filter, layer.c_str()); - if (ok && f != NULL) { + if (ok && f != nullptr) { ok = eval(feature, f, exclude_attributes, unidecode_data) > 0; } f = json_hash_get(filter, "*"); - if (ok && f != NULL) { + if (ok && f != nullptr) { ok = eval(feature, f, exclude_attributes, unidecode_data) > 0; } return ok; } -json_object *read_filter(const char *fname) { +json_object_ptr read_filter(const char *fname) { FILE *fp = fopen(fname, "r"); if (fp == NULL) { perror(fname); exit(EXIT_OPEN); } - json_pull *jp = json_begin_file(fp); - json_object *filter = json_read_tree(jp); - if (filter == NULL) { + json_pull_ptr jp = json_begin_file(fp); + json_object_ptr filter = json_read_tree(jp); + if (filter == nullptr) { fprintf(stderr, "%s: %s\n", fname, jp->error); exit(EXIT_JSON); } json_disconnect(filter); - json_end(jp); fclose(fp); return filter; } -json_object *parse_filter(const char *s) { - json_pull *jp = json_begin_string(s); - json_object *filter = json_read_tree(jp); - if (filter == NULL) { +json_object_ptr parse_filter(const char *s) { + json_pull_ptr jp = json_begin_string(s); + json_object_ptr filter = json_read_tree(jp); + if (filter == nullptr) { fprintf(stderr, "Could not parse filter %s\n", s); fprintf(stderr, "%s\n", jp->error); exit(EXIT_JSON); } json_disconnect(filter); - json_end(jp); return filter; } -bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { +bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object_ptr filter, std::set &exclude_attributes, std::vector const &unidecode_data) { std::function getter = [&](std::string const &key) { auto f = feature.find(key); if (f != feature.end()) { @@ -412,7 +404,7 @@ bool evaluate(std::unordered_map const &feature, std::st return evaluate(getter, layer, filter, exclude_attributes, unidecode_data); } -bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data) { +bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object_ptr filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data) { std::function getter = [&](std::string const &key) { const static std::string dollar_id = "$id"; if (key == dollar_id && feat.has_id) { diff --git a/evaluator.hpp b/evaluator.hpp index bc6be00ce..99a91fb76 100644 --- a/evaluator.hpp +++ b/evaluator.hpp @@ -7,10 +7,10 @@ #include "jsonpull/jsonpull.h" #include "mvt.hpp" -bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data); -json_object *parse_filter(const char *s); -json_object *read_filter(const char *fname); +bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object_ptr filter, std::set &exclude_attributes, std::vector const &unidecode_data); +json_object_ptr parse_filter(const char *s); +json_object_ptr read_filter(const char *fname); -bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data); +bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object_ptr filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data); #endif diff --git a/geobuf.cpp b/geobuf.cpp index 02d8ded33..c06b73c9d 100644 --- a/geobuf.cpp +++ b/geobuf.cpp @@ -394,28 +394,25 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vectorsecond.s.c_str()); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(tip->second.s.c_str()); + json_object_ptr o = json_read_tree(jp); - if (o != NULL) { - json_object *min = json_hash_get(o, "minzoom"); - if (min != NULL && (min->type == JSON_NUMBER)) { + if (o != nullptr) { + json_object_ptr min = json_hash_get(o, "minzoom"); + if (min != nullptr && (min->type == JSON_NUMBER)) { sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); } - json_object *max = json_hash_get(o, "maxzoom"); - if (max != NULL && (max->type == JSON_NUMBER)) { + json_object_ptr max = json_hash_get(o, "maxzoom"); + if (max != nullptr && (max->type == JSON_NUMBER)) { sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); } - json_object *tlayer = json_hash_get(o, "layer"); - if (tlayer != NULL && (tlayer->type == JSON_STRING)) { + json_object_ptr tlayer = json_hash_get(o, "layer"); + if (tlayer != nullptr && (tlayer->type == JSON_STRING)) { layername = tlayer->value.string.string; } } - - json_free(o); - json_end(jp); } serialize_feature(sst, sf, layername); diff --git a/geojson-loop.cpp b/geojson-loop.cpp index 199c65315..adc0b0a1c 100644 --- a/geojson-loop.cpp +++ b/geojson-loop.cpp @@ -25,35 +25,35 @@ static const char *geometry_names[GEOM_TYPES] = { }; // XXX duplicated -static void json_context(json_object *j) { - char *s = json_stringify(j); +static void json_context(json_object_ptr j) { + std::string s = json_stringify(j); - if (strlen(s) >= 500) { - snprintf(s + 497, strlen(s) + 1 - 497, "..."); + if (s.size() >= 500) { + s.resize(497); + s.append("..."); } - fprintf(stderr, "in JSON object %s\n", s); - free(s); // stringify + fprintf(stderr, "in JSON object %s\n", s.c_str()); } -void parse_json(json_feature_action *jfa, json_pull *jp) { +void parse_json(json_feature_action *jfa, json_pull_ptr jp) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; while (1) { - json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + json_object_ptr j = json_read(jp); + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "%s:%d: %s: ", jfa->fname.c_str(), jp->line, jp->error); - if (jp->root != NULL) { + if (jp->root != nullptr) { json_context(jp->root); } else { fprintf(stderr, "\n"); } } - json_free(jp->root); + jp->root.reset(); break; } @@ -65,8 +65,8 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } } - json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + json_object_ptr type = json_hash_get(j, "type"); + if (type == nullptr || type->type != JSON_STRING) { continue; } @@ -74,25 +74,25 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { int i; int is_geometry = 0; for (i = 0; i < GEOM_TYPES; i++) { - if (strcmp(type->value.string.string, geometry_names[i]) == 0) { + if (type->value.string.string == geometry_names[i]) { is_geometry = 1; break; } } if (is_geometry) { - if (j->parent != NULL) { - if (j->parent->type == JSON_ARRAY && j->parent->parent != NULL) { + if (j->parent != nullptr) { + if (j->parent->type == JSON_ARRAY && j->parent->parent != nullptr) { if (j->parent->parent->type == JSON_HASH) { - json_object *geometries = json_hash_get(j->parent->parent, "geometries"); - if (geometries != NULL) { + json_object_ptr geometries = json_hash_get(j->parent->parent->shared_from_this(), "geometries"); + if (geometries != nullptr) { // Parent of Parent must be a GeometryCollection is_geometry = 0; } } } else if (j->parent->type == JSON_HASH) { - json_object *geometry = json_hash_get(j->parent, "geometry"); - if (geometry != NULL) { + json_object_ptr geometry = json_hash_get(j->parent->shared_from_this(), "geometry"); + if (geometry != nullptr) { // Parent must be a Feature is_geometry = 0; } @@ -101,10 +101,10 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } if (is_geometry) { - json_object *jo = j; - while (jo != NULL) { - if (jo->parent != NULL && jo->parent->type == JSON_HASH) { - if (json_hash_get(jo->parent, "properties") == jo) { + json_object *jo = j.get(); + while (jo != nullptr) { + if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { + if (json_hash_get(jo->parent->shared_from_this(), "properties").get() == jo) { // Ancestor is the value corresponding to a properties key is_geometry = 0; break; @@ -120,14 +120,14 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } found_geometries++; - jfa->add_feature(j, false, NULL, NULL, NULL, j); + jfa->add_feature(j, false, nullptr, nullptr, nullptr, j); json_free(j); continue; } } - if (strcmp(type->value.string.string, "Feature") != 0) { - if (strcmp(type->value.string.string, "FeatureCollection") == 0) { + if (type->value.string.string != "Feature") { + if (type->value.string.string == "FeatureCollection") { jfa->check_crs(j); json_free(j); } @@ -140,16 +140,16 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } found_features++; - json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + json_object_ptr geometry = json_hash_get(j, "geometry"); + if (geometry == nullptr) { fprintf(stderr, "%s:%d: feature with no geometry: ", jfa->fname.c_str(), jp->line); json_context(j); json_free(j); continue; } - json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + json_object_ptr properties = json_hash_get(j, "properties"); + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "%s:%d: feature without properties hash: ", jfa->fname.c_str(), jp->line); json_context(j); json_free(j); @@ -158,10 +158,10 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { bool is_feature = true; { - json_object *jo = j; - while (jo != NULL) { - if (jo->parent != NULL && jo->parent->type == JSON_HASH) { - if (json_hash_get(jo->parent, "properties") == jo) { + json_object *jo = j.get(); + while (jo != nullptr) { + if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { + if (json_hash_get(jo->parent->shared_from_this(), "properties").get() == jo) { // Ancestor is the value corresponding to a properties key is_feature = false; break; @@ -174,11 +174,11 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { continue; } - json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - json_object *id = json_hash_get(j, "id"); + json_object_ptr tippecanoe = json_hash_get(j, "tippecanoe"); + json_object_ptr id = json_hash_get(j, "id"); - json_object *geometries = json_hash_get(geometry, "geometries"); - if (geometries != NULL && geometries->type == JSON_ARRAY) { + json_object_ptr geometries = json_hash_get(geometry, "geometries"); + if (geometries != nullptr && geometries->type == JSON_ARRAY) { jfa->add_feature(geometries, true, properties, id, tippecanoe, j); } else { jfa->add_feature(geometry, false, properties, id, tippecanoe, j); diff --git a/geojson-loop.hpp b/geojson-loop.hpp index 3d82be8ec..f1b26584b 100644 --- a/geojson-loop.hpp +++ b/geojson-loop.hpp @@ -4,8 +4,8 @@ struct json_feature_action { std::string fname; - virtual int add_feature(json_object *geometry, bool geometrycollection, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) = 0; - virtual void check_crs(json_object *j) = 0; + virtual int add_feature(json_object_ptr geometry, bool geometrycollection, json_object_ptr properties, json_object_ptr id, json_object_ptr tippecanoe, json_object_ptr feature) = 0; + virtual void check_crs(json_object_ptr j) = 0; }; -void parse_json(json_feature_action *action, json_pull *jp); +void parse_json(json_feature_action *action, json_pull_ptr jp); diff --git a/geojson.cpp b/geojson.cpp index 3798d14be..6e287d737 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -40,9 +40,9 @@ #include "milo/dtoa_milo.h" #include "errors.hpp" -int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string const &layername) { - json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { +int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr geometry, json_object_ptr properties, json_object_ptr id, int layer, json_object_ptr tippecanoe, json_object_ptr feature, std::string const &layername) { + json_object_ptr geometry_type = json_hash_get(geometry, "type"); + if (geometry_type == nullptr) { static int warned = 0; if (!warned) { fprintf(stderr, "%s:%d: null geometry (additional not reported): ", sst->fname, sst->line); @@ -59,8 +59,8 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom return 0; } - json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + json_object_ptr coordinates = json_hash_get(geometry, "coordinates"); + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: feature without coordinates array: ", sst->fname, sst->line); json_context(feature); return 0; @@ -68,12 +68,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->value.string.string == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->value.string.string); + fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->value.string.string.c_str()); json_context(feature); return 0; } @@ -82,26 +82,26 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int tippecanoe_maxzoom = -1; std::string tippecanoe_layername = layername; - if (tippecanoe != NULL) { - json_object *min = json_hash_get(tippecanoe, "minzoom"); - if (min != NULL && (min->type == JSON_NUMBER)) { + if (tippecanoe != nullptr) { + json_object_ptr min = json_hash_get(tippecanoe, "minzoom"); + if (min != nullptr && (min->type == JSON_NUMBER)) { tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); } - json_object *max = json_hash_get(tippecanoe, "maxzoom"); - if (max != NULL && (max->type == JSON_NUMBER)) { + json_object_ptr max = json_hash_get(tippecanoe, "maxzoom"); + if (max != nullptr && (max->type == JSON_NUMBER)) { tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); } - json_object *ln = json_hash_get(tippecanoe, "layer"); - if (ln != NULL && (ln->type == JSON_STRING)) { - tippecanoe_layername = std::string(ln->value.string.string); + json_object_ptr ln = json_hash_get(tippecanoe, "layer"); + if (ln != nullptr && (ln->type == JSON_STRING)) { + tippecanoe_layername = ln->value.string.string; } } bool has_id = false; unsigned long long id_value = 0; - if (id != NULL) { + if (id != nullptr) { if (id->type == JSON_NUMBER) { if (id->value.number.number >= 0) { char *err = NULL; @@ -142,20 +142,20 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom if (additional[A_CONVERT_NUMERIC_IDS] && id->type == JSON_STRING) { char *err = NULL; - id_value = strtoull(id->value.string.string, &err, 10); + id_value = strtoull(id->value.string.string.c_str(), &err, 10); if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->value.string.string); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->value.string.string.c_str()); warned_frac = true; } } else if (std::to_string(id_value) != id->value.string.string) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->value.string.string); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->value.string.string.c_str()); warned = true; } } else { @@ -168,9 +168,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom static bool warned_nan = false; if (!warned_nan) { - char *s = json_stringify(id); - fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", s); - free(s); // stringify + fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", json_stringify(id).c_str()); warned_nan = true; } } @@ -178,8 +176,8 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom } size_t nprop = 0; - if (properties != NULL && properties->type == JSON_HASH) { - nprop = properties->value.object.length; + if (properties != nullptr && properties->type == JSON_HASH) { + nprop = properties->value.object.keys.size(); } std::vector> full_keys; @@ -193,7 +191,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom if (properties->value.object.keys[i]->type == JSON_STRING) { serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature); - full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string)); + full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string.c_str())); values.push_back(std::move(sv)); } } @@ -218,16 +216,16 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom return serialize_feature(sst, sf, tippecanoe_layername); } -void check_crs(json_object *j, const char *reading) { - json_object *crs = json_hash_get(j, "crs"); - if (crs != NULL) { - json_object *properties = json_hash_get(crs, "properties"); - if (properties != NULL) { - json_object *name = json_hash_get(properties, "name"); - if (name != NULL && name->type == JSON_STRING) { - if (strcmp(name->value.string.string, projection->alias) != 0) { +void check_crs(json_object_ptr j, const char *reading) { + json_object_ptr crs = json_hash_get(j, "crs"); + if (crs != nullptr) { + json_object_ptr properties = json_hash_get(crs, "properties"); + if (properties != nullptr) { + json_object_ptr name = json_hash_get(properties, "name"); + if (name != nullptr && name->type == JSON_STRING) { + if (name->value.string.string != projection->alias) { if (!quiet) { - fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->value.string.string, projection->alias); + fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->value.string.string.c_str(), projection->alias); fprintf(stderr, "%s: If \"%s\" is not the expected projection, use -s to specify the right one.\n", reading, projection->alias); } } @@ -241,11 +239,11 @@ struct json_serialize_action : json_feature_action { int layer; std::string layername; - int add_feature(json_object *geometry, bool geometrycollection, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) { + int add_feature(json_object_ptr geometry, bool geometrycollection, json_object_ptr properties, json_object_ptr id, json_object_ptr tippecanoe, json_object_ptr feature) { sst->line = geometry->parser->line; if (geometrycollection) { int ret = 1; - for (size_t g = 0; g < geometry->value.array.length; g++) { + for (size_t g = 0; g < geometry->value.array.array.size(); g++) { ret &= serialize_geojson_feature(sst, geometry->value.array.array[g], properties, id, layer, tippecanoe, feature, layername); } return ret; @@ -254,12 +252,12 @@ struct json_serialize_action : json_feature_action { } } - void check_crs(json_object *j) { + void check_crs(json_object_ptr j) { ::check_crs(j, fname.c_str()); } }; -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) { +void parse_json(struct serialization_state *sst, json_pull_ptr jp, int layer, std::string layername) { json_serialize_action jsa; jsa.fname = sst->fname; jsa.sst = sst; @@ -296,7 +294,7 @@ ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) { return n; } -struct json_pull *json_begin_map(char *map, long long len) { +json_pull_ptr json_begin_map(char *map, long long len) { struct jsonmap *jm = new jsonmap; if (jm == NULL) { perror("Out of memory"); @@ -310,7 +308,7 @@ struct json_pull *json_begin_map(char *map, long long len) { return json_begin(json_map_read, jm); } -void json_end_map(struct json_pull *jp) { +void json_end_map(json_pull_ptr jp) { delete (struct jsonmap *) jp->source; json_end(jp); } diff --git a/geojson.hpp b/geojson.hpp index 664ea2e84..aca480082 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -10,21 +10,21 @@ #include "serial.hpp" struct parse_json_args { - json_pull *jp; + json_pull_ptr jp; int layer; std::string *layername; struct serialization_state *sst; - parse_json_args(json_pull *jp1, int layer1, std::string *layername1, struct serialization_state *sst1) + parse_json_args(json_pull_ptr jp1, int layer1, std::string *layername1, struct serialization_state *sst1) : jp(jp1), layer(layer1), layername(layername1), sst(sst1) { } }; -struct json_pull *json_begin_map(char *map, long long len); -void json_end_map(struct json_pull *jp); +json_pull_ptr json_begin_map(char *map, long long len); +void json_end_map(json_pull_ptr jp); -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername); +void parse_json(struct serialization_state *sst, json_pull_ptr jp, int layer, std::string layername); void *run_parse_json(void *v); #endif diff --git a/geometry.hpp b/geometry.hpp index 454cea611..3aab6c189 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -141,7 +141,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object *filter, bool preserve_input_order, + bool demultiply, json_object_ptr filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, @@ -157,7 +157,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object *filter, bool preserve_input_order, + bool demultiply, json_object_ptr filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index e69de29bb..9c061d576 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -0,0 +1,819 @@ +#define _GNU_SOURCE // for asprintf() +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "jsonpull.h" +#include "../milo/milo.h" + +#define BUFFER 10000 + +json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source) { + auto j = std::make_shared(); + j->read = read; + j->source = source; + j->buffer.resize(BUFFER); + return j; +} + +static inline int peek(json_pull *j) { + if (j->buffer_head < j->buffer_tail) { + return (unsigned char) j->buffer[j->buffer_head]; + } else { + j->buffer_head = 0; + j->buffer_tail = j->read(j, j->buffer.data(), BUFFER); + if (j->buffer_head >= j->buffer_tail) { + return EOF; + } + return (unsigned char) j->buffer[j->buffer_head]; + } +} + +static inline int next(json_pull *j) { + if (j->buffer_head < j->buffer_tail) { + return (unsigned char) j->buffer[j->buffer_head++]; + } else { + j->buffer_head = 0; + j->buffer_tail = j->read(j, j->buffer.data(), BUFFER); + if (j->buffer_head >= j->buffer_tail) { + return EOF; + } + return (unsigned char) j->buffer[j->buffer_head++]; + } +} + +static ssize_t read_file(json_pull *j, char *buffer, size_t n) { + return fread(buffer, 1, n, (FILE *) j->source); +} + +json_pull_ptr json_begin_file(FILE *f) { + return json_begin(read_file, f); +} + +static ssize_t read_string(json_pull *j, char *buffer, size_t n) { + const char *cp = (const char *) j->source; + size_t out = 0; + + while (out < n && cp[out] != '\0') { + buffer[out] = cp[out]; + out++; + } + + j->source = (void *) (cp + out); + return out; +} + +json_pull_ptr json_begin_string(const char *s) { + return json_begin(read_string, (void *) s); +} + +void json_end(json_pull_ptr &p) { + p.reset(); +} + +static inline int read_wrap(json_pull *j) { + int c = next(j); + + if (c == '\n') { + j->line++; + } + + return c; +} + +static json_object_ptr fabricate_object(json_pull *jp, json_object *parent, json_type type) { + auto o = std::make_shared(); + o->type = type; + o->parent = parent; + o->parser = jp; + return o; +} + +static json_object_ptr add_object(json_pull *j, json_type type) { + json_object *c = j->container.get(); + json_object_ptr o = fabricate_object(j, c, type); + + if (c != nullptr) { + if (c->type == JSON_ARRAY) { + if (c->expect == JSON_ITEM) { + c->value.array.array.push_back(o); + c->expect = JSON_COMMA; + } else { + j->error = "Expected a comma, not a list item"; + return nullptr; + } + } else if (c->type == JSON_HASH) { + if (c->expect == JSON_VALUE) { + c->value.object.values.back() = o; + c->expect = JSON_COMMA; + } else if (c->expect == JSON_KEY) { + if (type != JSON_STRING) { + j->error = "Hash key is not a string"; + return nullptr; + } + + c->value.object.keys.push_back(o); + c->value.object.values.push_back(nullptr); + c->expect = JSON_COLON; + } else { + j->error = "Expected a comma or colon"; + return nullptr; + } + } + } else { + // Drop the previous top-level value; replacing the parser's root + // shared_ptr will free it if no one else holds a reference. + j->root = o; + } + + return o; +} + +json_object_ptr json_hash_get(json_object_ptr o, const char *s) { + if (o == nullptr || o->type != JSON_HASH) { + return nullptr; + } + + for (size_t i = 0; i < o->value.object.keys.size(); i++) { + const auto &key = o->value.object.keys[i]; + if (key != nullptr && key->type == JSON_STRING) { + if (key->value.string.string == s) { + return o->value.object.values[i]; + } + } + } + + return nullptr; +} + +json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback cb, void *state) { + int c; + json_pull *j = jp.get(); + + // In case there is an error at the top level + if (j->container == nullptr) { + j->root.reset(); + } + +again: + c = read_wrap(j); + if (c == EOF) { + if (j->container != nullptr) { + j->error = "Reached EOF without all containers being closed"; + } + + return nullptr; + } + + switch (c) { + /////////////////////////// Byte order mark + + case 0xEF: { + int c2 = peek(j); + if (c2 == 0xBB) { + c2 = read_wrap(j); + c2 = peek(j); + if (c2 == 0xBF) { + c2 = read_wrap(j); + c = ' '; + goto again; + } + } + j->error = "Corrupt byte-order mark found"; + return nullptr; + } + + /////////////////////////// Whitespace + + case ' ': + case '\t': + case '\r': + case '\n': + case 0x1E: + goto again; + + /////////////////////////// Arrays + + case '[': { + json_object_ptr o = add_object(j, JSON_ARRAY); + if (o == nullptr) { + return nullptr; + } + j->container = o; + j->container->expect = JSON_ITEM; + + if (cb != nullptr) { + cb(JSON_ARRAY, j, state); + } + + goto again; + } + + case ']': { + if (j->container == nullptr) { + j->error = "Found ] at top level"; + return nullptr; + } + + if (j->container->type != JSON_ARRAY) { + j->error = "Found ] not in an array"; + return nullptr; + } + + if (j->container->expect != JSON_COMMA) { + if (!(j->container->expect == JSON_ITEM && j->container->value.array.array.size() == 0)) { + j->error = "Found ] without final element"; + return nullptr; + } + } + + json_object_ptr ret = j->container; + // Walk up to the parent container. The parent (if any) still owns + // `ret` via its own array vector, so the raw `parent` pointer is + // still valid and we can resurrect a shared_ptr to it. + if (ret->parent != nullptr) { + j->container = ret->parent->shared_from_this(); + } else { + j->container.reset(); + } + return ret; + } + + /////////////////////////// Hashes + + case '{': { + json_object_ptr o = add_object(j, JSON_HASH); + if (o == nullptr) { + return nullptr; + } + j->container = o; + j->container->expect = JSON_KEY; + + if (cb != nullptr) { + cb(JSON_HASH, j, state); + } + + goto again; + } + + case '}': { + if (j->container == nullptr) { + j->error = "Found } at top level"; + return nullptr; + } + + if (j->container->type != JSON_HASH) { + j->error = "Found } not in a hash"; + return nullptr; + } + + if (j->container->expect != JSON_COMMA) { + if (!(j->container->expect == JSON_KEY && j->container->value.object.keys.size() == 0)) { + j->error = "Found } without final element"; + return nullptr; + } + } + + json_object_ptr ret = j->container; + if (ret->parent != nullptr) { + j->container = ret->parent->shared_from_this(); + } else { + j->container.reset(); + } + return ret; + } + + /////////////////////////// Null + + case 'n': { + if (read_wrap(j) != 'u' || read_wrap(j) != 'l' || read_wrap(j) != 'l') { + j->error = "Found misspelling of null"; + return nullptr; + } + + return add_object(j, JSON_NULL); + } + + /////////////////////////// NaN + + case 'N': { + if (read_wrap(j) != 'a' || read_wrap(j) != 'N') { + j->error = "Found misspelling of NaN"; + return nullptr; + } + + j->error = "JSON does not allow NaN"; + return nullptr; + } + + /////////////////////////// Infinity + + case 'I': { + if (read_wrap(j) != 'n' || read_wrap(j) != 'f' || read_wrap(j) != 'i' || + read_wrap(j) != 'n' || read_wrap(j) != 'i' || read_wrap(j) != 't' || + read_wrap(j) != 'y') { + j->error = "Found misspelling of Infinity"; + return nullptr; + } + + j->error = "JSON does not allow Infinity"; + return nullptr; + } + + /////////////////////////// True + + case 't': { + if (read_wrap(j) != 'r' || read_wrap(j) != 'u' || read_wrap(j) != 'e') { + j->error = "Found misspelling of true"; + return nullptr; + } + + return add_object(j, JSON_TRUE); + } + + /////////////////////////// False + + case 'f': { + if (read_wrap(j) != 'a' || read_wrap(j) != 'l' || read_wrap(j) != 's' || read_wrap(j) != 'e') { + j->error = "Found misspelling of false"; + return nullptr; + } + + return add_object(j, JSON_FALSE); + } + + /////////////////////////// Comma + + case ',': { + if (j->container != nullptr) { + if (j->container->expect != JSON_COMMA) { + j->error = "Found unexpected comma"; + return nullptr; + } + + if (j->container->type == JSON_HASH) { + j->container->expect = JSON_KEY; + } else { + j->container->expect = JSON_ITEM; + } + } + + if (cb != nullptr) { + cb(JSON_COMMA, j, state); + } + + goto again; + } + + /////////////////////////// Colon + + case ':': { + if (j->container == nullptr) { + j->error = "Found colon at top level"; + return nullptr; + } + + if (j->container->expect != JSON_COLON) { + j->error = "Found unexpected colon"; + return nullptr; + } + + j->container->expect = JSON_VALUE; + + if (cb != nullptr) { + cb(JSON_COLON, j, state); + } + + goto again; + } + + /////////////////////////// Numbers + + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + j->number_buffer.clear(); + int decimal = 0; + + if (c == '-') { + j->number_buffer.push_back(c); + c = read_wrap(j); + } + + if (c == '0') { + j->number_buffer.push_back(c); + } else if (c >= '1' && c <= '9') { + j->number_buffer.push_back(c); + c = peek(j); + + while (c >= '0' && c <= '9') { + j->number_buffer.push_back(read_wrap(j)); + c = peek(j); + } + } + + if (peek(j) == '.') { + j->number_buffer.push_back(read_wrap(j)); + decimal = 1; + + c = peek(j); + if (c < '0' || c > '9') { + j->error = "Decimal point without digits"; + return nullptr; + } + while (c >= '0' && c <= '9') { + j->number_buffer.push_back(read_wrap(j)); + c = peek(j); + } + } + + c = peek(j); + if (c == 'e' || c == 'E') { + j->number_buffer.push_back(read_wrap(j)); + decimal = 1; + + c = peek(j); + if (c == '+' || c == '-') { + j->number_buffer.push_back(read_wrap(j)); + } + + c = peek(j); + if (c < '0' || c > '9') { + j->error = "Exponent without digits"; + return nullptr; + } + while (c >= '0' && c <= '9') { + j->number_buffer.push_back(read_wrap(j)); + c = peek(j); + } + } + + json_object_ptr n = add_object(j, JSON_NUMBER); + if (n != nullptr) { + n->value.number.number = atof(j->number_buffer.c_str()); + n->value.number.large_signed = 0; + n->value.number.large_unsigned = 0; + +#define MAX_SAFE_INTEGER 9007199254740991.0 +#define MIN_SAFE_INTEGER -9007199254740991.0 + + if (!decimal && n->value.number.number > MAX_SAFE_INTEGER) { + errno = 0; + char *err = nullptr; + unsigned long long ull = strtoull(j->number_buffer.c_str(), &err, 10); + if (errno == 0 && (err == nullptr || *err == '\0')) { + n->value.number.large_unsigned = ull; + } + } + if (!decimal && n->value.number.number < MIN_SAFE_INTEGER) { + errno = 0; + char *err = nullptr; + long long ll = strtoll(j->number_buffer.c_str(), &err, 10); + if (errno == 0 && (err == nullptr || *err == '\0')) { + n->value.number.large_signed = ll; + } + } + } + return n; + } + + /////////////////////////// Strings + + case '"': { + std::string val; + + int surrogate = -1; + while ((c = read_wrap(j)) != EOF) { + if (c == '"') { + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + break; + } else if (c == '\\') { + c = read_wrap(j); + + if (c == 'u') { + char hex[5] = "aaaa"; + int i; + for (i = 0; i < 4; i++) { + hex[i] = read_wrap(j); + if (hex[i] < '0' || (hex[i] > '9' && hex[i] < 'A') || (hex[i] > 'F' && hex[i] < 'a') || hex[i] > 'f') { + j->error = "Invalid \\u hex character"; + return nullptr; + } + } + + unsigned long ch = strtoul(hex, nullptr, 16); + if (ch >= 0xd800 && ch <= 0xdbff) { + if (surrogate < 0) { + surrogate = ch; + } else { + // Impossible surrogate, so output the first half, + // keep what might be a legitimate new first half. + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = ch; + } + continue; + } else if (ch >= 0xdc00 && c <= 0xdfff) { + if (surrogate >= 0) { + long c1 = surrogate - 0xd800; + long c2 = ch - 0xdc00; + ch = ((c1 << 10) | c2) + 0x010000; + surrogate = -1; + } + } + + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + if (ch <= 0x7F) { + val.push_back(ch); + } else if (ch <= 0x7FF) { + val.push_back(0xC0 | (ch >> 6)); + val.push_back(0x80 | (ch & 0x3F)); + } else if (ch < 0xFFFF) { + val.push_back(0xE0 | (ch >> 12)); + val.push_back(0x80 | ((ch >> 6) & 0x3F)); + val.push_back(0x80 | (ch & 0x3F)); + } else { + val.push_back(0xF0 | (ch >> 18)); + val.push_back(0x80 | ((ch >> 12) & 0x3F)); + val.push_back(0x80 | ((ch >> 6) & 0x3F)); + val.push_back(0x80 | (ch & 0x3F)); + } + } else { + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + if (c == '"') { + val.push_back('"'); + } else if (c == '\\') { + val.push_back('\\'); + } else if (c == '/') { + val.push_back('/'); + } else if (c == 'b') { + val.push_back('\b'); + } else if (c == 'f') { + val.push_back('\f'); + } else if (c == 'n') { + val.push_back('\n'); + } else if (c == 'r') { + val.push_back('\r'); + } else if (c == 't') { + val.push_back('\t'); + } else { + j->error = "Found backslash followed by unknown character"; + return nullptr; + } + } + } else if (c < ' ') { + j->error = "Found control character in string"; + return nullptr; + } else { + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + val.push_back(c); + } + } + if (c == EOF) { + j->error = "String without closing quote mark"; + return nullptr; + } + + json_object_ptr s = add_object(j, JSON_STRING); + if (s != nullptr) { + s->value.string.string = std::move(val); + s->value.string.refcon = nullptr; + } + return s; + } + } + + j->error = "Found unexpected character"; + return nullptr; +} + +json_object_ptr json_read(json_pull_ptr j) { + return json_read_separators(j, nullptr, nullptr); +} + +json_object_ptr json_read_tree(json_pull_ptr p) { + json_object_ptr j; + + while ((j = json_read(p)) != nullptr) { + if (j->parent == nullptr) { + return j; + } + } + + return nullptr; +} + +void json_free(json_object_ptr &o) { + o.reset(); +} + +// Walk the subtree clearing parent/parser back-pointers so the detached +// subtree can outlive the original parser. +static void clear_back_pointers(json_object *o) { + if (o == nullptr) { + return; + } + + if (o->type == JSON_HASH) { + for (size_t i = 0; i < o->value.object.keys.size(); i++) { + clear_back_pointers(o->value.object.keys[i].get()); + clear_back_pointers(o->value.object.values[i].get()); + } + } else if (o->type == JSON_ARRAY) { + for (size_t i = 0; i < o->value.array.array.size(); i++) { + clear_back_pointers(o->value.array.array[i].get()); + } + } + + o->parent = nullptr; + o->parser = nullptr; +} + +void json_disconnect(json_object_ptr o) { + if (o == nullptr) { + return; + } + + // Splice o out of its parent's array or object. The parent's vector + // holds the shared_ptr to this child; erasing it removes one reference, + // but the caller still holds `o`, so the subtree stays alive. + + json_object *parent = o->parent; + if (parent != nullptr) { + if (parent->type == JSON_ARRAY) { + auto &arr = parent->value.array.array; + for (size_t i = 0; i < arr.size(); i++) { + if (arr[i].get() == o.get()) { + arr.erase(arr.begin() + i); + break; + } + } + } else if (parent->type == JSON_HASH) { + auto &keys = parent->value.object.keys; + auto &vals = parent->value.object.values; + + for (size_t i = 0; i < keys.size(); i++) { + if (keys[i].get() == o.get()) { + // Leave a NULL placeholder in the key slot so the + // surrounding value isn't shifted; if the corresponding + // value is also detached the pair is removed below. + keys[i] = fabricate_object(parent->parser, parent, JSON_NULL); + + if (vals[i] != nullptr && vals[i]->type == JSON_NULL && keys[i]->type == JSON_NULL) { + keys.erase(keys.begin() + i); + vals.erase(vals.begin() + i); + } + break; + } + if (vals[i].get() == o.get()) { + vals[i] = fabricate_object(parent->parser, parent, JSON_NULL); + + if (keys[i] != nullptr && keys[i]->type == JSON_NULL && vals[i]->type == JSON_NULL) { + keys.erase(keys.begin() + i); + vals.erase(vals.begin() + i); + } + break; + } + } + } + } + + // Drop the parser's reference to this subtree if it was the root. + json_pull *parser = o->parser; + if (parser != nullptr && parser->root.get() == o.get()) { + parser->root.reset(); + } + + clear_back_pointers(o.get()); +} + +static void string_append_c(std::string &val, char c) { + val.push_back(c); +} + +static void string_append(std::string &val, const char *add) { + val.append(add); +} + +static void json_print_one(std::string &val, json_object *o) { + if (o == nullptr) { + string_append(val, "..."); + } else if (o->type == JSON_STRING) { + string_append_c(val, '\"'); + + for (const char *cp = o->value.string.string.c_str(); *cp != '\0'; cp++) { + if (*cp == '\\' || *cp == '"') { + string_append_c(val, '\\'); + string_append_c(val, *cp); + } else if (*cp >= 0 && *cp < ' ') { + char *s; + if (asprintf(&s, "\\u%04x", *cp) >= 0) { + string_append(val, s); + free(s); + } + } else { + string_append_c(val, *cp); + } + } + + string_append_c(val, '\"'); + } else if (o->type == JSON_NUMBER) { + if (o->value.number.large_signed != 0) { + char s[65]; + snprintf(s, sizeof(s), "%lld", o->value.number.large_signed); + string_append(val, s); + } else if (o->value.number.large_unsigned != 0) { + char s[65]; + snprintf(s, sizeof(s), "%llu", o->value.number.large_unsigned); + string_append(val, s); + } else { + char *s = dtoa_milo(o->value.number.number); + string_append(val, s); + free(s); + } + } else if (o->type == JSON_NULL) { + string_append(val, "null"); + } else if (o->type == JSON_TRUE) { + string_append(val, "true"); + } else if (o->type == JSON_FALSE) { + string_append(val, "false"); + } else if (o->type == JSON_HASH) { + string_append_c(val, '}'); + } else if (o->type == JSON_ARRAY) { + string_append_c(val, ']'); + } +} + +static void json_print(std::string &val, json_object *o) { + if (o == nullptr) { + // Hash value in incompletely read hash + string_append(val, "..."); + } else if (o->type == JSON_HASH) { + string_append_c(val, '{'); + + for (size_t i = 0; i < o->value.object.keys.size(); i++) { + json_print(val, o->value.object.keys[i].get()); + string_append_c(val, ':'); + json_print(val, o->value.object.values[i].get()); + if (i + 1 < o->value.object.keys.size()) { + string_append_c(val, ','); + } + } + string_append_c(val, '}'); + } else if (o->type == JSON_ARRAY) { + string_append_c(val, '['); + for (size_t i = 0; i < o->value.array.array.size(); i++) { + json_print(val, o->value.array.array[i].get()); + if (i + 1 < o->value.array.array.size()) { + string_append_c(val, ','); + } + } + string_append_c(val, ']'); + } else { + json_print_one(val, o); + } +} + +std::string json_stringify(json_object_ptr o) { + std::string val; + json_print(val, o.get()); + return val; +} diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index b19e0e8e9..3ccb0fae1 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -1,9 +1,11 @@ #ifndef JSONPULL_H #define JSONPULL_H -#ifdef __cplusplus -extern "C" { -#endif +#include +#include +#include +#include +#include typedef enum json_type { // These types can be returned by json_read() @@ -25,74 +27,98 @@ typedef enum json_type { JSON_VALUE, } json_type; -typedef struct json_object { - struct json_object *parent; - struct json_pull *parser; +struct json_object; +struct json_pull; + +typedef std::shared_ptr json_object_ptr; +typedef std::shared_ptr json_pull_ptr; + +// json_object owns its descendants via std::shared_ptr in std::vector<>s, +// and keeps raw back-pointers to its parent and to the parser. The back-pointers +// remain valid as long as the node is attached to the tree (the parent is kept +// alive by holding a shared_ptr to this child, and the parser is kept alive by +// the caller's json_pull_ptr). json_disconnect() splices a node out of its +// parent and clears those back-pointers in the detached subtree, so the +// detached subtree can outlive the original parser. + +struct json_object : public std::enable_shared_from_this { + json_object *parent = nullptr; + json_pull *parser = nullptr; - union { + json_type type = JSON_NULL; + int expect = 0; + + // Members named to match the previous C union layout so that existing + // access paths like `o->value.string.string` and `o->value.array.array[i]` + // continue to work. This is no longer a union because std::string and + // std::vector have non-trivial destructors. + struct value_t { struct { - double number; - unsigned long long large_unsigned; - long long large_signed; + double number = 0; + unsigned long long large_unsigned = 0; + long long large_signed = 0; } number; struct { - char *string; - void *refcon; // reference constant for caller's use + std::string string; + void *refcon = nullptr; // reference constant for caller's use } string; struct { - struct json_object **array; - size_t length; + std::vector array; } array; struct { - struct json_object **keys; - struct json_object **values; - size_t length; + std::vector keys; + std::vector values; } object; } value; +}; - json_type type; - int expect; -} json_object; +struct json_pull { + const char *error = nullptr; // points at a string literal; no allocation + int line = 1; -typedef struct json_pull { - char *error; - int line; + ssize_t (*read)(struct json_pull *, char *buf, size_t n) = nullptr; + void *source = nullptr; + std::vector buffer; + ssize_t buffer_tail = 0; + ssize_t buffer_head = 0; - ssize_t (*read)(struct json_pull *, char *buf, size_t n); - void *source; - char *buffer; - ssize_t buffer_tail; - ssize_t buffer_head; + json_object_ptr container; + json_object_ptr root; - json_object *container; - json_object *root; + std::string number_buffer; +}; - struct string *number_buffer; -} json_pull; +json_pull_ptr json_begin_file(FILE *f); +json_pull_ptr json_begin_string(const char *s); -json_pull *json_begin_file(FILE *f); -json_pull *json_begin_string(const char *s); +json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); -json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); -void json_end(json_pull *p); +// json_end is now a thin convenience that resets the caller's json_pull_ptr. +// The parser (and any tree it still owns) is freed when the last shared_ptr +// to it is dropped, so calling json_end is optional if the json_pull_ptr will +// go out of scope on its own. +void json_end(json_pull_ptr &p); typedef void (*json_separator_callback)(json_type type, json_pull *j, void *state); -json_object *json_read_tree(json_pull *j); -json_object *json_read(json_pull *j); -json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state); -void json_free(json_object *j); -void json_disconnect(json_object *j); +json_object_ptr json_read_tree(json_pull_ptr j); +json_object_ptr json_read(json_pull_ptr j); +json_object_ptr json_read_separators(json_pull_ptr j, json_separator_callback cb, void *state); -json_object *json_hash_get(json_object *o, const char *s); +// json_free now just resets the caller's json_object_ptr. The subtree is +// destroyed when the last shared_ptr to it is dropped (typically by also +// being removed from its parent or parser). +void json_free(json_object_ptr &j); -char *json_stringify(json_object *o); +// Splice o out of its parent's array/object and clear parent/parser back-pointers +// throughout the detached subtree so it can outlive the original parser. +void json_disconnect(json_object_ptr j); -#ifdef __cplusplus -} -#endif +json_object_ptr json_hash_get(json_object_ptr o, const char *s); + +std::string json_stringify(json_object_ptr o); #endif diff --git a/jsontool.cpp b/jsontool.cpp index bb5341b80..5c35206f4 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -140,23 +140,18 @@ std::string sort_quote(const char *s) { return ret; } -void out(std::string const &s, int type, json_object *properties) { +void out(std::string const &s, int type, json_object_ptr properties) { if (extract != NULL) { std::string extracted = sort_quote("null"); bool found = false; - json_object *o = json_hash_get(properties, extract); - if (o != NULL) { + json_object_ptr o = json_hash_get(properties, extract); + if (o != nullptr) { found = true; if (o->type == JSON_STRING || o->type == JSON_NUMBER) { - extracted = sort_quote(o->value.string.string); + extracted = sort_quote(o->value.string.string.c_str()); } else { - // Don't really know what to do about sort quoting - // for arbitrary objects - - const char *out = json_stringify(o); - extracted = sort_quote(out); - free((void *) out); + extracted = sort_quote(json_stringify(o).c_str()); } } @@ -205,7 +200,7 @@ void out(std::string const &s, int type, json_object *properties) { std::string prev_joinkey; -void join_csv(json_object *j) { +void join_csv(json_object_ptr j) { if (header.size() == 0) { std::string s = csv_getline(csvfile); if (s.size() == 0) { @@ -231,14 +226,14 @@ void join_csv(json_object *j) { } } - json_object *properties = json_hash_get(j, "properties"); - json_object *key = NULL; + json_object_ptr properties = json_hash_get(j, "properties"); + json_object_ptr key; - if (properties != NULL) { + if (properties != nullptr) { key = json_hash_get(properties, header[0].c_str()); } - if (key == NULL) { + if (key == nullptr) { static bool warned = false; if (!warned) { fprintf(stderr, "Warning: couldn't find CSV key \"%s\" in JSON\n", header[0].c_str()); @@ -253,9 +248,7 @@ void join_csv(json_object *j) { } else if (key->type == JSON_NUMBER) { joinkey = milo::dtoa_milo(key->value.number.number); } else { - const char *s = json_stringify(key); - joinkey = s; - free((void *) s); + joinkey = json_stringify(key); } if (joinkey < prev_joinkey) { @@ -305,14 +298,8 @@ void join_csv(json_object *j) { } if (fields.size() > 0 && joinkey == fields[0]) { - // This knows more about the structure of JSON objects than it ought to - // The 8 is to round up at least as much as SIZE_FOR in json_pull.c - properties->value.object.keys = (json_object **) realloc((void *) properties->value.object.keys, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *)); - properties->value.object.values = (json_object **) realloc((void *) properties->value.object.values, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *)); - if (properties->value.object.keys == NULL || properties->value.object.values == NULL) { - perror("realloc"); - exit(EXIT_MEMORY); - } + properties->value.object.keys.reserve(properties->value.object.keys.size() + fields.size()); + properties->value.object.values.reserve(properties->value.object.values.size() + fields.size()); for (size_t i = 1; i < fields.size(); i++) { std::string k = header[i]; @@ -330,35 +317,20 @@ void join_csv(json_object *j) { } if (attr_type != JSON_NULL) { - // This knows more about the structure of JSON objects than it ought to - - json_object *ko = (json_object *) malloc(sizeof(json_object)); - json_object *vo = (json_object *) malloc(sizeof(json_object)); - if (ko == NULL || vo == NULL) { - perror("malloc"); - exit(EXIT_MEMORY); - } + auto ko = std::make_shared(); + auto vo = std::make_shared(); ko->type = JSON_STRING; - ko->parent = properties; + ko->parent = properties.get(); ko->parser = properties->parser; - - ko->value.string.string = strdup(k.c_str()); - if (ko->value.string.string == NULL) { - perror("strdup"); - exit(EXIT_MEMORY); - } + ko->value.string.string = k; vo->type = attr_type; - vo->parent = properties; + vo->parent = properties.get(); vo->parser = properties->parser; if (attr_type == JSON_STRING) { - vo->value.string.string = strdup(v.c_str()); - if (vo->value.string.string == NULL) { - perror("strdup"); - exit(EXIT_MEMORY); - } + vo->value.string.string = v; } else if (attr_type == JSON_NUMBER) { vo->value.number.number = atof(v.c_str()); vo->value.number.large_unsigned = 0; @@ -367,44 +339,38 @@ void join_csv(json_object *j) { abort(); } - properties->value.object.keys[properties->value.object.length] = ko; - properties->value.object.values[properties->value.object.length] = vo; - properties->value.object.length++; + properties->value.object.keys.push_back(ko); + properties->value.object.values.push_back(vo); } } } } struct json_join_action : json_feature_action { - int add_feature(json_object *geometry, bool, json_object *, json_object *, json_object *, json_object *feature) { + int add_feature(json_object_ptr geometry, bool, json_object_ptr, json_object_ptr, json_object_ptr, json_object_ptr feature) { if (feature != geometry) { // a real feature, not a bare geometry if (csvfile != NULL) { join_csv(feature); } - char *s = json_stringify(feature); - out(s, 1, json_hash_get(feature, "properties")); - free(s); + out(json_stringify(feature), 1, json_hash_get(feature, "properties")); } else { - char *s = json_stringify(geometry); - out(s, 2, NULL); - free(s); + out(json_stringify(geometry), 2, nullptr); } return 1; } - void check_crs(json_object *) { + void check_crs(json_object_ptr) { } }; void process(FILE *fp, const char *fname) { - json_pull *jp = json_begin_file(fp); + json_pull_ptr jp = json_begin_file(fp); json_join_action jja; jja.fname = fname; parse_json(&jja, jp); - json_end(jp); } int main(int argc, char **argv) { diff --git a/main.cpp b/main.cpp index 1f8b8f832..1fc7d4977 100644 --- a/main.cpp +++ b/main.cpp @@ -586,7 +586,7 @@ struct STREAM { } } - json_pull *json_begin() { + json_pull_ptr json_begin() { return ::json_begin(read_stream, this); } }; @@ -1215,7 +1215,7 @@ double round_droprate(double r) { return std::round(r * 100000.0) / 100000.0; } -std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { +std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object_ptr filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { int ret = EXIT_SUCCESS; std::vector readers; @@ -1818,7 +1818,7 @@ std::pair read_input(std::vector &sources, char *fname, i // Plain serial reading std::atomic layer_seq(overall_offset); - json_pull *jp = fp->json_begin(); + json_pull_ptr jp = fp->json_begin(); struct serialization_state sst; sst.fname = reading.c_str(); @@ -1845,7 +1845,6 @@ std::pair read_input(std::vector &sources, char *fname, i sst.attribute_types = attribute_types; parse_json(&sst, jp, layer, sources[layer].layer); - json_end(jp); overall_offset = layer_seq; checkdisk(&readers); } @@ -2873,10 +2872,10 @@ void set_attribute_type(std::unordered_map &attribute_types, c void set_attribute_value(const char *arg) { if (*arg == '{') { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: --set-attribute %s: %s\n", *av, arg, jp->error); exit(EXIT_JSON); } @@ -2886,9 +2885,9 @@ void set_attribute_value(const char *arg) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - json_object *k = o->value.object.keys[i]; - json_object *v = o->value.object.values[i]; + for (size_t i = 0; i < o->value.object.keys.size(); i++) { + json_object_ptr k = o->value.object.keys[i]; + json_object_ptr v = o->value.object.values[i]; if (k->type != JSON_STRING) { fprintf(stderr, "%s: --set-attribute %s: key %zu not a string\n", *av, arg, i); @@ -2899,8 +2898,6 @@ void set_attribute_value(const char *arg) { set_attributes.emplace(k->value.string.string, val); } - json_free(o); - json_end(jp); return; } @@ -2925,10 +2922,10 @@ void set_attribute_value(const char *arg) { } void parse_json_source(const char *arg, struct source &src) { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: -L%s: %s\n", *av, arg, jp->error); exit(EXIT_JSON); } @@ -2938,31 +2935,28 @@ void parse_json_source(const char *arg, struct source &src) { exit(EXIT_JSON); } - json_object *fname = json_hash_get(o, "file"); - if (fname == NULL || fname->type != JSON_STRING) { + json_object_ptr fname = json_hash_get(o, "file"); + if (fname == nullptr || fname->type != JSON_STRING) { fprintf(stderr, "%s: -L%s: requires \"file\": filename\n", *av, arg); exit(EXIT_JSON); } - src.file = std::string(fname->value.string.string); + src.file = fname->value.string.string; - json_object *layer = json_hash_get(o, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - src.layer = std::string(layer->value.string.string); + json_object_ptr layer = json_hash_get(o, "layer"); + if (layer != nullptr && layer->type == JSON_STRING) { + src.layer = layer->value.string.string; } - json_object *description = json_hash_get(o, "description"); - if (description != NULL && description->type == JSON_STRING) { - src.description = std::string(description->value.string.string); + json_object_ptr description = json_hash_get(o, "description"); + if (description != nullptr && description->type == JSON_STRING) { + src.description = description->value.string.string; } - json_object *format = json_hash_get(o, "format"); - if (format != NULL && format->type == JSON_STRING) { - src.format = std::string(format->value.string.string); + json_object_ptr format = json_hash_get(o, "format"); + if (format != nullptr && format->type == JSON_STRING) { + src.format = format->value.string.string; } - - json_free(o); - json_end(jp); } int main(int argc, char **argv) { @@ -3008,7 +3002,7 @@ int main(int argc, char **argv) { int exclude_all = 0; int read_parallel = 0; int files_open_at_start; - json_object *filter = NULL; + json_object_ptr filter; memsize = calc_memsize(); @@ -3876,9 +3870,7 @@ int main(int argc, char **argv) { exit(EXIT_IMPOSSIBLE); } - if (filter != NULL) { - json_free(filter); - } + filter.reset(); return ret; } diff --git a/overzoom.cpp b/overzoom.cpp index cdef1dd82..b0b16910f 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -238,7 +238,7 @@ int main(int argc, char **argv) { std::string out; { - json_object *json_filter = NULL; + json_object_ptr json_filter; if (filter.size() > 0) { json_filter = parse_filter(filter.c_str()); } diff --git a/plugin.cpp b/plugin.cpp index cfe065938..b90fbfe71 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -27,9 +27,7 @@ #include "errors.hpp" #include "thread.hpp" -extern "C" { #include "jsonpull/jsonpull.h" -} #include "plugin.hpp" #include "write_json.hpp" @@ -145,15 +143,15 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } // Reads from the prefilter -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { +serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { serial_feature sf; while (1) { - json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + json_object_ptr j = json_read(jp); + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); - if (jp->root != NULL) { + if (jp->root != nullptr) { json_context(jp->root); } else { fprintf(stderr, "\n"); @@ -161,37 +159,35 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: exit(EXIT_JSON); } - json_free(jp->root); + jp->root.reset(); sf.t = -1; return sf; } - json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + json_object_ptr type = json_hash_get(j, "type"); + if (type == nullptr || type->type != JSON_STRING) { continue; } - if (strcmp(type->value.string.string, "Feature") != 0) { + if (type->value.string.string != "Feature") { continue; } - json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + json_object_ptr geometry = json_hash_get(j, "geometry"); + if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } - json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + json_object_ptr properties = json_hash_get(j, "properties"); + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } - json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + json_object_ptr geometry_type = json_hash_get(geometry, "type"); + if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); exit(EXIT_JSON); @@ -203,8 +199,8 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: exit(EXIT_JSON); } - json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + json_object_ptr coordinates = json_hash_get(geometry, "coordinates"); + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: feature without coordinates array: ", jp->line); json_context(j); exit(EXIT_JSON); @@ -212,12 +208,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->value.string.string == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string.c_str()); json_context(j); exit(EXIT_JSON); } @@ -252,30 +248,30 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.has_id = false; std::string layername = "unknown"; - json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - if (tippecanoe != NULL) { - json_object *layer = json_hash_get(tippecanoe, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - layername = std::string(layer->value.string.string); + json_object_ptr tippecanoe = json_hash_get(j, "tippecanoe"); + if (tippecanoe != nullptr) { + json_object_ptr layer = json_hash_get(tippecanoe, "layer"); + if (layer != nullptr && layer->type == JSON_STRING) { + layername = layer->value.string.string; } - json_object *index = json_hash_get(tippecanoe, "index"); - if (index != NULL && index->type == JSON_NUMBER) { + json_object_ptr index = json_hash_get(tippecanoe, "index"); + if (index != nullptr && index->type == JSON_NUMBER) { sf.index = index->value.number.number; } - json_object *sequence = json_hash_get(tippecanoe, "sequence"); - if (sequence != NULL && sequence->type == JSON_NUMBER) { + json_object_ptr sequence = json_hash_get(tippecanoe, "sequence"); + if (sequence != nullptr && sequence->type == JSON_NUMBER) { sf.seq = sequence->value.number.number; } - json_object *extent = json_hash_get(tippecanoe, "extent"); - if (extent != NULL && extent->type == JSON_NUMBER) { + json_object_ptr extent = json_hash_get(tippecanoe, "extent"); + if (extent != nullptr && extent->type == JSON_NUMBER) { sf.extent = extent->value.number.number; } - json_object *dropped = json_hash_get(tippecanoe, "dropped"); - if (dropped != NULL && dropped->type == JSON_TRUE) { + json_object_ptr dropped = json_hash_get(tippecanoe, "dropped"); + if (dropped != nullptr && dropped->type == JSON_TRUE) { sf.dropped = FEATURE_DROPPED; // dropped } else { sf.dropped = FEATURE_KEPT; // kept @@ -299,8 +295,8 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } } - json_object *id = json_hash_get(j, "id"); - if (id != NULL && id->type == JSON_NUMBER) { + json_object_ptr id = json_hash_get(j, "id"); + if (id != nullptr && id->type == JSON_NUMBER) { sf.id = id->value.number.number; if (id->value.number.large_unsigned > 0) { sf.id = id->value.number.large_unsigned; @@ -347,27 +343,24 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } } - for (size_t i = 0; i < properties->value.object.length; i++) { + for (size_t i = 0; i < properties->value.object.keys.size(); i++) { serial_val v = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); // Nulls can be excluded here because the expression evaluation filter // would have already run before prefiltering if (v.type != mvt_null) { - sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string))); + sf.full_keys.push_back(key_pool.pool(properties->value.object.keys[i]->value.string.string)); sf.full_values.push_back(v); if (!postfilter) { - add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), v); + add_to_tilestats(ts->second.tilestats, properties->value.object.keys[i]->value.string.string, v); } } } - json_free(j); return sf; } - - json_free(j); } } diff --git a/plugin.hpp b/plugin.hpp index fc901daf1..2a1feab62 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,4 +1,4 @@ struct key_pool; std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); +serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index e86e538b5..499d6ceeb 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -397,9 +397,9 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { exit(EXIT_OPEN); } - json_pull *jp = json_begin_string(decompressed_json.c_str()); - json_object *o = json_read_tree(jp); - if (o == NULL) { + json_pull_ptr jp = json_begin_string(decompressed_json.c_str()); + json_object_ptr o = json_read_tree(jp); + if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", fname, jp->error); exit(EXIT_JSON); } @@ -415,45 +415,44 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { state.nospace = true; state.json_write_hash(); - for (size_t i = 0; i < o->value.object.length; i++) { - const char *key = o->value.object.keys[i]->value.string.string; - if (strcmp(key, "vector_layers") == 0 && o->value.object.values[i]->type == JSON_ARRAY) { + for (size_t i = 0; i < o->value.object.keys.size(); i++) { + const std::string &key = o->value.object.keys[i]->value.string.string; + if (key == "vector_layers" && o->value.object.values[i]->type == JSON_ARRAY) { has_json = true; state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (strcmp(key, "tilestats") == 0 && o->value.object.values[i]->type == JSON_HASH) { + } else if (key == "tilestats" && o->value.object.values[i]->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (strcmp(key, "strategies") == 0 && o->value.object.values[i]->type == JSON_ARRAY) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->value.object.values[i])); + } else if (key == "strategies" && o->value.object.values[i]->type == JSON_ARRAY) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->value.object.values[i]).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (strcmp(key, "tippecanoe_decisions") == 0 && o->value.object.values[i]->type == JSON_HASH) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i])); + } else if (key == "tippecanoe_decisions" && o->value.object.values[i]->type == JSON_HASH) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i]).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } else if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { - fprintf(stderr, "%s\n", key); + fprintf(stderr, "%s\n", key.c_str()); fprintf(stderr, "%s: non-string in metadata\n", fname); } else { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key, o->value.object.values[i]->value.string.string); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), o->value.object.values[i]->value.string.string.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } } - json_end(jp); state.nospace = true; state.json_end_hash(); diff --git a/read_json.cpp b/read_json.cpp index 63329acb8..08ff2f147 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -42,19 +42,19 @@ int mb_geometry[GEOM_TYPES] = { VT_POLYGON, }; -void json_context(json_object *j) { - char *s = json_stringify(j); +void json_context(json_object_ptr j) { + std::string s = json_stringify(j); - if (strlen(s) >= 500) { - snprintf(s + 497, strlen(s) + 1 - 497, "..."); + if (s.size() >= 500) { + s.resize(497); + s.append("..."); } - fprintf(stderr, "in JSON object %s\n", s); - free(s); // stringify + fprintf(stderr, "in JSON object %s\n", s.c_str()); } -void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature) { - if (j == NULL || j->type != JSON_ARRAY) { +void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const char *fname, int line, json_object_ptr feature) { + if (j == nullptr || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for geometry type %d: ", fname, line, t); json_context(feature); return; @@ -63,7 +63,7 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * int within = geometry_within[t]; if (within >= 0) { size_t i; - for (i = 0; i < j->value.array.length; i++) { + for (i = 0; i < j->value.array.array.size(); i++) { if (within == GEOM_POINT) { if (i == 0 || mb_geometry[t] == VT_POINT) { op = VT_MOVETO; @@ -75,13 +75,13 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * parse_coordinates(within, j->value.array.array[i], out, op, fname, line, feature); } } else { - if (j->value.array.length >= 2 && j->value.array.array[0]->type == JSON_NUMBER && j->value.array.array[1]->type == JSON_NUMBER) { + if (j->value.array.array.size() >= 2 && j->value.array.array[0]->type == JSON_NUMBER && j->value.array.array[1]->type == JSON_NUMBER) { long long x, y; double lon = j->value.array.array[0]->value.number.number; double lat = j->value.array.array[1]->value.number.number; projection->project(lon, lat, 32, &x, &y); - if (j->value.array.length > 2) { + if (j->value.array.array.size() > 2) { static int warned = 0; if (!warned) { @@ -121,10 +121,10 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * // type and stringified value. All numeric values, even if they are integers, // even integers that are too large to fit in a double but will still be // stringified with their original precision, are recorded here as mvt_double. -serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature) { +serial_val stringify_value(json_object_ptr value, const char *reading, int line, json_object_ptr feature) { serial_val sv; - if (value != NULL) { + if (value != nullptr) { int vt = value->type; if (vt == JSON_STRING) { @@ -158,9 +158,7 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js sv.s = "null"; } else { sv.type = mvt_string; - const char *v = json_stringify(value); - sv.s = std::string(v); - free((void *) v); // stringify + sv.s = json_stringify(value); } } @@ -178,10 +176,10 @@ static std::vector to_feature(drawvec &geom) { return out; } -std::pair parse_geometry(json_object *geometry, json_pull *jp, json_object *j, +std::pair parse_geometry(json_object_ptr geometry, json_pull_ptr jp, json_object_ptr j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style) { - json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + json_object_ptr geometry_type = json_hash_get(geometry, "type"); + if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); exit(EXIT_JSON); @@ -193,8 +191,8 @@ std::pair parse_geometry(json_object *geometry, json_pull *jp, jso exit(EXIT_JSON); } - json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + json_object_ptr coordinates = json_hash_get(geometry, "coordinates"); + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: geometry without coordinates array: ", jp->line); json_context(j); exit(EXIT_JSON); @@ -202,12 +200,12 @@ std::pair parse_geometry(json_object *geometry, json_pull *jp, jso int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->value.string.string == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string.c_str()); json_context(j); exit(EXIT_JSON); } @@ -305,13 +303,13 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int std::map ret; std::shared_ptr tile_stringpool = std::make_shared(); - json_pull *jp = json_begin_file(fp); + json_pull_ptr jp = json_begin_file(fp); while (1) { - json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + json_object_ptr j = json_read(jp); + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); - if (jp->root != NULL) { + if (jp->root != nullptr) { json_context(jp->root); } else { fprintf(stderr, "\n"); @@ -319,33 +317,32 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int exit(EXIT_JSON); } - json_free(jp->root); + jp->root.reset(); break; } - json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + json_object_ptr type = json_hash_get(j, "type"); + if (type == nullptr || type->type != JSON_STRING) { continue; } - if (strcmp(type->value.string.string, "Feature") != 0) { + if (type->value.string.string != "Feature") { continue; } - json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + json_object_ptr properties = json_hash_get(j, "properties"); + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } std::string layername = "unknown"; - json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - json_object *layer = NULL; - if (tippecanoe != NULL) { + json_object_ptr tippecanoe = json_hash_get(j, "tippecanoe"); + json_object_ptr layer; + if (tippecanoe != nullptr) { layer = json_hash_get(tippecanoe, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - layername = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + layername = layer->value.string.string; } } @@ -359,11 +356,10 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int } auto l = ret.find(layername); - json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + json_object_ptr geometry = json_hash_get(j, "geometry"); + if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } @@ -377,8 +373,8 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int feature.type = mb_geometry[t]; feature.geometry = to_feature(dv); - json_object *id = json_hash_get(j, "id"); - if (id != NULL && id->type == JSON_NUMBER) { + json_object_ptr id = json_hash_get(j, "id"); + if (id != nullptr && id->type == JSON_NUMBER) { feature.id = id->value.number.number; if (id->value.number.large_unsigned > 0) { feature.id = id->value.number.large_unsigned; @@ -386,7 +382,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int feature.has_id = true; } - for (size_t i = 0; i < properties->value.object.length; i++) { + for (size_t i = 0; i < properties->value.object.keys.size(); i++) { serial_val sv = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); // Nulls can be excluded here because this is the postfilter @@ -394,18 +390,14 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int if (sv.type != mvt_null) { mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v); + l->second.tag(feature, properties->value.object.keys[i]->value.string.string, v); } } l->second.features.push_back(feature); } - - json_free(j); } - json_end(jp); - std::vector final; for (auto a : ret) { final.push_back(a.second); diff --git a/read_json.hpp b/read_json.hpp index a5d5d7b28..4b8f9fb83 100644 --- a/read_json.hpp +++ b/read_json.hpp @@ -10,10 +10,10 @@ extern const char *geometry_names[GEOM_TYPES]; extern int geometry_within[GEOM_TYPES]; extern int mb_geometry[GEOM_TYPES]; -void json_context(json_object *j); -void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature); -std::pair parse_geometry(json_object *geometry, json_pull *jp, json_object *j, +void json_context(json_object_ptr j); +void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const char *fname, int line, json_object_ptr feature); +std::pair parse_geometry(json_object_ptr geometry, json_pull_ptr jp, json_object_ptr j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style); std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int extent, bool fix_longitudes); -serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature); +serial_val stringify_value(json_object_ptr value, const char *reading, int line, json_object_ptr feature); diff --git a/tile-join.cpp b/tile-join.cpp index 1b057577e..24d96167e 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -89,7 +89,7 @@ struct arg { std::set *keep_layers = NULL; std::set *remove_layers = NULL; int ifmatched = 0; - json_object *filter = NULL; + json_object_ptr filter; struct tileset_reader *readers = NULL; double minlat, minlon; @@ -97,7 +97,7 @@ struct arg { double minlon2, maxlon2; }; -void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, sqlite3 * /* db */, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter, struct arg *a) { +void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, sqlite3 * /* db */, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object_ptr filter, struct arg *a) { mvt_tile tile; int features_added = 0; bool was_compressed; @@ -891,7 +891,7 @@ void *join_worker(void *v) { return NULL; } -void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) { +void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object_ptr filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) { pthread_t pthreads[CPUS]; std::vector args; @@ -965,16 +965,16 @@ void dispatch_tasks(std::map> &tasks, std::vector< } void handle_strategies(const unsigned char *s, std::vector *st) { - json_pull *jp = json_begin_string((const char *) s); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string((const char *) s); + json_object_ptr o = json_read_tree(jp); - if (o != NULL && o->type == JSON_ARRAY) { - for (size_t i = 0; i < o->value.array.length; i++) { - json_object *h = o->value.array.array[i]; + if (o != nullptr && o->type == JSON_ARRAY) { + for (size_t i = 0; i < o->value.array.array.size(); i++) { + json_object_ptr h = o->value.array.array[i]; if (h->type == JSON_HASH) { - for (size_t j = 0; j < h->value.object.length; j++) { - json_object *k = h->value.object.keys[j]; - json_object *v = h->value.object.values[j]; + for (size_t j = 0; j < h->value.object.keys.size(); j++) { + json_object_ptr k = h->value.object.keys[j]; + json_object_ptr v = h->value.object.values[j]; if (k->type != JSON_STRING) { fprintf(stderr, "Key %zu of %zu is not a string: %s\n", j, i, s); @@ -985,23 +985,24 @@ void handle_strategies(const unsigned char *s, std::vector *st) { st->resize(i + 1); } - if (strcmp(k->value.string.string, "dropped_by_rate") == 0) { + const std::string &key = k->value.string.string; + if (key == "dropped_by_rate") { (*st)[i].dropped_by_rate += v->value.number.number; - } else if (strcmp(k->value.string.string, "dropped_by_gamma") == 0) { + } else if (key == "dropped_by_gamma") { (*st)[i].dropped_by_gamma += v->value.number.number; - } else if (strcmp(k->value.string.string, "dropped_as_needed") == 0) { + } else if (key == "dropped_as_needed") { (*st)[i].dropped_as_needed += v->value.number.number; - } else if (strcmp(k->value.string.string, "coalesced_as_needed") == 0) { + } else if (key == "coalesced_as_needed") { (*st)[i].coalesced_as_needed += v->value.number.number; - } else if (strcmp(k->value.string.string, "truncated_zooms") == 0) { + } else if (key == "truncated_zooms") { (*st)[i].truncated_zooms += v->value.number.number; - } else if (strcmp(k->value.string.string, "detail_reduced") == 0) { + } else if (key == "detail_reduced") { (*st)[i].detail_reduced += v->value.number.number; - } else if (strcmp(k->value.string.string, "tiny_polygons") == 0) { + } else if (key == "tiny_polygons") { (*st)[i].tiny_polygons += v->value.number.number; - } else if (strcmp(k->value.string.string, "tile_size_desired") == 0) { + } else if (key == "tile_size_desired") { (*st)[i].tile_size += v->value.number.number; - } else if (strcmp(k->value.string.string, "feature_count_desired") == 0) { + } else if (key == "feature_count_desired") { (*st)[i].feature_count += v->value.number.number; } } @@ -1010,22 +1011,19 @@ void handle_strategies(const unsigned char *s, std::vector *st) { fprintf(stderr, "Element %zu is not a hash: %s\n", i, s); } } - json_free(o); } - - json_end(jp); } -void handle_vector_layers(json_object *vector_layers, std::map &layermap, std::map &attribute_descriptions) { - if (vector_layers != NULL && vector_layers->type == JSON_ARRAY) { - for (size_t i = 0; i < vector_layers->value.array.length; i++) { +void handle_vector_layers(json_object_ptr vector_layers, std::map &layermap, std::map &attribute_descriptions) { + if (vector_layers != nullptr && vector_layers->type == JSON_ARRAY) { + for (size_t i = 0; i < vector_layers->value.array.array.size(); i++) { if (vector_layers->value.array.array[i]->type == JSON_HASH) { - json_object *id = json_hash_get(vector_layers->value.array.array[i], "id"); - json_object *desc = json_hash_get(vector_layers->value.array.array[i], "description"); + json_object_ptr id = json_hash_get(vector_layers->value.array.array[i], "id"); + json_object_ptr desc = json_hash_get(vector_layers->value.array.array[i], "description"); - if (id != NULL && desc != NULL && id->type == JSON_STRING && desc->type == JSON_STRING) { - std::string sid = id->value.string.string; - std::string sdesc = desc->value.string.string; + if (id != nullptr && desc != nullptr && id->type == JSON_STRING && desc->type == JSON_STRING) { + const std::string &sid = id->value.string.string; + const std::string &sdesc = desc->value.string.string; if (sdesc.size() != 0) { auto f = layermap.find(sid); @@ -1035,16 +1033,16 @@ void handle_vector_layers(json_object *vector_layers, std::mapvalue.array.array[i], "fields"); - if (fields != NULL && fields->type == JSON_HASH) { - for (size_t j = 0; j < fields->value.object.length; j++) { + json_object_ptr fields = json_hash_get(vector_layers->value.array.array[i], "fields"); + if (fields != nullptr && fields->type == JSON_HASH) { + for (size_t j = 0; j < fields->value.object.keys.size(); j++) { if (fields->value.object.keys[j]->type == JSON_STRING && fields->value.object.values[j]->type) { - const char *desc2 = fields->value.object.values[j]->value.string.string; + const std::string &desc2 = fields->value.object.values[j]->value.string.string; - if (strcmp(desc2, "Number") != 0 && - strcmp(desc2, "String") != 0 && - strcmp(desc2, "Boolean") != 0 && - strcmp(desc2, "Mixed") != 0) { + if (desc2 != "Number" && + desc2 != "String" && + desc2 != "Boolean" && + desc2 != "Mixed") { attribute_descriptions.insert(std::pair(fields->value.object.keys[j]->value.string.string, desc2)); } } @@ -1055,7 +1053,7 @@ void handle_vector_layers(json_object *vector_layers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { +void decode(struct tileset_reader *readers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object_ptr filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -1205,17 +1203,14 @@ void decode(struct tileset_reader *readers, std::maptype == JSON_HASH) { - json_object *vector_layers = json_hash_get(o, "vector_layers"); + if (o != nullptr && o->type == JSON_HASH) { + json_object_ptr vector_layers = json_hash_get(o, "vector_layers"); handle_vector_layers(vector_layers, layermap, attribute_descriptions); - json_free(o); } - - json_end(jp); } } @@ -1266,7 +1261,7 @@ int main(int argc, char **argv) { int force = 0; int ifmatched = 0; int filearg = 0; - json_object *filter = NULL; + json_object_ptr filter; std::string join_sqlite_fname; @@ -1648,9 +1643,7 @@ int main(int argc, char **argv) { mbtiles_close(outdb, argv[0]); } - if (filter != NULL) { - json_free(filter); - } + filter.reset(); if (pmtiles_has_suffix(out_mbtiles)) { mbtiles_map_image_to_pmtiles(out_mbtiles, m, !pC, quiet, false); diff --git a/tile.cpp b/tile.cpp index a4f12c758..309d2e929 100644 --- a/tile.cpp +++ b/tile.cpp @@ -941,7 +941,7 @@ struct write_tile_args { bool still_dropping = false; int wrote_zoom = 0; size_t tiling_seg = 0; - json_object *filter = NULL; + json_object_ptr filter; std::vector const *unidecode_data; std::atomic *dropped_count = NULL; atomic_strategy *strategy = NULL; @@ -1102,7 +1102,7 @@ struct next_feature_state { // This function is called repeatedly from write_tile() to retrieve the next feature // from the input stream. If the stream is at an end, it returns a feature with the // geometry type set to -2. -static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { +static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object_ptr filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate); while (1) { @@ -1350,7 +1350,7 @@ struct run_prefilter_args { char *global_stringpool = NULL; long long *pool_off = NULL; FILE *prefilter_fp = NULL; - json_object *filter = NULL; + json_object_ptr filter; std::vector const *unidecode_data; bool first_time = false; bool compressed = false; @@ -1641,7 +1641,7 @@ void skip_tile(decompressor *geoms, std::atomic *geompos_in, bool com } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object_ptr filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -1775,7 +1775,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch pthread_t prefilter_writer; run_prefilter_args rpa; // here so it stays in scope until joined FILE *prefilter_read_fp = NULL; - json_pull *prefilter_jp = NULL; + json_pull_ptr prefilter_jp; if (z < minzoom) { prefilter = NULL; @@ -3213,7 +3213,7 @@ exit(EXIT_IMPOSSIBLE); return err_or_null; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object_ptr filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending) { last_progress = 0; // The existing layermaps are one table per input thread. diff --git a/tile.hpp b/tile.hpp index 8f266c0b2..e9c7d42a2 100644 --- a/tile.hpp +++ b/tile.hpp @@ -62,7 +62,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object_ptr filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); From f366b2c4aa95882e55be16f8ddb766cb016857c2 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 09:47:22 -0700 Subject: [PATCH 03/13] Subclass json_object so primitives shrink from 168 to 24 bytes The previous "every member in a struct" layout cost 168 bytes per json_object, even for JSON_NULL / JSON_TRUE / JSON_FALSE nodes that have no payload. Splitting json_object into a small base class plus json_number / json_string / json_array / json_hash subclasses brings each instance down to just the size of its actual contents: json_object (base, TRUE / FALSE / NULL) 24 bytes json_number 48 bytes json_string 48 bytes json_array (empty) 48 bytes json_hash (empty) 72 bytes Other size wins along the way: * Drop enable_shared_from_this (its embedded weak_ptr was 16 bytes per node). json_pull now keeps an explicit container_stack and the parser no longer needs to resurrect a shared_ptr from a raw `parent` walk. * Remove the unused `refcon` slot from the string variant. * No virtual destructor: shared_ptr keeps the deleter from the original std::make_shared call, so destroying a shared_ptr still runs the right subclass dtor. The base class exposes type-tagged accessors (o->string(), o->number(), o->array(), o->keys(), o->values(), o->large_signed(), o->large_unsigned()) that assert the type matches and downcast to the appropriate subclass storage. All call sites were swept from the old `o->value.X.Y` field paths to these accessors. A raw-pointer overload of json_hash_get() replaces the few external uses of shared_from_this() that survived in geojson-loop.cpp. Co-authored-by: Cursor --- attribute.cpp | 8 +- dirtiles.cpp | 8 +- evaluator.cpp | 56 ++++++------ geobuf.cpp | 6 +- geojson-loop.cpp | 14 +-- geojson.cpp | 50 +++++------ jsonpull/jsonpull.cpp | 195 ++++++++++++++++++++++++------------------ jsonpull/jsonpull.h | 186 ++++++++++++++++++++++++++++++++-------- jsontool.cpp | 38 ++++---- main.cpp | 16 ++-- plugin.cpp | 28 +++--- pmtiles_file.cpp | 24 +++--- read_json.cpp | 44 +++++----- tile-join.cpp | 52 +++++------ 14 files changed, 429 insertions(+), 296 deletions(-) diff --git a/attribute.cpp b/attribute.cpp index bf53d399d..2794e5b79 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -55,9 +55,9 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - json_object_ptr k = o->value.object.keys[i]; - json_object_ptr v = o->value.object.values[i]; + for (size_t i = 0; i < o->keys().size(); i++) { + json_object_ptr k = o->keys()[i]; + json_object_ptr v = o->values()[i]; if (k->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: key %zu not a string\n", *argv, arg, i); @@ -68,7 +68,7 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - set_attribute_accum(attribute_accum, k->value.string.string.c_str(), v->value.string.string.c_str()); + set_attribute_accum(attribute_accum, k->string().c_str(), v->string().c_str()); } return; diff --git a/dirtiles.cpp b/dirtiles.cpp index da7c82af2..de3f86d48 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -260,14 +260,14 @@ sqlite3 *dirmeta2tmp(const char *fname) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { + for (size_t i = 0; i < o->keys().size(); i++) { + if (o->keys()[i]->type != JSON_STRING || o->values()[i]->type != JSON_STRING) { fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); } - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->value.object.keys[i]->value.string.string.c_str(), o->value.object.values[i]->value.string.string.c_str()); + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys()[i]->string().c_str(), o->values()[i]->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->value.object.keys[i]->value.string.string.c_str(), err); + fprintf(stderr, "set %s in metadata: %s\n", o->keys()[i]->string().c_str(), err); } sqlite3_free(sql); } diff --git a/evaluator.cpp b/evaluator.cpp index 0d99409c1..efd27746b 100644 --- a/evaluator.cpp +++ b/evaluator.cpp @@ -17,7 +17,7 @@ int compare(mvt_value const &one, json_object_ptr two, bool &fail) { return false; // string vs non-string } - return strcmp(one.c_str(), two->value.string.string.c_str()); + return strcmp(one.c_str(), two->string().c_str()); case mvt_double: case mvt_float: @@ -52,9 +52,9 @@ int compare(mvt_value const &one, json_object_ptr two, bool &fail) { exit(EXIT_IMPOSSIBLE); } - if (v < two->value.number.number) { + if (v < two->number()) { return -1; - } else if (v > two->value.number.number) { + } else if (v > two->number()) { return 1; } else { return 0; @@ -102,7 +102,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_NUMBER) { - if (f->value.number.number == 0) { + if (f->number() == 0) { return 0; } else { return 1; @@ -110,7 +110,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_STRING) { - if (f->value.string.string.empty()) { + if (f->string().empty()) { return 0; } else { return 1; @@ -123,39 +123,39 @@ static int eval(std::function feature, json_obje exit(EXIT_FILTER); } - if (f->value.array.array.size() < 1) { + if (f->array().size() < 1) { fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[0]->type != JSON_STRING) { + if (f->array()[0]->type != JSON_STRING) { fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - const std::string &op = f->value.array.array[0]->value.string.string; + const std::string &op = f->array()[0]->string(); if (op == "has" || op == "!has") { - if (f->value.array.array.size() != 2) { + if (f->array().size() != 2) { fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } if (op == "has") { - if (f->value.array.array[1]->type != JSON_STRING) { + if (f->array()[1]->type != JSON_STRING) { fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(f->value.array.array[1]->value.string.string).type != mvt_no_such_key; + return feature(f->array()[1]->string()).type != mvt_no_such_key; } if (op == "!has") { - if (f->value.array.array[1]->type != JSON_STRING) { + if (f->array()[1]->type != JSON_STRING) { fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(f->value.array.array[1]->value.string.string).type == mvt_no_such_key; + return feature(f->array()[1]->string()).type == mvt_no_such_key; } } @@ -165,16 +165,16 @@ static int eval(std::function feature, json_obje op == ">=" || op == "<" || op == "<=") { - if (f->value.array.array.size() != 3) { + if (f->array().size() != 3) { fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { + if (f->array()[1]->type != JSON_STRING) { fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(f->value.array.array[1]->value.string.string); + mvt_value ff = feature(f->array()[1]->string()); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { @@ -188,7 +188,7 @@ static int eval(std::function feature, json_obje } bool fail = false; - int cmp = compare(ff, f->value.array.array[2], fail); + int cmp = compare(ff, f->array()[2], fail); if (fail) { static bool warned = false; @@ -236,8 +236,8 @@ static int eval(std::function feature, json_obje v = false; } - for (size_t i = 1; i < f->value.array.array.size(); i++) { - int out = eval(feature, f->value.array.array[i], exclude_attributes, unidecode_data); + for (size_t i = 1; i < f->array().size(); i++) { + int out = eval(feature, f->array()[i], exclude_attributes, unidecode_data); if (out >= 0) { // nulls are ignored in boolean and/or expressions if (op == "all") { @@ -263,17 +263,17 @@ static int eval(std::function feature, json_obje if (op == "in" || op == "!in") { - if (f->value.array.array.size() < 2) { + if (f->array().size() < 2) { fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { + if (f->array()[1]->type != JSON_STRING) { fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(f->value.array.array[1]->value.string.string); + mvt_value ff = feature(f->array()[1]->string()); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { @@ -287,9 +287,9 @@ static int eval(std::function feature, json_obje } bool found = false; - for (size_t i = 2; i < f->value.array.array.size(); i++) { + for (size_t i = 2; i < f->array().size(); i++) { bool fail = false; - int cmp = compare(ff, f->value.array.array[i], fail); + int cmp = compare(ff, f->array()[i], fail); if (fail) { static bool warned = false; @@ -314,19 +314,19 @@ static int eval(std::function feature, json_obje } if (op == "attribute-filter") { - if (f->value.array.array.size() != 3) { + if (f->array().size() != 3) { fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { + if (f->array()[1]->type != JSON_STRING) { fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - bool ok = eval(feature, f->value.array.array[2], exclude_attributes, unidecode_data) > 0; + bool ok = eval(feature, f->array()[2], exclude_attributes, unidecode_data) > 0; if (!ok) { - exclude_attributes.insert(f->value.array.array[1]->value.string.string); + exclude_attributes.insert(f->array()[1]->string()); } return true; diff --git a/geobuf.cpp b/geobuf.cpp index c06b73c9d..721c675cc 100644 --- a/geobuf.cpp +++ b/geobuf.cpp @@ -400,17 +400,17 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vectortype == JSON_NUMBER)) { - sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); + sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } json_object_ptr max = json_hash_get(o, "maxzoom"); if (max != nullptr && (max->type == JSON_NUMBER)) { - sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); + sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } json_object_ptr tlayer = json_hash_get(o, "layer"); if (tlayer != nullptr && (tlayer->type == JSON_STRING)) { - layername = tlayer->value.string.string; + layername = tlayer->string(); } } } diff --git a/geojson-loop.cpp b/geojson-loop.cpp index adc0b0a1c..75149e94a 100644 --- a/geojson-loop.cpp +++ b/geojson-loop.cpp @@ -74,7 +74,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { int i; int is_geometry = 0; for (i = 0; i < GEOM_TYPES; i++) { - if (type->value.string.string == geometry_names[i]) { + if (type->string() == geometry_names[i]) { is_geometry = 1; break; } @@ -84,14 +84,14 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { if (j->parent != nullptr) { if (j->parent->type == JSON_ARRAY && j->parent->parent != nullptr) { if (j->parent->parent->type == JSON_HASH) { - json_object_ptr geometries = json_hash_get(j->parent->parent->shared_from_this(), "geometries"); + json_object_ptr geometries = json_hash_get(j->parent->parent, "geometries"); if (geometries != nullptr) { // Parent of Parent must be a GeometryCollection is_geometry = 0; } } } else if (j->parent->type == JSON_HASH) { - json_object_ptr geometry = json_hash_get(j->parent->shared_from_this(), "geometry"); + json_object_ptr geometry = json_hash_get(j->parent, "geometry"); if (geometry != nullptr) { // Parent must be a Feature is_geometry = 0; @@ -104,7 +104,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { json_object *jo = j.get(); while (jo != nullptr) { if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { - if (json_hash_get(jo->parent->shared_from_this(), "properties").get() == jo) { + if (json_hash_get(jo->parent, "properties").get() == jo) { // Ancestor is the value corresponding to a properties key is_geometry = 0; break; @@ -126,8 +126,8 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { } } - if (type->value.string.string != "Feature") { - if (type->value.string.string == "FeatureCollection") { + if (type->string() != "Feature") { + if (type->string() == "FeatureCollection") { jfa->check_crs(j); json_free(j); } @@ -161,7 +161,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { json_object *jo = j.get(); while (jo != nullptr) { if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { - if (json_hash_get(jo->parent->shared_from_this(), "properties").get() == jo) { + if (json_hash_get(jo->parent, "properties").get() == jo) { // Ancestor is the value corresponding to a properties key is_feature = false; break; diff --git a/geojson.cpp b/geojson.cpp index 6e287d737..4d16088fc 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -68,12 +68,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g int t; for (t = 0; t < GEOM_TYPES; t++) { - if (geometry_type->value.string.string == geometry_names[t]) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->value.string.string.c_str()); + fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->string().c_str()); json_context(feature); return 0; } @@ -85,17 +85,17 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g if (tippecanoe != nullptr) { json_object_ptr min = json_hash_get(tippecanoe, "minzoom"); if (min != nullptr && (min->type == JSON_NUMBER)) { - tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); + tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } json_object_ptr max = json_hash_get(tippecanoe, "maxzoom"); if (max != nullptr && (max->type == JSON_NUMBER)) { - tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); + tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } json_object_ptr ln = json_hash_get(tippecanoe, "layer"); if (ln != nullptr && (ln->type == JSON_STRING)) { - tippecanoe_layername = ln->value.string.string; + tippecanoe_layername = ln->string(); } } @@ -103,27 +103,27 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g unsigned long long id_value = 0; if (id != nullptr) { if (id->type == JSON_NUMBER) { - if (id->value.number.number >= 0) { + if (id->number() >= 0) { char *err = NULL; - std::string id_number = milo::dtoa_milo(id->value.number.number); + std::string id_number = milo::dtoa_milo(id->number()); id_value = strtoull(id_number.c_str(), &err, 10); - if (id->value.number.large_unsigned != 0) { - id_value = id->value.number.large_unsigned; + if (id->large_unsigned() != 0) { + id_value = id->large_unsigned(); } if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned_frac = true; } - } else if (id->value.number.large_unsigned == 0 && std::to_string(id_value) != milo::dtoa_milo(id->value.number.number)) { + } else if (id->large_unsigned() == 0 && std::to_string(id_value) != milo::dtoa_milo(id->number())) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned = true; } } else { @@ -133,7 +133,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g static bool warned_neg = false; if (!warned_neg) { - fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned_neg = true; } } @@ -142,20 +142,20 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g if (additional[A_CONVERT_NUMERIC_IDS] && id->type == JSON_STRING) { char *err = NULL; - id_value = strtoull(id->value.string.string.c_str(), &err, 10); + id_value = strtoull(id->string().c_str(), &err, 10); if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->value.string.string.c_str()); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->string().c_str()); warned_frac = true; } - } else if (std::to_string(id_value) != id->value.string.string) { + } else if (std::to_string(id_value) != id->string()) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->value.string.string.c_str()); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->string().c_str()); warned = true; } } else { @@ -177,7 +177,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g size_t nprop = 0; if (properties != nullptr && properties->type == JSON_HASH) { - nprop = properties->value.object.keys.size(); + nprop = properties->keys().size(); } std::vector> full_keys; @@ -188,10 +188,10 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g key_pool key_pool; for (size_t i = 0; i < nprop; i++) { - if (properties->value.object.keys[i]->type == JSON_STRING) { - serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature); + if (properties->keys()[i]->type == JSON_STRING) { + serial_val sv = stringify_value(properties->values()[i], sst->fname, sst->line, feature); - full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string.c_str())); + full_keys.emplace_back(key_pool.pool(properties->keys()[i]->string().c_str())); values.push_back(std::move(sv)); } } @@ -223,9 +223,9 @@ void check_crs(json_object_ptr j, const char *reading) { if (properties != nullptr) { json_object_ptr name = json_hash_get(properties, "name"); if (name != nullptr && name->type == JSON_STRING) { - if (name->value.string.string != projection->alias) { + if (name->string() != projection->alias) { if (!quiet) { - fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->value.string.string.c_str(), projection->alias); + fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->string().c_str(), projection->alias); fprintf(stderr, "%s: If \"%s\" is not the expected projection, use -s to specify the right one.\n", reading, projection->alias); } } @@ -243,8 +243,8 @@ struct json_serialize_action : json_feature_action { sst->line = geometry->parser->line; if (geometrycollection) { int ret = 1; - for (size_t g = 0; g < geometry->value.array.array.size(); g++) { - ret &= serialize_geojson_feature(sst, geometry->value.array.array[g], properties, id, layer, tippecanoe, feature, layername); + for (size_t g = 0; g < geometry->array().size(); g++) { + ret &= serialize_geojson_feature(sst, geometry->array()[g], properties, id, layer, tippecanoe, feature, layername); } return ret; } else { diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 9c061d576..d35bf8d39 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -86,22 +86,47 @@ static inline int read_wrap(json_pull *j) { return c; } -static json_object_ptr fabricate_object(json_pull *jp, json_object *parent, json_type type) { - auto o = std::make_shared(); - o->type = type; - o->parent = parent; - o->parser = jp; +// Construct an instance of the right subclass for the given type. +// JSON_TRUE / JSON_FALSE / JSON_NULL and the parse-token types are bare +// json_objects; the value-bearing types each get their own subclass. +static json_object_ptr make_object(json_type type, json_object *parent, json_pull *jp) { + json_object_ptr o; + switch (type) { + case JSON_NUMBER: + o = std::make_shared(parent, jp); + break; + case JSON_STRING: + o = std::make_shared(parent, jp); + break; + case JSON_ARRAY: + o = std::make_shared(parent, jp); + break; + case JSON_HASH: + o = std::make_shared(parent, jp); + break; + default: + o = std::make_shared(type, parent, jp); + break; + } return o; } +static json_object_ptr fabricate_object(json_pull *jp, json_object *parent, json_type type) { + return make_object(type, parent, jp); +} + +static inline json_object *current_container(json_pull *j) { + return j->container_stack.empty() ? nullptr : j->container_stack.back().get(); +} + static json_object_ptr add_object(json_pull *j, json_type type) { - json_object *c = j->container.get(); - json_object_ptr o = fabricate_object(j, c, type); + json_object *c = current_container(j); + json_object_ptr o = make_object(type, c, j); if (c != nullptr) { if (c->type == JSON_ARRAY) { if (c->expect == JSON_ITEM) { - c->value.array.array.push_back(o); + c->array().push_back(o); c->expect = JSON_COMMA; } else { j->error = "Expected a comma, not a list item"; @@ -109,7 +134,7 @@ static json_object_ptr add_object(json_pull *j, json_type type) { } } else if (c->type == JSON_HASH) { if (c->expect == JSON_VALUE) { - c->value.object.values.back() = o; + c->values().back() = o; c->expect = JSON_COMMA; } else if (c->expect == JSON_KEY) { if (type != JSON_STRING) { @@ -117,8 +142,8 @@ static json_object_ptr add_object(json_pull *j, json_type type) { return nullptr; } - c->value.object.keys.push_back(o); - c->value.object.values.push_back(nullptr); + c->keys().push_back(o); + c->values().push_back(nullptr); c->expect = JSON_COLON; } else { j->error = "Expected a comma or colon"; @@ -134,16 +159,18 @@ static json_object_ptr add_object(json_pull *j, json_type type) { return o; } -json_object_ptr json_hash_get(json_object_ptr o, const char *s) { +json_object_ptr json_hash_get(json_object *o, const char *s) { if (o == nullptr || o->type != JSON_HASH) { return nullptr; } - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - const auto &key = o->value.object.keys[i]; + const auto &keys = o->keys(); + const auto &vals = o->values(); + for (size_t i = 0; i < keys.size(); i++) { + const auto &key = keys[i]; if (key != nullptr && key->type == JSON_STRING) { - if (key->value.string.string == s) { - return o->value.object.values[i]; + if (key->string() == s) { + return vals[i]; } } } @@ -151,19 +178,23 @@ json_object_ptr json_hash_get(json_object_ptr o, const char *s) { return nullptr; } +json_object_ptr json_hash_get(json_object_ptr o, const char *s) { + return json_hash_get(o.get(), s); +} + json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback cb, void *state) { int c; json_pull *j = jp.get(); // In case there is an error at the top level - if (j->container == nullptr) { + if (j->container_stack.empty()) { j->root.reset(); } again: c = read_wrap(j); if (c == EOF) { - if (j->container != nullptr) { + if (!j->container_stack.empty()) { j->error = "Reached EOF without all containers being closed"; } @@ -204,8 +235,8 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c if (o == nullptr) { return nullptr; } - j->container = o; - j->container->expect = JSON_ITEM; + o->expect = JSON_ITEM; + j->container_stack.push_back(o); if (cb != nullptr) { cb(JSON_ARRAY, j, state); @@ -215,32 +246,26 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } case ']': { - if (j->container == nullptr) { + json_object *cc = current_container(j); + if (cc == nullptr) { j->error = "Found ] at top level"; return nullptr; } - if (j->container->type != JSON_ARRAY) { + if (cc->type != JSON_ARRAY) { j->error = "Found ] not in an array"; return nullptr; } - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_ITEM && j->container->value.array.array.size() == 0)) { + if (cc->expect != JSON_COMMA) { + if (!(cc->expect == JSON_ITEM && cc->array().size() == 0)) { j->error = "Found ] without final element"; return nullptr; } } - json_object_ptr ret = j->container; - // Walk up to the parent container. The parent (if any) still owns - // `ret` via its own array vector, so the raw `parent` pointer is - // still valid and we can resurrect a shared_ptr to it. - if (ret->parent != nullptr) { - j->container = ret->parent->shared_from_this(); - } else { - j->container.reset(); - } + json_object_ptr ret = j->container_stack.back(); + j->container_stack.pop_back(); return ret; } @@ -251,8 +276,8 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c if (o == nullptr) { return nullptr; } - j->container = o; - j->container->expect = JSON_KEY; + o->expect = JSON_KEY; + j->container_stack.push_back(o); if (cb != nullptr) { cb(JSON_HASH, j, state); @@ -262,29 +287,26 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } case '}': { - if (j->container == nullptr) { + json_object *cc = current_container(j); + if (cc == nullptr) { j->error = "Found } at top level"; return nullptr; } - if (j->container->type != JSON_HASH) { + if (cc->type != JSON_HASH) { j->error = "Found } not in a hash"; return nullptr; } - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_KEY && j->container->value.object.keys.size() == 0)) { + if (cc->expect != JSON_COMMA) { + if (!(cc->expect == JSON_KEY && cc->keys().size() == 0)) { j->error = "Found } without final element"; return nullptr; } } - json_object_ptr ret = j->container; - if (ret->parent != nullptr) { - j->container = ret->parent->shared_from_this(); - } else { - j->container.reset(); - } + json_object_ptr ret = j->container_stack.back(); + j->container_stack.pop_back(); return ret; } @@ -350,16 +372,17 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c /////////////////////////// Comma case ',': { - if (j->container != nullptr) { - if (j->container->expect != JSON_COMMA) { + json_object *cc = current_container(j); + if (cc != nullptr) { + if (cc->expect != JSON_COMMA) { j->error = "Found unexpected comma"; return nullptr; } - if (j->container->type == JSON_HASH) { - j->container->expect = JSON_KEY; + if (cc->type == JSON_HASH) { + cc->expect = JSON_KEY; } else { - j->container->expect = JSON_ITEM; + cc->expect = JSON_ITEM; } } @@ -373,17 +396,18 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c /////////////////////////// Colon case ':': { - if (j->container == nullptr) { + json_object *cc = current_container(j); + if (cc == nullptr) { j->error = "Found colon at top level"; return nullptr; } - if (j->container->expect != JSON_COLON) { + if (cc->expect != JSON_COLON) { j->error = "Found unexpected colon"; return nullptr; } - j->container->expect = JSON_VALUE; + cc->expect = JSON_VALUE; if (cb != nullptr) { cb(JSON_COLON, j, state); @@ -463,27 +487,27 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c json_object_ptr n = add_object(j, JSON_NUMBER); if (n != nullptr) { - n->value.number.number = atof(j->number_buffer.c_str()); - n->value.number.large_signed = 0; - n->value.number.large_unsigned = 0; + n->number() = atof(j->number_buffer.c_str()); + n->large_signed() = 0; + n->large_unsigned() = 0; #define MAX_SAFE_INTEGER 9007199254740991.0 #define MIN_SAFE_INTEGER -9007199254740991.0 - if (!decimal && n->value.number.number > MAX_SAFE_INTEGER) { + if (!decimal && n->number() > MAX_SAFE_INTEGER) { errno = 0; char *err = nullptr; unsigned long long ull = strtoull(j->number_buffer.c_str(), &err, 10); if (errno == 0 && (err == nullptr || *err == '\0')) { - n->value.number.large_unsigned = ull; + n->large_unsigned() = ull; } } - if (!decimal && n->value.number.number < MIN_SAFE_INTEGER) { + if (!decimal && n->number() < MIN_SAFE_INTEGER) { errno = 0; char *err = nullptr; long long ll = strtoll(j->number_buffer.c_str(), &err, 10); if (errno == 0 && (err == nullptr || *err == '\0')) { - n->value.number.large_signed = ll; + n->large_signed() = ll; } } } @@ -614,8 +638,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c json_object_ptr s = add_object(j, JSON_STRING); if (s != nullptr) { - s->value.string.string = std::move(val); - s->value.string.refcon = nullptr; + s->string() = std::move(val); } return s; } @@ -653,13 +676,16 @@ static void clear_back_pointers(json_object *o) { } if (o->type == JSON_HASH) { - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - clear_back_pointers(o->value.object.keys[i].get()); - clear_back_pointers(o->value.object.values[i].get()); + const auto &keys = o->keys(); + const auto &vals = o->values(); + for (size_t i = 0; i < keys.size(); i++) { + clear_back_pointers(keys[i].get()); + clear_back_pointers(vals[i].get()); } } else if (o->type == JSON_ARRAY) { - for (size_t i = 0; i < o->value.array.array.size(); i++) { - clear_back_pointers(o->value.array.array[i].get()); + const auto &arr = o->array(); + for (size_t i = 0; i < arr.size(); i++) { + clear_back_pointers(arr[i].get()); } } @@ -679,7 +705,7 @@ void json_disconnect(json_object_ptr o) { json_object *parent = o->parent; if (parent != nullptr) { if (parent->type == JSON_ARRAY) { - auto &arr = parent->value.array.array; + auto &arr = parent->array(); for (size_t i = 0; i < arr.size(); i++) { if (arr[i].get() == o.get()) { arr.erase(arr.begin() + i); @@ -687,8 +713,8 @@ void json_disconnect(json_object_ptr o) { } } } else if (parent->type == JSON_HASH) { - auto &keys = parent->value.object.keys; - auto &vals = parent->value.object.values; + auto &keys = parent->keys(); + auto &vals = parent->values(); for (size_t i = 0; i < keys.size(); i++) { if (keys[i].get() == o.get()) { @@ -739,7 +765,7 @@ static void json_print_one(std::string &val, json_object *o) { } else if (o->type == JSON_STRING) { string_append_c(val, '\"'); - for (const char *cp = o->value.string.string.c_str(); *cp != '\0'; cp++) { + for (const char *cp = o->string().c_str(); *cp != '\0'; cp++) { if (*cp == '\\' || *cp == '"') { string_append_c(val, '\\'); string_append_c(val, *cp); @@ -756,16 +782,16 @@ static void json_print_one(std::string &val, json_object *o) { string_append_c(val, '\"'); } else if (o->type == JSON_NUMBER) { - if (o->value.number.large_signed != 0) { + if (o->large_signed() != 0) { char s[65]; - snprintf(s, sizeof(s), "%lld", o->value.number.large_signed); + snprintf(s, sizeof(s), "%lld", o->large_signed()); string_append(val, s); - } else if (o->value.number.large_unsigned != 0) { + } else if (o->large_unsigned() != 0) { char s[65]; - snprintf(s, sizeof(s), "%llu", o->value.number.large_unsigned); + snprintf(s, sizeof(s), "%llu", o->large_unsigned()); string_append(val, s); } else { - char *s = dtoa_milo(o->value.number.number); + char *s = dtoa_milo(o->number()); string_append(val, s); free(s); } @@ -789,20 +815,23 @@ static void json_print(std::string &val, json_object *o) { } else if (o->type == JSON_HASH) { string_append_c(val, '{'); - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - json_print(val, o->value.object.keys[i].get()); + const auto &keys = o->keys(); + const auto &vals = o->values(); + for (size_t i = 0; i < keys.size(); i++) { + json_print(val, keys[i].get()); string_append_c(val, ':'); - json_print(val, o->value.object.values[i].get()); - if (i + 1 < o->value.object.keys.size()) { + json_print(val, vals[i].get()); + if (i + 1 < keys.size()) { string_append_c(val, ','); } } string_append_c(val, '}'); } else if (o->type == JSON_ARRAY) { string_append_c(val, '['); - for (size_t i = 0; i < o->value.array.array.size(); i++) { - json_print(val, o->value.array.array[i].get()); - if (i + 1 < o->value.array.array.size()) { + const auto &arr = o->array(); + for (size_t i = 0; i < arr.size(); i++) { + json_print(val, arr[i].get()); + if (i + 1 < arr.size()) { string_append_c(val, ','); } } diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index 3ccb0fae1..fa9b265f5 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -33,48 +34,152 @@ struct json_pull; typedef std::shared_ptr json_object_ptr; typedef std::shared_ptr json_pull_ptr; -// json_object owns its descendants via std::shared_ptr in std::vector<>s, -// and keeps raw back-pointers to its parent and to the parser. The back-pointers -// remain valid as long as the node is attached to the tree (the parent is kept -// alive by holding a shared_ptr to this child, and the parser is kept alive by -// the caller's json_pull_ptr). json_disconnect() splices a node out of its -// parent and clears those back-pointers in the detached subtree, so the -// detached subtree can outlive the original parser. - -struct json_object : public std::enable_shared_from_this { +// json_object is a small base type that just records the JSON type and +// the back-pointers to its parent and parser. The actual value payload +// lives in a type-specific subclass (json_number, json_string, json_array, +// json_hash), so that JSON_TRUE / JSON_FALSE / JSON_NULL nodes pay only +// the base-class cost and a JSON_HASH does not also drag along a string +// or a number field. Type-tagged accessor methods on the base class +// downcast and return references to the underlying subclass storage. +// +// Children are owned by their parent (via std::vector +// inside json_array / json_hash); the raw `parent` and `parser` +// back-pointers stay valid as long as the node is attached to the tree. +// json_disconnect() splices a node out of its parent and walks the +// detached subtree clearing those back-pointers so the subtree can +// outlive the original parser. +// +// json_object intentionally has no virtual functions and no virtual +// destructor: subclasses are constructed via std::make_shared(), +// and std::shared_ptr remembers the deleter from the original type, so +// destroying a shared_ptr that actually points at a +// json_string still runs ~json_string(). Dispatch on `type` is what the +// rest of the code already does. The accessor methods assert at debug +// time that the type matches before downcasting. + +struct json_object { json_object *parent = nullptr; json_pull *parser = nullptr; - json_type type = JSON_NULL; - int expect = 0; - - // Members named to match the previous C union layout so that existing - // access paths like `o->value.string.string` and `o->value.array.array[i]` - // continue to work. This is no longer a union because std::string and - // std::vector have non-trivial destructors. - struct value_t { - struct { - double number = 0; - unsigned long long large_unsigned = 0; - long long large_signed = 0; - } number; - - struct { - std::string string; - void *refcon = nullptr; // reference constant for caller's use - } string; - - struct { - std::vector array; - } array; - - struct { - std::vector keys; - std::vector values; - } object; - } value; + json_type type; + int expect = 0; // used by the parser on JSON_ARRAY / JSON_HASH nodes + + json_object(json_type t) : type(t) {} + json_object(json_type t, json_object *p, json_pull *pl) : parent(p), parser(pl), type(t) {} + + // Type-tagged accessors. Each one asserts that the receiver is of + // the right kind, then downcasts to the storage in the appropriate + // subclass. Inline so the assert and cast disappear at -O. + inline std::string &string(); + inline const std::string &string() const; + + inline double &number(); + inline double number() const; + inline unsigned long long &large_unsigned(); + inline unsigned long long large_unsigned() const; + inline long long &large_signed(); + inline long long large_signed() const; + + inline std::vector &array(); + inline const std::vector &array() const; + + inline std::vector &keys(); + inline const std::vector &keys() const; + inline std::vector &values(); + inline const std::vector &values() const; }; +struct json_number : json_object { + double number_value = 0; + unsigned long long large_unsigned_value = 0; + long long large_signed_value = 0; + + json_number() : json_object(JSON_NUMBER) {} + json_number(json_object *p, json_pull *pl) : json_object(JSON_NUMBER, p, pl) {} +}; + +struct json_string : json_object { + std::string string_value; + + json_string() : json_object(JSON_STRING) {} + json_string(json_object *p, json_pull *pl) : json_object(JSON_STRING, p, pl) {} +}; + +struct json_array : json_object { + std::vector array_value; + + json_array() : json_object(JSON_ARRAY) {} + json_array(json_object *p, json_pull *pl) : json_object(JSON_ARRAY, p, pl) {} +}; + +struct json_hash : json_object { + std::vector keys_value; + std::vector values_value; + + json_hash() : json_object(JSON_HASH) {} + json_hash(json_object *p, json_pull *pl) : json_object(JSON_HASH, p, pl) {} +}; + +inline std::string &json_object::string() { + assert(type == JSON_STRING); + return static_cast(this)->string_value; +} +inline const std::string &json_object::string() const { + assert(type == JSON_STRING); + return static_cast(this)->string_value; +} + +inline double &json_object::number() { + assert(type == JSON_NUMBER); + return static_cast(this)->number_value; +} +inline double json_object::number() const { + assert(type == JSON_NUMBER); + return static_cast(this)->number_value; +} +inline unsigned long long &json_object::large_unsigned() { + assert(type == JSON_NUMBER); + return static_cast(this)->large_unsigned_value; +} +inline unsigned long long json_object::large_unsigned() const { + assert(type == JSON_NUMBER); + return static_cast(this)->large_unsigned_value; +} +inline long long &json_object::large_signed() { + assert(type == JSON_NUMBER); + return static_cast(this)->large_signed_value; +} +inline long long json_object::large_signed() const { + assert(type == JSON_NUMBER); + return static_cast(this)->large_signed_value; +} + +inline std::vector &json_object::array() { + assert(type == JSON_ARRAY); + return static_cast(this)->array_value; +} +inline const std::vector &json_object::array() const { + assert(type == JSON_ARRAY); + return static_cast(this)->array_value; +} + +inline std::vector &json_object::keys() { + assert(type == JSON_HASH); + return static_cast(this)->keys_value; +} +inline const std::vector &json_object::keys() const { + assert(type == JSON_HASH); + return static_cast(this)->keys_value; +} +inline std::vector &json_object::values() { + assert(type == JSON_HASH); + return static_cast(this)->values_value; +} +inline const std::vector &json_object::values() const { + assert(type == JSON_HASH); + return static_cast(this)->values_value; +} + struct json_pull { const char *error = nullptr; // points at a string literal; no allocation int line = 1; @@ -85,7 +190,11 @@ struct json_pull { ssize_t buffer_tail = 0; ssize_t buffer_head = 0; - json_object_ptr container; + // Stack of currently-open containers; the top is the innermost container + // being parsed. Replaces the previous single `container` pointer / parent + // walk, which previously required enable_shared_from_this + // on every json_object instance (16 extra bytes per node). + std::vector container_stack; json_object_ptr root; std::string number_buffer; @@ -118,6 +227,7 @@ void json_free(json_object_ptr &j); void json_disconnect(json_object_ptr j); json_object_ptr json_hash_get(json_object_ptr o, const char *s); +json_object_ptr json_hash_get(json_object *o, const char *s); std::string json_stringify(json_object_ptr o); diff --git a/jsontool.cpp b/jsontool.cpp index 5c35206f4..e83ab5a83 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -149,7 +149,7 @@ void out(std::string const &s, int type, json_object_ptr properties) { if (o != nullptr) { found = true; if (o->type == JSON_STRING || o->type == JSON_NUMBER) { - extracted = sort_quote(o->value.string.string.c_str()); + extracted = sort_quote(o->string().c_str()); } else { extracted = sort_quote(json_stringify(o).c_str()); } @@ -244,9 +244,9 @@ void join_csv(json_object_ptr j) { std::string joinkey; if (key->type == JSON_STRING) { - joinkey = key->value.string.string; + joinkey = key->string(); } else if (key->type == JSON_NUMBER) { - joinkey = milo::dtoa_milo(key->value.number.number); + joinkey = milo::dtoa_milo(key->number()); } else { joinkey = json_stringify(key); } @@ -298,8 +298,8 @@ void join_csv(json_object_ptr j) { } if (fields.size() > 0 && joinkey == fields[0]) { - properties->value.object.keys.reserve(properties->value.object.keys.size() + fields.size()); - properties->value.object.values.reserve(properties->value.object.values.size() + fields.size()); + properties->keys().reserve(properties->keys().size() + fields.size()); + properties->values().reserve(properties->values().size() + fields.size()); for (size_t i = 1; i < fields.size(); i++) { std::string k = header[i]; @@ -317,30 +317,24 @@ void join_csv(json_object_ptr j) { } if (attr_type != JSON_NULL) { - auto ko = std::make_shared(); - auto vo = std::make_shared(); - - ko->type = JSON_STRING; - ko->parent = properties.get(); - ko->parser = properties->parser; - ko->value.string.string = k; - - vo->type = attr_type; - vo->parent = properties.get(); - vo->parser = properties->parser; + auto ko = std::make_shared(properties.get(), properties->parser); + ko->string_value = k; + json_object_ptr vo; if (attr_type == JSON_STRING) { - vo->value.string.string = v; + auto s = std::make_shared(properties.get(), properties->parser); + s->string_value = v; + vo = s; } else if (attr_type == JSON_NUMBER) { - vo->value.number.number = atof(v.c_str()); - vo->value.number.large_unsigned = 0; - vo->value.number.large_signed = 0; + auto n = std::make_shared(properties.get(), properties->parser); + n->number_value = atof(v.c_str()); + vo = n; } else { abort(); } - properties->value.object.keys.push_back(ko); - properties->value.object.values.push_back(vo); + properties->keys().push_back(ko); + properties->values().push_back(vo); } } } diff --git a/main.cpp b/main.cpp index 1fc7d4977..fae5b50e2 100644 --- a/main.cpp +++ b/main.cpp @@ -2885,9 +2885,9 @@ void set_attribute_value(const char *arg) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - json_object_ptr k = o->value.object.keys[i]; - json_object_ptr v = o->value.object.values[i]; + for (size_t i = 0; i < o->keys().size(); i++) { + json_object_ptr k = o->keys()[i]; + json_object_ptr v = o->values()[i]; if (k->type != JSON_STRING) { fprintf(stderr, "%s: --set-attribute %s: key %zu not a string\n", *av, arg, i); @@ -2895,7 +2895,7 @@ void set_attribute_value(const char *arg) { } serial_val val = stringify_value(v, "json", 1, o); - set_attributes.emplace(k->value.string.string, val); + set_attributes.emplace(k->string(), val); } return; @@ -2941,21 +2941,21 @@ void parse_json_source(const char *arg, struct source &src) { exit(EXIT_JSON); } - src.file = fname->value.string.string; + src.file = fname->string(); json_object_ptr layer = json_hash_get(o, "layer"); if (layer != nullptr && layer->type == JSON_STRING) { - src.layer = layer->value.string.string; + src.layer = layer->string(); } json_object_ptr description = json_hash_get(o, "description"); if (description != nullptr && description->type == JSON_STRING) { - src.description = description->value.string.string; + src.description = description->string(); } json_object_ptr format = json_hash_get(o, "format"); if (format != nullptr && format->type == JSON_STRING) { - src.format = format->value.string.string; + src.format = format->string(); } } diff --git a/plugin.cpp b/plugin.cpp index b90fbfe71..10e8d9c36 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -168,7 +168,7 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st if (type == nullptr || type->type != JSON_STRING) { continue; } - if (type->value.string.string != "Feature") { + if (type->string() != "Feature") { continue; } @@ -208,12 +208,12 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st int t; for (t = 0; t < GEOM_TYPES; t++) { - if (geometry_type->value.string.string == geometry_names[t]) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string.c_str()); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str()); json_context(j); exit(EXIT_JSON); } @@ -252,22 +252,22 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st if (tippecanoe != nullptr) { json_object_ptr layer = json_hash_get(tippecanoe, "layer"); if (layer != nullptr && layer->type == JSON_STRING) { - layername = layer->value.string.string; + layername = layer->string(); } json_object_ptr index = json_hash_get(tippecanoe, "index"); if (index != nullptr && index->type == JSON_NUMBER) { - sf.index = index->value.number.number; + sf.index = index->number(); } json_object_ptr sequence = json_hash_get(tippecanoe, "sequence"); if (sequence != nullptr && sequence->type == JSON_NUMBER) { - sf.seq = sequence->value.number.number; + sf.seq = sequence->number(); } json_object_ptr extent = json_hash_get(tippecanoe, "extent"); if (extent != nullptr && extent->type == JSON_NUMBER) { - sf.extent = extent->value.number.number; + sf.extent = extent->number(); } json_object_ptr dropped = json_hash_get(tippecanoe, "dropped"); @@ -297,9 +297,9 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st json_object_ptr id = json_hash_get(j, "id"); if (id != nullptr && id->type == JSON_NUMBER) { - sf.id = id->value.number.number; - if (id->value.number.large_unsigned > 0) { - sf.id = id->value.number.large_unsigned; + sf.id = id->number(); + if (id->large_unsigned() > 0) { + sf.id = id->large_unsigned(); } sf.has_id = true; } @@ -343,18 +343,18 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st } } - for (size_t i = 0; i < properties->value.object.keys.size(); i++) { - serial_val v = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); + for (size_t i = 0; i < properties->keys().size(); i++) { + serial_val v = stringify_value(properties->values()[i], "Filter output", jp->line, j); // Nulls can be excluded here because the expression evaluation filter // would have already run before prefiltering if (v.type != mvt_null) { - sf.full_keys.push_back(key_pool.pool(properties->value.object.keys[i]->value.string.string)); + sf.full_keys.push_back(key_pool.pool(properties->keys()[i]->string())); sf.full_values.push_back(v); if (!postfilter) { - add_to_tilestats(ts->second.tilestats, properties->value.object.keys[i]->value.string.string, v); + add_to_tilestats(ts->second.tilestats, properties->keys()[i]->string(), v); } } } diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 499d6ceeb..39ed4f44d 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -415,37 +415,37 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { state.nospace = true; state.json_write_hash(); - for (size_t i = 0; i < o->value.object.keys.size(); i++) { - const std::string &key = o->value.object.keys[i]->value.string.string; - if (key == "vector_layers" && o->value.object.values[i]->type == JSON_ARRAY) { + for (size_t i = 0; i < o->keys().size(); i++) { + const std::string &key = o->keys()[i]->string(); + if (key == "vector_layers" && o->values()[i]->type == JSON_ARRAY) { has_json = true; state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; - state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (key == "tilestats" && o->value.object.values[i]->type == JSON_HASH) { + state.json_write_json(json_stringify(o->values()[i])); + } else if (key == "tilestats" && o->values()[i]->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; - state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (key == "strategies" && o->value.object.values[i]->type == JSON_ARRAY) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->value.object.values[i]).c_str()); + state.json_write_json(json_stringify(o->values()[i])); + } else if (key == "strategies" && o->values()[i]->type == JSON_ARRAY) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->values()[i]).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (key == "tippecanoe_decisions" && o->value.object.values[i]->type == JSON_HASH) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i]).c_str()); + } else if (key == "tippecanoe_decisions" && o->values()[i]->type == JSON_HASH) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->values()[i]).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { + } else if (o->keys()[i]->type != JSON_STRING || o->values()[i]->type != JSON_STRING) { fprintf(stderr, "%s\n", key.c_str()); fprintf(stderr, "%s: non-string in metadata\n", fname); } else { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), o->value.object.values[i]->value.string.string.c_str()); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), o->values()[i]->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } diff --git a/read_json.cpp b/read_json.cpp index 08ff2f147..613041539 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -63,7 +63,7 @@ void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const cha int within = geometry_within[t]; if (within >= 0) { size_t i; - for (i = 0; i < j->value.array.array.size(); i++) { + for (i = 0; i < j->array().size(); i++) { if (within == GEOM_POINT) { if (i == 0 || mb_geometry[t] == VT_POINT) { op = VT_MOVETO; @@ -72,16 +72,16 @@ void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const cha } } - parse_coordinates(within, j->value.array.array[i], out, op, fname, line, feature); + parse_coordinates(within, j->array()[i], out, op, fname, line, feature); } } else { - if (j->value.array.array.size() >= 2 && j->value.array.array[0]->type == JSON_NUMBER && j->value.array.array[1]->type == JSON_NUMBER) { + if (j->array().size() >= 2 && j->array()[0]->type == JSON_NUMBER && j->array()[1]->type == JSON_NUMBER) { long long x, y; - double lon = j->value.array.array[0]->value.number.number; - double lat = j->value.array.array[1]->value.number.number; + double lon = j->array()[0]->number(); + double lat = j->array()[1]->number(); projection->project(lon, lat, 32, &x, &y); - if (j->value.array.array.size() > 2) { + if (j->array().size() > 2) { static int warned = 0; if (!warned) { @@ -129,7 +129,7 @@ serial_val stringify_value(json_object_ptr value, const char *reading, int line, if (vt == JSON_STRING) { sv.type = mvt_string; - sv.s = value->value.string.string; + sv.s = value->string(); std::string err = check_utf8(sv.s); if (err.size() > 0) { @@ -140,12 +140,12 @@ serial_val stringify_value(json_object_ptr value, const char *reading, int line, } else if (vt == JSON_NUMBER) { sv.type = mvt_double; - if (value->value.number.large_unsigned != 0) { - sv.s = std::to_string(value->value.number.large_unsigned); - } else if (value->value.number.large_signed != 0) { - sv.s = std::to_string(value->value.number.large_signed); + if (value->large_unsigned() != 0) { + sv.s = std::to_string(value->large_unsigned()); + } else if (value->large_signed() != 0) { + sv.s = std::to_string(value->large_signed()); } else { - sv.s = milo::dtoa_milo(value->value.number.number); + sv.s = milo::dtoa_milo(value->number()); } } else if (vt == JSON_TRUE) { sv.type = mvt_bool; @@ -200,12 +200,12 @@ std::pair parse_geometry(json_object_ptr geometry, json_pull_ptr j int t; for (t = 0; t < GEOM_TYPES; t++) { - if (geometry_type->value.string.string == geometry_names[t]) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string.c_str()); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str()); json_context(j); exit(EXIT_JSON); } @@ -325,7 +325,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int if (type == nullptr || type->type != JSON_STRING) { continue; } - if (type->value.string.string != "Feature") { + if (type->string() != "Feature") { continue; } @@ -342,7 +342,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int if (tippecanoe != nullptr) { layer = json_hash_get(tippecanoe, "layer"); if (layer != nullptr && layer->type == JSON_STRING) { - layername = layer->value.string.string; + layername = layer->string(); } } @@ -375,22 +375,22 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int json_object_ptr id = json_hash_get(j, "id"); if (id != nullptr && id->type == JSON_NUMBER) { - feature.id = id->value.number.number; - if (id->value.number.large_unsigned > 0) { - feature.id = id->value.number.large_unsigned; + feature.id = id->number(); + if (id->large_unsigned() > 0) { + feature.id = id->large_unsigned(); } feature.has_id = true; } - for (size_t i = 0; i < properties->value.object.keys.size(); i++) { - serial_val sv = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); + for (size_t i = 0; i < properties->keys().size(); i++) { + serial_val sv = stringify_value(properties->values()[i], "Filter output", jp->line, j); // Nulls can be excluded here because this is the postfilter // and it is nearly time to create the vector representation if (sv.type != mvt_null) { mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - l->second.tag(feature, properties->value.object.keys[i]->value.string.string, v); + l->second.tag(feature, properties->keys()[i]->string(), v); } } diff --git a/tile-join.cpp b/tile-join.cpp index 24d96167e..cf4eb8335 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -969,12 +969,12 @@ void handle_strategies(const unsigned char *s, std::vector *st) { json_object_ptr o = json_read_tree(jp); if (o != nullptr && o->type == JSON_ARRAY) { - for (size_t i = 0; i < o->value.array.array.size(); i++) { - json_object_ptr h = o->value.array.array[i]; + for (size_t i = 0; i < o->array().size(); i++) { + json_object_ptr h = o->array()[i]; if (h->type == JSON_HASH) { - for (size_t j = 0; j < h->value.object.keys.size(); j++) { - json_object_ptr k = h->value.object.keys[j]; - json_object_ptr v = h->value.object.values[j]; + for (size_t j = 0; j < h->keys().size(); j++) { + json_object_ptr k = h->keys()[j]; + json_object_ptr v = h->values()[j]; if (k->type != JSON_STRING) { fprintf(stderr, "Key %zu of %zu is not a string: %s\n", j, i, s); @@ -985,25 +985,25 @@ void handle_strategies(const unsigned char *s, std::vector *st) { st->resize(i + 1); } - const std::string &key = k->value.string.string; + const std::string &key = k->string(); if (key == "dropped_by_rate") { - (*st)[i].dropped_by_rate += v->value.number.number; + (*st)[i].dropped_by_rate += v->number(); } else if (key == "dropped_by_gamma") { - (*st)[i].dropped_by_gamma += v->value.number.number; + (*st)[i].dropped_by_gamma += v->number(); } else if (key == "dropped_as_needed") { - (*st)[i].dropped_as_needed += v->value.number.number; + (*st)[i].dropped_as_needed += v->number(); } else if (key == "coalesced_as_needed") { - (*st)[i].coalesced_as_needed += v->value.number.number; + (*st)[i].coalesced_as_needed += v->number(); } else if (key == "truncated_zooms") { - (*st)[i].truncated_zooms += v->value.number.number; + (*st)[i].truncated_zooms += v->number(); } else if (key == "detail_reduced") { - (*st)[i].detail_reduced += v->value.number.number; + (*st)[i].detail_reduced += v->number(); } else if (key == "tiny_polygons") { - (*st)[i].tiny_polygons += v->value.number.number; + (*st)[i].tiny_polygons += v->number(); } else if (key == "tile_size_desired") { - (*st)[i].tile_size += v->value.number.number; + (*st)[i].tile_size += v->number(); } else if (key == "feature_count_desired") { - (*st)[i].feature_count += v->value.number.number; + (*st)[i].feature_count += v->number(); } } } @@ -1016,14 +1016,14 @@ void handle_strategies(const unsigned char *s, std::vector *st) { void handle_vector_layers(json_object_ptr vector_layers, std::map &layermap, std::map &attribute_descriptions) { if (vector_layers != nullptr && vector_layers->type == JSON_ARRAY) { - for (size_t i = 0; i < vector_layers->value.array.array.size(); i++) { - if (vector_layers->value.array.array[i]->type == JSON_HASH) { - json_object_ptr id = json_hash_get(vector_layers->value.array.array[i], "id"); - json_object_ptr desc = json_hash_get(vector_layers->value.array.array[i], "description"); + for (size_t i = 0; i < vector_layers->array().size(); i++) { + if (vector_layers->array()[i]->type == JSON_HASH) { + json_object_ptr id = json_hash_get(vector_layers->array()[i], "id"); + json_object_ptr desc = json_hash_get(vector_layers->array()[i], "description"); if (id != nullptr && desc != nullptr && id->type == JSON_STRING && desc->type == JSON_STRING) { - const std::string &sid = id->value.string.string; - const std::string &sdesc = desc->value.string.string; + const std::string &sid = id->string(); + const std::string &sdesc = desc->string(); if (sdesc.size() != 0) { auto f = layermap.find(sid); @@ -1033,17 +1033,17 @@ void handle_vector_layers(json_object_ptr vector_layers, std::mapvalue.array.array[i], "fields"); + json_object_ptr fields = json_hash_get(vector_layers->array()[i], "fields"); if (fields != nullptr && fields->type == JSON_HASH) { - for (size_t j = 0; j < fields->value.object.keys.size(); j++) { - if (fields->value.object.keys[j]->type == JSON_STRING && fields->value.object.values[j]->type) { - const std::string &desc2 = fields->value.object.values[j]->value.string.string; + for (size_t j = 0; j < fields->keys().size(); j++) { + if (fields->keys()[j]->type == JSON_STRING && fields->values()[j]->type) { + const std::string &desc2 = fields->values()[j]->string(); if (desc2 != "Number" && desc2 != "String" && desc2 != "Boolean" && desc2 != "Mixed") { - attribute_descriptions.insert(std::pair(fields->value.object.keys[j]->value.string.string, desc2)); + attribute_descriptions.insert(std::pair(fields->keys()[j]->string(), desc2)); } } } From 4d9a48c3d44cad99c5392ac7c9515acc0ef11e33 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 10:08:52 -0700 Subject: [PATCH 04/13] Store hash key/value pairs in one ordered vector Replace the parallel std::vector keys / values on json_hash with a single std::vector, where json_entry is a small {key, value} aggregate. This still preserves insertion order (the property the parallel vectors were providing) but removes the "keep two vectors in lockstep" pattern, and call sites can now use range-for with structured bindings: for (auto &[k, v] : o->entries()) { ... } Side effects: * sizeof(json_hash) drops from 72 to 48 bytes (one fewer vector header), matching json_array. * The keys() and values() accessors on json_object are replaced by a single entries() accessor returning std::vector&. * All call sites were swept from the old paired-index pattern (`o->keys()[i]` / `o->values()[i]`) to entry-based access. Where the original pattern relied on `nprop = 0` to short-circuit iteration on a null or non-hash `properties`, the rewrite now guards the loop explicitly with `if (o->type == JSON_HASH)` so that calling entries() doesn't trip the asserting downcast. Co-authored-by: Cursor --- attribute.cpp | 13 +++++---- dirtiles.cpp | 8 +++--- geojson.cpp | 24 ++++++++--------- jsonpull/jsonpull.cpp | 61 ++++++++++++++++++------------------------- jsonpull/jsonpull.h | 35 ++++++++++++------------- jsontool.cpp | 6 ++--- main.cpp | 13 +++++---- plugin.cpp | 20 +++++++------- pmtiles_file.cpp | 24 ++++++++--------- read_json.cpp | 16 +++++++----- tile-join.cpp | 39 ++++++++++++++------------- 11 files changed, 122 insertions(+), 137 deletions(-) diff --git a/attribute.cpp b/attribute.cpp index 2794e5b79..e4deb9329 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -55,20 +55,19 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - for (size_t i = 0; i < o->keys().size(); i++) { - json_object_ptr k = o->keys()[i]; - json_object_ptr v = o->values()[i]; - - if (k->type != JSON_STRING) { + size_t i = 0; + for (const auto &e : o->entries()) { + if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: key %zu not a string\n", *argv, arg, i); exit(EXIT_JSON); } - if (v->type != JSON_STRING) { + if (e.value->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: value %zu not a string\n", *argv, arg, i); exit(EXIT_JSON); } - set_attribute_accum(attribute_accum, k->string().c_str(), v->string().c_str()); + set_attribute_accum(attribute_accum, e.key->string().c_str(), e.value->string().c_str()); + i++; } return; diff --git a/dirtiles.cpp b/dirtiles.cpp index de3f86d48..452ecf9e7 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -260,14 +260,14 @@ sqlite3 *dirmeta2tmp(const char *fname) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->keys().size(); i++) { - if (o->keys()[i]->type != JSON_STRING || o->values()[i]->type != JSON_STRING) { + for (const auto &e : o->entries()) { + if (e.key->type != JSON_STRING || e.value->type != JSON_STRING) { fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); } - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys()[i]->string().c_str(), o->values()[i]->string().c_str()); + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", e.key->string().c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->keys()[i]->string().c_str(), err); + fprintf(stderr, "set %s in metadata: %s\n", e.key->string().c_str(), err); } sqlite3_free(sql); } diff --git a/geojson.cpp b/geojson.cpp index 4d16088fc..66cad85bb 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -175,24 +175,22 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g } } - size_t nprop = 0; - if (properties != nullptr && properties->type == JSON_HASH) { - nprop = properties->keys().size(); - } - std::vector> full_keys; std::vector values; - - full_keys.reserve(nprop); - values.reserve(nprop); key_pool key_pool; - for (size_t i = 0; i < nprop; i++) { - if (properties->keys()[i]->type == JSON_STRING) { - serial_val sv = stringify_value(properties->values()[i], sst->fname, sst->line, feature); + if (properties != nullptr && properties->type == JSON_HASH) { + const auto &entries = properties->entries(); + full_keys.reserve(entries.size()); + values.reserve(entries.size()); + + for (const auto &e : entries) { + if (e.key->type == JSON_STRING) { + serial_val sv = stringify_value(e.value, sst->fname, sst->line, feature); - full_keys.emplace_back(key_pool.pool(properties->keys()[i]->string().c_str())); - values.push_back(std::move(sv)); + full_keys.emplace_back(key_pool.pool(e.key->string().c_str())); + values.push_back(std::move(sv)); + } } } diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index d35bf8d39..53704ae3f 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -134,7 +134,7 @@ static json_object_ptr add_object(json_pull *j, json_type type) { } } else if (c->type == JSON_HASH) { if (c->expect == JSON_VALUE) { - c->values().back() = o; + c->entries().back().value = o; c->expect = JSON_COMMA; } else if (c->expect == JSON_KEY) { if (type != JSON_STRING) { @@ -142,8 +142,7 @@ static json_object_ptr add_object(json_pull *j, json_type type) { return nullptr; } - c->keys().push_back(o); - c->values().push_back(nullptr); + c->entries().push_back({o, nullptr}); c->expect = JSON_COLON; } else { j->error = "Expected a comma or colon"; @@ -164,14 +163,9 @@ json_object_ptr json_hash_get(json_object *o, const char *s) { return nullptr; } - const auto &keys = o->keys(); - const auto &vals = o->values(); - for (size_t i = 0; i < keys.size(); i++) { - const auto &key = keys[i]; - if (key != nullptr && key->type == JSON_STRING) { - if (key->string() == s) { - return vals[i]; - } + for (const auto &e : o->entries()) { + if (e.key != nullptr && e.key->type == JSON_STRING && e.key->string() == s) { + return e.value; } } @@ -299,7 +293,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } if (cc->expect != JSON_COMMA) { - if (!(cc->expect == JSON_KEY && cc->keys().size() == 0)) { + if (!(cc->expect == JSON_KEY && cc->entries().size() == 0)) { j->error = "Found } without final element"; return nullptr; } @@ -676,11 +670,9 @@ static void clear_back_pointers(json_object *o) { } if (o->type == JSON_HASH) { - const auto &keys = o->keys(); - const auto &vals = o->values(); - for (size_t i = 0; i < keys.size(); i++) { - clear_back_pointers(keys[i].get()); - clear_back_pointers(vals[i].get()); + for (const auto &e : o->entries()) { + clear_back_pointers(e.key.get()); + clear_back_pointers(e.value.get()); } } else if (o->type == JSON_ARRAY) { const auto &arr = o->array(); @@ -713,28 +705,26 @@ void json_disconnect(json_object_ptr o) { } } } else if (parent->type == JSON_HASH) { - auto &keys = parent->keys(); - auto &vals = parent->values(); + auto &entries = parent->entries(); - for (size_t i = 0; i < keys.size(); i++) { - if (keys[i].get() == o.get()) { + for (size_t i = 0; i < entries.size(); i++) { + auto &e = entries[i]; + if (e.key.get() == o.get()) { // Leave a NULL placeholder in the key slot so the // surrounding value isn't shifted; if the corresponding // value is also detached the pair is removed below. - keys[i] = fabricate_object(parent->parser, parent, JSON_NULL); + e.key = fabricate_object(parent->parser, parent, JSON_NULL); - if (vals[i] != nullptr && vals[i]->type == JSON_NULL && keys[i]->type == JSON_NULL) { - keys.erase(keys.begin() + i); - vals.erase(vals.begin() + i); + if (e.value != nullptr && e.value->type == JSON_NULL && e.key->type == JSON_NULL) { + entries.erase(entries.begin() + i); } break; } - if (vals[i].get() == o.get()) { - vals[i] = fabricate_object(parent->parser, parent, JSON_NULL); + if (e.value.get() == o.get()) { + e.value = fabricate_object(parent->parser, parent, JSON_NULL); - if (keys[i] != nullptr && keys[i]->type == JSON_NULL && vals[i]->type == JSON_NULL) { - keys.erase(keys.begin() + i); - vals.erase(vals.begin() + i); + if (e.key != nullptr && e.key->type == JSON_NULL && e.value->type == JSON_NULL) { + entries.erase(entries.begin() + i); } break; } @@ -815,13 +805,12 @@ static void json_print(std::string &val, json_object *o) { } else if (o->type == JSON_HASH) { string_append_c(val, '{'); - const auto &keys = o->keys(); - const auto &vals = o->values(); - for (size_t i = 0; i < keys.size(); i++) { - json_print(val, keys[i].get()); + const auto &entries = o->entries(); + for (size_t i = 0; i < entries.size(); i++) { + json_print(val, entries[i].key.get()); string_append_c(val, ':'); - json_print(val, vals[i].get()); - if (i + 1 < keys.size()) { + json_print(val, entries[i].value.get()); + if (i + 1 < entries.size()) { string_append_c(val, ','); } } diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index fa9b265f5..b4ac51a66 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -34,6 +34,16 @@ struct json_pull; typedef std::shared_ptr json_object_ptr; typedef std::shared_ptr json_pull_ptr; +// A single key/value pair inside a JSON_HASH. The pairs are stored in +// insertion order in a single std::vector on json_hash, so +// callers can range-for over `o->entries()` with structured bindings +// (`for (auto &[k, v] : o->entries()) ...`) while still preserving the +// order keys appeared in the source document. +struct json_entry { + json_object_ptr key; + json_object_ptr value; +}; + // json_object is a small base type that just records the JSON type and // the back-pointers to its parent and parser. The actual value payload // lives in a type-specific subclass (json_number, json_string, json_array, @@ -83,10 +93,8 @@ struct json_object { inline std::vector &array(); inline const std::vector &array() const; - inline std::vector &keys(); - inline const std::vector &keys() const; - inline std::vector &values(); - inline const std::vector &values() const; + inline std::vector &entries(); + inline const std::vector &entries() const; }; struct json_number : json_object { @@ -113,8 +121,7 @@ struct json_array : json_object { }; struct json_hash : json_object { - std::vector keys_value; - std::vector values_value; + std::vector entries_value; json_hash() : json_object(JSON_HASH) {} json_hash(json_object *p, json_pull *pl) : json_object(JSON_HASH, p, pl) {} @@ -163,21 +170,13 @@ inline const std::vector &json_object::array() const { return static_cast(this)->array_value; } -inline std::vector &json_object::keys() { - assert(type == JSON_HASH); - return static_cast(this)->keys_value; -} -inline const std::vector &json_object::keys() const { - assert(type == JSON_HASH); - return static_cast(this)->keys_value; -} -inline std::vector &json_object::values() { +inline std::vector &json_object::entries() { assert(type == JSON_HASH); - return static_cast(this)->values_value; + return static_cast(this)->entries_value; } -inline const std::vector &json_object::values() const { +inline const std::vector &json_object::entries() const { assert(type == JSON_HASH); - return static_cast(this)->values_value; + return static_cast(this)->entries_value; } struct json_pull { diff --git a/jsontool.cpp b/jsontool.cpp index e83ab5a83..9a8dbfc31 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -298,8 +298,7 @@ void join_csv(json_object_ptr j) { } if (fields.size() > 0 && joinkey == fields[0]) { - properties->keys().reserve(properties->keys().size() + fields.size()); - properties->values().reserve(properties->values().size() + fields.size()); + properties->entries().reserve(properties->entries().size() + fields.size()); for (size_t i = 1; i < fields.size(); i++) { std::string k = header[i]; @@ -333,8 +332,7 @@ void join_csv(json_object_ptr j) { abort(); } - properties->keys().push_back(ko); - properties->values().push_back(vo); + properties->entries().push_back({ko, vo}); } } } diff --git a/main.cpp b/main.cpp index fae5b50e2..95a4096bb 100644 --- a/main.cpp +++ b/main.cpp @@ -2885,17 +2885,16 @@ void set_attribute_value(const char *arg) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->keys().size(); i++) { - json_object_ptr k = o->keys()[i]; - json_object_ptr v = o->values()[i]; - - if (k->type != JSON_STRING) { + size_t i = 0; + for (const auto &e : o->entries()) { + if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: --set-attribute %s: key %zu not a string\n", *av, arg, i); exit(EXIT_JSON); } - serial_val val = stringify_value(v, "json", 1, o); - set_attributes.emplace(k->string(), val); + serial_val val = stringify_value(e.value, "json", 1, o); + set_attributes.emplace(e.key->string(), val); + i++; } return; diff --git a/plugin.cpp b/plugin.cpp index 10e8d9c36..b9d5f2dea 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -343,18 +343,20 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st } } - for (size_t i = 0; i < properties->keys().size(); i++) { - serial_val v = stringify_value(properties->values()[i], "Filter output", jp->line, j); + if (properties->type == JSON_HASH) { + for (const auto &e : properties->entries()) { + serial_val v = stringify_value(e.value, "Filter output", jp->line, j); - // Nulls can be excluded here because the expression evaluation filter - // would have already run before prefiltering + // Nulls can be excluded here because the expression evaluation filter + // would have already run before prefiltering - if (v.type != mvt_null) { - sf.full_keys.push_back(key_pool.pool(properties->keys()[i]->string())); - sf.full_values.push_back(v); + if (v.type != mvt_null) { + sf.full_keys.push_back(key_pool.pool(e.key->string())); + sf.full_values.push_back(v); - if (!postfilter) { - add_to_tilestats(ts->second.tilestats, properties->keys()[i]->string(), v); + if (!postfilter) { + add_to_tilestats(ts->second.tilestats, e.key->string(), v); + } } } } diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 39ed4f44d..9a8c62a1a 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -415,37 +415,37 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { state.nospace = true; state.json_write_hash(); - for (size_t i = 0; i < o->keys().size(); i++) { - const std::string &key = o->keys()[i]->string(); - if (key == "vector_layers" && o->values()[i]->type == JSON_ARRAY) { + for (const auto &e : o->entries()) { + const std::string &key = e.key->string(); + if (key == "vector_layers" && e.value->type == JSON_ARRAY) { has_json = true; state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; - state.json_write_json(json_stringify(o->values()[i])); - } else if (key == "tilestats" && o->values()[i]->type == JSON_HASH) { + state.json_write_json(json_stringify(e.value)); + } else if (key == "tilestats" && e.value->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; - state.json_write_json(json_stringify(o->values()[i])); - } else if (key == "strategies" && o->values()[i]->type == JSON_ARRAY) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->values()[i]).c_str()); + state.json_write_json(json_stringify(e.value)); + } else if (key == "strategies" && e.value->type == JSON_ARRAY) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (key == "tippecanoe_decisions" && o->values()[i]->type == JSON_HASH) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->values()[i]).c_str()); + } else if (key == "tippecanoe_decisions" && e.value->type == JSON_HASH) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (o->keys()[i]->type != JSON_STRING || o->values()[i]->type != JSON_STRING) { + } else if (e.key->type != JSON_STRING || e.value->type != JSON_STRING) { fprintf(stderr, "%s\n", key.c_str()); fprintf(stderr, "%s: non-string in metadata\n", fname); } else { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), o->values()[i]->string().c_str()); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } diff --git a/read_json.cpp b/read_json.cpp index 613041539..45a533f4a 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -382,15 +382,17 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int feature.has_id = true; } - for (size_t i = 0; i < properties->keys().size(); i++) { - serial_val sv = stringify_value(properties->values()[i], "Filter output", jp->line, j); + if (properties->type == JSON_HASH) { + for (const auto &e : properties->entries()) { + serial_val sv = stringify_value(e.value, "Filter output", jp->line, j); - // Nulls can be excluded here because this is the postfilter - // and it is nearly time to create the vector representation + // Nulls can be excluded here because this is the postfilter + // and it is nearly time to create the vector representation - if (sv.type != mvt_null) { - mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - l->second.tag(feature, properties->keys()[i]->string(), v); + if (sv.type != mvt_null) { + mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); + l->second.tag(feature, e.key->string(), v); + } } } diff --git a/tile-join.cpp b/tile-join.cpp index cf4eb8335..5f7af3686 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -972,40 +972,39 @@ void handle_strategies(const unsigned char *s, std::vector *st) { for (size_t i = 0; i < o->array().size(); i++) { json_object_ptr h = o->array()[i]; if (h->type == JSON_HASH) { - for (size_t j = 0; j < h->keys().size(); j++) { - json_object_ptr k = h->keys()[j]; - json_object_ptr v = h->values()[j]; - - if (k->type != JSON_STRING) { + size_t j = 0; + for (const auto &kv : h->entries()) { + if (kv.key->type != JSON_STRING) { fprintf(stderr, "Key %zu of %zu is not a string: %s\n", j, i, s); - } else if (v->type != JSON_NUMBER) { + } else if (kv.value->type != JSON_NUMBER) { fprintf(stderr, "Value %zu of %zu is not a number: %s\n", j, i, s); } else { if (i >= st->size()) { st->resize(i + 1); } - const std::string &key = k->string(); + const std::string &key = kv.key->string(); if (key == "dropped_by_rate") { - (*st)[i].dropped_by_rate += v->number(); + (*st)[i].dropped_by_rate += kv.value->number(); } else if (key == "dropped_by_gamma") { - (*st)[i].dropped_by_gamma += v->number(); + (*st)[i].dropped_by_gamma += kv.value->number(); } else if (key == "dropped_as_needed") { - (*st)[i].dropped_as_needed += v->number(); + (*st)[i].dropped_as_needed += kv.value->number(); } else if (key == "coalesced_as_needed") { - (*st)[i].coalesced_as_needed += v->number(); + (*st)[i].coalesced_as_needed += kv.value->number(); } else if (key == "truncated_zooms") { - (*st)[i].truncated_zooms += v->number(); + (*st)[i].truncated_zooms += kv.value->number(); } else if (key == "detail_reduced") { - (*st)[i].detail_reduced += v->number(); + (*st)[i].detail_reduced += kv.value->number(); } else if (key == "tiny_polygons") { - (*st)[i].tiny_polygons += v->number(); + (*st)[i].tiny_polygons += kv.value->number(); } else if (key == "tile_size_desired") { - (*st)[i].tile_size += v->number(); + (*st)[i].tile_size += kv.value->number(); } else if (key == "feature_count_desired") { - (*st)[i].feature_count += v->number(); + (*st)[i].feature_count += kv.value->number(); } } + j++; } } else { fprintf(stderr, "Element %zu is not a hash: %s\n", i, s); @@ -1035,15 +1034,15 @@ void handle_vector_layers(json_object_ptr vector_layers, std::maparray()[i], "fields"); if (fields != nullptr && fields->type == JSON_HASH) { - for (size_t j = 0; j < fields->keys().size(); j++) { - if (fields->keys()[j]->type == JSON_STRING && fields->values()[j]->type) { - const std::string &desc2 = fields->values()[j]->string(); + for (const auto &e : fields->entries()) { + if (e.key->type == JSON_STRING && e.value->type) { + const std::string &desc2 = e.value->string(); if (desc2 != "Number" && desc2 != "String" && desc2 != "Boolean" && desc2 != "Mixed") { - attribute_descriptions.insert(std::pair(fields->keys()[j]->string(), desc2)); + attribute_descriptions.insert(std::pair(e.key->string(), desc2)); } } } From bd90f0b4fe428c1c75da185d727f4a85a538ec51 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 10:16:56 -0700 Subject: [PATCH 05/13] Move parser-only `expect` state out of json_object `expect` was only meaningful while the parser was building a container, and only ever read or written from jsonpull.cpp itself; once parsing finished it was dead weight on every JSON_ARRAY and JSON_HASH (and present-but-unused on every primitive too). Move it into the parser's container stack, alongside the shared_ptr to the container it pertains to: struct json_pull::parse_frame { json_object_ptr container; json_type expect; }; std::vector container_stack; The base class now only carries data-model state (parent, parser, type). No external caller depended on `expect`, so no sweep was needed outside jsonpull.cpp. This change does not, in itself, shrink any json_object: the 4-byte `expect` field used to live at offset 20 inside the base, where it was already being eaten by alignment padding for the 8-byte-aligned first member of every subclass (std::string, std::vector, double). The win is in the data model, not the byte count -- the 4-byte hole is still there, but it is now available for a future subclass whose first member is small enough to slot into it. Co-authored-by: Cursor --- jsonpull/jsonpull.cpp | 69 ++++++++++++++++++++++--------------------- jsonpull/jsonpull.h | 16 ++++++---- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 53704ae3f..0f2d99fe1 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -115,35 +115,36 @@ static json_object_ptr fabricate_object(json_pull *jp, json_object *parent, json return make_object(type, parent, jp); } -static inline json_object *current_container(json_pull *j) { - return j->container_stack.empty() ? nullptr : j->container_stack.back().get(); +static inline json_pull::parse_frame *current_frame(json_pull *j) { + return j->container_stack.empty() ? nullptr : &j->container_stack.back(); } static json_object_ptr add_object(json_pull *j, json_type type) { - json_object *c = current_container(j); + json_pull::parse_frame *f = current_frame(j); + json_object *c = f ? f->container.get() : nullptr; json_object_ptr o = make_object(type, c, j); - if (c != nullptr) { + if (f != nullptr) { if (c->type == JSON_ARRAY) { - if (c->expect == JSON_ITEM) { + if (f->expect == JSON_ITEM) { c->array().push_back(o); - c->expect = JSON_COMMA; + f->expect = JSON_COMMA; } else { j->error = "Expected a comma, not a list item"; return nullptr; } } else if (c->type == JSON_HASH) { - if (c->expect == JSON_VALUE) { + if (f->expect == JSON_VALUE) { c->entries().back().value = o; - c->expect = JSON_COMMA; - } else if (c->expect == JSON_KEY) { + f->expect = JSON_COMMA; + } else if (f->expect == JSON_KEY) { if (type != JSON_STRING) { j->error = "Hash key is not a string"; return nullptr; } c->entries().push_back({o, nullptr}); - c->expect = JSON_COLON; + f->expect = JSON_COLON; } else { j->error = "Expected a comma or colon"; return nullptr; @@ -229,8 +230,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c if (o == nullptr) { return nullptr; } - o->expect = JSON_ITEM; - j->container_stack.push_back(o); + j->container_stack.push_back({o, JSON_ITEM}); if (cb != nullptr) { cb(JSON_ARRAY, j, state); @@ -240,25 +240,26 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } case ']': { - json_object *cc = current_container(j); - if (cc == nullptr) { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { j->error = "Found ] at top level"; return nullptr; } + json_object *cc = f->container.get(); if (cc->type != JSON_ARRAY) { j->error = "Found ] not in an array"; return nullptr; } - if (cc->expect != JSON_COMMA) { - if (!(cc->expect == JSON_ITEM && cc->array().size() == 0)) { + if (f->expect != JSON_COMMA) { + if (!(f->expect == JSON_ITEM && cc->array().size() == 0)) { j->error = "Found ] without final element"; return nullptr; } } - json_object_ptr ret = j->container_stack.back(); + json_object_ptr ret = f->container; j->container_stack.pop_back(); return ret; } @@ -270,8 +271,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c if (o == nullptr) { return nullptr; } - o->expect = JSON_KEY; - j->container_stack.push_back(o); + j->container_stack.push_back({o, JSON_KEY}); if (cb != nullptr) { cb(JSON_HASH, j, state); @@ -281,25 +281,26 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } case '}': { - json_object *cc = current_container(j); - if (cc == nullptr) { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { j->error = "Found } at top level"; return nullptr; } + json_object *cc = f->container.get(); if (cc->type != JSON_HASH) { j->error = "Found } not in a hash"; return nullptr; } - if (cc->expect != JSON_COMMA) { - if (!(cc->expect == JSON_KEY && cc->entries().size() == 0)) { + if (f->expect != JSON_COMMA) { + if (!(f->expect == JSON_KEY && cc->entries().size() == 0)) { j->error = "Found } without final element"; return nullptr; } } - json_object_ptr ret = j->container_stack.back(); + json_object_ptr ret = f->container; j->container_stack.pop_back(); return ret; } @@ -366,17 +367,17 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c /////////////////////////// Comma case ',': { - json_object *cc = current_container(j); - if (cc != nullptr) { - if (cc->expect != JSON_COMMA) { + json_pull::parse_frame *f = current_frame(j); + if (f != nullptr) { + if (f->expect != JSON_COMMA) { j->error = "Found unexpected comma"; return nullptr; } - if (cc->type == JSON_HASH) { - cc->expect = JSON_KEY; + if (f->container->type == JSON_HASH) { + f->expect = JSON_KEY; } else { - cc->expect = JSON_ITEM; + f->expect = JSON_ITEM; } } @@ -390,18 +391,18 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c /////////////////////////// Colon case ':': { - json_object *cc = current_container(j); - if (cc == nullptr) { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { j->error = "Found colon at top level"; return nullptr; } - if (cc->expect != JSON_COLON) { + if (f->expect != JSON_COLON) { j->error = "Found unexpected colon"; return nullptr; } - cc->expect = JSON_VALUE; + f->expect = JSON_VALUE; if (cb != nullptr) { cb(JSON_COLON, j, state); diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index b4ac51a66..e21a656da 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -72,7 +72,6 @@ struct json_object { json_pull *parser = nullptr; json_type type; - int expect = 0; // used by the parser on JSON_ARRAY / JSON_HASH nodes json_object(json_type t) : type(t) {} json_object(json_type t, json_object *p, json_pull *pl) : parent(p), parser(pl), type(t) {} @@ -190,10 +189,17 @@ struct json_pull { ssize_t buffer_head = 0; // Stack of currently-open containers; the top is the innermost container - // being parsed. Replaces the previous single `container` pointer / parent - // walk, which previously required enable_shared_from_this - // on every json_object instance (16 extra bytes per node). - std::vector container_stack; + // being parsed. Each frame also remembers what token is expected next + // (an item, a comma, a key, a colon, or a value). This stack is the + // only place the parser-only `expect` state lives, so it does not + // pollute json_object once parsing finishes. Replaces the previous + // single `container` pointer / parent walk, which previously required + // enable_shared_from_this on every json_object instance. + struct parse_frame { + json_object_ptr container; + json_type expect; + }; + std::vector container_stack; json_object_ptr root; std::string number_buffer; From 1bf18d39cec5687bf85df9cda31bd73cfdcac55a Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 10:47:00 -0700 Subject: [PATCH 06/13] Discriminate json_number's three numeric slots into one union json_number used to carry three parallel 8-byte fields (a double plus both a 64-bit unsigned and a 64-bit signed slot for the large-integer cases) even though at most one of the integer slots is ever the canonical value for any given number. Collapse them into a discriminated union: enum repr_t { REPR_DOUBLE, REPR_LARGE_UNSIGNED, REPR_LARGE_SIGNED }; repr_t repr; union { double d; unsigned long long u; long long s; } value; Callers keep the same read API: number() returns the appropriate double, large_unsigned() returns the ull (or 0 if not currently stored that way), large_signed() likewise. Writes go through new set_number / set_large_unsigned / set_large_signed methods that keep the discriminator and the union value in sync. This was prompted by an observation that moving json_type to the end of the object should shrink things via tail-padding reuse. Empirically the type-at-end rearrangement saves nothing on its own (every subclass payload is 8-byte aligned so it can't slot into the 4-byte tail), but the discriminated-number redesign hits the same idea from a different direction: adding the 4-byte `repr` to json_number makes the class non-standard-layout, which lets the Itanium ABI pack `repr` into the base's 4-byte tail padding at offset 20. The union value then starts at the natural offset 24, and json_number ends at offset 32 -- a 33% reduction. Per-node sizes: json_object (TRUE/FALSE/NULL) 24 bytes json_number 32 bytes (was 48) json_string 48 bytes json_array 48 bytes json_hash 48 bytes Numbers dominate real GeoJSON (every coordinate is one), so the net memory win on a typical parse is substantial. Co-authored-by: Cursor --- jsonpull/jsonpull.cpp | 13 ++++----- jsonpull/jsonpull.h | 66 +++++++++++++++++++++++++++++++------------ jsontool.cpp | 2 +- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 0f2d99fe1..22e0cd94c 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -482,27 +482,26 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c json_object_ptr n = add_object(j, JSON_NUMBER); if (n != nullptr) { - n->number() = atof(j->number_buffer.c_str()); - n->large_signed() = 0; - n->large_unsigned() = 0; + double d = atof(j->number_buffer.c_str()); + n->set_number(d); #define MAX_SAFE_INTEGER 9007199254740991.0 #define MIN_SAFE_INTEGER -9007199254740991.0 - if (!decimal && n->number() > MAX_SAFE_INTEGER) { + if (!decimal && d > MAX_SAFE_INTEGER) { errno = 0; char *err = nullptr; unsigned long long ull = strtoull(j->number_buffer.c_str(), &err, 10); if (errno == 0 && (err == nullptr || *err == '\0')) { - n->large_unsigned() = ull; + n->set_large_unsigned(ull); } } - if (!decimal && n->number() < MIN_SAFE_INTEGER) { + if (!decimal && d < MIN_SAFE_INTEGER) { errno = 0; char *err = nullptr; long long ll = strtoll(j->number_buffer.c_str(), &err, 10); if (errno == 0 && (err == nullptr || *err == '\0')) { - n->large_signed() = ll; + n->set_large_signed(ll); } } } diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index e21a656da..4153b4b6b 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -82,12 +82,17 @@ struct json_object { inline std::string &string(); inline const std::string &string() const; - inline double &number(); + // Numbers are stored in a discriminated union (double / unsigned / + // signed) so a json_number is only 40 bytes instead of 48. The + // large_*() accessors return 0 when the number is not currently + // stored in that representation, matching the prior convention + // where "0" meant "not set, fall through to the next slot". inline double number() const; - inline unsigned long long &large_unsigned(); inline unsigned long long large_unsigned() const; - inline long long &large_signed(); inline long long large_signed() const; + inline void set_number(double d); + inline void set_large_unsigned(unsigned long long u); + inline void set_large_signed(long long s); inline std::vector &array(); inline const std::vector &array() const; @@ -97,9 +102,17 @@ struct json_object { }; struct json_number : json_object { - double number_value = 0; - unsigned long long large_unsigned_value = 0; - long long large_signed_value = 0; + enum repr_t { REPR_DOUBLE, + REPR_LARGE_UNSIGNED, + REPR_LARGE_SIGNED }; + + repr_t repr = REPR_DOUBLE; + union value_t { + double d; + unsigned long long u; + long long s; + value_t() : d(0) {} + } value; json_number() : json_object(JSON_NUMBER) {} json_number(json_object *p, json_pull *pl) : json_object(JSON_NUMBER, p, pl) {} @@ -135,29 +148,46 @@ inline const std::string &json_object::string() const { return static_cast(this)->string_value; } -inline double &json_object::number() { +inline double json_object::number() const { assert(type == JSON_NUMBER); - return static_cast(this)->number_value; + auto *n = static_cast(this); + switch (n->repr) { + case json_number::REPR_LARGE_UNSIGNED: + return static_cast(n->value.u); + case json_number::REPR_LARGE_SIGNED: + return static_cast(n->value.s); + case json_number::REPR_DOUBLE: + default: + return n->value.d; + } } -inline double json_object::number() const { +inline unsigned long long json_object::large_unsigned() const { assert(type == JSON_NUMBER); - return static_cast(this)->number_value; + auto *n = static_cast(this); + return n->repr == json_number::REPR_LARGE_UNSIGNED ? n->value.u : 0; } -inline unsigned long long &json_object::large_unsigned() { +inline long long json_object::large_signed() const { assert(type == JSON_NUMBER); - return static_cast(this)->large_unsigned_value; + auto *n = static_cast(this); + return n->repr == json_number::REPR_LARGE_SIGNED ? n->value.s : 0; } -inline unsigned long long json_object::large_unsigned() const { +inline void json_object::set_number(double d) { assert(type == JSON_NUMBER); - return static_cast(this)->large_unsigned_value; + auto *n = static_cast(this); + n->repr = json_number::REPR_DOUBLE; + n->value.d = d; } -inline long long &json_object::large_signed() { +inline void json_object::set_large_unsigned(unsigned long long u) { assert(type == JSON_NUMBER); - return static_cast(this)->large_signed_value; + auto *n = static_cast(this); + n->repr = json_number::REPR_LARGE_UNSIGNED; + n->value.u = u; } -inline long long json_object::large_signed() const { +inline void json_object::set_large_signed(long long s) { assert(type == JSON_NUMBER); - return static_cast(this)->large_signed_value; + auto *n = static_cast(this); + n->repr = json_number::REPR_LARGE_SIGNED; + n->value.s = s; } inline std::vector &json_object::array() { diff --git a/jsontool.cpp b/jsontool.cpp index 9a8dbfc31..79077396c 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -326,7 +326,7 @@ void join_csv(json_object_ptr j) { vo = s; } else if (attr_type == JSON_NUMBER) { auto n = std::make_shared(properties.get(), properties->parser); - n->number_value = atof(v.c_str()); + n->set_number(atof(v.c_str())); vo = n; } else { abort(); From 05b17620771c7f6d3e736d7766c58638cacb5a45 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 14:25:39 -0700 Subject: [PATCH 07/13] Fix bugs flagged in code review of jsonpull C++ port - jsontool.cpp `out()`: route JSON_NUMBER (and anything else non-string) through `json_stringify` instead of `o->string()`, which now asserts on a non-string type and would crash `--extract` on numeric attributes. - geojson.{hpp,cpp} `json_end_map`: take `json_pull_ptr` by reference so the caller's shared_ptr is released, null-guard before touching `jp->source`, and clear `jp->source` after delete to avoid a dangling pointer. - jsonpull/jsonpull.cpp: low-surrogate range check was comparing the outer-loop byte `c` instead of the parsed code unit `ch`, breaking surrogate-pair decoding for some \\uXXXX escapes. Pre-existing bug preserved across the port. - tile-join.cpp `handle_vector_layers`: require the field value to have type JSON_STRING (and the key to be non-null) before calling `string()`; the previous truthy `type` check would assert on a non-string value. Co-authored-by: Cursor --- geojson.cpp | 6 +++++- geojson.hpp | 2 +- jsonpull/jsonpull.cpp | 2 +- jsontool.cpp | 6 +++++- tile-join.cpp | 3 ++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index 66cad85bb..3a64e08f7 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -306,7 +306,11 @@ json_pull_ptr json_begin_map(char *map, long long len) { return json_begin(json_map_read, jm); } -void json_end_map(json_pull_ptr jp) { +void json_end_map(json_pull_ptr &jp) { + if (jp == nullptr) { + return; + } delete (struct jsonmap *) jp->source; + jp->source = nullptr; json_end(jp); } diff --git a/geojson.hpp b/geojson.hpp index aca480082..cb0776a46 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -22,7 +22,7 @@ struct parse_json_args { }; json_pull_ptr json_begin_map(char *map, long long len); -void json_end_map(json_pull_ptr jp); +void json_end_map(json_pull_ptr &jp); void parse_json(struct serialization_state *sst, json_pull_ptr jp, int layer, std::string layername); void *run_parse_json(void *v); diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 22e0cd94c..3929e1f02 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -551,7 +551,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c surrogate = ch; } continue; - } else if (ch >= 0xdc00 && c <= 0xdfff) { + } else if (ch >= 0xdc00 && ch <= 0xdfff) { if (surrogate >= 0) { long c1 = surrogate - 0xd800; long c2 = ch - 0xdc00; diff --git a/jsontool.cpp b/jsontool.cpp index 79077396c..52cb1c334 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -148,9 +148,13 @@ void out(std::string const &s, int type, json_object_ptr properties) { json_object_ptr o = json_hash_get(properties, extract); if (o != nullptr) { found = true; - if (o->type == JSON_STRING || o->type == JSON_NUMBER) { + if (o->type == JSON_STRING) { extracted = sort_quote(o->string().c_str()); } else { + // Numbers, booleans, null, and any other non-string + // values are rendered via json_stringify(); calling + // o->string() here would assert because the type-tagged + // accessor requires JSON_STRING. extracted = sort_quote(json_stringify(o).c_str()); } } diff --git a/tile-join.cpp b/tile-join.cpp index 5f7af3686..b729230c2 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -1035,7 +1035,8 @@ void handle_vector_layers(json_object_ptr vector_layers, std::maparray()[i], "fields"); if (fields != nullptr && fields->type == JSON_HASH) { for (const auto &e : fields->entries()) { - if (e.key->type == JSON_STRING && e.value->type) { + if (e.key != nullptr && e.key->type == JSON_STRING && + e.value != nullptr && e.value->type == JSON_STRING) { const std::string &desc2 = e.value->string(); if (desc2 != "Number" && From 29b5dff33392142f57dff0d168090f7da90e860e Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 14:33:34 -0700 Subject: [PATCH 08/13] Add jsonpull regression test for surrogate-pair decoding Covers the `c` vs `ch` bug fixed in the previous commit: parsing "\uD83D\uE000" (a valid high surrogate followed by a non-surrogate BMP code point) used to mis-classify U+E000 as a low surrogate and combine the two units into U+1F400 (F0 9F 90 80). The fixed code flushes the stale high surrogate as standalone CESU-8 (ED A0 BD) and then encodes U+E000 normally as EE 80 80. Verified the test fails under the pre-fix logic. Co-authored-by: Cursor --- unit.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/unit.cpp b/unit.cpp index fff6b7c4f..0db13ee94 100644 --- a/unit.cpp +++ b/unit.cpp @@ -6,6 +6,7 @@ #include "mvt.hpp" #include "projection.hpp" #include "geometry.hpp" +#include "jsonpull/jsonpull.h" #include #include @@ -136,3 +137,28 @@ TEST_CASE("line_is_too_small") { dv.emplace_back(VT_LINETO, -51864809, 2683873977); REQUIRE(line_is_too_small(dv, 0, 10)); } + +// Regression test for the surrogate-decoding bug that compared the leftover +// outer-loop byte `c` against `0xdfff` instead of the parsed code unit `ch`. +// For a string like "\uD83D\uE000" (a valid high surrogate followed by a +// non-surrogate BMP code point) the buggy version would mis-classify +// U+E000 as a low surrogate and combine the two units into the four-byte +// UTF-8 sequence F0 9F 90 80 (U+1F400). The fixed version flushes the +// stale high surrogate as standalone CESU-8 (ED A0 BD) and then encodes +// U+E000 normally as EE 80 80. +TEST_CASE("jsonpull surrogate-pair regression", "[jsonpull][surrogate]") { + json_pull_ptr jp = json_begin_string("\"\\uD83D\\uE000\""); + json_object_ptr o = json_read_tree(jp); + + REQUIRE(jp->error == nullptr); + REQUIRE(o != nullptr); + REQUIRE(o->type == JSON_STRING); + + const std::string expected = "\xED\xA0\xBD\xEE\x80\x80"; + REQUIRE(o->string() == expected); + + // Sanity check: the buggy output (a single 4-byte UTF-8 sequence for + // U+1F400) must not be what we got. + const std::string buggy = "\xF0\x9F\x90\x80"; + REQUIRE(o->string() != buggy); +} From 3f526ccad79d3301c4c956f54ce674c8218f5315 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 18:28:11 -0700 Subject: [PATCH 09/13] Cheap perf wins in jsonpull C++ port Profiling tl_2022_us_county.json (sample(1) on Apple Silicon) showed ~38% of parse time in allocator work and ~14% in std::string::push_back during string-token construction. These changes target the low-hanging fruit from that profile: - Pre-reserve 2 slots in json_array and 4 slots in json_hash so coordinate `[x, y]` pairs and typical GeoJSON property maps avoid the 0 -> 1 -> 2 -> 4 vector-growth chain (and the shared_ptr copies it incurs). - Reuse a parser-wide std::string buffer for JSON_STRING tokens instead of constructing a fresh local std::string per token. The buffer is cleared (capacity preserved) at the start of each token and copied into the final json_string, so once it has grown to the longest string seen it stops reallocating entirely. - std::move the freshly-created container shared_ptr into the parser container stack in the `[` and `{` handlers, and move it out of the frame on the matching `]` / `}`. Each move skips one atomic inc/dec round-trip per container open and close. On a tl_2022_us_county.json benchmark (4-iter user-time mean, Apple Silicon, /usr/bin/time): - main baseline: ~8.17s - jsonpull-cpp before these changes: ~10.90s (+33%) - jsonpull-cpp with these changes: ~9.33s (+14%) So this commit recovers roughly half of the post-port regression. The remaining gap is dominated by shared_ptr atomic refcount traffic on the parse tree and per-node heap allocations, which would require the larger unique_ptr/arena reworks to address. Co-authored-by: Cursor --- jsonpull/jsonpull.cpp | 29 +++++++++++++++++++++++------ jsonpull/jsonpull.h | 25 +++++++++++++++++++++---- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 3929e1f02..c96adcc09 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -230,7 +230,10 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c if (o == nullptr) { return nullptr; } - j->container_stack.push_back({o, JSON_ITEM}); + // add_object already installed `o` in the parent (or the + // parser's root); moving the local copy into the frame + // avoids one shared_ptr atomic inc/dec pair per container. + j->container_stack.push_back({std::move(o), JSON_ITEM}); if (cb != nullptr) { cb(JSON_ARRAY, j, state); @@ -259,7 +262,9 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } } - json_object_ptr ret = f->container; + // Move the container out of the frame so pop_back doesn't + // drop the last reference; saves one atomic inc/dec. + json_object_ptr ret = std::move(f->container); j->container_stack.pop_back(); return ret; } @@ -271,7 +276,9 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c if (o == nullptr) { return nullptr; } - j->container_stack.push_back({o, JSON_KEY}); + // See the [ case above: move into the frame to skip a + // shared_ptr atomic inc/dec round-trip. + j->container_stack.push_back({std::move(o), JSON_KEY}); if (cb != nullptr) { cb(JSON_HASH, j, state); @@ -300,7 +307,8 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } } - json_object_ptr ret = f->container; + // See the ] case: move out to skip an atomic refcount round-trip. + json_object_ptr ret = std::move(f->container); j->container_stack.pop_back(); return ret; } @@ -511,7 +519,11 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c /////////////////////////// Strings case '"': { - std::string val; + // Reuse the parser-wide string buffer so we don't construct a + // fresh std::string (with its inevitable SSO->heap promotion + // and capacity doublings) for every JSON_STRING token. + std::string &val = j->string_buffer; + val.clear(); int surrogate = -1; while ((c = read_wrap(j)) != EOF) { @@ -632,7 +644,12 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c json_object_ptr s = add_object(j, JSON_STRING); if (s != nullptr) { - s->string() = std::move(val); + // Copy (don't move) so j->string_buffer retains its + // grown capacity for the next token. The copy is a + // single right-sized allocation plus one memcpy, which + // is cheaper than the multiple capacity doublings the + // per-token std::string would otherwise incur. + s->string() = val; } return s; } diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index 4153b4b6b..a6d5cc6ce 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -128,15 +128,25 @@ struct json_string : json_object { struct json_array : json_object { std::vector array_value; - json_array() : json_object(JSON_ARRAY) {} - json_array(json_object *p, json_pull *pl) : json_object(JSON_ARRAY, p, pl) {} + // Coordinate-heavy GeoJSON dominates the parse workload, and every + // `[x, y]` (or `[x, y, z]`) pair would otherwise force the inner + // vector through 0 -> 1 -> 2 -> 4 growths plus the matching + // shared_ptr copies. Reserving 2 slots up front eliminates those + // reallocations for the common case and adds only a single small + // allocation for larger rings (which still grow geometrically). + json_array() : json_object(JSON_ARRAY) { array_value.reserve(2); } + json_array(json_object *p, json_pull *pl) : json_object(JSON_ARRAY, p, pl) { array_value.reserve(2); } }; struct json_hash : json_object { std::vector entries_value; - json_hash() : json_object(JSON_HASH) {} - json_hash(json_object *p, json_pull *pl) : json_object(JSON_HASH, p, pl) {} + // Most GeoJSON property hashes have a handful of keys (type, id, + // properties, geometry, plus a few attribute fields). Reserving 4 + // slots avoids the 0 -> 1 -> 2 -> 4 growth chain for the typical + // case while only modestly over-allocating for one-key hashes. + json_hash() : json_object(JSON_HASH) { entries_value.reserve(4); } + json_hash(json_object *p, json_pull *pl) : json_object(JSON_HASH, p, pl) { entries_value.reserve(4); } }; inline std::string &json_object::string() { @@ -232,7 +242,14 @@ struct json_pull { std::vector container_stack; json_object_ptr root; + // Scratch buffers reused across tokens so we don't reallocate per + // number/string. number_buffer accumulates raw digits before atof(); + // string_buffer accumulates decoded bytes before being copied into + // the final json_string. Both are cleared (capacity preserved) at + // the start of each token, so once they grow to the largest seen + // size they stop reallocating entirely. std::string number_buffer; + std::string string_buffer; }; json_pull_ptr json_begin_file(FILE *f); From 1a7075e3ab3701597fafceddc80548eab1b1daec Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 19:23:47 -0700 Subject: [PATCH 10/13] Make json_free actually free the subtree In the C++ port, json_free was just `o.reset()`, which dropped the caller's reference but left the subtree alive: the parent's vector slot kept it allocated, and for line-delimited streams the parser's jp->root co-owned it until the next top-level value started parsing. That defeated the geojson-loop pattern of calling json_free on each feature after serializing it, which is supposed to release the feature so it doesn't sit in memory while subsequent ones are parsed. Restore the historical "remove this from the tree" semantics by splicing the node out of its parent (sharing splice_from_parent with json_disconnect) and clearing jp->root when the node is the parser's current top-level value, then dropping the caller's reference. Two unit tests pin this down: a pruning test parses "[[1, 2], [3, 4], [5, 6]]" element-wise and confirms that calling json_free on [3, 4] leaves the outer array with just [1, 2] and [5, 6]; a top-level test uses a weak_ptr observer to confirm that json_free on the parser's root really destroys the tree. Co-authored-by: Cursor --- jsonpull/jsonpull.cpp | 113 ++++++++++++++++++++++++++++-------------- unit.cpp | 71 ++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 38 deletions(-) diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index c96adcc09..6e8a534db 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -675,7 +675,81 @@ json_object_ptr json_read_tree(json_pull_ptr p) { return nullptr; } +// Splice `o` out of its parent's array or object, dropping the +// parent's owning reference. After this returns, the parent no longer +// holds any pointer to `o`; the caller's reference is the only thing +// keeping the subtree alive. +// +// For a hash, removing a single key or value individually would +// disturb the surrounding key/value pairing, so we replace the removed +// half with a placeholder JSON_NULL and only erase the entry once both +// halves have been detached. This matches the historical +// json_disconnect semantics. +static void splice_from_parent(json_object *o) { + if (o == nullptr) { + return; + } + + json_object *parent = o->parent; + if (parent == nullptr) { + return; + } + + if (parent->type == JSON_ARRAY) { + auto &arr = parent->array(); + for (size_t i = 0; i < arr.size(); i++) { + if (arr[i].get() == o) { + arr.erase(arr.begin() + i); + break; + } + } + } else if (parent->type == JSON_HASH) { + auto &entries = parent->entries(); + for (size_t i = 0; i < entries.size(); i++) { + auto &e = entries[i]; + if (e.key.get() == o) { + e.key = fabricate_object(parent->parser, parent, JSON_NULL); + if (e.value != nullptr && e.value->type == JSON_NULL && e.key->type == JSON_NULL) { + entries.erase(entries.begin() + i); + } + break; + } + if (e.value.get() == o) { + e.value = fabricate_object(parent->parser, parent, JSON_NULL); + if (e.key != nullptr && e.key->type == JSON_NULL && e.value->type == JSON_NULL) { + entries.erase(entries.begin() + i); + } + break; + } + } + } +} + +// json_free splices `o` out of its parent (if any) so that the +// parent no longer keeps the subtree alive, then drops the caller's +// reference. The subtree is freed when the last reference is gone +// (typically right here, since the parent's reference was just +// dropped). geojson-loop.cpp relies on this to release each feature +// after it has been serialized, so that already-serialized features +// don't sit in memory while subsequent features are parsed. +// +// If `o` is the parser's current root (the most recently completed +// top-level value), drop the parser's reference too -- otherwise a +// line-delimited stream would always hold the previously-completed +// feature until the next one started parsing. +// +// Unlike json_disconnect, this does NOT walk the subtree clearing +// parent/parser back-pointers, because the subtree is about to be +// destroyed and those pointers will never be observed again. void json_free(json_object_ptr &o) { + if (o != nullptr) { + splice_from_parent(o.get()); + + json_pull *parser = o->parser; + if (parser != nullptr && parser->root.get() == o.get()) { + parser->root.reset(); + } + } o.reset(); } @@ -710,44 +784,7 @@ void json_disconnect(json_object_ptr o) { // Splice o out of its parent's array or object. The parent's vector // holds the shared_ptr to this child; erasing it removes one reference, // but the caller still holds `o`, so the subtree stays alive. - - json_object *parent = o->parent; - if (parent != nullptr) { - if (parent->type == JSON_ARRAY) { - auto &arr = parent->array(); - for (size_t i = 0; i < arr.size(); i++) { - if (arr[i].get() == o.get()) { - arr.erase(arr.begin() + i); - break; - } - } - } else if (parent->type == JSON_HASH) { - auto &entries = parent->entries(); - - for (size_t i = 0; i < entries.size(); i++) { - auto &e = entries[i]; - if (e.key.get() == o.get()) { - // Leave a NULL placeholder in the key slot so the - // surrounding value isn't shifted; if the corresponding - // value is also detached the pair is removed below. - e.key = fabricate_object(parent->parser, parent, JSON_NULL); - - if (e.value != nullptr && e.value->type == JSON_NULL && e.key->type == JSON_NULL) { - entries.erase(entries.begin() + i); - } - break; - } - if (e.value.get() == o.get()) { - e.value = fabricate_object(parent->parser, parent, JSON_NULL); - - if (e.key != nullptr && e.key->type == JSON_NULL && e.value->type == JSON_NULL) { - entries.erase(entries.begin() + i); - } - break; - } - } - } - } + splice_from_parent(o.get()); // Drop the parser's reference to this subtree if it was the root. json_pull *parser = o->parser; diff --git a/unit.cpp b/unit.cpp index 0db13ee94..bbd1a6d2a 100644 --- a/unit.cpp +++ b/unit.cpp @@ -162,3 +162,74 @@ TEST_CASE("jsonpull surrogate-pair regression", "[jsonpull][surrogate]") { const std::string buggy = "\xF0\x9F\x90\x80"; REQUIRE(o->string() != buggy); } + +// geojson-loop.cpp calls json_free(j) after jfa->add_feature has +// serialized the feature, intending to drop the JSON subtree from the +// in-progress parse tree so that already-serialized features don't sit +// in memory while subsequent features are parsed. That intent was +// never tested; this test pins it down. The pre-fix behavior of +// json_free was a bare unique_ptr/shared_ptr reset that only dropped +// the caller's local reference; the parent container kept the subtree +// alive, so memory grew until the top-level parse completed. +TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") { + json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]"); + + json_object_ptr outer; + int arrays_seen = 0; + + json_object_ptr j; + while ((j = json_read(jp)) != nullptr) { + if (j->type != JSON_ARRAY) { + continue; + } + arrays_seen++; + if (arrays_seen == 2) { + // This is [3, 4]; verify, then ask the parser to drop it. + REQUIRE(j->array().size() == 2); + REQUIRE(j->array()[0]->number() == 3); + REQUIRE(j->array()[1]->number() == 4); + json_free(j); + } else if (j->parent == nullptr) { + // The completed outer array. + outer = j; + break; + } + } + + REQUIRE(outer != nullptr); + REQUIRE(outer->type == JSON_ARRAY); + REQUIRE(outer->array().size() == 2); + + // First surviving element: [1, 2]. + REQUIRE(outer->array()[0]->type == JSON_ARRAY); + REQUIRE(outer->array()[0]->array().size() == 2); + REQUIRE(outer->array()[0]->array()[0]->number() == 1); + REQUIRE(outer->array()[0]->array()[1]->number() == 2); + + // Second surviving element (previously third): [5, 6]. + REQUIRE(outer->array()[1]->type == JSON_ARRAY); + REQUIRE(outer->array()[1]->array().size() == 2); + REQUIRE(outer->array()[1]->array()[0]->number() == 5); + REQUIRE(outer->array()[1]->array()[1]->number() == 6); +} + +// The companion case to the pruning test above: in a line-delimited +// stream, each feature returned by json_read is a top-level value +// with no parent, but the parser still co-owns it via jp->root. +// json_free must drop that parser reference too, otherwise the +// just-serialized feature stays in memory until the next feature +// starts parsing. +TEST_CASE("json_free releases a top-level value held by the parser", "[jsonpull][memory]") { + std::weak_ptr observer; + + json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": [2, 3]})"); + json_object_ptr j = json_read_tree(jp); + REQUIRE(j != nullptr); + REQUIRE(j->type == JSON_HASH); + REQUIRE(j->parent == nullptr); + observer = j; + + json_free(j); + + REQUIRE(observer.expired()); +} From 65beb3f0d7c4e0c37884f3a7435e97777ea7fa41 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 20:44:25 -0700 Subject: [PATCH 11/13] Migrate jsonpull to unique_ptr ownership Replaces the shared_ptr-based json_object_ptr with a unique_ptr that has a stateless custom deleter dispatching on json_object::type before calling the right subclass destructor. Eliminates per-node atomic reference-counting and the control-block allocation that shared_ptr required for every node in the tree. API now distinguishes owning and borrowing pointers explicitly: - json_read / json_read_separators / json_hash_get return raw json_object * (borrowed from the parser-owned tree). - json_read_tree / json_disconnect return json_object_ptr (caller takes ownership; back-pointers are cleared so the subtree can outlive the parser). - json_free / json_context / json_stringify take raw pointers. - The parser's container_stack holds raw pointers; jp->root keeps unique_ptr ownership of the most recent top-level value. Internally, take_from_owner moves the unique_ptr out of whichever parent vector / hash entry / parser root owned it, which both json_free and json_disconnect rely on. In the streaming parsers (parse_feature, parse_layers, the geojson-loop callback), we are careful to free `j` only after we have processed a complete Feature: json_read returns each token as the tree is being built up, and freeing an intermediate node would splice it out of the surrounding hash and corrupt the in-progress feature. Benchmark (tl_2022_us_county.json, -z0 --extend-zooms-if-still-dropping, median of 5 runs on macOS arm64): 8.5s, vs 10.6s with shared_ptr and 8.7s on the pre-refactor C baseline. Co-authored-by: Cursor --- clip.cpp | 4 +- evaluator.cpp | 22 ++--- evaluator.hpp | 8 +- geobuf.cpp | 6 +- geojson-loop.cpp | 32 +++---- geojson-loop.hpp | 6 +- geojson.cpp | 30 +++--- geojson.hpp | 2 +- geometry.hpp | 4 +- jsonpull/jsonpull.cpp | 213 +++++++++++++++++++++--------------------- jsonpull/jsonpull.h | 156 ++++++++++++++++++++++++------- jsontool.cpp | 30 +++--- main.cpp | 14 +-- overzoom.cpp | 2 +- plugin.cpp | 42 +++++---- plugin.hpp | 2 +- pmtiles_file.cpp | 8 +- read_json.cpp | 39 ++++---- read_json.hpp | 8 +- tile-join.cpp | 22 ++--- tile.cpp | 10 +- tile.hpp | 2 +- unit.cpp | 43 ++++++--- 23 files changed, 412 insertions(+), 293 deletions(-) diff --git a/clip.cpp b/clip.cpp index a50a7eda6..72a1fb3a6 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1221,7 +1221,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object_ptr filter, bool preserve_input_order, + bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, @@ -1457,7 +1457,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object_ptr filter, bool preserve_input_order, + bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, diff --git a/evaluator.cpp b/evaluator.cpp index efd27746b..ce1e576c1 100644 --- a/evaluator.cpp +++ b/evaluator.cpp @@ -9,7 +9,7 @@ #include "milo/dtoa_milo.h" #include "text.hpp" -int compare(mvt_value const &one, json_object_ptr two, bool &fail) { +int compare(mvt_value const &one, json_object *two, bool &fail) { switch (one.type) { case mvt_string: if (two->type != JSON_STRING) { @@ -91,7 +91,7 @@ int compare(mvt_value const &one, json_object_ptr two, bool &fail) { // 0: false // 1: true // -1: incomparable (sql null), treated as false in final output -static int eval(std::function feature, json_object_ptr f, std::set &exclude_attributes, std::vector const &unidecode_data) { +static int eval(std::function feature, json_object *f, std::set &exclude_attributes, std::vector const &unidecode_data) { if (f != nullptr) { if (f->type == JSON_TRUE) { return 1; @@ -188,7 +188,7 @@ static int eval(std::function feature, json_obje } bool fail = false; - int cmp = compare(ff, f->array()[2], fail); + int cmp = compare(ff, f->array()[2].get(), fail); if (fail) { static bool warned = false; @@ -237,7 +237,7 @@ static int eval(std::function feature, json_obje } for (size_t i = 1; i < f->array().size(); i++) { - int out = eval(feature, f->array()[i], exclude_attributes, unidecode_data); + int out = eval(feature, f->array()[i].get(), exclude_attributes, unidecode_data); if (out >= 0) { // nulls are ignored in boolean and/or expressions if (op == "all") { @@ -289,7 +289,7 @@ static int eval(std::function feature, json_obje bool found = false; for (size_t i = 2; i < f->array().size(); i++) { bool fail = false; - int cmp = compare(ff, f->array()[i], fail); + int cmp = compare(ff, f->array()[i].get(), fail); if (fail) { static bool warned = false; @@ -324,7 +324,7 @@ static int eval(std::function feature, json_obje exit(EXIT_FILTER); } - bool ok = eval(feature, f->array()[2], exclude_attributes, unidecode_data) > 0; + bool ok = eval(feature, f->array()[2].get(), exclude_attributes, unidecode_data) > 0; if (!ok) { exclude_attributes.insert(f->array()[1]->string()); } @@ -336,14 +336,14 @@ static int eval(std::function feature, json_obje exit(EXIT_FILTER); } -bool evaluate(std::function feature, std::string const &layer, json_object_ptr filter, std::set &exclude_attributes, std::vector const &unidecode_data) { +static bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { if (filter == nullptr || filter->type != JSON_HASH) { fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter).c_str()); exit(EXIT_JSON); } bool ok = true; - json_object_ptr f; + json_object *f; f = json_hash_get(filter, layer.c_str()); if (ok && f != nullptr) { @@ -371,7 +371,6 @@ json_object_ptr read_filter(const char *fname) { fprintf(stderr, "%s: %s\n", fname, jp->error); exit(EXIT_JSON); } - json_disconnect(filter); fclose(fp); return filter; } @@ -384,11 +383,10 @@ json_object_ptr parse_filter(const char *s) { fprintf(stderr, "%s\n", jp->error); exit(EXIT_JSON); } - json_disconnect(filter); return filter; } -bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object_ptr filter, std::set &exclude_attributes, std::vector const &unidecode_data) { +bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { std::function getter = [&](std::string const &key) { auto f = feature.find(key); if (f != feature.end()) { @@ -404,7 +402,7 @@ bool evaluate(std::unordered_map const &feature, std::st return evaluate(getter, layer, filter, exclude_attributes, unidecode_data); } -bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object_ptr filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data) { +bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data) { std::function getter = [&](std::string const &key) { const static std::string dollar_id = "$id"; if (key == dollar_id && feat.has_id) { diff --git a/evaluator.hpp b/evaluator.hpp index 99a91fb76..a2c234fb9 100644 --- a/evaluator.hpp +++ b/evaluator.hpp @@ -7,10 +7,14 @@ #include "jsonpull/jsonpull.h" #include "mvt.hpp" -bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object_ptr filter, std::set &exclude_attributes, std::vector const &unidecode_data); +// The `filter` parameters take a borrowed pointer; the caller (in +// main.cpp, tile-join, overzoom) keeps the json_object_ptr alive +// across every per-feature evaluate() call. A raw pointer avoids +// touching unique_ptr at all on this hot path. +bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data); json_object_ptr parse_filter(const char *s); json_object_ptr read_filter(const char *fname); -bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object_ptr filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data); +bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data); #endif diff --git a/geobuf.cpp b/geobuf.cpp index 721c675cc..6481695e5 100644 --- a/geobuf.cpp +++ b/geobuf.cpp @@ -398,17 +398,17 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vectortype == JSON_NUMBER)) { sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } - json_object_ptr max = json_hash_get(o, "maxzoom"); + json_object *max = json_hash_get(o, "maxzoom"); if (max != nullptr && (max->type == JSON_NUMBER)) { sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } - json_object_ptr tlayer = json_hash_get(o, "layer"); + json_object *tlayer = json_hash_get(o, "layer"); if (tlayer != nullptr && (tlayer->type == JSON_STRING)) { layername = tlayer->string(); } diff --git a/geojson-loop.cpp b/geojson-loop.cpp index 75149e94a..e4f2f40b9 100644 --- a/geojson-loop.cpp +++ b/geojson-loop.cpp @@ -25,7 +25,7 @@ static const char *geometry_names[GEOM_TYPES] = { }; // XXX duplicated -static void json_context(json_object_ptr j) { +static void json_context(json_object *j) { std::string s = json_stringify(j); if (s.size() >= 500) { @@ -36,18 +36,18 @@ static void json_context(json_object_ptr j) { fprintf(stderr, "in JSON object %s\n", s.c_str()); } -void parse_json(json_feature_action *jfa, json_pull_ptr jp) { +void parse_json(json_feature_action *jfa, json_pull_ptr &jp) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; while (1) { - json_object_ptr j = json_read(jp); + json_object *j = json_read(jp); if (j == nullptr) { if (jp->error != nullptr) { fprintf(stderr, "%s:%d: %s: ", jfa->fname.c_str(), jp->line, jp->error); if (jp->root != nullptr) { - json_context(jp->root); + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } @@ -65,7 +65,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { } } - json_object_ptr type = json_hash_get(j, "type"); + json_object *type = json_hash_get(j, "type"); if (type == nullptr || type->type != JSON_STRING) { continue; } @@ -84,14 +84,14 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { if (j->parent != nullptr) { if (j->parent->type == JSON_ARRAY && j->parent->parent != nullptr) { if (j->parent->parent->type == JSON_HASH) { - json_object_ptr geometries = json_hash_get(j->parent->parent, "geometries"); + json_object *geometries = json_hash_get(j->parent->parent, "geometries"); if (geometries != nullptr) { // Parent of Parent must be a GeometryCollection is_geometry = 0; } } } else if (j->parent->type == JSON_HASH) { - json_object_ptr geometry = json_hash_get(j->parent, "geometry"); + json_object *geometry = json_hash_get(j->parent, "geometry"); if (geometry != nullptr) { // Parent must be a Feature is_geometry = 0; @@ -101,10 +101,10 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { } if (is_geometry) { - json_object *jo = j.get(); + json_object *jo = j; while (jo != nullptr) { if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { - if (json_hash_get(jo->parent, "properties").get() == jo) { + if (json_hash_get(jo->parent, "properties") == jo) { // Ancestor is the value corresponding to a properties key is_geometry = 0; break; @@ -140,7 +140,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { } found_features++; - json_object_ptr geometry = json_hash_get(j, "geometry"); + json_object *geometry = json_hash_get(j, "geometry"); if (geometry == nullptr) { fprintf(stderr, "%s:%d: feature with no geometry: ", jfa->fname.c_str(), jp->line); json_context(j); @@ -148,7 +148,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { continue; } - json_object_ptr properties = json_hash_get(j, "properties"); + json_object *properties = json_hash_get(j, "properties"); if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "%s:%d: feature without properties hash: ", jfa->fname.c_str(), jp->line); json_context(j); @@ -158,10 +158,10 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { bool is_feature = true; { - json_object *jo = j.get(); + json_object *jo = j; while (jo != nullptr) { if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { - if (json_hash_get(jo->parent, "properties").get() == jo) { + if (json_hash_get(jo->parent, "properties") == jo) { // Ancestor is the value corresponding to a properties key is_feature = false; break; @@ -174,10 +174,10 @@ void parse_json(json_feature_action *jfa, json_pull_ptr jp) { continue; } - json_object_ptr tippecanoe = json_hash_get(j, "tippecanoe"); - json_object_ptr id = json_hash_get(j, "id"); + json_object *tippecanoe = json_hash_get(j, "tippecanoe"); + json_object *id = json_hash_get(j, "id"); - json_object_ptr geometries = json_hash_get(geometry, "geometries"); + json_object *geometries = json_hash_get(geometry, "geometries"); if (geometries != nullptr && geometries->type == JSON_ARRAY) { jfa->add_feature(geometries, true, properties, id, tippecanoe, j); } else { diff --git a/geojson-loop.hpp b/geojson-loop.hpp index f1b26584b..acdb43d7f 100644 --- a/geojson-loop.hpp +++ b/geojson-loop.hpp @@ -4,8 +4,8 @@ struct json_feature_action { std::string fname; - virtual int add_feature(json_object_ptr geometry, bool geometrycollection, json_object_ptr properties, json_object_ptr id, json_object_ptr tippecanoe, json_object_ptr feature) = 0; - virtual void check_crs(json_object_ptr j) = 0; + virtual int add_feature(json_object *geometry, bool geometrycollection, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) = 0; + virtual void check_crs(json_object *j) = 0; }; -void parse_json(json_feature_action *action, json_pull_ptr jp); +void parse_json(json_feature_action *action, json_pull_ptr &jp); diff --git a/geojson.cpp b/geojson.cpp index 3a64e08f7..8997c5a28 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -40,8 +40,8 @@ #include "milo/dtoa_milo.h" #include "errors.hpp" -int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr geometry, json_object_ptr properties, json_object_ptr id, int layer, json_object_ptr tippecanoe, json_object_ptr feature, std::string const &layername) { - json_object_ptr geometry_type = json_hash_get(geometry, "type"); +int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string const &layername) { + json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == nullptr) { static int warned = 0; if (!warned) { @@ -59,7 +59,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g return 0; } - json_object_ptr coordinates = json_hash_get(geometry, "coordinates"); + json_object *coordinates = json_hash_get(geometry, "coordinates"); if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: feature without coordinates array: ", sst->fname, sst->line); json_context(feature); @@ -83,17 +83,17 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g std::string tippecanoe_layername = layername; if (tippecanoe != nullptr) { - json_object_ptr min = json_hash_get(tippecanoe, "minzoom"); + json_object *min = json_hash_get(tippecanoe, "minzoom"); if (min != nullptr && (min->type == JSON_NUMBER)) { tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } - json_object_ptr max = json_hash_get(tippecanoe, "maxzoom"); + json_object *max = json_hash_get(tippecanoe, "maxzoom"); if (max != nullptr && (max->type == JSON_NUMBER)) { tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } - json_object_ptr ln = json_hash_get(tippecanoe, "layer"); + json_object *ln = json_hash_get(tippecanoe, "layer"); if (ln != nullptr && (ln->type == JSON_STRING)) { tippecanoe_layername = ln->string(); } @@ -186,7 +186,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g for (const auto &e : entries) { if (e.key->type == JSON_STRING) { - serial_val sv = stringify_value(e.value, sst->fname, sst->line, feature); + serial_val sv = stringify_value(e.value.get(), sst->fname, sst->line, feature); full_keys.emplace_back(key_pool.pool(e.key->string().c_str())); values.push_back(std::move(sv)); @@ -214,12 +214,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object_ptr g return serialize_feature(sst, sf, tippecanoe_layername); } -void check_crs(json_object_ptr j, const char *reading) { - json_object_ptr crs = json_hash_get(j, "crs"); +void check_crs(json_object *j, const char *reading) { + json_object *crs = json_hash_get(j, "crs"); if (crs != nullptr) { - json_object_ptr properties = json_hash_get(crs, "properties"); + json_object *properties = json_hash_get(crs, "properties"); if (properties != nullptr) { - json_object_ptr name = json_hash_get(properties, "name"); + json_object *name = json_hash_get(properties, "name"); if (name != nullptr && name->type == JSON_STRING) { if (name->string() != projection->alias) { if (!quiet) { @@ -237,12 +237,12 @@ struct json_serialize_action : json_feature_action { int layer; std::string layername; - int add_feature(json_object_ptr geometry, bool geometrycollection, json_object_ptr properties, json_object_ptr id, json_object_ptr tippecanoe, json_object_ptr feature) { + int add_feature(json_object *geometry, bool geometrycollection, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) { sst->line = geometry->parser->line; if (geometrycollection) { int ret = 1; for (size_t g = 0; g < geometry->array().size(); g++) { - ret &= serialize_geojson_feature(sst, geometry->array()[g], properties, id, layer, tippecanoe, feature, layername); + ret &= serialize_geojson_feature(sst, geometry->array()[g].get(), properties, id, layer, tippecanoe, feature, layername); } return ret; } else { @@ -250,12 +250,12 @@ struct json_serialize_action : json_feature_action { } } - void check_crs(json_object_ptr j) { + void check_crs(json_object *j) { ::check_crs(j, fname.c_str()); } }; -void parse_json(struct serialization_state *sst, json_pull_ptr jp, int layer, std::string layername) { +void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername) { json_serialize_action jsa; jsa.fname = sst->fname; jsa.sst = sst; diff --git a/geojson.hpp b/geojson.hpp index cb0776a46..8c63318c4 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -24,7 +24,7 @@ struct parse_json_args { json_pull_ptr json_begin_map(char *map, long long len); void json_end_map(json_pull_ptr &jp); -void parse_json(struct serialization_state *sst, json_pull_ptr jp, int layer, std::string layername); +void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername); void *run_parse_json(void *v); #endif diff --git a/geometry.hpp b/geometry.hpp index 3aab6c189..454cea611 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -141,7 +141,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object_ptr filter, bool preserve_input_order, + bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, @@ -157,7 +157,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n std::vector const &exclude_prefix, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object_ptr filter, bool preserve_input_order, + bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp index 6e8a534db..8b23ba75c 100644 --- a/jsonpull/jsonpull.cpp +++ b/jsonpull/jsonpull.cpp @@ -89,26 +89,23 @@ static inline int read_wrap(json_pull *j) { // Construct an instance of the right subclass for the given type. // JSON_TRUE / JSON_FALSE / JSON_NULL and the parse-token types are bare // json_objects; the value-bearing types each get their own subclass. +// +// Returns a json_object_ptr (unique_ptr with a type-dispatching deleter, +// see jsonpull.h), so the caller doesn't have to remember which subclass +// was constructed when it eventually deletes. static json_object_ptr make_object(json_type type, json_object *parent, json_pull *jp) { - json_object_ptr o; switch (type) { case JSON_NUMBER: - o = std::make_shared(parent, jp); - break; + return json_object_ptr(new json_number(parent, jp)); case JSON_STRING: - o = std::make_shared(parent, jp); - break; + return json_object_ptr(new json_string(parent, jp)); case JSON_ARRAY: - o = std::make_shared(parent, jp); - break; + return json_object_ptr(new json_array(parent, jp)); case JSON_HASH: - o = std::make_shared(parent, jp); - break; + return json_object_ptr(new json_hash(parent, jp)); default: - o = std::make_shared(type, parent, jp); - break; + return json_object_ptr(new json_object(type, parent, jp)); } - return o; } static json_object_ptr fabricate_object(json_pull *jp, json_object *parent, json_type type) { @@ -119,15 +116,22 @@ static inline json_pull::parse_frame *current_frame(json_pull *j) { return j->container_stack.empty() ? nullptr : &j->container_stack.back(); } -static json_object_ptr add_object(json_pull *j, json_type type) { +// Construct a new node of `type` and install it as a child of the +// current container (or as the parser's root, if the container stack +// is empty). Returns a borrowed pointer into the parser-owned tree; +// the unique_ptr that owns the node lives in whichever vector slot +// we just pushed it into. Returns nullptr on error after setting +// j->error. +static json_object *add_object(json_pull *j, json_type type) { json_pull::parse_frame *f = current_frame(j); - json_object *c = f ? f->container.get() : nullptr; + json_object *c = f ? f->container : nullptr; json_object_ptr o = make_object(type, c, j); + json_object *raw = o.get(); if (f != nullptr) { if (c->type == JSON_ARRAY) { if (f->expect == JSON_ITEM) { - c->array().push_back(o); + c->array().push_back(std::move(o)); f->expect = JSON_COMMA; } else { j->error = "Expected a comma, not a list item"; @@ -135,7 +139,7 @@ static json_object_ptr add_object(json_pull *j, json_type type) { } } else if (c->type == JSON_HASH) { if (f->expect == JSON_VALUE) { - c->entries().back().value = o; + c->entries().back().value = std::move(o); f->expect = JSON_COMMA; } else if (f->expect == JSON_KEY) { if (type != JSON_STRING) { @@ -143,7 +147,7 @@ static json_object_ptr add_object(json_pull *j, json_type type) { return nullptr; } - c->entries().push_back({o, nullptr}); + c->entries().push_back({std::move(o), nullptr}); f->expect = JSON_COLON; } else { j->error = "Expected a comma or colon"; @@ -151,33 +155,34 @@ static json_object_ptr add_object(json_pull *j, json_type type) { } } } else { - // Drop the previous top-level value; replacing the parser's root - // shared_ptr will free it if no one else holds a reference. - j->root = o; + // Replacing the parser's root destroys the previous top-level + // value (if no one called json_disconnect / json_read_tree to + // take ownership of it). + j->root = std::move(o); } - return o; + return raw; } -json_object_ptr json_hash_get(json_object *o, const char *s) { +json_object *json_hash_get(json_object *o, const char *s) { if (o == nullptr || o->type != JSON_HASH) { return nullptr; } for (const auto &e : o->entries()) { if (e.key != nullptr && e.key->type == JSON_STRING && e.key->string() == s) { - return e.value; + return e.value.get(); } } return nullptr; } -json_object_ptr json_hash_get(json_object_ptr o, const char *s) { +json_object *json_hash_get(const json_object_ptr &o, const char *s) { return json_hash_get(o.get(), s); } -json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback cb, void *state) { +json_object *json_read_separators(json_pull_ptr &jp, json_separator_callback cb, void *state) { int c; json_pull *j = jp.get(); @@ -226,14 +231,13 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c /////////////////////////// Arrays case '[': { - json_object_ptr o = add_object(j, JSON_ARRAY); + json_object *o = add_object(j, JSON_ARRAY); if (o == nullptr) { return nullptr; } // add_object already installed `o` in the parent (or the - // parser's root); moving the local copy into the frame - // avoids one shared_ptr atomic inc/dec pair per container. - j->container_stack.push_back({std::move(o), JSON_ITEM}); + // parser's root) as a unique_ptr; the frame just borrows. + j->container_stack.push_back({o, JSON_ITEM}); if (cb != nullptr) { cb(JSON_ARRAY, j, state); @@ -249,7 +253,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c return nullptr; } - json_object *cc = f->container.get(); + json_object *cc = f->container; if (cc->type != JSON_ARRAY) { j->error = "Found ] not in an array"; return nullptr; @@ -262,23 +266,20 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } } - // Move the container out of the frame so pop_back doesn't - // drop the last reference; saves one atomic inc/dec. - json_object_ptr ret = std::move(f->container); + // Pop the frame; ownership of `cc` stays with whatever + // surrounding container (or jp->root) installed it. j->container_stack.pop_back(); - return ret; + return cc; } /////////////////////////// Hashes case '{': { - json_object_ptr o = add_object(j, JSON_HASH); + json_object *o = add_object(j, JSON_HASH); if (o == nullptr) { return nullptr; } - // See the [ case above: move into the frame to skip a - // shared_ptr atomic inc/dec round-trip. - j->container_stack.push_back({std::move(o), JSON_KEY}); + j->container_stack.push_back({o, JSON_KEY}); if (cb != nullptr) { cb(JSON_HASH, j, state); @@ -294,7 +295,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c return nullptr; } - json_object *cc = f->container.get(); + json_object *cc = f->container; if (cc->type != JSON_HASH) { j->error = "Found } not in a hash"; return nullptr; @@ -307,10 +308,8 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } } - // See the ] case: move out to skip an atomic refcount round-trip. - json_object_ptr ret = std::move(f->container); j->container_stack.pop_back(); - return ret; + return cc; } /////////////////////////// Null @@ -488,7 +487,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c } } - json_object_ptr n = add_object(j, JSON_NUMBER); + json_object *n = add_object(j, JSON_NUMBER); if (n != nullptr) { double d = atof(j->number_buffer.c_str()); n->set_number(d); @@ -642,7 +641,7 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c return nullptr; } - json_object_ptr s = add_object(j, JSON_STRING); + json_object *s = add_object(j, JSON_STRING); if (s != nullptr) { // Copy (don't move) so j->string_buffer retains its // grown capacity for the next token. The copy is a @@ -659,48 +658,71 @@ json_object_ptr json_read_separators(json_pull_ptr jp, json_separator_callback c return nullptr; } -json_object_ptr json_read(json_pull_ptr j) { +json_object *json_read(json_pull_ptr &j) { return json_read_separators(j, nullptr, nullptr); } -json_object_ptr json_read_tree(json_pull_ptr p) { - json_object_ptr j; +// Forward declaration so json_read_tree can clear back-pointers on +// the tree it hands out -- this lets callers (like the filter loaders) +// keep the returned tree past the parser's lifetime without having to +// follow up with a separate json_disconnect call. +static void clear_back_pointers(json_object *o); + +json_object_ptr json_read_tree(json_pull_ptr &p) { + json_object *j; while ((j = json_read(p)) != nullptr) { if (j->parent == nullptr) { - return j; + // The parser owns the top-level value via p->root; + // transfer ownership out to the caller and detach + // the subtree from the parser so the caller can + // outlive the json_pull. + json_object_ptr tree = std::move(p->root); + clear_back_pointers(tree.get()); + return tree; } } return nullptr; } -// Splice `o` out of its parent's array or object, dropping the -// parent's owning reference. After this returns, the parent no longer -// holds any pointer to `o`; the caller's reference is the only thing -// keeping the subtree alive. +// Take ownership of `o` away from its parent (or from the parser's +// root) by moving the owning json_object_ptr out of whatever vector +// slot or hash entry holds it. Returns the unique_ptr to the caller, +// who is now solely responsible for it. Returns an empty +// json_object_ptr if `o` is not currently owned by a parent or by +// the parser (e.g. already detached, or only borrowed from somewhere +// untracked). // // For a hash, removing a single key or value individually would -// disturb the surrounding key/value pairing, so we replace the removed -// half with a placeholder JSON_NULL and only erase the entry once both -// halves have been detached. This matches the historical -// json_disconnect semantics. -static void splice_from_parent(json_object *o) { +// disturb the surrounding key/value pairing, so we replace the +// extracted half with a fresh JSON_NULL placeholder and only erase +// the entry once both halves have been detached. This matches the +// historical json_disconnect semantics for partially-disconnected +// pairs. +static json_object_ptr take_from_owner(json_object *o) { if (o == nullptr) { - return; + return nullptr; } json_object *parent = o->parent; if (parent == nullptr) { - return; + // Top-level value: the parser owns it via root, unless the + // caller already moved it out. + json_pull *parser = o->parser; + if (parser != nullptr && parser->root.get() == o) { + return std::move(parser->root); + } + return nullptr; } if (parent->type == JSON_ARRAY) { auto &arr = parent->array(); for (size_t i = 0; i < arr.size(); i++) { if (arr[i].get() == o) { + json_object_ptr taken = std::move(arr[i]); arr.erase(arr.begin() + i); - break; + return taken; } } } else if (parent->type == JSON_HASH) { @@ -708,49 +730,43 @@ static void splice_from_parent(json_object *o) { for (size_t i = 0; i < entries.size(); i++) { auto &e = entries[i]; if (e.key.get() == o) { + json_object_ptr taken = std::move(e.key); e.key = fabricate_object(parent->parser, parent, JSON_NULL); if (e.value != nullptr && e.value->type == JSON_NULL && e.key->type == JSON_NULL) { entries.erase(entries.begin() + i); } - break; + return taken; } if (e.value.get() == o) { + json_object_ptr taken = std::move(e.value); e.value = fabricate_object(parent->parser, parent, JSON_NULL); if (e.key != nullptr && e.key->type == JSON_NULL && e.value->type == JSON_NULL) { entries.erase(entries.begin() + i); } - break; + return taken; } } } + + return nullptr; } -// json_free splices `o` out of its parent (if any) so that the -// parent no longer keeps the subtree alive, then drops the caller's -// reference. The subtree is freed when the last reference is gone -// (typically right here, since the parent's reference was just -// dropped). geojson-loop.cpp relies on this to release each feature -// after it has been serialized, so that already-serialized features -// don't sit in memory while subsequent features are parsed. +// json_free splices `o` out of its parent (if any), or out of the +// parser's root (if `o` is the most recently completed top-level +// value), and destroys the subtree. After this call, `o` is a +// dangling pointer and must not be used. // -// If `o` is the parser's current root (the most recently completed -// top-level value), drop the parser's reference too -- otherwise a -// line-delimited stream would always hold the previously-completed -// feature until the next one started parsing. +// geojson-loop.cpp relies on this to release each feature after it +// has been serialized, so that already-serialized features don't sit +// in memory while subsequent features are parsed. // // Unlike json_disconnect, this does NOT walk the subtree clearing // parent/parser back-pointers, because the subtree is about to be -// destroyed and those pointers will never be observed again. -void json_free(json_object_ptr &o) { - if (o != nullptr) { - splice_from_parent(o.get()); - - json_pull *parser = o->parser; - if (parser != nullptr && parser->root.get() == o.get()) { - parser->root.reset(); - } - } - o.reset(); +// destroyed and those pointers will never be observed again -- the +// unique_ptr returned by take_from_owner goes out of scope at the end +// of this function and runs the type-dispatching deleter. +void json_free(json_object *o) { + (void) take_from_owner(o); } // Walk the subtree clearing parent/parser back-pointers so the detached @@ -776,23 +792,12 @@ static void clear_back_pointers(json_object *o) { o->parser = nullptr; } -void json_disconnect(json_object_ptr o) { - if (o == nullptr) { - return; +json_object_ptr json_disconnect(json_object *o) { + json_object_ptr taken = take_from_owner(o); + if (taken != nullptr) { + clear_back_pointers(taken.get()); } - - // Splice o out of its parent's array or object. The parent's vector - // holds the shared_ptr to this child; erasing it removes one reference, - // but the caller still holds `o`, so the subtree stays alive. - splice_from_parent(o.get()); - - // Drop the parser's reference to this subtree if it was the root. - json_pull *parser = o->parser; - if (parser != nullptr && parser->root.get() == o.get()) { - parser->root.reset(); - } - - clear_back_pointers(o.get()); + return taken; } static void string_append_c(std::string &val, char c) { @@ -803,7 +808,7 @@ static void string_append(std::string &val, const char *add) { val.append(add); } -static void json_print_one(std::string &val, json_object *o) { +static void json_print_one(std::string &val, const json_object *o) { if (o == nullptr) { string_append(val, "..."); } else if (o->type == JSON_STRING) { @@ -852,7 +857,7 @@ static void json_print_one(std::string &val, json_object *o) { } } -static void json_print(std::string &val, json_object *o) { +static void json_print(std::string &val, const json_object *o) { if (o == nullptr) { // Hash value in incompletely read hash string_append(val, "..."); @@ -884,8 +889,8 @@ static void json_print(std::string &val, json_object *o) { } } -std::string json_stringify(json_object_ptr o) { +std::string json_stringify(const json_object *o) { std::string val; - json_print(val, o.get()); + json_print(val, o); return val; } diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index a6d5cc6ce..170a8d216 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -31,7 +31,31 @@ typedef enum json_type { struct json_object; struct json_pull; -typedef std::shared_ptr json_object_ptr; +// json_object is non-virtual so that JSON_TRUE / JSON_FALSE / JSON_NULL +// nodes don't have to pay for a vptr, but the typed subclasses +// (json_number, json_string, json_array, json_hash) have non-trivial +// destructors that need to run to free their std::vector / std::string +// members. So json_object_ptr is given a custom empty deleter that +// dispatches on `type` and static_casts to the right subclass before +// `delete`. The deleter is stateless, so the unique_ptr stays one +// pointer wide. +struct json_object_deleter { + void operator()(json_object *p) const noexcept; +}; + +// Ownership of a JSON subtree is unique: every node has a single owner, +// which is either its parent (via a json_object_ptr in the parent's +// vector or hash entry) or, for the root, the parser (via jp->root) or +// the caller (after json_read_tree / json_disconnect). +// +// Callers receive borrowed `json_object *` views from json_read, +// json_hash_get, etc.; those pointers stay valid as long as the owning +// container is intact (which, for json_read results, means "until the +// next json_read, json_free, or json_disconnect call on that subtree"). +// +// json_pull_ptr stays a shared_ptr because the parser is created once +// and freed once and the cost of shared_ptr there is irrelevant. +typedef std::unique_ptr json_object_ptr; typedef std::shared_ptr json_pull_ptr; // A single key/value pair inside a JSON_HASH. The pairs are stored in @@ -60,12 +84,12 @@ struct json_entry { // outlive the original parser. // // json_object intentionally has no virtual functions and no virtual -// destructor: subclasses are constructed via std::make_shared(), -// and std::shared_ptr remembers the deleter from the original type, so -// destroying a shared_ptr that actually points at a -// json_string still runs ~json_string(). Dispatch on `type` is what the -// rest of the code already does. The accessor methods assert at debug -// time that the type matches before downcasting. +// destructor; the json_object_ptr deleter (see below in this header) +// switches on `type` and static_casts to the correct subclass before +// `delete`, so each subclass's destructor still runs without costing +// a vptr per node. Dispatch on `type` is what the rest of the code +// already does. The accessor methods assert at debug time that the +// type matches before downcasting. struct json_object { json_object *parent = nullptr; @@ -218,6 +242,35 @@ inline const std::vector &json_object::entries() const { return static_cast(this)->entries_value; } +inline void json_object_deleter::operator()(json_object *p) const noexcept { + if (p == nullptr) { + return; + } + // Dispatch on the discriminator so the correct subclass destructor + // runs. json_object has no virtual destructor, so a bare `delete p` + // would skip the std::vector / std::string members of the subclass. + switch (p->type) { + case JSON_NUMBER: + delete static_cast(p); + break; + case JSON_STRING: + delete static_cast(p); + break; + case JSON_ARRAY: + delete static_cast(p); + break; + case JSON_HASH: + delete static_cast(p); + break; + default: + // JSON_TRUE / JSON_FALSE / JSON_NULL (and the parse-token + // types, which never appear as owned nodes) are bare + // json_objects with no extra fields. + delete p; + break; + } +} + struct json_pull { const char *error = nullptr; // points at a string literal; no allocation int line = 1; @@ -228,18 +281,25 @@ struct json_pull { ssize_t buffer_tail = 0; ssize_t buffer_head = 0; - // Stack of currently-open containers; the top is the innermost container - // being parsed. Each frame also remembers what token is expected next - // (an item, a comma, a key, a colon, or a value). This stack is the - // only place the parser-only `expect` state lives, so it does not - // pollute json_object once parsing finishes. Replaces the previous - // single `container` pointer / parent walk, which previously required - // enable_shared_from_this on every json_object instance. + // Stack of currently-open containers; the top is the innermost + // container being parsed. Each frame also remembers what token is + // expected next (an item, a comma, a key, a colon, or a value). + // The frame's `container` is a borrowed raw pointer; actual + // ownership of the in-progress container lives in either the + // surrounding container's vector (for nested containers) or + // `root` (for the outermost container). struct parse_frame { - json_object_ptr container; + json_object *container; json_type expect; }; std::vector container_stack; + + // The most recently completed top-level value. The parser owns + // it (as a unique_ptr) until either: the next top-level value + // starts parsing (the old root is destroyed), the caller calls + // json_read_tree (ownership is transferred out), or the caller + // calls json_free / json_disconnect (the parser's reference is + // dropped explicitly). json_object_ptr root; // Scratch buffers reused across tokens so we don't reallocate per @@ -257,30 +317,54 @@ json_pull_ptr json_begin_string(const char *s); json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); -// json_end is now a thin convenience that resets the caller's json_pull_ptr. -// The parser (and any tree it still owns) is freed when the last shared_ptr -// to it is dropped, so calling json_end is optional if the json_pull_ptr will -// go out of scope on its own. +// json_end is a thin convenience that resets the caller's json_pull_ptr. +// The parser (and any tree it still owns) is freed when the last +// shared_ptr to it is dropped, so calling json_end is optional if the +// json_pull_ptr will go out of scope on its own. void json_end(json_pull_ptr &p); typedef void (*json_separator_callback)(json_type type, json_pull *j, void *state); -json_object_ptr json_read_tree(json_pull_ptr j); -json_object_ptr json_read(json_pull_ptr j); -json_object_ptr json_read_separators(json_pull_ptr j, json_separator_callback cb, void *state); - -// json_free now just resets the caller's json_object_ptr. The subtree is -// destroyed when the last shared_ptr to it is dropped (typically by also -// being removed from its parent or parser). -void json_free(json_object_ptr &j); - -// Splice o out of its parent's array/object and clear parent/parser back-pointers -// throughout the detached subtree so it can outlive the original parser. -void json_disconnect(json_object_ptr j); - -json_object_ptr json_hash_get(json_object_ptr o, const char *s); -json_object_ptr json_hash_get(json_object *o, const char *s); - -std::string json_stringify(json_object_ptr o); +// json_read returns a borrowed pointer to the next completed JSON node +// in the stream. The returned pointer is valid until the next call that +// extends or trims the parser's tree (the next json_read on the same +// parser, a json_free on the same node, or a json_disconnect that +// extracts the node). Returns nullptr at end of input or on error. +// +// For top-level values, ownership stays with the parser (via jp->root); +// for nested values, ownership stays with the enclosing container. +json_object *json_read(json_pull_ptr &j); +json_object *json_read_separators(json_pull_ptr &j, json_separator_callback cb, void *state); + +// json_read_tree drains the next top-level value out of the parser +// and hands ownership to the caller. After it returns, jp->root is +// empty, the parent/parser back-pointers throughout the subtree have +// been cleared, and the caller's json_object_ptr is the only thing +// keeping the tree alive. The returned tree can outlive the +// json_pull it was parsed from. +json_object_ptr json_read_tree(json_pull_ptr &j); + +// json_free splices `o` out of its parent (if any), or clears the +// parser's root if `o` is the parser's current top-level value, and +// destroys the subtree. After this call, `o` is a dangling pointer +// that must not be used. Safe to call with nullptr. +void json_free(json_object *o); + +// Splice `o` out of its parent's array/object (or out of the parser's +// root), walk the detached subtree clearing parent/parser back-pointers, +// and return ownership of the subtree to the caller as a +// json_object_ptr. After this returns, the parser no longer references +// any node in the subtree, and the subtree can outlive the original +// parser. +json_object_ptr json_disconnect(json_object *o); + +// Look up `s` in the hash `o`. Returns a borrowed pointer; ownership +// stays with the hash. nullptr if `o` is not a hash, or `s` is absent, +// or the matching value is null. Accepts a json_object_ptr by reference +// as a convenience so callers don't have to write `.get()`. +json_object *json_hash_get(const json_object_ptr &o, const char *s); +json_object *json_hash_get(json_object *o, const char *s); + +std::string json_stringify(const json_object *o); #endif diff --git a/jsontool.cpp b/jsontool.cpp index 52cb1c334..014b1f6ee 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -140,12 +140,12 @@ std::string sort_quote(const char *s) { return ret; } -void out(std::string const &s, int type, json_object_ptr properties) { +void out(std::string const &s, int type, json_object *properties) { if (extract != NULL) { std::string extracted = sort_quote("null"); bool found = false; - json_object_ptr o = json_hash_get(properties, extract); + json_object *o = json_hash_get(properties, extract); if (o != nullptr) { found = true; if (o->type == JSON_STRING) { @@ -204,7 +204,7 @@ void out(std::string const &s, int type, json_object_ptr properties) { std::string prev_joinkey; -void join_csv(json_object_ptr j) { +void join_csv(json_object *j) { if (header.size() == 0) { std::string s = csv_getline(csvfile); if (s.size() == 0) { @@ -230,8 +230,8 @@ void join_csv(json_object_ptr j) { } } - json_object_ptr properties = json_hash_get(j, "properties"); - json_object_ptr key; + json_object *properties = json_hash_get(j, "properties"); + json_object *key = nullptr; if (properties != nullptr) { key = json_hash_get(properties, header[0].c_str()); @@ -320,30 +320,28 @@ void join_csv(json_object_ptr j) { } if (attr_type != JSON_NULL) { - auto ko = std::make_shared(properties.get(), properties->parser); - ko->string_value = k; + json_object_ptr ko(new json_string(properties, properties->parser)); + ko->string() = k; json_object_ptr vo; if (attr_type == JSON_STRING) { - auto s = std::make_shared(properties.get(), properties->parser); - s->string_value = v; - vo = s; + vo = json_object_ptr(new json_string(properties, properties->parser)); + vo->string() = v; } else if (attr_type == JSON_NUMBER) { - auto n = std::make_shared(properties.get(), properties->parser); - n->set_number(atof(v.c_str())); - vo = n; + vo = json_object_ptr(new json_number(properties, properties->parser)); + vo->set_number(atof(v.c_str())); } else { abort(); } - properties->entries().push_back({ko, vo}); + properties->entries().push_back({std::move(ko), std::move(vo)}); } } } } struct json_join_action : json_feature_action { - int add_feature(json_object_ptr geometry, bool, json_object_ptr, json_object_ptr, json_object_ptr, json_object_ptr feature) { + int add_feature(json_object *geometry, bool, json_object *, json_object *, json_object *, json_object *feature) { if (feature != geometry) { // a real feature, not a bare geometry if (csvfile != NULL) { join_csv(feature); @@ -357,7 +355,7 @@ struct json_join_action : json_feature_action { return 1; } - void check_crs(json_object_ptr) { + void check_crs(json_object *) { } }; diff --git a/main.cpp b/main.cpp index 95a4096bb..c11c3456d 100644 --- a/main.cpp +++ b/main.cpp @@ -1215,7 +1215,7 @@ double round_droprate(double r) { return std::round(r * 100000.0) / 100000.0; } -std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object_ptr filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { +std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { int ret = EXIT_SUCCESS; std::vector readers; @@ -2892,7 +2892,7 @@ void set_attribute_value(const char *arg) { exit(EXIT_JSON); } - serial_val val = stringify_value(e.value, "json", 1, o); + serial_val val = stringify_value(e.value.get(), "json", 1, o.get()); set_attributes.emplace(e.key->string(), val); i++; } @@ -2934,7 +2934,7 @@ void parse_json_source(const char *arg, struct source &src) { exit(EXIT_JSON); } - json_object_ptr fname = json_hash_get(o, "file"); + json_object *fname = json_hash_get(o, "file"); if (fname == nullptr || fname->type != JSON_STRING) { fprintf(stderr, "%s: -L%s: requires \"file\": filename\n", *av, arg); exit(EXIT_JSON); @@ -2942,17 +2942,17 @@ void parse_json_source(const char *arg, struct source &src) { src.file = fname->string(); - json_object_ptr layer = json_hash_get(o, "layer"); + json_object *layer = json_hash_get(o, "layer"); if (layer != nullptr && layer->type == JSON_STRING) { src.layer = layer->string(); } - json_object_ptr description = json_hash_get(o, "description"); + json_object *description = json_hash_get(o, "description"); if (description != nullptr && description->type == JSON_STRING) { src.description = description->string(); } - json_object_ptr format = json_hash_get(o, "format"); + json_object *format = json_hash_get(o, "format"); if (format != nullptr && format->type == JSON_STRING) { src.format = format->string(); } @@ -3846,7 +3846,7 @@ int main(int argc, char **argv) { auto input_ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, - maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); + maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter.get(), droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); ret = std::get<0>(input_ret); diff --git a/overzoom.cpp b/overzoom.cpp index b0b16910f..05dce3e75 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -264,7 +264,7 @@ int main(int argc, char **argv) { its.push_back(std::move(t)); } - out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector(), "", "", SIZE_MAX, std::vector(), deduplicate_by_id); + out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter.get(), preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector(), "", "", SIZE_MAX, std::vector(), deduplicate_by_id); } FILE *f = fopen(outfile, "wb"); diff --git a/plugin.cpp b/plugin.cpp index b9d5f2dea..7e07dea3e 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -143,16 +143,23 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } // Reads from the prefilter -serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { +serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { serial_feature sf; while (1) { - json_object_ptr j = json_read(jp); + // json_read returns each token as the parser produces it, including + // intermediate (still incomplete) container nodes. We must NOT free + // these intermediates here: they belong to the larger feature hash + // still being assembled, and freeing them would splice them out of + // the parent and corrupt the in-progress tree. We only free `j` + // after we have successfully processed a complete Feature hash + // (just before returning), or `jp->root` when the stream ends. + json_object *j = json_read(jp); if (j == nullptr) { if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); if (jp->root != nullptr) { - json_context(jp->root); + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } @@ -164,7 +171,7 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st return sf; } - json_object_ptr type = json_hash_get(j, "type"); + json_object *type = json_hash_get(j, "type"); if (type == nullptr || type->type != JSON_STRING) { continue; } @@ -172,21 +179,21 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st continue; } - json_object_ptr geometry = json_hash_get(j, "geometry"); + json_object *geometry = json_hash_get(j, "geometry"); if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); exit(EXIT_JSON); } - json_object_ptr properties = json_hash_get(j, "properties"); + json_object *properties = json_hash_get(j, "properties"); if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); exit(EXIT_JSON); } - json_object_ptr geometry_type = json_hash_get(geometry, "type"); + json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); @@ -199,7 +206,7 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st exit(EXIT_JSON); } - json_object_ptr coordinates = json_hash_get(geometry, "coordinates"); + json_object *coordinates = json_hash_get(geometry, "coordinates"); if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: feature without coordinates array: ", jp->line); json_context(j); @@ -248,29 +255,29 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st sf.has_id = false; std::string layername = "unknown"; - json_object_ptr tippecanoe = json_hash_get(j, "tippecanoe"); + json_object *tippecanoe = json_hash_get(j, "tippecanoe"); if (tippecanoe != nullptr) { - json_object_ptr layer = json_hash_get(tippecanoe, "layer"); + json_object *layer = json_hash_get(tippecanoe, "layer"); if (layer != nullptr && layer->type == JSON_STRING) { layername = layer->string(); } - json_object_ptr index = json_hash_get(tippecanoe, "index"); + json_object *index = json_hash_get(tippecanoe, "index"); if (index != nullptr && index->type == JSON_NUMBER) { sf.index = index->number(); } - json_object_ptr sequence = json_hash_get(tippecanoe, "sequence"); + json_object *sequence = json_hash_get(tippecanoe, "sequence"); if (sequence != nullptr && sequence->type == JSON_NUMBER) { sf.seq = sequence->number(); } - json_object_ptr extent = json_hash_get(tippecanoe, "extent"); + json_object *extent = json_hash_get(tippecanoe, "extent"); if (extent != nullptr && extent->type == JSON_NUMBER) { sf.extent = extent->number(); } - json_object_ptr dropped = json_hash_get(tippecanoe, "dropped"); + json_object *dropped = json_hash_get(tippecanoe, "dropped"); if (dropped != nullptr && dropped->type == JSON_TRUE) { sf.dropped = FEATURE_DROPPED; // dropped } else { @@ -295,7 +302,7 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st } } - json_object_ptr id = json_hash_get(j, "id"); + json_object *id = json_hash_get(j, "id"); if (id != nullptr && id->type == JSON_NUMBER) { sf.id = id->number(); if (id->large_unsigned() > 0) { @@ -345,7 +352,7 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st if (properties->type == JSON_HASH) { for (const auto &e : properties->entries()) { - serial_val v = stringify_value(e.value, "Filter output", jp->line, j); + serial_val v = stringify_value(e.value.get(), "Filter output", jp->line, j); // Nulls can be excluded here because the expression evaluation filter // would have already run before prefiltering @@ -361,8 +368,11 @@ serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, st } } + json_free(j); return sf; } + + json_free(j); } } diff --git a/plugin.hpp b/plugin.hpp index 2a1feab62..5ad99092c 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,4 +1,4 @@ struct key_pool; std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull_ptr jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); +serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 9a8c62a1a..57695b1a2 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -422,21 +422,21 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; - state.json_write_json(json_stringify(e.value)); + state.json_write_json(json_stringify(e.value.get())); } else if (key == "tilestats" && e.value->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; - state.json_write_json(json_stringify(e.value)); + state.json_write_json(json_stringify(e.value.get())); } else if (key == "strategies" && e.value->type == JSON_ARRAY) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value).c_str()); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } else if (key == "tippecanoe_decisions" && e.value->type == JSON_HASH) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value).c_str()); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } diff --git a/read_json.cpp b/read_json.cpp index 45a533f4a..13798ee6e 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -42,7 +42,7 @@ int mb_geometry[GEOM_TYPES] = { VT_POLYGON, }; -void json_context(json_object_ptr j) { +void json_context(json_object *j) { std::string s = json_stringify(j); if (s.size() >= 500) { @@ -53,7 +53,7 @@ void json_context(json_object_ptr j) { fprintf(stderr, "in JSON object %s\n", s.c_str()); } -void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const char *fname, int line, json_object_ptr feature) { +void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature) { if (j == nullptr || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for geometry type %d: ", fname, line, t); json_context(feature); @@ -72,7 +72,7 @@ void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const cha } } - parse_coordinates(within, j->array()[i], out, op, fname, line, feature); + parse_coordinates(within, j->array()[i].get(), out, op, fname, line, feature); } } else { if (j->array().size() >= 2 && j->array()[0]->type == JSON_NUMBER && j->array()[1]->type == JSON_NUMBER) { @@ -121,7 +121,7 @@ void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const cha // type and stringified value. All numeric values, even if they are integers, // even integers that are too large to fit in a double but will still be // stringified with their original precision, are recorded here as mvt_double. -serial_val stringify_value(json_object_ptr value, const char *reading, int line, json_object_ptr feature) { +serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature) { serial_val sv; if (value != nullptr) { @@ -176,9 +176,9 @@ static std::vector to_feature(drawvec &geom) { return out; } -std::pair parse_geometry(json_object_ptr geometry, json_pull_ptr jp, json_object_ptr j, +std::pair parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style) { - json_object_ptr geometry_type = json_hash_get(geometry, "type"); + json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); @@ -191,7 +191,7 @@ std::pair parse_geometry(json_object_ptr geometry, json_pull_ptr j exit(EXIT_JSON); } - json_object_ptr coordinates = json_hash_get(geometry, "coordinates"); + json_object *coordinates = json_hash_get(geometry, "coordinates"); if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: geometry without coordinates array: ", jp->line); json_context(j); @@ -305,12 +305,12 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int json_pull_ptr jp = json_begin_file(fp); while (1) { - json_object_ptr j = json_read(jp); + json_object *j = json_read(jp); if (j == nullptr) { if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); if (jp->root != nullptr) { - json_context(jp->root); + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } @@ -321,7 +321,12 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int break; } - json_object_ptr type = json_hash_get(j, "type"); + // json_read returns each parser token in sequence, including + // intermediate (still-incomplete) container nodes. Freeing those + // here would splice them out of the feature hash being built + // up, so only free `j` once we have processed a complete + // Feature (or `jp->root` when the stream ends). + json_object *type = json_hash_get(j, "type"); if (type == nullptr || type->type != JSON_STRING) { continue; } @@ -329,7 +334,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int continue; } - json_object_ptr properties = json_hash_get(j, "properties"); + json_object *properties = json_hash_get(j, "properties"); if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); @@ -337,8 +342,8 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int } std::string layername = "unknown"; - json_object_ptr tippecanoe = json_hash_get(j, "tippecanoe"); - json_object_ptr layer; + json_object *tippecanoe = json_hash_get(j, "tippecanoe"); + json_object *layer = nullptr; if (tippecanoe != nullptr) { layer = json_hash_get(tippecanoe, "layer"); if (layer != nullptr && layer->type == JSON_STRING) { @@ -356,7 +361,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int } auto l = ret.find(layername); - json_object_ptr geometry = json_hash_get(j, "geometry"); + json_object *geometry = json_hash_get(j, "geometry"); if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); @@ -373,7 +378,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int feature.type = mb_geometry[t]; feature.geometry = to_feature(dv); - json_object_ptr id = json_hash_get(j, "id"); + json_object *id = json_hash_get(j, "id"); if (id != nullptr && id->type == JSON_NUMBER) { feature.id = id->number(); if (id->large_unsigned() > 0) { @@ -384,7 +389,7 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int if (properties->type == JSON_HASH) { for (const auto &e : properties->entries()) { - serial_val sv = stringify_value(e.value, "Filter output", jp->line, j); + serial_val sv = stringify_value(e.value.get(), "Filter output", jp->line, j); // Nulls can be excluded here because this is the postfilter // and it is nearly time to create the vector representation @@ -398,6 +403,8 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int l->second.features.push_back(feature); } + + json_free(j); } std::vector final; diff --git a/read_json.hpp b/read_json.hpp index 4b8f9fb83..f254e354c 100644 --- a/read_json.hpp +++ b/read_json.hpp @@ -10,10 +10,10 @@ extern const char *geometry_names[GEOM_TYPES]; extern int geometry_within[GEOM_TYPES]; extern int mb_geometry[GEOM_TYPES]; -void json_context(json_object_ptr j); -void parse_coordinates(int t, json_object_ptr j, drawvec &out, int op, const char *fname, int line, json_object_ptr feature); -std::pair parse_geometry(json_object_ptr geometry, json_pull_ptr jp, json_object_ptr j, +void json_context(json_object *j); +void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature); +std::pair parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style); std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int extent, bool fix_longitudes); -serial_val stringify_value(json_object_ptr value, const char *reading, int line, json_object_ptr feature); +serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature); diff --git a/tile-join.cpp b/tile-join.cpp index b729230c2..fec371aa8 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -89,7 +89,7 @@ struct arg { std::set *keep_layers = NULL; std::set *remove_layers = NULL; int ifmatched = 0; - json_object_ptr filter; + json_object *filter = NULL; struct tileset_reader *readers = NULL; double minlat, minlon; @@ -97,7 +97,7 @@ struct arg { double minlon2, maxlon2; }; -void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, sqlite3 * /* db */, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object_ptr filter, struct arg *a) { +void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, sqlite3 * /* db */, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter, struct arg *a) { mvt_tile tile; int features_added = 0; bool was_compressed; @@ -891,7 +891,7 @@ void *join_worker(void *v) { return NULL; } -void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object_ptr filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) { +void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) { pthread_t pthreads[CPUS]; std::vector args; @@ -970,7 +970,7 @@ void handle_strategies(const unsigned char *s, std::vector *st) { if (o != nullptr && o->type == JSON_ARRAY) { for (size_t i = 0; i < o->array().size(); i++) { - json_object_ptr h = o->array()[i]; + const json_object_ptr &h = o->array()[i]; if (h->type == JSON_HASH) { size_t j = 0; for (const auto &kv : h->entries()) { @@ -1013,12 +1013,12 @@ void handle_strategies(const unsigned char *s, std::vector *st) { } } -void handle_vector_layers(json_object_ptr vector_layers, std::map &layermap, std::map &attribute_descriptions) { +void handle_vector_layers(json_object *vector_layers, std::map &layermap, std::map &attribute_descriptions) { if (vector_layers != nullptr && vector_layers->type == JSON_ARRAY) { for (size_t i = 0; i < vector_layers->array().size(); i++) { if (vector_layers->array()[i]->type == JSON_HASH) { - json_object_ptr id = json_hash_get(vector_layers->array()[i], "id"); - json_object_ptr desc = json_hash_get(vector_layers->array()[i], "description"); + json_object *id = json_hash_get(vector_layers->array()[i].get(), "id"); + json_object *desc = json_hash_get(vector_layers->array()[i].get(), "description"); if (id != nullptr && desc != nullptr && id->type == JSON_STRING && desc->type == JSON_STRING) { const std::string &sid = id->string(); @@ -1032,7 +1032,7 @@ void handle_vector_layers(json_object_ptr vector_layers, std::maparray()[i], "fields"); + json_object *fields = json_hash_get(vector_layers->array()[i].get(), "fields"); if (fields != nullptr && fields->type == JSON_HASH) { for (const auto &e : fields->entries()) { if (e.key != nullptr && e.key->type == JSON_STRING && @@ -1053,7 +1053,7 @@ void handle_vector_layers(json_object_ptr vector_layers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object_ptr filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { +void decode(struct tileset_reader *readers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -1207,7 +1207,7 @@ void decode(struct tileset_reader *readers, std::maptype == JSON_HASH) { - json_object_ptr vector_layers = json_hash_get(o, "vector_layers"); + json_object *vector_layers = json_hash_get(o, "vector_layers"); handle_vector_layers(vector_layers, layermap, attribute_descriptions); } @@ -1591,7 +1591,7 @@ int main(int argc, char **argv) { std::string generator_options; std::vector strategies; - decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); + decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter.get(), attribute_descriptions, generator_options, &strategies); if (set_attribution.size() != 0) { attribution = set_attribution; diff --git a/tile.cpp b/tile.cpp index 309d2e929..fd03a7e96 100644 --- a/tile.cpp +++ b/tile.cpp @@ -941,7 +941,7 @@ struct write_tile_args { bool still_dropping = false; int wrote_zoom = 0; size_t tiling_seg = 0; - json_object_ptr filter; + json_object *filter = NULL; std::vector const *unidecode_data; std::atomic *dropped_count = NULL; atomic_strategy *strategy = NULL; @@ -1102,7 +1102,7 @@ struct next_feature_state { // This function is called repeatedly from write_tile() to retrieve the next feature // from the input stream. If the stream is at an end, it returns a feature with the // geometry type set to -2. -static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object_ptr filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { +static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate); while (1) { @@ -1350,7 +1350,7 @@ struct run_prefilter_args { char *global_stringpool = NULL; long long *pool_off = NULL; FILE *prefilter_fp = NULL; - json_object_ptr filter; + json_object *filter = NULL; std::vector const *unidecode_data; bool first_time = false; bool compressed = false; @@ -1641,7 +1641,7 @@ void skip_tile(decompressor *geoms, std::atomic *geompos_in, bool com } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object_ptr filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -3213,7 +3213,7 @@ exit(EXIT_IMPOSSIBLE); return err_or_null; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object_ptr filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending) { last_progress = 0; // The existing layermaps are one table per input thread. diff --git a/tile.hpp b/tile.hpp index e9c7d42a2..8b72e3fb0 100644 --- a/tile.hpp +++ b/tile.hpp @@ -62,7 +62,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object_ptr filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/unit.cpp b/unit.cpp index bbd1a6d2a..e30ae227d 100644 --- a/unit.cpp +++ b/unit.cpp @@ -174,10 +174,10 @@ TEST_CASE("jsonpull surrogate-pair regression", "[jsonpull][surrogate]") { TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") { json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]"); - json_object_ptr outer; + json_object *outer = nullptr; int arrays_seen = 0; - json_object_ptr j; + json_object *j; while ((j = json_read(jp)) != nullptr) { if (j->type != JSON_ARRAY) { continue; @@ -190,7 +190,8 @@ TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") { REQUIRE(j->array()[1]->number() == 4); json_free(j); } else if (j->parent == nullptr) { - // The completed outer array. + // The completed outer array; the parser still owns it + // via jp->root, so the borrowed pointer stays valid. outer = j; break; } @@ -215,21 +216,33 @@ TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") { // The companion case to the pruning test above: in a line-delimited // stream, each feature returned by json_read is a top-level value -// with no parent, but the parser still co-owns it via jp->root. +// with no parent, but the parser still owns it via jp->root. // json_free must drop that parser reference too, otherwise the -// just-serialized feature stays in memory until the next feature -// starts parsing. +// just-serialized feature would sit in memory until the next feature +// started parsing. Under the unique_ptr ownership model, the only +// owner is jp->root, so verifying that jp->root is empty after the +// json_free call is also a guarantee that the subtree itself has +// been destroyed. TEST_CASE("json_free releases a top-level value held by the parser", "[jsonpull][memory]") { - std::weak_ptr observer; - json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": [2, 3]})"); - json_object_ptr j = json_read_tree(jp); - REQUIRE(j != nullptr); - REQUIRE(j->type == JSON_HASH); - REQUIRE(j->parent == nullptr); - observer = j; - json_free(j); + // json_read streams atoms first (1, 2, 3, [2,3], ...); the top-level + // hash is returned by the final `}` token. + json_object *top = nullptr; + json_object *j; + while ((j = json_read(jp)) != nullptr) { + if (j->parent == nullptr) { + top = j; + break; + } + } + + REQUIRE(top != nullptr); + REQUIRE(top->type == JSON_HASH); + REQUIRE(jp->root.get() == top); + + json_free(top); + // top is dangling now; do not dereference. - REQUIRE(observer.expired()); + REQUIRE(jp->root == nullptr); } From c4e06dddbdd6bea9b40e5f9d9854418531e60078 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 21:21:15 -0700 Subject: [PATCH 12/13] Fix preprocessor mistakes identified by Copilot --- evaluator.hpp | 2 +- tile.cpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/evaluator.hpp b/evaluator.hpp index a2c234fb9..4be1582d4 100644 --- a/evaluator.hpp +++ b/evaluator.hpp @@ -1,5 +1,5 @@ #ifndef EVALUATOR_HPP -#define EVALUATOR HPP +#define EVALUATOR_HPP #include #include diff --git a/tile.cpp b/tile.cpp index fd03a7e96..cdfba946d 100644 --- a/tile.cpp +++ b/tile.cpp @@ -48,11 +48,7 @@ #include "attribute.hpp" #include "thread.hpp" #include "shared_borders.hpp" - -extern "C" { #include "jsonpull/jsonpull.h" -} - #include "plugin.hpp" #define CMD_BITS 3 From 535519b665d90c4aeec9790127081862e54b65c7 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Sat, 30 May 2026 21:22:08 -0700 Subject: [PATCH 13/13] Make indent --- jsonpull/jsonpull.h | 48 ++++++++++++++++++++++++++++++++++----------- mvt.cpp | 2 +- tile-join.cpp | 2 +- tile.cpp | 14 ++++++------- 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index 170a8d216..059dde0d3 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -97,8 +97,12 @@ struct json_object { json_type type; - json_object(json_type t) : type(t) {} - json_object(json_type t, json_object *p, json_pull *pl) : parent(p), parser(pl), type(t) {} + json_object(json_type t) + : type(t) { + } + json_object(json_type t, json_object *p, json_pull *pl) + : parent(p), parser(pl), type(t) { + } // Type-tagged accessors. Each one asserts that the receiver is of // the right kind, then downcasts to the storage in the appropriate @@ -135,18 +139,28 @@ struct json_number : json_object { double d; unsigned long long u; long long s; - value_t() : d(0) {} + value_t() + : d(0) { + } } value; - json_number() : json_object(JSON_NUMBER) {} - json_number(json_object *p, json_pull *pl) : json_object(JSON_NUMBER, p, pl) {} + json_number() + : json_object(JSON_NUMBER) { + } + json_number(json_object *p, json_pull *pl) + : json_object(JSON_NUMBER, p, pl) { + } }; struct json_string : json_object { std::string string_value; - json_string() : json_object(JSON_STRING) {} - json_string(json_object *p, json_pull *pl) : json_object(JSON_STRING, p, pl) {} + json_string() + : json_object(JSON_STRING) { + } + json_string(json_object *p, json_pull *pl) + : json_object(JSON_STRING, p, pl) { + } }; struct json_array : json_object { @@ -158,8 +172,14 @@ struct json_array : json_object { // shared_ptr copies. Reserving 2 slots up front eliminates those // reallocations for the common case and adds only a single small // allocation for larger rings (which still grow geometrically). - json_array() : json_object(JSON_ARRAY) { array_value.reserve(2); } - json_array(json_object *p, json_pull *pl) : json_object(JSON_ARRAY, p, pl) { array_value.reserve(2); } + json_array() + : json_object(JSON_ARRAY) { + array_value.reserve(2); + } + json_array(json_object *p, json_pull *pl) + : json_object(JSON_ARRAY, p, pl) { + array_value.reserve(2); + } }; struct json_hash : json_object { @@ -169,8 +189,14 @@ struct json_hash : json_object { // properties, geometry, plus a few attribute fields). Reserving 4 // slots avoids the 0 -> 1 -> 2 -> 4 growth chain for the typical // case while only modestly over-allocating for one-key hashes. - json_hash() : json_object(JSON_HASH) { entries_value.reserve(4); } - json_hash(json_object *p, json_pull *pl) : json_object(JSON_HASH, p, pl) { entries_value.reserve(4); } + json_hash() + : json_object(JSON_HASH) { + entries_value.reserve(4); + } + json_hash(json_object *p, json_pull *pl) + : json_object(JSON_HASH, p, pl) { + entries_value.reserve(4); + } }; inline std::string &json_object::string() { diff --git a/mvt.cpp b/mvt.cpp index 18f239d27..8042db7cd 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -407,7 +407,7 @@ std::string mvt_tile::encode() { std::string feature_string; protozero::pbf_writer feature_writer(feature_string); - if (layers[i].features[f].type >= 0) + if (layers[i].features[f].type >= 0) feature_writer.add_enum(3, layers[i].features[f].type); std::vector sorted_tags = layers[i].features[f].tags; diff --git a/tile-join.cpp b/tile-join.cpp index fec371aa8..034647145 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -335,7 +335,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &attribute_values, double if (descending) { // For descending: drop features > threshold, keep features <= threshold // ix points at the last value to keep - size_t ix = (size_t)((attribute_values.size() - 1) * f); + size_t ix = (size_t) ((attribute_values.size() - 1) * f); while (ix > 0 && attribute_values[ix] >= existing_attribute) { ix--; } @@ -844,7 +844,7 @@ static double choose_minattribute(std::vector &attribute_values, double } else { // For ascending: drop features < threshold, keep features >= threshold // ix points at the first value to keep - size_t ix = (size_t)ceil((double)(attribute_values.size() - 1) * (1 - f)); + size_t ix = (size_t) ceil((double) (attribute_values.size() - 1) * (1 - f)); if (ix >= attribute_values.size()) { ix = attribute_values.size() - 1; } @@ -2106,8 +2106,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (attr_valid) { add_sample_to(attribute_values, attr_numeric, attribute_values_increment, seq); bool should_drop = arg->drop_by_attribute_descending - ? (minattribute != HUGE_VAL && attr_numeric > minattribute) - : (minattribute != -HUGE_VAL && attr_numeric < minattribute); + ? (minattribute != HUGE_VAL && attr_numeric > minattribute) + : (minattribute != -HUGE_VAL && attr_numeric < minattribute); if (should_drop) { can_stop_early = false; if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) { @@ -2770,7 +2770,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } } else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) { minattribute_fraction = minattribute_fraction * - adjusted_max_tile_features / adjusted_feature_count * 0.75; + adjusted_max_tile_features / adjusted_feature_count * 0.75; if (minattribute_fraction > 0.80) { if (!quiet) { fprintf(stderr, @@ -3452,8 +3452,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: again = true; } bool attr_propagate = drop_by_attribute_descending - ? args[thread].minattribute_out < zoom_minattribute - : args[thread].minattribute_out > zoom_minattribute; + ? args[thread].minattribute_out < zoom_minattribute + : args[thread].minattribute_out > zoom_minattribute; if (attr_propagate) { zoom_minattribute = args[thread].minattribute_out; again = true;