Skip to content

Use a hash chain when deduplicating tile layer keys and values #306

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
e-n-f wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Use a hash chain when deduplicating tile layer keys and values #306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 22 additions & 10 deletions mvt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,22 +607,34 @@ std::string mvt_value::toString() const {

void mvt_layer::tag(mvt_feature &feature, std::string const &key, mvt_value const &value) {
size_t key_hash = fnv1a(key) % key_dedup.size();
if (key_dedup[key_hash] >= 0 &&
keys[key_dedup[key_hash]] == key) {
} else {
key_dedup[key_hash] = keys.size();
ssize_t key_index = -1;

for (auto const &k : key_dedup[key_hash]) {
if (keys[k] == key) {
key_index = k;
break;
}
}
if (key_index < 0) {
key_dedup[key_hash].push_back(key_index = keys.size());
keys.push_back(key);
}
feature.tags.push_back(key_dedup[key_hash]);
feature.tags.push_back(key_index);

size_t value_hash = std::hash<mvt_value>()(value) % value_dedup.size();
if (value_dedup[value_hash] >= 0 &&
values[value_dedup[value_hash]] == value) {
} else {
value_dedup[value_hash] = values.size();
ssize_t value_index = -1;

for (auto const &v : value_dedup[value_hash]) {
if (values[v] == value) {
value_index = v;
break;
}
}
if (value_index < 0) {
value_dedup[value_hash].push_back(value_index = values.size());
values.push_back(value);
}
feature.tags.push_back(value_dedup[value_hash]);
feature.tags.push_back(value_index);
}

bool is_integer(const char *s, long long *v) {
Expand Down
4 changes: 2 additions & 2 deletions mvt.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,8 @@ struct mvt_layer {
void tag(mvt_feature &feature, std::string const &key, mvt_value const &value);

// For tracking the key-value constants already used in this layer
std::vector<ssize_t> key_dedup = std::vector<ssize_t>(65536, -1);
std::vector<ssize_t> value_dedup = std::vector<ssize_t>(65536, -1);
std::vector<std::vector<ssize_t>> key_dedup = std::vector<std::vector<ssize_t>>(500, std::vector<ssize_t>());
std::vector<std::vector<ssize_t>> value_dedup = std::vector<std::vector<ssize_t>>(500, std::vector<ssize_t>());

int detail() const {
return std::round(std::log(extent) / std::log(2));
Expand Down
4 changes: 3 additions & 1 deletion ...M10000_--drop-densest-as-needed_--extend-zooms-if-still-dropping-maximum_2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"maxzoom": "2",
"minzoom": "0",
"name": "tests/ne_110m_populated_places/out/-z0_-M10000_--drop-densest-as-needed_--extend-zooms-if-still-dropping-maximum_2.json.check.mbtiles",
"strategies": "[{\"dropped_as_needed\":223,\"tile_size_desired\":75535},{\"dropped_as_needed\":230,\"tile_size_desired\":44899},{\"dropped_as_needed\":195,\"tile_size_desired\":35505}]",
"strategies": "[{\"dropped_as_needed\":222,\"tile_size_desired\":74734},{\"dropped_as_needed\":230,\"tile_size_desired\":44415},{\"dropped_as_needed\":195,\"tile_size_desired\":35130}]",
"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
"type": "overlay",
"version": "2"
Expand All @@ -18,6 +18,8 @@
{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
{ "type": "Feature", "properties": { "SCALERANK": 1, "NATSCALE": 300, "LABELRANK": 2, "FEATURECLA": "Populated place", "NAME": "Vancouver", "DIFFASCII": 0, "NAMEASCII": "Vancouver", "ADM0CAP": 0, "CAPALT": 0, "WORLDCITY": 0, "MEGACITY": 1, "SOV0NAME": "Canada", "SOV_A3": "CAN", "ADM0NAME": "Canada", "ADM0_A3": "CAN", "ADM1NAME": "British Columbia", "ISO_A2": "CA", "LATITUDE": 49.273417, "LONGITUDE": -123.121644, "CHANGED": 5, "NAMEDIFF": 0, "DIFFNOTE": "Changed scale rank.", "POP_MAX": 2313328, "POP_MIN": 603502, "POP_OTHER": 482002, "RANK_MAX": 12, "RANK_MIN": 11, "GEONAMEID": 6173331, "MEGANAME": "Vancouver", "LS_NAME": "Vancouver2", "LS_MATCH": 1, "CHECKME": 0, "MAX_POP10": 1590116, "MAX_POP20": 1588839, "MAX_POP50": 1590116, "MAX_POP300": 1590116, "MAX_POP310": 0, "MAX_NATSCA": 100, "MIN_AREAKM": 706, "MAX_AREAKM": 708, "MIN_AREAMI": 273, "MAX_AREAMI": 273, "MIN_PERKM": 398, "MAX_PERKM": 405, "MIN_PERMI": 248, "MAX_PERMI": 251, "MIN_BBXMIN": -123.283333, "MAX_BBXMIN": -123.283333, "MIN_BBXMAX": -122.708333, "MAX_BBXMAX": -122.708333, "MIN_BBYMIN": 49.1, "MAX_BBYMIN": 49.1, "MIN_BBYMAX": 49.383333, "MAX_BBYMAX": 49.383333, "MEAN_BBXC": -122.982768, "MEAN_BBYC": 49.228888, "COMPARE": 0, "GN_ASCII": "Vancouver", "FEATURE_CL": "P", "FEATURE_CO": "PPL", "ADMIN1_COD": 2, "GN_POP": 1837969, "ELEVATION": 0, "GTOPO30": 63, "TIMEZONE": "America/Vancouver", "GEONAMESNO": "Geonames ascii name + lat.d + long.d matching.", "UN_FID": 15, "UN_ADM0": "Canada", "UN_LAT": 49.27, "UN_LONG": -122.96, "POP1950": 556, "POP1955": 588, "POP1960": 620, "POP1965": 836, "POP1970": 1045, "POP1975": 1150, "POP1980": 1247, "POP1985": 1359, "POP1990": 1559, "POP1995": 1789, "POP2000": 1959, "POP2005": 2093, "POP2010": 2146, "POP2015": 2219, "POP2020": 2310, "POP2025": 2380, "POP2050": 2444 }, "geometry": { "type": "Point", "coordinates": [ -123.134766, 49.267805 ] } }
,
{ "type": "Feature", "properties": { "SCALERANK": 1, "NATSCALE": 300, "LABELRANK": 1, "FEATURECLA": "Populated place", "NAME": "Chicago", "DIFFASCII": 0, "NAMEASCII": "Chicago", "ADM0CAP": 0, "CAPALT": 0, "WORLDCITY": 1, "MEGACITY": 1, "SOV0NAME": "United States", "SOV_A3": "USA", "ADM0NAME": "United States of America", "ADM0_A3": "USA", "ADM1NAME": "Illinois", "ISO_A2": "US", "LATITUDE": 41.829991, "LONGITUDE": -87.750055, "CHANGED": 0, "NAMEDIFF": 0, "POP_MAX": 8990000, "POP_MIN": 2841952, "POP_OTHER": 3635101, "RANK_MAX": 13, "RANK_MIN": 12, "GEONAMEID": 4887398, "MEGANAME": "Chicago", "LS_NAME": "Chicago", "LS_MATCH": 1, "CHECKME": 0, "MAX_POP10": 3747798, "MAX_POP20": 5069998, "MAX_POP50": 8416660, "MAX_POP300": 8416660, "MAX_POP310": 8450289, "MAX_NATSCA": 300, "MIN_AREAKM": 1345, "MAX_AREAKM": 4804, "MIN_AREAMI": 519, "MAX_AREAMI": 1855, "MIN_PERKM": 471, "MAX_PERKM": 2946, "MIN_PERMI": 293, "MAX_PERMI": 1830, "MIN_BBXMIN": -88.408333, "MAX_BBXMIN": -88.03629, "MIN_BBXMAX": -87.528138, "MAX_BBXMAX": -87.125, "MIN_BBYMIN": 41.391667, "MAX_BBYMIN": 41.458333, "MIN_BBYMAX": 42.000972, "MAX_BBYMAX": 42.491667, "MEAN_BBXC": -87.85874, "MEAN_BBYC": 41.832719, "COMPARE": 0, "GN_ASCII": "Chicago", "FEATURE_CL": "P", "FEATURE_CO": "PPL", "ADMIN1_COD": 0, "GN_POP": 2841952, "ELEVATION": 179, "GTOPO30": 181, "TIMEZONE": "America/Chicago", "GEONAMESNO": "GeoNames match with ascii name + lat + long whole numbers.", "UN_FID": 531, "UN_ADM0": "United States of America", "UN_LAT": 41.82, "UN_LONG": -87.64, "POP1950": 4999, "POP1955": 5565, "POP1960": 6183, "POP1965": 6639, "POP1970": 7106, "POP1975": 7160, "POP1980": 7216, "POP1985": 7285, "POP1990": 7374, "POP1995": 7839, "POP2000": 8333, "POP2005": 8820, "POP2010": 8990, "POP2015": 9211, "POP2020": 9516, "POP2025": 9756, "POP2050": 9932 }, "geometry": { "type": "Point", "coordinates": [ -87.714844, 41.836828 ] } }
,
{ "type": "Feature", "properties": { "SCALERANK": 3, "NATSCALE": 110, "LABELRANK": 8, "FEATURECLA": "Admin-0 capital", "NAME": "Reykjavík", "DIFFASCII": 1, "NAMEASCII": "Reykjavik", "ADM0CAP": 1, "CAPALT": 0, "WORLDCITY": 0, "MEGACITY": 0, "SOV0NAME": "Iceland", "SOV_A3": "ISL", "ADM0NAME": "Iceland", "ADM0_A3": "ISL", "ADM1NAME": "Suðurnes", "ISO_A2": "IS", "LATITUDE": 64.150024, "LONGITUDE": -21.950014, "CHANGED": 0, "NAMEDIFF": 0, "POP_MAX": 166212, "POP_MIN": 113906, "POP_OTHER": 160116, "RANK_MAX": 9, "RANK_MIN": 9, "GEONAMEID": 3413829, "LS_NAME": "Reykjavik", "LS_MATCH": 1, "CHECKME": 0, "MAX_POP10": 166212, "MAX_POP20": 166212, "MAX_POP50": 166212, "MAX_POP300": 166212, "MAX_POP310": 0, "MAX_NATSCA": 100, "MIN_AREAKM": 75, "MAX_AREAKM": 75, "MIN_AREAMI": 29, "MAX_AREAMI": 29, "MIN_PERKM": 119, "MAX_PERKM": 119, "MIN_PERMI": 74, "MAX_PERMI": 74, "MIN_BBXMIN": -22.008333, "MAX_BBXMIN": -22.008333, "MIN_BBXMAX": -21.75, "MAX_BBXMAX": -21.75, "MIN_BBYMIN": 64.05, "MAX_BBYMIN": 64.05, "MIN_BBYMAX": 64.166667, "MAX_BBYMAX": 64.166667, "MEAN_BBXC": -21.8825, "MEAN_BBYC": 64.116125, "COMPARE": 0, "GN_ASCII": "Reykjavik", "FEATURE_CL": "P", "FEATURE_CO": "PPLC", "ADMIN1_COD": 39, "GN_POP": 113906, "ELEVATION": 0, "GTOPO30": 16, "TIMEZONE": "Atlantic/Reykjavik", "GEONAMESNO": "GeoNames match general.", "UN_FID": 0, "UN_LAT": 0, "UN_LONG": 0, "POP1950": 0, "POP1955": 0, "POP1960": 0, "POP1965": 0, "POP1970": 0, "POP1975": 0, "POP1980": 0, "POP1985": 0, "POP1990": 0, "POP1995": 0, "POP2000": 0, "POP2005": 0, "POP2010": 0, "POP2015": 0, "POP2020": 0, "POP2025": 0, "POP2050": 0 }, "geometry": { "type": "Point", "coordinates": [ -21.972656, 64.168107 ] } }
,
{ "type": "Feature", "properties": { "SCALERANK": 3, "NATSCALE": 110, "LABELRANK": 0, "FEATURECLA": "Admin-0 capital", "NAME": "Praia", "DIFFASCII": 0, "NAMEASCII": "Praia", "ADM0CAP": 1, "CAPALT": 0, "WORLDCITY": 0, "MEGACITY": 0, "SOV0NAME": "Cape Verde", "SOV_A3": "CPV", "ADM0NAME": "Cape Verde", "ADM0_A3": "CPV", "ISO_A2": "CV", "LATITUDE": 14.916698, "LONGITUDE": -23.516689, "CHANGED": 0, "NAMEDIFF": 0, "POP_MAX": 113364, "POP_MIN": 88859, "POP_OTHER": 89205, "RANK_MAX": 9, "RANK_MIN": 8, "GEONAMEID": 3374333, "LS_NAME": "Praia", "LS_MATCH": 1, "CHECKME": 5, "MAX_POP10": 88859, "MAX_POP20": 88859, "MAX_POP50": 88859, "MAX_POP300": 88859, "MAX_POP310": 0, "MAX_NATSCA": 100, "MIN_AREAKM": 37, "MAX_AREAKM": 37, "MIN_AREAMI": 14, "MAX_AREAMI": 14, "MIN_PERKM": 40, "MAX_PERKM": 40, "MIN_PERMI": 25, "MAX_PERMI": 25, "MIN_BBXMIN": -23.541667, "MAX_BBXMIN": -23.541667, "MIN_BBXMAX": -23.483333, "MAX_BBXMAX": -23.483333, "MIN_BBYMIN": 14.9, "MAX_BBYMIN": 14.9, "MIN_BBYMAX": 14.983333, "MAX_BBYMAX": 14.983333, "MEAN_BBXC": -23.514907, "MEAN_BBYC": 14.938056, "COMPARE": 0, "GN_ASCII": "Praia", "FEATURE_CL": "P", "FEATURE_CO": "PPLC", "ADMIN1_COD": 0, "GN_POP": 113364, "ELEVATION": 0, "GTOPO30": -9999, "TIMEZONE": "Atlantic/Cape_Verde", "GEONAMESNO": "GeoNames match general.", "UN_FID": 0, "UN_LAT": 0, "UN_LONG": 0, "POP1950": 0, "POP1955": 0, "POP1960": 0, "POP1965": 0, "POP1970": 0, "POP1975": 0, "POP1980": 0, "POP1985": 0, "POP1990": 0, "POP1995": 0, "POP2000": 0, "POP2005": 0, "POP2010": 0, "POP2015": 0, "POP2020": 0, "POP2025": 0, "POP2050": 0 }, "geometry": { "type": "Point", "coordinates": [ -23.554688, 14.944785 ] } }
Expand Down
Loading