diff --git a/index.js b/index.js deleted file mode 100644 index 179c8b7..0000000 --- a/index.js +++ /dev/null @@ -1,2 +0,0 @@ -const esmRequire = require('esm')(module); -module.exports = esmRequire('./main.js').default; diff --git a/main.cjs b/main.cjs new file mode 100644 index 0000000..206fd24 --- /dev/null +++ b/main.cjs @@ -0,0 +1,901 @@ +'use strict'; + +var sqlite = require('sqlite'); +var VTpbf = require('vt-pbf'); +var nodeGzip = require('node-gzip'); +var perf_hooks = require('perf_hooks'); +var fs = require('fs'); +var sqlite3 = require('sqlite3'); + +const ARRAY_TYPES = [ + Int8Array, Uint8Array, Uint8ClampedArray, Int16Array, Uint16Array, + Int32Array, Uint32Array, Float32Array, Float64Array +]; + +/** @typedef {Int8ArrayConstructor | Uint8ArrayConstructor | Uint8ClampedArrayConstructor | Int16ArrayConstructor | Uint16ArrayConstructor | Int32ArrayConstructor | Uint32ArrayConstructor | Float32ArrayConstructor | Float64ArrayConstructor} TypedArrayConstructor */ + +const VERSION = 1; // serialized format version +const HEADER_SIZE = 8; + +class KDBush { + + /** + * Creates an index from raw `ArrayBuffer` data. + * @param {ArrayBuffer} data + */ + static from(data) { + if (!(data instanceof ArrayBuffer)) { + throw new Error('Data must be an instance of ArrayBuffer.'); + } + const [magic, versionAndType] = new Uint8Array(data, 0, 2); + if (magic !== 0xdb) { + throw new Error('Data does not appear to be in a KDBush format.'); + } + const version = versionAndType >> 4; + if (version !== VERSION) { + throw new Error(`Got v${version} data when expected v${VERSION}.`); + } + const ArrayType = ARRAY_TYPES[versionAndType & 0x0f]; + if (!ArrayType) { + throw new Error('Unrecognized array type.'); + } + const [nodeSize] = new Uint16Array(data, 2, 1); + const [numItems] = new Uint32Array(data, 4, 1); + + return new KDBush(numItems, nodeSize, ArrayType, data); + } + + /** + * Creates an index that will hold a given number of items. + * @param {number} numItems + * @param {number} [nodeSize=64] Size of the KD-tree node (64 by default). + * @param {TypedArrayConstructor} [ArrayType=Float64Array] The array type used for coordinates storage (`Float64Array` by default). + * @param {ArrayBuffer} [data] (For internal use only) + */ + constructor(numItems, nodeSize = 64, ArrayType = Float64Array, data) { + if (isNaN(numItems) || numItems < 0) throw new Error(`Unpexpected numItems value: ${numItems}.`); + + this.numItems = +numItems; + this.nodeSize = Math.min(Math.max(+nodeSize, 2), 65535); + this.ArrayType = ArrayType; + this.IndexArrayType = numItems < 65536 ? Uint16Array : Uint32Array; + + const arrayTypeIndex = ARRAY_TYPES.indexOf(this.ArrayType); + const coordsByteSize = numItems * 2 * this.ArrayType.BYTES_PER_ELEMENT; + const idsByteSize = numItems * this.IndexArrayType.BYTES_PER_ELEMENT; + const padCoords = (8 - idsByteSize % 8) % 8; + + if (arrayTypeIndex < 0) { + throw new Error(`Unexpected typed array class: ${ArrayType}.`); + } + + if (data && (data instanceof ArrayBuffer)) { // reconstruct an index from a buffer + this.data = data; + this.ids = new this.IndexArrayType(this.data, HEADER_SIZE, numItems); + this.coords = new this.ArrayType(this.data, HEADER_SIZE + idsByteSize + padCoords, numItems * 2); + this._pos = numItems * 2; + this._finished = true; + } else { // initialize a new index + this.data = new ArrayBuffer(HEADER_SIZE + coordsByteSize + idsByteSize + padCoords); + this.ids = new this.IndexArrayType(this.data, HEADER_SIZE, numItems); + this.coords = new this.ArrayType(this.data, HEADER_SIZE + idsByteSize + padCoords, numItems * 2); + this._pos = 0; + this._finished = false; + + // set header + new Uint8Array(this.data, 0, 2).set([0xdb, (VERSION << 4) + arrayTypeIndex]); + new Uint16Array(this.data, 2, 1)[0] = nodeSize; + new Uint32Array(this.data, 4, 1)[0] = numItems; + } + } + + /** + * Add a point to the index. + * @param {number} x + * @param {number} y + * @returns {number} An incremental index associated with the added item (starting from `0`). + */ + add(x, y) { + const index = this._pos >> 1; + this.ids[index] = index; + this.coords[this._pos++] = x; + this.coords[this._pos++] = y; + return index; + } + + /** + * Perform indexing of the added points. + */ + finish() { + const numAdded = this._pos >> 1; + if (numAdded !== this.numItems) { + throw new Error(`Added ${numAdded} items when expected ${this.numItems}.`); + } + // kd-sort both arrays for efficient search + sort(this.ids, this.coords, this.nodeSize, 0, this.numItems - 1, 0); + + this._finished = true; + return this; + } + + /** + * Search the index for items within a given bounding box. + * @param {number} minX + * @param {number} minY + * @param {number} maxX + * @param {number} maxY + * @returns {number[]} An array of indices correponding to the found items. + */ + range(minX, minY, maxX, maxY) { + if (!this._finished) throw new Error('Data not yet indexed - call index.finish().'); + + const {ids, coords, nodeSize} = this; + const stack = [0, ids.length - 1, 0]; + const result = []; + + // recursively search for items in range in the kd-sorted arrays + while (stack.length) { + const axis = stack.pop() || 0; + const right = stack.pop() || 0; + const left = stack.pop() || 0; + + // if we reached "tree node", search linearly + if (right - left <= nodeSize) { + for (let i = left; i <= right; i++) { + const x = coords[2 * i]; + const y = coords[2 * i + 1]; + if (x >= minX && x <= maxX && y >= minY && y <= maxY) result.push(ids[i]); + } + continue; + } + + // otherwise find the middle index + const m = (left + right) >> 1; + + // include the middle item if it's in range + const x = coords[2 * m]; + const y = coords[2 * m + 1]; + if (x >= minX && x <= maxX && y >= minY && y <= maxY) result.push(ids[m]); + + // queue search in halves that intersect the query + if (axis === 0 ? minX <= x : minY <= y) { + stack.push(left); + stack.push(m - 1); + stack.push(1 - axis); + } + if (axis === 0 ? maxX >= x : maxY >= y) { + stack.push(m + 1); + stack.push(right); + stack.push(1 - axis); + } + } + + return result; + } + + /** + * Search the index for items within a given radius. + * @param {number} qx + * @param {number} qy + * @param {number} r Query radius. + * @returns {number[]} An array of indices correponding to the found items. + */ + within(qx, qy, r) { + if (!this._finished) throw new Error('Data not yet indexed - call index.finish().'); + + const {ids, coords, nodeSize} = this; + const stack = [0, ids.length - 1, 0]; + const result = []; + const r2 = r * r; + + // recursively search for items within radius in the kd-sorted arrays + while (stack.length) { + const axis = stack.pop() || 0; + const right = stack.pop() || 0; + const left = stack.pop() || 0; + + // if we reached "tree node", search linearly + if (right - left <= nodeSize) { + for (let i = left; i <= right; i++) { + if (sqDist(coords[2 * i], coords[2 * i + 1], qx, qy) <= r2) result.push(ids[i]); + } + continue; + } + + // otherwise find the middle index + const m = (left + right) >> 1; + + // include the middle item if it's in range + const x = coords[2 * m]; + const y = coords[2 * m + 1]; + if (sqDist(x, y, qx, qy) <= r2) result.push(ids[m]); + + // queue search in halves that intersect the query + if (axis === 0 ? qx - r <= x : qy - r <= y) { + stack.push(left); + stack.push(m - 1); + stack.push(1 - axis); + } + if (axis === 0 ? qx + r >= x : qy + r >= y) { + stack.push(m + 1); + stack.push(right); + stack.push(1 - axis); + } + } + + return result; + } +} + +/** + * @param {Uint16Array | Uint32Array} ids + * @param {InstanceType} coords + * @param {number} nodeSize + * @param {number} left + * @param {number} right + * @param {number} axis + */ +function sort(ids, coords, nodeSize, left, right, axis) { + if (right - left <= nodeSize) return; + + const m = (left + right) >> 1; // middle index + + // sort ids and coords around the middle index so that the halves lie + // either left/right or top/bottom correspondingly (taking turns) + select(ids, coords, m, left, right, axis); + + // recursively kd-sort first half and second half on the opposite axis + sort(ids, coords, nodeSize, left, m - 1, 1 - axis); + sort(ids, coords, nodeSize, m + 1, right, 1 - axis); +} + +/** + * Custom Floyd-Rivest selection algorithm: sort ids and coords so that + * [left..k-1] items are smaller than k-th item (on either x or y axis) + * @param {Uint16Array | Uint32Array} ids + * @param {InstanceType} coords + * @param {number} k + * @param {number} left + * @param {number} right + * @param {number} axis + */ +function select(ids, coords, k, left, right, axis) { + + while (right > left) { + if (right - left > 600) { + const n = right - left + 1; + const m = k - left + 1; + const z = Math.log(n); + const s = 0.5 * Math.exp(2 * z / 3); + const sd = 0.5 * Math.sqrt(z * s * (n - s) / n) * (m - n / 2 < 0 ? -1 : 1); + const newLeft = Math.max(left, Math.floor(k - m * s / n + sd)); + const newRight = Math.min(right, Math.floor(k + (n - m) * s / n + sd)); + select(ids, coords, k, newLeft, newRight, axis); + } + + const t = coords[2 * k + axis]; + let i = left; + let j = right; + + swapItem(ids, coords, left, k); + if (coords[2 * right + axis] > t) swapItem(ids, coords, left, right); + + while (i < j) { + swapItem(ids, coords, i, j); + i++; + j--; + while (coords[2 * i + axis] < t) i++; + while (coords[2 * j + axis] > t) j--; + } + + if (coords[2 * left + axis] === t) swapItem(ids, coords, left, j); + else { + j++; + swapItem(ids, coords, j, right); + } + + if (j <= k) left = j + 1; + if (k <= j) right = j - 1; + } +} + +/** + * @param {Uint16Array | Uint32Array} ids + * @param {InstanceType} coords + * @param {number} i + * @param {number} j + */ +function swapItem(ids, coords, i, j) { + swap(ids, i, j); + swap(coords, 2 * i, 2 * j); + swap(coords, 2 * i + 1, 2 * j + 1); +} + +/** + * @param {InstanceType} arr + * @param {number} i + * @param {number} j + */ +function swap(arr, i, j) { + const tmp = arr[i]; + arr[i] = arr[j]; + arr[j] = tmp; +} + +/** + * @param {number} ax + * @param {number} ay + * @param {number} bx + * @param {number} by + */ +function sqDist(ax, ay, bx, by) { + const dx = ax - bx; + const dy = ay - by; + return dx * dx + dy * dy; +} + +const defaultOptions$1 = { + minZoom: 0, // min zoom to generate clusters on + maxZoom: 16, // max zoom level to cluster the points on + minPoints: 2, // minimum points to form a cluster + radius: 40, // cluster radius in pixels + extent: 512, // tile extent (radius is calculated relative to it) + nodeSize: 64, // size of the KD-tree leaf node, affects performance + log: false, // whether to log timing info + + // whether to generate numeric ids for input features (in vector tiles) + generateId: false, + + // a reduce function for calculating custom cluster properties + reduce: null, // (accumulated, props) => { accumulated.sum += props.sum; } + + // properties to use for individual points when running the reducer + map: props => props // props => ({sum: props.my_value}) +}; + +const fround = Math.fround || (tmp => ((x) => { tmp[0] = +x; return tmp[0]; }))(new Float32Array(1)); + +const OFFSET_ZOOM = 2; +const OFFSET_ID = 3; +const OFFSET_PARENT = 4; +const OFFSET_NUM = 5; +const OFFSET_PROP = 6; + +class Supercluster { + constructor(options) { + this.options = Object.assign(Object.create(defaultOptions$1), options); + this.trees = new Array(this.options.maxZoom + 1); + this.stride = this.options.reduce ? 7 : 6; + this.clusterProps = []; + } + + load(points) { + const {log, minZoom, maxZoom} = this.options; + + if (log) console.time('total time'); + + const timerId = `prepare ${ points.length } points`; + if (log) console.time(timerId); + + this.points = points; + + // generate a cluster object for each point and index input points into a KD-tree + const data = []; + + for (let i = 0; i < points.length; i++) { + const p = points[i]; + if (!p.geometry) continue; + + const [lng, lat] = p.geometry.coordinates; + const x = fround(lngX(lng)); + const y = fround(latY(lat)); + // store internal point/cluster data in flat numeric arrays for performance + data.push( + x, y, // projected point coordinates + Infinity, // the last zoom the point was processed at + i, // index of the source feature in the original input array + -1, // parent cluster id + 1 // number of points in a cluster + ); + if (this.options.reduce) data.push(0); // noop + } + let tree = this.trees[maxZoom + 1] = this._createTree(data); + + if (log) console.timeEnd(timerId); + + // cluster points on max zoom, then cluster the results on previous zoom, etc.; + // results in a cluster hierarchy across zoom levels + for (let z = maxZoom; z >= minZoom; z--) { + const now = +Date.now(); + + // create a new set of clusters for the zoom and index them with a KD-tree + tree = this.trees[z] = this._createTree(this._cluster(tree, z)); + + if (log) console.log('z%d: %d clusters in %dms', z, tree.numItems, +Date.now() - now); + } + + if (log) console.timeEnd('total time'); + + return this; + } + + getClusters(bbox, zoom) { + let minLng = ((bbox[0] + 180) % 360 + 360) % 360 - 180; + const minLat = Math.max(-90, Math.min(90, bbox[1])); + let maxLng = bbox[2] === 180 ? 180 : ((bbox[2] + 180) % 360 + 360) % 360 - 180; + const maxLat = Math.max(-90, Math.min(90, bbox[3])); + + if (bbox[2] - bbox[0] >= 360) { + minLng = -180; + maxLng = 180; + } else if (minLng > maxLng) { + const easternHem = this.getClusters([minLng, minLat, 180, maxLat], zoom); + const westernHem = this.getClusters([-180, minLat, maxLng, maxLat], zoom); + return easternHem.concat(westernHem); + } + + const tree = this.trees[this._limitZoom(zoom)]; + const ids = tree.range(lngX(minLng), latY(maxLat), lngX(maxLng), latY(minLat)); + const data = tree.data; + const clusters = []; + for (const id of ids) { + const k = this.stride * id; + clusters.push(data[k + OFFSET_NUM] > 1 ? getClusterJSON(data, k, this.clusterProps) : this.points[data[k + OFFSET_ID]]); + } + return clusters; + } + + getChildren(clusterId) { + const originId = this._getOriginId(clusterId); + const originZoom = this._getOriginZoom(clusterId); + const errorMsg = 'No cluster with the specified id.'; + + const tree = this.trees[originZoom]; + if (!tree) throw new Error(errorMsg); + + const data = tree.data; + if (originId * this.stride >= data.length) throw new Error(errorMsg); + + const r = this.options.radius / (this.options.extent * Math.pow(2, originZoom - 1)); + const x = data[originId * this.stride]; + const y = data[originId * this.stride + 1]; + const ids = tree.within(x, y, r); + const children = []; + for (const id of ids) { + const k = id * this.stride; + if (data[k + OFFSET_PARENT] === clusterId) { + children.push(data[k + OFFSET_NUM] > 1 ? getClusterJSON(data, k, this.clusterProps) : this.points[data[k + OFFSET_ID]]); + } + } + + if (children.length === 0) throw new Error(errorMsg); + + return children; + } + + getLeaves(clusterId, limit, offset) { + limit = limit || 10; + offset = offset || 0; + + const leaves = []; + this._appendLeaves(leaves, clusterId, limit, offset, 0); + + return leaves; + } + + getTile(z, x, y) { + const tree = this.trees[this._limitZoom(z)]; + const z2 = Math.pow(2, z); + const {extent, radius} = this.options; + const p = radius / extent; + const top = (y - p) / z2; + const bottom = (y + 1 + p) / z2; + + const tile = { + features: [] + }; + + this._addTileFeatures( + tree.range((x - p) / z2, top, (x + 1 + p) / z2, bottom), + tree.data, x, y, z2, tile); + + if (x === 0) { + this._addTileFeatures( + tree.range(1 - p / z2, top, 1, bottom), + tree.data, z2, y, z2, tile); + } + if (x === z2 - 1) { + this._addTileFeatures( + tree.range(0, top, p / z2, bottom), + tree.data, -1, y, z2, tile); + } + + return tile.features.length ? tile : null; + } + + getClusterExpansionZoom(clusterId) { + let expansionZoom = this._getOriginZoom(clusterId) - 1; + while (expansionZoom <= this.options.maxZoom) { + const children = this.getChildren(clusterId); + expansionZoom++; + if (children.length !== 1) break; + clusterId = children[0].properties.cluster_id; + } + return expansionZoom; + } + + _appendLeaves(result, clusterId, limit, offset, skipped) { + const children = this.getChildren(clusterId); + + for (const child of children) { + const props = child.properties; + + if (props && props.cluster) { + if (skipped + props.point_count <= offset) { + // skip the whole cluster + skipped += props.point_count; + } else { + // enter the cluster + skipped = this._appendLeaves(result, props.cluster_id, limit, offset, skipped); + // exit the cluster + } + } else if (skipped < offset) { + // skip a single point + skipped++; + } else { + // add a single point + result.push(child); + } + if (result.length === limit) break; + } + + return skipped; + } + + _createTree(data) { + const tree = new KDBush(data.length / this.stride | 0, this.options.nodeSize, Float32Array); + for (let i = 0; i < data.length; i += this.stride) tree.add(data[i], data[i + 1]); + tree.finish(); + tree.data = data; + return tree; + } + + _addTileFeatures(ids, data, x, y, z2, tile) { + for (const i of ids) { + const k = i * this.stride; + const isCluster = data[k + OFFSET_NUM] > 1; + + let tags, px, py; + if (isCluster) { + tags = getClusterProperties(data, k, this.clusterProps); + px = data[k]; + py = data[k + 1]; + } else { + const p = this.points[data[k + OFFSET_ID]]; + tags = p.properties; + const [lng, lat] = p.geometry.coordinates; + px = lngX(lng); + py = latY(lat); + } + + const f = { + type: 1, + geometry: [[ + Math.round(this.options.extent * (px * z2 - x)), + Math.round(this.options.extent * (py * z2 - y)) + ]], + tags + }; + + // assign id + let id; + if (isCluster || this.options.generateId) { + // optionally generate id for points + id = data[k + OFFSET_ID]; + } else { + // keep id if already assigned + id = this.points[data[k + OFFSET_ID]].id; + } + + if (id !== undefined) f.id = id; + + tile.features.push(f); + } + } + + _limitZoom(z) { + return Math.max(this.options.minZoom, Math.min(Math.floor(+z), this.options.maxZoom + 1)); + } + + _cluster(tree, zoom) { + const {radius, extent, reduce, minPoints} = this.options; + const r = radius / (extent * Math.pow(2, zoom)); + const data = tree.data; + const nextData = []; + const stride = this.stride; + + // loop through each point + for (let i = 0; i < data.length; i += stride) { + // if we've already visited the point at this zoom level, skip it + if (data[i + OFFSET_ZOOM] <= zoom) continue; + data[i + OFFSET_ZOOM] = zoom; + + // find all nearby points + const x = data[i]; + const y = data[i + 1]; + const neighborIds = tree.within(data[i], data[i + 1], r); + + const numPointsOrigin = data[i + OFFSET_NUM]; + let numPoints = numPointsOrigin; + + // count the number of points in a potential cluster + for (const neighborId of neighborIds) { + const k = neighborId * stride; + // filter out neighbors that are already processed + if (data[k + OFFSET_ZOOM] > zoom) numPoints += data[k + OFFSET_NUM]; + } + + // if there were neighbors to merge, and there are enough points to form a cluster + if (numPoints > numPointsOrigin && numPoints >= minPoints) { + let wx = x * numPointsOrigin; + let wy = y * numPointsOrigin; + + let clusterProperties; + let clusterPropIndex = -1; + + // encode both zoom and point index on which the cluster originated -- offset by total length of features + const id = ((i / stride | 0) << 5) + (zoom + 1) + this.points.length; + + for (const neighborId of neighborIds) { + const k = neighborId * stride; + + if (data[k + OFFSET_ZOOM] <= zoom) continue; + data[k + OFFSET_ZOOM] = zoom; // save the zoom (so it doesn't get processed twice) + + const numPoints2 = data[k + OFFSET_NUM]; + wx += data[k] * numPoints2; // accumulate coordinates for calculating weighted center + wy += data[k + 1] * numPoints2; + + data[k + OFFSET_PARENT] = id; + + if (reduce) { + if (!clusterProperties) { + clusterProperties = this._map(data, i, true); + clusterPropIndex = this.clusterProps.length; + this.clusterProps.push(clusterProperties); + } + reduce(clusterProperties, this._map(data, k)); + } + } + + data[i + OFFSET_PARENT] = id; + nextData.push(wx / numPoints, wy / numPoints, Infinity, id, -1, numPoints); + if (reduce) nextData.push(clusterPropIndex); + + } else { // left points as unclustered + for (let j = 0; j < stride; j++) nextData.push(data[i + j]); + + if (numPoints > 1) { + for (const neighborId of neighborIds) { + const k = neighborId * stride; + if (data[k + OFFSET_ZOOM] <= zoom) continue; + data[k + OFFSET_ZOOM] = zoom; + for (let j = 0; j < stride; j++) nextData.push(data[k + j]); + } + } + } + } + + return nextData; + } + + // get index of the point from which the cluster originated + _getOriginId(clusterId) { + return (clusterId - this.points.length) >> 5; + } + + // get zoom of the point from which the cluster originated + _getOriginZoom(clusterId) { + return (clusterId - this.points.length) % 32; + } + + _map(data, i, clone) { + if (data[i + OFFSET_NUM] > 1) { + const props = this.clusterProps[data[i + OFFSET_PROP]]; + return clone ? Object.assign({}, props) : props; + } + const original = this.points[data[i + OFFSET_ID]].properties; + const result = this.options.map(original); + return clone && result === original ? Object.assign({}, result) : result; + } +} + +function getClusterJSON(data, i, clusterProps) { + return { + type: 'Feature', + id: data[i + OFFSET_ID], + properties: getClusterProperties(data, i, clusterProps), + geometry: { + type: 'Point', + coordinates: [xLng(data[i]), yLat(data[i + 1])] + } + }; +} + +function getClusterProperties(data, i, clusterProps) { + const count = data[i + OFFSET_NUM]; + const abbrev = + count >= 10000 ? `${Math.round(count / 1000) }k` : + count >= 1000 ? `${Math.round(count / 100) / 10 }k` : count; + const propIndex = data[i + OFFSET_PROP]; + const properties = propIndex === -1 ? {} : Object.assign({}, clusterProps[propIndex]); + return Object.assign(properties, { + cluster: true, + cluster_id: data[i + OFFSET_ID], + point_count: count, + point_count_abbreviated: abbrev + }); +} + +// longitude/latitude to spherical mercator in [0..1] range +function lngX(lng) { + return lng / 360 + 0.5; +} +function latY(lat) { + const sin = Math.sin(lat * Math.PI / 180); + const y = (0.5 - 0.25 * Math.log((1 + sin) / (1 - sin)) / Math.PI); + return y < 0 ? 0 : y > 1 ? 1 : y; +} + +// spherical mercator to longitude/latitude +function xLng(x) { + return (x - 0.5) * 360; +} +function yLat(y) { + const y2 = (180 - y * 360) * Math.PI / 180; + return 360 * Math.atan(Math.exp(y2)) / Math.PI - 90; +} + +const defaultOptions = { + // For Supercluster + minZoom: 0, // min zoom to generate clusters on + maxZoom: 8, // max zoom level to cluster the points on + radius: 40, // cluster radius in pixels + extent: 512, // tile extent (radius is calculated relative to it) + nodeSize: 64, // size of the KD-tree leaf node, affects performance + log: false, // whether to log timing info + // a reduce function for calculating custom cluster properties + reduce: null, // (accumulated, props) => { accumulated.sum += props.sum; } + // properties to use for individual points when running the reducer + map: props => props, // props => ({sum: props.my_value}) + storeClusterExpansionZoom: false, + + // For mbtiles + bounds: '-180.0,-85,180,85', + center: '0,0,0', + tileSpecVersion: 2 +}; + +function extend(dest, src) { + for (const id in src) dest[id] = src[id]; + return dest; +} + +function main (options) { + options = extend(Object.create(defaultOptions), options); + const clustered = new Supercluster({ + minZoom: options.minZoom, + maxZoom: options.maxZoom, + radius: options.radius, + extent: options.extent, + nodeSize: options.nodeSize, + map: options.map, + reduce: options.reduce + }).load(JSON.parse(fs.readFileSync(options.input)).features); + + if (options.logPerformance) { + console.log(`Finished clustering at ${perf_hooks.performance.now()}`); + } + if (fs.existsSync(options.output)) { + // Clear previous MBTiles, if it exists + fs.unlinkSync(options.output); + } + const filter = options.filter; + return sqlite.open({filename: options.output, driver: sqlite3.Database}, {Promise}).then(db => Promise.all([ + db.run('CREATE TABLE metadata (name text, value text)'), + db.run('CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob)') + ]).then(() => { + // Build metadata table + db.run('INSERT INTO metadata (name, value) VALUES ("name", ?)', options.output); + db.run('INSERT INTO metadata (name, value) VALUES ("format", "pbf")'); + db.run('INSERT INTO metadata (name, value) VALUES ("minZoom", ?)', options.minZoom); + db.run('INSERT INTO metadata (name, value) VALUES ("maxZoom", ?)', options.maxZoom + (options.includeUnclustered ? 1 : 0)); + db.run('INSERT INTO metadata (name, value) VALUES ("bounds", ?)', options.bounds); + db.run('INSERT INTO metadata (name, value) VALUES ("center", ?)', options.center); + db.run('INSERT INTO metadata (name, value) VALUES ("type", "overlay")'); // See MBTiles spec: I think "overlay" is most appropriate here + db.run('INSERT INTO metadata (name, value) VALUES ("version", ?)', options.tileSpecVersion); + if (options.attribution) { + db.run('INSERT INTO metadata (name, value) VALUES ("attribution", ?)', options.attribution); + } + if (options.description) { + db.run('INSERT INTO metadata (name, value) VALUES ("description", ?)', options.description); + } + + const fields = {}; + const statements = []; + const compressedTiles = []; + // Insert tiles + for (let z = options.minZoom; z <= options.maxZoom + (options.includeUnclustered ? 1 : 0); z++) { + const zoomDimension = Math.pow(2, z); + // TODO: No need to process tiles outside of bounds + // TODO: Stop zoom descent for tiles that don't have any clusters + for (let x = 0; x < zoomDimension; x++) { + for (let y = 0; y < zoomDimension; y++) { + const tile = clustered.getTile(z, x, y); + if (!tile || !tile.features) { + // Don't serialize empty tiles + continue; + } + // Apply feature filter + if (filter) { + tile.features = tile.features.filter(feature => filter(feature.tags)); + } + + if (tile.features.length === 0) { + // Don't serialize empty tiles (after our custom feature filter) + continue; + } + + // Collect field information for metadata + for (const feature of tile.features) { + for (const property in feature.tags) { + fields[property] = typeof feature.tags[property]; + } + } + if (options.storeClusterExpansionZoom) { + for (const feature of tile.features) { + if (feature.tags.cluster_id) { + feature.tags['clusterExpansionZoom'] = clustered.getClusterExpansionZoom(feature.tags.cluster_id); + } + } + } + + // Convert to PBF and compress before insertion + compressedTiles.push( + nodeGzip.gzip(VTpbf.fromGeojsonVt({'geojsonLayer': tile}, {version: options.tileSpecVersion, extent: options.extent})).then((compressed) => { + if (compressed.length > 500000) { + return Promise.reject(new Error(`Tile z:${z}, x:${x}, y:${y} greater than 500KB compressed. Try increasing radius or max zoom, or try including fewer cluster properties.`)); + } + statements.push( + db.run( + 'INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES(?, ?, ?, ?)', + z, x, zoomDimension - 1 - y, compressed)); + return Promise.resolve(); + }) + ); + } + } + } + + // Complete metadata table by adding layer definition + const vectorJson = { + 'vector_layers': + [{ + 'id': 'geojsonLayer', + 'description': 'Point layer imported from GeoJSON.', + fields + }] + }; + statements.push( + db.run('INSERT INTO metadata (name, value) VALUES ("json", ?)', JSON.stringify(vectorJson))); + + return Promise.all(compressedTiles).then(() => Promise.all(statements).then(() => { + // TODO include stats? + if (options.logPerformance) { + console.log(`Finished generating MBTiles at ${perf_hooks.performance.now()}.`); + } + })); + })); +} + +module.exports = main; diff --git a/main.js b/main.js index 4deec54..bbf8d02 100644 --- a/main.js +++ b/main.js @@ -1,9 +1,10 @@ import Supercluster from 'supercluster'; -import Sqlite from 'sqlite'; +import {open} from 'sqlite'; import VTpbf from 'vt-pbf'; import {gzip} from 'node-gzip'; import {performance} from 'perf_hooks'; import fs from 'fs'; +import sqlite3 from 'sqlite3'; const defaultOptions = { // For Supercluster @@ -50,7 +51,7 @@ export default function (options) { fs.unlinkSync(options.output); } const filter = options.filter; - return Sqlite.open(options.output, {Promise}).then(db => Promise.all([ + return open({filename: options.output, driver: sqlite3.Database}, {Promise}).then(db => Promise.all([ db.run('CREATE TABLE metadata (name text, value text)'), db.run('CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob)') ]).then(() => { diff --git a/package.json b/package.json index 375cdb5..915f87e 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,14 @@ "name": "supertiler", "version": "0.0.4", "description": "GeoJSON -> MBTiles with Supercluster", - "module": "index.js", + "type": "module", + "module": "main.js", + "types": "supertiler.d.ts", "scripts": { - "pretest": "eslint index.js main.js supertiler.js test/test.js", - "test": "tap --node-arg=-r --node-arg=esm test", - "cov": "tap --node-arg=-r --node-arg=esm test --cov", - "prepublishOnly": "npm run test" + "pretest": "eslint main.js supertiler.js test/test.js", + "test": "tap test", + "build": "rollup -c", + "cov": "tap test --cov" }, "bin": { "supertiler": "supertiler.js" @@ -21,23 +23,30 @@ "mbtiles" ], "dependencies": { - "esm": "^3.1.0", "node-gzip": "^1.1.2", - "sqlite": "^3.0.3", - "sqlite3": "^4.1.0", - "supercluster": "^6.0.1", - "vt-pbf": "^3.1.1", - "yargs": "^13.2.1" + "sqlite": "^4.2.1", + "sqlite3": "^5.1.6", + "supercluster": "^8.0.1", + "vt-pbf": "^3.1.3", + "yargs": "^17.7.2" }, "devDependencies": { - "eslint": "^5.12.0", + "@rollup/plugin-node-resolve": "^15.1.0", + "eslint": "^8.44.0", "eslint-config-mourner": "^3.0.0", - "tap": "^12.1.1" + "rollup": "^3.26.2", + "tap": "^16.3.7" }, "eslintConfig": { "extends": "mourner", "rules": { "camelcase": 0 } + }, + "exports": { + ".": { + "import": "./main.js", + "require": "./main.cjs" + } } } diff --git a/rollup.config.js b/rollup.config.js new file mode 100644 index 0000000..ec30e09 --- /dev/null +++ b/rollup.config.js @@ -0,0 +1,10 @@ +import { nodeResolve } from '@rollup/plugin-node-resolve'; + +export default { + input: './main.js', + output: { + file: './main.cjs', + format: 'cjs' + }, + plugins: [nodeResolve({resolveOnly: ['supercluster', 'kdbush']})] +} \ No newline at end of file diff --git a/supertiler.d.ts b/supertiler.d.ts new file mode 100644 index 0000000..ac224e1 --- /dev/null +++ b/supertiler.d.ts @@ -0,0 +1,15 @@ +declare module 'supertiler' { + import { AnyProps, Options } from 'supercluster' + + export type SupertilerOptions = Options & { + input: string; + output: string; + + storeClusterExpansionZoom?: boolean; + bounds?: string; + center?: string; + tileSpecVersion?: number; + } + + export default function supertiler(options: SupertilerOptions): Promise; +} \ No newline at end of file diff --git a/test/test.cjs b/test/test.cjs new file mode 100644 index 0000000..465a228 --- /dev/null +++ b/test/test.cjs @@ -0,0 +1,46 @@ +// import tap from 'tap'; +const tap = require('tap'); +const supertiler = require('../main.cjs'); +const {open} = require('sqlite'); +const sqlite3 = require('sqlite3'); +// import {open} from 'sqlite'; +// import sqlite3 from 'sqlite3'; + +const test = tap.test; + +test('mbtiles sanity test', (t) => { + supertiler({ + input: './test/places.geojson', + output: './test/basic.mbtiles', + maxZoom: 2 + }).then(() => { + open({filename: './test/basic.mbtiles', driver: sqlite3.Database}, {Promise}).then((db) => { + Promise.all([ + db.all('SELECT * FROM metadata'), + db.all('SELECT * from tiles') + ]).then((results) => { + const metadata = results[0]; + const tiles = results[1]; + const expectedMetadata = { + 'name': './test/basic.mbtiles', + 'bounds': '-180.0,-85,180,85', + 'center': '0,0,0', + 'type': 'overlay', + 'version': '2', + 'maxZoom': '2', + 'format': 'pbf', + 'minZoom': '0', + 'json': '{"vector_layers":[{"id":"geojsonLayer","description":"Point layer imported from GeoJSON.","fields":{"cluster":"boolean","cluster_id":"number","point_count":"number","point_count_abbreviated":"number","scalerank":"number","name":"string","comment":"object","name_alt":"object","lat_y":"number","long_x":"number","region":"string","subregion":"string","featureclass":"string"}}]}' + }; + const roundTripMetadata = {}; + for (const row of metadata) { + roundTripMetadata[row.name] = row.value; + } + t.same(roundTripMetadata, expectedMetadata); + + t.equals(tiles.length, 19); + t.end(); + }); + }); + }); +}); diff --git a/test/test.js b/test/test.js index b04030b..554e9aa 100644 --- a/test/test.js +++ b/test/test.js @@ -1,6 +1,7 @@ import tap from 'tap'; -import supertiler from '../index.js'; -import Sqlite from 'sqlite'; +import supertiler from '../main.js'; +import {open} from 'sqlite'; +import sqlite3 from 'sqlite3'; const test = tap.test; @@ -10,7 +11,7 @@ test('mbtiles sanity test', (t) => { output: './test/basic.mbtiles', maxZoom: 2 }).then(() => { - Sqlite.open('./test/basic.mbtiles', {Promise}).then((db) => { + open({filename: './test/basic.mbtiles', driver: sqlite3.Database}, {Promise}).then((db) => { Promise.all([ db.all('SELECT * FROM metadata'), db.all('SELECT * from tiles')