Add MLT output format

.github/workflows/test.yml

@@ -11,6 +11,8 @@ jobs:
         version: ['Release', 'Debug']
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Install dependencies (Ubuntu)
         if: runner.os == 'Linux'
         run: sudo apt-get install libsqlite3-dev

.gitignore

@@ -43,6 +43,8 @@ tests/**/*.mbtiles
 tests/**/*.check
 tests/**/*.geobuf
 
+mlt-build/
+
 # Vim
 *.swp
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "maplibre-tile-spec"]
+	path = maplibre-tile-spec
+	url = https://github.com/maplibre/maplibre-tile-spec.git

Makefile

@@ -57,9 +57,27 @@ H = $(wildcard *.h) $(wildcard *.hpp)
 C = $(wildcard *.c) $(wildcard *.cpp)
 
 INCLUDES = -I/usr/local/include -I. -Iclipper2/include
+MLT_INCLUDES = -Imaplibre-tile-spec/cpp/include -isystem maplibre-tile-spec/cpp/vendor/fsst
+MLT_LIBS = mlt-build/libmlt-cpp-encoder.a mlt-build/libfsst-lib.a mlt-build/fastpfor/libFastPFOR.a
+MLT_BUILD_STAMP = mlt-build/.built
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
+$(MLT_BUILD_STAMP): maplibre-tile-spec/cpp/CMakeLists.txt
+	cmake -S maplibre-tile-spec/cpp -B mlt-build \
+		-DCMAKE_BUILD_TYPE=$(BUILDTYPE) \
+		-DMLT_WITH_FASTPFOR=ON \
+		-DMLT_WITH_FASTPFOR_SIMD=OFF \
+		-DMLT_WITH_JSON=OFF \
+		-DMLT_WITH_TESTS=OFF \
+		-DMLT_WITH_TOOLS=OFF \
+		-DCMAKE_CXX_STANDARD=20 \
+		$(if $(VERBOSE),,--log-level=WARNING) > /dev/null
+	cmake --build mlt-build --target mlt-cpp-encoder $(if $(VERBOSE),,-- -s) > /dev/null
+	touch $@
+
+$(MLT_LIBS): $(MLT_BUILD_STAMP)
+
+tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o mlt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o $(MLT_LIBS)
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-enumerate: enumerate.o
@@ -85,19 +103,22 @@ tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o tex
 %.o: %.c
 	$(CC) -MMD $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c -o $@ $<
 
+mlt.o: mlt.cpp mlt-build/libmlt-cpp-encoder.a
+	$(CXX) -MMD $(PG) $(INCLUDES) $(MLT_INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -std=c++20 -c -o $@ $<
+
 %.o: %.cpp
 	$(CXX) -MMD $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c -o $@ $<
 
 clean:
-	rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check
+	rm -rf ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check mlt-build
 
 indent:
 	clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/*.[ch]
 
 TESTS = $(wildcard tests/*/out/*.json)
 SPACE = $(NULL) $(NULL)
 
-test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test overzoom-test
+test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test overzoom-test mlt-test
 	./unit
 
 suffixes = json json.gz
@@ -584,6 +605,21 @@ layer-json-test: tippecanoe tippecanoe-decode
 	cmp tests/layer-json/out.mbtiles.json.check tests/layer-json/out.mbtiles.json
 	rm -f tests/layer-json/out.mbtiles.json.check tests/layer-json/out.mbtiles
 
+mlt-test: tippecanoe
+	./tippecanoe -q --output-format=mlt -z5 -f -o tests/mlt/points.mbtiles tests/mlt/points.geojson
+	./tippecanoe -q -z5 -f -o tests/mlt/points-mvt.mbtiles tests/mlt/points.geojson
+	@test $$(sqlite3 tests/mlt/points.mbtiles "SELECT COUNT(*) FROM tiles") -eq $$(sqlite3 tests/mlt/points-mvt.mbtiles "SELECT COUNT(*) FROM tiles") || (echo "FAIL: MLT and MVT tile counts differ" && exit 1)
+	@test "$$(sqlite3 tests/mlt/points.mbtiles "SELECT value FROM metadata WHERE name='format'")" = "mlt" || (echo "FAIL: format metadata is not 'mlt'" && exit 1)
+	@sqlite3 tests/mlt/points.mbtiles "SELECT hex(substr(tile_data, 1, 2)) FROM tiles LIMIT 1" | grep -q "1F8B" || (echo "FAIL: MLT tiles not gzip compressed" && exit 1)
+	rm -rf tests/mlt/dir-out
+	./tippecanoe -q --output-format=mlt -z2 -f -e tests/mlt/dir-out tests/mlt/points.geojson
+	@test $$(find tests/mlt/dir-out -name '*.mlt' | wc -l) -gt 0 || (echo "FAIL: No .mlt files in directory output" && exit 1)
+	@test $$(find tests/mlt/dir-out -name '*.pbf' | wc -l) -eq 0 || (echo "FAIL: .pbf files in MLT directory output" && exit 1)
+	./tippecanoe -q --output-format=mlt --pretessellate -z5 -f -o tests/mlt/points-tess.mbtiles tests/mlt/points.geojson
+	@test $$(sqlite3 tests/mlt/points-tess.mbtiles "SELECT COUNT(*) FROM tiles") -gt 0 || (echo "FAIL: No tiles with pretessellate" && exit 1)
+	rm -f tests/mlt/points.mbtiles tests/mlt/points-mvt.mbtiles tests/mlt/points-tess.mbtiles
+	rm -rf tests/mlt/dir-out
+
 # Use this target to regenerate the standards that the tests are compared against
 # after making a change that legitimately changes their output
 

README.md

@@ -44,7 +44,7 @@ $ brew install tippecanoe
 On Ubuntu it will usually be easiest to build from the source repository:
 
 ```sh
-$ git clone https://github.com/felt/tippecanoe.git
+$ git clone --recurse-submodules https://github.com/felt/tippecanoe.git
 $ cd tippecanoe
 $ make -j
 $ make install
@@ -552,7 +552,10 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
  * `--limit-tile-feature-count-at-maximum-zoom=`_features_: Abruptly limit each tile at the maximum zoom level to the specified number of _features_, after ordering them if specified.
  * `-pf` or `--no-feature-limit`: Don't limit tiles to 200,000 features
  * `-pk` or `--no-tile-size-limit`: Don't limit tiles to 500K bytes
- * `-pC` or `--no-tile-compression`: Don't compress the PBF vector tile data. If you are getting "Unimplemented type 3" error messages from a renderer, it is probably because it expects uncompressed tiles using this option rather than the normal gzip-compressed tiles.
+ * `-pC` or `--no-tile-compression`: Don't compress the vector tile data. If you are getting "Unimplemented type 3" error messages from a renderer, it is probably because it expects uncompressed tiles using this option rather than the normal gzip-compressed tiles.
+ * `--output-format=`*format*: Set the tile encoding format. Supported values: `mvt` (default, Mapbox Vector Tiles) or `mlt` ([MapLibre Tiles](https://github.com/maplibre/maplibre-tile-spec)).
+ * `--pretessellate`: When using `--output-format=mlt`, pre-triangulate polygon geometries. Only applies to layers where all features are polygons.
+ * `--no-mlt-feature-sort`: When using `--output-format=mlt`, disable within-tile spatial sorting of features by Hilbert curve index. Sorting is on by default.
  * `-pg` or `--no-tile-stats`: Don't generate the `tilestats` row in the tileset metadata. Uploads without [tilestats](https://github.com/mapbox/mapbox-geostats) will take longer to process.
  * `--tile-stats-attributes-limit=`*count*: Include `tilestats` information about at most *count* attributes instead of the default 1000.
  * `--tile-stats-sample-values-limit=`*count*: Calculate `tilestats` attribute statistics based on *count* values instead of the default 1000.
@@ -821,7 +824,7 @@ The options are:
 ### Setting or disabling tile size limits
 
  * `-pk` or `--no-tile-size-limit`: Don't skip tiles larger than 500K.
- * `-pC` or `--no-tile-compression`: Don't compress the PBF vector tile data.
+ * `-pC` or `--no-tile-compression`: Don't compress the vector tile data.
  * `-pg` or `--no-tile-stats`: Don't generate the `tilestats` row in the tileset metadata. Uploads without [tilestats](https://github.com/mapbox/mapbox-geostats) will take longer to process.
  * `--tile-stats-attributes-limit=`*count*: Include `tilestats` information about at most *count* attributes instead of the default 1000.
  * `--tile-stats-sample-values-limit=`*count*: Calculate `tilestats` attribute statistics based on *count* values instead of the default 1000.

dirtiles.cpp

@@ -25,7 +25,7 @@ std::string dir_read_tile(std::string base, struct zxy tile) {
 	return (contents.str());
 }
 
-void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) {
+void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf, const char *ext) {
 	// Don't check mkdir error returns, since most of these calls to
 	// mkdir will be creating directories that already exist.
 	mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO);
@@ -39,7 +39,7 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const
 	newdir = newdir + "/" + std::to_string(tx);
 	mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
 
-	newdir = newdir + "/" + std::to_string(ty) + ".pbf";
+	newdir = newdir + "/" + std::to_string(ty) + ext;
 
 	struct stat st;
 	if (stat(newdir.c_str(), &st) == 0) {
@@ -81,7 +81,7 @@ static bool pbfname(const char *s) {
 		s++;
 	}
 
-	return strcmp(s, ".pbf") == 0 || strcmp(s, ".mvt") == 0;
+	return strcmp(s, ".pbf") == 0 || strcmp(s, ".mvt") == 0 || strcmp(s, ".mlt") == 0;
 }
 
 void check_dir(const char *dir, char **argv, bool force, bool forcetable) {

dirtiles.hpp

@@ -6,7 +6,7 @@
 #ifndef DIRTILES_HPP
 #define DIRTILES_HPP
 
-void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
+void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf, const char *ext = ".pbf");
 void dir_erase_zoom(const char *outdir, int z);
 void dir_write_metadata(const char *outdir, const metadata &m);
 

main.cpp

@@ -104,6 +104,9 @@ bool drop_by_attribute_descending = false;
 std::vector<order_field> order_by;
 bool order_reverse;
 bool order_by_size = false;
+int output_format = OUTPUT_MVT;
+bool mlt_sort_features = true;
+bool mlt_pretessellate = false;
 
 int prevent[256];
 int additional[256];
@@ -2824,7 +2827,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 		ai->second.maxzoom = maxzoom;
 	}
 
-	metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, minlat2, minlon2, maxlat2, maxlon2, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies, basezoom, droprate, retain_points_multiplier);
+	const char *tile_format = (output_format == OUTPUT_MLT) ? "mlt" : "pbf";
+	metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, minlat2, minlon2, maxlat2, maxlon2, midlat, midlon, attribution, merged_lm, tile_format, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies, basezoom, droprate, retain_points_multiplier);
 	if (outdb != NULL) {
 		mbtiles_write_metadata(outdb, m, forcetable);
 	} else {
@@ -3160,6 +3164,9 @@ int main(int argc, char **argv) {
 		{"no-feature-limit", no_argument, &prevent[P_FEATURE_LIMIT], 1},
 		{"no-tile-size-limit", no_argument, &prevent[P_KILOBYTE_LIMIT], 1},
 		{"no-tile-compression", no_argument, &prevent[P_TILE_COMPRESSION], 1},
+		{"output-format", required_argument, 0, '~'},
+		{"pretessellate", no_argument, 0, '~'},
+		{"no-mlt-feature-sort", no_argument, 0, '~'},
 		{"no-tile-stats", no_argument, &prevent[P_TILE_STATS], 1},
 		{"tile-stats-attributes-limit", required_argument, 0, '~'},
 		{"tile-stats-sample-values-limit", required_argument, 0, '~'},
@@ -3327,6 +3334,19 @@ int main(int argc, char **argv) {
 				unidecode_data = read_unidecode(optarg);
 			} else if (strcmp(opt, "maximum-string-attribute-length") == 0) {
 				maximum_string_attribute_length = atoll_require(optarg, "Maximum string attribute length");
+			} else if (strcmp(opt, "output-format") == 0) {
+				if (strcmp(optarg, "mvt") == 0 || strcmp(optarg, "pbf") == 0) {
+					output_format = OUTPUT_MVT;
+				} else if (strcmp(optarg, "mlt") == 0) {
+					output_format = OUTPUT_MLT;
+				} else {
+					fprintf(stderr, "%s: --output-format must be 'mvt' or 'mlt'\n", argv[0]);
+					exit(EXIT_ARGS);
+				}
+			} else if (strcmp(opt, "pretessellate") == 0) {
+				mlt_pretessellate = true;
+			} else if (strcmp(opt, "no-mlt-feature-sort") == 0) {
+				mlt_sort_features = false;
 			} else {
 				fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
 				exit(EXIT_ARGS);

main.hpp

@@ -76,4 +76,11 @@ bool progress_time();
 
 #define MAX_ZOOM 24
 
+#define OUTPUT_MVT 0
+#define OUTPUT_MLT 1
+
+extern int output_format;
+extern bool mlt_sort_features;
+extern bool mlt_pretessellate;
+
 #endif

mbtiles.cpp

@@ -665,14 +665,14 @@ std::string version_str() {
 	return s;
 }
 
-metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier) {
+metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, const char *tile_format, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier) {
 	metadata m;
 
 	m.name = fname;
 	m.description = description != NULL ? description : fname;
 	m.version = 2;
 	m.type = "overlay";
-	m.format = vector ? "pbf" : "png";
+	m.format = tile_format;
 
 	m.minzoom = minzoom;
 	m.maxzoom = maxzoom;
@@ -711,7 +711,8 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
 				   std::string("}");
 	}
 
-	if (vector) {
+	bool is_vector_format = (strcmp(tile_format, "pbf") == 0 || strcmp(tile_format, "mlt") == 0);
+	if (is_vector_format) {
 		{
 			json_writer state(&m.vector_layers_json);
 

mbtiles.hpp

@@ -68,7 +68,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 void mbtiles_erase_zoom(sqlite3 *outdb, int z);
 
-metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier);
+metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, const char *tile_format, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier);
 void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable);
 
 void mbtiles_close(sqlite3 *outdb, const char *pgm);