From ed7c87dc35fd2a10e1ceff59421bad256e83aed5 Mon Sep 17 00:00:00 2001 From: Mast Date: Wed, 4 Dec 2024 11:40:30 -0800 Subject: [PATCH 1/8] Use C++17 std::filesystem to create directories and check file stats --- dirtiles.cpp | 63 +++++++++++++++++++++++++++--------------------- main.cpp | 6 ++++- pmtiles_file.cpp | 6 +++-- tile-join.cpp | 8 +++--- 4 files changed, 50 insertions(+), 33 deletions(-) diff --git a/dirtiles.cpp b/dirtiles.cpp index 98138bd5a..a602d6a95 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -16,6 +17,8 @@ #include "errors.hpp" #include "write_json.hpp" +namespace fs = std::filesystem; + std::string dir_read_tile(std::string base, struct zxy tile) { std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary); std::ostringstream contents; @@ -26,40 +29,43 @@ std::string dir_read_tile(std::string base, struct zxy tile) { } void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) { - // Don't check mkdir error returns, since most of these calls to - // mkdir will be creating directories that already exist. - mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); - - std::string curdir(outdir); - std::string slash("/"); - - std::string newdir = curdir + slash + std::to_string(z); - mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); - - newdir = newdir + "/" + std::to_string(tx); - mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + fs::path dirPath(outdir); + fs::path zPath = dirPath / std::to_string(z); + fs::path xPath = zPath / std::to_string(tx); + fs::path tilePath = xPath / (std::to_string(ty) + ".pbf"); + + // create_directories() does not treat a directory already existing + // as an error, which is fine since most directories will already exist. + std::error_code ec; + fs::create_directories(xPath, ec); + if (ec) { + fprintf(stderr, "Failed to create directories: %s\n", ec.message().c_str()); + exit(EXIT_WRITE); + } - newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + // Set permissions equivalent to S_IRWXU | S_IRWXG | S_IRWXO (0777) + fs::permissions(dirPath, fs::perms::all, fs::perm_options::add, ec); + fs::permissions(zPath, fs::perms::all, fs::perm_options::add, ec); + fs::permissions(xPath, fs::perms::all, fs::perm_options::add, ec); - struct stat st; - if (stat(newdir.c_str(), &st) == 0) { - fprintf(stderr, "Can't write tile to already existing %s\n", newdir.c_str()); + if (fs::exists(tilePath)) { + fprintf(stderr, "Can't write tile to already existing %s\n", tilePath.c_str()); exit(EXIT_EXISTS); } - FILE *fp = fopen(newdir.c_str(), "wb"); + FILE *fp = fopen(tilePath.c_str(), "wb"); if (fp == NULL) { - fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno)); + fprintf(stderr, "%s: %s\n", tilePath.c_str(), strerror(errno)); exit(EXIT_WRITE); } if (fwrite(pbf.c_str(), sizeof(char), pbf.size(), fp) != pbf.size()) { - fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno)); + fprintf(stderr, "%s: %s\n", tilePath.c_str(), strerror(errno)); exit(EXIT_WRITE); } if (fclose(fp) != 0) { - fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno)); + fprintf(stderr, "%s: %s\n", tilePath.c_str(), strerror(errno)); exit(EXIT_CLOSE); } } @@ -85,14 +91,18 @@ static bool pbfname(const char *s) { } void check_dir(const char *dir, char **argv, bool force, bool forcetable) { - struct stat st; - - mkdir(dir, S_IRWXU | S_IRWXG | S_IRWXO); - std::string meta = std::string(dir) + "/" + "metadata.json"; + std::error_code ec; + fs::create_directories(dir, ec); + if (ec) { + fprintf(stderr, "Failed to create directory: %s\n", ec.message().c_str()); + exit(EXIT_WRITE); + } + fs::permissions(dir, fs::perms::all, fs::perm_options::add, ec); + std::string meta = fs::path(dir) / "metadata.json"; if (force) { unlink(meta.c_str()); // error OK since it may not exist; } else { - if (stat(meta.c_str(), &st) == 0) { + if (fs::exists(meta)) { fprintf(stderr, "%s: Tileset \"%s\" already exists. You can use --force if you want to delete the old tileset.\n", argv[0], dir); fprintf(stderr, "%s: %s: file exists\n", argv[0], meta.c_str()); if (!forcetable) { @@ -288,8 +298,7 @@ static void out(json_writer &state, std::string k, std::string v) { void dir_write_metadata(const char *outdir, const metadata &m) { std::string metadata = std::string(outdir) + "/metadata.json"; - struct stat st; - if (stat(metadata.c_str(), &st) == 0) { + if (fs::exists(metadata)) { // Leave existing metadata in place with --allow-existing } else { FILE *fp = fopen(metadata.c_str(), "w"); diff --git a/main.cpp b/main.cpp index aa6c49f27..68b744996 100644 --- a/main.cpp +++ b/main.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -68,6 +69,8 @@ #include "thread.hpp" #include "platform.hpp" +namespace fs = std::filesystem; + static int low_detail = 12; static int full_detail = -1; static int min_detail = 7; @@ -2991,7 +2994,8 @@ int main(int argc, char **argv) { double droprate = 2.5; double gamma = 0; int buffer = 5; - const char *tmpdir = "/tmp"; + std::string tmpdir_path = fs::temp_directory_path().string(); + const char *tmpdir = tmpdir_path.c_str(); const char *attribution = NULL; std::vector sources; const char *prefilter = NULL; diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index e86e538b5..3763b2924 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -12,6 +13,8 @@ #include "write_json.hpp" #include "main.hpp" +namespace fs = std::filesystem; + bool pmtiles_has_suffix(const char *filename) { if (filename == nullptr) { return false; @@ -27,8 +30,7 @@ bool pmtiles_has_suffix(const char *filename) { } void check_pmtiles(const char *filename, char **argv, bool forcetable) { - struct stat st; - if (stat(filename, &st) == 0) { + if (fs::exists(filename)) { fprintf(stderr, "%s: Tileset \"%s\" already exists. You can use --force if you want to delete the old tileset.\n", argv[0], filename); fprintf(stderr, "%s: %s: file exists\n", argv[0], filename); diff --git a/tile-join.cpp b/tile-join.cpp index d9484a281..e74794ed0 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include "jsonpull/jsonpull.h" #include "milo/dtoa_milo.h" #include "errors.hpp" @@ -45,6 +46,8 @@ #include "thread.hpp" #include "platform.hpp" +namespace fs = std::filesystem; + int pk = false; int pC = false; int pg = false; @@ -643,8 +646,7 @@ struct tileset_reader { tileset_reader(const char *fname) { name = fname; - struct stat st; - if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { + if (fs::is_directory(fname)) { db = NULL; stmt = NULL; next = NULL; @@ -653,7 +655,7 @@ struct tileset_reader { dirbase = fname; } else if (pmtiles_has_suffix(fname)) { int pmtiles_fd = open(fname, O_RDONLY | O_CLOEXEC); - pmtiles_map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0); + pmtiles_map = (char *) mmap(NULL, fs::file_size(fname), PROT_READ, MAP_PRIVATE, pmtiles_fd, 0); if (pmtiles_map == MAP_FAILED) { perror("mmap in decode"); From 6fd79746ec9ce97d6f3fd0f229961976c35927bf Mon Sep 17 00:00:00 2001 From: Mast Date: Wed, 4 Dec 2024 12:15:28 -0800 Subject: [PATCH 2/8] Use std::filesystem to check free disk space available --- main.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main.cpp b/main.cpp index 68b744996..4289446f6 100644 --- a/main.cpp +++ b/main.cpp @@ -121,7 +121,7 @@ size_t CPUS; size_t TEMP_FILES; long long MAX_FILES; size_t memsize; -static long long diskfree; +static uintmax_t diskfree; char **av; std::vector clipbboxes; @@ -1323,13 +1323,13 @@ std::pair read_input(std::vector &sources, char *fname, i r->file_bbox[2] = r->file_bbox[3] = 0; } - struct statfs fsstat; - if (fstatfs(readers[0].geomfd, &fsstat) != 0) { - perror("Warning: fstatfs"); + std::error_code ec; + fs::space_info si = fs::space(fs::path(tmpdir), ec); + if (ec) { fprintf(stderr, "Tippecanoe cannot check whether disk space will run out during tiling.\n"); - diskfree = LLONG_MAX; + diskfree = UINTMAX_MAX; } else { - diskfree = (long long) fsstat.f_bsize * fsstat.f_bavail; + diskfree = si.available; } std::atomic progress_seq(0); From 992dc522003d064316381458d516114ffa174363 Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Wed, 4 Dec 2024 13:07:37 -0800 Subject: [PATCH 3/8] Remove platform-specific headers that were previously used to check free disk space --- main.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/main.cpp b/main.cpp index 4289446f6..0be47f1bd 100644 --- a/main.cpp +++ b/main.cpp @@ -38,9 +38,6 @@ #include #include #include -#include -#else -#include #endif #include "jsonpull/jsonpull.h" From 5b6b3d80bb7ae3a938e7180abdbb8160109fefc3 Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Thu, 5 Dec 2024 18:14:34 -0800 Subject: [PATCH 4/8] Use std::filesystem::remove instead of unlink and std::string for file names instead of VLAs --- dirtiles.cpp | 10 ++- main.cpp | 189 +++++++++++++++++++++++------------------------ pmtiles_file.cpp | 2 +- sort.cpp | 11 ++- tile-join.cpp | 2 +- tile.cpp | 19 +++-- unit.cpp | 11 ++- 7 files changed, 123 insertions(+), 121 deletions(-) diff --git a/dirtiles.cpp b/dirtiles.cpp index a602d6a95..385f13dac 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -100,7 +100,7 @@ void check_dir(const char *dir, char **argv, bool force, bool forcetable) { fs::permissions(dir, fs::perms::all, fs::perm_options::add, ec); std::string meta = fs::path(dir) / "metadata.json"; if (force) { - unlink(meta.c_str()); // error OK since it may not exist; + fs::remove(meta, ec); // error OK since it may not exist } else { if (fs::exists(meta)) { fprintf(stderr, "%s: Tileset \"%s\" already exists. You can use --force if you want to delete the old tileset.\n", argv[0], dir); @@ -122,7 +122,9 @@ void check_dir(const char *dir, char **argv, bool force, bool forcetable) { std::string fn = std::string(dir) + "/" + tiles[i].path(); if (force) { - if (unlink(fn.c_str()) != 0) { + std::error_code ec; + fs::remove(fn, ec); + if (ec) { perror(fn.c_str()); exit(EXIT_UNLINK); } @@ -219,7 +221,9 @@ void dir_erase_zoom(const char *fname, int zoom) { while ((dp3 = readdir(d3)) != NULL) { if (pbfname(dp3->d_name)) { std::string y = x + "/" + dp3->d_name; - if (unlink(y.c_str()) != 0) { + std::error_code ec; + fs::remove(y, ec); + if (ec) { perror(y.c_str()); exit(EXIT_UNLINK); } diff --git a/main.cpp b/main.cpp index 0be47f1bd..90f830db6 100644 --- a/main.cpp +++ b/main.cpp @@ -755,37 +755,35 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split for (i = 0; i < splits; i++) { sub_geompos[i] = 0; - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; - snprintf(geomname, sizeof(geomname), "%s%s", tmpdir, "/geom.XXXXXXXX"); - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - snprintf(indexname, sizeof(indexname), "%s%s", tmpdir, "/index.XXXXXXXX"); + std::string geomname = std::string(tmpdir) + "/geom.XXXXXXXX"; + std::string indexname = std::string(tmpdir) + "/index.XXXXXXXX"; - geomfds[i] = mkstemp_cloexec(geomname); + geomfds[i] = mkstemp_cloexec(geomname.data()); if (geomfds[i] < 0) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } - indexfds[i] = mkstemp_cloexec(indexname); + indexfds[i] = mkstemp_cloexec(indexname.data()); if (indexfds[i] < 0) { - perror(indexname); + perror(indexname.c_str()); exit(EXIT_OPEN); } - geomfiles[i] = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC); + geomfiles[i] = fopen_oflag(geomname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (geomfiles[i] == NULL) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } - indexfiles[i] = fopen_oflag(indexname, "wb", O_WRONLY | O_CLOEXEC); + indexfiles[i] = fopen_oflag(indexname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (indexfiles[i] == NULL) { - perror(indexname); + perror(indexname.c_str()); exit(EXIT_OPEN); } *availfiles -= 4; - unlink(geomname); - unlink(indexname); + fs::remove(geomname); + fs::remove(indexname); } for (i = 0; i < inputs; i++) { @@ -1222,79 +1220,74 @@ std::pair read_input(std::vector &sources, char *fname, i for (size_t i = 0; i < CPUS; i++) { struct reader *r = &readers[i]; - char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; - char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1]; - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - char vertexname[strlen(tmpdir) + strlen("/vertex.XXXXXXXX") + 1]; - char nodename[strlen(tmpdir) + strlen("/node.XXXXXXXX") + 1]; - - snprintf(poolname, sizeof(poolname), "%s%s", tmpdir, "/pool.XXXXXXXX"); - snprintf(treename, sizeof(treename), "%s%s", tmpdir, "/tree.XXXXXXXX"); - snprintf(geomname, sizeof(geomname), "%s%s", tmpdir, "/geom.XXXXXXXX"); - snprintf(indexname, sizeof(indexname), "%s%s", tmpdir, "/index.XXXXXXXX"); - snprintf(vertexname, sizeof(vertexname), "%s%s", tmpdir, "/vertex.XXXXXXXX"); - snprintf(nodename, sizeof(nodename), "%s%s", tmpdir, "/node.XXXXXXXX"); - - r->poolfd = mkstemp_cloexec(poolname); + std::string poolname = std::string(tmpdir) + "/pool.XXXXXXXX"; + std::string treename = std::string(tmpdir) + "/tree.XXXXXXXX"; + std::string geomname = std::string(tmpdir) + "/geom.XXXXXXXX"; + std::string indexname = std::string(tmpdir) + "/index.XXXXXXXX"; + std::string vertexname = std::string(tmpdir) + "/vertex.XXXXXXXX"; + std::string nodename = std::string(tmpdir) + "/node.XXXXXXXX"; + + // Since C++11, the returned array from data() is null-terminated + r->poolfd = mkstemp_cloexec(poolname.data()); if (r->poolfd < 0) { - perror(poolname); + perror(poolname.c_str()); exit(EXIT_OPEN); } - r->treefd = mkstemp_cloexec(treename); + r->treefd = mkstemp_cloexec(treename.data()); if (r->treefd < 0) { - perror(treename); + perror(treename.c_str()); exit(EXIT_OPEN); } - r->geomfd = mkstemp_cloexec(geomname); + r->geomfd = mkstemp_cloexec(geomname.data()); if (r->geomfd < 0) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } - r->indexfd = mkstemp_cloexec(indexname); + r->indexfd = mkstemp_cloexec(indexname.data()); if (r->indexfd < 0) { - perror(indexname); + perror(indexname.c_str()); exit(EXIT_OPEN); } - r->vertexfd = mkstemp_cloexec(vertexname); + r->vertexfd = mkstemp_cloexec(vertexname.data()); if (r->vertexfd < 0) { - perror(vertexname); + perror(vertexname.c_str()); exit(EXIT_OPEN); } - r->nodefd = mkstemp_cloexec(nodename); + r->nodefd = mkstemp_cloexec(nodename.data()); if (r->nodefd < 0) { - perror(nodename); + perror(nodename.c_str()); exit(EXIT_OPEN); } r->poolfile = memfile_open(r->poolfd); if (r->poolfile == NULL) { - perror(poolname); + perror(poolname.c_str()); exit(EXIT_OPEN); } r->treefile = memfile_open(r->treefd); if (r->treefile == NULL) { - perror(treename); + perror(treename.c_str()); exit(EXIT_OPEN); } - r->geomfile = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC); + r->geomfile = fopen_oflag(geomname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (r->geomfile == NULL) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } - r->indexfile = fopen_oflag(indexname, "wb", O_WRONLY | O_CLOEXEC); + r->indexfile = fopen_oflag(indexname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (r->indexfile == NULL) { - perror(indexname); + perror(indexname.c_str()); exit(EXIT_OPEN); } - r->vertexfile = fopen_oflag(vertexname, "w+b", O_RDWR | O_CLOEXEC); + r->vertexfile = fopen_oflag(vertexname.c_str(), "w+b", O_RDWR | O_CLOEXEC); if (r->vertexfile == NULL) { - perror(("open vertexfile " + std::string(vertexname)).c_str()); + std::string err = "open vertexfile " + vertexname; + perror(err.c_str()); exit(EXIT_OPEN); } - r->nodefile = fopen_oflag(nodename, "w+b", O_RDWR | O_CLOEXEC); + r->nodefile = fopen_oflag(nodename.c_str(), "w+b", O_RDWR | O_CLOEXEC); if (r->nodefile == NULL) { - perror(nodename); + perror(nodename.c_str()); exit(EXIT_OPEN); } r->geompos = 0; @@ -1302,12 +1295,13 @@ std::pair read_input(std::vector &sources, char *fname, i r->vertexpos = 0; r->nodepos = 0; - unlink(poolname); - unlink(treename); - unlink(geomname); - unlink(indexname); - unlink(vertexname); - unlink(nodename); + fs::remove(poolname); + fs::remove(treename); + fs::remove(geomname); + fs::remove(indexname); + fs::remove(vertexname); + fs::remove(nodename); + // To distinguish a null value { @@ -1717,19 +1711,18 @@ std::pair read_input(std::vector &sources, char *fname, i if (read_parallel_this) { // Serial reading of chunks that are then parsed in parallel - char readname[strlen(tmpdir) + strlen("/read.XXXXXXXX") + 1]; - snprintf(readname, sizeof(readname), "%s%s", tmpdir, "/read.XXXXXXXX"); - int readfd = mkstemp_cloexec(readname); + std::string readname = std::string(tmpdir) + "/read.XXXXXXXX"; + int readfd = mkstemp_cloexec(readname.data()); if (readfd < 0) { - perror(readname); + perror(readname.c_str()); exit(EXIT_OPEN); } FILE *readfp = fdopen(readfd, "w"); if (readfp == NULL) { - perror(readname); + perror(readname.c_str()); exit(EXIT_OPEN); } - unlink(readname); + fs::remove(readname); std::atomic is_parsing(0); long long ahead = 0; @@ -1771,18 +1764,18 @@ std::pair read_input(std::vector &sources, char *fname, i checkdisk(&readers); ahead = 0; - snprintf(readname, sizeof(readname), "%s%s", tmpdir, "/read.XXXXXXXX"); - readfd = mkstemp_cloexec(readname); + readname = std::string(tmpdir) + "/read.XXXXXXXX"; + readfd = mkstemp_cloexec(readname.data()); if (readfd < 0) { - perror(readname); + perror(readname.c_str()); exit(EXIT_OPEN); } readfp = fdopen(readfd, "w"); if (readfp == NULL) { - perror(readname); + perror(readname.c_str()); exit(EXIT_OPEN); } - unlink(readname); + fs::remove(readname); } } } @@ -1915,22 +1908,21 @@ std::pair read_input(std::vector &sources, char *fname, i pool_off[i] = 0; } - char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; - snprintf(poolname, sizeof(poolname), "%s%s", tmpdir, "/pool.XXXXXXXX"); + std::string poolname = std::string(tmpdir) + "/pool.XXXXXXXX"; - int poolfd = mkstemp_cloexec(poolname); + int poolfd = mkstemp_cloexec(poolname.data()); if (poolfd < 0) { - perror(poolname); + perror(poolname.c_str()); exit(EXIT_OPEN); } - FILE *poolfile = fopen_oflag(poolname, "wb", O_WRONLY | O_CLOEXEC); + FILE *poolfile = fopen_oflag(poolname.data(), "wb", O_WRONLY | O_CLOEXEC); if (poolfile == NULL) { - perror(poolname); + perror(poolname.c_str()); exit(EXIT_OPEN); } - unlink(poolname); + fs::remove(poolname); std::atomic poolpos(0); for (size_t i = 0; i < CPUS; i++) { @@ -2002,12 +1994,13 @@ std::pair read_input(std::vector &sources, char *fname, i // find nodes where the same central point is part of two different vertices { std::string tmpname = std::string(tmpdir) + "/vertex2.XXXXXX"; - int vertexfd = mkstemp((char *) tmpname.c_str()); + int vertexfd = mkstemp(tmpname.data()); if (vertexfd < 0) { - perror(("mkstemp vertexfile " + std::string(tmpname)).c_str()); + std::string err = "mkstemp vertexfile " + tmpname; + perror(err.c_str()); exit(EXIT_OPEN); } - unlink(tmpname.c_str()); + fs::remove(tmpname); FILE *vertex_out = fdopen(vertexfd, "w+b"); if (vertex_out == NULL) { perror(tmpname.c_str()); @@ -2069,12 +2062,13 @@ std::pair read_input(std::vector &sources, char *fname, i // sort std::string tmpname = std::string(tmpdir) + "/node2.XXXXXX"; - int nodefd = mkstemp((char *) tmpname.c_str()); + int nodefd = mkstemp(tmpname.data()); if (nodefd < 0) { - perror(("mkstemp nodefile " + std::string(tmpname)).c_str()); + std::string err = "mkstemp nodefile " + tmpname; + perror(tmpname.c_str()); exit(EXIT_OPEN); } - unlink(tmpname.c_str()); + fs::remove(tmpname); FILE *node_out; node_out = fdopen(nodefd, "w+b"); if (node_out == NULL) { @@ -2102,12 +2096,13 @@ std::pair read_input(std::vector &sources, char *fname, i // scan tmpname = std::string(tmpdir) + "/node3.XXXXXX"; - nodefd = mkstemp((char *) tmpname.c_str()); + nodefd = mkstemp(tmpname.data()); if (nodefd < 0) { - perror(("mkstemp nodefile " + std::string(tmpname)).c_str()); + std::string err = "mkstemp nodefile " + tmpname; + perror(tmpname.c_str()); exit(EXIT_OPEN); } - unlink(tmpname.c_str()); + fs::remove(tmpname); shared_nodes = fdopen(nodefd, "w+b"); if (shared_nodes == NULL) { perror(tmpname.c_str()); @@ -2158,36 +2153,34 @@ std::pair read_input(std::vector &sources, char *fname, i fprintf(stderr, "Merging index \r"); } - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - snprintf(indexname, sizeof(indexname), "%s%s", tmpdir, "/index.XXXXXXXX"); + std::string indexname = std::string(tmpdir) + "/index.XXXXXXXX"; - int indexfd = mkstemp_cloexec(indexname); + int indexfd = mkstemp_cloexec(indexname.data()); if (indexfd < 0) { - perror(indexname); + perror(indexname.c_str()); exit(EXIT_OPEN); } - FILE *indexfile = fopen_oflag(indexname, "wb", O_WRONLY | O_CLOEXEC); + FILE *indexfile = fopen_oflag(indexname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (indexfile == NULL) { - perror(indexname); + perror(indexname.c_str()); exit(EXIT_OPEN); } - unlink(indexname); + fs::remove(indexname); - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; - snprintf(geomname, sizeof(geomname), "%s%s", tmpdir, "/geom.XXXXXXXX"); + std::string geomname = std::string(tmpdir) + "/geom.XXXXXXXX"; - int geomfd = mkstemp_cloexec(geomname); + int geomfd = mkstemp_cloexec(geomname.data()); if (geomfd < 0) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_CLOSE); } - FILE *geomfile = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC); + FILE *geomfile = fopen_oflag(geomname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (geomfile == NULL) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } - unlink(geomname); + fs::remove(geomname); unsigned iz = 0, ix = 0, iy = 0; choose_first_zoom(file_bbox, file_bbox1, file_bbox2, readers, &iz, &ix, &iy, minzoom, buffer); @@ -3790,7 +3783,7 @@ int main(int argc, char **argv) { if (out_mbtiles != NULL) { if (force) { - unlink(out_mbtiles); + fs::remove(out_mbtiles); } else { if (pmtiles_has_suffix(out_mbtiles)) { check_pmtiles(out_mbtiles, argv, forcetable); diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 3763b2924..a9d2dec3b 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -332,7 +332,7 @@ void mbtiles_map_image_to_pmtiles(char *fname, metadata m, bool tile_compression ostream << tmp_istream.rdbuf(); tmp_istream.close(); - unlink(tmpname.c_str()); + fs::remove(tmpname); ostream.close(); } } diff --git a/sort.cpp b/sort.cpp index c5350c8be..aee5326cb 100644 --- a/sort.cpp +++ b/sort.cpp @@ -1,9 +1,12 @@ #include #include #include +#include #include #include +namespace fs = std::filesystem; + #define MAX_MEMORY (1024 * 1024 * 1024) // 1 GB void fqsort(std::vector &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem) { @@ -70,10 +73,10 @@ void fqsort(std::vector &inputs, size_t width, int (*cmp)(const void *, std::string t1 = "/tmp/sort1.XXXXXX"; std::string t2 = "/tmp/sort2.XXXXXX"; - int fd1 = mkstemp((char *) t1.c_str()); - unlink(t1.c_str()); - int fd2 = mkstemp((char *) t2.c_str()); - unlink(t2.c_str()); + int fd1 = mkstemp(t1.data()); + fs::remove(t1); + int fd2 = mkstemp(t2.data()); + fs::remove(t2); fp1 = fdopen(fd1, "w+b"); if (fp1 == NULL) { diff --git a/tile-join.cpp b/tile-join.cpp index e74794ed0..e94907ea4 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -1724,7 +1724,7 @@ int main(int argc, char **argv) { if (out_mbtiles != NULL) { if (force) { - unlink(out_mbtiles); + fs::remove(out_mbtiles); } else { if (pmtiles_has_suffix(out_mbtiles)) { check_pmtiles(out_mbtiles, argv, false); diff --git a/tile.cpp b/tile.cpp index 108da5347..cc7e33fb1 100644 --- a/tile.cpp +++ b/tile.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -55,15 +56,14 @@ extern "C" { #include "plugin.hpp" +namespace fs = std::filesystem; + #define CMD_BITS 3 // Offset coordinates to keep them positive #define COORD_OFFSET (4LL << 32) #define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale))) -#define XSTRINGIFY(s) STRINGIFY(s) -#define STRINGIFY(s) #s - pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t task_lock = PTHREAD_MUTEX_INITIALIZER; @@ -3065,23 +3065,22 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: std::atomic subpos[TEMP_FILES]; int subfd[TEMP_FILES]; for (size_t j = 0; j < TEMP_FILES; j++) { - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1]; - snprintf(geomname, sizeof(geomname), "%s/geom%zu.XXXXXXXX", tmpdir, j); - subfd[j] = mkstemp_cloexec(geomname); + std::string geomname = std::string(tmpdir) + "/geom" + std::to_string(j) + ".XXXXXXXX"; + subfd[j] = mkstemp_cloexec(geomname.data()); // printf("%s\n", geomname); if (subfd[j] < 0) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } - FILE *fp = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC); + FILE *fp = fopen_oflag(geomname.c_str(), "wb", O_WRONLY | O_CLOEXEC); if (fp == NULL) { - perror(geomname); + perror(geomname.c_str()); exit(EXIT_OPEN); } compressors[j] = compressor(fp); sub[j] = &compressors[j]; subpos[j] = 0; - unlink(geomname); + fs::remove(geomname); } size_t useful_threads = 0; diff --git a/unit.cpp b/unit.cpp index c8f364914..3c9becc31 100644 --- a/unit.cpp +++ b/unit.cpp @@ -8,6 +8,9 @@ #include "geometry.hpp" #include #include +#include + +namespace fs = std::filesystem; TEST_CASE("UTF-8 enforcement", "[utf8]") { REQUIRE(check_utf8("") == std::string("")); @@ -55,8 +58,8 @@ TEST_CASE("External quicksort", "fqsort") { size_t written = 0; for (size_t i = 0; i < 5; i++) { std::string tmpname = "/tmp/in.XXXXXXX"; - int fd = mkstemp((char *) tmpname.c_str()); - unlink(tmpname.c_str()); + int fd = mkstemp(tmpname.data()); + fs::remove(tmpname); FILE *f = fdopen(fd, "w+b"); inputs.emplace_back(f); size_t iterations = 2000 + rand() % 200; @@ -69,8 +72,8 @@ TEST_CASE("External quicksort", "fqsort") { } std::string tmpname = "/tmp/out.XXXXXX"; - int fd = mkstemp((char *) tmpname.c_str()); - unlink(tmpname.c_str()); + int fd = mkstemp(tmpname.data()); + fs::remove(tmpname); FILE *f = fdopen(fd, "w+b"); fqsort(inputs, sizeof(int), intcmp, f, 256); From 314598b5563c07b9846036bb4b3eb7161f78d7d4 Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Thu, 5 Dec 2024 18:36:25 -0800 Subject: [PATCH 5/8] Replace remaining call to stat with std::filesystem --- decode.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/decode.cpp b/decode.cpp index 534e6219d..32657fe21 100644 --- a/decode.cpp +++ b/decode.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,8 @@ #include "pmtiles_file.hpp" #include "errors.hpp" +namespace fs = std::filesystem; + int minzoom = 0; int maxzoom = 32; bool force = false; @@ -278,21 +281,19 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co perror(fname); } - struct stat st; std::vector tiles; - char *pmtiles_map = NULL; std::vector entries; bool is_pmtiles = false; - if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { + if (fs::is_directory(fname)) { isdir = true; - db = dirmeta2tmp(fname); tiles = enumerate_dirtiles(fname, minzoom, maxzoom); } else if (pmtiles_has_suffix(fname)) { + auto file_size = fs::file_size(fname); int pmtiles_fd = open(fname, O_RDONLY | O_CLOEXEC); - pmtiles_map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0); + pmtiles_map = (char *) mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0); if (pmtiles_map == MAP_FAILED) { perror("mmap in decode"); exit(EXIT_MEMORY); From f4621d4ea4c7b8a1f20be76a453e39158f0cb97a Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Thu, 5 Dec 2024 18:48:11 -0800 Subject: [PATCH 6/8] Remove sys/stat.h from files that no longer need it --- dirtiles.cpp | 1 - geojson.cpp | 1 - mbtiles.cpp | 1 - pmtiles_file.cpp | 1 - tile-join.cpp | 1 - 5 files changed, 5 deletions(-) diff --git a/dirtiles.cpp b/dirtiles.cpp index 385f13dac..5dcd37db5 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include "jsonpull/jsonpull.h" #include "mbtiles.hpp" diff --git a/geojson.cpp b/geojson.cpp index 3798d14be..7f9be672d 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/mbtiles.cpp b/mbtiles.cpp index c7a615b0a..dbaa6d423 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include "mvt.hpp" #include "mbtiles.hpp" #include "text.hpp" diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index a9d2dec3b..9647282fc 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include "errors.hpp" #include "pmtiles_file.hpp" diff --git a/tile-join.cpp b/tile-join.cpp index e94907ea4..998dd4c89 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include From bce61c11d79831f840d95718807b42b16aa5f4c9 Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Thu, 5 Dec 2024 19:19:20 -0800 Subject: [PATCH 7/8] Fix format specifier warning for diskfree type --- main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.cpp b/main.cpp index 90f830db6..827e2063b 100644 --- a/main.cpp +++ b/main.cpp @@ -134,7 +134,7 @@ void checkdisk(std::vector *r) { static int warned = 0; if (used > diskfree * .9 && !warned) { - fprintf(stderr, "You will probably run out of disk space.\n%lld bytes used or committed, of %lld originally available\n", used, diskfree); + fprintf(stderr, "You will probably run out of disk space.\n%lld bytes used or committed, of %ju originally available\n", used, diskfree); warned = 1; } }; From 8625110a7b565f4a59aabee931122ca0849d927a Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Thu, 5 Dec 2024 19:20:20 -0800 Subject: [PATCH 8/8] Remove shadow re-declaration of ec variable --- dirtiles.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dirtiles.cpp b/dirtiles.cpp index 5dcd37db5..73abb6706 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -121,7 +121,6 @@ void check_dir(const char *dir, char **argv, bool force, bool forcetable) { std::string fn = std::string(dir) + "/" + tiles[i].path(); if (force) { - std::error_code ec; fs::remove(fn, ec); if (ec) { perror(fn.c_str());