Skip to content

Refactor various filesystem related operations to use std::filesystem #309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nightlark wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Refactor various filesystem related operations to use std::filesystem #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ed7c87d
Use C++17 std::filesystem to create directories and check file stats
nightlark Dec 4, 2024
6fd7974
Use std::filesystem to check free disk space available
nightlark Dec 4, 2024
992dc52
Remove platform-specific headers that were previously used to check f…
nightlark Dec 4, 2024
5b6b3d8
Use std::filesystem::remove instead of unlink and std::string for fil…
nightlark Dec 6, 2024
314598b
Replace remaining call to stat with std::filesystem
nightlark Dec 6, 2024
f4621d4
Remove sys/stat.h from files that no longer need it
nightlark Dec 6, 2024
bce61c1
Fix format specifier warning for diskfree type
nightlark Dec 6, 2024
8625110
Remove shadow re-declaration of ec variable
nightlark Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions decode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <getopt.h>
#include <string>
#include <vector>
#include <filesystem>
#include <map>
#include <set>
#include <zlib.h>
Expand All @@ -25,6 +26,8 @@
#include "pmtiles_file.hpp"
#include "errors.hpp"

namespace fs = std::filesystem;

int minzoom = 0;
int maxzoom = 32;
bool force = false;
Expand Down Expand Up @@ -278,21 +281,19 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
perror(fname);
}

struct stat st;
std::vector<zxy> tiles;

char *pmtiles_map = NULL;
std::vector<pmtiles::entry_zxy> entries;
bool is_pmtiles = false;

if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) {
if (fs::is_directory(fname)) {
isdir = true;

db = dirmeta2tmp(fname);
tiles = enumerate_dirtiles(fname, minzoom, maxzoom);
} else if (pmtiles_has_suffix(fname)) {
auto file_size = fs::file_size(fname);
int pmtiles_fd = open(fname, O_RDONLY | O_CLOEXEC);
pmtiles_map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0);
pmtiles_map = (char *) mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0);
if (pmtiles_map == MAP_FAILED) {
perror("mmap in decode");
exit(EXIT_MEMORY);
Expand Down
73 changes: 42 additions & 31 deletions dirtiles.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <iostream>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <string>
Expand All @@ -8,14 +9,15 @@
#include <unistd.h>
#include <dirent.h>
#include <limits.h>
#include <sys/stat.h>
#include <sqlite3.h>
#include "jsonpull/jsonpull.h"
#include "mbtiles.hpp"
#include "dirtiles.hpp"
#include "errors.hpp"
#include "write_json.hpp"

namespace fs = std::filesystem;

std::string dir_read_tile(std::string base, struct zxy tile) {
std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary);
std::ostringstream contents;
Expand All @@ -26,40 +28,43 @@ std::string dir_read_tile(std::string base, struct zxy tile) {
}

void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) {
// Don't check mkdir error returns, since most of these calls to
// mkdir will be creating directories that already exist.
mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO);

std::string curdir(outdir);
std::string slash("/");

std::string newdir = curdir + slash + std::to_string(z);
mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);

newdir = newdir + "/" + std::to_string(tx);
mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
fs::path dirPath(outdir);
fs::path zPath = dirPath / std::to_string(z);
fs::path xPath = zPath / std::to_string(tx);
fs::path tilePath = xPath / (std::to_string(ty) + ".pbf");

// create_directories() does not treat a directory already existing
// as an error, which is fine since most directories will already exist.
std::error_code ec;
fs::create_directories(xPath, ec);
if (ec) {
fprintf(stderr, "Failed to create directories: %s\n", ec.message().c_str());
exit(EXIT_WRITE);
}

newdir = newdir + "/" + std::to_string(ty) + ".pbf";
// Set permissions equivalent to S_IRWXU | S_IRWXG | S_IRWXO (0777)
fs::permissions(dirPath, fs::perms::all, fs::perm_options::add, ec);
fs::permissions(zPath, fs::perms::all, fs::perm_options::add, ec);
fs::permissions(xPath, fs::perms::all, fs::perm_options::add, ec);

struct stat st;
if (stat(newdir.c_str(), &st) == 0) {
fprintf(stderr, "Can't write tile to already existing %s\n", newdir.c_str());
if (fs::exists(tilePath)) {
fprintf(stderr, "Can't write tile to already existing %s\n", tilePath.c_str());
exit(EXIT_EXISTS);
}

FILE *fp = fopen(newdir.c_str(), "wb");
FILE *fp = fopen(tilePath.c_str(), "wb");
if (fp == NULL) {
fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno));
fprintf(stderr, "%s: %s\n", tilePath.c_str(), strerror(errno));
exit(EXIT_WRITE);
}

if (fwrite(pbf.c_str(), sizeof(char), pbf.size(), fp) != pbf.size()) {
fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno));
fprintf(stderr, "%s: %s\n", tilePath.c_str(), strerror(errno));
exit(EXIT_WRITE);
}

if (fclose(fp) != 0) {
fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno));
fprintf(stderr, "%s: %s\n", tilePath.c_str(), strerror(errno));
exit(EXIT_CLOSE);
}
}
Expand All @@ -85,14 +90,18 @@ static bool pbfname(const char *s) {
}

void check_dir(const char *dir, char **argv, bool force, bool forcetable) {
struct stat st;

mkdir(dir, S_IRWXU | S_IRWXG | S_IRWXO);
std::string meta = std::string(dir) + "/" + "metadata.json";
std::error_code ec;
fs::create_directories(dir, ec);
if (ec) {
fprintf(stderr, "Failed to create directory: %s\n", ec.message().c_str());
exit(EXIT_WRITE);
}
fs::permissions(dir, fs::perms::all, fs::perm_options::add, ec);
std::string meta = fs::path(dir) / "metadata.json";
if (force) {
unlink(meta.c_str()); // error OK since it may not exist;
fs::remove(meta, ec); // error OK since it may not exist
} else {
if (stat(meta.c_str(), &st) == 0) {
if (fs::exists(meta)) {
fprintf(stderr, "%s: Tileset \"%s\" already exists. You can use --force if you want to delete the old tileset.\n", argv[0], dir);
fprintf(stderr, "%s: %s: file exists\n", argv[0], meta.c_str());
if (!forcetable) {
Expand All @@ -112,7 +121,8 @@ void check_dir(const char *dir, char **argv, bool force, bool forcetable) {
std::string fn = std::string(dir) + "/" + tiles[i].path();

if (force) {
if (unlink(fn.c_str()) != 0) {
fs::remove(fn, ec);
if (ec) {
perror(fn.c_str());
exit(EXIT_UNLINK);
}
Expand Down Expand Up @@ -209,7 +219,9 @@ void dir_erase_zoom(const char *fname, int zoom) {
while ((dp3 = readdir(d3)) != NULL) {
if (pbfname(dp3->d_name)) {
std::string y = x + "/" + dp3->d_name;
if (unlink(y.c_str()) != 0) {
std::error_code ec;
fs::remove(y, ec);
if (ec) {
perror(y.c_str());
exit(EXIT_UNLINK);
}
Expand Down Expand Up @@ -288,8 +300,7 @@ static void out(json_writer &state, std::string k, std::string v) {
void dir_write_metadata(const char *outdir, const metadata &m) {
std::string metadata = std::string(outdir) + "/metadata.json";

struct stat st;
if (stat(metadata.c_str(), &st) == 0) {
if (fs::exists(metadata)) {
// Leave existing metadata in place with --allow-existing
} else {
FILE *fp = fopen(metadata.c_str(), "w");
Expand Down
1 change: 0 additions & 1 deletion geojson.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
#include <math.h>
#include <string.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <fcntl.h>
Expand Down
Loading