Skip to content

Commit c7f3035

Browse files
committed
fix: restore self-hosted URL wiring and add data maintenance workflows
- derive pnpm version in Dockerfiles from root packageManager for sync safety - inject internal OVERPASS_URL/NOMINATIM_URL for app-api when services are co-deployed - add >50GB planet-scale guards for OSRM/OTP builds - add `openmapx data update [region]` full refresh workflow - add `openmapx data clean <target|all>` and offline-capable `data status` - add `openmapx services recreate <ids...>` (pull + force-recreate with render/link prep)
1 parent 844ae5a commit c7f3035

16 files changed

Lines changed: 1172 additions & 117 deletions

File tree

apps/api/Dockerfile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
FROM node:24-alpine AS base
2-
# Keep in sync with the root `packageManager` field so lockfile resolution
3-
# stays reproducible across local and CI Docker builds.
4-
ARG PNPM_VERSION=9.15.9
5-
RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate
2+
# Read the pnpm version directly from the repo root package.json to ensure
3+
# lockfile resolution stays reproducible across local and CI Docker builds.
4+
COPY package.json /tmp/package.json
5+
RUN corepack enable && corepack prepare $(node -e "console.log(require('/tmp/package.json').packageManager)") --activate
66

77
# Install all dependencies (for building)
88
FROM base AS deps

apps/web/Dockerfile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
FROM node:24-alpine AS base
2-
# Keep in sync with the root `packageManager` field so lockfile resolution
3-
# stays reproducible across local and CI Docker builds.
4-
ARG PNPM_VERSION=9.15.9
5-
RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate
2+
# Read the pnpm version directly from the repo root package.json to ensure
3+
# lockfile resolution stays reproducible across local and CI Docker builds.
4+
COPY package.json /tmp/package.json
5+
RUN corepack enable && corepack prepare $(node -e "console.log(require('/tmp/package.json').packageManager)") --activate
66

77
# Install dependencies
88
FROM base AS deps
Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
5+
import {
6+
applyDataCleanup,
7+
collectOfflineDataStatus,
8+
planDataCleanup,
9+
pruneDataManagerStateForCleanup,
10+
summarizePath,
11+
} from "../src/lib/data-local";
12+
13+
let tmp: string;
14+
15+
function writeManifest(slug: string, body: Record<string, unknown>) {
16+
const dir = join(tmp, "services", slug);
17+
mkdirSync(dir, { recursive: true });
18+
writeFileSync(join(dir, "service.json"), JSON.stringify(body), "utf-8");
19+
}
20+
21+
const baseManifest = {
22+
name: "Test",
23+
version: "1.0.0",
24+
quality: "built-in",
25+
container: { image: "t/x", tag: "latest", expose: [80] },
26+
};
27+
28+
beforeEach(() => {
29+
tmp = mkdtempSync(join(tmpdir(), "openmapx-cli-data-local-"));
30+
writeFileSync(join(tmp, "pnpm-workspace.yaml"), "packages: []\n");
31+
mkdirSync(join(tmp, "services"), { recursive: true });
32+
mkdirSync(join(tmp, "infra", "docker", "data"), { recursive: true });
33+
});
34+
35+
afterEach(() => {
36+
rmSync(tmp, { recursive: true, force: true });
37+
});
38+
39+
describe("collectOfflineDataStatus", () => {
40+
it("summarizes OSM + GTFS files and top-level directory usage", () => {
41+
const dataRoot = join(tmp, "infra", "docker", "data");
42+
mkdirSync(join(dataRoot, "osm"), { recursive: true });
43+
mkdirSync(join(dataRoot, "gtfs"), { recursive: true });
44+
mkdirSync(join(dataRoot, "valhalla", "osm-pbf"), { recursive: true });
45+
46+
writeFileSync(join(dataRoot, "osm", "planet.osm.pbf"), "PBF");
47+
writeFileSync(join(dataRoot, "gtfs", "feed-a.zip"), "A");
48+
writeFileSync(join(dataRoot, "gtfs", "feed-b.zip"), "BB");
49+
writeFileSync(join(dataRoot, "valhalla", "osm-pbf", "planet.osm.pbf"), "PBF");
50+
51+
const status = collectOfflineDataStatus(tmp);
52+
53+
expect(status.osmPbfFiles.map((file) => file.name)).toEqual(["planet.osm.pbf"]);
54+
expect(status.gtfsZipFiles.map((file) => file.name)).toEqual(["feed-a.zip", "feed-b.zip"]);
55+
expect(status.directories.map((dir) => dir.name)).toEqual(["gtfs", "osm", "valhalla"]);
56+
expect(status.totalFiles).toBe(4);
57+
});
58+
});
59+
60+
describe("planDataCleanup", () => {
61+
it("maps a concrete data type to producer + consumer paths", async () => {
62+
writeManifest("data-manager", {
63+
...baseManifest,
64+
id: "data-manager",
65+
provides: ["osm-data"],
66+
produces: [{ type: "osm-pbf", sourceDir: "data/osm" }],
67+
});
68+
writeManifest("valhalla", {
69+
...baseManifest,
70+
id: "valhalla",
71+
consumes: [{ type: "osm-pbf", mountAt: "/custom_files", required: true }],
72+
});
73+
74+
const plan = await planDataCleanup("osm-pbf", tmp);
75+
76+
expect(plan.all).toBe(false);
77+
expect(plan.normalizedTypes).toEqual(["osm-pbf"]);
78+
expect(plan.paths).toEqual([
79+
join(tmp, "infra", "docker", "data", "osm"),
80+
join(tmp, "infra", "docker", "data", "valhalla", "osm-pbf"),
81+
]);
82+
});
83+
84+
it("expands style alias into tile-fonts + tile-styles", async () => {
85+
writeManifest("data-manager", {
86+
...baseManifest,
87+
id: "data-manager",
88+
provides: ["tile-asset-data"],
89+
produces: [
90+
{ type: "tile-fonts", sourceDir: "data/tile-fonts" },
91+
{ type: "tile-styles", sourceDir: "data/tile-styles" },
92+
],
93+
});
94+
writeManifest("tileserver", {
95+
...baseManifest,
96+
id: "tileserver",
97+
consumes: [
98+
{ type: "tile-fonts", mountAt: "/data/fonts", required: true },
99+
{ type: "tile-styles", mountAt: "/data/styles", required: true },
100+
],
101+
});
102+
103+
const plan = await planDataCleanup("style", tmp);
104+
105+
expect(plan.normalizedTypes.sort()).toEqual(["tile-fonts", "tile-styles"]);
106+
expect(plan.paths).toEqual([
107+
join(tmp, "infra", "docker", "data", "tile-fonts"),
108+
join(tmp, "infra", "docker", "data", "tile-styles"),
109+
join(tmp, "infra", "docker", "data", "tileserver", "tile-fonts"),
110+
join(tmp, "infra", "docker", "data", "tileserver", "tile-styles"),
111+
]);
112+
});
113+
114+
it("supports all cleanup as the data-root path", async () => {
115+
writeManifest("data-manager", {
116+
...baseManifest,
117+
id: "data-manager",
118+
provides: ["osm-data"],
119+
produces: [{ type: "osm-pbf", sourceDir: "data/osm" }],
120+
});
121+
122+
const plan = await planDataCleanup("all", tmp);
123+
124+
expect(plan.all).toBe(true);
125+
expect(plan.paths).toEqual([join(tmp, "infra", "docker", "data")]);
126+
});
127+
128+
it("includes instance-scoped consumer target paths", async () => {
129+
writeManifest("data-manager", {
130+
...baseManifest,
131+
id: "data-manager",
132+
produces: [{ type: "otp-graph", sourceDir: "data/otp-graph" }],
133+
});
134+
writeManifest("otp", {
135+
...baseManifest,
136+
id: "otp",
137+
consumes: [
138+
{ type: "otp-graph", instance: "de", mountAt: "/var/opentripplanner", required: true },
139+
],
140+
});
141+
142+
const plan = await planDataCleanup("otp-graph", tmp);
143+
144+
expect(plan.paths).toEqual([
145+
join(tmp, "infra", "docker", "data", "otp-graph"),
146+
join(tmp, "infra", "docker", "data", "otp", "otp-graph", "de"),
147+
]);
148+
});
149+
150+
it("throws on unknown cleanup target", async () => {
151+
writeManifest("data-manager", {
152+
...baseManifest,
153+
id: "data-manager",
154+
provides: ["osm-data"],
155+
produces: [{ type: "osm-pbf", sourceDir: "data/osm" }],
156+
});
157+
158+
await expect(planDataCleanup("unknown-type", tmp)).rejects.toThrow(/Unknown clean target/);
159+
});
160+
});
161+
162+
describe("applyDataCleanup", () => {
163+
it("removes existing paths and reports removed files/bytes", () => {
164+
const dataRoot = join(tmp, "infra", "docker", "data");
165+
const source = join(dataRoot, "osm");
166+
const target = join(dataRoot, "valhalla", "osm-pbf");
167+
168+
mkdirSync(source, { recursive: true });
169+
mkdirSync(target, { recursive: true });
170+
writeFileSync(join(source, "planet.osm.pbf"), "PBF");
171+
writeFileSync(join(target, "planet.osm.pbf"), "PBF");
172+
173+
const before = summarizePath(dataRoot);
174+
const result = applyDataCleanup([source, target]);
175+
176+
expect(result.removedPaths).toBe(2);
177+
expect(result.removedFiles).toBe(2);
178+
expect(result.removedBytes).toBeGreaterThan(0);
179+
expect(summarizePath(dataRoot).files).toBe(before.files - 2);
180+
});
181+
});
182+
183+
describe("pruneDataManagerStateForCleanup", () => {
184+
it("drops matching datasets from .data-manager-state.json", async () => {
185+
writeManifest("data-manager", {
186+
...baseManifest,
187+
id: "data-manager",
188+
produces: [{ type: "osm-pbf", sourceDir: "data/osm" }],
189+
});
190+
writeManifest("valhalla", {
191+
...baseManifest,
192+
id: "valhalla",
193+
consumes: [{ type: "osm-pbf", mountAt: "/custom_files", required: true }],
194+
});
195+
196+
const dataRoot = join(tmp, "infra", "docker", "data");
197+
writeFileSync(
198+
join(dataRoot, ".data-manager-state.json"),
199+
JSON.stringify(
200+
{
201+
datasets: [
202+
{
203+
type: "osm-pbf",
204+
id: "europe-germany",
205+
path: "/data/osm/europe-germany.osm.pbf",
206+
sizeBytes: 123,
207+
downloadedAt: "2026-01-01T00:00:00.000Z",
208+
},
209+
{
210+
type: "gtfs",
211+
id: "de_db",
212+
path: "/data/gtfs/de_db.zip",
213+
sizeBytes: 10,
214+
downloadedAt: "2026-01-01T00:00:00.000Z",
215+
},
216+
],
217+
},
218+
null,
219+
2,
220+
),
221+
"utf-8",
222+
);
223+
224+
const plan = await planDataCleanup("osm-pbf", tmp);
225+
const result = pruneDataManagerStateForCleanup(plan);
226+
expect(result.updated).toBe(true);
227+
expect(result.removedDatasets).toBe(1);
228+
229+
const state = JSON.parse(readFileSync(join(dataRoot, ".data-manager-state.json"), "utf-8")) as {
230+
datasets: Array<{ type: string; id: string }>;
231+
};
232+
expect(state.datasets.map((dataset) => ({ type: dataset.type, id: dataset.id }))).toEqual([
233+
{ type: "gtfs", id: "de_db" },
234+
]);
235+
});
236+
});

0 commit comments

Comments
 (0)