diff --git a/docs/openapi.json b/docs/openapi.json index a4c7c40..ff2bab8 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -2653,6 +2653,147 @@ } } }, + "/api/v1/servers/{id}/map/perf": { + "get": { + "operationId": "getServersIdMapPerf", + "summary": "What the live map is allowed to cost on this server.", + "description": "Scope `settings` on the server.", + "tags": [ + "players" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Server id, as returned by GET /servers.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success." + }, + "400": { + "description": "Malformed request, or a missing confirmation." + }, + "401": { + "description": "No usable credential." + }, + "403": { + "description": "Authenticated, but not permitted — the body names what was needed." + }, + "404": { + "description": "No such server, or no such route." + }, + "409": { + "description": "Conflicts with the current state (running server, name taken, …)." + }, + "429": { + "description": "Rate limited. `Retry-After` says for how long." + } + } + }, + "post": { + "operationId": "postServersIdMapPerf", + "summary": "Change it. Values are clamped on the way in.", + "description": "Scope `settings` on the server.\n\nBody fields:\n- `cache` — Keep parsed tiles on disk (default on).\n- `memoryRegions` — Regions held in memory, 2-64.\n- `parseGapMs` — Minimum gap between region parses, 0-5000.\n- `cacheLimitMB` — On-disk ceiling, oldest evicted first.", + "tags": [ + "players" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Server id, as returned by GET /servers.", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "description": "Success." + }, + "400": { + "description": "Malformed request, or a missing confirmation." + }, + "401": { + "description": "No usable credential." + }, + "403": { + "description": "Authenticated, but not permitted — the body names what was needed." + }, + "404": { + "description": "No such server, or no such route." + }, + "409": { + "description": "Conflicts with the current state (running server, name taken, …)." + }, + "429": { + "description": "Rate limited. `Retry-After` says for how long." + } + } + } + }, + "/api/v1/servers/{id}/map/cache": { + "delete": { + "operationId": "deleteServersIdMapCache", + "summary": "Drop this server's cached map tiles.", + "description": "Scope `settings` on the server.\n\nOnly this server's: the cache filename carries the owner so one server's clear cannot take another's with it.", + "tags": [ + "players" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Server id, as returned by GET /servers.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success." + }, + "400": { + "description": "Malformed request, or a missing confirmation." + }, + "401": { + "description": "No usable credential." + }, + "403": { + "description": "Authenticated, but not permitted — the body names what was needed." + }, + "404": { + "description": "No such server, or no such route." + }, + "409": { + "description": "Conflicts with the current state (running server, name taken, …)." + }, + "429": { + "description": "Rate limited. `Retry-After` says for how long." + } + } + } + }, "/api/v1/servers/{id}/map/tiles": { "get": { "operationId": "getServersIdMapTiles", diff --git a/src/main/core/worldTiles.ts b/src/main/core/worldTiles.ts index dc146ed..ac14424 100644 --- a/src/main/core/worldTiles.ts +++ b/src/main/core/worldTiles.ts @@ -12,9 +12,22 @@ * parse a region synchronously — that is the amplification closed in #107, and * a region is three orders of magnitude more work than a player file. */ -import { existsSync, readFileSync, statSync } from 'node:fs' -import { inflateSync, gunzipSync } from 'node:zlib' +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { createHash } from 'node:crypto' +import { inflateSync, gunzipSync, gzipSync } from 'node:zlib' import { join } from 'node:path' +import { cacheDir } from '../paths' +import { decodeRegionTiles, encodeRegionTiles, normalizeMapPerf } from '@shared/tileCache' +import type { MapPerfConfig } from '@shared/tileCache' import * as nbt from 'prismarine-nbt' import { getServer } from './serverRegistry' import { readProperties } from './serverFiles' @@ -58,13 +71,149 @@ interface RegionEntry { } const regions = new Map() -/** A region is a few MB parsed; this is the ceiling on what is kept resident. */ -const MAX_REGIONS = 12 + +/** + * The tuning for the server whose region is being read. + * + * Looked up per call rather than captured: an operator changing the setting + * should not have to restart to see it take effect, and these are cheap reads + * off the in-memory config. + */ +function perfFor(serverId: string): MapPerfConfig { + return normalizeMapPerf(getServer(serverId)?.map) +} export function _resetWorldTiles(): void { regions.clear() } +// ---- the on-disk cache (#133) ---- +// +// #119 kept parsed regions in memory only, so every restart re-parsed the +// world: 180ms per region just to decompress and parse the NBT, before any +// surface extraction. The encoded form of the same region gunzips in about a +// millisecond. + +function cacheDirFor(): string { + return ensureDir(join(cacheDir(), 'worldtiles')) +} + +function ensureDir(p: string): string { + mkdirSync(p, { recursive: true }) + return p +} + +/** + * A stable filename for a region path. + * + * Hashed rather than sanitised: a region path contains drive letters, colons + * and separators, and every scheme for flattening those into a filename either + * collides or produces something unreadable. The version is in the CONTENT, not + * the name, so a stale file is refused on read and then overwritten rather than + * accumulating one file per version. + */ +function cacheFileFor(serverId: string, path: string): string { + // The server id is a visible PREFIX rather than part of the hash, because + // "clear this server's cache" has to be answerable from the filenames alone — + // a hash cannot be reversed, so a single hashed key made the clear button + // wipe every server's cache while claiming to clear one. + const owner = createHash('sha1').update(serverId).digest('hex').slice(0, 12) + return join(cacheDirFor(), owner + '-' + createHash('sha1').update(path).digest('hex') + '.tiles') +} + +function readCachedRegion(serverId: string, path: string, mtimeMs: number): RegionEntry | null { + try { + const f = cacheFileFor(serverId, path) + if (!existsSync(f)) return null + const decoded = decodeRegionTiles(gunzipSync(readFileSync(f))) + // The mtime says the world has not changed; the version inside the file + // says we still draw it the same way. Both have to hold. + if (!decoded || decoded.mtimeMs !== mtimeMs) return null + return { at: Date.now(), mtimeMs, tiles: new Map(decoded.tiles) } + } catch { + // A corrupt or half-written cache is not an error, it is a cache miss. + return null + } +} + +function writeCachedRegion(serverId: string, path: string, entry: RegionEntry): void { + try { + const usable = new Map() + for (const [slot, tile] of entry.tiles) if (tile) usable.set(slot, tile) + const buf = gzipSync(encodeRegionTiles({ mtimeMs: entry.mtimeMs, tiles: usable }), { level: 6 }) + writtenSinceSweep += buf.length + const f = cacheFileFor(serverId, path) + // Through a temp file: a reader hitting a half-written cache would decode + // garbage, and "garbage" here means a wrong map rather than an error. + writeFileSync(f + '.tmp', buf) + renameSync(f + '.tmp', f) + } catch { + /* a cache that cannot be written still leaves a working map */ + } +} + +/** + * Keep the cache under its ceiling, oldest first. + * + * Swept after a write rather than on a timer: the only moment it can grow is + * the moment something was added, and a timer would be one more thing running + * in a process that already has enough of them. + */ +/** + * Bytes added since the last sweep. + * + * A sweep stats every file in the directory, and doing that after each region + * means a directory scan per parse — a cost inside the thing that exists to + * remove cost. Only worth doing once enough has been added to matter. + */ +let writtenSinceSweep = 0 +const SWEEP_AFTER_BYTES = 32 * 1024 * 1024 + +function sweepCache(limitMB: number): void { + if (writtenSinceSweep < SWEEP_AFTER_BYTES) return + writtenSinceSweep = 0 + try { + const dir = cacheDirFor() + const files = readdirSync(dir) + .filter((n) => n.endsWith('.tiles')) + .map((n) => { + const p = join(dir, n) + const s = statSync(p) + return { p, size: s.size, at: s.mtimeMs } + }) + let total = files.reduce((a, f) => a + f.size, 0) + const limit = limitMB * 1024 * 1024 + if (total <= limit) return + for (const f of files.sort((a, b) => a.at - b.at)) { + if (total <= limit) break + rmSync(f.p, { force: true }) + total -= f.size + } + } catch { + /* sweeping is housekeeping; failing at it must not fail a map */ + } +} + +/** Drop cached regions for one server, or every server when given nothing. */ +export function clearTileCache(serverId?: string): number { + let n = 0 + try { + const dir = cacheDirFor() + const prefix = serverId ? createHash('sha1').update(serverId).digest('hex').slice(0, 12) + '-' : '' + for (const name of readdirSync(dir)) { + if (!name.endsWith('.tiles') || !name.startsWith(prefix)) continue + rmSync(join(dir, name), { force: true }) + n++ + } + } catch { + /* nothing to clear */ + } + // The memory cache is keyed by region path with no owner, so it goes whole. + // Dropping too much of an optimisation is free; keeping a stale entry is not. + regions.clear() + return n +} + /** Matches the private copies in players.ts and backups.ts. */ function levelName(id: string): string { const map = Object.fromEntries(readProperties(id).entries.map((e) => [e.key, e.value])) @@ -248,8 +397,9 @@ const REGION_PARSE_GAP_MS = 250 let lastParseAt = 0 /** Whether a parse is allowed to start right now. */ -export function parseBudgetReady(now = Date.now()): boolean { - return now - lastParseAt >= REGION_PARSE_GAP_MS +export function parseBudgetReady(serverId?: string, now = Date.now()): boolean { + const gap = serverId ? perfFor(serverId).parseGapMs : REGION_PARSE_GAP_MS + return now - lastParseAt >= gap } /** @@ -258,7 +408,7 @@ export function parseBudgetReady(now = Date.now()): boolean { * Synchronous and slow by design — the callers are expected to keep this off * any request path, and to respect `parseBudgetReady`. */ -function loadRegion(path: string): RegionEntry | null { +function loadRegion(serverId: string, path: string, perf: MapPerfConfig): RegionEntry | null { if (!existsSync(path)) return null let mtimeMs = 0 try { @@ -272,6 +422,17 @@ function loadRegion(path: string): RegionEntry | null { return hit } + // Disk before work. A region the server has not rewritten is the same region, + // and re-parsing it is the cost this whole cache exists to avoid. + if (perf.cache) { + const cached = readCachedRegion(serverId, path, mtimeMs) + if (cached) { + regions.set(path, cached) + trimMemory(perf.memoryRegions) + return cached + } + } + lastParseAt = Date.now() let file: Buffer try { @@ -301,14 +462,23 @@ function loadRegion(path: string): RegionEntry | null { const entry: RegionEntry = { at: Date.now(), mtimeMs, tiles } regions.set(path, entry) - if (regions.size > MAX_REGIONS) { - const oldest = [...regions.entries()].sort((a, b) => a[1].at - b[1].at)[0] - if (oldest) regions.delete(oldest[0]) + trimMemory(perf.memoryRegions) + if (perf.cache) { + writeCachedRegion(serverId, path, entry) + sweepCache(perf.cacheLimitMB) } log.info(`World tiles: parsed ${tiles.size} chunks from ${path.split(/[\\/]/).pop()}`) return entry } +function trimMemory(keep: number): void { + while (regions.size > keep) { + const oldest = [...regions.entries()].sort((a, b) => a[1].at - b[1].at)[0] + if (!oldest) break + regions.delete(oldest[0]) + } +} + const SECTOR_HEADER = 8192 /** @@ -389,7 +559,7 @@ async function drain(): Promise { while (queue.length) { // Yielding between chunks is not enough on its own: the first chunk of an // unseen region parses the whole file. Wait for the parse budget. - if (!parseBudgetReady()) { + if (!parseBudgetReady(queue[0]?.serverId)) { await new Promise((r) => setTimeout(r, 60)) continue } @@ -431,7 +601,7 @@ export function chunkTile( ): ChunkTile | null { const path = regionPath(serverId, dim, regionOf(chunkX), regionOf(chunkZ)) if (!path) return null - const region = loadRegion(path) + const region = loadRegion(serverId, path, perfFor(serverId)) if (!region) return null return region.tiles.get(chunkSlot(localChunk(chunkX), localChunk(chunkZ))) ?? null } diff --git a/src/main/ipc/register.ts b/src/main/ipc/register.ts index c08ec07..e35be95 100644 --- a/src/main/ipc/register.ts +++ b/src/main/ipc/register.ts @@ -302,6 +302,7 @@ export function registerIpc(): void { H(IPC.bridgeInstall, (_e, id: string) => bridgeInstall.installBridge(id, { by: 'desktop', source: 'panel' }) ) + H(IPC.mapCacheClear, () => worldTiles.clearTileCache()) H( IPC.mapTiles, (_e, id: string, dim: string, chunks: { cx: number; cz: number }[], marks?: boolean) => diff --git a/src/main/smoke.ts b/src/main/smoke.ts index 5024015..e60257d 100644 --- a/src/main/smoke.ts +++ b/src/main/smoke.ts @@ -79,6 +79,14 @@ import * as schedulerMod from './core/scheduler' import * as modsMod from './core/mods' import * as bridgeInstallMod from './core/bridgeInstall' import * as worldTilesMod from './core/worldTiles' +import { + decodeRegionTiles, + encodeRegionTiles, + normalizeMapPerf, + MAP_PERF_DEFAULTS, + TILE_CACHE_VERSION +} from '@shared/tileCache' +import type { CachedRegion, CachedTile } from '@shared/tileCache' import { bridgeNeed, bridgeVersionOf, @@ -1145,7 +1153,97 @@ export async function runModUpdateSmoke(): Promise { } } - console.log('MODUPDATE-SMOKE: region decoding OK (1.16 packing split, real NBT chunk renders, foliage seen through)') + // ---- the on-disk tile cache (#133) ---- + // + // A decoder that mis-reads its own file produces a plausible map rather + // than an error, and then serves it until somebody deletes the cache by + // hand. So the codec round-trips, and every way it can be handed + // nonsense answers null. + { + const mk = (n: number): CachedTile => ({ + colour: Array.from({ length: 256 }, (_, i) => (i % 7 === 0 ? -1 : (i * 977 + n) & 0xffffff)), + height: Array.from({ length: 256 }, (_, i) => ((i * 13 + n) % 384) - 64) + }) + const region: CachedRegion = { + mtimeMs: 1_700_000_000_123, + tiles: new Map([ + [0, mk(1)], + [511, { ...mk(2), marks: [{ kind: 'village', id: 'village_plains', x: 8, z: -24 }] }], + [1023, { ...mk(3), marks: [ + { kind: 'dungeon', id: 'ancient_city', x: -1_000_000, z: 2_000_000 }, + { kind: 'mine', id: 'mineshaft', x: 0, z: 0 } + ] }] + ]) + } + const back = decodeRegionTiles(encodeRegionTiles(region)) + if (!back) return fail('a freshly encoded region did not decode') + if (back.mtimeMs !== region.mtimeMs) return fail('the mtime did not survive the round trip') + if (back.tiles.size !== 3) return fail('a chunk was lost: ' + back.tiles.size) + for (const [slot, want] of region.tiles) { + const got = back.tiles.get(slot) + if (!got) return fail('chunk ' + slot + ' vanished') + for (let i = 0; i < 256; i++) { + // Transparent columns are the ones that matter: encoded as black + // they would paint the void over every ungenerated gap. + if (got.colour[i] !== want.colour[i]) { + return fail('colour ' + i + ' of chunk ' + slot + ': ' + got.colour[i] + ' vs ' + want.colour[i]) + } + // Heights run -64..319, which does not fit a byte. + if (got.height[i] !== want.height[i]) { + return fail('height ' + i + ' of chunk ' + slot + ': ' + got.height[i] + ' vs ' + want.height[i]) + } + } + if ((got.marks ?? []).length !== (want.marks ?? []).length) { + return fail('marks lost on chunk ' + slot) + } + for (let mi = 0; mi < (want.marks ?? []).length; mi++) { + const a = (want.marks ?? [])[mi] + const b = (got.marks ?? [])[mi] + if (a.kind !== b.kind || a.id !== b.id || a.x !== b.x || a.z !== b.z) { + return fail('mark ' + mi + ' of chunk ' + slot + ' changed: ' + JSON.stringify(b)) + } + } + } + + // A cache written by an older renderer must be REFUSED, not decoded. + // Keying on the world's mtime alone says "the world has not changed", + // which is true and beside the point when what changed is how we draw + // it — every existing cache would serve the old colours forever. + const stale = encodeRegionTiles(region) + new DataView(stale.buffer).setUint16(4, TILE_CACHE_VERSION - 1) + if (decodeRegionTiles(stale)) return fail('a cache from an older format version was accepted') + + // Nonsense of every shape is a miss, never a throw and never half a map. + const good = encodeRegionTiles(region) + const wrongMagic = good.slice() + wrongMagic[0] ^= 0xff + if (decodeRegionTiles(wrongMagic)) return fail('a file with the wrong magic was accepted') + if (decodeRegionTiles(new Uint8Array(0))) return fail('an empty buffer was accepted') + if (decodeRegionTiles(new Uint8Array(8))) return fail('a runt buffer was accepted') + for (const cut of [17, 200, good.length - 1]) { + if (decodeRegionTiles(good.slice(0, cut))) { + return fail('a buffer truncated to ' + cut + ' bytes was accepted') + } + } + // A count that claims more chunks than the bytes hold. + const lying = good.slice() + new DataView(lying.buffer).setUint16(14, 900) + if (decodeRegionTiles(lying)) return fail('a chunk count past the end of the buffer was accepted') + + // The tuning is clamped: every one of these is a way to hang the + // process, and they arrive from a config file an operator can edit. + const wild = normalizeMapPerf({ memoryRegions: 1e9, parseGapMs: -5, cacheLimitMB: -1 }) + if (wild.memoryRegions > 64) return fail('memoryRegions was not clamped: ' + wild.memoryRegions) + if (wild.parseGapMs < 0) return fail('a negative parse gap survived') + if (wild.cacheLimitMB < 0) return fail('a negative cache limit survived') + if (normalizeMapPerf({}).cache !== true) return fail('caching must default to on') + if (normalizeMapPerf({ cache: false }).cache !== false) return fail('caching cannot be turned off') + if (normalizeMapPerf(null).memoryRegions !== MAP_PERF_DEFAULTS.memoryRegions) { + return fail('an absent config did not fall back to the defaults') + } + } + + console.log('MODUPDATE-SMOKE: region decoding OK (1.16 packing split, real NBT chunk renders, foliage seen through, cache round-trips)') } // ---- the Bridge plugin installer (#103) ---- diff --git a/src/main/web/panelHtml.ts b/src/main/web/panelHtml.ts index 1db1845..e1fd185 100644 --- a/src/main/web/panelHtml.ts +++ b/src/main/web/panelHtml.ts @@ -340,7 +340,34 @@ h2{margin:8px 0;font-weight:800;letter-spacing:-.4px}
- +