From 7974237f52d69ce536c46b5195cffb6f803aec29 Mon Sep 17 00:00:00 2001 From: CaYatur Date: Wed, 29 Jul 2026 13:17:39 +0300 Subject: [PATCH 1/2] Cache parsed tiles on disk, and make the map's cost an operator setting (#133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on a real Paper 1.21.6 world before writing anything. One 5.4 MB region of 811 chunks and 19 769 sections: total 180ms | decompress 39ms | nbt parse 141ms ...before any surface extraction, and repeated **every time the app starts**, because #119's cache was a `Map` in memory holding twelve regions. The disk half was in that issue's scope and I only built the memory half. This is the rest. The encoded form of a region is about 1 MB, gzips in 2-3 ms and gunzips in about one. So a region now costs a few hundred milliseconds once, and single-digit milliseconds forever after — parsed again only when the server rewrites it. `shared/tileCache.ts` holds the format, pure, because this is where a mistake is silent: a decoder that mis-reads its own file produces a *plausible* map rather than an error, and then serves it until somebody deletes the cache by hand. Fixed-width columns rather than a hand-rolled packing — the caller gzips it, and gzip finds the runs better than a scheme I could get wrong. **The key carries a format version as well as the mtime.** The mtime says the world has not changed, which is true and beside the point when what changed is how we draw it: without the version, every cache written before the colour and foliage work would go on serving the old picture forever, and an operator staring at it has no way to connect it to an update. Changing the colours, the foliage rule or the shading and not bumping that number is the bug this field exists to prevent. **Per-server settings**, because a box running twenty servers and a laptop running one want different answers: - *Cache tiles on disk* — on by default; this is the one that matters. - *Regions in memory* — the working set. - *Delay between parses* — the brake that keeps the main thread responsive. - *Cache limit* — oldest evicted first, plus a way to clear it outright. They live on the server's own config so they survive a restart and apply to every surface, and they are clamped on the way IN rather than on the way out: a value only fixed when read is still a wrong number in the file, and every one of these is a way to hang the process — a parse gap of zero removes the brake, a memory limit of a million holds a whole world resident. Asserted: the codec round-trips colours, heights across the full -64..319 range, the transparent columns that must stay transparent, and structure marks with their ids. A cache from an older format version is refused rather than decoded. Wrong magic, an empty buffer, three different truncations and a chunk count that runs past the end all answer null rather than throwing or returning half a map. Proved failable — encoding transparent columns as present gives `FAIL - colour 0 of chunk 0: 0 vs -1`, which is the void painted over every ungenerated gap. --- src/main/core/worldTiles.ts | 173 ++++++++++++++++++-- src/main/ipc/register.ts | 1 + src/main/smoke.ts | 100 +++++++++++- src/preload/index.ts | 1 + src/renderer/src/components/LiveMap.tsx | 97 ++++++++++- src/renderer/src/locales/en.ts | 11 ++ src/renderer/src/locales/tr.ts | 11 ++ src/shared/ipc.ts | 3 + src/shared/tileCache.ts | 206 ++++++++++++++++++++++++ src/shared/types.ts | 8 + 10 files changed, 597 insertions(+), 14 deletions(-) create mode 100644 src/shared/tileCache.ts diff --git a/src/main/core/worldTiles.ts b/src/main/core/worldTiles.ts index dc146ed..6bab3d2 100644 --- a/src/main/core/worldTiles.ts +++ b/src/main/core/worldTiles.ts @@ -12,9 +12,22 @@ * parse a region synchronously — that is the amplification closed in #107, and * a region is three orders of magnitude more work than a player file. */ -import { existsSync, readFileSync, statSync } from 'node:fs' -import { inflateSync, gunzipSync } from 'node:zlib' +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { createHash } from 'node:crypto' +import { inflateSync, gunzipSync, gzipSync } from 'node:zlib' import { join } from 'node:path' +import { cacheDir } from '../paths' +import { decodeRegionTiles, encodeRegionTiles, normalizeMapPerf } from '@shared/tileCache' +import type { MapPerfConfig } from '@shared/tileCache' import * as nbt from 'prismarine-nbt' import { getServer } from './serverRegistry' import { readProperties } from './serverFiles' @@ -58,13 +71,128 @@ interface RegionEntry { } const regions = new Map() -/** A region is a few MB parsed; this is the ceiling on what is kept resident. */ -const MAX_REGIONS = 12 + +/** + * The tuning for the server whose region is being read. + * + * Looked up per call rather than captured: an operator changing the setting + * should not have to restart to see it take effect, and these are cheap reads + * off the in-memory config. + */ +function perfFor(serverId: string): MapPerfConfig { + return normalizeMapPerf(getServer(serverId)?.map) +} export function _resetWorldTiles(): void { regions.clear() } +// ---- the on-disk cache (#133) ---- +// +// #119 kept parsed regions in memory only, so every restart re-parsed the +// world: 180ms per region just to decompress and parse the NBT, before any +// surface extraction. The encoded form of the same region gunzips in about a +// millisecond. + +function cacheDirFor(): string { + return ensureDir(join(cacheDir(), 'worldtiles')) +} + +function ensureDir(p: string): string { + mkdirSync(p, { recursive: true }) + return p +} + +/** + * A stable filename for a region path. + * + * Hashed rather than sanitised: a region path contains drive letters, colons + * and separators, and every scheme for flattening those into a filename either + * collides or produces something unreadable. The version is in the CONTENT, not + * the name, so a stale file is refused on read and then overwritten rather than + * accumulating one file per version. + */ +function cacheFileFor(path: string): string { + return join(cacheDirFor(), createHash('sha1').update(path).digest('hex') + '.tiles') +} + +function readCachedRegion(path: string, mtimeMs: number): RegionEntry | null { + try { + const f = cacheFileFor(path) + if (!existsSync(f)) return null + const decoded = decodeRegionTiles(gunzipSync(readFileSync(f))) + // The mtime says the world has not changed; the version inside the file + // says we still draw it the same way. Both have to hold. + if (!decoded || decoded.mtimeMs !== mtimeMs) return null + return { at: Date.now(), mtimeMs, tiles: new Map(decoded.tiles) } + } catch { + // A corrupt or half-written cache is not an error, it is a cache miss. + return null + } +} + +function writeCachedRegion(path: string, entry: RegionEntry): void { + try { + const usable = new Map() + for (const [slot, tile] of entry.tiles) if (tile) usable.set(slot, tile) + const buf = gzipSync(encodeRegionTiles({ mtimeMs: entry.mtimeMs, tiles: usable }), { level: 6 }) + const f = cacheFileFor(path) + // Through a temp file: a reader hitting a half-written cache would decode + // garbage, and "garbage" here means a wrong map rather than an error. + writeFileSync(f + '.tmp', buf) + renameSync(f + '.tmp', f) + } catch { + /* a cache that cannot be written still leaves a working map */ + } +} + +/** + * Keep the cache under its ceiling, oldest first. + * + * Swept after a write rather than on a timer: the only moment it can grow is + * the moment something was added, and a timer would be one more thing running + * in a process that already has enough of them. + */ +function sweepCache(limitMB: number): void { + try { + const dir = cacheDirFor() + const files = readdirSync(dir) + .filter((n) => n.endsWith('.tiles')) + .map((n) => { + const p = join(dir, n) + const s = statSync(p) + return { p, size: s.size, at: s.mtimeMs } + }) + let total = files.reduce((a, f) => a + f.size, 0) + const limit = limitMB * 1024 * 1024 + if (total <= limit) return + for (const f of files.sort((a, b) => a.at - b.at)) { + if (total <= limit) break + rmSync(f.p, { force: true }) + total -= f.size + } + } catch { + /* sweeping is housekeeping; failing at it must not fail a map */ + } +} + +/** Drop every cached region for one server, or all of them. */ +export function clearTileCache(): number { + let n = 0 + try { + const dir = cacheDirFor() + for (const name of readdirSync(dir)) { + if (!name.endsWith('.tiles')) continue + rmSync(join(dir, name), { force: true }) + n++ + } + } catch { + /* nothing to clear */ + } + regions.clear() + return n +} + /** Matches the private copies in players.ts and backups.ts. */ function levelName(id: string): string { const map = Object.fromEntries(readProperties(id).entries.map((e) => [e.key, e.value])) @@ -248,8 +376,9 @@ const REGION_PARSE_GAP_MS = 250 let lastParseAt = 0 /** Whether a parse is allowed to start right now. */ -export function parseBudgetReady(now = Date.now()): boolean { - return now - lastParseAt >= REGION_PARSE_GAP_MS +export function parseBudgetReady(serverId?: string, now = Date.now()): boolean { + const gap = serverId ? perfFor(serverId).parseGapMs : REGION_PARSE_GAP_MS + return now - lastParseAt >= gap } /** @@ -258,7 +387,7 @@ export function parseBudgetReady(now = Date.now()): boolean { * Synchronous and slow by design — the callers are expected to keep this off * any request path, and to respect `parseBudgetReady`. */ -function loadRegion(path: string): RegionEntry | null { +function loadRegion(path: string, perf: MapPerfConfig): RegionEntry | null { if (!existsSync(path)) return null let mtimeMs = 0 try { @@ -272,6 +401,17 @@ function loadRegion(path: string): RegionEntry | null { return hit } + // Disk before work. A region the server has not rewritten is the same region, + // and re-parsing it is the cost this whole cache exists to avoid. + if (perf.cache) { + const cached = readCachedRegion(path, mtimeMs) + if (cached) { + regions.set(path, cached) + trimMemory(perf.memoryRegions) + return cached + } + } + lastParseAt = Date.now() let file: Buffer try { @@ -301,14 +441,23 @@ function loadRegion(path: string): RegionEntry | null { const entry: RegionEntry = { at: Date.now(), mtimeMs, tiles } regions.set(path, entry) - if (regions.size > MAX_REGIONS) { - const oldest = [...regions.entries()].sort((a, b) => a[1].at - b[1].at)[0] - if (oldest) regions.delete(oldest[0]) + trimMemory(perf.memoryRegions) + if (perf.cache) { + writeCachedRegion(path, entry) + sweepCache(perf.cacheLimitMB) } log.info(`World tiles: parsed ${tiles.size} chunks from ${path.split(/[\\/]/).pop()}`) return entry } +function trimMemory(keep: number): void { + while (regions.size > keep) { + const oldest = [...regions.entries()].sort((a, b) => a[1].at - b[1].at)[0] + if (!oldest) break + regions.delete(oldest[0]) + } +} + const SECTOR_HEADER = 8192 /** @@ -389,7 +538,7 @@ async function drain(): Promise { while (queue.length) { // Yielding between chunks is not enough on its own: the first chunk of an // unseen region parses the whole file. Wait for the parse budget. - if (!parseBudgetReady()) { + if (!parseBudgetReady(queue[0]?.serverId)) { await new Promise((r) => setTimeout(r, 60)) continue } @@ -431,7 +580,7 @@ export function chunkTile( ): ChunkTile | null { const path = regionPath(serverId, dim, regionOf(chunkX), regionOf(chunkZ)) if (!path) return null - const region = loadRegion(path) + const region = loadRegion(path, perfFor(serverId)) if (!region) return null return region.tiles.get(chunkSlot(localChunk(chunkX), localChunk(chunkZ))) ?? null } diff --git a/src/main/ipc/register.ts b/src/main/ipc/register.ts index c08ec07..e35be95 100644 --- a/src/main/ipc/register.ts +++ b/src/main/ipc/register.ts @@ -302,6 +302,7 @@ export function registerIpc(): void { H(IPC.bridgeInstall, (_e, id: string) => bridgeInstall.installBridge(id, { by: 'desktop', source: 'panel' }) ) + H(IPC.mapCacheClear, () => worldTiles.clearTileCache()) H( IPC.mapTiles, (_e, id: string, dim: string, chunks: { cx: number; cz: number }[], marks?: boolean) => diff --git a/src/main/smoke.ts b/src/main/smoke.ts index 5024015..e60257d 100644 --- a/src/main/smoke.ts +++ b/src/main/smoke.ts @@ -79,6 +79,14 @@ import * as schedulerMod from './core/scheduler' import * as modsMod from './core/mods' import * as bridgeInstallMod from './core/bridgeInstall' import * as worldTilesMod from './core/worldTiles' +import { + decodeRegionTiles, + encodeRegionTiles, + normalizeMapPerf, + MAP_PERF_DEFAULTS, + TILE_CACHE_VERSION +} from '@shared/tileCache' +import type { CachedRegion, CachedTile } from '@shared/tileCache' import { bridgeNeed, bridgeVersionOf, @@ -1145,7 +1153,97 @@ export async function runModUpdateSmoke(): Promise { } } - console.log('MODUPDATE-SMOKE: region decoding OK (1.16 packing split, real NBT chunk renders, foliage seen through)') + // ---- the on-disk tile cache (#133) ---- + // + // A decoder that mis-reads its own file produces a plausible map rather + // than an error, and then serves it until somebody deletes the cache by + // hand. So the codec round-trips, and every way it can be handed + // nonsense answers null. + { + const mk = (n: number): CachedTile => ({ + colour: Array.from({ length: 256 }, (_, i) => (i % 7 === 0 ? -1 : (i * 977 + n) & 0xffffff)), + height: Array.from({ length: 256 }, (_, i) => ((i * 13 + n) % 384) - 64) + }) + const region: CachedRegion = { + mtimeMs: 1_700_000_000_123, + tiles: new Map([ + [0, mk(1)], + [511, { ...mk(2), marks: [{ kind: 'village', id: 'village_plains', x: 8, z: -24 }] }], + [1023, { ...mk(3), marks: [ + { kind: 'dungeon', id: 'ancient_city', x: -1_000_000, z: 2_000_000 }, + { kind: 'mine', id: 'mineshaft', x: 0, z: 0 } + ] }] + ]) + } + const back = decodeRegionTiles(encodeRegionTiles(region)) + if (!back) return fail('a freshly encoded region did not decode') + if (back.mtimeMs !== region.mtimeMs) return fail('the mtime did not survive the round trip') + if (back.tiles.size !== 3) return fail('a chunk was lost: ' + back.tiles.size) + for (const [slot, want] of region.tiles) { + const got = back.tiles.get(slot) + if (!got) return fail('chunk ' + slot + ' vanished') + for (let i = 0; i < 256; i++) { + // Transparent columns are the ones that matter: encoded as black + // they would paint the void over every ungenerated gap. + if (got.colour[i] !== want.colour[i]) { + return fail('colour ' + i + ' of chunk ' + slot + ': ' + got.colour[i] + ' vs ' + want.colour[i]) + } + // Heights run -64..319, which does not fit a byte. + if (got.height[i] !== want.height[i]) { + return fail('height ' + i + ' of chunk ' + slot + ': ' + got.height[i] + ' vs ' + want.height[i]) + } + } + if ((got.marks ?? []).length !== (want.marks ?? []).length) { + return fail('marks lost on chunk ' + slot) + } + for (let mi = 0; mi < (want.marks ?? []).length; mi++) { + const a = (want.marks ?? [])[mi] + const b = (got.marks ?? [])[mi] + if (a.kind !== b.kind || a.id !== b.id || a.x !== b.x || a.z !== b.z) { + return fail('mark ' + mi + ' of chunk ' + slot + ' changed: ' + JSON.stringify(b)) + } + } + } + + // A cache written by an older renderer must be REFUSED, not decoded. + // Keying on the world's mtime alone says "the world has not changed", + // which is true and beside the point when what changed is how we draw + // it — every existing cache would serve the old colours forever. + const stale = encodeRegionTiles(region) + new DataView(stale.buffer).setUint16(4, TILE_CACHE_VERSION - 1) + if (decodeRegionTiles(stale)) return fail('a cache from an older format version was accepted') + + // Nonsense of every shape is a miss, never a throw and never half a map. + const good = encodeRegionTiles(region) + const wrongMagic = good.slice() + wrongMagic[0] ^= 0xff + if (decodeRegionTiles(wrongMagic)) return fail('a file with the wrong magic was accepted') + if (decodeRegionTiles(new Uint8Array(0))) return fail('an empty buffer was accepted') + if (decodeRegionTiles(new Uint8Array(8))) return fail('a runt buffer was accepted') + for (const cut of [17, 200, good.length - 1]) { + if (decodeRegionTiles(good.slice(0, cut))) { + return fail('a buffer truncated to ' + cut + ' bytes was accepted') + } + } + // A count that claims more chunks than the bytes hold. + const lying = good.slice() + new DataView(lying.buffer).setUint16(14, 900) + if (decodeRegionTiles(lying)) return fail('a chunk count past the end of the buffer was accepted') + + // The tuning is clamped: every one of these is a way to hang the + // process, and they arrive from a config file an operator can edit. + const wild = normalizeMapPerf({ memoryRegions: 1e9, parseGapMs: -5, cacheLimitMB: -1 }) + if (wild.memoryRegions > 64) return fail('memoryRegions was not clamped: ' + wild.memoryRegions) + if (wild.parseGapMs < 0) return fail('a negative parse gap survived') + if (wild.cacheLimitMB < 0) return fail('a negative cache limit survived') + if (normalizeMapPerf({}).cache !== true) return fail('caching must default to on') + if (normalizeMapPerf({ cache: false }).cache !== false) return fail('caching cannot be turned off') + if (normalizeMapPerf(null).memoryRegions !== MAP_PERF_DEFAULTS.memoryRegions) { + return fail('an absent config did not fall back to the defaults') + } + } + + console.log('MODUPDATE-SMOKE: region decoding OK (1.16 packing split, real NBT chunk renders, foliage seen through, cache round-trips)') } // ---- the Bridge plugin installer (#103) ---- diff --git a/src/preload/index.ts b/src/preload/index.ts index 7f9bf5f..6fa310d 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -81,6 +81,7 @@ const api: MsmsApi = { bridgeStatus: (id) => ipcRenderer.invoke(IPC.bridgeStatus, id), installBridge: (id) => ipcRenderer.invoke(IPC.bridgeInstall, id), mapTiles: (id, dim, chunks, marks) => ipcRenderer.invoke(IPC.mapTiles, id, dim, chunks, marks), + clearMapCache: () => ipcRenderer.invoke(IPC.mapCacheClear), listJava: (refresh) => ipcRenderer.invoke(IPC.javaList, refresh), resolveJava: (override) => ipcRenderer.invoke(IPC.javaResolve, override), diff --git a/src/renderer/src/components/LiveMap.tsx b/src/renderer/src/components/LiveMap.tsx index 13f7bdc..7ef656e 100644 --- a/src/renderer/src/components/LiveMap.tsx +++ b/src/renderer/src/components/LiveMap.tsx @@ -1,6 +1,9 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' -import { Map as MapIcon, Flame } from 'lucide-react' +import { Map as MapIcon, Flame, Gauge } from 'lucide-react' +import { useStore } from '../store' +import { normalizeMapPerf } from '@shared/tileCache' +import type { MapPerfConfig } from '@shared/tileCache' import { fitView, heatmap, mapBounds, panBy, screenToWorld, worldToScreen, zoomAt } from '@shared/livemap' import type { LivePlayer, MapView, Viewport } from '@shared/livemap' import { avatarUrl } from '@shared/profile' @@ -192,6 +195,29 @@ export function LiveMap({ serverId }: { serverId: string }): JSX.Element { const [marks, setMarks] = useState(false) const [markKind, setMarkKind] = useState('') const markStore = useRef(new Map()) + + // Per-server map tuning (#133), read from the server's own config so it + // survives a restart and applies to every surface, not just this one. + const servers = useStore((s) => s.servers) + const updateServer = useStore((s) => s.updateServer) + const perf = useMemo( + () => normalizeMapPerf(servers.find((s) => s.id === serverId)?.map), + [servers, serverId] + ) + const [showPerf, setShowPerf] = useState(false) + const [cleared, setCleared] = useState(null) + + const savePerf = (patch: Partial): void => { + // Normalised before it is stored, not after it is read: a value that only + // gets clamped on the way out is still a wrong number in the config file. + void updateServer(serverId, { map: normalizeMapPerf({ ...perf, ...patch }) }) + } + const clearCache = async (): Promise => { + setCleared(await window.msms.clearMapCache()) + tiles.current.clear() + markStore.current.clear() + setTick2((n) => n + 1) + } const headCache = useRef(new Map()) const tiles = useRef(new Map()) const tilesPending = useRef(false) @@ -470,8 +496,77 @@ export function LiveMap({ serverId }: { serverId: string }): JSX.Element { + + {/* Per-server, persisted, and applied without a restart — the map is where + an operator meets the cost, so it is where the dials belong (#133). */} + {showPerf && ( +
+ +

+ {t('map.perfCacheHint')} +

+
+
+
+ {t('map.perfMemory')} +
+ savePerf({ memoryRegions: Number(e.target.value) })} + /> +
+
+
+ {t('map.perfGap')} +
+ savePerf({ parseGapMs: Number(e.target.value) })} + /> +
+
+
+ {t('map.perfLimit')} +
+ savePerf({ cacheLimitMB: Number(e.target.value) })} + /> +
+ +
+

{t('map.perfGapHint')}

+ {cleared !== null &&

{t('map.perfCleared', { n: cleared })}

} +
+ )} +
pending: number }> + /** Drop every cached region. Returns how many files went. */ + clearMapCache(): Promise listJava(refresh?: boolean): Promise /** The Java that will actually launch, given a per-server override ('' = auto). */ diff --git a/src/shared/tileCache.ts b/src/shared/tileCache.ts new file mode 100644 index 0000000..a9dcd2a --- /dev/null +++ b/src/shared/tileCache.ts @@ -0,0 +1,206 @@ +/** + * The on-disk format for parsed map tiles (#133). + * + * Pure, because this is the part where a mistake is silent: a decoder that + * mis-reads its own file produces a *plausible* map rather than an error, and + * the cache then serves that wrong picture until somebody deletes it by hand. + * + * Why it is worth having at all, measured on a real world: one 5.4 MB region of + * 811 chunks costs 180 ms to decompress and NBT-parse before any surface work, + * and #119's cache was a Map in memory — so every restart paid it again. The + * encoded form of the same region is about 1 MB, gzips in 2-3 ms and gunzips in + * about 1. + */ + +import type { StructureKind } from './regionFormat' +import { STRUCTURE_KINDS } from './regionFormat' + +/** + * Bumped whenever the RENDERED OUTPUT changes — the colour table, the foliage + * rule, the shading, the column layout. + * + * This is the field that stops a cache outliving its meaning. Keying on the + * region's mtime alone says "the world has not changed", which is true and + * beside the point when what changed is how we draw it: every existing cache + * would go on serving the old picture forever, and an operator staring at a + * map with the old colours has no way to connect it to an update they + * installed. Changing any of those things and NOT bumping this is the bug. + */ +export const TILE_CACHE_VERSION = 3 + +const MAGIC = 0x4d53544c // 'MSTL' +const COLUMNS = 256 +/** flags, r, g, b, then height as an i16. */ +const BYTES_PER_COLUMN = 6 +const HEADER_BYTES = 4 + 2 + 8 + 2 + +export interface CachedTile { + colour: number[] + height: number[] + marks?: { kind: StructureKind; id: string; x: number; z: number }[] +} + +export interface CachedRegion { + mtimeMs: number + /** Chunk slot (0..1023) to tile. A slot absent here was never generated. */ + tiles: Map +} + +/** + * Encode a region's tiles. + * + * Fixed-width per column rather than a packed run-length: the caller gzips this, + * and gzip finds the runs itself far better than a hand-rolled scheme would — + * with none of the ways a hand-rolled one can be wrong. + */ +export function encodeRegionTiles(region: CachedRegion): Uint8Array { + const parts: Uint8Array[] = [] + let bodyBytes = 0 + for (const [slot, tile] of region.tiles) { + const marks = tile.marks ?? [] + let markBytes = 0 + const markBufs: Uint8Array[] = [] + for (const m of marks) { + const id = new TextEncoder().encode(m.id.slice(0, 200)) + const b = new Uint8Array(1 + 4 + 4 + 1 + id.length) + const dv = new DataView(b.buffer) + b[0] = Math.max(0, STRUCTURE_KINDS.indexOf(m.kind)) + dv.setInt32(1, Math.trunc(m.x)) + dv.setInt32(5, Math.trunc(m.z)) + b[9] = id.length + b.set(id, 10) + markBufs.push(b) + markBytes += b.length + } + const chunk = new Uint8Array(2 + 2 + COLUMNS * BYTES_PER_COLUMN + markBytes) + const dv = new DataView(chunk.buffer) + dv.setUint16(0, slot) + dv.setUint16(2, markBufs.length) + let o = 4 + for (let i = 0; i < COLUMNS; i++) { + const c = tile.colour[i] + // A column with no colour is transparent and MUST round-trip as one — + // encoding it as black would paint the void over every ungenerated gap. + chunk[o] = c >= 0 ? 1 : 0 + chunk[o + 1] = c >= 0 ? (c >> 16) & 255 : 0 + chunk[o + 2] = c >= 0 ? (c >> 8) & 255 : 0 + chunk[o + 3] = c >= 0 ? c & 255 : 0 + // Heights run -64..319 in modern worlds, which does not fit a byte. + dv.setInt16(o + 4, Math.max(-32768, Math.min(32767, Math.trunc(tile.height[i] ?? 0)))) + o += BYTES_PER_COLUMN + } + for (const b of markBufs) { + chunk.set(b, o) + o += b.length + } + parts.push(chunk) + bodyBytes += chunk.length + } + + const out = new Uint8Array(HEADER_BYTES + bodyBytes) + const dv = new DataView(out.buffer) + dv.setUint32(0, MAGIC) + dv.setUint16(4, TILE_CACHE_VERSION) + dv.setFloat64(6, region.mtimeMs) + dv.setUint16(14, region.tiles.size) + let at = HEADER_BYTES + for (const p of parts) { + out.set(p, at) + at += p.length + } + return out +} + +/** + * Decode, or null. + * + * Null for every reason: wrong magic, wrong version, truncated, a length that + * runs past the end. A cache is an optimisation, so the only correct response + * to one that does not make sense is to ignore it and parse the world again — + * never to throw, and never to return half of it. + */ +export function decodeRegionTiles(buf: Uint8Array): CachedRegion | null { + if (buf.length < HEADER_BYTES) return null + const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength) + if (dv.getUint32(0) !== MAGIC) return null + if (dv.getUint16(4) !== TILE_CACHE_VERSION) return null + const mtimeMs = dv.getFloat64(6) + const count = dv.getUint16(14) + + const tiles = new Map() + let o = HEADER_BYTES + for (let n = 0; n < count; n++) { + if (o + 4 + COLUMNS * BYTES_PER_COLUMN > buf.length) return null + const slot = dv.getUint16(o) + const markCount = dv.getUint16(o + 2) + o += 4 + const colour = new Array(COLUMNS) + const height = new Array(COLUMNS) + for (let i = 0; i < COLUMNS; i++) { + colour[i] = buf[o] ? (buf[o + 1] << 16) | (buf[o + 2] << 8) | buf[o + 3] : -1 + height[i] = dv.getInt16(o + 4) + o += BYTES_PER_COLUMN + } + const marks: CachedTile['marks'] = [] + for (let mi = 0; mi < markCount; mi++) { + if (o + 10 > buf.length) return null + const kind = STRUCTURE_KINDS[buf[o]] ?? 'other' + const x = dv.getInt32(o + 1) + const z = dv.getInt32(o + 5) + const idLen = buf[o + 9] + o += 10 + if (o + idLen > buf.length) return null + const id = new TextDecoder().decode(buf.subarray(o, o + idLen)) + o += idLen + marks.push({ kind, id, x, z }) + } + tiles.set(slot, { colour, height, ...(marks.length ? { marks } : {}) }) + } + return { mtimeMs, tiles } +} + +// ---- per-server tuning ---- + +/** + * What the map is allowed to cost on one server. + * + * Per server because a box running twenty of them and a laptop running one want + * different answers, and the person who knows which is the operator. + */ +export interface MapPerfConfig { + /** Keep parsed tiles on disk so a restart does not re-parse the world. */ + cache: boolean + /** Regions held in memory. The working set; more is faster and heavier. */ + memoryRegions: number + /** Minimum gap between two region parses, in ms. The politeness brake. */ + parseGapMs: number + /** Ceiling on the on-disk cache, in MB. Oldest evicted first. */ + cacheLimitMB: number +} + +export const MAP_PERF_DEFAULTS: MapPerfConfig = { + cache: true, + memoryRegions: 12, + parseGapMs: 250, + cacheLimitMB: 512 +} + +/** + * Clamp, because these arrive from a config file an operator can hand-edit and + * every one of them is a way to hang the process: a parse gap of zero removes + * the brake that keeps the console feed responsive, and a memory limit of a + * million holds every region of a big world at once. + */ +export function normalizeMapPerf(raw: unknown): MapPerfConfig { + const r = (raw ?? {}) as Partial + const num = (v: unknown, lo: number, hi: number, dflt: number): number => { + const n = typeof v === 'number' && Number.isFinite(v) ? Math.round(v) : dflt + return Math.min(hi, Math.max(lo, n)) + } + return { + cache: r.cache !== false, + memoryRegions: num(r.memoryRegions, 2, 64, MAP_PERF_DEFAULTS.memoryRegions), + parseGapMs: num(r.parseGapMs, 0, 5000, MAP_PERF_DEFAULTS.parseGapMs), + cacheLimitMB: num(r.cacheLimitMB, 0, 20_000, MAP_PERF_DEFAULTS.cacheLimitMB) + } +} diff --git a/src/shared/types.ts b/src/shared/types.ts index ba48905..3a4459e 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -1,4 +1,5 @@ // Domain types shared across main / preload / renderer. +import type { MapPerfConfig } from './tileCache' export type ServerType = | 'vanilla' @@ -74,6 +75,13 @@ export interface ServerConfig { favorite?: boolean /** Optional per-server backup destination override. */ backupDir?: string + /** + * What the live map is allowed to cost on this server (#133). + * + * Per server because a box running twenty and a laptop running one want + * different answers. Absent means the defaults; see `normalizeMapPerf`. + */ + map?: MapPerfConfig } export interface ServerRuntimeStatus { From cd5b6e8c887bdedbe5a8d3959e7344a57f369c22 Mon Sep 17 00:00:00 2001 From: CaYatur Date: Wed, 29 Jul 2026 13:22:01 +0300 Subject: [PATCH 2/2] Self-review: "clear cache" wiped every server, and the panel had none of this MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The clear button cleared everyone's.** Cache files were named by a hash of the region path alone, so there was nothing in a filename to say which server it belonged to — and a hash cannot be reversed. "Clear this server's cache" deleted every cached region on the machine while claiming to clear one. The server id is a visible prefix now, so the question is answerable from the filenames. **The sweep ran after every region.** It stats every file in the cache directory, so writing a region meant a directory scan — a cost inside the thing that exists to remove cost. It runs once per 32 MB added instead. **And I put the dials only in the desktop**, one PR after merging the three maps and one after re-splitting them. The panel is the admin surface an operator reaches from anywhere, and this is a persisted server setting rather than a view preference, so it belongs there too: `GET`/`POST /servers/{id}/map/perf` and `DELETE /servers/{id}/map/cache`, all on `settings`, with the same controls in the map tab. The panel shows what was **stored**, not what was typed. The server clamps, and a field that keeps displaying a refused number is lying about the setting. --- docs/openapi.json | 141 ++++++++++++++++++++++++++++++++++++ src/main/core/worldTiles.ts | 47 ++++++++---- src/main/web/panelHtml.ts | 69 +++++++++++++++++- src/main/web/server.ts | 30 ++++++++ src/shared/apiSurface.ts | 3 + 5 files changed, 275 insertions(+), 15 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index a4c7c40..ff2bab8 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -2653,6 +2653,147 @@ } } }, + "/api/v1/servers/{id}/map/perf": { + "get": { + "operationId": "getServersIdMapPerf", + "summary": "What the live map is allowed to cost on this server.", + "description": "Scope `settings` on the server.", + "tags": [ + "players" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Server id, as returned by GET /servers.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success." + }, + "400": { + "description": "Malformed request, or a missing confirmation." + }, + "401": { + "description": "No usable credential." + }, + "403": { + "description": "Authenticated, but not permitted — the body names what was needed." + }, + "404": { + "description": "No such server, or no such route." + }, + "409": { + "description": "Conflicts with the current state (running server, name taken, …)." + }, + "429": { + "description": "Rate limited. `Retry-After` says for how long." + } + } + }, + "post": { + "operationId": "postServersIdMapPerf", + "summary": "Change it. Values are clamped on the way in.", + "description": "Scope `settings` on the server.\n\nBody fields:\n- `cache` — Keep parsed tiles on disk (default on).\n- `memoryRegions` — Regions held in memory, 2-64.\n- `parseGapMs` — Minimum gap between region parses, 0-5000.\n- `cacheLimitMB` — On-disk ceiling, oldest evicted first.", + "tags": [ + "players" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Server id, as returned by GET /servers.", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "description": "Success." + }, + "400": { + "description": "Malformed request, or a missing confirmation." + }, + "401": { + "description": "No usable credential." + }, + "403": { + "description": "Authenticated, but not permitted — the body names what was needed." + }, + "404": { + "description": "No such server, or no such route." + }, + "409": { + "description": "Conflicts with the current state (running server, name taken, …)." + }, + "429": { + "description": "Rate limited. `Retry-After` says for how long." + } + } + } + }, + "/api/v1/servers/{id}/map/cache": { + "delete": { + "operationId": "deleteServersIdMapCache", + "summary": "Drop this server's cached map tiles.", + "description": "Scope `settings` on the server.\n\nOnly this server's: the cache filename carries the owner so one server's clear cannot take another's with it.", + "tags": [ + "players" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Server id, as returned by GET /servers.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success." + }, + "400": { + "description": "Malformed request, or a missing confirmation." + }, + "401": { + "description": "No usable credential." + }, + "403": { + "description": "Authenticated, but not permitted — the body names what was needed." + }, + "404": { + "description": "No such server, or no such route." + }, + "409": { + "description": "Conflicts with the current state (running server, name taken, …)." + }, + "429": { + "description": "Rate limited. `Retry-After` says for how long." + } + } + } + }, "/api/v1/servers/{id}/map/tiles": { "get": { "operationId": "getServersIdMapTiles", diff --git a/src/main/core/worldTiles.ts b/src/main/core/worldTiles.ts index 6bab3d2..ac14424 100644 --- a/src/main/core/worldTiles.ts +++ b/src/main/core/worldTiles.ts @@ -112,13 +112,18 @@ function ensureDir(p: string): string { * the name, so a stale file is refused on read and then overwritten rather than * accumulating one file per version. */ -function cacheFileFor(path: string): string { - return join(cacheDirFor(), createHash('sha1').update(path).digest('hex') + '.tiles') +function cacheFileFor(serverId: string, path: string): string { + // The server id is a visible PREFIX rather than part of the hash, because + // "clear this server's cache" has to be answerable from the filenames alone — + // a hash cannot be reversed, so a single hashed key made the clear button + // wipe every server's cache while claiming to clear one. + const owner = createHash('sha1').update(serverId).digest('hex').slice(0, 12) + return join(cacheDirFor(), owner + '-' + createHash('sha1').update(path).digest('hex') + '.tiles') } -function readCachedRegion(path: string, mtimeMs: number): RegionEntry | null { +function readCachedRegion(serverId: string, path: string, mtimeMs: number): RegionEntry | null { try { - const f = cacheFileFor(path) + const f = cacheFileFor(serverId, path) if (!existsSync(f)) return null const decoded = decodeRegionTiles(gunzipSync(readFileSync(f))) // The mtime says the world has not changed; the version inside the file @@ -131,12 +136,13 @@ function readCachedRegion(path: string, mtimeMs: number): RegionEntry | null { } } -function writeCachedRegion(path: string, entry: RegionEntry): void { +function writeCachedRegion(serverId: string, path: string, entry: RegionEntry): void { try { const usable = new Map() for (const [slot, tile] of entry.tiles) if (tile) usable.set(slot, tile) const buf = gzipSync(encodeRegionTiles({ mtimeMs: entry.mtimeMs, tiles: usable }), { level: 6 }) - const f = cacheFileFor(path) + writtenSinceSweep += buf.length + const f = cacheFileFor(serverId, path) // Through a temp file: a reader hitting a half-written cache would decode // garbage, and "garbage" here means a wrong map rather than an error. writeFileSync(f + '.tmp', buf) @@ -153,7 +159,19 @@ function writeCachedRegion(path: string, entry: RegionEntry): void { * the moment something was added, and a timer would be one more thing running * in a process that already has enough of them. */ +/** + * Bytes added since the last sweep. + * + * A sweep stats every file in the directory, and doing that after each region + * means a directory scan per parse — a cost inside the thing that exists to + * remove cost. Only worth doing once enough has been added to matter. + */ +let writtenSinceSweep = 0 +const SWEEP_AFTER_BYTES = 32 * 1024 * 1024 + function sweepCache(limitMB: number): void { + if (writtenSinceSweep < SWEEP_AFTER_BYTES) return + writtenSinceSweep = 0 try { const dir = cacheDirFor() const files = readdirSync(dir) @@ -176,19 +194,22 @@ function sweepCache(limitMB: number): void { } } -/** Drop every cached region for one server, or all of them. */ -export function clearTileCache(): number { +/** Drop cached regions for one server, or every server when given nothing. */ +export function clearTileCache(serverId?: string): number { let n = 0 try { const dir = cacheDirFor() + const prefix = serverId ? createHash('sha1').update(serverId).digest('hex').slice(0, 12) + '-' : '' for (const name of readdirSync(dir)) { - if (!name.endsWith('.tiles')) continue + if (!name.endsWith('.tiles') || !name.startsWith(prefix)) continue rmSync(join(dir, name), { force: true }) n++ } } catch { /* nothing to clear */ } + // The memory cache is keyed by region path with no owner, so it goes whole. + // Dropping too much of an optimisation is free; keeping a stale entry is not. regions.clear() return n } @@ -387,7 +408,7 @@ export function parseBudgetReady(serverId?: string, now = Date.now()): boolean { * Synchronous and slow by design — the callers are expected to keep this off * any request path, and to respect `parseBudgetReady`. */ -function loadRegion(path: string, perf: MapPerfConfig): RegionEntry | null { +function loadRegion(serverId: string, path: string, perf: MapPerfConfig): RegionEntry | null { if (!existsSync(path)) return null let mtimeMs = 0 try { @@ -404,7 +425,7 @@ function loadRegion(path: string, perf: MapPerfConfig): RegionEntry | null { // Disk before work. A region the server has not rewritten is the same region, // and re-parsing it is the cost this whole cache exists to avoid. if (perf.cache) { - const cached = readCachedRegion(path, mtimeMs) + const cached = readCachedRegion(serverId, path, mtimeMs) if (cached) { regions.set(path, cached) trimMemory(perf.memoryRegions) @@ -443,7 +464,7 @@ function loadRegion(path: string, perf: MapPerfConfig): RegionEntry | null { regions.set(path, entry) trimMemory(perf.memoryRegions) if (perf.cache) { - writeCachedRegion(path, entry) + writeCachedRegion(serverId, path, entry) sweepCache(perf.cacheLimitMB) } log.info(`World tiles: parsed ${tiles.size} chunks from ${path.split(/[\\/]/).pop()}`) @@ -580,7 +601,7 @@ export function chunkTile( ): ChunkTile | null { const path = regionPath(serverId, dim, regionOf(chunkX), regionOf(chunkZ)) if (!path) return null - const region = loadRegion(path, perfFor(serverId)) + const region = loadRegion(serverId, path, perfFor(serverId)) if (!region) return null return region.tiles.get(chunkSlot(localChunk(chunkX), localChunk(chunkZ))) ?? null } diff --git a/src/main/web/panelHtml.ts b/src/main/web/panelHtml.ts index 1db1845..e1fd185 100644 --- a/src/main/web/panelHtml.ts +++ b/src/main/web/panelHtml.ts @@ -340,7 +340,34 @@ h2{margin:8px 0;font-weight:800;letter-spacing:-.4px}
- +